From 269cda016dd6ea92b66e55ebe283965924e67bc1 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:55 +0000 Subject: [PATCH 0001/1648] lang/c/msgpack: added Messagepack, a binary-based efficient data interchange format. git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@48 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- README | 20 + c/Makefile | 17 + c/bench.c | 338 +++++++ c/bench_inline.c | 329 +++++++ c/msgpack | 1 + c/pack.c | 40 + c/pack.h | 38 + c/pack_inline.h | 14 + c/unpack.c | 56 ++ c/unpack.h | 59 ++ c/unpack_context.h | 30 + c/unpack_inline.c | 82 ++ cpp/unpack.cc | 30 + cpp/unpack.h | 126 +++ cpp/unpack_context.h | 13 + msgpack/pack/inline_context.h | 2 + msgpack/pack/inline_impl.h | 287 ++++++ msgpack/unpack/callback.h | 24 + msgpack/unpack/inline_context.h | 52 + msgpack/unpack/inline_impl.h | 438 +++++++++ ruby/Makefile | 153 +++ ruby/bench.rb | 60 ++ ruby/extconf.rb | 4 + ruby/gem.sh | 19 + ruby/gem/AUTHORS | 1 + ruby/gem/History.txt | 0 ruby/gem/License.txt | 13 + ruby/gem/Manifest.txt | 26 + ruby/gem/PostInstall.txt | 1 + ruby/gem/Rakefile | 4 + ruby/gem/config/hoe.rb | 75 ++ ruby/gem/config/requirements.rb | 15 + ruby/gem/lib/msgpack/version.rb | 9 + ruby/gem/script/console | 10 + ruby/gem/script/destroy | 14 + ruby/gem/script/generate | 14 + ruby/gem/script/txt2html | 82 ++ ruby/gem/setup.rb | 1585 +++++++++++++++++++++++++++++++ ruby/gem/tasks/deployment.rake | 34 + ruby/gem/tasks/environment.rake | 7 + ruby/pack.c | 131 +++ ruby/pack.h | 26 + ruby/pack_inline.h | 33 + ruby/rbinit.c | 29 + ruby/test_format.rb | 128 +++ ruby/test_pack.rb | 56 ++ ruby/unpack.c | 202 ++++ ruby/unpack.h | 26 + ruby/unpack_context.h | 35 + ruby/unpack_inline.c | 81 ++ 50 files changed, 4869 insertions(+) create mode 100644 README create mode 100644 c/Makefile create mode 100644 c/bench.c create mode 100644 c/bench_inline.c create mode 120000 c/msgpack create mode 100644 c/pack.c create mode 100644 c/pack.h create mode 100644 c/pack_inline.h create mode 100644 c/unpack.c create mode 100644 c/unpack.h create mode 100644 c/unpack_context.h create mode 100644 c/unpack_inline.c create mode 100644 cpp/unpack.cc create mode 100644 cpp/unpack.h create mode 100644 cpp/unpack_context.h create mode 100644 msgpack/pack/inline_context.h create mode 100644 msgpack/pack/inline_impl.h create mode 100644 msgpack/unpack/callback.h create mode 100644 msgpack/unpack/inline_context.h create mode 100644 msgpack/unpack/inline_impl.h create mode 100644 ruby/Makefile create mode 100644 ruby/bench.rb create mode 100644 ruby/extconf.rb create mode 100755 ruby/gem.sh create mode 100644 ruby/gem/AUTHORS create mode 100644 ruby/gem/History.txt create mode 100644 ruby/gem/License.txt create mode 100644 ruby/gem/Manifest.txt create mode 100644 ruby/gem/PostInstall.txt create mode 100644 ruby/gem/Rakefile create mode 100644 ruby/gem/config/hoe.rb create mode 100644 ruby/gem/config/requirements.rb create mode 100644 ruby/gem/lib/msgpack/version.rb create mode 100755 ruby/gem/script/console create mode 100755 ruby/gem/script/destroy create mode 100755 ruby/gem/script/generate create mode 100755 ruby/gem/script/txt2html create mode 100644 ruby/gem/setup.rb create mode 100644 ruby/gem/tasks/deployment.rake create mode 100644 ruby/gem/tasks/environment.rake create mode 100644 ruby/pack.c create mode 100644 ruby/pack.h create mode 100644 ruby/pack_inline.h create mode 100644 ruby/rbinit.c create mode 100644 ruby/test_format.rb create mode 100644 ruby/test_pack.rb create mode 100644 ruby/unpack.c create mode 100644 ruby/unpack.h create mode 100644 ruby/unpack_context.h create mode 100644 ruby/unpack_inline.c diff --git a/README b/README new file mode 100644 index 0000000..31a482a --- /dev/null +++ b/README @@ -0,0 +1,20 @@ +MessagePack +----------- +Binary-based efficient data interchange format. + + + +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/c/Makefile b/c/Makefile new file mode 100644 index 0000000..d4b2226 --- /dev/null +++ b/c/Makefile @@ -0,0 +1,17 @@ + +CFLAGS = -I.. -Wall -g -O4 +LDFLAGS = -L. + +all: bench + +bench: pack.o unpack.o unpack_inline.o bench.o pack.h pack_inline.h unpack.h unpack_context.h + $(CC) $(LDFLAGS) unpack.o unpack_inline.o pack.o bench.o -lyajl_s -o $@ + +bench_inline: pack.o bench_inline.o pack.h pack_inline.h + $(CC) $(LDFLAGS) pack.o bench_inline.o -lyajl_s -o $@ + +.PHONY: clean +clean: + $(RM) unpack.o unpack_inline.o pack.o test.o bench.o bench_inline.o + $(RM) bench bench_inline + diff --git a/c/bench.c b/c/bench.c new file mode 100644 index 0000000..f27350a --- /dev/null +++ b/c/bench.c @@ -0,0 +1,338 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + + +static struct timeval g_timer; + +void reset_timer() +{ + gettimeofday(&g_timer, NULL); +} + +double show_timer() +{ + struct timeval endtime; + gettimeofday(&endtime, NULL); + double sec = (endtime.tv_sec - g_timer.tv_sec) + + (double)(endtime.tv_usec - g_timer.tv_usec) / 1000 / 1000; + printf("%f sec\n", sec); + return sec; +} + + +static int reformat_null(void * ctx) { return 1; } +static int reformat_boolean(void * ctx, int boolean) { return 1; } +static int reformat_number(void * ctx, const char * s, unsigned int l) { return 1; } +static int reformat_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } +static int reformat_map_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } +static int reformat_start_map(void * ctx) { return 1; } +static int reformat_end_map(void * ctx) { return 1; } +static int reformat_start_array(void * ctx) { return 1; } +static int reformat_end_array(void * ctx) { return 1; } + + +static void* unpack_unsigned_int_8(void* data, uint8_t d) { return NULL; } +static void* unpack_unsigned_int_16(void* data, uint16_t d) { return NULL; } +static void* unpack_unsigned_int_32(void* data, uint32_t d) { return NULL; } +static void* unpack_unsigned_int_64(void* data, uint64_t d) { return NULL; } +static void* unpack_signed_int_8(void* data, int8_t d) { return NULL; } +static void* unpack_signed_int_16(void* data, int16_t d) { return NULL; } +static void* unpack_signed_int_32(void* data, int32_t d) { return NULL; } +static void* unpack_signed_int_64(void* data, int64_t d) { return NULL; } +static void* unpack_float(void* data, float d) { return NULL; } +static void* unpack_double(void* data, double d) { return NULL; } +static void* unpack_nil(void* data) { return NULL; } +static void* unpack_true(void* data) { return NULL; } +static void* unpack_false(void* data) { return NULL; } +static void* unpack_array_start(void* data, unsigned int n) { return NULL; } +static void unpack_array_item(void* data, void* c, void* o) { } +static void* unpack_map_start(void* data, unsigned int n) { return NULL; } +static void unpack_map_item(void* data, void* c, void* k, void* v) { } +static void* unpack_string(void* data, const void* b, size_t l) { return NULL; } +static void* unpack_raw(void* data, const void* b, size_t l) { /*printf("unpack raw %p %lu\n",b,l);*/ return NULL; } + +typedef struct { + size_t allocated; + size_t length; + char* buffer; +} pack_buffer; + +static const size_t PACK_INITIAL_BUFFER_SIZE = 512; + +static void pack_buffer_init(pack_buffer* data) +{ + data->buffer = malloc(PACK_INITIAL_BUFFER_SIZE); + data->length = 0; + data->allocated = PACK_INITIAL_BUFFER_SIZE; +} + +static void pack_buffer_reset(pack_buffer* data) +{ + data->buffer = realloc(data->buffer, PACK_INITIAL_BUFFER_SIZE); + data->allocated = PACK_INITIAL_BUFFER_SIZE; + data->length = 0; +} + +static void pack_buffer_free(pack_buffer* data) +{ + free(data->buffer); +} + +static void pack_append_buffer(void* user, const unsigned char* b, unsigned int l) +{ + pack_buffer* data = (pack_buffer*)user; + if(data->allocated - data->length < l) { + data->buffer = realloc(data->buffer, data->allocated*2); + data->allocated *= 2; + } + memcpy(data->buffer + data->length, b, l); + data->length += l; +} + + +static const unsigned int TASK_INT_NUM = 1<<24; +static const unsigned int TASK_STR_LEN = 1<<15; +//static const unsigned int TASK_INT_NUM = 1<<20; +//static const unsigned int TASK_STR_LEN = 1<<12; +static const char* TASK_STR_PTR; + + +void bench_json(void) +{ + puts("== JSON =="); + + + yajl_gen_config gcfg = {0, NULL}; + yajl_gen g = yajl_gen_alloc(&gcfg); + + yajl_parser_config hcfg = { 0, 0 }; + yajl_callbacks callbacks = { + reformat_null, + reformat_boolean, + NULL, + NULL, + reformat_number, + reformat_string, + reformat_start_map, + reformat_map_key, + reformat_end_map, + reformat_start_array, + reformat_end_array + }; + yajl_handle h = yajl_alloc(&callbacks, &hcfg, NULL); + + + double sec; + const unsigned char * buf; + unsigned int len; + + + puts("generate integer"); + reset_timer(); + { + unsigned int i; + yajl_gen_array_open(g); + for(i=0; i < TASK_INT_NUM; ++i) { + yajl_gen_integer(g, i); + } + yajl_gen_array_close(g); + } + sec = show_timer(); + + yajl_gen_get_buf(g, &buf, &len); + printf("%u KB\n", len / 1024); + printf("%f MB/s\n", len / sec / 1024 / 1024); + + puts("----"); + puts("parse integer"); + reset_timer(); + { + yajl_status stat = yajl_parse(h, buf, len); + if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { + unsigned char * str = yajl_get_error(h, 1, buf, len); + fprintf(stderr, (const char *) str); + } + } + sec = show_timer(); + + printf("%f MB/s\n", len / sec / 1024 / 1024); + + + //yajl_gen_clear(g); + yajl_gen_free(g); + g = yajl_gen_alloc(&gcfg); + yajl_free(h); + h = yajl_alloc(&callbacks, &hcfg, NULL); + + + puts("----"); + puts("generate string"); + reset_timer(); + { + unsigned int i; + yajl_gen_array_open(g); + for(i=0; i < TASK_STR_LEN; ++i) { + yajl_gen_string(g, (const unsigned char*)TASK_STR_PTR, i); + } + yajl_gen_array_close(g); + } + sec = show_timer(); + + yajl_gen_get_buf(g, &buf, &len); + printf("%u KB\n", len / 1024); + printf("%f MB/s\n", len / sec / 1024 / 1024); + + puts("----"); + puts("parse string"); + reset_timer(); + { + yajl_status stat = yajl_parse(h, buf, len); + if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { + unsigned char * str = yajl_get_error(h, 1, buf, len); + fprintf(stderr, (const char *) str); + } + } + sec = show_timer(); + + printf("%f MB/s\n", len / sec / 1024 / 1024); + + + yajl_gen_free(g); + yajl_free(h); +} + +void bench_msgpack(void) +{ + puts("== MessagePack =="); + + + pack_buffer mpkbuf; + pack_buffer_init(&mpkbuf); + msgpack_pack_t* mpk = msgpack_pack_new( + &mpkbuf, pack_append_buffer); + + msgpack_unpack_callback cb = { + unpack_unsigned_int_8, + unpack_unsigned_int_16, + unpack_unsigned_int_32, + unpack_unsigned_int_64, + unpack_signed_int_8, + unpack_signed_int_16, + unpack_signed_int_32, + unpack_signed_int_64, + unpack_float, + unpack_double, + unpack_nil, + unpack_true, + unpack_false, + unpack_array_start, + unpack_array_item, + unpack_map_start, + unpack_map_item, + unpack_string, + unpack_raw, + }; + msgpack_unpack_t* mupk = msgpack_unpack_new(NULL, &cb); + + + double sec; + size_t len; + const char* buf; + + + puts("pack integer"); + reset_timer(); + { + unsigned int i; + msgpack_pack_array(mpk, TASK_INT_NUM); + for(i=0; i < TASK_INT_NUM; ++i) { + msgpack_pack_unsigned_int(mpk, i); + } + } + sec = show_timer(); + + len = mpkbuf.length; + buf = mpkbuf.buffer; + printf("%lu KB\n", len / 1024); + printf("%f MB/s\n", len / sec / 1024 / 1024); + + puts("----"); + puts("unpack integer"); + reset_timer(); + { + size_t off = 0; + int ret = msgpack_unpack_execute(mupk, buf, len, &off); + if(ret < 0) { + fprintf(stderr, "Parse error.\n"); + } else if(ret == 0) { + fprintf(stderr, "Not finished.\n"); + } + } + sec = show_timer(); + + printf("%f MB/s\n", len / sec / 1024 / 1024); + + + pack_buffer_reset(&mpkbuf); + msgpack_unpack_reset(mupk); + + + puts("----"); + puts("pack string"); + reset_timer(); + { + unsigned int i; + msgpack_pack_array(mpk, TASK_STR_LEN); + for(i=0; i < TASK_STR_LEN; ++i) { + msgpack_pack_raw(mpk, TASK_STR_PTR, i); + } + } + sec = show_timer(); + + len = mpkbuf.length; + buf = mpkbuf.buffer; + printf("%lu KB\n", len / 1024); + printf("%f MB/s\n", len / sec / 1024 / 1024); + + puts("----"); + puts("unpack string"); + reset_timer(); + { + size_t off = 0; + int ret = msgpack_unpack_execute(mupk, buf, len, &off); + if(ret < 0) { + fprintf(stderr, "Parse error.\n"); + } else if(ret == 0) { + fprintf(stderr, "Not finished.\n"); + } + } + sec = show_timer(); + + printf("%f MB/s\n", len / sec / 1024 / 1024); + + + msgpack_unpack_free(mupk); + msgpack_pack_free(mpk); + pack_buffer_free(&mpkbuf); +} + +int main(int argc, char* argv[]) +{ + char* str = malloc(TASK_STR_LEN); + memset(str, 'a', TASK_STR_LEN); + TASK_STR_PTR = str; + + bench_msgpack(); + bench_json(); + + return 0; +} + + diff --git a/c/bench_inline.c b/c/bench_inline.c new file mode 100644 index 0000000..2901508 --- /dev/null +++ b/c/bench_inline.c @@ -0,0 +1,329 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + + +static struct timeval g_timer; + +void reset_timer() +{ + gettimeofday(&g_timer, NULL); +} + +double show_timer() +{ + struct timeval endtime; + gettimeofday(&endtime, NULL); + double sec = (endtime.tv_sec - g_timer.tv_sec) + + (double)(endtime.tv_usec - g_timer.tv_usec) / 1000 / 1000; + printf("%f sec\n", sec); + return sec; +} + + +static int reformat_null(void * ctx) { return 1; } +static int reformat_boolean(void * ctx, int boolean) { return 1; } +static int reformat_number(void * ctx, const char * s, unsigned int l) { return 1; } +static int reformat_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } +static int reformat_map_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } +static int reformat_start_map(void * ctx) { return 1; } +static int reformat_end_map(void * ctx) { return 1; } +static int reformat_start_array(void * ctx) { return 1; } +static int reformat_end_array(void * ctx) { return 1; } + + +typedef void* msgpack_object; + +typedef struct { +} msgpack_unpack_context; + +#include "msgpack/unpack/inline_context.h" + +static inline void* msgpack_unpack_init(msgpack_unpack_context* x) { return NULL; } +static inline void* msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d) { return NULL; } +static inline void* msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d) { return NULL; } +static inline void* msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d) { return NULL; } +static inline void* msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d) { return NULL; } +static inline void* msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d) { return NULL; } +static inline void* msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d) { return NULL; } +static inline void* msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d) { return NULL; } +static inline void* msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) { return NULL; } +static inline void* msgpack_unpack_float(msgpack_unpack_context* x, float d) { return NULL; } +static inline void* msgpack_unpack_double(msgpack_unpack_context* x, double d) { return NULL; } +static inline void* msgpack_unpack_big_int(msgpack_unpack_context* x, const void* b, unsigned int l) { return NULL; } +static inline void* msgpack_unpack_big_float(msgpack_unpack_context* x, const void* b, unsigned int l) { return NULL; } +static inline void* msgpack_unpack_nil(msgpack_unpack_context* x) { return NULL; } +static inline void* msgpack_unpack_true(msgpack_unpack_context* x) { return NULL; } +static inline void* msgpack_unpack_false(msgpack_unpack_context* x) { return NULL; } +static inline void* msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n) { return NULL; } +static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, void* c, void* o) { } +static inline void* msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) { return NULL; } +static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, void* c, void* k, void* v) { } +static inline void* msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l) { return NULL; } +static inline void* msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l) { return NULL; } + +#include "msgpack/unpack/inline_impl.h" + +typedef struct { + size_t allocated; + size_t length; + char* buffer; +} pack_buffer; + +static const size_t PACK_INITIAL_BUFFER_SIZE = 512; + +static void pack_buffer_init(pack_buffer* data) +{ + data->buffer = malloc(PACK_INITIAL_BUFFER_SIZE); + data->length = 0; + data->allocated = PACK_INITIAL_BUFFER_SIZE; +} + +static void pack_buffer_reset(pack_buffer* data) +{ + data->buffer = realloc(data->buffer, PACK_INITIAL_BUFFER_SIZE); + data->allocated = PACK_INITIAL_BUFFER_SIZE; + data->length = 0; +} + +static void pack_buffer_free(pack_buffer* data) +{ + free(data->buffer); +} + +static void pack_append_buffer(void* user, const unsigned char* b, unsigned int l) +{ + pack_buffer* data = (pack_buffer*)user; + if(data->allocated - data->length < l) { + data->buffer = realloc(data->buffer, data->allocated*2); + data->allocated *= 2; + } + memcpy(data->buffer + data->length, b, l); + data->length += l; +} + + +static const unsigned int TASK_INT_NUM = 1<<24; +static const unsigned int TASK_STR_LEN = 1<<15; +//static const unsigned int TASK_INT_NUM = 1<<20; +//static const unsigned int TASK_STR_LEN = 1<<12; +static const char* TASK_STR_PTR; + + +void bench_json(void) +{ + puts("== JSON =="); + + + yajl_gen_config gcfg = {0, NULL}; + yajl_gen g = yajl_gen_alloc(&gcfg); + + yajl_parser_config hcfg = { 0, 0 }; + yajl_callbacks callbacks = { + reformat_null, + reformat_boolean, + NULL, + NULL, + reformat_number, + reformat_string, + reformat_start_map, + reformat_map_key, + reformat_end_map, + reformat_start_array, + reformat_end_array + }; + yajl_handle h = yajl_alloc(&callbacks, &hcfg, NULL); + + + double sec; + const unsigned char * buf; + unsigned int len; + + + puts("generate integer"); + reset_timer(); + { + unsigned int i; + yajl_gen_array_open(g); + for(i=0; i < TASK_INT_NUM; ++i) { + yajl_gen_integer(g, i); + } + yajl_gen_array_close(g); + } + sec = show_timer(); + + yajl_gen_get_buf(g, &buf, &len); + printf("%u KB\n", len / 1024); + printf("%f Mbps\n", len / sec / 1024 / 1024); + + puts("----"); + puts("parse integer"); + reset_timer(); + { + yajl_status stat = yajl_parse(h, buf, len); + if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { + unsigned char * str = yajl_get_error(h, 1, buf, len); + fprintf(stderr, (const char *) str); + } + } + sec = show_timer(); + + printf("%f Mbps\n", len / sec / 1024 / 1024); + + + //yajl_gen_clear(g); + yajl_gen_free(g); + g = yajl_gen_alloc(&gcfg); + yajl_free(h); + h = yajl_alloc(&callbacks, &hcfg, NULL); + + + puts("----"); + puts("generate string"); + reset_timer(); + { + unsigned int i; + yajl_gen_array_open(g); + for(i=0; i < TASK_STR_LEN; ++i) { + yajl_gen_string(g, (const unsigned char*)TASK_STR_PTR, i); + } + yajl_gen_array_close(g); + } + sec = show_timer(); + + yajl_gen_get_buf(g, &buf, &len); + printf("%u KB\n", len / 1024); + printf("%f Mbps\n", len / sec / 1024 / 1024); + + puts("----"); + puts("parse string"); + reset_timer(); + { + yajl_status stat = yajl_parse(h, buf, len); + if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { + unsigned char * str = yajl_get_error(h, 1, buf, len); + fprintf(stderr, (const char *) str); + } + } + sec = show_timer(); + + printf("%f Mbps\n", len / sec / 1024 / 1024); + + + yajl_gen_free(g); + yajl_free(h); +} + +void bench_msgpack(void) +{ + puts("== MessagePack =="); + + + pack_buffer mpkbuf; + pack_buffer_init(&mpkbuf); + msgpack_pack_t* mpk = msgpack_pack_new( + &mpkbuf, pack_append_buffer); + + msgpack_unpacker mupk; + msgpack_unpacker_init(&mupk); + + double sec; + size_t len; + const char* buf; + + + puts("pack integer"); + reset_timer(); + { + unsigned int i; + msgpack_pack_array(mpk, TASK_INT_NUM); + for(i=0; i < TASK_INT_NUM; ++i) { + msgpack_pack_unsigned_int(mpk, i); + } + } + sec = show_timer(); + + len = mpkbuf.length; + buf = mpkbuf.buffer; + printf("%lu KB\n", len / 1024); + printf("%f Mbps\n", len / sec / 1024 / 1024); + + puts("----"); + puts("unpack integer"); + reset_timer(); + { + size_t off = 0; + int ret = msgpack_unpacker_execute(&mupk, buf, len, &off); + if(ret < 0) { + fprintf(stderr, "Parse error.\n"); + } else if(ret == 0) { + fprintf(stderr, "Not finished.\n"); + } + } + sec = show_timer(); + + printf("%f Mbps\n", len / sec / 1024 / 1024); + + + pack_buffer_reset(&mpkbuf); + msgpack_unpacker_init(&mupk); + + + puts("----"); + puts("pack string"); + reset_timer(); + { + unsigned int i; + msgpack_pack_array(mpk, TASK_STR_LEN); + for(i=0; i < TASK_STR_LEN; ++i) { + msgpack_pack_raw(mpk, TASK_STR_PTR, i); + } + } + sec = show_timer(); + + len = mpkbuf.length; + buf = mpkbuf.buffer; + printf("%lu KB\n", len / 1024); + printf("%f Mbps\n", len / sec / 1024 / 1024); + + puts("----"); + puts("unpack string"); + reset_timer(); + { + size_t off = 0; + int ret = msgpack_unpacker_execute(&mupk, buf, len, &off); + if(ret < 0) { + fprintf(stderr, "Parse error.\n"); + } else if(ret == 0) { + fprintf(stderr, "Not finished.\n"); + } + } + sec = show_timer(); + + printf("%f Mbps\n", len / sec / 1024 / 1024); + + + msgpack_pack_free(mpk); + pack_buffer_free(&mpkbuf); +} + +int main(int argc, char* argv[]) +{ + char* str = malloc(TASK_STR_LEN); + memset(str, 'a', TASK_STR_LEN); + TASK_STR_PTR = str; + + bench_msgpack(); + //bench_json(); + + return 0; +} + + + diff --git a/c/msgpack b/c/msgpack new file mode 120000 index 0000000..945c9b4 --- /dev/null +++ b/c/msgpack @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/c/pack.c b/c/pack.c new file mode 100644 index 0000000..05bd38d --- /dev/null +++ b/c/pack.c @@ -0,0 +1,40 @@ +/* + * MessagePack packing routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pack_inline.h" +#include +#include + +msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_callback_t callback) +{ + msgpack_pack_t* ctx = calloc(1, sizeof(msgpack_pack_t)); + if(!ctx) { return NULL; } + ctx->data = data; + ctx->callback = callback; + return ctx; +} + +void msgpack_pack_free(msgpack_pack_t* ctx) +{ + free(ctx); +} + +static inline void msgpack_pack_append_buffer(msgpack_pack_t* ctx, const unsigned char* b, unsigned int l) +{ + ctx->callback(ctx->data, b, l); +} + diff --git a/c/pack.h b/c/pack.h new file mode 100644 index 0000000..8107d9f --- /dev/null +++ b/c/pack.h @@ -0,0 +1,38 @@ +#ifndef MSGPACK_PACK_H__ +#define MSGPACK_PACK_H__ + +#include +#include + +typedef void (*msgpack_pack_callback_t)(void* data, const unsigned char* b, unsigned int i); + +typedef struct { + void* data; + msgpack_pack_callback_t callback; +} msgpack_pack_t; + +msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_callback_t callback); +void msgpack_pack_free(msgpack_pack_t* ctx); + +void msgpack_pack_int(msgpack_pack_t* ctx, int d); +void msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); +void msgpack_pack_unsigned_int_8(msgpack_pack_t* ctx, uint8_t d); +void msgpack_pack_unsigned_int_16(msgpack_pack_t* ctx, uint16_t d); +void msgpack_pack_unsigned_int_32(msgpack_pack_t* ctx, uint32_t d); +void msgpack_pack_unsigned_int_64(msgpack_pack_t* ctx, uint64_t d); +void msgpack_pack_signed_int_8(msgpack_pack_t* ctx, int8_t d); +void msgpack_pack_signed_int_16(msgpack_pack_t* ctx, int16_t d); +void msgpack_pack_signed_int_32(msgpack_pack_t* ctx, int32_t d); +void msgpack_pack_signed_int_64(msgpack_pack_t* ctx, int64_t d); +void msgpack_pack_float(msgpack_pack_t* ctx, float d); +void msgpack_pack_double(msgpack_pack_t* ctx, double d); +void msgpack_pack_nil(msgpack_pack_t* ctx); +void msgpack_pack_true(msgpack_pack_t* ctx); +void msgpack_pack_false(msgpack_pack_t* ctx); +void msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); +void msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); +void msgpack_pack_string(msgpack_pack_t* ctx, const char* b); +void msgpack_pack_raw(msgpack_pack_t* ctx, const void* b, size_t l); + +#endif /* msgpack/pack.h */ + diff --git a/c/pack_inline.h b/c/pack_inline.h new file mode 100644 index 0000000..d943464 --- /dev/null +++ b/c/pack_inline.h @@ -0,0 +1,14 @@ +#ifndef PACK_INLINE_H__ +#define PACK_INLINE_H__ + +#include "pack.h" + +typedef msgpack_pack_t* msgpack_pack_context; + +static inline void msgpack_pack_append_buffer(msgpack_pack_t* x, const unsigned char* b, unsigned int l); + +#include "msgpack/pack/inline_impl.h" + + +#endif /* pack_inline.h */ + diff --git a/c/unpack.c b/c/unpack.c new file mode 100644 index 0000000..9fe3695 --- /dev/null +++ b/c/unpack.c @@ -0,0 +1,56 @@ +/* + * MessagePack packing routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "unpack.h" +#include "unpack_context.h" +#include + +msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback) +{ + msgpack_unpacker* ctx; + ctx = (msgpack_unpacker*)calloc(1, sizeof(msgpack_unpacker)); + if(ctx == NULL) { return NULL; } + msgpack_unpacker_init(ctx); + ((msgpack_unpack_t*)ctx)->data = data; + ((msgpack_unpack_t*)ctx)->callback = *callback; + return (msgpack_unpack_t*)ctx; +} + +void msgpack_unpack_free(msgpack_unpack_t* ctx) +{ + free((msgpack_unpacker*)ctx); +} + +void* msgpack_unpack_data(msgpack_unpack_t* ctx) +{ + return msgpack_unpacker_data((msgpack_unpacker*)ctx); +} + +void msgpack_unpack_reset(msgpack_unpack_t* ctx) +{ + msgpack_unpack_t x = ((msgpack_unpacker*)ctx)->user; + msgpack_unpacker_init((msgpack_unpacker*)ctx); + ((msgpack_unpacker*)ctx)->user = x; +} + +int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off) +{ + return msgpack_unpacker_execute( + (msgpack_unpacker*)ctx, + data, len, off); +} + diff --git a/c/unpack.h b/c/unpack.h new file mode 100644 index 0000000..3230a0d --- /dev/null +++ b/c/unpack.h @@ -0,0 +1,59 @@ +/* + * MessagePack unpacking routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_H__ +#define MSGPACK_UNPACK_H__ + +#include +#include + +typedef struct { + void* (*unpack_unsigned_int_8)(void* data, uint8_t d); + void* (*unpack_unsigned_int_16)(void* data, uint16_t d); + void* (*unpack_unsigned_int_32)(void* data, uint32_t d); + void* (*unpack_unsigned_int_64)(void* data, uint64_t d); + void* (*unpack_signed_int_8)(void* data, int8_t d); + void* (*unpack_signed_int_16)(void* data, int16_t d); + void* (*unpack_signed_int_32)(void* data, int32_t d); + void* (*unpack_signed_int_64)(void* data, int64_t d); + void* (*unpack_float)(void* data, float d); + void* (*unpack_double)(void* data, double d); + void* (*unpack_nil)(void* data); + void* (*unpack_true)(void* data); + void* (*unpack_false)(void* data); + void* (*unpack_array_start)(void* data, unsigned int n); + void (*unpack_array_item)(void* data, void* c, void* o); + void* (*unpack_map_start)(void* data, unsigned int n); + void (*unpack_map_item)(void* data, void* c, void* k, void* v); + void* (*unpack_string)(void* data, const void* b, size_t l); + void* (*unpack_raw)(void* data, const void* b, size_t l); +} msgpack_unpack_callback; + +typedef struct { + void* data; + msgpack_unpack_callback callback; +} msgpack_unpack_t; + +msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback); +void msgpack_unpack_free(msgpack_unpack_t* ctx); +void msgpack_unpack_reset(msgpack_unpack_t* ctx); + +int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off); +void* msgpack_unpack_data(msgpack_unpack_t* ctx); + +#endif /* msgpack/unpack.h */ + diff --git a/c/unpack_context.h b/c/unpack_context.h new file mode 100644 index 0000000..7337c9e --- /dev/null +++ b/c/unpack_context.h @@ -0,0 +1,30 @@ +/* + * MessagePack unpacking routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef UNPACK_CONTEXT_H__ +#define UNPACK_CONTEXT_H__ + +#include "unpack.h" + +typedef void* msgpack_object; + +typedef msgpack_unpack_t msgpack_unpack_context; + +#include "msgpack/unpack/inline_context.h" + +#endif /* unpack_context.h */ + diff --git a/c/unpack_inline.c b/c/unpack_inline.c new file mode 100644 index 0000000..3525468 --- /dev/null +++ b/c/unpack_inline.c @@ -0,0 +1,82 @@ +/* + * MessagePack unpacking routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "unpack_context.h" + +static inline void* msgpack_unpack_init(msgpack_unpack_t* x) +{ return NULL; } + +static inline void* msgpack_unpack_unsigned_int_8(msgpack_unpack_t* x, uint8_t d) +{ return x->callback.unpack_unsigned_int_8(x->data, d); } + +static inline void* msgpack_unpack_unsigned_int_16(msgpack_unpack_t* x, uint16_t d) +{ return x->callback.unpack_unsigned_int_16(x->data, d); } + +static inline void* msgpack_unpack_unsigned_int_32(msgpack_unpack_t* x, uint32_t d) +{ return x->callback.unpack_unsigned_int_32(x->data, d); } + +static inline void* msgpack_unpack_unsigned_int_64(msgpack_unpack_t* x, uint64_t d) +{ return x->callback.unpack_unsigned_int_64(x->data, d); } + +static inline void* msgpack_unpack_signed_int_8(msgpack_unpack_t* x, int8_t d) +{ return x->callback.unpack_signed_int_8(x->data, d); } + +static inline void* msgpack_unpack_signed_int_16(msgpack_unpack_t* x, int16_t d) +{ return x->callback.unpack_signed_int_16(x->data, d); } + +static inline void* msgpack_unpack_signed_int_32(msgpack_unpack_t* x, int32_t d) +{ return x->callback.unpack_signed_int_32(x->data, d); } + +static inline void* msgpack_unpack_signed_int_64(msgpack_unpack_t* x, int64_t d) +{ return x->callback.unpack_signed_int_64(x->data, d); } + +static inline void* msgpack_unpack_float(msgpack_unpack_t* x, float d) +{ return x->callback.unpack_float(x->data, d); } + +static inline void* msgpack_unpack_double(msgpack_unpack_t* x, double d) +{ return x->callback.unpack_double(x->data, d); } + +static inline void* msgpack_unpack_nil(msgpack_unpack_t* x) +{ return x->callback.unpack_nil(x->data); } + +static inline void* msgpack_unpack_true(msgpack_unpack_t* x) +{ return x->callback.unpack_true(x->data); } + +static inline void* msgpack_unpack_false(msgpack_unpack_t* x) +{ return x->callback.unpack_false(x->data); } + +static inline void* msgpack_unpack_array_start(msgpack_unpack_t* x, unsigned int n) +{ return x->callback.unpack_array_start(x->data, n); } + +static inline void msgpack_unpack_array_item(msgpack_unpack_t* x, void* c, void* o) +{ x->callback.unpack_array_item(x->data, c, o); } + +static inline void* msgpack_unpack_map_start(msgpack_unpack_t* x, unsigned int n) +{ return x->callback.unpack_map_start(x->data, n); } + +static inline void msgpack_unpack_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) +{ x->callback.unpack_map_item(x->data, c, k, v); } + +static inline void* msgpack_unpack_string(msgpack_unpack_t* x, const void* b, size_t l) +{ return x->callback.unpack_string(x->data, b, l); } + +static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const void* b, size_t l) +{ return x->callback.unpack_raw(x->data, b, l); } + + +#include "msgpack/unpack/inline_impl.h" + diff --git a/cpp/unpack.cc b/cpp/unpack.cc new file mode 100644 index 0000000..ce57e67 --- /dev/null +++ b/cpp/unpack.cc @@ -0,0 +1,30 @@ +#include "unpack.h" +#include "unpack_context.h" +#include + +msgpack_unpack_t* msgpack_unpack_new(void) +{ + msgpack_unpacker* ctx; + ctx = (msgpack_unpacker*)calloc(1, sizeof(msgpack_unpacker)); + if(ctx == NULL) { return NULL; } + msgpack_unpacker_init(ctx); + return (msgpack_unpack_t*)ctx; +} + +void msgpack_unpack_free(msgpack_unpack_t* ctx) +{ + free((msgpack_unpacker*)ctx); +} + +int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off) +{ + return msgpack_unpacker_execute( + (msgpack_unpacker*)ctx, + data, len, off); +} + +void* msgpack_unpack_data(msgpack_unpack_t* ctx) +{ + return msgpack_unpacker_data((msgpack_unpacker*)ctx); +} + diff --git a/cpp/unpack.h b/cpp/unpack.h new file mode 100644 index 0000000..cf7a168 --- /dev/null +++ b/cpp/unpack.h @@ -0,0 +1,126 @@ +#ifndef MSGPACK_UNPACK_H__ +#define MSGPACK_UNPACK_H__ + +#include +#include + +namespace MessagePack { + +class Unpacker { + class object { + template + object(const T& x) : m_obj(new holder(x)) {} + }; + + class type_error : public std::exception { }; + class cast_error : public type_error { }; + class overflow_error : public type_error { }; + class underflow_error : public type_error { }; + + struct object { + virtual ~object() {} + virtual bool isnil() const { return false; } + virtual bool xbool() const { throw cast_error(); } + virtual uint8_t xu8() const { throw cast_error(); } + virtual uint16_t xu16() const { throw cast_error(); } + virtual uint32_t xu32() const { throw cast_error(); } + virtual uint64_t xu64() const { throw cast_error(); } + virtual int8_t xi8() const { throw cast_error(); } + virtual int16_t xi16() const { throw cast_error(); } + virtual int32_t xi32() const { throw cast_error(); } + virtual int64_t xi64() const { throw cast_error(); } + virtual float xfloat() const { throw cast_error(); } + virtual double xdouble() const { throw cast_error(); } + virtual std::map& xmap() const { throw cast_error(); } + virtual std::string& xstring() const { throw cast_error(); } + virtual std::pair xraw() const { throw cast_error(); } + public: + template + inline void check_overflow(X x) { + if(std::numeric_limits::max() < x) { throw overflow_error(); } + } + template + inline void check_underflow(X x) { + if(std::numeric_limits::min() > x) { throw overflow_error(); } + } + }; + +private: + struct object_nil : object { + bool isnil() const { return true; } + }; + + struct object_true : object { + bool xbool() const { return true; } + }; + + struct object_false : object { + bool xbool() const { return false; } + }; + + struct object_u8 : object { + object_u8(uint8_t val) : m_val(val) {} + uint8_t xu8() const { return m_val; } + uint16_t xu16() const { return static_cast(m_val); } + uint32_t xu32() const { return static_cast(m_val); } + uint64_t xu64() const { return static_cast(m_val); } + int8_t xi8() const { check_overflow(m_val); return m_val; } + int16_t xi16() const { return static_cast(m_val); } + int32_t xi32() const { return static_cast(m_val); } + int64_t xi64() const { return static_cast(m_val); } + private: + uint8_t m_val; + }; + + struct object_u16 : object { + object_u16(uint16_t val) : m_val(val) {} + uint8_t xu8() const { check_overflow(m_val); return m_val; } + uint16_t xu16() const { return m_val; } + uint32_t xu32() const { return static_cast(m_val); } + uint64_t xu64() const { return static_cast(m_val); } + int8_t xi8() const { check_overflow< int8_t>(m_val); return m_val; } + int16_t xi16() const { check_overflow(m_val); return m_val; } + int32_t xi32() const { return static_cast(m_val); } + int64_t xi64() const { return static_cast(m_val); } + private: + uint16_t m_val; + }; + + ... +}; + +} // namespace MessagePack + +typedef struct { + void* (*unpack_unsigned_int_8)(void* data, uint8_t d); + void* (*unpack_unsigned_int_16)(void* data, uint16_t d); + void* (*unpack_unsigned_int_32)(void* data, uint32_t d); + void* (*unpack_unsigned_int_64)(void* data, uint64_t d); + void* (*unpack_signed_int_8)(void* data, int8_t d); + void* (*unpack_signed_int_16)(void* data, int16_t d); + void* (*unpack_signed_int_32)(void* data, int32_t d); + void* (*unpack_signed_int_64)(void* data, int64_t d); + void* (*unpack_float)(void* data, float d); + void* (*unpack_double)(void* data, double d); + void* (*unpack_big_int)(void* data, const void* b, unsigned int l); + void* (*unpack_big_float)(void* data, const void* b, unsigned int l); + void* (*unpack_nil)(void* data); + void* (*unpack_true)(void* data); + void* (*unpack_false)(void* data); + void* (*unpack_array_start)(void* data, unsigned int n); + void (*unpack_array_item)(void* data, void* c, void* o); + void* (*unpack_map_start)(void* data, unsigned int n); + void (*unpack_map_item)(void* data, void* c, void* k, void* v); + void* (*unpack_string)(void* data, const void* b, size_t l); + void* (*unpack_raw)(void* data, const void* b, size_t l); + void* data; +} msgpack_unpack_t; + +msgpack_unpack_t* msgpack_unpack_new(void); +void msgpack_unpack_free(msgpack_unpack_t* ctx); +int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off); +void* msgpack_unpack_data(msgpack_unpack_t* ctx); + +#endif /* msgpack/unpack.h */ + + diff --git a/cpp/unpack_context.h b/cpp/unpack_context.h new file mode 100644 index 0000000..caf2271 --- /dev/null +++ b/cpp/unpack_context.h @@ -0,0 +1,13 @@ +#ifndef UNPACK_CONTEXT_H__ +#define UNPACK_CONTEXT_H__ + +#include "unpack.h" + +typedef void* msgpack_object; + +typedef msgpack_unpack_t msgpack_unpack_context; + +#include "msgpack/unpack/inline_context.h" + +#endif /* unpack_context.h */ + diff --git a/msgpack/pack/inline_context.h b/msgpack/pack/inline_context.h new file mode 100644 index 0000000..139597f --- /dev/null +++ b/msgpack/pack/inline_context.h @@ -0,0 +1,2 @@ + + diff --git a/msgpack/pack/inline_impl.h b/msgpack/pack/inline_impl.h new file mode 100644 index 0000000..5c4bfed --- /dev/null +++ b/msgpack/pack/inline_impl.h @@ -0,0 +1,287 @@ +/* + * MessagePack packing routine + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_INLINE_IMPL_H__ +#define MSGPACK_PACK_INLINE_IMPL_H__ + +#include +#include + +#ifdef __LITTLE_ENDIAN__ + +#define STORE_16(d) \ + ((char*)&d)[1], ((char*)&d)[0] + +#define STORE_32(d) \ + ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] + +#define STORE_64(d) \ + ((char*)&d)[7], ((char*)&d)[6], ((char*)&d)[5], ((char*)&d)[4], \ + ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] + +#elif __BIG_ENDIAN__ + +#define STORE_16(d) \ + ((char*)&d)[2], ((char*)&d)[3] + +#define STORE_32(d) \ + ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3] + +#define STORE_32(d) \ + ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3], \ + ((char*)&d)[4], ((char*)&d)[5], ((char*)&d)[6], ((char*)&d)[7] + +#endif + + +/* + * Integer + */ + +// wrapper +inline void msgpack_pack_int(msgpack_pack_context x, int d) +{ + if(d < -32) { + if(d < -32768) { // signed 32 + const unsigned char buf[5] = {0xd2, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } else if(d < -128) { // signed 16 + const unsigned char buf[3] = {0xd1, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { // signed 8 + const unsigned char buf[2] = {0xd0, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } + } else if(d < 128) { // fixnum + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + } else { + if(d < 256) { + // unsigned 8 + const unsigned char buf[2] = {0xcc, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } else if(d < 65536) { + // unsigned 16 + const unsigned char buf[3] = {0xcd, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + // unsigned 32 + const unsigned char buf[5] = {0xce, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } + } +} + +// wrapper +inline void msgpack_pack_unsigned_int(msgpack_pack_context x, unsigned int d) +{ + if(d < 128) { + // fixnum + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + } else if(d < 256) { + // unsigned 8 + const unsigned char buf[2] = {0xcc, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } else if(d < 65536) { + // unsigned 16 + const unsigned char buf[3] = {0xcd, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + // unsigned 32 + const unsigned char buf[5] = {0xce, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + +inline void msgpack_pack_unsigned_int_8(msgpack_pack_context x, uint8_t d) +{ + if(d < 128) { + msgpack_pack_append_buffer(x, &d, 1); + } else { + const unsigned char buf[2] = {0xcc, d}; + msgpack_pack_append_buffer(x, buf, 2); + } +} + +inline void msgpack_pack_unsigned_int_16(msgpack_pack_context x, uint16_t d) +{ + const unsigned char buf[3] = {0xcd, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +inline void msgpack_pack_unsigned_int_32(msgpack_pack_context x, uint32_t d) +{ + const unsigned char buf[5] = {0xce, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +inline void msgpack_pack_unsigned_int_64(msgpack_pack_context x, uint64_t d) +{ + // FIXME + const unsigned char buf[9] = {0xcf, STORE_64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +inline void msgpack_pack_signed_int_8(msgpack_pack_context x, int8_t d) +{ + if(d > 0) { + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + } else if(d >= -32) { + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + } else { + const unsigned char buf[2] = {0xd0, d}; + msgpack_pack_append_buffer(x, buf, 2); + } +} + +inline void msgpack_pack_signed_int_16(msgpack_pack_context x, int16_t d) +{ + const unsigned char buf[3] = {0xd1, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +inline void msgpack_pack_signed_int_32(msgpack_pack_context x, int32_t d) +{ + const unsigned char buf[5] = {0xd2, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +inline void msgpack_pack_signed_int_64(msgpack_pack_context x, int64_t d) +{ + // FIXME + const unsigned char buf[9] = {0xd3, STORE_64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + + +/* + * Float + */ + +inline void msgpack_pack_float(msgpack_pack_context x, float d) +{ + uint32_t n = *((uint32_t*)&d); // FIXME + const unsigned char buf[5] = {0xca, STORE_32(n)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +inline void msgpack_pack_double(msgpack_pack_context x, double d) +{ + uint64_t n = *((uint64_t*)&d); // FIXME + const unsigned char buf[9] = {0xcb, STORE_64(n)}; + msgpack_pack_append_buffer(x, buf, 9); +} + + +/* + * Nil + */ + +inline void msgpack_pack_nil(msgpack_pack_context x) +{ + static const unsigned char d = 0xc0; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Boolean + */ +inline void msgpack_pack_true(msgpack_pack_context x) +{ + static const unsigned char d = 0xc3; + msgpack_pack_append_buffer(x, &d, 1); +} + +inline void msgpack_pack_false(msgpack_pack_context x) +{ + static const unsigned char d = 0xc2; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Array + */ + +inline void msgpack_pack_array(msgpack_pack_context x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x90 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xdc, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdd, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Map + */ + +inline void msgpack_pack_map(msgpack_pack_context x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x80 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xde, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdf, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * String + */ + +inline void msgpack_pack_string(msgpack_pack_context x, const char* b) +{ + uint32_t l = strlen(b); + msgpack_pack_append_buffer(x, (const unsigned char*)b, l+1); +} + +inline void msgpack_pack_raw(msgpack_pack_context x, const void* b, size_t l) +{ + if(l < 32) { + unsigned char d = 0xa0 | l; + msgpack_pack_append_buffer(x, &d, 1); + } else if(l < 65536) { + uint16_t d = (uint16_t)l; + unsigned char buf[3] = {0xda, STORE_16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)l; + unsigned char buf[5] = {0xdb, STORE_32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } + msgpack_pack_append_buffer(x, b, l); +} + + +#endif /* msgpack/pack/inline_impl.h */ + diff --git a/msgpack/unpack/callback.h b/msgpack/unpack/callback.h new file mode 100644 index 0000000..b058a15 --- /dev/null +++ b/msgpack/unpack/callback.h @@ -0,0 +1,24 @@ + +msgpack_object msgpack_unpack_init(msgpack_unpack_context* x); +msgpack_object msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d); +msgpack_object msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d); +msgpack_object msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d); +msgpack_object msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d); +msgpack_object msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d); +msgpack_object msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d); +msgpack_object msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d); +msgpack_object msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d); +msgpack_object msgpack_unpack_float(msgpack_unpack_context* x, float d); +msgpack_object msgpack_unpack_double(msgpack_unpack_context* x, double d); +msgpack_object msgpack_unpack_big_int(msgpack_unpack_context* x, const void* b, unsigned int l); +msgpack_object msgpack_unpack_big_float(msgpack_unpack_context* x, const void* b, unsigned int l); +msgpack_object msgpack_unpack_nil(msgpack_unpack_context* x); +msgpack_object msgpack_unpack_true(msgpack_unpack_context* x); +msgpack_object msgpack_unpack_false(msgpack_unpack_context* x); +msgpack_object msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n); + void msgpack_unpack_array_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object o); +msgpack_object msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n); + void msgpack_unpack_map_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object k, msgpack_object v); +msgpack_object msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l); +msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l); + diff --git a/msgpack/unpack/inline_context.h b/msgpack/unpack/inline_context.h new file mode 100644 index 0000000..aecd566 --- /dev/null +++ b/msgpack/unpack/inline_context.h @@ -0,0 +1,52 @@ +/* + * MessagePack unpacking routine + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_INLINE_CONTEXT_H__ +#define MSGPACK_UNPACK_INLINE_CONTEXT_H__ + +#include +#include + +#ifndef MSG_STACK_SIZE +#define MSG_STACK_SIZE 16 +#endif + +typedef struct { + msgpack_object obj; + size_t count; + unsigned int ct; + union { + const unsigned char* terminal_trail_start; + msgpack_object map_key; + } tmp; +} msgpack_unpacker_stack; + +typedef struct { + msgpack_unpack_context user; // must be first + unsigned int cs; + size_t trail; + unsigned int top; + msgpack_unpacker_stack stack[MSG_STACK_SIZE]; +} msgpack_unpacker; + +void msgpack_unpacker_init(msgpack_unpacker* ctx); +int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len, size_t* off); +#define msgpack_unpacker_data(unpacker) (unpacker)->stack[0].obj + + +#endif /* msgpack/unpack/inline_context.h */ + diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h new file mode 100644 index 0000000..ec7f0fc --- /dev/null +++ b/msgpack/unpack/inline_impl.h @@ -0,0 +1,438 @@ +/* + * MessagePack unpacking routine + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_INLINE_IMPL_H__ +#define MSGPACK_UNPACK_INLINE_IMPL_H__ + +#include +#include +#include +/*#include */ + +// Positive FixNum 0xxxxxxx 0x00 - 0x7f +// Negative FixNum 111xxxxx 0xe0 - 0xff +// Variable 110xxxxx 0xc0 - 0xdf +// nil 00000 0xc0 +// string 00001 0xc1 +// false 00010 0xc2 +// true 00011 0xc3 +// (?) 00100 0xc4 +// (?) 00101 0xc5 +// (?) 00110 0xc6 +// (?) 00111 0xc7 +// (?) 01000 0xc8 +// (?) 01001 0xc9 +// float 01010 0xca +// double 01011 0xcb +// uint 8 01100 0xcc +// uint 16 01101 0xcd +// uint 32 01110 0xce +// uint 64 01111 0xcf +// int 8 10000 0xd0 +// int 16 10001 0xd1 +// int 32 10010 0xd2 +// int 64 10011 0xd3 +// (?) 10100 0xd4 +// (?) 10101 0xd5 +// (big float 16) 10110 0xd6 +// (big float 32) 10111 0xd7 +// (big integer 16) 11000 0xd8 +// (big integer 32) 11001 0xd9 +// raw 16 11010 0xda +// raw 32 11011 0xdb +// array 16 11100 0xdc +// array 32 11101 0xdd +// map 16 11110 0xde +// map 32 11111 0xdf +// FixRaw 101xxxxx 0xa0 - 0xbf +// FixArray 1001xxxx 0x90 - 0x9f +// FixMap 1000xxxx 0x80 - 0x8f + + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +static inline uint64_t ntohll(uint64_t x) { +#ifdef __LITTLE_ENDIAN__ // FIXME +#if defined(__bswap_64) + return __bswap_64(x); +#elif defined(__DARWIN_OSSwapInt64) + return __DARWIN_OSSwapInt64(x); +#else + return ((x << 56) & 0xff00000000000000ULL ) | + ((x << 40) & 0x00ff000000000000ULL ) | + ((x << 24) & 0x0000ff0000000000ULL ) | + ((x << 8) & 0x000000ff00000000ULL ) | + ((x >> 8) & 0x00000000ff000000ULL ) | + ((x >> 24) & 0x0000000000ff0000ULL ) | + ((x >> 40) & 0x000000000000ff00ULL ) | + ((x >> 56) & 0x00000000000000ffULL ) ; +#endif +#else + return x; +#endif +} + +typedef enum { + CS_HEADER = 0x00, // nil + + CS_STRING = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true + + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, + + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UNSIGNED_INT_8 = 0x0c, + CS_UNSIGNED_INT_16 = 0x0d, + CS_UNSIGNED_INT_32 = 0x0e, + CS_UNSIGNED_INT_64 = 0x0f, + CS_SIGNED_INT_8 = 0x10, + CS_SIGNED_INT_16 = 0x11, + CS_SIGNED_INT_32 = 0x12, + CS_SIGNED_INT_64 = 0x13, + + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, + + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, +} current_state_t; + + +typedef enum { + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, +} container_type_t; + + +void msgpack_unpacker_init(msgpack_unpacker* ctx) +{ + memset(ctx, 0, sizeof(msgpack_unpacker)); // FIXME init ctx->user? + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + ctx->stack[0].obj = msgpack_unpack_init(&ctx->user); +} + +int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; + + size_t trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + msgpack_unpacker_stack* stack = ctx->stack; + msgpack_unpack_context* user = &ctx->user; + + msgpack_object obj; + + int ret; + +#define push_simple_value(func) \ + obj = func(user); \ + /*printf("obj %d\n",obj);*/ \ + goto _push +#define push_fixed_value(func, arg) \ + obj = func(user, arg); \ + /*printf("obj %d\n",obj);*/ \ + goto _push +#define push_variable_value(func, arg, arglen) \ + obj = func(user, arg, arglen); \ + /*printf("obj %d\n",obj);*/ \ + goto _push + +#define again_terminal_trail(_cs, from) \ + cs = _cs; \ + stack[top].tmp.terminal_trail_start = from; \ + goto _terminal_trail_again +#define again_fixed_trail(_cs, trail_len) \ + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again +#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again + +#define start_container(func, count_, ct_) \ + stack[top].obj = func(user, count_); \ + if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if(top >= MSG_STACK_SIZE) { goto _failed; } \ + stack[top].ct = ct_; \ + stack[top].count = count_; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + ++top; \ + goto _header_again + +#define NEXT_CS(p) \ + ((unsigned int)*p & 0x1f) + +#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) +#define PTR_CAST_16(ptr) ntohs(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) ntohl(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) ntohll(*(uint64_t*)ptr) + + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + switch(*p) { + case 0x00 ... 0x7f: // Positive Fixnum + push_fixed_value(msgpack_unpack_unsigned_int_8, *(uint8_t*)p); + case 0xe0 ... 0xff: // Negative Fixnum + push_fixed_value(msgpack_unpack_signed_int_8, *(int8_t*)p); + case 0xc0 ... 0xdf: // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(msgpack_unpack_nil); + case 0xc1: // string + again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(msgpack_unpack_false); + case 0xc3: // true + push_simple_value(msgpack_unpack_true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + case 0xa0 ... 0xbf: // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + case 0x90 ... 0x9f: // FixArray + start_container(msgpack_unpack_array_start, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + case 0x80 ... 0x8f: // FixMap + start_container(msgpack_unpack_map_start, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + + default: + goto _failed; + } + // end CS_HEADER + + + _terminal_trail_again: + ++p; + + case CS_STRING: + if(*p == 0) { + const unsigned char* start = stack[top].tmp.terminal_trail_start; + obj = msgpack_unpack_string(user, start, p-start); + goto _push; + } + goto _terminal_trail_again; + + + _fixed_trail_again: + ++p; + + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + uint32_t x = PTR_CAST_32(n); // FIXME + push_fixed_value(msgpack_unpack_float, *((float*)&x)); } + case CS_DOUBLE: { + uint64_t x = PTR_CAST_64(n); // FIXME + push_fixed_value(msgpack_unpack_double, *((double*)&x)); } + case CS_UNSIGNED_INT_8: + push_fixed_value(msgpack_unpack_unsigned_int_8, (uint8_t)PTR_CAST_8(n)); + case CS_UNSIGNED_INT_16: + push_fixed_value(msgpack_unpack_unsigned_int_16, (uint16_t)PTR_CAST_16(n)); + case CS_UNSIGNED_INT_32: + push_fixed_value(msgpack_unpack_unsigned_int_32, (uint32_t)PTR_CAST_32(n)); + case CS_UNSIGNED_INT_64: + push_fixed_value(msgpack_unpack_unsigned_int_64, (uint64_t)PTR_CAST_64(n)); + + case CS_SIGNED_INT_8: + push_fixed_value(msgpack_unpack_signed_int_8, (int8_t)PTR_CAST_8(n)); + case CS_SIGNED_INT_16: + push_fixed_value(msgpack_unpack_signed_int_16, (int16_t)PTR_CAST_16(n)); + case CS_SIGNED_INT_32: + push_fixed_value(msgpack_unpack_signed_int_32, (int32_t)PTR_CAST_32(n)); + case CS_SIGNED_INT_64: + push_fixed_value(msgpack_unpack_signed_int_64, (int64_t)PTR_CAST_64(n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(msgpack_unpack_big_int, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(msgpack_unpack_big_float, n, trail); + + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(msgpack_unpack_raw, n, trail); + + case CS_ARRAY_16: + start_container(msgpack_unpack_array_start, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + start_container(msgpack_unpack_array_start, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + + case CS_MAP_16: + start_container(msgpack_unpack_map_start, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + case CS_MAP_32: + start_container(msgpack_unpack_map_start, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + + default: + goto _failed; + } + } + +_push: + if(top == 0) { goto _finish; } + msgpack_unpacker_stack* c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + msgpack_unpack_array_item(user, c->obj, obj); + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->tmp.map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + msgpack_unpack_map_item(user, c->obj, c->tmp.map_key, obj); + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } + +_header_again: + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; + + +_finish: + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; + +_failed: + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; + +_out: + ret = 0; + goto _end; + +_end: + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; + + return ret; +} + + +#endif /* msgpack/unpack/inline_impl.h */ + diff --git a/ruby/Makefile b/ruby/Makefile new file mode 100644 index 0000000..6ef04b0 --- /dev/null +++ b/ruby/Makefile @@ -0,0 +1,153 @@ + +SHELL = /bin/sh + +#### Start of system configuration section. #### + +srcdir = . +topdir = /Users/frsyuki/ports/lib/ruby/1.8/i686-darwin9.1.0 +hdrdir = $(topdir) +VPATH = $(srcdir):$(topdir):$(hdrdir) +prefix = $(DESTDIR)/Users/frsyuki/ports +exec_prefix = $(prefix) +sitedir = $(prefix)/lib/ruby/site_ruby +rubylibdir = $(libdir)/ruby/$(ruby_version) +docdir = $(datarootdir)/doc/$(PACKAGE) +dvidir = $(docdir) +datarootdir = $(prefix)/share +archdir = $(rubylibdir)/$(arch) +sbindir = $(exec_prefix)/sbin +psdir = $(docdir) +vendordir = $(prefix)/lib/ruby/vendor_ruby +localedir = $(datarootdir)/locale +htmldir = $(docdir) +datadir = $(datarootdir) +includedir = $(prefix)/include +infodir = $(datarootdir)/info +sysconfdir = $(prefix)/etc +mandir = $(DESTDIR)/Users/frsyuki/ports/share/man +libdir = $(exec_prefix)/lib +sharedstatedir = $(prefix)/com +oldincludedir = $(DESTDIR)/usr/include +pdfdir = $(docdir) +sitearchdir = $(sitelibdir)/$(sitearch) +vendorarchdir = $(vendorlibdir)/$(vendorarch) +bindir = $(exec_prefix)/bin +localstatedir = $(prefix)/var +vendorlibdir = $(vendordir)/$(ruby_version) +sitelibdir = $(sitedir)/$(ruby_version) +libexecdir = $(exec_prefix)/libexec + +CC = /usr/bin/gcc-4.0 +LIBRUBY = $(LIBRUBY_SO) +LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a +LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME) +LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static + +RUBY_EXTCONF_H = +CFLAGS = -fno-common -O2 -fno-common -pipe -fno-common -I.. -Wall -O9 +INCFLAGS = -I. -I$(topdir) -I$(hdrdir) -I$(srcdir) +CPPFLAGS = -I/Users/frsyuki/ports/include +CXXFLAGS = $(CFLAGS) +DLDFLAGS = -L. -L/Users/frsyuki/ports/lib +LDSHARED = cc -dynamic -bundle -undefined suppress -flat_namespace +AR = ar +EXEEXT = + +RUBY_INSTALL_NAME = ruby +RUBY_SO_NAME = ruby +arch = i686-darwin9.1.0 +sitearch = i686-darwin9.1.0 +vendorarch = i686-darwin9.1.0 +ruby_version = 1.8 +ruby = /Users/frsyuki/ports/bin/ruby +RUBY = $(ruby) +RM = rm -f +MAKEDIRS = mkdir -p +INSTALL = /usr/bin/install +INSTALL_PROG = $(INSTALL) -m 0755 +INSTALL_DATA = $(INSTALL) -m 644 +COPY = cp + +#### End of system configuration section. #### + +preload = + +libpath = . $(libdir) +LIBPATH = -L"." -L"$(libdir)" +DEFFILE = + +CLEANFILES = mkmf.log +DISTCLEANFILES = + +extout = +extout_prefix = +target_prefix = +LOCAL_LIBS = +LIBS = $(LIBRUBYARG_SHARED) -lpthread -ldl -lobjc +SRCS = pack.c rbinit.c unpack.c unpack_inline.c +OBJS = pack.o rbinit.o unpack.o unpack_inline.o +TARGET = msgpack +DLLIB = $(TARGET).bundle +EXTSTATIC = +STATIC_LIB = + +RUBYCOMMONDIR = $(sitedir)$(target_prefix) +RUBYLIBDIR = $(sitelibdir)$(target_prefix) +RUBYARCHDIR = $(sitearchdir)$(target_prefix) + +TARGET_SO = $(DLLIB) +CLEANLIBS = $(TARGET).bundle $(TARGET).il? $(TARGET).tds $(TARGET).map +CLEANOBJS = *.o *.a *.s[ol] *.pdb *.exp *.bak + +all: $(DLLIB) +static: $(STATIC_LIB) + +clean: + @-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) + +distclean: clean + @-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log + @-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES) + +realclean: distclean +install: install-so install-rb + +install-so: $(RUBYARCHDIR) +install-so: $(RUBYARCHDIR)/$(DLLIB) +$(RUBYARCHDIR)/$(DLLIB): $(DLLIB) + $(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR) +install-rb: pre-install-rb install-rb-default +install-rb-default: pre-install-rb-default +pre-install-rb: Makefile +pre-install-rb-default: Makefile +$(RUBYARCHDIR): + $(MAKEDIRS) $@ + +site-install: site-install-so site-install-rb +site-install-so: install-so +site-install-rb: install-rb + +.SUFFIXES: .c .m .cc .cxx .cpp .C .o + +.cc.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.cxx.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.cpp.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.C.o: + $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< + +.c.o: + $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $< + +$(DLLIB): $(OBJS) + @-$(RM) $@ + $(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS) + + + +$(OBJS): ruby.h defines.h diff --git a/ruby/bench.rb b/ruby/bench.rb new file mode 100644 index 0000000..1e6e27b --- /dev/null +++ b/ruby/bench.rb @@ -0,0 +1,60 @@ +require 'rubygems' +require 'json' +require 'msgpack' + +def show10(str) + puts "#{str.length/1024} KB" + puts str[0, 10].unpack('C*').map{|x|"%02x"%x}.join(' ') + " ..." +end + +ary = [] +i = 0 +while i < (1<<23) + ary << (1<<23) + #ary << i + i += 1 +end + +GC.start + +puts "----" +puts "MessagePack" +a = Time.now +packed = MessagePack::pack(ary) +b = Time.now +show10(packed) +puts "#{b-a} sec." + +GC.start + +puts "----" +puts "JSON" +a = Time.now +json = ary.to_json +b = Time.now +show10(json) +puts "#{b-a} sec." + +ary = nil +GC.start + + +puts "----" +puts "MessagePack" +a = Time.now +ary = MessagePack::unpack(packed) +b = Time.now +puts "#{b-a} sec." + +ary = nil +GC.start + + +puts "----" +puts "JSON" +a = Time.now +ary = JSON::load(json) +b = Time.now +puts "#{b-a} sec." + + diff --git a/ruby/extconf.rb b/ruby/extconf.rb new file mode 100644 index 0000000..88abb55 --- /dev/null +++ b/ruby/extconf.rb @@ -0,0 +1,4 @@ +require 'mkmf' +$CFLAGS << " -I.. -Wall -O9" +create_makefile('msgpack') + diff --git a/ruby/gem.sh b/ruby/gem.sh new file mode 100755 index 0000000..9d9f429 --- /dev/null +++ b/ruby/gem.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +cp extconf.rb gem/ext/ +cp pack.c gem/ext/ +cp pack.h gem/ext/ +cp pack_inline.h gem/ext/ +cp rbinit.c gem/ext/ +cp unpack.c gem/ext/ +cp unpack.h gem/ext/ +cp unpack_context.h gem/ext/ +cp unpack_inline.c gem/ext/ +cp ../README gem/README.txt +cp ../msgpack/pack/inline_context.h gem/msgpack/pack/ +cp ../msgpack/pack/inline_impl.h gem/msgpack/pack/ +cp ../msgpack/unpack/inline_context.h gem/msgpack/unpack/ +cp ../msgpack/unpack/inline_impl.h gem/msgpack/unpack/ + +cd gem && rake --trace package + diff --git a/ruby/gem/AUTHORS b/ruby/gem/AUTHORS new file mode 100644 index 0000000..ababacb --- /dev/null +++ b/ruby/gem/AUTHORS @@ -0,0 +1 @@ +FURUHASHI Sadayuki diff --git a/ruby/gem/History.txt b/ruby/gem/History.txt new file mode 100644 index 0000000..e69de29 diff --git a/ruby/gem/License.txt b/ruby/gem/License.txt new file mode 100644 index 0000000..f4000b7 --- /dev/null +++ b/ruby/gem/License.txt @@ -0,0 +1,13 @@ +Copyright 2008 Furuhashi Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ruby/gem/Manifest.txt b/ruby/gem/Manifest.txt new file mode 100644 index 0000000..fac8555 --- /dev/null +++ b/ruby/gem/Manifest.txt @@ -0,0 +1,26 @@ +License.txt +Manifest.txt +README.txt +Rakefile +config/hoe.rb +config/requirements.rb +ext/extconf.rb +ext/pack.c +ext/pack.h +ext/pack_inline.h +ext/rbinit.c +ext/unpack.c +ext/unpack.h +ext/unpack_context.h +ext/unpack_inline.c +msgpack/pack/inline_context.h +msgpack/pack/inline_impl.h +msgpack/unpack/inline_context.h +msgpack/unpack/inline_impl.h +lib/msgpack/version.rb +script/console +script/destroy +script/generate +setup.rb +tasks/deployment.rake +tasks/environment.rake diff --git a/ruby/gem/PostInstall.txt b/ruby/gem/PostInstall.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ruby/gem/PostInstall.txt @@ -0,0 +1 @@ + diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile new file mode 100644 index 0000000..e469154 --- /dev/null +++ b/ruby/gem/Rakefile @@ -0,0 +1,4 @@ +require 'config/requirements' +require 'config/hoe' # setup Hoe + all gem configuration + +Dir['tasks/**/*.rake'].each { |rake| load rake } \ No newline at end of file diff --git a/ruby/gem/config/hoe.rb b/ruby/gem/config/hoe.rb new file mode 100644 index 0000000..8500acf --- /dev/null +++ b/ruby/gem/config/hoe.rb @@ -0,0 +1,75 @@ +require 'msgpack/version' + +AUTHOR = 'FURUHASHI Sadayuki' # can also be an array of Authors +EMAIL = "fr _at_ syuki.skr.jp" +DESCRIPTION = "An object-oriented parser generator based on Parser Expression Grammar" +GEM_NAME = 'msgpack' # what ppl will type to install your gem +RUBYFORGE_PROJECT = 'msgpack' # The unix name for your project +HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" +DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}" +EXTRA_DEPENDENCIES = [ +# ['activesupport', '>= 1.3.1'] +] # An array of rubygem dependencies [name, version] + +@config_file = "~/.rubyforge/user-config.yml" +@config = nil +RUBYFORGE_USERNAME = "unknown" +def rubyforge_username + unless @config + begin + @config = YAML.load(File.read(File.expand_path(@config_file))) + rescue + puts <<-EOS +ERROR: No rubyforge config file found: #{@config_file} +Run 'rubyforge setup' to prepare your env for access to Rubyforge + - See http://newgem.rubyforge.org/rubyforge.html for more details + EOS + exit + end + end + RUBYFORGE_USERNAME.replace @config["username"] +end + + +REV = nil +# UNCOMMENT IF REQUIRED: +# REV = YAML.load(`svn info`)['Revision'] +VERS = MessagePack::VERSION::STRING + (REV ? ".#{REV}" : "") +RDOC_OPTS = ['--quiet', '--title', 'msgpack documentation', + "--opname", "index.html", + "--line-numbers", + "--main", "README", + "--inline-source"] + +class Hoe + def extra_deps + @extra_deps.reject! { |x| Array(x).first == 'hoe' } + @extra_deps + end +end + +# Generate all the Rake tasks +# Run 'rake -T' to see list of generated tasks (from gem root directory) +$hoe = Hoe.new(GEM_NAME, VERS) do |p| + p.developer(AUTHOR, EMAIL) + p.description = DESCRIPTION + p.summary = DESCRIPTION + p.url = HOMEPATH + p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT + p.test_globs = ["test/**/test_*.rb"] + p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store'] #An array of file patterns to delete on clean. + + # == Optional + p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n") + #p.extra_deps = EXTRA_DEPENDENCIES + + p.spec_extras = { # A hash of extra values to set in the gemspec. + :extensions => %w[ext/extconf.rb] + } +end + +CHANGES = $hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n") +PATH = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}" +$hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc') +$hoe.rsync_args = '-av --delete --ignore-errors' +$hoe.spec.post_install_message = File.open(File.dirname(__FILE__) + "/../PostInstall.txt").read rescue "" diff --git a/ruby/gem/config/requirements.rb b/ruby/gem/config/requirements.rb new file mode 100644 index 0000000..9292b69 --- /dev/null +++ b/ruby/gem/config/requirements.rb @@ -0,0 +1,15 @@ +require 'fileutils' +include FileUtils + +require 'rubygems' +%w[rake hoe newgem rubigen].each do |req_gem| + begin + require req_gem + rescue LoadError + puts "This Rakefile requires the '#{req_gem}' RubyGem." + puts "Installation: gem install #{req_gem} -y" + exit + end +end + +$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib])) diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb new file mode 100644 index 0000000..44d1dc0 --- /dev/null +++ b/ruby/gem/lib/msgpack/version.rb @@ -0,0 +1,9 @@ +module MessagePack + module VERSION #:nodoc: + MAJOR = 0 + MINOR = 0 + TINY = 1 + + STRING = [MAJOR, MINOR, TINY].join('.') + end +end diff --git a/ruby/gem/script/console b/ruby/gem/script/console new file mode 100755 index 0000000..76f32a0 --- /dev/null +++ b/ruby/gem/script/console @@ -0,0 +1,10 @@ +#!/usr/bin/env ruby +# File: script/console +irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb' + +libs = " -r irb/completion" +# Perhaps use a console_lib to store any extra methods I may want available in the cosole +# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}" +libs << " -r #{File.dirname(__FILE__) + '/../lib/msgpack.rb'}" +puts "Loading msgpack gem" +exec "#{irb} #{libs} --simple-prompt" diff --git a/ruby/gem/script/destroy b/ruby/gem/script/destroy new file mode 100755 index 0000000..e48464d --- /dev/null +++ b/ruby/gem/script/destroy @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) + +begin + require 'rubigen' +rescue LoadError + require 'rubygems' + require 'rubigen' +end +require 'rubigen/scripts/destroy' + +ARGV.shift if ['--help', '-h'].include?(ARGV[0]) +RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit] +RubiGen::Scripts::Destroy.new.run(ARGV) diff --git a/ruby/gem/script/generate b/ruby/gem/script/generate new file mode 100755 index 0000000..c27f655 --- /dev/null +++ b/ruby/gem/script/generate @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) + +begin + require 'rubigen' +rescue LoadError + require 'rubygems' + require 'rubigen' +end +require 'rubigen/scripts/generate' + +ARGV.shift if ['--help', '-h'].include?(ARGV[0]) +RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit] +RubiGen::Scripts::Generate.new.run(ARGV) diff --git a/ruby/gem/script/txt2html b/ruby/gem/script/txt2html new file mode 100755 index 0000000..09c583f --- /dev/null +++ b/ruby/gem/script/txt2html @@ -0,0 +1,82 @@ +#!/usr/bin/env ruby + +GEM_NAME = 'msgpack' # what ppl will type to install your gem +RUBYFORGE_PROJECT = 'msgpack' + +require 'rubygems' +begin + require 'newgem' + require 'rubyforge' +rescue LoadError + puts "\n\nGenerating the website requires the newgem RubyGem" + puts "Install: gem install newgem\n\n" + exit(1) +end +require 'redcloth' +require 'syntax/convertors/html' +require 'erb' +require File.dirname(__FILE__) + "/../lib/#{GEM_NAME}/version.rb" + +version = MessagePack::VERSION::STRING +download = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}" + +def rubyforge_project_id + RubyForge.new.autoconfig["group_ids"][RUBYFORGE_PROJECT] +end + +class Fixnum + def ordinal + # teens + return 'th' if (10..19).include?(self % 100) + # others + case self % 10 + when 1: return 'st' + when 2: return 'nd' + when 3: return 'rd' + else return 'th' + end + end +end + +class Time + def pretty + return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}" + end +end + +def convert_syntax(syntax, source) + return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^
|
$!,'') +end + +if ARGV.length >= 1 + src, template = ARGV + template ||= File.join(File.dirname(__FILE__), '/../website/template.html.erb') +else + puts("Usage: #{File.split($0).last} source.txt [template.html.erb] > output.html") + exit! +end + +template = ERB.new(File.open(template).read) + +title = nil +body = nil +File.open(src) do |fsrc| + title_text = fsrc.readline + body_text_template = fsrc.read + body_text = ERB.new(body_text_template).result(binding) + syntax_items = [] + body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)!m){ + ident = syntax_items.length + element, syntax, source = $1, $2, $3 + syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}" + "syntax-temp-#{ident}" + } + title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip + body = RedCloth.new(body_text).to_html + body.gsub!(%r!(?:
)?syntax-temp-(\d+)(?:
)?!){ syntax_items[$1.to_i] } +end +stat = File.stat(src) +created = stat.ctime +modified = stat.mtime + +$stdout << template.result(binding) diff --git a/ruby/gem/setup.rb b/ruby/gem/setup.rb new file mode 100644 index 0000000..424a5f3 --- /dev/null +++ b/ruby/gem/setup.rb @@ -0,0 +1,1585 @@ +# +# setup.rb +# +# Copyright (c) 2000-2005 Minero Aoki +# +# This program is free software. +# You can distribute/modify this program under the terms of +# the GNU LGPL, Lesser General Public License version 2.1. +# + +unless Enumerable.method_defined?(:map) # Ruby 1.4.6 + module Enumerable + alias map collect + end +end + +unless File.respond_to?(:read) # Ruby 1.6 + def File.read(fname) + open(fname) {|f| + return f.read + } + end +end + +unless Errno.const_defined?(:ENOTEMPTY) # Windows? + module Errno + class ENOTEMPTY + # We do not raise this exception, implementation is not needed. + end + end +end + +def File.binread(fname) + open(fname, 'rb') {|f| + return f.read + } +end + +# for corrupted Windows' stat(2) +def File.dir?(path) + File.directory?((path[-1,1] == '/') ? path : path + '/') +end + + +class ConfigTable + + include Enumerable + + def initialize(rbconfig) + @rbconfig = rbconfig + @items = [] + @table = {} + # options + @install_prefix = nil + @config_opt = nil + @verbose = true + @no_harm = false + end + + attr_accessor :install_prefix + attr_accessor :config_opt + + attr_writer :verbose + + def verbose? + @verbose + end + + attr_writer :no_harm + + def no_harm? + @no_harm + end + + def [](key) + lookup(key).resolve(self) + end + + def []=(key, val) + lookup(key).set val + end + + def names + @items.map {|i| i.name } + end + + def each(&block) + @items.each(&block) + end + + def key?(name) + @table.key?(name) + end + + def lookup(name) + @table[name] or setup_rb_error "no such config item: #{name}" + end + + def add(item) + @items.push item + @table[item.name] = item + end + + def remove(name) + item = lookup(name) + @items.delete_if {|i| i.name == name } + @table.delete_if {|name, i| i.name == name } + item + end + + def load_script(path, inst = nil) + if File.file?(path) + MetaConfigEnvironment.new(self, inst).instance_eval File.read(path), path + end + end + + def savefile + '.config' + end + + def load_savefile + begin + File.foreach(savefile()) do |line| + k, v = *line.split(/=/, 2) + self[k] = v.strip + end + rescue Errno::ENOENT + setup_rb_error $!.message + "\n#{File.basename($0)} config first" + end + end + + def save + @items.each {|i| i.value } + File.open(savefile(), 'w') {|f| + @items.each do |i| + f.printf "%s=%s\n", i.name, i.value if i.value? and i.value + end + } + end + + def load_standard_entries + standard_entries(@rbconfig).each do |ent| + add ent + end + end + + def standard_entries(rbconfig) + c = rbconfig + + rubypath = File.join(c['bindir'], c['ruby_install_name'] + c['EXEEXT']) + + major = c['MAJOR'].to_i + minor = c['MINOR'].to_i + teeny = c['TEENY'].to_i + version = "#{major}.#{minor}" + + # ruby ver. >= 1.4.4? + newpath_p = ((major >= 2) or + ((major == 1) and + ((minor >= 5) or + ((minor == 4) and (teeny >= 4))))) + + if c['rubylibdir'] + # V > 1.6.3 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = c['rubylibdir'] + librubyverarch = c['archdir'] + siteruby = c['sitedir'] + siterubyver = c['sitelibdir'] + siterubyverarch = c['sitearchdir'] + elsif newpath_p + # 1.4.4 <= V <= 1.6.3 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = "#{c['prefix']}/lib/ruby/#{version}" + librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" + siteruby = c['sitedir'] + siterubyver = "$siteruby/#{version}" + siterubyverarch = "$siterubyver/#{c['arch']}" + else + # V < 1.4.4 + libruby = "#{c['prefix']}/lib/ruby" + librubyver = "#{c['prefix']}/lib/ruby/#{version}" + librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" + siteruby = "#{c['prefix']}/lib/ruby/#{version}/site_ruby" + siterubyver = siteruby + siterubyverarch = "$siterubyver/#{c['arch']}" + end + parameterize = lambda {|path| + path.sub(/\A#{Regexp.quote(c['prefix'])}/, '$prefix') + } + + if arg = c['configure_args'].split.detect {|arg| /--with-make-prog=/ =~ arg } + makeprog = arg.sub(/'/, '').split(/=/, 2)[1] + else + makeprog = 'make' + end + + [ + ExecItem.new('installdirs', 'std/site/home', + 'std: install under libruby; site: install under site_ruby; home: install under $HOME')\ + {|val, table| + case val + when 'std' + table['rbdir'] = '$librubyver' + table['sodir'] = '$librubyverarch' + when 'site' + table['rbdir'] = '$siterubyver' + table['sodir'] = '$siterubyverarch' + when 'home' + setup_rb_error '$HOME was not set' unless ENV['HOME'] + table['prefix'] = ENV['HOME'] + table['rbdir'] = '$libdir/ruby' + table['sodir'] = '$libdir/ruby' + end + }, + PathItem.new('prefix', 'path', c['prefix'], + 'path prefix of target environment'), + PathItem.new('bindir', 'path', parameterize.call(c['bindir']), + 'the directory for commands'), + PathItem.new('libdir', 'path', parameterize.call(c['libdir']), + 'the directory for libraries'), + PathItem.new('datadir', 'path', parameterize.call(c['datadir']), + 'the directory for shared data'), + PathItem.new('mandir', 'path', parameterize.call(c['mandir']), + 'the directory for man pages'), + PathItem.new('sysconfdir', 'path', parameterize.call(c['sysconfdir']), + 'the directory for system configuration files'), + PathItem.new('localstatedir', 'path', parameterize.call(c['localstatedir']), + 'the directory for local state data'), + PathItem.new('libruby', 'path', libruby, + 'the directory for ruby libraries'), + PathItem.new('librubyver', 'path', librubyver, + 'the directory for standard ruby libraries'), + PathItem.new('librubyverarch', 'path', librubyverarch, + 'the directory for standard ruby extensions'), + PathItem.new('siteruby', 'path', siteruby, + 'the directory for version-independent aux ruby libraries'), + PathItem.new('siterubyver', 'path', siterubyver, + 'the directory for aux ruby libraries'), + PathItem.new('siterubyverarch', 'path', siterubyverarch, + 'the directory for aux ruby binaries'), + PathItem.new('rbdir', 'path', '$siterubyver', + 'the directory for ruby scripts'), + PathItem.new('sodir', 'path', '$siterubyverarch', + 'the directory for ruby extentions'), + PathItem.new('rubypath', 'path', rubypath, + 'the path to set to #! line'), + ProgramItem.new('rubyprog', 'name', rubypath, + 'the ruby program using for installation'), + ProgramItem.new('makeprog', 'name', makeprog, + 'the make program to compile ruby extentions'), + SelectItem.new('shebang', 'all/ruby/never', 'ruby', + 'shebang line (#!) editing mode'), + BoolItem.new('without-ext', 'yes/no', 'no', + 'does not compile/install ruby extentions') + ] + end + private :standard_entries + + def load_multipackage_entries + multipackage_entries().each do |ent| + add ent + end + end + + def multipackage_entries + [ + PackageSelectionItem.new('with', 'name,name...', '', 'ALL', + 'package names that you want to install'), + PackageSelectionItem.new('without', 'name,name...', '', 'NONE', + 'package names that you do not want to install') + ] + end + private :multipackage_entries + + ALIASES = { + 'std-ruby' => 'librubyver', + 'stdruby' => 'librubyver', + 'rubylibdir' => 'librubyver', + 'archdir' => 'librubyverarch', + 'site-ruby-common' => 'siteruby', # For backward compatibility + 'site-ruby' => 'siterubyver', # For backward compatibility + 'bin-dir' => 'bindir', + 'bin-dir' => 'bindir', + 'rb-dir' => 'rbdir', + 'so-dir' => 'sodir', + 'data-dir' => 'datadir', + 'ruby-path' => 'rubypath', + 'ruby-prog' => 'rubyprog', + 'ruby' => 'rubyprog', + 'make-prog' => 'makeprog', + 'make' => 'makeprog' + } + + def fixup + ALIASES.each do |ali, name| + @table[ali] = @table[name] + end + @items.freeze + @table.freeze + @options_re = /\A--(#{@table.keys.join('|')})(?:=(.*))?\z/ + end + + def parse_opt(opt) + m = @options_re.match(opt) or setup_rb_error "config: unknown option #{opt}" + m.to_a[1,2] + end + + def dllext + @rbconfig['DLEXT'] + end + + def value_config?(name) + lookup(name).value? + end + + class Item + def initialize(name, template, default, desc) + @name = name.freeze + @template = template + @value = default + @default = default + @description = desc + end + + attr_reader :name + attr_reader :description + + attr_accessor :default + alias help_default default + + def help_opt + "--#{@name}=#{@template}" + end + + def value? + true + end + + def value + @value + end + + def resolve(table) + @value.gsub(%r<\$([^/]+)>) { table[$1] } + end + + def set(val) + @value = check(val) + end + + private + + def check(val) + setup_rb_error "config: --#{name} requires argument" unless val + val + end + end + + class BoolItem < Item + def config_type + 'bool' + end + + def help_opt + "--#{@name}" + end + + private + + def check(val) + return 'yes' unless val + case val + when /\Ay(es)?\z/i, /\At(rue)?\z/i then 'yes' + when /\An(o)?\z/i, /\Af(alse)\z/i then 'no' + else + setup_rb_error "config: --#{@name} accepts only yes/no for argument" + end + end + end + + class PathItem < Item + def config_type + 'path' + end + + private + + def check(path) + setup_rb_error "config: --#{@name} requires argument" unless path + path[0,1] == '$' ? path : File.expand_path(path) + end + end + + class ProgramItem < Item + def config_type + 'program' + end + end + + class SelectItem < Item + def initialize(name, selection, default, desc) + super + @ok = selection.split('/') + end + + def config_type + 'select' + end + + private + + def check(val) + unless @ok.include?(val.strip) + setup_rb_error "config: use --#{@name}=#{@template} (#{val})" + end + val.strip + end + end + + class ExecItem < Item + def initialize(name, selection, desc, &block) + super name, selection, nil, desc + @ok = selection.split('/') + @action = block + end + + def config_type + 'exec' + end + + def value? + false + end + + def resolve(table) + setup_rb_error "$#{name()} wrongly used as option value" + end + + undef set + + def evaluate(val, table) + v = val.strip.downcase + unless @ok.include?(v) + setup_rb_error "invalid option --#{@name}=#{val} (use #{@template})" + end + @action.call v, table + end + end + + class PackageSelectionItem < Item + def initialize(name, template, default, help_default, desc) + super name, template, default, desc + @help_default = help_default + end + + attr_reader :help_default + + def config_type + 'package' + end + + private + + def check(val) + unless File.dir?("packages/#{val}") + setup_rb_error "config: no such package: #{val}" + end + val + end + end + + class MetaConfigEnvironment + def initialize(config, installer) + @config = config + @installer = installer + end + + def config_names + @config.names + end + + def config?(name) + @config.key?(name) + end + + def bool_config?(name) + @config.lookup(name).config_type == 'bool' + end + + def path_config?(name) + @config.lookup(name).config_type == 'path' + end + + def value_config?(name) + @config.lookup(name).config_type != 'exec' + end + + def add_config(item) + @config.add item + end + + def add_bool_config(name, default, desc) + @config.add BoolItem.new(name, 'yes/no', default ? 'yes' : 'no', desc) + end + + def add_path_config(name, default, desc) + @config.add PathItem.new(name, 'path', default, desc) + end + + def set_config_default(name, default) + @config.lookup(name).default = default + end + + def remove_config(name) + @config.remove(name) + end + + # For only multipackage + def packages + raise '[setup.rb fatal] multi-package metaconfig API packages() called for single-package; contact application package vendor' unless @installer + @installer.packages + end + + # For only multipackage + def declare_packages(list) + raise '[setup.rb fatal] multi-package metaconfig API declare_packages() called for single-package; contact application package vendor' unless @installer + @installer.packages = list + end + end + +end # class ConfigTable + + +# This module requires: #verbose?, #no_harm? +module FileOperations + + def mkdir_p(dirname, prefix = nil) + dirname = prefix + File.expand_path(dirname) if prefix + $stderr.puts "mkdir -p #{dirname}" if verbose? + return if no_harm? + + # Does not check '/', it's too abnormal. + dirs = File.expand_path(dirname).split(%r<(?=/)>) + if /\A[a-z]:\z/i =~ dirs[0] + disk = dirs.shift + dirs[0] = disk + dirs[0] + end + dirs.each_index do |idx| + path = dirs[0..idx].join('') + Dir.mkdir path unless File.dir?(path) + end + end + + def rm_f(path) + $stderr.puts "rm -f #{path}" if verbose? + return if no_harm? + force_remove_file path + end + + def rm_rf(path) + $stderr.puts "rm -rf #{path}" if verbose? + return if no_harm? + remove_tree path + end + + def remove_tree(path) + if File.symlink?(path) + remove_file path + elsif File.dir?(path) + remove_tree0 path + else + force_remove_file path + end + end + + def remove_tree0(path) + Dir.foreach(path) do |ent| + next if ent == '.' + next if ent == '..' + entpath = "#{path}/#{ent}" + if File.symlink?(entpath) + remove_file entpath + elsif File.dir?(entpath) + remove_tree0 entpath + else + force_remove_file entpath + end + end + begin + Dir.rmdir path + rescue Errno::ENOTEMPTY + # directory may not be empty + end + end + + def move_file(src, dest) + force_remove_file dest + begin + File.rename src, dest + rescue + File.open(dest, 'wb') {|f| + f.write File.binread(src) + } + File.chmod File.stat(src).mode, dest + File.unlink src + end + end + + def force_remove_file(path) + begin + remove_file path + rescue + end + end + + def remove_file(path) + File.chmod 0777, path + File.unlink path + end + + def install(from, dest, mode, prefix = nil) + $stderr.puts "install #{from} #{dest}" if verbose? + return if no_harm? + + realdest = prefix ? prefix + File.expand_path(dest) : dest + realdest = File.join(realdest, File.basename(from)) if File.dir?(realdest) + str = File.binread(from) + if diff?(str, realdest) + verbose_off { + rm_f realdest if File.exist?(realdest) + } + File.open(realdest, 'wb') {|f| + f.write str + } + File.chmod mode, realdest + + File.open("#{objdir_root()}/InstalledFiles", 'a') {|f| + if prefix + f.puts realdest.sub(prefix, '') + else + f.puts realdest + end + } + end + end + + def diff?(new_content, path) + return true unless File.exist?(path) + new_content != File.binread(path) + end + + def command(*args) + $stderr.puts args.join(' ') if verbose? + system(*args) or raise RuntimeError, + "system(#{args.map{|a| a.inspect }.join(' ')}) failed" + end + + def ruby(*args) + command config('rubyprog'), *args + end + + def make(task = nil) + command(*[config('makeprog'), task].compact) + end + + def extdir?(dir) + File.exist?("#{dir}/MANIFEST") or File.exist?("#{dir}/extconf.rb") + end + + def files_of(dir) + Dir.open(dir) {|d| + return d.select {|ent| File.file?("#{dir}/#{ent}") } + } + end + + DIR_REJECT = %w( . .. CVS SCCS RCS CVS.adm .svn ) + + def directories_of(dir) + Dir.open(dir) {|d| + return d.select {|ent| File.dir?("#{dir}/#{ent}") } - DIR_REJECT + } + end + +end + + +# This module requires: #srcdir_root, #objdir_root, #relpath +module HookScriptAPI + + def get_config(key) + @config[key] + end + + alias config get_config + + # obsolete: use metaconfig to change configuration + def set_config(key, val) + @config[key] = val + end + + # + # srcdir/objdir (works only in the package directory) + # + + def curr_srcdir + "#{srcdir_root()}/#{relpath()}" + end + + def curr_objdir + "#{objdir_root()}/#{relpath()}" + end + + def srcfile(path) + "#{curr_srcdir()}/#{path}" + end + + def srcexist?(path) + File.exist?(srcfile(path)) + end + + def srcdirectory?(path) + File.dir?(srcfile(path)) + end + + def srcfile?(path) + File.file?(srcfile(path)) + end + + def srcentries(path = '.') + Dir.open("#{curr_srcdir()}/#{path}") {|d| + return d.to_a - %w(. ..) + } + end + + def srcfiles(path = '.') + srcentries(path).select {|fname| + File.file?(File.join(curr_srcdir(), path, fname)) + } + end + + def srcdirectories(path = '.') + srcentries(path).select {|fname| + File.dir?(File.join(curr_srcdir(), path, fname)) + } + end + +end + + +class ToplevelInstaller + + Version = '3.4.1' + Copyright = 'Copyright (c) 2000-2005 Minero Aoki' + + TASKS = [ + [ 'all', 'do config, setup, then install' ], + [ 'config', 'saves your configurations' ], + [ 'show', 'shows current configuration' ], + [ 'setup', 'compiles ruby extentions and others' ], + [ 'install', 'installs files' ], + [ 'test', 'run all tests in test/' ], + [ 'clean', "does `make clean' for each extention" ], + [ 'distclean',"does `make distclean' for each extention" ] + ] + + def ToplevelInstaller.invoke + config = ConfigTable.new(load_rbconfig()) + config.load_standard_entries + config.load_multipackage_entries if multipackage? + config.fixup + klass = (multipackage?() ? ToplevelInstallerMulti : ToplevelInstaller) + klass.new(File.dirname($0), config).invoke + end + + def ToplevelInstaller.multipackage? + File.dir?(File.dirname($0) + '/packages') + end + + def ToplevelInstaller.load_rbconfig + if arg = ARGV.detect {|arg| /\A--rbconfig=/ =~ arg } + ARGV.delete(arg) + load File.expand_path(arg.split(/=/, 2)[1]) + $".push 'rbconfig.rb' + else + require 'rbconfig' + end + ::Config::CONFIG + end + + def initialize(ardir_root, config) + @ardir = File.expand_path(ardir_root) + @config = config + # cache + @valid_task_re = nil + end + + def config(key) + @config[key] + end + + def inspect + "#<#{self.class} #{__id__()}>" + end + + def invoke + run_metaconfigs + case task = parsearg_global() + when nil, 'all' + parsearg_config + init_installers + exec_config + exec_setup + exec_install + else + case task + when 'config', 'test' + ; + when 'clean', 'distclean' + @config.load_savefile if File.exist?(@config.savefile) + else + @config.load_savefile + end + __send__ "parsearg_#{task}" + init_installers + __send__ "exec_#{task}" + end + end + + def run_metaconfigs + @config.load_script "#{@ardir}/metaconfig" + end + + def init_installers + @installer = Installer.new(@config, @ardir, File.expand_path('.')) + end + + # + # Hook Script API bases + # + + def srcdir_root + @ardir + end + + def objdir_root + '.' + end + + def relpath + '.' + end + + # + # Option Parsing + # + + def parsearg_global + while arg = ARGV.shift + case arg + when /\A\w+\z/ + setup_rb_error "invalid task: #{arg}" unless valid_task?(arg) + return arg + when '-q', '--quiet' + @config.verbose = false + when '--verbose' + @config.verbose = true + when '--help' + print_usage $stdout + exit 0 + when '--version' + puts "#{File.basename($0)} version #{Version}" + exit 0 + when '--copyright' + puts Copyright + exit 0 + else + setup_rb_error "unknown global option '#{arg}'" + end + end + nil + end + + def valid_task?(t) + valid_task_re() =~ t + end + + def valid_task_re + @valid_task_re ||= /\A(?:#{TASKS.map {|task,desc| task }.join('|')})\z/ + end + + def parsearg_no_options + unless ARGV.empty? + task = caller(0).first.slice(%r<`parsearg_(\w+)'>, 1) + setup_rb_error "#{task}: unknown options: #{ARGV.join(' ')}" + end + end + + alias parsearg_show parsearg_no_options + alias parsearg_setup parsearg_no_options + alias parsearg_test parsearg_no_options + alias parsearg_clean parsearg_no_options + alias parsearg_distclean parsearg_no_options + + def parsearg_config + evalopt = [] + set = [] + @config.config_opt = [] + while i = ARGV.shift + if /\A--?\z/ =~ i + @config.config_opt = ARGV.dup + break + end + name, value = *@config.parse_opt(i) + if @config.value_config?(name) + @config[name] = value + else + evalopt.push [name, value] + end + set.push name + end + evalopt.each do |name, value| + @config.lookup(name).evaluate value, @config + end + # Check if configuration is valid + set.each do |n| + @config[n] if @config.value_config?(n) + end + end + + def parsearg_install + @config.no_harm = false + @config.install_prefix = '' + while a = ARGV.shift + case a + when '--no-harm' + @config.no_harm = true + when /\A--prefix=/ + path = a.split(/=/, 2)[1] + path = File.expand_path(path) unless path[0,1] == '/' + @config.install_prefix = path + else + setup_rb_error "install: unknown option #{a}" + end + end + end + + def print_usage(out) + out.puts 'Typical Installation Procedure:' + out.puts " $ ruby #{File.basename $0} config" + out.puts " $ ruby #{File.basename $0} setup" + out.puts " # ruby #{File.basename $0} install (may require root privilege)" + out.puts + out.puts 'Detailed Usage:' + out.puts " ruby #{File.basename $0} " + out.puts " ruby #{File.basename $0} [] []" + + fmt = " %-24s %s\n" + out.puts + out.puts 'Global options:' + out.printf fmt, '-q,--quiet', 'suppress message outputs' + out.printf fmt, ' --verbose', 'output messages verbosely' + out.printf fmt, ' --help', 'print this message' + out.printf fmt, ' --version', 'print version and quit' + out.printf fmt, ' --copyright', 'print copyright and quit' + out.puts + out.puts 'Tasks:' + TASKS.each do |name, desc| + out.printf fmt, name, desc + end + + fmt = " %-24s %s [%s]\n" + out.puts + out.puts 'Options for CONFIG or ALL:' + @config.each do |item| + out.printf fmt, item.help_opt, item.description, item.help_default + end + out.printf fmt, '--rbconfig=path', 'rbconfig.rb to load',"running ruby's" + out.puts + out.puts 'Options for INSTALL:' + out.printf fmt, '--no-harm', 'only display what to do if given', 'off' + out.printf fmt, '--prefix=path', 'install path prefix', '' + out.puts + end + + # + # Task Handlers + # + + def exec_config + @installer.exec_config + @config.save # must be final + end + + def exec_setup + @installer.exec_setup + end + + def exec_install + @installer.exec_install + end + + def exec_test + @installer.exec_test + end + + def exec_show + @config.each do |i| + printf "%-20s %s\n", i.name, i.value if i.value? + end + end + + def exec_clean + @installer.exec_clean + end + + def exec_distclean + @installer.exec_distclean + end + +end # class ToplevelInstaller + + +class ToplevelInstallerMulti < ToplevelInstaller + + include FileOperations + + def initialize(ardir_root, config) + super + @packages = directories_of("#{@ardir}/packages") + raise 'no package exists' if @packages.empty? + @root_installer = Installer.new(@config, @ardir, File.expand_path('.')) + end + + def run_metaconfigs + @config.load_script "#{@ardir}/metaconfig", self + @packages.each do |name| + @config.load_script "#{@ardir}/packages/#{name}/metaconfig" + end + end + + attr_reader :packages + + def packages=(list) + raise 'package list is empty' if list.empty? + list.each do |name| + raise "directory packages/#{name} does not exist"\ + unless File.dir?("#{@ardir}/packages/#{name}") + end + @packages = list + end + + def init_installers + @installers = {} + @packages.each do |pack| + @installers[pack] = Installer.new(@config, + "#{@ardir}/packages/#{pack}", + "packages/#{pack}") + end + with = extract_selection(config('with')) + without = extract_selection(config('without')) + @selected = @installers.keys.select {|name| + (with.empty? or with.include?(name)) \ + and not without.include?(name) + } + end + + def extract_selection(list) + a = list.split(/,/) + a.each do |name| + setup_rb_error "no such package: #{name}" unless @installers.key?(name) + end + a + end + + def print_usage(f) + super + f.puts 'Inluded packages:' + f.puts ' ' + @packages.sort.join(' ') + f.puts + end + + # + # Task Handlers + # + + def exec_config + run_hook 'pre-config' + each_selected_installers {|inst| inst.exec_config } + run_hook 'post-config' + @config.save # must be final + end + + def exec_setup + run_hook 'pre-setup' + each_selected_installers {|inst| inst.exec_setup } + run_hook 'post-setup' + end + + def exec_install + run_hook 'pre-install' + each_selected_installers {|inst| inst.exec_install } + run_hook 'post-install' + end + + def exec_test + run_hook 'pre-test' + each_selected_installers {|inst| inst.exec_test } + run_hook 'post-test' + end + + def exec_clean + rm_f @config.savefile + run_hook 'pre-clean' + each_selected_installers {|inst| inst.exec_clean } + run_hook 'post-clean' + end + + def exec_distclean + rm_f @config.savefile + run_hook 'pre-distclean' + each_selected_installers {|inst| inst.exec_distclean } + run_hook 'post-distclean' + end + + # + # lib + # + + def each_selected_installers + Dir.mkdir 'packages' unless File.dir?('packages') + @selected.each do |pack| + $stderr.puts "Processing the package `#{pack}' ..." if verbose? + Dir.mkdir "packages/#{pack}" unless File.dir?("packages/#{pack}") + Dir.chdir "packages/#{pack}" + yield @installers[pack] + Dir.chdir '../..' + end + end + + def run_hook(id) + @root_installer.run_hook id + end + + # module FileOperations requires this + def verbose? + @config.verbose? + end + + # module FileOperations requires this + def no_harm? + @config.no_harm? + end + +end # class ToplevelInstallerMulti + + +class Installer + + FILETYPES = %w( bin lib ext data conf man ) + + include FileOperations + include HookScriptAPI + + def initialize(config, srcroot, objroot) + @config = config + @srcdir = File.expand_path(srcroot) + @objdir = File.expand_path(objroot) + @currdir = '.' + end + + def inspect + "#<#{self.class} #{File.basename(@srcdir)}>" + end + + def noop(rel) + end + + # + # Hook Script API base methods + # + + def srcdir_root + @srcdir + end + + def objdir_root + @objdir + end + + def relpath + @currdir + end + + # + # Config Access + # + + # module FileOperations requires this + def verbose? + @config.verbose? + end + + # module FileOperations requires this + def no_harm? + @config.no_harm? + end + + def verbose_off + begin + save, @config.verbose = @config.verbose?, false + yield + ensure + @config.verbose = save + end + end + + # + # TASK config + # + + def exec_config + exec_task_traverse 'config' + end + + alias config_dir_bin noop + alias config_dir_lib noop + + def config_dir_ext(rel) + extconf if extdir?(curr_srcdir()) + end + + alias config_dir_data noop + alias config_dir_conf noop + alias config_dir_man noop + + def extconf + ruby "#{curr_srcdir()}/extconf.rb", *@config.config_opt + end + + # + # TASK setup + # + + def exec_setup + exec_task_traverse 'setup' + end + + def setup_dir_bin(rel) + files_of(curr_srcdir()).each do |fname| + update_shebang_line "#{curr_srcdir()}/#{fname}" + end + end + + alias setup_dir_lib noop + + def setup_dir_ext(rel) + make if extdir?(curr_srcdir()) + end + + alias setup_dir_data noop + alias setup_dir_conf noop + alias setup_dir_man noop + + def update_shebang_line(path) + return if no_harm? + return if config('shebang') == 'never' + old = Shebang.load(path) + if old + $stderr.puts "warning: #{path}: Shebang line includes too many args. It is not portable and your program may not work." if old.args.size > 1 + new = new_shebang(old) + return if new.to_s == old.to_s + else + return unless config('shebang') == 'all' + new = Shebang.new(config('rubypath')) + end + $stderr.puts "updating shebang: #{File.basename(path)}" if verbose? + open_atomic_writer(path) {|output| + File.open(path, 'rb') {|f| + f.gets if old # discard + output.puts new.to_s + output.print f.read + } + } + end + + def new_shebang(old) + if /\Aruby/ =~ File.basename(old.cmd) + Shebang.new(config('rubypath'), old.args) + elsif File.basename(old.cmd) == 'env' and old.args.first == 'ruby' + Shebang.new(config('rubypath'), old.args[1..-1]) + else + return old unless config('shebang') == 'all' + Shebang.new(config('rubypath')) + end + end + + def open_atomic_writer(path, &block) + tmpfile = File.basename(path) + '.tmp' + begin + File.open(tmpfile, 'wb', &block) + File.rename tmpfile, File.basename(path) + ensure + File.unlink tmpfile if File.exist?(tmpfile) + end + end + + class Shebang + def Shebang.load(path) + line = nil + File.open(path) {|f| + line = f.gets + } + return nil unless /\A#!/ =~ line + parse(line) + end + + def Shebang.parse(line) + cmd, *args = *line.strip.sub(/\A\#!/, '').split(' ') + new(cmd, args) + end + + def initialize(cmd, args = []) + @cmd = cmd + @args = args + end + + attr_reader :cmd + attr_reader :args + + def to_s + "#! #{@cmd}" + (@args.empty? ? '' : " #{@args.join(' ')}") + end + end + + # + # TASK install + # + + def exec_install + rm_f 'InstalledFiles' + exec_task_traverse 'install' + end + + def install_dir_bin(rel) + install_files targetfiles(), "#{config('bindir')}/#{rel}", 0755 + end + + def install_dir_lib(rel) + install_files libfiles(), "#{config('rbdir')}/#{rel}", 0644 + end + + def install_dir_ext(rel) + return unless extdir?(curr_srcdir()) + install_files rubyextentions('.'), + "#{config('sodir')}/#{File.dirname(rel)}", + 0555 + end + + def install_dir_data(rel) + install_files targetfiles(), "#{config('datadir')}/#{rel}", 0644 + end + + def install_dir_conf(rel) + # FIXME: should not remove current config files + # (rename previous file to .old/.org) + install_files targetfiles(), "#{config('sysconfdir')}/#{rel}", 0644 + end + + def install_dir_man(rel) + install_files targetfiles(), "#{config('mandir')}/#{rel}", 0644 + end + + def install_files(list, dest, mode) + mkdir_p dest, @config.install_prefix + list.each do |fname| + install fname, dest, mode, @config.install_prefix + end + end + + def libfiles + glob_reject(%w(*.y *.output), targetfiles()) + end + + def rubyextentions(dir) + ents = glob_select("*.#{@config.dllext}", targetfiles()) + if ents.empty? + setup_rb_error "no ruby extention exists: 'ruby #{$0} setup' first" + end + ents + end + + def targetfiles + mapdir(existfiles() - hookfiles()) + end + + def mapdir(ents) + ents.map {|ent| + if File.exist?(ent) + then ent # objdir + else "#{curr_srcdir()}/#{ent}" # srcdir + end + } + end + + # picked up many entries from cvs-1.11.1/src/ignore.c + JUNK_FILES = %w( + core RCSLOG tags TAGS .make.state + .nse_depinfo #* .#* cvslog.* ,* .del-* *.olb + *~ *.old *.bak *.BAK *.orig *.rej _$* *$ + + *.org *.in .* + ) + + def existfiles + glob_reject(JUNK_FILES, (files_of(curr_srcdir()) | files_of('.'))) + end + + def hookfiles + %w( pre-%s post-%s pre-%s.rb post-%s.rb ).map {|fmt| + %w( config setup install clean ).map {|t| sprintf(fmt, t) } + }.flatten + end + + def glob_select(pat, ents) + re = globs2re([pat]) + ents.select {|ent| re =~ ent } + end + + def glob_reject(pats, ents) + re = globs2re(pats) + ents.reject {|ent| re =~ ent } + end + + GLOB2REGEX = { + '.' => '\.', + '$' => '\$', + '#' => '\#', + '*' => '.*' + } + + def globs2re(pats) + /\A(?:#{ + pats.map {|pat| pat.gsub(/[\.\$\#\*]/) {|ch| GLOB2REGEX[ch] } }.join('|') + })\z/ + end + + # + # TASK test + # + + TESTDIR = 'test' + + def exec_test + unless File.directory?('test') + $stderr.puts 'no test in this package' if verbose? + return + end + $stderr.puts 'Running tests...' if verbose? + begin + require 'test/unit' + rescue LoadError + setup_rb_error 'test/unit cannot loaded. You need Ruby 1.8 or later to invoke this task.' + end + runner = Test::Unit::AutoRunner.new(true) + runner.to_run << TESTDIR + runner.run + end + + # + # TASK clean + # + + def exec_clean + exec_task_traverse 'clean' + rm_f @config.savefile + rm_f 'InstalledFiles' + end + + alias clean_dir_bin noop + alias clean_dir_lib noop + alias clean_dir_data noop + alias clean_dir_conf noop + alias clean_dir_man noop + + def clean_dir_ext(rel) + return unless extdir?(curr_srcdir()) + make 'clean' if File.file?('Makefile') + end + + # + # TASK distclean + # + + def exec_distclean + exec_task_traverse 'distclean' + rm_f @config.savefile + rm_f 'InstalledFiles' + end + + alias distclean_dir_bin noop + alias distclean_dir_lib noop + + def distclean_dir_ext(rel) + return unless extdir?(curr_srcdir()) + make 'distclean' if File.file?('Makefile') + end + + alias distclean_dir_data noop + alias distclean_dir_conf noop + alias distclean_dir_man noop + + # + # Traversing + # + + def exec_task_traverse(task) + run_hook "pre-#{task}" + FILETYPES.each do |type| + if type == 'ext' and config('without-ext') == 'yes' + $stderr.puts 'skipping ext/* by user option' if verbose? + next + end + traverse task, type, "#{task}_dir_#{type}" + end + run_hook "post-#{task}" + end + + def traverse(task, rel, mid) + dive_into(rel) { + run_hook "pre-#{task}" + __send__ mid, rel.sub(%r[\A.*?(?:/|\z)], '') + directories_of(curr_srcdir()).each do |d| + traverse task, "#{rel}/#{d}", mid + end + run_hook "post-#{task}" + } + end + + def dive_into(rel) + return unless File.dir?("#{@srcdir}/#{rel}") + + dir = File.basename(rel) + Dir.mkdir dir unless File.dir?(dir) + prevdir = Dir.pwd + Dir.chdir dir + $stderr.puts '---> ' + rel if verbose? + @currdir = rel + yield + Dir.chdir prevdir + $stderr.puts '<--- ' + rel if verbose? + @currdir = File.dirname(rel) + end + + def run_hook(id) + path = [ "#{curr_srcdir()}/#{id}", + "#{curr_srcdir()}/#{id}.rb" ].detect {|cand| File.file?(cand) } + return unless path + begin + instance_eval File.read(path), path, 1 + rescue + raise if $DEBUG + setup_rb_error "hook #{path} failed:\n" + $!.message + end + end + +end # class Installer + + +class SetupError < StandardError; end + +def setup_rb_error(msg) + raise SetupError, msg +end + +if $0 == __FILE__ + begin + ToplevelInstaller.invoke + rescue SetupError + raise if $DEBUG + $stderr.puts $!.message + $stderr.puts "Try 'ruby #{$0} --help' for detailed usage." + exit 1 + end +end diff --git a/ruby/gem/tasks/deployment.rake b/ruby/gem/tasks/deployment.rake new file mode 100644 index 0000000..2f43742 --- /dev/null +++ b/ruby/gem/tasks/deployment.rake @@ -0,0 +1,34 @@ +desc 'Release the website and new gem version' +task :deploy => [:check_version, :website, :release] do + puts "Remember to create SVN tag:" + puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " + + "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} " + puts "Suggested comment:" + puts "Tagging release #{CHANGES}" +end + +desc 'Runs tasks website_generate and install_gem as a local deployment of the gem' +task :local_deploy => [:website_generate, :install_gem] + +task :check_version do + unless ENV['VERSION'] + puts 'Must pass a VERSION=x.y.z release version' + exit + end + unless ENV['VERSION'] == VERS + puts "Please update your version.rb to match the release version, currently #{VERS}" + exit + end +end + +desc 'Install the package as a gem, without generating documentation(ri/rdoc)' +task :install_gem_no_doc => [:clean, :package] do + sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri" +end + +namespace :manifest do + desc 'Recreate Manifest.txt to include ALL files' + task :refresh do + `rake check_manifest | patch -p0 > Manifest.txt` + end +end \ No newline at end of file diff --git a/ruby/gem/tasks/environment.rake b/ruby/gem/tasks/environment.rake new file mode 100644 index 0000000..691ed3b --- /dev/null +++ b/ruby/gem/tasks/environment.rake @@ -0,0 +1,7 @@ +task :ruby_env do + RUBY_APP = if RUBY_PLATFORM =~ /java/ + "jruby" + else + "ruby" + end unless defined? RUBY_APP +end diff --git a/ruby/pack.c b/ruby/pack.c new file mode 100644 index 0000000..3520f9f --- /dev/null +++ b/ruby/pack.c @@ -0,0 +1,131 @@ +/* + * MessagePack packing routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pack_inline.h" + +#ifndef RUBY_VM +#include "st.h" // ruby hash +#endif + +static ID s_to_msgpack; + +#define ARG_BUFFER(name, argc, argv) \ + VALUE name; \ + if(argc == 1) { \ + name = argv[0]; \ + } else if(argc == 0) { \ + name = rb_str_buf_new(0); \ + } else { \ + rb_raise(rb_eArgError, "wrong number of arguments (%d for 0)", argc); \ + } + +static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_nil(out); + return out; +} + +static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_true(out); + return out; +} + +static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_false(out); + return out; +} + + +static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_int(out, FIX2INT(self)); + return out; +} + +static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_double(out, rb_num2dbl(self)); + return out; +} + +static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_raw(out, RSTRING_PTR(self), RSTRING_LEN(self)); + return out; +} + +static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_array(out, RARRAY_LEN(self)); + VALUE* p = RARRAY_PTR(self); + VALUE* const pend = p + RARRAY_LEN(self); + for(;p != pend; ++p) { + rb_funcall(*p, s_to_msgpack, 1, out); + } + return out; +} + +#ifndef RHASH_SIZE // Ruby 1.8 +#define RHASH_SIZE(h) (RHASH(h)->tbl ? RHASH(h)->tbl->num_entries : 0) +#endif + +static int MessagePack_Hash_to_msgpack_foreach(VALUE key, VALUE value, VALUE out) +{ + if (key == Qundef) { return ST_CONTINUE; } + rb_funcall(key, s_to_msgpack, 1, out); + rb_funcall(value, s_to_msgpack, 1, out); + return ST_CONTINUE; +} + +static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + msgpack_pack_map(out, RHASH_SIZE(self)); + rb_hash_foreach(self, MessagePack_Hash_to_msgpack_foreach, out); + return out; +} + + +static VALUE MessagePack_pack(VALUE self, VALUE data) +{ + return rb_funcall(data, s_to_msgpack, 0); +} + + +void Init_msgpack_pack(VALUE mMessagePack) +{ + s_to_msgpack = rb_intern("to_msgpack"); + rb_define_method_id(rb_cNilClass, s_to_msgpack, MessagePack_NilClass_to_msgpack, -1); + rb_define_method_id(rb_cTrueClass, s_to_msgpack, MessagePack_TrueClass_to_msgpack, -1); + rb_define_method_id(rb_cFalseClass, s_to_msgpack, MessagePack_FalseClass_to_msgpack, -1); + rb_define_method_id(rb_cFixnum, s_to_msgpack, MessagePack_Fixnum_to_msgpack, -1); + rb_define_method_id(rb_cFloat, s_to_msgpack, MessagePack_Float_to_msgpack, -1); + rb_define_method_id(rb_cString, s_to_msgpack, MessagePack_String_to_msgpack, -1); + rb_define_method_id(rb_cArray, s_to_msgpack, MessagePack_Array_to_msgpack, -1); + rb_define_method_id(rb_cHash, s_to_msgpack, MessagePack_Hash_to_msgpack, -1); + rb_define_module_function(mMessagePack, "pack", MessagePack_pack, 1); +} + diff --git a/ruby/pack.h b/ruby/pack.h new file mode 100644 index 0000000..c38ac48 --- /dev/null +++ b/ruby/pack.h @@ -0,0 +1,26 @@ +/* + * MessagePack packing routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef PACK_H__ +#define PACK_H__ + +#include "ruby.h" + +void Init_msgpack_pack(VALUE mMessagePack); + +#endif /* pack.h */ + diff --git a/ruby/pack_inline.h b/ruby/pack_inline.h new file mode 100644 index 0000000..ab4b092 --- /dev/null +++ b/ruby/pack_inline.h @@ -0,0 +1,33 @@ +/* + * MessagePack packing routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef PACK_INLINE_H__ +#define PACK_INLINE_H__ + +#include "ruby.h" + +typedef VALUE msgpack_pack_context; + +static inline void msgpack_pack_append_buffer(VALUE x, const unsigned char* b, unsigned int l) +{ + rb_str_buf_cat(x, (const void*)b, l); +} + +#include "msgpack/pack/inline_impl.h" + +#endif /* pack_inline.h */ + diff --git a/ruby/rbinit.c b/ruby/rbinit.c new file mode 100644 index 0000000..7ef92fb --- /dev/null +++ b/ruby/rbinit.c @@ -0,0 +1,29 @@ +/* + * MessagePack for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pack.h" +#include "unpack.h" + +static VALUE mMessagePack; + +void Init_msgpack(void) +{ + mMessagePack = rb_define_module("MessagePack"); + Init_msgpack_unpack(mMessagePack); + Init_msgpack_pack(mMessagePack); +} + diff --git a/ruby/test_format.rb b/ruby/test_format.rb new file mode 100644 index 0000000..7c2e8fc --- /dev/null +++ b/ruby/test_format.rb @@ -0,0 +1,128 @@ +require 'msgpack' + +@up = MessagePack::Unpacker.new + +def check(bytes, should) + puts "----" + @up.reset + src = bytes.pack('C*') + ret = @up.execute(src, 0) + if ret != src.length + puts "** EXTRA BYTES **" + end + puts bytes.map{|x|"%x"%x}.join(' ') + data = @up.data + p data + if data != should + puts "** TEST FAILED **" + p should + end +end + +# SimpleValue +check([ + 0x93, 0xc0, 0xc2, 0xc3, +], [nil,false,true]) + +# Fixnum +check([ + 0x92, + 0x93, 0x00, 0x40, 0x7f, + 0x93, 0xe0, 0xf0, 0xff, +], [[0,64,127], [-32,-16,-1]]) + +# FixArray +check([ + 0x92, + 0x90, + 0x91, + 0x91, 0xc0, +], [[],[[nil]]]) + + +# FixRaw +check([ + 0x94, + 0xa0, + 0xa1, ?a, + 0xa2, ?b, ?c, + 0xa3, ?d, ?e, ?f, +], ["","a","bc","def"]) + +# FixMap +check([ + 0x82, + 0xc2, 0x81, + 0xc0, 0xc0, + 0xc3, 0x81, + 0xc0, 0x80, +], {false=>{nil=>nil}, true=>{nil=>{}}}) + +# unsigned int +check([ + 0x99, + 0xcc, 0, + 0xcc, 128, + 0xcc, 255, + 0xcd, 0x00, 0x00, + 0xcd, 0x80, 0x00, + 0xcd, 0xff, 0xff, + 0xce, 0x00, 0x00, 0x00, 0x00, + 0xce, 0x80, 0x00, 0x00, 0x00, + 0xce, 0xff, 0xff, 0xff, 0xff, +], [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295]) + +# signed int +check([ + 0x99, + 0xd0, 0, + 0xd0, 128, + 0xd0, 255, + 0xd1, 0x00, 0x00, + 0xd1, 0x80, 0x00, + 0xd1, 0xff, 0xff, + 0xd2, 0x00, 0x00, 0x00, 0x00, + 0xd2, 0x80, 0x00, 0x00, 0x00, + 0xd2, 0xff, 0xff, 0xff, 0xff, +], [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) + +# raw +check([ + 0x96, + 0xda, 0x00, 0x00, + 0xda, 0x00, 0x01, ?a, + 0xda, 0x00, 0x02, ?a, ?b, + 0xdb, 0x00, 0x00, 0x00, 0x00, + 0xdb, 0x00, 0x00, 0x00, 0x01, ?a, + 0xdb, 0x00, 0x00, 0x00, 0x02, ?a, ?b, +], ["", "a", "ab", "", "a", "ab"]) + +# array +check([ + 0x96, + 0xdc, 0x00, 0x00, + 0xdc, 0x00, 0x01, 0xc0, + 0xdc, 0x00, 0x02, 0xc2, 0xc3, + 0xdd, 0x00, 0x00, 0x00, 0x00, + 0xdd, 0x00, 0x00, 0x00, 0x01, 0xc0, + 0xdd, 0x00, 0x00, 0x00, 0x02, 0xc2, 0xc3 +], [[], [nil], [false,true], [], [nil], [false,true]]) + +# map +check([ + 0x96, + 0xde, 0x00, 0x00, + 0xde, 0x00, 0x01, 0xc0, 0xc2, + 0xde, 0x00, 0x02, 0xc0, 0xc2, 0xc3, 0xc2, + 0xdf, 0x00, 0x00, 0x00, 0x00, + 0xdf, 0x00, 0x00, 0x00, 0x01, 0xc0, 0xc2, + 0xdf, 0x00, 0x00, 0x00, 0x02, 0xc0, 0xc2, 0xc3, 0xc2, +], [{}, {nil=>false}, {true=>false, nil=>false}, {}, {nil=>false}, {true=>false, nil=>false}]) + +# string +check([ + 0x92, + 0xc1, 0x00, + 0xc1, ?a, ?b, ?c, 0x00, +], ["", "abc"]) + diff --git a/ruby/test_pack.rb b/ruby/test_pack.rb new file mode 100644 index 0000000..16a8ccf --- /dev/null +++ b/ruby/test_pack.rb @@ -0,0 +1,56 @@ +require 'msgpack' + +def check(data) + puts "---" + pack = data.to_msgpack + p data + puts pack.unpack('C*').map{|x|"%02x"%x}.join(' ') + re = MessagePack::unpack(pack) + if re != data + p re + puts "** TEST FAILED **" + end +end + +check 0 +check 1 +check 127 +check 128 +check 255 +check 256 +check 65535 +check 65536 +check -1 +check -128 +check -129 +check -32768 +check -32769 + +check 1.0 + +check "" +check "a" +check "a"*31 +check "a"*32 + +check nil +check true +check false + +check [] +check [[]] +check [[], nil] + +check( {nil=>0} ) + +check (1<<23) +__END__ + +ary = [] +i = 0 +while i < (1<<16) + ary << i + i += 1 +end +check ary + diff --git a/ruby/unpack.c b/ruby/unpack.c new file mode 100644 index 0000000..fa2996d --- /dev/null +++ b/ruby/unpack.c @@ -0,0 +1,202 @@ +/* + * MessagePack unpacking routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "ruby.h" +#include "unpack_context.h" +#include + +#define UNPACKER(from, name) \ + msgpack_unpacker *name = NULL; \ + Data_Get_Struct(from, msgpack_unpacker, name); \ + if(name == NULL) { \ + rb_raise(rb_eArgError, "NULL found for " # name " when shouldn't be."); \ + } + +#define CHECK_STRING_TYPE(value) \ + value = rb_check_string_type(value); \ + if( NIL_P(value) ) { \ + rb_raise(rb_eTypeError, "instance of String needed"); \ + } + +static VALUE cUnpacker; +static VALUE eUnpackError; + +static void MessagePack_Unpacker_free(void* data) +{ + if(data) { free(data); } +} + +static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) +{ + unsigned int i; + for(i=0; i < mp->top; ++i) { + rb_gc_mark(mp->stack[i].obj); + rb_gc_mark(mp->stack[i].tmp.map_key); + } +} + +static VALUE MessagePack_Unpacker_alloc(VALUE klass) +{ + VALUE obj; + msgpack_unpacker* mp = ALLOC_N(msgpack_unpacker, 1); + obj = Data_Wrap_Struct(klass, MessagePack_Unpacker_mark, + MessagePack_Unpacker_free, mp); + return obj; +} + +static VALUE MessagePack_Unpacker_reset(VALUE self) +{ + UNPACKER(self, mp); + mp->user.finished = false; + msgpack_unpacker_init(mp); + return self; +} + +static VALUE MessagePack_Unpacker_initialize(VALUE self) +{ + return MessagePack_Unpacker_reset(self); +} + + +static VALUE MessagePack_Unpacker_execute_impl(VALUE args) +{ + VALUE self = ((VALUE*)args)[0]; + VALUE data = ((VALUE*)args)[1]; + VALUE off = ((VALUE*)args)[2]; + + UNPACKER(self, mp); + size_t from = NUM2UINT(off); + char* dptr = RSTRING_PTR(data); + long dlen = RSTRING_LEN(data); + int ret; + + if(from >= dlen) { + rb_raise(eUnpackError, "Requested start is after data buffer end."); + } + + ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); + + if(ret < 0) { + rb_raise(eUnpackError, "Parse error."); + } else if(ret > 0) { + mp->user.finished = true; + return ULONG2NUM(from); + } else { + mp->user.finished = false; + return ULONG2NUM(from); + } +} + +static VALUE MessagePack_Unpacker_execute_rescue(VALUE nouse) +{ + rb_gc_enable(); +#ifdef RUBY_VM + rb_exc_raise(rb_errinfo()); +#else + rb_exc_raise(ruby_errinfo); +#endif +} + +static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) +{ + // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ + rb_gc_disable(); + VALUE args[3] = {self, data, off}; + VALUE ret = rb_rescue(MessagePack_Unpacker_execute_impl, (VALUE)args, + MessagePack_Unpacker_execute_rescue, Qnil); + rb_gc_enable(); + return ret; +} + +static VALUE MessagePack_Unpacker_finished_p(VALUE self) +{ + UNPACKER(self, mp); + if(mp->user.finished) { + return Qtrue; + } + return Qfalse; +} + +static VALUE MessagePack_Unpacker_data(VALUE self) +{ + UNPACKER(self, mp); + return msgpack_unpacker_data(mp); +} + + +static VALUE MessagePack_unpack_impl(VALUE args) +{ + msgpack_unpacker* mp = (msgpack_unpacker*)((VALUE*)args)[0]; + VALUE data = ((VALUE*)args)[1]; + + size_t from = 0; + char* dptr = RSTRING_PTR(data); + long dlen = RSTRING_LEN(data); + int ret; + + ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); + + if(ret < 0) { + rb_raise(eUnpackError, "Parse error."); + } else if(ret == 0) { + rb_raise(eUnpackError, "Insufficient bytes."); + } else { + if(from < dlen) { + rb_raise(eUnpackError, "Extra bytes."); + } + return msgpack_unpacker_data(mp); + } +} + +static VALUE MessagePack_unpack_rescue(VALUE args) +{ + rb_gc_enable(); +#ifdef RUBY_VM + rb_exc_raise(rb_errinfo()); +#else + rb_exc_raise(ruby_errinfo); +#endif +} + +static VALUE MessagePack_unpack(VALUE self, VALUE data) +{ + CHECK_STRING_TYPE(data); + msgpack_unpacker mp; + msgpack_unpacker_init(&mp); + rb_gc_disable(); + VALUE args[2] = {(VALUE)&mp, data}; + VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, + MessagePack_unpack_rescue, Qnil); + rb_gc_enable(); + return ret; +} + + +void Init_msgpack_unpack(VALUE mMessagePack) +{ + eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); + cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); + rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); + rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, 0); + rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); + rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); + rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); + rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); + rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); +} + + diff --git a/ruby/unpack.h b/ruby/unpack.h new file mode 100644 index 0000000..0fe01ec --- /dev/null +++ b/ruby/unpack.h @@ -0,0 +1,26 @@ +/* + * MessagePack unpacking routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef UNPACK_H__ +#define UNPACK_H__ + +#include "ruby.h" + +void Init_msgpack_unpack(VALUE mMessagePack); + +#endif /* unpack.h */ + diff --git a/ruby/unpack_context.h b/ruby/unpack_context.h new file mode 100644 index 0000000..35e0132 --- /dev/null +++ b/ruby/unpack_context.h @@ -0,0 +1,35 @@ +/* + * MessagePack unpacking routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef UNPACK_CONTEXT_H__ +#define UNPACK_CONTEXT_H__ + +#include "ruby.h" +#include +#include + +typedef VALUE msgpack_object; + +typedef struct { + bool finished; +} msgpack_unpack_context; + + +#include "msgpack/unpack/inline_context.h" + +#endif /* unpack_context.h */ + diff --git a/ruby/unpack_inline.c b/ruby/unpack_inline.c new file mode 100644 index 0000000..fa684c9 --- /dev/null +++ b/ruby/unpack_inline.c @@ -0,0 +1,81 @@ +/* + * MessagePack unpacking routine for Ruby + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "unpack_context.h" + +static inline VALUE msgpack_unpack_init(msgpack_unpack_context* x) +{ return Qnil; } + +static inline VALUE msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d) +{ return INT2FIX(d); } + +static inline VALUE msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d) +{ return INT2FIX(d); } + +static inline VALUE msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d) +{ return UINT2NUM(d); } + +static inline VALUE msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d) +{ return UINT2NUM(d); } // FIXME + +static inline VALUE msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d) +{ return INT2FIX((long)d); } + +static inline VALUE msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d) +{ return INT2FIX((long)d); } + +static inline VALUE msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d) +{ return INT2NUM((long)d); } + +static inline VALUE msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) +{ return INT2NUM(d); } // FIXME + +static inline VALUE msgpack_unpack_float(msgpack_unpack_context* x, float d) +{ return rb_float_new(d); } + +static inline VALUE msgpack_unpack_double(msgpack_unpack_context* x, double d) +{ return rb_float_new(d); } + +static inline VALUE msgpack_unpack_nil(msgpack_unpack_context* x) +{ return Qnil; } + +static inline VALUE msgpack_unpack_true(msgpack_unpack_context* x) +{ return Qtrue; } + +static inline VALUE msgpack_unpack_false(msgpack_unpack_context* x) +{ return Qfalse; } + +static inline VALUE msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n) +{ return rb_ary_new2(n); } + +static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, VALUE c, VALUE o) +{ rb_ary_push(c, o); } + +static inline VALUE msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) +{ return rb_hash_new(); } + +static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) +{ rb_hash_aset(c, k, v); } + +static inline VALUE msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l) +{ return rb_str_new(b, l); } + +static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l) +{ return rb_str_new(b, l); } + +#include "msgpack/unpack/inline_impl.h" + From 9f460f17d73e749d58c2f58beb621e1f7292cfa6 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:55 +0000 Subject: [PATCH 0002/1648] lang/c/msgpack: uint64_t, int64_t support for ruby git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@49 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- ruby/Makefile | 153 ------------------------------------------- ruby/pack.c | 18 +++++ ruby/unpack_inline.c | 6 +- 3 files changed, 21 insertions(+), 156 deletions(-) delete mode 100644 ruby/Makefile diff --git a/ruby/Makefile b/ruby/Makefile deleted file mode 100644 index 6ef04b0..0000000 --- a/ruby/Makefile +++ /dev/null @@ -1,153 +0,0 @@ - -SHELL = /bin/sh - -#### Start of system configuration section. #### - -srcdir = . -topdir = /Users/frsyuki/ports/lib/ruby/1.8/i686-darwin9.1.0 -hdrdir = $(topdir) -VPATH = $(srcdir):$(topdir):$(hdrdir) -prefix = $(DESTDIR)/Users/frsyuki/ports -exec_prefix = $(prefix) -sitedir = $(prefix)/lib/ruby/site_ruby -rubylibdir = $(libdir)/ruby/$(ruby_version) -docdir = $(datarootdir)/doc/$(PACKAGE) -dvidir = $(docdir) -datarootdir = $(prefix)/share -archdir = $(rubylibdir)/$(arch) -sbindir = $(exec_prefix)/sbin -psdir = $(docdir) -vendordir = $(prefix)/lib/ruby/vendor_ruby -localedir = $(datarootdir)/locale -htmldir = $(docdir) -datadir = $(datarootdir) -includedir = $(prefix)/include -infodir = $(datarootdir)/info -sysconfdir = $(prefix)/etc -mandir = $(DESTDIR)/Users/frsyuki/ports/share/man -libdir = $(exec_prefix)/lib -sharedstatedir = $(prefix)/com -oldincludedir = $(DESTDIR)/usr/include -pdfdir = $(docdir) -sitearchdir = $(sitelibdir)/$(sitearch) -vendorarchdir = $(vendorlibdir)/$(vendorarch) -bindir = $(exec_prefix)/bin -localstatedir = $(prefix)/var -vendorlibdir = $(vendordir)/$(ruby_version) -sitelibdir = $(sitedir)/$(ruby_version) -libexecdir = $(exec_prefix)/libexec - -CC = /usr/bin/gcc-4.0 -LIBRUBY = $(LIBRUBY_SO) -LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a -LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME) -LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static - -RUBY_EXTCONF_H = -CFLAGS = -fno-common -O2 -fno-common -pipe -fno-common -I.. -Wall -O9 -INCFLAGS = -I. -I$(topdir) -I$(hdrdir) -I$(srcdir) -CPPFLAGS = -I/Users/frsyuki/ports/include -CXXFLAGS = $(CFLAGS) -DLDFLAGS = -L. -L/Users/frsyuki/ports/lib -LDSHARED = cc -dynamic -bundle -undefined suppress -flat_namespace -AR = ar -EXEEXT = - -RUBY_INSTALL_NAME = ruby -RUBY_SO_NAME = ruby -arch = i686-darwin9.1.0 -sitearch = i686-darwin9.1.0 -vendorarch = i686-darwin9.1.0 -ruby_version = 1.8 -ruby = /Users/frsyuki/ports/bin/ruby -RUBY = $(ruby) -RM = rm -f -MAKEDIRS = mkdir -p -INSTALL = /usr/bin/install -INSTALL_PROG = $(INSTALL) -m 0755 -INSTALL_DATA = $(INSTALL) -m 644 -COPY = cp - -#### End of system configuration section. #### - -preload = - -libpath = . $(libdir) -LIBPATH = -L"." -L"$(libdir)" -DEFFILE = - -CLEANFILES = mkmf.log -DISTCLEANFILES = - -extout = -extout_prefix = -target_prefix = -LOCAL_LIBS = -LIBS = $(LIBRUBYARG_SHARED) -lpthread -ldl -lobjc -SRCS = pack.c rbinit.c unpack.c unpack_inline.c -OBJS = pack.o rbinit.o unpack.o unpack_inline.o -TARGET = msgpack -DLLIB = $(TARGET).bundle -EXTSTATIC = -STATIC_LIB = - -RUBYCOMMONDIR = $(sitedir)$(target_prefix) -RUBYLIBDIR = $(sitelibdir)$(target_prefix) -RUBYARCHDIR = $(sitearchdir)$(target_prefix) - -TARGET_SO = $(DLLIB) -CLEANLIBS = $(TARGET).bundle $(TARGET).il? $(TARGET).tds $(TARGET).map -CLEANOBJS = *.o *.a *.s[ol] *.pdb *.exp *.bak - -all: $(DLLIB) -static: $(STATIC_LIB) - -clean: - @-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) - -distclean: clean - @-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log - @-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES) - -realclean: distclean -install: install-so install-rb - -install-so: $(RUBYARCHDIR) -install-so: $(RUBYARCHDIR)/$(DLLIB) -$(RUBYARCHDIR)/$(DLLIB): $(DLLIB) - $(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR) -install-rb: pre-install-rb install-rb-default -install-rb-default: pre-install-rb-default -pre-install-rb: Makefile -pre-install-rb-default: Makefile -$(RUBYARCHDIR): - $(MAKEDIRS) $@ - -site-install: site-install-so site-install-rb -site-install-so: install-so -site-install-rb: install-rb - -.SUFFIXES: .c .m .cc .cxx .cpp .C .o - -.cc.o: - $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< - -.cxx.o: - $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< - -.cpp.o: - $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< - -.C.o: - $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $< - -.c.o: - $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $< - -$(DLLIB): $(OBJS) - @-$(RM) $@ - $(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS) - - - -$(OBJS): ruby.h defines.h diff --git a/ruby/pack.c b/ruby/pack.c index 3520f9f..3d71776 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -62,6 +62,23 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +#ifndef RBIGNUM_SIGN // Ruby 1.8 +#define RBIGNUM_SIGN(b) (RBIGNUM(b)->sign) +#endif + +static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + // FIXME bignum + if(RBIGNUM_SIGN(self)) { // positive + msgpack_pack_unsigned_int_64(out, rb_big2ull(self)); + } else { // negative + msgpack_pack_signed_int_64(out, rb_big2ll(self)); + } + return out; +} + static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -122,6 +139,7 @@ void Init_msgpack_pack(VALUE mMessagePack) rb_define_method_id(rb_cTrueClass, s_to_msgpack, MessagePack_TrueClass_to_msgpack, -1); rb_define_method_id(rb_cFalseClass, s_to_msgpack, MessagePack_FalseClass_to_msgpack, -1); rb_define_method_id(rb_cFixnum, s_to_msgpack, MessagePack_Fixnum_to_msgpack, -1); + rb_define_method_id(rb_cBignum, s_to_msgpack, MessagePack_Bignum_to_msgpack, -1); rb_define_method_id(rb_cFloat, s_to_msgpack, MessagePack_Float_to_msgpack, -1); rb_define_method_id(rb_cString, s_to_msgpack, MessagePack_String_to_msgpack, -1); rb_define_method_id(rb_cArray, s_to_msgpack, MessagePack_Array_to_msgpack, -1); diff --git a/ruby/unpack_inline.c b/ruby/unpack_inline.c index fa684c9..f6715d9 100644 --- a/ruby/unpack_inline.c +++ b/ruby/unpack_inline.c @@ -30,7 +30,7 @@ static inline VALUE msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, ui { return UINT2NUM(d); } static inline VALUE msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d) -{ return UINT2NUM(d); } // FIXME +{ return rb_ull2inum(d); } static inline VALUE msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d) { return INT2FIX((long)d); } @@ -42,7 +42,7 @@ static inline VALUE msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int3 { return INT2NUM((long)d); } static inline VALUE msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) -{ return INT2NUM(d); } // FIXME +{ return rb_ll2inum(d); } static inline VALUE msgpack_unpack_float(msgpack_unpack_context* x, float d) { return rb_float_new(d); } @@ -63,7 +63,7 @@ static inline VALUE msgpack_unpack_array_start(msgpack_unpack_context* x, unsign { return rb_ary_new2(n); } static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, VALUE c, VALUE o) -{ rb_ary_push(c, o); } +{ rb_ary_push(c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] static inline VALUE msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) { return rb_hash_new(); } From 529a50633dffc91dd5ce58ae5a905a0ac4a5fdf9 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0003/1648] lang/c/msgpack: added C++ binding git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@50 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile | 21 + cpp/msgpack | 1 + cpp/object.hpp | 760 ++++++++++++++++++++++++++++++++ cpp/test.cpp | 110 +++++ cpp/unpack.cc | 30 -- cpp/unpack.cpp | 96 ++++ cpp/unpack.h | 126 ------ cpp/unpack.hpp | 48 ++ cpp/unpack_context.h | 13 - cpp/unpack_context.hpp | 14 + cpp/unpack_inline.cpp | 72 +++ cpp/zone.cpp | 40 ++ cpp/zone.hpp.erb | 123 ++++++ msgpack/pack/inline_context.h | 2 - msgpack/pack/inline_impl.h | 66 +-- msgpack/unpack/inline_context.h | 7 + msgpack/unpack/inline_impl.h | 42 +- 17 files changed, 1361 insertions(+), 210 deletions(-) create mode 100644 cpp/Makefile create mode 120000 cpp/msgpack create mode 100644 cpp/object.hpp create mode 100644 cpp/test.cpp delete mode 100644 cpp/unpack.cc create mode 100644 cpp/unpack.cpp delete mode 100644 cpp/unpack.h create mode 100644 cpp/unpack.hpp delete mode 100644 cpp/unpack_context.h create mode 100644 cpp/unpack_context.hpp create mode 100644 cpp/unpack_inline.cpp create mode 100644 cpp/zone.cpp create mode 100644 cpp/zone.hpp.erb delete mode 100644 msgpack/pack/inline_context.h diff --git a/cpp/Makefile b/cpp/Makefile new file mode 100644 index 0000000..eafa683 --- /dev/null +++ b/cpp/Makefile @@ -0,0 +1,21 @@ + +CXXFLAGS = -I.. -I. -Wall -g -O4 +LDFLAGS = -L. + +NEED_PREPROCESS = zone.hpp + +all: test + +%.hpp: %.hpp.erb + erb $< > $@ + +test: $(NEED_PREPROCESS) unpack.o unpack_inline.o zone.o test.o object.hpp unpack.hpp + $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o test.o -o $@ + +.PHONY: clean +clean: + $(RM) unpack.o unpack_inline.o zone.o + $(RM) test.o + $(RM) test + $(RM) $(NEED_PREPROCESS) + diff --git a/cpp/msgpack b/cpp/msgpack new file mode 120000 index 0000000..945c9b4 --- /dev/null +++ b/cpp/msgpack @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/cpp/object.hpp b/cpp/object.hpp new file mode 100644 index 0000000..c4e94d7 --- /dev/null +++ b/cpp/object.hpp @@ -0,0 +1,760 @@ +#ifndef MSGPACK_OBJECT_HPP__ +#define MSGPACK_OBJECT_HPP__ +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace msgpack { + + +class type_error : public std::bad_cast { }; +class cast_error : public type_error { }; +class overflow_error : public type_error { }; +class underflow_error : public type_error { }; +class positive_overflow_error : public overflow_error { }; +class negative_overflow_error : public overflow_error { }; + + +struct raw { + explicit raw() : ptr(NULL), len(0) {} + explicit raw(void* p, size_t l) : ptr(p), len(l) {} +public: + void* ptr; + size_t len; +public: + std::string str() { return std::string((const char*)ptr, len); } +}; + +struct const_raw { + const_raw() : ptr(NULL), len(0) {} + const_raw(const void* p, size_t l) : ptr(p), len(l) {} +public: + const void* ptr; + size_t len; +public: + std::string str() { return std::string((const char*)ptr, len); } +}; + + +struct object; + +typedef std::map map; +typedef std::vector array; + + +template +struct numeric_overflow_signed_impl; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast( std::numeric_limits::max()) < x ) { return 1; } + if( static_cast(-std::numeric_limits::max()) > x ) { return -1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + if( static_cast(-std::numeric_limits::max()) > x) { return -1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) <= sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( static_cast(0) > x ) { return -1; } + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow { + static int test(X x) { + return numeric_overflow_signed_impl::is_signed, std::numeric_limits::is_signed>::test(x); + } + static void check(X x) { + int r = test(x); + if(r == 1) { throw positive_overflow_error(); } + if(r == -1) { throw negative_overflow_error(); } + } +}; + + +template +struct numeric_underflow { + static bool test(X x) { + return static_cast(static_cast(x)) != x; + } + static void check(X x) { + if(test(x)) { throw underflow_error(); } + } +}; + + +struct object_class { + virtual ~object_class() {} + virtual bool isnil() const { return false; } + virtual bool xbool() const { throw cast_error(); } + virtual uint8_t xu8() const { throw cast_error(); } + virtual uint16_t xu16() const { throw cast_error(); } + virtual uint32_t xu32() const { throw cast_error(); } + virtual uint64_t xu64() const { throw cast_error(); } + virtual int8_t xi8() const { throw cast_error(); } + virtual int16_t xi16() const { throw cast_error(); } + virtual int32_t xi32() const { throw cast_error(); } + virtual int64_t xi64() const { throw cast_error(); } + virtual float xfloat() const { throw cast_error(); } + virtual double xdouble() const { throw cast_error(); } + virtual raw xraw() { throw cast_error(); } + virtual array& xarray() { throw cast_error(); } + virtual map& xmap() { throw cast_error(); } + virtual const_raw xraw() const { throw cast_error(); } + virtual const array& xarray() const { throw cast_error(); } + virtual const map& xmap() const { throw cast_error(); } + virtual bool operator== (const object_class* x) const { return false; } + bool operator!= (const object_class* x) const { return !(this->operator==(x)); } + virtual bool operator< (const object_class* x) const { throw cast_error(); } + virtual bool operator> (const object_class* x) const { throw cast_error(); } + operator bool() const { return xbool(); } // FIXME !isnil(); + operator uint8_t() const { return xu8(); } + operator uint16_t() const { return xu16(); } + operator uint32_t() const { return xu32(); } + operator uint64_t() const { return xu64(); } + operator int8_t() const { return xi8(); } + operator int16_t() const { return xi16(); } + operator int32_t() const { return xi32(); } + operator int64_t() const { return xi64(); } + operator float() const { return xfloat(); } + operator double() const { return xdouble(); } + operator raw() { return xraw(); } + operator array&() { return xarray(); } + operator map&() { return xmap(); } + operator const_raw() const { return xraw(); } + operator const array&() const { return xarray(); } + operator const map&() const { return xmap(); } + virtual const object_class* inspect(std::ostream& s) const + { s << '<' << typeid(*this).name() << '>'; return this; } +protected: + template + static void check_overflow(X x) { numeric_overflow::check(x); } + template + static void check_underflow(X x) { numeric_underflow::check(x); } +}; + +inline std::ostream& operator<< (std::ostream& s, const object_class* o) + { o->inspect(s); return s; } + + +struct object_container_mixin {}; +struct object_constructor_mixin {}; + + +struct object { + explicit object() : val(NULL) {} + object(object_class* v) : val(v) {} + //object(object_class& v) : val(&v) {} + ~object() {} + bool isnil() const { return val->isnil(); } + bool xbool() const { return val->xbool(); } + uint8_t xu8() const { return val->xu8(); } + uint16_t xu16() const { return val->xu16(); } + uint32_t xu32() const { return val->xu32(); } + uint64_t xu64() const { return val->xu64(); } + int8_t xi8() const { return val->xi8(); } + int16_t xi16() const { return val->xi16(); } + int32_t xi32() const { return val->xi32(); } + int64_t xi64() const { return val->xi64(); } + float xfloat() const { return val->xfloat(); } + double xdouble() const { return val->xdouble(); } + raw xraw() { return val->xraw(); } + array& xarray() { return val->xarray(); } + map& xmap() { return val->xmap(); } + const_raw xraw() const { return const_cast(val)->xraw(); } + const array& xarray() const { return const_cast(val)->xarray(); } + const map& xmap() const { return const_cast(val)->xmap(); } + bool operator== (object x) const { return val->operator== (x.val); } + bool operator!= (object x) const { return val->operator!= (x.val); } + bool operator< (object x) const { return val->operator< (x.val); } + bool operator> (object x) const { return val->operator> (x.val); } + operator bool() const { return val->operator bool(); } + operator uint8_t() const { return val->operator uint8_t(); } + operator uint16_t() const { return val->operator uint16_t(); } + operator uint32_t() const { return val->operator uint32_t(); } + operator uint64_t() const { return val->operator uint64_t(); } + operator int8_t() const { return val->operator int8_t(); } + operator int16_t() const { return val->operator int16_t(); } + operator int32_t() const { return val->operator int32_t(); } + operator int64_t() const { return val->operator int64_t(); } + operator float() const { return val->operator float(); } + operator double() const { return val->operator double(); } + operator raw() { return val->operator raw(); } + operator array&() { return val->operator array&(); } + operator map&() { return val->operator map&(); } + operator raw() const { return val->operator raw(); } + operator array&() const { return val->operator array&(); } + operator map&() const { return val->operator map&(); } + const object& inspect(std::ostream& s) const + { val->inspect(s); return *this; } +private: + friend class object_container_mixin; + friend class object_constructor_mixin; + object_class* val; +}; + +inline std::ostream& operator<< (std::ostream& s, const object& o) + { o.inspect(s); return s; } + + +struct object_nil : object_class { + bool isnil() const { return true; } + bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } + const object_class* inspect(std::ostream& s) const + { s << "nil"; return this; } +}; + +struct object_true : object_class { + bool xbool() const { return true; } + bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } + const object_class* inspect(std::ostream& s) const + { s << "true"; return this; } +}; + +struct object_false : object_class { + bool xbool() const { return false; } + bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } + const object_class* inspect(std::ostream& s) const + { s << "false"; return this; } +}; + +struct object_u8 : object_class { + explicit object_u8(uint8_t v) : val(v) {} + uint8_t xu8() const { return val; } + uint16_t xu16() const { return static_cast(val); } + uint32_t xu32() const { return static_cast(val); } + uint64_t xu64() const { return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { return static_cast(val); } + int32_t xi32() const { return static_cast(val); } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xu8(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xu8(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xu8(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << (uint16_t)val; return this; } +private: + uint8_t val; +}; + +struct object_u16 : object_class { + explicit object_u16(uint16_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { return val; } + uint32_t xu32() const { return static_cast(val); } + uint64_t xu64() const { return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { return static_cast(val); } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xu16(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xu16(); } + catch (positive_overflow_error&) { return true; } + catch (type_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xu16(); } + catch (negative_overflow_error&) { return true; } + catch (type_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + uint16_t val; +}; + +struct object_u32 : object_class { + explicit object_u32(uint32_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { return val; } + uint64_t xu64() const { return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { check_overflow(val); + return static_cast(val); } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { check_underflow(val); + return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xu32(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xu32(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xu32(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + uint32_t val; +}; + +struct object_u64 : object_class { + explicit object_u64(uint64_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { return val; } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { check_overflow(val); + return static_cast(val); } + int64_t xi64() const { check_overflow(val); + return static_cast(val); } + float xfloat() const { check_underflow(val); + return static_cast(val); } + double xdouble() const { check_underflow(val); + return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xu64(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xu64(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xu64(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + uint64_t val; +}; + +struct object_i8 : object_class { + explicit object_i8(int8_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { return val; } + int16_t xi16() const { return static_cast(val); } + int32_t xi32() const { return static_cast(val); } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xi8(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xi8(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xi8(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << (int16_t)val; return this; } +private: + int8_t val; +}; + +struct object_i16 : object_class { + explicit object_i16(int16_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { return val; } + int32_t xi32() const { return static_cast(val); } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xi16(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xi16(); } + catch (positive_overflow_error&) { return true; } + catch (type_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xi16(); } + catch (negative_overflow_error&) { return true; } + catch (type_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + int16_t val; +}; + +struct object_i32 : object_class { + explicit object_i32(int32_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { return val; } + int64_t xi64() const { return static_cast(val); } + float xfloat() const { check_underflow(val); + return static_cast(val); } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xi32(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xi32(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xi32(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + int32_t val; +}; + +struct object_i64 : object_class { + explicit object_i64(int64_t v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { check_overflow(val); + return static_cast(val); } + int64_t xi64() const { return val; } + float xfloat() const { check_underflow(val); + return static_cast(val); } + double xdouble() const { check_underflow(val); + return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xi64(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xi64(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + bool operator> (const object_class* x) const { try { return val > x->xi64(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + int64_t val; +}; + + +struct object_float : object_class { + object_float(float v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { check_overflow(val); + return static_cast(val); } + int64_t xi64() const { check_overflow(val); + return static_cast(val); } + float xfloat() const { return val; } + double xdouble() const { return static_cast(val); } + bool operator== (const object_class* x) const { try { return val == x->xfloat(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return static_cast(val) < x->xdouble(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + catch (underflow_error&) { + if(val < 0.0) { + if(numeric_overflow::test(val) == -1) { return true; } + try { return static_cast(val) < x->xi64(); } + catch (type_error&) { return true; } + } else { + if(numeric_overflow::test(val) == 1) { return false; } + try { return static_cast(val) < x->xu64(); } + catch (type_error&) { return false; } + } + } } + bool operator> (const object_class* x) const { try { return static_cast(val) > x->xdouble(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + catch (underflow_error&) { + if(val < 0.0) { + if(numeric_overflow::test(val) == -1) { return false; } + try { return static_cast(val) > x->xi64(); } + catch (type_error&) { return false; } + } else { + if(numeric_overflow::test(val) == 1) { return true; } + try { return static_cast(val) > x->xu64(); } + catch (type_error&) { return true; } + } + } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + float val; +}; + + +struct object_double : object_class { + object_double(double v) : val(v) {} + uint8_t xu8() const { check_overflow(val); + return static_cast(val); } + uint16_t xu16() const { check_overflow(val); + return static_cast(val); } + uint32_t xu32() const { check_overflow(val); + return static_cast(val); } + uint64_t xu64() const { check_overflow(val); + return static_cast(val); } + int8_t xi8() const { check_overflow(val); + return static_cast(val); } + int16_t xi16() const { check_overflow(val); + return static_cast(val); } + int32_t xi32() const { check_overflow(val); + return static_cast(val); } + int64_t xi64() const { check_overflow(val); + return static_cast(val); } + float xfloat() const { check_overflow(val); + check_underflow(val); + return static_cast(val); } + double xdouble() const { return val; } + bool operator== (const object_class* x) const { try { return val == x->xdouble(); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { try { return val < x->xdouble(); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + catch (underflow_error&) { + if(val < 0.0) { + if(numeric_overflow::test(val) == -1) { return true; } + try { return static_cast(val) < x->xi64(); } + catch (type_error&) { return true; } + } else { + if(numeric_overflow::test(val) == 1) { return false; } + try { return static_cast(val) < x->xu64(); } + catch (type_error&) { return false; } + } + } } + bool operator> (const object_class* x) const { try { return val > x->xdouble(); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + catch (underflow_error&) { + if(val < 0.0) { + if(numeric_overflow::test(val) == -1) { return false; } + try { return static_cast(val) > x->xi64(); } + catch (type_error&) { return false; } + } else { + if(numeric_overflow::test(val) == 1) { return true; } + try { return static_cast(val) > x->xu64(); } + catch (type_error&) { return true; } + } + } } + const object_class* inspect(std::ostream& s) const + { s << val; return this; } +private: + double val; +}; + + +struct object_raw : object_class { + explicit object_raw(void* p, uint32_t l) : ptr(p), len(l) {} + raw xraw() { return raw(ptr, len); } + const_raw xraw() const { return const_raw(ptr, len); } + bool operator== (const object_class* x) const { try { const_raw xr(x->xraw()); + return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { const_raw xr(x->xraw()); + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } + else { return len < xr.len; } } + bool operator> (const object_class* x) const { const_raw xr(x->xraw()); + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } + else { return len > xr.len; } } + const object_class* inspect(std::ostream& s) const + { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape +private: + void* ptr; + uint32_t len; +}; + +struct object_const_raw : object_class { + explicit object_const_raw(const void* p, uint32_t l) : ptr(p), len(l) {} + const_raw xraw() const { return const_raw(ptr, len); } + bool operator== (const object_class* x) const { try { const_raw xr(x->xraw()); + return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); } + catch (type_error&) { return false; } } + bool operator< (const object_class* x) const { const_raw xr(x->xraw()); + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } + else { return len < xr.len; } } + bool operator> (const object_class* x) const { const_raw xr(x->xraw()); + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } + else { return len > xr.len; } } + const object_class* inspect(std::ostream& s) const + { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape +private: + const void* ptr; + uint32_t len; +}; + +struct object_array : object_class, object_container_mixin { + explicit object_array() {} + explicit object_array(uint32_t n) { val.reserve(n); } + array& xarray() { return val; } + const array& xarray() const { return val; } + bool operator== (const object_class* x) const { try { + const std::vector& xa(x->xarray()); + if(val.size() != xa.size()) { return false; } + for(std::vector::const_iterator iv(val.begin()), iv_end(val.end()), ix(xa.begin()); + iv != iv_end; + ++iv, ++ix) { + if(*iv != *ix) { return false; } + } + return true; + } catch (type_error&) { return false; } } + // FIXME operator< operator> + const object_class* inspect(std::ostream& s) const { + s << '['; + if(!val.empty()) { + std::vector::const_iterator it(val.begin()); + s << *it; + ++it; + for(std::vector::const_iterator it_end(val.end()); + it != it_end; + ++it) { + s << ", " << *it; + } + } + s << ']'; + return this; } +public: + void push_back(object o) { val.push_back(o); } +private: + std::vector val; +}; + +// FIXME hash, operator==: nil, true, false, containerを入れられãªã„ +struct object_map : object_class, object_container_mixin { + explicit object_map() {} + map& xmap() { return val; } + const map& xmap() const { return val; } + bool operator== (const object_class* x) const { try { + const std::map& xm(x->xmap()); + if(val.size() != xm.size()) { return false; } + for(std::map::const_iterator iv(val.begin()), iv_end(val.end()), ix(xm.begin()); + iv != iv_end; + ++iv, ++ix) { + if(iv->first != ix->first || iv->second != ix->first) { return false; } + } + return true; + } catch (type_error&) { return false; } } + // FIXME operator< operator> + const object_class* inspect(std::ostream& s) const { + s << '{'; + if(!val.empty()) { + std::map::const_iterator it(val.begin()); + s << it->first << "=>" << it->second; + ++it; + for(std::map::const_iterator it_end(val.end()); + it != it_end; + ++it) { + s << ", " << it->first << "=>" << it->second; + } + } + s << '}'; + return this; } +public: + void store(object k, object v) { val[k] = v; } +private: + std::map val; +}; + + +} // namespace msgpack + +#endif /* msgpack/object.hpp */ + diff --git a/cpp/test.cpp b/cpp/test.cpp new file mode 100644 index 0000000..423a6bd --- /dev/null +++ b/cpp/test.cpp @@ -0,0 +1,110 @@ +#include +#include + +class checker { +public: + void check(const char* d, size_t len, msgpack::object should) { + try { + std::cout << "----" << std::endl; + msgpack::object o; + try { + o = msgpack::unpack(d, len, m_zone); + } catch (std::runtime_error& e) { + std::cout << should << std::endl; + std::cout << "**" << e.what() << "**" << std::endl; + return; + } + std::cout << o << std::endl; + if(o != should) { + std::cout << "** TEST FAILED **" << std::endl; + } + } catch (...) { m_zone.clear(); throw; } + m_zone.clear(); + } +private: + msgpack::zone m_zone; +}; + +int main(void) +{ + +checker c; + +{ // SimpleValue + msgpack::zone z; + const char d[] = { + 0x93, 0xc0, 0xc2, 0xc3, + }; + c.check(d, sizeof(d), + z.narray( + z.nnil(), z.nfalse(), z.ntrue() + ) + ); +} + +{ // Fixnum + msgpack::zone z; + const char d[] = { + 0x92, + 0x93, 0x00, 0x40, 0x7f, + 0x93, 0xe0, 0xf0, 0xff, + }; + c.check(d, sizeof(d), + z.narray( + z.narray( + z.nu8(0), + z.nu8(64), + z.nu8(127) + ), + z.narray( + z.ni8(-32), + z.ni8(-16), + z.ni8(-1) + ) + ) + ); +} + +{ // FixArray + msgpack::zone z; + const char d[] = { + 0x92, + 0x90, + 0x91, + 0x91, 0xc0, + }; + c.check(d, sizeof(d), + z.narray( + z.narray(), + z.narray( + z.narray( + z.nnil() + ) + ) + ) + ); +} + +{ // FixRaw + msgpack::zone z; + const char d[] = { + 0x94, + 0xa0, + 0xa1, 'a', + 0xa2, 'b', 'c', + 0xa3, 'd', 'e', 'f', + }; + c.check(d, sizeof(d), + z.narray( + z.nraw("", 0), + z.nraw("a", 1), + z.nraw("bc", 2), + z.nraw("def", 3) + ) + ); +} + + +return 0; +} + diff --git a/cpp/unpack.cc b/cpp/unpack.cc deleted file mode 100644 index ce57e67..0000000 --- a/cpp/unpack.cc +++ /dev/null @@ -1,30 +0,0 @@ -#include "unpack.h" -#include "unpack_context.h" -#include - -msgpack_unpack_t* msgpack_unpack_new(void) -{ - msgpack_unpacker* ctx; - ctx = (msgpack_unpacker*)calloc(1, sizeof(msgpack_unpacker)); - if(ctx == NULL) { return NULL; } - msgpack_unpacker_init(ctx); - return (msgpack_unpack_t*)ctx; -} - -void msgpack_unpack_free(msgpack_unpack_t* ctx) -{ - free((msgpack_unpacker*)ctx); -} - -int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off) -{ - return msgpack_unpacker_execute( - (msgpack_unpacker*)ctx, - data, len, off); -} - -void* msgpack_unpack_data(msgpack_unpack_t* ctx) -{ - return msgpack_unpacker_data((msgpack_unpacker*)ctx); -} - diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp new file mode 100644 index 0000000..0f02d3c --- /dev/null +++ b/cpp/unpack.cpp @@ -0,0 +1,96 @@ +#include "msgpack/unpack.hpp" +#include "unpack_context.hpp" +#include + +namespace msgpack { + +struct unpacker::context { + context(zone& z) + { + msgpack_unpacker_init(&m_ctx); + m_ctx.user = &z; + } + + ~context() { } + + int execute(const void* data, size_t len, size_t* off) + { + return msgpack_unpacker_execute(&m_ctx, (const char*)data, len, off); + } + + object_class* data() + { + return msgpack_unpacker_data(&m_ctx); + } + + void reset() + { + zone* z = m_ctx.user; + msgpack_unpacker_init(&m_ctx); + m_ctx.user = z; + } + +private: + msgpack_unpacker m_ctx; + +private: + context(); + context(const context&); +}; + + +unpacker::unpacker(zone& z) : + m_ctx(new context(z)), + m_zone(z), + m_finished(false) +{ } + + +unpacker::~unpacker() { delete m_ctx; } + + +size_t unpacker::execute(const void* data, size_t len, size_t off) +{ + int ret = m_ctx->execute(data, len, &off); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret > 0) { + m_finished = true; + return off; + } else { + m_finished = false; + return off; + } +} + + +object unpacker::data() +{ + return object(m_ctx->data()); +} + + +void unpacker::reset() +{ + m_ctx->reset(); +} + + +object unpacker::unpack(const void* data, size_t len, zone& z) +{ + context ctx(z); + size_t off = 0; + int ret = ctx.execute(data, len, &off); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + throw unpack_error("insufficient bytes"); + } else if(off < len) { + throw unpack_error("extra bytes"); + } + return ctx.data(); +} + + +} // namespace msgpack + diff --git a/cpp/unpack.h b/cpp/unpack.h deleted file mode 100644 index cf7a168..0000000 --- a/cpp/unpack.h +++ /dev/null @@ -1,126 +0,0 @@ -#ifndef MSGPACK_UNPACK_H__ -#define MSGPACK_UNPACK_H__ - -#include -#include - -namespace MessagePack { - -class Unpacker { - class object { - template - object(const T& x) : m_obj(new holder(x)) {} - }; - - class type_error : public std::exception { }; - class cast_error : public type_error { }; - class overflow_error : public type_error { }; - class underflow_error : public type_error { }; - - struct object { - virtual ~object() {} - virtual bool isnil() const { return false; } - virtual bool xbool() const { throw cast_error(); } - virtual uint8_t xu8() const { throw cast_error(); } - virtual uint16_t xu16() const { throw cast_error(); } - virtual uint32_t xu32() const { throw cast_error(); } - virtual uint64_t xu64() const { throw cast_error(); } - virtual int8_t xi8() const { throw cast_error(); } - virtual int16_t xi16() const { throw cast_error(); } - virtual int32_t xi32() const { throw cast_error(); } - virtual int64_t xi64() const { throw cast_error(); } - virtual float xfloat() const { throw cast_error(); } - virtual double xdouble() const { throw cast_error(); } - virtual std::map& xmap() const { throw cast_error(); } - virtual std::string& xstring() const { throw cast_error(); } - virtual std::pair xraw() const { throw cast_error(); } - public: - template - inline void check_overflow(X x) { - if(std::numeric_limits::max() < x) { throw overflow_error(); } - } - template - inline void check_underflow(X x) { - if(std::numeric_limits::min() > x) { throw overflow_error(); } - } - }; - -private: - struct object_nil : object { - bool isnil() const { return true; } - }; - - struct object_true : object { - bool xbool() const { return true; } - }; - - struct object_false : object { - bool xbool() const { return false; } - }; - - struct object_u8 : object { - object_u8(uint8_t val) : m_val(val) {} - uint8_t xu8() const { return m_val; } - uint16_t xu16() const { return static_cast(m_val); } - uint32_t xu32() const { return static_cast(m_val); } - uint64_t xu64() const { return static_cast(m_val); } - int8_t xi8() const { check_overflow(m_val); return m_val; } - int16_t xi16() const { return static_cast(m_val); } - int32_t xi32() const { return static_cast(m_val); } - int64_t xi64() const { return static_cast(m_val); } - private: - uint8_t m_val; - }; - - struct object_u16 : object { - object_u16(uint16_t val) : m_val(val) {} - uint8_t xu8() const { check_overflow(m_val); return m_val; } - uint16_t xu16() const { return m_val; } - uint32_t xu32() const { return static_cast(m_val); } - uint64_t xu64() const { return static_cast(m_val); } - int8_t xi8() const { check_overflow< int8_t>(m_val); return m_val; } - int16_t xi16() const { check_overflow(m_val); return m_val; } - int32_t xi32() const { return static_cast(m_val); } - int64_t xi64() const { return static_cast(m_val); } - private: - uint16_t m_val; - }; - - ... -}; - -} // namespace MessagePack - -typedef struct { - void* (*unpack_unsigned_int_8)(void* data, uint8_t d); - void* (*unpack_unsigned_int_16)(void* data, uint16_t d); - void* (*unpack_unsigned_int_32)(void* data, uint32_t d); - void* (*unpack_unsigned_int_64)(void* data, uint64_t d); - void* (*unpack_signed_int_8)(void* data, int8_t d); - void* (*unpack_signed_int_16)(void* data, int16_t d); - void* (*unpack_signed_int_32)(void* data, int32_t d); - void* (*unpack_signed_int_64)(void* data, int64_t d); - void* (*unpack_float)(void* data, float d); - void* (*unpack_double)(void* data, double d); - void* (*unpack_big_int)(void* data, const void* b, unsigned int l); - void* (*unpack_big_float)(void* data, const void* b, unsigned int l); - void* (*unpack_nil)(void* data); - void* (*unpack_true)(void* data); - void* (*unpack_false)(void* data); - void* (*unpack_array_start)(void* data, unsigned int n); - void (*unpack_array_item)(void* data, void* c, void* o); - void* (*unpack_map_start)(void* data, unsigned int n); - void (*unpack_map_item)(void* data, void* c, void* k, void* v); - void* (*unpack_string)(void* data, const void* b, size_t l); - void* (*unpack_raw)(void* data, const void* b, size_t l); - void* data; -} msgpack_unpack_t; - -msgpack_unpack_t* msgpack_unpack_new(void); -void msgpack_unpack_free(msgpack_unpack_t* ctx); -int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off); -void* msgpack_unpack_data(msgpack_unpack_t* ctx); - -#endif /* msgpack/unpack.h */ - - diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp new file mode 100644 index 0000000..61ba781 --- /dev/null +++ b/cpp/unpack.hpp @@ -0,0 +1,48 @@ +#ifndef MSGPACK_UNPACK_HPP__ +#define MSGPACK_UNPACK_HPP__ + +#include "msgpack/object.hpp" +#include "msgpack/zone.hpp" +#include + +namespace msgpack { + + +struct unpack_error : public std::runtime_error { + unpack_error(const std::string& msg) : + std::runtime_error(msg) { } +}; + + +class unpacker { +public: + unpacker(zone& z); + ~unpacker(); +public: + size_t execute(const void* data, size_t len, size_t off); + bool is_finished() { return m_finished; } + object data(); + void reset(); +private: + struct context; + context* m_ctx; + zone& m_zone; + bool m_finished; +private: + unpacker(); + unpacker(const unpacker&); +public: + static object unpack(const void* data, size_t len, zone& z); +}; + + +inline object unpack(const void* data, size_t len, zone& z) +{ + return unpacker::unpack(data, len, z); +} + + +} // namespace msgpack + +#endif /* msgpack/unpack.hpp */ + diff --git a/cpp/unpack_context.h b/cpp/unpack_context.h deleted file mode 100644 index caf2271..0000000 --- a/cpp/unpack_context.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef UNPACK_CONTEXT_H__ -#define UNPACK_CONTEXT_H__ - -#include "unpack.h" - -typedef void* msgpack_object; - -typedef msgpack_unpack_t msgpack_unpack_context; - -#include "msgpack/unpack/inline_context.h" - -#endif /* unpack_context.h */ - diff --git a/cpp/unpack_context.hpp b/cpp/unpack_context.hpp new file mode 100644 index 0000000..49a93e4 --- /dev/null +++ b/cpp/unpack_context.hpp @@ -0,0 +1,14 @@ +#ifndef UNPACK_CONTEXT_HPP__ +#define UNPACK_CONTEXT_HPP__ + +#include "msgpack/zone.hpp" +#include "msgpack/object.hpp" + +typedef msgpack::object_class* msgpack_object; + +typedef msgpack::zone* msgpack_unpack_context; + +#include "msgpack/unpack/inline_context.h" + +#endif /* unpack_context.h */ + diff --git a/cpp/unpack_inline.cpp b/cpp/unpack_inline.cpp new file mode 100644 index 0000000..82f6e7a --- /dev/null +++ b/cpp/unpack_inline.cpp @@ -0,0 +1,72 @@ +#include "unpack_context.hpp" + + +extern "C" { +using namespace msgpack; + + +static inline object_class* msgpack_unpack_init(zone** z) +{ return NULL; } + +static inline object_class* msgpack_unpack_unsigned_int_8(zone** z, uint8_t d) +{ return (*z)->nu8(d); } + +static inline object_class* msgpack_unpack_unsigned_int_16(zone** z, uint16_t d) +{ return (*z)->nu16(d); } + +static inline object_class* msgpack_unpack_unsigned_int_32(zone** z, uint32_t d) +{ return (*z)->nu32(d); } + +static inline object_class* msgpack_unpack_unsigned_int_64(zone** z, uint64_t d) +{ return (*z)->nu64(d); } + +static inline object_class* msgpack_unpack_signed_int_8(zone** z, int8_t d) +{ return (*z)->ni8(d); } + +static inline object_class* msgpack_unpack_signed_int_16(zone** z, int16_t d) +{ return (*z)->ni16(d); } + +static inline object_class* msgpack_unpack_signed_int_32(zone** z, int32_t d) +{ return (*z)->ni32(d); } + +static inline object_class* msgpack_unpack_signed_int_64(zone** z, int64_t d) +{ return (*z)->ni64(d); } + +static inline object_class* msgpack_unpack_float(zone** z, float d) +{ return (*z)->nfloat(d); } + +static inline object_class* msgpack_unpack_double(zone** z, double d) +{ return (*z)->ndouble(d); } + +static inline object_class* msgpack_unpack_nil(zone** z) +{ return (*z)->nnil(); } + +static inline object_class* msgpack_unpack_true(zone** z) +{ return (*z)->ntrue(); } + +static inline object_class* msgpack_unpack_false(zone** z) +{ return (*z)->nfalse(); } + +static inline object_class* msgpack_unpack_array_start(zone** z, unsigned int n) +{ return (*z)->narray(n); } + +static inline void msgpack_unpack_array_item(zone** z, object_class* c, object_class* o) +{ reinterpret_cast(c)->push_back(o); } + +static inline object_class* msgpack_unpack_map_start(zone** z, unsigned int n) +{ return (*z)->narray(); } + +static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) +{ reinterpret_cast(c)->store(k, v); } + +static inline object_class* msgpack_unpack_string(zone** z, const void* b, size_t l) +{ return (*z)->nraw(b, l); } + +static inline object_class* msgpack_unpack_raw(zone** z, const void* b, size_t l) +{ return (*z)->nraw(b, l); } + + +} // extern "C" + +#include "msgpack/unpack/inline_impl.h" + diff --git a/cpp/zone.cpp b/cpp/zone.cpp new file mode 100644 index 0000000..5031467 --- /dev/null +++ b/cpp/zone.cpp @@ -0,0 +1,40 @@ +#include "zone.hpp" + +namespace msgpack { + + +void* zone::alloc() +{ + if(m_used >= m_pool.size()*MSGPACK_ZONE_CHUNK_SIZE) { + m_pool.push_back(chunk_t()); + } + void* data = m_pool[m_used/MSGPACK_ZONE_CHUNK_SIZE].cells[m_used%MSGPACK_ZONE_CHUNK_SIZE].data; + ++m_used; + return data; +} + +void zone::clear() +{ + for(size_t b=0; b < m_used/MSGPACK_ZONE_CHUNK_SIZE; ++b) { + cell_t* c(m_pool[b].cells); + for(size_t e=0; e < MSGPACK_ZONE_CHUNK_SIZE; ++e) { + reinterpret_cast(c[e].data)->~object_class(); + } + } + cell_t* c(m_pool.back().cells); + for(size_t e=0; e < m_used%MSGPACK_ZONE_CHUNK_SIZE; ++e) { + reinterpret_cast(c[e].data)->~object_class(); + } + m_used = 0; + m_pool.resize(1); + for(user_finalizer_t::reverse_iterator it(m_user_finalizer.rbegin()), it_end(m_user_finalizer.rend()); + it != it_end; + ++it) { + it->call(); + } + m_user_finalizer.clear(); +} + + +} // namespace msgpack + diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb new file mode 100644 index 0000000..1a941af --- /dev/null +++ b/cpp/zone.hpp.erb @@ -0,0 +1,123 @@ +#ifndef MSGPACK_ZONE_HPP__ +#define MSGPACK_ZONE_HPP__ + +#include "msgpack/object.hpp" +#include + +#ifndef MSGPACK_ZONE_CHUNK_SIZE +#define MSGPACK_ZONE_CHUNK_SIZE 64 +#endif + +namespace msgpack { + + +class zone { +public: +zone() : m_used(0), m_pool(1) { } +~zone() { clear(); } + +public: + template + void push_finalizer(void (*func)(void* obj, void* user), T* obj, void* user); + +public: + object_nil* nnil() { return new (alloc()) object_nil(); } + object_true* ntrue() { return new (alloc()) object_true(); } + object_false* nfalse() { return new (alloc()) object_false(); } + object_u8* nu8( uint8_t v) { return new (alloc()) object_u8(v); } + object_u16* nu16(uint16_t v) { return new (alloc()) object_u16(v); } + object_u32* nu32(uint32_t v) { return new (alloc()) object_u32(v); } + object_u64* nu64(uint64_t v) { return new (alloc()) object_u64(v); } + object_i8* ni8( int8_t v) { return new (alloc()) object_i8(v); } + object_i16* ni16( int16_t v) { return new (alloc()) object_i16(v); } + object_i32* ni32( int32_t v) { return new (alloc()) object_i32(v); } + object_i64* ni64( int64_t v) { return new (alloc()) object_i64(v); } + object_float* nfloat( float v) { return new (alloc()) object_float(v); } + object_double* ndouble( double v) { return new (alloc()) object_double(v); } + + object_raw* nraw(void* ptr, uint32_t len) + { return new (alloc()) object_raw(ptr, len); } + + object_const_raw* nraw(const void* ptr, uint32_t len) + { return new (alloc()) object_const_raw(ptr, len); } + + object_array* narray() + { return new (alloc()) object_array(); } + + object_array* narray(size_t reserve_size) + { return new (alloc()) object_array(reserve_size); } + + object_map* nmap() + { return new (alloc()) object_map(); } + +<% GENERATION_SIZE = 16 %> +<% 1.upto(GENERATION_SIZE) {|i| %> + object_array* narray(<% 1.upto(i-1) {|n| %>object o<%=n%>, <% } %>object o<%=i%>) + { object_array* a = new (alloc()) object_array(<%=i%>); + <% 1.upto(i) {|n| %>a->push_back(o<%=n%>); + <% } %>return a; } +<% } %> + +<% 1.upto(GENERATION_SIZE) {|i| %> + object_map* nmap(<% 1.upto(i-1) {|n| %>object k<%=n%>, object v<%=n%>, <% } %>object k<%=i%>, object v<%=i%>) + { object_map* m = new (alloc()) object_map(); + <% 1.upto(i) {|n| %>m->store(k<%=n%>, v<%=n%>); + <% } %>return m; } +<% } %> + +public: + void clear(); + +private: + void* alloc(); + +private: + size_t m_used; + + static const size_t MAX_OBJECT_SIZE = + sizeof(object_raw) > sizeof(object_array) + ? ( sizeof(object_raw) > sizeof(object_map) + ? sizeof(object_raw) + : sizeof(object_map) + ) + : ( sizeof(object_array) > sizeof(object_map) + ? sizeof(object_array) + : sizeof(object_map) + ) + ; + + struct cell_t { + char data[MAX_OBJECT_SIZE]; + }; + + struct chunk_t { + cell_t cells[MSGPACK_ZONE_CHUNK_SIZE]; + }; + + typedef std::vector pool_t; + pool_t m_pool; + + + class finalizer { + public: + finalizer(void (*func)(void*, void*), void* obj, void* user) : + m_obj(obj), m_user(user), m_func(func) {} + void call() { (*m_func)(m_obj, m_user); } + private: + void* m_obj; + void* m_user; + void (*m_func)(void*, void*); + }; + + typedef std::vector user_finalizer_t; + user_finalizer_t m_user_finalizer; + +private: + zone(const zone&); +}; + + +} // namespace msgpack + +#endif /* msgpack/zone.hpp */ + diff --git a/msgpack/pack/inline_context.h b/msgpack/pack/inline_context.h deleted file mode 100644 index 139597f..0000000 --- a/msgpack/pack/inline_context.h +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/msgpack/pack/inline_impl.h b/msgpack/pack/inline_impl.h index 5c4bfed..08a5bc1 100644 --- a/msgpack/pack/inline_impl.h +++ b/msgpack/pack/inline_impl.h @@ -23,25 +23,25 @@ #ifdef __LITTLE_ENDIAN__ -#define STORE_16(d) \ +#define STORE_BE16(d) \ ((char*)&d)[1], ((char*)&d)[0] -#define STORE_32(d) \ +#define STORE_BE32(d) \ ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] -#define STORE_64(d) \ +#define STORE_BE64(d) \ ((char*)&d)[7], ((char*)&d)[6], ((char*)&d)[5], ((char*)&d)[4], \ ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] #elif __BIG_ENDIAN__ -#define STORE_16(d) \ - ((char*)&d)[2], ((char*)&d)[3] +#define STORE_BE16(d) \ + ((char*)&d)[0], ((char*)&d)[1] -#define STORE_32(d) \ +#define STORE_BE32(d) \ ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3] -#define STORE_32(d) \ +#define STORE_BE64(d) \ ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3], \ ((char*)&d)[4], ((char*)&d)[5], ((char*)&d)[6], ((char*)&d)[7] @@ -57,10 +57,10 @@ inline void msgpack_pack_int(msgpack_pack_context x, int d) { if(d < -32) { if(d < -32768) { // signed 32 - const unsigned char buf[5] = {0xd2, STORE_32(d)}; + const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } else if(d < -128) { // signed 16 - const unsigned char buf[3] = {0xd1, STORE_16(d)}; + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { // signed 8 const unsigned char buf[2] = {0xd0, (uint8_t)d}; @@ -75,11 +75,11 @@ inline void msgpack_pack_int(msgpack_pack_context x, int d) msgpack_pack_append_buffer(x, buf, 2); } else if(d < 65536) { // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_16(d)}; + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_32(d)}; + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -97,11 +97,11 @@ inline void msgpack_pack_unsigned_int(msgpack_pack_context x, unsigned int d) msgpack_pack_append_buffer(x, buf, 2); } else if(d < 65536) { // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_16(d)}; + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_32(d)}; + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -118,20 +118,20 @@ inline void msgpack_pack_unsigned_int_8(msgpack_pack_context x, uint8_t d) inline void msgpack_pack_unsigned_int_16(msgpack_pack_context x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE_16(d)}; + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } inline void msgpack_pack_unsigned_int_32(msgpack_pack_context x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE_32(d)}; + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } inline void msgpack_pack_unsigned_int_64(msgpack_pack_context x, uint64_t d) { - // FIXME - const unsigned char buf[9] = {0xcf, STORE_64(d)}; + // FIXME optimization + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -149,20 +149,20 @@ inline void msgpack_pack_signed_int_8(msgpack_pack_context x, int8_t d) inline void msgpack_pack_signed_int_16(msgpack_pack_context x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE_16(d)}; + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } inline void msgpack_pack_signed_int_32(msgpack_pack_context x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE_32(d)}; + const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } inline void msgpack_pack_signed_int_64(msgpack_pack_context x, int64_t d) { - // FIXME - const unsigned char buf[9] = {0xd3, STORE_64(d)}; + // FIXME optimization + const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -174,14 +174,14 @@ inline void msgpack_pack_signed_int_64(msgpack_pack_context x, int64_t d) inline void msgpack_pack_float(msgpack_pack_context x, float d) { uint32_t n = *((uint32_t*)&d); // FIXME - const unsigned char buf[5] = {0xca, STORE_32(n)}; + const unsigned char buf[5] = {0xca, STORE_BE32(n)}; msgpack_pack_append_buffer(x, buf, 5); } inline void msgpack_pack_double(msgpack_pack_context x, double d) { uint64_t n = *((uint64_t*)&d); // FIXME - const unsigned char buf[9] = {0xcb, STORE_64(n)}; + const unsigned char buf[9] = {0xcb, STORE_BE64(n)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -200,6 +200,7 @@ inline void msgpack_pack_nil(msgpack_pack_context x) /* * Boolean */ + inline void msgpack_pack_true(msgpack_pack_context x) { static const unsigned char d = 0xc3; @@ -224,11 +225,11 @@ inline void msgpack_pack_array(msgpack_pack_context x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE_16(d)}; + unsigned char buf[3] = {0xdc, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE_32(d)}; + unsigned char buf[5] = {0xdd, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -245,18 +246,18 @@ inline void msgpack_pack_map(msgpack_pack_context x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE_16(d)}; + unsigned char buf[3] = {0xde, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE_32(d)}; + unsigned char buf[5] = {0xdf, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } /* - * String + * Raw */ inline void msgpack_pack_string(msgpack_pack_context x, const char* b) @@ -272,16 +273,21 @@ inline void msgpack_pack_raw(msgpack_pack_context x, const void* b, size_t l) msgpack_pack_append_buffer(x, &d, 1); } else if(l < 65536) { uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE_16(d)}; + unsigned char buf[3] = {0xda, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE_32(d)}; + unsigned char buf[5] = {0xdb, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_append_buffer(x, b, l); } +#undef STORE_BE16(d) +#undef STORE_BE32(d) +#undef STORE_BE64(d) + + #endif /* msgpack/pack/inline_impl.h */ diff --git a/msgpack/unpack/inline_context.h b/msgpack/unpack/inline_context.h index aecd566..d6558a3 100644 --- a/msgpack/unpack/inline_context.h +++ b/msgpack/unpack/inline_context.h @@ -25,6 +25,10 @@ #define MSG_STACK_SIZE 16 #endif +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { msgpack_object obj; size_t count; @@ -47,6 +51,9 @@ void msgpack_unpacker_init(msgpack_unpacker* ctx); int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len, size_t* off); #define msgpack_unpacker_data(unpacker) (unpacker)->stack[0].obj +#ifdef __cplusplus +} +#endif #endif /* msgpack/unpack/inline_context.h */ diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h index ec7f0fc..f610dd7 100644 --- a/msgpack/unpack/inline_impl.h +++ b/msgpack/unpack/inline_impl.h @@ -23,6 +23,10 @@ #include /*#include */ +#ifdef __cplusplus +extern "C" { +#endif + // Positive FixNum 0xxxxxxx 0x00 - 0x7f // Negative FixNum 111xxxxx 0xe0 - 0xff // Variable 110xxxxx 0xc0 - 0xdf @@ -71,13 +75,16 @@ #endif #endif -static inline uint64_t ntohll(uint64_t x) { -#ifdef __LITTLE_ENDIAN__ // FIXME +#define betoh16(x) ntohs(x) +#define betoh32(x) ntohl(x) + +#ifdef __LITTLE_ENDIAN__ #if defined(__bswap_64) - return __bswap_64(x); +# define betoh64(x) __bswap_64(x) #elif defined(__DARWIN_OSSwapInt64) - return __DARWIN_OSSwapInt64(x); +# define betoh64(x) __DARWIN_OSSwapInt64(x) #else +static inline uint64_t betoh64(uint64_t x) { return ((x << 56) & 0xff00000000000000ULL ) | ((x << 40) & 0x00ff000000000000ULL ) | ((x << 24) & 0x0000ff0000000000ULL ) | @@ -86,11 +93,12 @@ static inline uint64_t ntohll(uint64_t x) { ((x >> 24) & 0x0000000000ff0000ULL ) | ((x >> 40) & 0x000000000000ff00ULL ) | ((x >> 56) & 0x00000000000000ffULL ) ; +} #endif #else - return x; +#define betoh64(x) (x) #endif -} + typedef enum { CS_HEADER = 0x00, // nil @@ -212,9 +220,9 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) ntohs(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) ntohl(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) ntohll(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) betoh16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) betoh32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) betoh64(*(uint64_t*)ptr) if(p == pe) { goto _out; } do { @@ -434,5 +442,21 @@ _end: } +#ifdef betoh16(x) +#undef betoh16(x) +#endif + +#ifdef betoh32(x) +#undef betoh32(x) +#endif + +#ifdef betoh64(x) +#undef betoh64(x) +#endif + +#ifdef __cplusplus +} +#endif + #endif /* msgpack/unpack/inline_impl.h */ From 7c427400a7071f0cb251edd102430945e142033f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0004/1648] lang/c/msgpack: update C++ code git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@51 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bench.c | 8 +- cpp/Makefile | 6 +- cpp/object.cpp | 340 ++++++++++++++++++++++++++ cpp/object.hpp | 652 +++++++------------------------------------------ 4 files changed, 439 insertions(+), 567 deletions(-) create mode 100644 cpp/object.cpp diff --git a/c/bench.c b/c/bench.c index f27350a..0f316dc 100644 --- a/c/bench.c +++ b/c/bench.c @@ -98,9 +98,9 @@ static void pack_append_buffer(void* user, const unsigned char* b, unsigned int static const unsigned int TASK_INT_NUM = 1<<24; -static const unsigned int TASK_STR_LEN = 1<<15; +//static const unsigned int TASK_STR_LEN = 1<<15; //static const unsigned int TASK_INT_NUM = 1<<20; -//static const unsigned int TASK_STR_LEN = 1<<12; +static const unsigned int TASK_STR_LEN = 1<<12; static const char* TASK_STR_PTR; @@ -284,6 +284,7 @@ void bench_msgpack(void) msgpack_unpack_reset(mupk); + /* puts("----"); puts("pack string"); reset_timer(); @@ -316,6 +317,7 @@ void bench_msgpack(void) sec = show_timer(); printf("%f MB/s\n", len / sec / 1024 / 1024); + */ msgpack_unpack_free(mupk); @@ -330,7 +332,7 @@ int main(int argc, char* argv[]) TASK_STR_PTR = str; bench_msgpack(); - bench_json(); +// bench_json(); return 0; } diff --git a/cpp/Makefile b/cpp/Makefile index eafa683..04b421c 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -9,12 +9,12 @@ all: test %.hpp: %.hpp.erb erb $< > $@ -test: $(NEED_PREPROCESS) unpack.o unpack_inline.o zone.o test.o object.hpp unpack.hpp - $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o test.o -o $@ +test: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o test.o object.hpp unpack.hpp + $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o test.o -o $@ .PHONY: clean clean: - $(RM) unpack.o unpack_inline.o zone.o + $(RM) unpack.o unpack_inline.o object.o zone.o $(RM) test.o $(RM) test $(RM) $(NEED_PREPROCESS) diff --git a/cpp/object.cpp b/cpp/object.cpp new file mode 100644 index 0000000..099b541 --- /dev/null +++ b/cpp/object.cpp @@ -0,0 +1,340 @@ +#include "msgpack/object.hpp" + +namespace msgpack { + +namespace { + +template +struct numeric_overflow_signed_impl; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast( std::numeric_limits::max()) < x ) { return 1; } + if( static_cast(-std::numeric_limits::max()) > x ) { return -1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + if( static_cast(-std::numeric_limits::max()) > x) { return -1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) <= sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( static_cast(0) > x ) { return -1; } + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast( std::numeric_limits::max()) < x) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow_signed_impl { + static int test(X x) { + if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || + (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { + if( sizeof(T) < sizeof(X) ) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + } else if(std::numeric_limits::is_integer) { + if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + } + return 0; + } +}; + +template +struct numeric_overflow { + static int test(X x) { + return numeric_overflow_signed_impl::is_signed, std::numeric_limits::is_signed>::test(x); + } + static void check(X x) { + int r = test(x); + if(r == 1) { throw positive_overflow_error(); } + if(r == -1) { throw negative_overflow_error(); } + } +}; + +template +struct numeric_underflow { + static bool test(X x) { + return static_cast(static_cast(x)) != x; + } + static void check(X x) { + if(test(x)) { throw underflow_error(); } + } +}; + +template +inline T integer_cast(X x) { + numeric_overflow::check(x); + return static_cast(x); } + +template +inline T float_cast(X x) { + numeric_overflow::check(x); + numeric_underflow::check(x); + return static_cast(x); } + +template +inline bool numequal(V v, const object_class* x) + try { return v == static_cast(*x); } + catch (type_error&) { return false; } + +template +inline bool numless(V v, const object_class* x) + try { return v < static_cast(*x); } + catch (positive_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + +template +inline bool numgreater(V v, const object_class* x) + try { return v > static_cast(*x); } + catch (negative_overflow_error&) { return true; } + catch (overflow_error&) { return false; } + +template +inline void numeric_inspect(V v, std::ostream& s) + { s << v; } + +template <> +inline void numeric_inspect(uint8_t v, std::ostream& s) + { s << (uint16_t)v; } + +template <> +inline void numeric_inspect(int8_t v, std::ostream& s) + { s << (int16_t)v; } + +} // noname namespace + + +bool object_nil::isnil() const { return true; } +bool object_nil::operator== (const object_class* x) const + { return typeid(*this) == typeid(*x); } +const object_class* object_nil::inspect(std::ostream& s) const + { s << "nil"; return this; } + +bool object_true::xbool() const { return true; } +bool object_true::operator== (const object_class* x) const + { return typeid(*this) == typeid(*x); } +const object_class* object_true::inspect(std::ostream& s) const + { s << "true"; return this; } + +bool object_false::xbool() const { return false; } +bool object_false::operator== (const object_class* x) const + { return typeid(*this) == typeid(*x); } +const object_class* object_false::inspect(std::ostream& s) const + { s << "false"; return this; } + + +#define INTEGER_OBJECT(NAME) \ +uint8_t object_##NAME::xu8 () const { return val; } \ +uint16_t object_##NAME::xu16 () const { return integer_cast(val); } \ +uint32_t object_##NAME::xu32 () const { return integer_cast(val); } \ +uint64_t object_##NAME::xu64 () const { return integer_cast(val); } \ +int8_t object_##NAME::xi8 () const { return integer_cast(val); } \ +int16_t object_##NAME::xi16 () const { return integer_cast(val); } \ +int32_t object_##NAME::xi32 () const { return integer_cast(val); } \ +int64_t object_##NAME::xi64 () const { return integer_cast(val); } \ +float object_##NAME::xfloat () const { return integer_cast(val); } \ +double object_##NAME::xdouble() const { return integer_cast(val); } \ +bool object_##NAME::operator== (const object_class* x) const \ + try { return val == x->x##NAME(); } \ + catch (type_error&) { return false; } \ +bool object_##NAME::operator< (const object_class* x) const \ + try { return val < x->x##NAME(); } \ + catch (positive_overflow_error&) { return true; } \ + catch (overflow_error&) { return false; } \ +bool object_##NAME::operator> (const object_class* x) const \ + try { return val > x->x##NAME(); } \ + catch (negative_overflow_error&) { return true; } \ + catch (overflow_error&) { return false; } \ +const object_class* object_##NAME::inspect(std::ostream& s) const \ + { numeric_inspect(val, s); return this; } \ + + +INTEGER_OBJECT(u8) +INTEGER_OBJECT(u16) +INTEGER_OBJECT(u32) +INTEGER_OBJECT(u64) +INTEGER_OBJECT(i8) +INTEGER_OBJECT(i16) +INTEGER_OBJECT(i32) +INTEGER_OBJECT(i64) + +#undef INTEGER_OBJECT(NAME) + + +#define FLOAT_OBJECT(NAME) \ +uint8_t object_##NAME::xu8 () const { return val; } \ +uint16_t object_##NAME::xu16 () const { return integer_cast(val); } \ +uint32_t object_##NAME::xu32 () const { return integer_cast(val); } \ +uint64_t object_##NAME::xu64 () const { return integer_cast(val); } \ +int8_t object_##NAME::xi8 () const { return integer_cast(val); } \ +int16_t object_##NAME::xi16 () const { return integer_cast(val); } \ +int32_t object_##NAME::xi32 () const { return integer_cast(val); } \ +int64_t object_##NAME::xi64 () const { return integer_cast(val); } \ +float object_##NAME::xfloat () const { return float_cast(val); } \ +double object_##NAME::xdouble() const { return float_cast(val); } \ +bool object_##NAME::operator== (const object_class* x) const \ + try { return val == x->x##NAME(); } \ + catch (type_error&) { return false; } \ +bool object_##NAME::operator< (const object_class* x) const { \ + try { return val < x->xdouble(); } \ + catch (positive_overflow_error&) { return true; } \ + catch (overflow_error&) { return false; } \ + catch (underflow_error&) { \ + if(val < 0.0) { \ + if(numeric_overflow::test(val) == -1) { return true; } \ + try { return static_cast(val) < x->xi64(); } \ + catch (type_error&) { return true; } \ + } else { \ + if(numeric_overflow::test(val) == 1) { return false; } \ + try { return static_cast(val) < x->xu64(); } \ + catch (type_error&) { return false; } \ + } \ + } } \ +bool object_##NAME::operator> (const object_class* x) const { \ + try { return val > x->xdouble(); } \ + catch (negative_overflow_error&) { return true; } \ + catch (overflow_error&) { return false; } \ + catch (underflow_error&) { \ + if(val < 0.0) { \ + if(numeric_overflow::test(val) == -1) { return false; } \ + try { return static_cast(val) > x->xi64(); } \ + catch (type_error&) { return false; } \ + } else { \ + if(numeric_overflow::test(val) == 1) { return true; } \ + try { return static_cast(val) > x->xu64(); } \ + catch (type_error&) { return true; } \ + } \ + } } \ +const object_class* object_##NAME::inspect(std::ostream& s) const \ + { s << val; return this; } \ + +FLOAT_OBJECT(float) +FLOAT_OBJECT(double) + +#undef FLOAT_OBJECT(NAME) + + +#define RAW_OBJECT(NAME, EXTRA) \ +EXTRA \ +bool object_##NAME::operator== (const object_class* x) const \ + try { \ + const_raw xr(x->xraw()); \ + return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); \ + } catch (type_error&) { return false; } \ +bool object_##NAME::operator< (const object_class* x) const { \ + const_raw xr(x->xraw()); \ + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } \ + else { return len < xr.len; } } \ +bool object_##NAME::operator> (const object_class* x) const { \ + const_raw xr(x->xraw()); \ + if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } \ + else { return len > xr.len; } } \ +const object_class* object_##NAME::inspect(std::ostream& s) const \ + { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape + + +RAW_OBJECT(raw, + raw object_raw::xraw() { return raw(ptr, len); } + const_raw object_raw::xraw() const { return const_raw(ptr, len); } ) + +RAW_OBJECT(const_raw, + const_raw object_const_raw::xraw() const { return const_raw(ptr, len); } ) + +#undef RAW_OBJECT(NAME, EXTRA) + + + array& object_array::xarray() { return val; } +const array& object_array::xarray() const { return val; } +bool object_array::operator== (const object_class* x) const + try { + const std::vector& xa(x->xarray()); + if(val.size() != xa.size()) { return false; } + for(std::vector::const_iterator iv(val.begin()), iv_end(val.end()), ix(xa.begin()); + iv != iv_end; + ++iv, ++ix) { + if(*iv != *ix) { return false; } + } + return true; + } catch (type_error&) { return false; } +const object_class* object_array::inspect(std::ostream& s) const +{ + s << '['; + if(!val.empty()) { + std::vector::const_iterator it(val.begin()); + s << *it; + ++it; + for(std::vector::const_iterator it_end(val.end()); + it != it_end; + ++it) { + s << ", " << *it; + } + } + s << ']'; + return this; +} + + + map& object_map::xmap() { return val; } +const map& object_map::xmap() const { return val; } +bool object_map::operator== (const object_class* x) const + try { + const std::map& xm(x->xmap()); + if(val.size() != xm.size()) { return false; } + for(std::map::const_iterator iv(val.begin()), iv_end(val.end()), ix(xm.begin()); + iv != iv_end; + ++iv, ++ix) { + if(iv->first != ix->first || iv->second != ix->first) { return false; } + } + return true; + } catch (type_error&) { return false; } +const object_class* object_map::inspect(std::ostream& s) const +{ + s << '{'; + if(!val.empty()) { + std::map::const_iterator it(val.begin()); + s << it->first << "=>" << it->second; + ++it; + for(std::map::const_iterator it_end(val.end()); + it != it_end; + ++it) { + s << ", " << it->first << "=>" << it->second; + } + } + s << '}'; + return this; +} + + +} // namespace msgpack + diff --git a/cpp/object.hpp b/cpp/object.hpp index c4e94d7..992ac1e 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -48,96 +48,6 @@ typedef std::map map; typedef std::vector array; -template -struct numeric_overflow_signed_impl; - -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast( std::numeric_limits::max()) < x ) { return 1; } - if( static_cast(-std::numeric_limits::max()) > x ) { return -1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } - if( static_cast(-std::numeric_limits::max()) > x) { return -1; } - } - return 0; - } -}; - -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) <= sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } - } - return 0; - } -}; - -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( static_cast(0) > x ) { return -1; } - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } - } - return 0; - } -}; - -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - return 0; - } -}; - -template -struct numeric_overflow { - static int test(X x) { - return numeric_overflow_signed_impl::is_signed, std::numeric_limits::is_signed>::test(x); - } - static void check(X x) { - int r = test(x); - if(r == 1) { throw positive_overflow_error(); } - if(r == -1) { throw negative_overflow_error(); } - } -}; - - -template -struct numeric_underflow { - static bool test(X x) { - return static_cast(static_cast(x)) != x; - } - static void check(X x) { - if(test(x)) { throw underflow_error(); } - } -}; - - struct object_class { virtual ~object_class() {} virtual bool isnil() const { return false; } @@ -181,11 +91,6 @@ struct object_class { operator const map&() const { return xmap(); } virtual const object_class* inspect(std::ostream& s) const { s << '<' << typeid(*this).name() << '>'; return this; } -protected: - template - static void check_overflow(X x) { numeric_overflow::check(x); } - template - static void check_underflow(X x) { numeric_underflow::check(x); } }; inline std::ostream& operator<< (std::ostream& s, const object_class* o) @@ -253,500 +158,125 @@ inline std::ostream& operator<< (std::ostream& s, const object& o) struct object_nil : object_class { - bool isnil() const { return true; } - bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } - const object_class* inspect(std::ostream& s) const - { s << "nil"; return this; } + bool isnil() const; + bool operator== (const object_class* x) const; + const object_class* inspect(std::ostream& s) const; }; struct object_true : object_class { - bool xbool() const { return true; } - bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } - const object_class* inspect(std::ostream& s) const - { s << "true"; return this; } + bool xbool() const; + bool operator== (const object_class* x) const; + const object_class* inspect(std::ostream& s) const; }; struct object_false : object_class { - bool xbool() const { return false; } - bool operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } - const object_class* inspect(std::ostream& s) const - { s << "false"; return this; } + bool xbool() const; + bool operator== (const object_class* x) const; + const object_class* inspect(std::ostream& s) const; }; -struct object_u8 : object_class { - explicit object_u8(uint8_t v) : val(v) {} - uint8_t xu8() const { return val; } - uint16_t xu16() const { return static_cast(val); } - uint32_t xu32() const { return static_cast(val); } - uint64_t xu64() const { return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { return static_cast(val); } - int32_t xi32() const { return static_cast(val); } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xu8(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xu8(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xu8(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << (uint16_t)val; return this; } -private: - uint8_t val; +#define INTEGER_CLASS(TYPE, NAME) \ +struct object_##NAME : object_class { \ + explicit object_##NAME(TYPE v) : val(v) {} \ + uint8_t xu8 () const; \ + uint16_t xu16 () const; \ + uint32_t xu32 () const; \ + uint64_t xu64 () const; \ + int8_t xi8 () const; \ + int16_t xi16 () const; \ + int32_t xi32 () const; \ + int64_t xi64 () const; \ + float xfloat () const; \ + double xdouble() const; \ + bool operator== (const object_class* x) const; \ + bool operator< (const object_class* x) const; \ + bool operator> (const object_class* x) const; \ + const object_class* inspect(std::ostream& s) const; \ +private: \ + TYPE val; \ }; -struct object_u16 : object_class { - explicit object_u16(uint16_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { return val; } - uint32_t xu32() const { return static_cast(val); } - uint64_t xu64() const { return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { return static_cast(val); } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xu16(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xu16(); } - catch (positive_overflow_error&) { return true; } - catch (type_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xu16(); } - catch (negative_overflow_error&) { return true; } - catch (type_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - uint16_t val; +INTEGER_CLASS(uint8_t, u8) +INTEGER_CLASS(uint16_t, u16) +INTEGER_CLASS(uint32_t, u32) +INTEGER_CLASS(uint64_t, u64) +INTEGER_CLASS(int8_t, i8) +INTEGER_CLASS(int16_t, i16) +INTEGER_CLASS(int32_t, i32) +INTEGER_CLASS(int64_t, i64) + +#undef INTEGER_CLASS(TYPE, NAME) + + +#define FLOAT_CLASS(TYPE, NAME) \ +struct object_##NAME : object_class { \ + object_##NAME(TYPE v) : val(v) {} \ + uint8_t xu8 () const; \ + uint16_t xu16 () const; \ + uint32_t xu32 () const; \ + uint64_t xu64 () const; \ + int8_t xi8 () const; \ + int16_t xi16 () const; \ + int32_t xi32 () const; \ + int64_t xi64 () const; \ + float xfloat () const; \ + double xdouble() const; \ + bool operator== (const object_class* x) const; \ + bool operator< (const object_class* x) const; \ + bool operator> (const object_class* x) const; \ + const object_class* inspect(std::ostream& s) const; \ +private: \ + TYPE val; \ }; -struct object_u32 : object_class { - explicit object_u32(uint32_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { return val; } - uint64_t xu64() const { return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { check_overflow(val); - return static_cast(val); } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { check_underflow(val); - return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xu32(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xu32(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xu32(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - uint32_t val; +FLOAT_CLASS(float, float) +FLOAT_CLASS(double, double) + +#undef FLOAT_CLASS(TYPE, NAME) + + +#define RAW_CLASS(NAME, TYPE, EXTRA) \ +struct object_##NAME : object_class { \ + explicit object_##NAME(TYPE p, uint32_t l) : ptr(p), len(l) {} \ + EXTRA \ + bool operator== (const object_class* x) const; \ + bool operator< (const object_class* x) const; \ + bool operator> (const object_class* x) const; \ + const object_class* inspect(std::ostream& s) const; \ +private: \ + TYPE ptr; \ + uint32_t len; \ }; -struct object_u64 : object_class { - explicit object_u64(uint64_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { return val; } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { check_overflow(val); - return static_cast(val); } - int64_t xi64() const { check_overflow(val); - return static_cast(val); } - float xfloat() const { check_underflow(val); - return static_cast(val); } - double xdouble() const { check_underflow(val); - return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xu64(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xu64(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xu64(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - uint64_t val; -}; +RAW_CLASS(raw, void*, raw xraw(); const_raw xraw() const; ) +RAW_CLASS(const_raw, const void*, const_raw xraw() const; ) -struct object_i8 : object_class { - explicit object_i8(int8_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { return val; } - int16_t xi16() const { return static_cast(val); } - int32_t xi32() const { return static_cast(val); } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xi8(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xi8(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xi8(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << (int16_t)val; return this; } -private: - int8_t val; -}; +#undef RAW_CLASS(NAME, TYPE, EXTRA) -struct object_i16 : object_class { - explicit object_i16(int16_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { return val; } - int32_t xi32() const { return static_cast(val); } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xi16(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xi16(); } - catch (positive_overflow_error&) { return true; } - catch (type_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xi16(); } - catch (negative_overflow_error&) { return true; } - catch (type_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - int16_t val; -}; - -struct object_i32 : object_class { - explicit object_i32(int32_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { return val; } - int64_t xi64() const { return static_cast(val); } - float xfloat() const { check_underflow(val); - return static_cast(val); } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xi32(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xi32(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xi32(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - int32_t val; -}; - -struct object_i64 : object_class { - explicit object_i64(int64_t v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { check_overflow(val); - return static_cast(val); } - int64_t xi64() const { return val; } - float xfloat() const { check_underflow(val); - return static_cast(val); } - double xdouble() const { check_underflow(val); - return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xi64(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xi64(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - bool operator> (const object_class* x) const { try { return val > x->xi64(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - int64_t val; -}; - - -struct object_float : object_class { - object_float(float v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { check_overflow(val); - return static_cast(val); } - int64_t xi64() const { check_overflow(val); - return static_cast(val); } - float xfloat() const { return val; } - double xdouble() const { return static_cast(val); } - bool operator== (const object_class* x) const { try { return val == x->xfloat(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return static_cast(val) < x->xdouble(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - catch (underflow_error&) { - if(val < 0.0) { - if(numeric_overflow::test(val) == -1) { return true; } - try { return static_cast(val) < x->xi64(); } - catch (type_error&) { return true; } - } else { - if(numeric_overflow::test(val) == 1) { return false; } - try { return static_cast(val) < x->xu64(); } - catch (type_error&) { return false; } - } - } } - bool operator> (const object_class* x) const { try { return static_cast(val) > x->xdouble(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - catch (underflow_error&) { - if(val < 0.0) { - if(numeric_overflow::test(val) == -1) { return false; } - try { return static_cast(val) > x->xi64(); } - catch (type_error&) { return false; } - } else { - if(numeric_overflow::test(val) == 1) { return true; } - try { return static_cast(val) > x->xu64(); } - catch (type_error&) { return true; } - } - } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - float val; -}; - - -struct object_double : object_class { - object_double(double v) : val(v) {} - uint8_t xu8() const { check_overflow(val); - return static_cast(val); } - uint16_t xu16() const { check_overflow(val); - return static_cast(val); } - uint32_t xu32() const { check_overflow(val); - return static_cast(val); } - uint64_t xu64() const { check_overflow(val); - return static_cast(val); } - int8_t xi8() const { check_overflow(val); - return static_cast(val); } - int16_t xi16() const { check_overflow(val); - return static_cast(val); } - int32_t xi32() const { check_overflow(val); - return static_cast(val); } - int64_t xi64() const { check_overflow(val); - return static_cast(val); } - float xfloat() const { check_overflow(val); - check_underflow(val); - return static_cast(val); } - double xdouble() const { return val; } - bool operator== (const object_class* x) const { try { return val == x->xdouble(); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { try { return val < x->xdouble(); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - catch (underflow_error&) { - if(val < 0.0) { - if(numeric_overflow::test(val) == -1) { return true; } - try { return static_cast(val) < x->xi64(); } - catch (type_error&) { return true; } - } else { - if(numeric_overflow::test(val) == 1) { return false; } - try { return static_cast(val) < x->xu64(); } - catch (type_error&) { return false; } - } - } } - bool operator> (const object_class* x) const { try { return val > x->xdouble(); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - catch (underflow_error&) { - if(val < 0.0) { - if(numeric_overflow::test(val) == -1) { return false; } - try { return static_cast(val) > x->xi64(); } - catch (type_error&) { return false; } - } else { - if(numeric_overflow::test(val) == 1) { return true; } - try { return static_cast(val) > x->xu64(); } - catch (type_error&) { return true; } - } - } } - const object_class* inspect(std::ostream& s) const - { s << val; return this; } -private: - double val; -}; - - -struct object_raw : object_class { - explicit object_raw(void* p, uint32_t l) : ptr(p), len(l) {} - raw xraw() { return raw(ptr, len); } - const_raw xraw() const { return const_raw(ptr, len); } - bool operator== (const object_class* x) const { try { const_raw xr(x->xraw()); - return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { const_raw xr(x->xraw()); - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } - else { return len < xr.len; } } - bool operator> (const object_class* x) const { const_raw xr(x->xraw()); - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } - else { return len > xr.len; } } - const object_class* inspect(std::ostream& s) const - { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape -private: - void* ptr; - uint32_t len; -}; - -struct object_const_raw : object_class { - explicit object_const_raw(const void* p, uint32_t l) : ptr(p), len(l) {} - const_raw xraw() const { return const_raw(ptr, len); } - bool operator== (const object_class* x) const { try { const_raw xr(x->xraw()); - return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); } - catch (type_error&) { return false; } } - bool operator< (const object_class* x) const { const_raw xr(x->xraw()); - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } - else { return len < xr.len; } } - bool operator> (const object_class* x) const { const_raw xr(x->xraw()); - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } - else { return len > xr.len; } } - const object_class* inspect(std::ostream& s) const - { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape -private: - const void* ptr; - uint32_t len; -}; struct object_array : object_class, object_container_mixin { explicit object_array() {} explicit object_array(uint32_t n) { val.reserve(n); } - array& xarray() { return val; } - const array& xarray() const { return val; } - bool operator== (const object_class* x) const { try { - const std::vector& xa(x->xarray()); - if(val.size() != xa.size()) { return false; } - for(std::vector::const_iterator iv(val.begin()), iv_end(val.end()), ix(xa.begin()); - iv != iv_end; - ++iv, ++ix) { - if(*iv != *ix) { return false; } - } - return true; - } catch (type_error&) { return false; } } - // FIXME operator< operator> - const object_class* inspect(std::ostream& s) const { - s << '['; - if(!val.empty()) { - std::vector::const_iterator it(val.begin()); - s << *it; - ++it; - for(std::vector::const_iterator it_end(val.end()); - it != it_end; - ++it) { - s << ", " << *it; - } - } - s << ']'; - return this; } + array& xarray(); + const array& xarray() const; + bool operator== (const object_class* x) const; + // FIXME operator<, operator> + const object_class* inspect(std::ostream& s) const; public: void push_back(object o) { val.push_back(o); } private: std::vector val; }; -// FIXME hash, operator==: nil, true, false, containerを入れられãªã„ + +// FIXME hash, operator==: nil, true, false, array, mapを入れられãªã„ struct object_map : object_class, object_container_mixin { explicit object_map() {} - map& xmap() { return val; } - const map& xmap() const { return val; } - bool operator== (const object_class* x) const { try { - const std::map& xm(x->xmap()); - if(val.size() != xm.size()) { return false; } - for(std::map::const_iterator iv(val.begin()), iv_end(val.end()), ix(xm.begin()); - iv != iv_end; - ++iv, ++ix) { - if(iv->first != ix->first || iv->second != ix->first) { return false; } - } - return true; - } catch (type_error&) { return false; } } - // FIXME operator< operator> - const object_class* inspect(std::ostream& s) const { - s << '{'; - if(!val.empty()) { - std::map::const_iterator it(val.begin()); - s << it->first << "=>" << it->second; - ++it; - for(std::map::const_iterator it_end(val.end()); - it != it_end; - ++it) { - s << ", " << it->first << "=>" << it->second; - } - } - s << '}'; - return this; } + map& xmap(); + const map& xmap() const; + bool operator== (const object_class* x) const; + // FIXME operator<, operator> + const object_class* inspect(std::ostream& s) const; public: void store(object k, object v) { val[k] = v; } private: From 9b95875d85fb802f6dd0fc1fe0544bef4be1fbc5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0005/1648] lang/c/msgpack: reverted c/bench.c git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@52 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bench.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/c/bench.c b/c/bench.c index 0f316dc..f27350a 100644 --- a/c/bench.c +++ b/c/bench.c @@ -98,9 +98,9 @@ static void pack_append_buffer(void* user, const unsigned char* b, unsigned int static const unsigned int TASK_INT_NUM = 1<<24; -//static const unsigned int TASK_STR_LEN = 1<<15; +static const unsigned int TASK_STR_LEN = 1<<15; //static const unsigned int TASK_INT_NUM = 1<<20; -static const unsigned int TASK_STR_LEN = 1<<12; +//static const unsigned int TASK_STR_LEN = 1<<12; static const char* TASK_STR_PTR; @@ -284,7 +284,6 @@ void bench_msgpack(void) msgpack_unpack_reset(mupk); - /* puts("----"); puts("pack string"); reset_timer(); @@ -317,7 +316,6 @@ void bench_msgpack(void) sec = show_timer(); printf("%f MB/s\n", len / sec / 1024 / 1024); - */ msgpack_unpack_free(mupk); @@ -332,7 +330,7 @@ int main(int argc, char* argv[]) TASK_STR_PTR = str; bench_msgpack(); -// bench_json(); + bench_json(); return 0; } From f41c20a2503411393012d01e3f3c3ea0e9c7dde8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0006/1648] lang/c/msgpack: added C++ binding msgpack::pack git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@53 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/pack_inline.h | 3 +- cpp/Makefile | 2 +- cpp/object.cpp | 77 ++++++++++++++- cpp/object.hpp | 49 ++++++---- cpp/pack.hpp | 192 +++++++++++++++++++++++++++++++++++++ cpp/test.cpp | 161 +++++++++++++++++-------------- msgpack/pack/inline_impl.h | 58 ++++++----- ruby/pack_inline.h | 2 + 8 files changed, 426 insertions(+), 118 deletions(-) create mode 100644 cpp/pack.hpp diff --git a/c/pack_inline.h b/c/pack_inline.h index d943464..b1f9e63 100644 --- a/c/pack_inline.h +++ b/c/pack_inline.h @@ -7,8 +7,9 @@ typedef msgpack_pack_t* msgpack_pack_context; static inline void msgpack_pack_append_buffer(msgpack_pack_t* x, const unsigned char* b, unsigned int l); +#include +#include /* __BYTE_ORDER */ #include "msgpack/pack/inline_impl.h" - #endif /* pack_inline.h */ diff --git a/cpp/Makefile b/cpp/Makefile index 04b421c..5ef7ecd 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -9,7 +9,7 @@ all: test %.hpp: %.hpp.erb erb $< > $@ -test: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o test.o object.hpp unpack.hpp +test: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o test.o object.hpp unpack.hpp pack.hpp $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o test.o -o $@ .PHONY: clean diff --git a/cpp/object.cpp b/cpp/object.cpp index 099b541..02e0438 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -1,4 +1,5 @@ #include "msgpack/object.hpp" +#include "msgpack/pack.hpp" namespace msgpack { @@ -132,24 +133,73 @@ template <> inline void numeric_inspect(int8_t v, std::ostream& s) { s << (int16_t)v; } +template +inline void numeric_pack(dynamic_packer& p, V v); + +template <> +inline void numeric_pack(dynamic_packer& p, uint8_t v) + { p.pack_unsigned_int_8(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, uint16_t v) + { p.pack_unsigned_int_16(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, uint32_t v) + { p.pack_unsigned_int_32(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, uint64_t v) + { p.pack_unsigned_int_64(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, int8_t v) + { p.pack_unsigned_int_8(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, int16_t v) + { p.pack_unsigned_int_16(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, int32_t v) + { p.pack_unsigned_int_32(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, int64_t v) + { p.pack_unsigned_int_64(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, float v) + { p.pack_float(v); } + +template <> +inline void numeric_pack(dynamic_packer& p, double v) + { p.pack_double(v); } + } // noname namespace bool object_nil::isnil() const { return true; } bool object_nil::operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } +void object_nil::pack(dynamic_packer& p) const + { p.pack_nil(); } const object_class* object_nil::inspect(std::ostream& s) const { s << "nil"; return this; } bool object_true::xbool() const { return true; } bool object_true::operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } +void object_true::pack(dynamic_packer& p) const + { p.pack_true(); } const object_class* object_true::inspect(std::ostream& s) const { s << "true"; return this; } bool object_false::xbool() const { return false; } bool object_false::operator== (const object_class* x) const { return typeid(*this) == typeid(*x); } +void object_false::pack(dynamic_packer& p) const + { p.pack_false(); } const object_class* object_false::inspect(std::ostream& s) const { s << "false"; return this; } @@ -176,6 +226,8 @@ bool object_##NAME::operator> (const object_class* x) const \ try { return val > x->x##NAME(); } \ catch (negative_overflow_error&) { return true; } \ catch (overflow_error&) { return false; } \ +void object_##NAME::pack(dynamic_packer& p) const \ + { numeric_pack(p, val); } \ const object_class* object_##NAME::inspect(std::ostream& s) const \ { numeric_inspect(val, s); return this; } \ @@ -236,7 +288,9 @@ bool object_##NAME::operator> (const object_class* x) const { \ catch (type_error&) { return true; } \ } \ } } \ -const object_class* object_##NAME::inspect(std::ostream& s) const \ +void object_##NAME::pack(dynamic_packer& p) const \ + { numeric_pack(p, val); } \ +const object_class* object_##NAME::inspect(std::ostream& s) const \ { s << val; return this; } \ FLOAT_OBJECT(float) @@ -260,6 +314,8 @@ bool object_##NAME::operator> (const object_class* x) const { \ const_raw xr(x->xraw()); \ if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } \ else { return len > xr.len; } } \ +void object_##NAME::pack(dynamic_packer& p) const \ + { p.pack_raw(ptr, len); } \ const object_class* object_##NAME::inspect(std::ostream& s) const \ { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape @@ -303,6 +359,15 @@ const object_class* object_array::inspect(std::ostream& s) const s << ']'; return this; } +void object_array::pack(dynamic_packer& p) const +{ + p.pack_array(val.size()); + for(std::vector::const_iterator it(val.begin()), it_end(val.end()); + it != it_end; + ++it) { + it->pack(p); + } +} map& object_map::xmap() { return val; } @@ -334,6 +399,16 @@ const object_class* object_map::inspect(std::ostream& s) const s << '}'; return this; } +void object_map::pack(dynamic_packer& p) const +{ + p.pack_map(val.size()); + for(std::map::const_iterator it(val.begin()), it_end(val.end()); + it != it_end; + ++it) { + it->first.pack(p); + it->second.pack(p); + } +} } // namespace msgpack diff --git a/cpp/object.hpp b/cpp/object.hpp index 992ac1e..3f22dfc 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -1,6 +1,5 @@ #ifndef MSGPACK_OBJECT_HPP__ #define MSGPACK_OBJECT_HPP__ -#include #include #include @@ -47,6 +46,8 @@ struct object; typedef std::map map; typedef std::vector array; +class dynamic_packer; + struct object_class { virtual ~object_class() {} @@ -72,6 +73,7 @@ struct object_class { bool operator!= (const object_class* x) const { return !(this->operator==(x)); } virtual bool operator< (const object_class* x) const { throw cast_error(); } virtual bool operator> (const object_class* x) const { throw cast_error(); } + virtual void pack(dynamic_packer& p) const = 0; operator bool() const { return xbool(); } // FIXME !isnil(); operator uint8_t() const { return xu8(); } operator uint16_t() const { return xu16(); } @@ -128,6 +130,7 @@ struct object { bool operator!= (object x) const { return val->operator!= (x.val); } bool operator< (object x) const { return val->operator< (x.val); } bool operator> (object x) const { return val->operator> (x.val); } + void pack(dynamic_packer& p) const { val->pack(p); } operator bool() const { return val->operator bool(); } operator uint8_t() const { return val->operator uint8_t(); } operator uint16_t() const { return val->operator uint16_t(); } @@ -160,18 +163,21 @@ inline std::ostream& operator<< (std::ostream& s, const object& o) struct object_nil : object_class { bool isnil() const; bool operator== (const object_class* x) const; + void pack(dynamic_packer& p) const; const object_class* inspect(std::ostream& s) const; }; struct object_true : object_class { bool xbool() const; bool operator== (const object_class* x) const; + void pack(dynamic_packer& p) const; const object_class* inspect(std::ostream& s) const; }; struct object_false : object_class { bool xbool() const; bool operator== (const object_class* x) const; + void pack(dynamic_packer& p) const; const object_class* inspect(std::ostream& s) const; }; @@ -191,6 +197,7 @@ struct object_##NAME : object_class { \ bool operator== (const object_class* x) const; \ bool operator< (const object_class* x) const; \ bool operator> (const object_class* x) const; \ + void pack(dynamic_packer& p) const; \ const object_class* inspect(std::ostream& s) const; \ private: \ TYPE val; \ @@ -209,24 +216,25 @@ INTEGER_CLASS(int64_t, i64) #define FLOAT_CLASS(TYPE, NAME) \ -struct object_##NAME : object_class { \ - object_##NAME(TYPE v) : val(v) {} \ - uint8_t xu8 () const; \ - uint16_t xu16 () const; \ - uint32_t xu32 () const; \ - uint64_t xu64 () const; \ - int8_t xi8 () const; \ - int16_t xi16 () const; \ - int32_t xi32 () const; \ - int64_t xi64 () const; \ - float xfloat () const; \ - double xdouble() const; \ - bool operator== (const object_class* x) const; \ - bool operator< (const object_class* x) const; \ - bool operator> (const object_class* x) const; \ - const object_class* inspect(std::ostream& s) const; \ -private: \ - TYPE val; \ +struct object_##NAME : object_class { \ + object_##NAME(TYPE v) : val(v) {} \ + uint8_t xu8 () const; \ + uint16_t xu16 () const; \ + uint32_t xu32 () const; \ + uint64_t xu64 () const; \ + int8_t xi8 () const; \ + int16_t xi16 () const; \ + int32_t xi32 () const; \ + int64_t xi64 () const; \ + float xfloat () const; \ + double xdouble() const; \ + bool operator== (const object_class* x) const; \ + bool operator< (const object_class* x) const; \ + bool operator> (const object_class* x) const; \ + void pack(dynamic_packer& p) const; \ + const object_class* inspect(std::ostream& s) const; \ +private: \ + TYPE val; \ }; FLOAT_CLASS(float, float) @@ -242,6 +250,7 @@ struct object_##NAME : object_class { \ bool operator== (const object_class* x) const; \ bool operator< (const object_class* x) const; \ bool operator> (const object_class* x) const; \ + void pack(dynamic_packer& p) const; \ const object_class* inspect(std::ostream& s) const; \ private: \ TYPE ptr; \ @@ -261,6 +270,7 @@ struct object_array : object_class, object_container_mixin { const array& xarray() const; bool operator== (const object_class* x) const; // FIXME operator<, operator> + void pack(dynamic_packer& p) const; const object_class* inspect(std::ostream& s) const; public: void push_back(object o) { val.push_back(o); } @@ -276,6 +286,7 @@ struct object_map : object_class, object_container_mixin { const map& xmap() const; bool operator== (const object_class* x) const; // FIXME operator<, operator> + void pack(dynamic_packer& p) const; const object_class* inspect(std::ostream& s) const; public: void store(object k, object v) { val[k] = v; } diff --git a/cpp/pack.hpp b/cpp/pack.hpp new file mode 100644 index 0000000..89f8cee --- /dev/null +++ b/cpp/pack.hpp @@ -0,0 +1,192 @@ +#ifndef MSGPACK_PACK_HPP__ +#define MSGPACK_PACK_HPP__ + +#include "msgpack/object.hpp" +#include "msgpack/zone.hpp" +#include // __BYTE_ORDER +#include + +namespace msgpack { + + +template +class packer { +public: + packer(Stream& s); + +public: + void pack_int(int d) { pack_int_impl(m_stream, d); } + void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } + void pack_unsigned_int_8(uint8_t d) { pack_unsigned_int_8_impl(m_stream, d); } + void pack_unsigned_int_16(uint16_t d) { pack_unsigned_int_16_impl(m_stream, d); } + void pack_unsigned_int_32(uint32_t d) { pack_unsigned_int_32_impl(m_stream, d); } + void pack_unsigned_int_64(uint64_t d) { pack_unsigned_int_64_impl(m_stream, d); } + void pack_signed_int_8(uint8_t d) { pack_signed_int_8_impl(m_stream, d); } + void pack_signed_int_16(uint16_t d) { pack_signed_int_16_impl(m_stream, d); } + void pack_signed_int_32(uint32_t d) { pack_signed_int_32_impl(m_stream, d); } + void pack_signed_int_64(uint64_t d) { pack_signed_int_64_impl(m_stream, d); } + void pack_float(float d) { pack_float_impl(m_stream, d); } + void pack_double(double d) { pack_double_impl(m_stream, d); } + void pack_nil() { pack_nil(m_stream); } + void pack_true() { pack_true(m_stream); } + void pack_false() { pack_false(m_stream); } + void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } + void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + void pack_string(const char* b) { pack_string_impl(m_stream, b); } + void pack_raw(const void* b, size_t l) { pack_raw_impl(m_stream, b, l); } + +private: + static void pack_int_impl(Stream& x, int d); + static void pack_unsigned_int_impl(Stream& x, unsigned int d); + static void pack_unsigned_int_8_impl(Stream& x, uint8_t d); + static void pack_unsigned_int_16_impl(Stream& x, uint16_t d); + static void pack_unsigned_int_32_impl(Stream& x, uint32_t d); + static void pack_unsigned_int_64_impl(Stream& x, uint64_t d); + static void pack_signed_int_8_impl(Stream& x, int8_t d); + static void pack_signed_int_16_impl(Stream& x, int16_t d); + static void pack_signed_int_32_impl(Stream& x, int32_t d); + static void pack_signed_int_64_impl(Stream& x, int64_t d); + static void pack_float_impl(Stream& x, float d); + static void pack_double_impl(Stream& x, double d); + static void pack_nil_impl(Stream& x); + static void pack_true_impl(Stream& x); + static void pack_false_impl(Stream& x); + static void pack_array_impl(Stream& x, unsigned int n); + static void pack_map_impl(Stream& x, unsigned int n); + static void pack_string_impl(Stream& x, const char* b); + static void pack_raw_impl(Stream& x, const void* b, size_t l); + static void append_buffer(Stream& x, const unsigned char* buf, unsigned int len) + { x.append((const char*)buf, len); } + +private: + Stream& m_stream; + +private: + packer(); +}; + +#define msgpack_pack_inline_func(name) \ + template \ + inline void packer::pack_ ## name ## _impl +#define msgpack_pack_context Stream& +#define msgpack_pack_append_buffer append_buffer +#include "msgpack/pack/inline_impl.h" +#undef msgpack_pack_context +#undef msgpack_pack_append_buffer + +template +packer::packer(Stream& s) : m_stream(s) { } + + +class dynamic_stream { +public: + template + dynamic_stream(Stream& s); +public: + dynamic_stream& append(const char* buf, size_t len) + { (*m_function)(m_object, buf, len); return *this; } +private: + void* m_object; + void (*m_function)(void* object, const char* buf, size_t len); +private: + template + static void append_trampoline(void* object, const char* buf, size_t len); +}; + + +class dynamic_packer { +public: + template + dynamic_packer(Stream& s); + +public: + void pack_int(int d) { pack_int_impl(m_stream, d); } + void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } + void pack_unsigned_int_8(uint8_t d) { pack_unsigned_int_8_impl(m_stream, d); } + void pack_unsigned_int_16(uint16_t d) { pack_unsigned_int_16_impl(m_stream, d); } + void pack_unsigned_int_32(uint32_t d) { pack_unsigned_int_32_impl(m_stream, d); } + void pack_unsigned_int_64(uint64_t d) { pack_unsigned_int_64_impl(m_stream, d); } + void pack_signed_int_8(uint8_t d) { pack_signed_int_8_impl(m_stream, d); } + void pack_signed_int_16(uint16_t d) { pack_signed_int_16_impl(m_stream, d); } + void pack_signed_int_32(uint32_t d) { pack_signed_int_32_impl(m_stream, d); } + void pack_signed_int_64(uint64_t d) { pack_signed_int_64_impl(m_stream, d); } + void pack_float(float d) { pack_float_impl(m_stream, d); } + void pack_double(double d) { pack_double_impl(m_stream, d); } + void pack_nil() { pack_nil_impl(m_stream); } + void pack_true() { pack_true_impl(m_stream); } + void pack_false() { pack_false_impl(m_stream); } + void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } + void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + void pack_string(const char* b) { pack_string_impl(m_stream, b); } + void pack_raw(const void* b, size_t l) { pack_raw_impl(m_stream, b, l); } + +public: + +private: + static void pack_int_impl(dynamic_stream& x, int d); + static void pack_unsigned_int_impl(dynamic_stream& x, unsigned int d); + static void pack_unsigned_int_8_impl(dynamic_stream& x, uint8_t d); + static void pack_unsigned_int_16_impl(dynamic_stream& x, uint16_t d); + static void pack_unsigned_int_32_impl(dynamic_stream& x, uint32_t d); + static void pack_unsigned_int_64_impl(dynamic_stream& x, uint64_t d); + static void pack_signed_int_8_impl(dynamic_stream& x, int8_t d); + static void pack_signed_int_16_impl(dynamic_stream& x, int16_t d); + static void pack_signed_int_32_impl(dynamic_stream& x, int32_t d); + static void pack_signed_int_64_impl(dynamic_stream& x, int64_t d); + static void pack_float_impl(dynamic_stream& x, float d); + static void pack_double_impl(dynamic_stream& x, double d); + static void pack_nil_impl(dynamic_stream& x); + static void pack_true_impl(dynamic_stream& x); + static void pack_false_impl(dynamic_stream& x); + static void pack_array_impl(dynamic_stream& x, unsigned int n); + static void pack_map_impl(dynamic_stream& x, unsigned int n); + static void pack_string_impl(dynamic_stream& x, const char* b); + static void pack_raw_impl(dynamic_stream& x, const void* b, size_t l); + static void append_buffer(dynamic_stream& x, const unsigned char* buf, unsigned int len) + { x.append((const char*)buf, len); } + +private: + dynamic_stream m_stream; + +private: + dynamic_packer(); +}; + +#undef MSGPACK_PACK_INLINE_IMPL_H__ +#define msgpack_pack_inline_func(name) \ + inline void dynamic_packer::pack_ ## name ## _impl +#define msgpack_pack_context dynamic_stream& +#define msgpack_pack_append_buffer append_buffer +#include "msgpack/pack/inline_impl.h" +#undef msgpack_pack_context +#undef msgpack_pack_append_buffer + +template +dynamic_packer::dynamic_packer(Stream& s) : m_stream(s) { } + +template +dynamic_stream::dynamic_stream(Stream& s) +{ + m_object = reinterpret_cast(&s); + m_function = &dynamic_stream::append_trampoline; +} + +template +void dynamic_stream::append_trampoline(void* object, const char* buf, size_t len) +{ + (reinterpret_cast(object)->*MemFun)(buf, len); +} + + +template +inline void pack(Stream& s, object o) +{ + dynamic_packer pk(s); + o.pack(pk); +} + + +} // namespace msgpack + +#endif /* msgpack/pack.hpp */ + diff --git a/cpp/test.cpp b/cpp/test.cpp index 423a6bd..12e9150 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -1,12 +1,16 @@ #include +#include #include +#include class checker { public: void check(const char* d, size_t len, msgpack::object should) { + using msgpack::object; try { std::cout << "----" << std::endl; - msgpack::object o; + + object o; try { o = msgpack::unpack(d, len, m_zone); } catch (std::runtime_error& e) { @@ -14,10 +18,25 @@ public: std::cout << "**" << e.what() << "**" << std::endl; return; } + std::cout << o << std::endl; if(o != should) { std::cout << "** TEST FAILED **" << std::endl; } + + try { + std::string s; + msgpack::pack(s, o); + object ro = msgpack::unpack(s.data(), s.size(), m_zone); + if(ro != o) { throw std::runtime_error("NOT MATCH"); } + } catch (std::runtime_error& e) { + std::cout << "** REUNPACK FAILED **" << std::endl; + std::cout << e.what() << std::endl; + } catch (...) { + std::cout << "** REUNPACK FAILED **" << std::endl; + std::cout << "unknown error" << std::endl; + } + } catch (...) { m_zone.clear(); throw; } m_zone.clear(); } @@ -27,84 +46,82 @@ private: int main(void) { + checker c; -checker c; - -{ // SimpleValue - msgpack::zone z; - const char d[] = { - 0x93, 0xc0, 0xc2, 0xc3, - }; - c.check(d, sizeof(d), - z.narray( - z.nnil(), z.nfalse(), z.ntrue() - ) - ); -} - -{ // Fixnum - msgpack::zone z; - const char d[] = { - 0x92, - 0x93, 0x00, 0x40, 0x7f, - 0x93, 0xe0, 0xf0, 0xff, - }; - c.check(d, sizeof(d), - z.narray( + { // SimpleValue + msgpack::zone z; + const char d[] = { + 0x93, 0xc0, 0xc2, 0xc3, + }; + c.check(d, sizeof(d), z.narray( - z.nu8(0), - z.nu8(64), - z.nu8(127) - ), - z.narray( - z.ni8(-32), - z.ni8(-16), - z.ni8(-1) + z.nnil(), z.nfalse(), z.ntrue() ) - ) - ); -} + ); + } -{ // FixArray - msgpack::zone z; - const char d[] = { - 0x92, - 0x90, - 0x91, - 0x91, 0xc0, - }; - c.check(d, sizeof(d), - z.narray( - z.narray(), + { // Fixnum + msgpack::zone z; + const char d[] = { + 0x92, + 0x93, 0x00, 0x40, 0x7f, + 0x93, 0xe0, 0xf0, 0xff, + }; + c.check(d, sizeof(d), z.narray( z.narray( - z.nnil() + z.nu8(0), + z.nu8(64), + z.nu8(127) + ), + z.narray( + z.ni8(-32), + z.ni8(-16), + z.ni8(-1) ) ) - ) - ); -} - -{ // FixRaw - msgpack::zone z; - const char d[] = { - 0x94, - 0xa0, - 0xa1, 'a', - 0xa2, 'b', 'c', - 0xa3, 'd', 'e', 'f', - }; - c.check(d, sizeof(d), - z.narray( - z.nraw("", 0), - z.nraw("a", 1), - z.nraw("bc", 2), - z.nraw("def", 3) - ) - ); -} - - -return 0; + ); + } + + { // FixArray + msgpack::zone z; + const char d[] = { + 0x92, + 0x90, + 0x91, + 0x91, 0xc0, + }; + c.check(d, sizeof(d), + z.narray( + z.narray(), + z.narray( + z.narray( + z.nnil() + ) + ) + ) + ); + } + + { // FixRaw + msgpack::zone z; + const char d[] = { + 0x94, + 0xa0, + 0xa1, 'a', + 0xa2, 'b', 'c', + 0xa3, 'd', 'e', 'f', + }; + c.check(d, sizeof(d), + z.narray( + z.nraw("", 0), + z.nraw("a", 1), + z.nraw("bc", 2), + z.nraw("def", 3) + ) + ); + } + + return 0; } diff --git a/msgpack/pack/inline_impl.h b/msgpack/pack/inline_impl.h index 08a5bc1..d4d5a5a 100644 --- a/msgpack/pack/inline_impl.h +++ b/msgpack/pack/inline_impl.h @@ -18,8 +18,13 @@ #ifndef MSGPACK_PACK_INLINE_IMPL_H__ #define MSGPACK_PACK_INLINE_IMPL_H__ -#include -#include +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif #ifdef __LITTLE_ENDIAN__ @@ -47,13 +52,17 @@ #endif +#ifndef msgpack_pack_inline_func(name) +#define msgpack_pack_inline_func(name) \ + inline void msgpack_pack_##name +#endif /* * Integer */ // wrapper -inline void msgpack_pack_int(msgpack_pack_context x, int d) +msgpack_pack_inline_func(int)(msgpack_pack_context x, int d) { if(d < -32) { if(d < -32768) { // signed 32 @@ -86,11 +95,11 @@ inline void msgpack_pack_int(msgpack_pack_context x, int d) } // wrapper -inline void msgpack_pack_unsigned_int(msgpack_pack_context x, unsigned int d) +msgpack_pack_inline_func(unsigned_int)(msgpack_pack_context x, unsigned int d) { if(d < 128) { // fixnum - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); } else if(d < 256) { // unsigned 8 const unsigned char buf[2] = {0xcc, (uint8_t)d}; @@ -106,7 +115,7 @@ inline void msgpack_pack_unsigned_int(msgpack_pack_context x, unsigned int d) } } -inline void msgpack_pack_unsigned_int_8(msgpack_pack_context x, uint8_t d) +msgpack_pack_inline_func(unsigned_int_8)(msgpack_pack_context x, uint8_t d) { if(d < 128) { msgpack_pack_append_buffer(x, &d, 1); @@ -116,26 +125,26 @@ inline void msgpack_pack_unsigned_int_8(msgpack_pack_context x, uint8_t d) } } -inline void msgpack_pack_unsigned_int_16(msgpack_pack_context x, uint16_t d) +msgpack_pack_inline_func(unsigned_int_16)(msgpack_pack_context x, uint16_t d) { const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -inline void msgpack_pack_unsigned_int_32(msgpack_pack_context x, uint32_t d) +msgpack_pack_inline_func(unsigned_int_32)(msgpack_pack_context x, uint32_t d) { const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -inline void msgpack_pack_unsigned_int_64(msgpack_pack_context x, uint64_t d) +msgpack_pack_inline_func(unsigned_int_64)(msgpack_pack_context x, uint64_t d) { // FIXME optimization const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } -inline void msgpack_pack_signed_int_8(msgpack_pack_context x, int8_t d) +msgpack_pack_inline_func(signed_int_8)(msgpack_pack_context x, int8_t d) { if(d > 0) { msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); @@ -147,19 +156,19 @@ inline void msgpack_pack_signed_int_8(msgpack_pack_context x, int8_t d) } } -inline void msgpack_pack_signed_int_16(msgpack_pack_context x, int16_t d) +msgpack_pack_inline_func(signed_int_16)(msgpack_pack_context x, int16_t d) { const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -inline void msgpack_pack_signed_int_32(msgpack_pack_context x, int32_t d) +msgpack_pack_inline_func(signed_int_32)(msgpack_pack_context x, int32_t d) { const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -inline void msgpack_pack_signed_int_64(msgpack_pack_context x, int64_t d) +msgpack_pack_inline_func(signed_int_64)(msgpack_pack_context x, int64_t d) { // FIXME optimization const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; @@ -171,14 +180,14 @@ inline void msgpack_pack_signed_int_64(msgpack_pack_context x, int64_t d) * Float */ -inline void msgpack_pack_float(msgpack_pack_context x, float d) +msgpack_pack_inline_func(float)(msgpack_pack_context x, float d) { uint32_t n = *((uint32_t*)&d); // FIXME const unsigned char buf[5] = {0xca, STORE_BE32(n)}; msgpack_pack_append_buffer(x, buf, 5); } -inline void msgpack_pack_double(msgpack_pack_context x, double d) +msgpack_pack_inline_func(double)(msgpack_pack_context x, double d) { uint64_t n = *((uint64_t*)&d); // FIXME const unsigned char buf[9] = {0xcb, STORE_BE64(n)}; @@ -190,7 +199,7 @@ inline void msgpack_pack_double(msgpack_pack_context x, double d) * Nil */ -inline void msgpack_pack_nil(msgpack_pack_context x) +msgpack_pack_inline_func(nil)(msgpack_pack_context x) { static const unsigned char d = 0xc0; msgpack_pack_append_buffer(x, &d, 1); @@ -201,13 +210,13 @@ inline void msgpack_pack_nil(msgpack_pack_context x) * Boolean */ -inline void msgpack_pack_true(msgpack_pack_context x) +msgpack_pack_inline_func(true)(msgpack_pack_context x) { static const unsigned char d = 0xc3; msgpack_pack_append_buffer(x, &d, 1); } -inline void msgpack_pack_false(msgpack_pack_context x) +msgpack_pack_inline_func(false)(msgpack_pack_context x) { static const unsigned char d = 0xc2; msgpack_pack_append_buffer(x, &d, 1); @@ -218,7 +227,7 @@ inline void msgpack_pack_false(msgpack_pack_context x) * Array */ -inline void msgpack_pack_array(msgpack_pack_context x, unsigned int n) +msgpack_pack_inline_func(array)(msgpack_pack_context x, unsigned int n) { if(n < 16) { unsigned char d = 0x90 | n; @@ -239,7 +248,7 @@ inline void msgpack_pack_array(msgpack_pack_context x, unsigned int n) * Map */ -inline void msgpack_pack_map(msgpack_pack_context x, unsigned int n) +msgpack_pack_inline_func(map)(msgpack_pack_context x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; @@ -260,13 +269,13 @@ inline void msgpack_pack_map(msgpack_pack_context x, unsigned int n) * Raw */ -inline void msgpack_pack_string(msgpack_pack_context x, const char* b) +msgpack_pack_inline_func(string)(msgpack_pack_context x, const char* b) { uint32_t l = strlen(b); msgpack_pack_append_buffer(x, (const unsigned char*)b, l+1); } -inline void msgpack_pack_raw(msgpack_pack_context x, const void* b, size_t l) +msgpack_pack_inline_func(raw)(msgpack_pack_context x, const void* b, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; @@ -280,14 +289,15 @@ inline void msgpack_pack_raw(msgpack_pack_context x, const void* b, size_t l) unsigned char buf[5] = {0xdb, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } - msgpack_pack_append_buffer(x, b, l); + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); } +#undef msgpack_pack_inline_func(name) + #undef STORE_BE16(d) #undef STORE_BE32(d) #undef STORE_BE64(d) - #endif /* msgpack/pack/inline_impl.h */ diff --git a/ruby/pack_inline.h b/ruby/pack_inline.h index ab4b092..bda74c3 100644 --- a/ruby/pack_inline.h +++ b/ruby/pack_inline.h @@ -27,6 +27,8 @@ static inline void msgpack_pack_append_buffer(VALUE x, const unsigned char* b, u rb_str_buf_cat(x, (const void*)b, l); } +#include +#include /* __BYTE_ORDER */ #include "msgpack/pack/inline_impl.h" #endif /* pack_inline.h */ From 249d3e9c908ea4a3fd2012a753ca7f0cea0b4ee5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0007/1648] lang/c/msgpack: removed string type git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@54 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bench.c | 6 ++---- c/bench_inline.c | 4 +--- c/unpack.h | 3 +-- c/unpack_inline.c | 7 ++----- msgpack/unpack/callback.h | 2 +- msgpack/unpack/inline_context.h | 2 +- msgpack/unpack/inline_impl.h | 36 +++++++++++++++++---------------- ruby/test_format.rb | 6 ------ ruby/unpack_inline.c | 7 ++----- 9 files changed, 29 insertions(+), 44 deletions(-) diff --git a/c/bench.c b/c/bench.c index f27350a..91f0c21 100644 --- a/c/bench.c +++ b/c/bench.c @@ -55,8 +55,7 @@ static void* unpack_array_start(void* data, unsigned int n) { return NULL; } static void unpack_array_item(void* data, void* c, void* o) { } static void* unpack_map_start(void* data, unsigned int n) { return NULL; } static void unpack_map_item(void* data, void* c, void* k, void* v) { } -static void* unpack_string(void* data, const void* b, size_t l) { return NULL; } -static void* unpack_raw(void* data, const void* b, size_t l) { /*printf("unpack raw %p %lu\n",b,l);*/ return NULL; } +static void* unpack_raw(void* data, const void* b, const void* p, size_t l) { /*printf("unpack raw %p %lu\n",p,l);*/ return NULL; } typedef struct { size_t allocated; @@ -119,7 +118,7 @@ void bench_json(void) NULL, NULL, reformat_number, - reformat_string, + reformat_string, reformat_start_map, reformat_map_key, reformat_end_map, @@ -236,7 +235,6 @@ void bench_msgpack(void) unpack_array_item, unpack_map_start, unpack_map_item, - unpack_string, unpack_raw, }; msgpack_unpack_t* mupk = msgpack_unpack_new(NULL, &cb); diff --git a/c/bench_inline.c b/c/bench_inline.c index 2901508..5390319 100644 --- a/c/bench_inline.c +++ b/c/bench_inline.c @@ -65,8 +65,7 @@ static inline void* msgpack_unpack_array_start(msgpack_unpack_context* x, unsign static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, void* c, void* o) { } static inline void* msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) { return NULL; } static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, void* c, void* k, void* v) { } -static inline void* msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l) { return NULL; } -static inline void* msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l) { return NULL; } +static inline void* msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l) { return NULL; } #include "msgpack/unpack/inline_impl.h" @@ -131,7 +130,6 @@ void bench_json(void) NULL, NULL, reformat_number, - reformat_string, reformat_start_map, reformat_map_key, reformat_end_map, diff --git a/c/unpack.h b/c/unpack.h index 3230a0d..094328e 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -39,8 +39,7 @@ typedef struct { void (*unpack_array_item)(void* data, void* c, void* o); void* (*unpack_map_start)(void* data, unsigned int n); void (*unpack_map_item)(void* data, void* c, void* k, void* v); - void* (*unpack_string)(void* data, const void* b, size_t l); - void* (*unpack_raw)(void* data, const void* b, size_t l); + void* (*unpack_raw)(void* data, const void* b, const void* p, size_t l); } msgpack_unpack_callback; typedef struct { diff --git a/c/unpack_inline.c b/c/unpack_inline.c index 3525468..d758d3e 100644 --- a/c/unpack_inline.c +++ b/c/unpack_inline.c @@ -71,11 +71,8 @@ static inline void* msgpack_unpack_map_start(msgpack_unpack_t* x, unsigned int n static inline void msgpack_unpack_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) { x->callback.unpack_map_item(x->data, c, k, v); } -static inline void* msgpack_unpack_string(msgpack_unpack_t* x, const void* b, size_t l) -{ return x->callback.unpack_string(x->data, b, l); } - -static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const void* b, size_t l) -{ return x->callback.unpack_raw(x->data, b, l); } +static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const void* b, const void* p, size_t l) +{ return x->callback.unpack_raw(x->data, b, p, l); } #include "msgpack/unpack/inline_impl.h" diff --git a/msgpack/unpack/callback.h b/msgpack/unpack/callback.h index b058a15..51508cc 100644 --- a/msgpack/unpack/callback.h +++ b/msgpack/unpack/callback.h @@ -20,5 +20,5 @@ msgpack_object msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned in msgpack_object msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n); void msgpack_unpack_map_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object k, msgpack_object v); msgpack_object msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l); -msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l); +msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l); diff --git a/msgpack/unpack/inline_context.h b/msgpack/unpack/inline_context.h index d6558a3..22698d5 100644 --- a/msgpack/unpack/inline_context.h +++ b/msgpack/unpack/inline_context.h @@ -34,7 +34,7 @@ typedef struct { size_t count; unsigned int ct; union { - const unsigned char* terminal_trail_start; + /*const unsigned char* terminal_trail_start;*/ msgpack_object map_key; } tmp; } msgpack_unpacker_stack; diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h index f610dd7..fdbe7f6 100644 --- a/msgpack/unpack/inline_impl.h +++ b/msgpack/unpack/inline_impl.h @@ -103,7 +103,7 @@ static inline uint64_t betoh64(uint64_t x) { typedef enum { CS_HEADER = 0x00, // nil - CS_STRING = 0x01, + //CS_STRING = 0x01, //CS_ = 0x02, // false //CS_ = 0x03, // true @@ -186,15 +186,17 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len obj = func(user, arg); \ /*printf("obj %d\n",obj);*/ \ goto _push -#define push_variable_value(func, arg, arglen) \ - obj = func(user, arg, arglen); \ +#define push_variable_value(func, base, pos, len) \ + obj = func(user, (const void*)base, (const void*)pos, len); \ /*printf("obj %d\n",obj);*/ \ goto _push +/* #define again_terminal_trail(_cs, from) \ cs = _cs; \ stack[top].tmp.terminal_trail_start = from; \ goto _terminal_trail_again +*/ #define again_fixed_trail(_cs, trail_len) \ trail = trail_len; \ cs = _cs; \ @@ -237,8 +239,8 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len switch(*p) { case 0xc0: // nil push_simple_value(msgpack_unpack_nil); - case 0xc1: // string - again_terminal_trail(NEXT_CS(p), p+1); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); case 0xc2: // false push_simple_value(msgpack_unpack_false); case 0xc3: // true @@ -289,16 +291,16 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len // end CS_HEADER - _terminal_trail_again: - ++p; + //_terminal_trail_again: + // ++p; - case CS_STRING: - if(*p == 0) { - const unsigned char* start = stack[top].tmp.terminal_trail_start; - obj = msgpack_unpack_string(user, start, p-start); - goto _push; - } - goto _terminal_trail_again; + //case CS_STRING: + // if(*p == 0) { + // const unsigned char* start = stack[top].tmp.terminal_trail_start; + // obj = msgpack_unpack_string(user, start, p-start); + // goto _push; + // } + // goto _terminal_trail_again; _fixed_trail_again: @@ -343,7 +345,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len //case ACS_BIG_INT_VALUE: //_big_int_zero: // // FIXME - // push_variable_value(msgpack_unpack_big_int, n, trail); + // push_variable_value(msgpack_unpack_big_int, data, n, trail); //case CS_BIG_FLOAT_16: // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); @@ -352,7 +354,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len //case ACS_BIG_FLOAT_VALUE: //_big_float_zero: // // FIXME - // push_variable_value(msgpack_unpack_big_float, n, trail); + // push_variable_value(msgpack_unpack_big_float, data, n, trail); case CS_RAW_16: again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); @@ -360,7 +362,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); case ACS_RAW_VALUE: _raw_zero: - push_variable_value(msgpack_unpack_raw, n, trail); + push_variable_value(msgpack_unpack_raw, data, n, trail); case CS_ARRAY_16: start_container(msgpack_unpack_array_start, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); diff --git a/ruby/test_format.rb b/ruby/test_format.rb index 7c2e8fc..99a27d1 100644 --- a/ruby/test_format.rb +++ b/ruby/test_format.rb @@ -119,10 +119,4 @@ check([ 0xdf, 0x00, 0x00, 0x00, 0x02, 0xc0, 0xc2, 0xc3, 0xc2, ], [{}, {nil=>false}, {true=>false, nil=>false}, {}, {nil=>false}, {true=>false, nil=>false}]) -# string -check([ - 0x92, - 0xc1, 0x00, - 0xc1, ?a, ?b, ?c, 0x00, -], ["", "abc"]) diff --git a/ruby/unpack_inline.c b/ruby/unpack_inline.c index f6715d9..b30754b 100644 --- a/ruby/unpack_inline.c +++ b/ruby/unpack_inline.c @@ -71,11 +71,8 @@ static inline VALUE msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) { rb_hash_aset(c, k, v); } -static inline VALUE msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l) -{ return rb_str_new(b, l); } - -static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, size_t l) -{ return rb_str_new(b, l); } +static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l) +{ return rb_str_new(p, l); } #include "msgpack/unpack/inline_impl.h" From 990ac38ccdb51acc520fa43c99115cb216ec95e6 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0008/1648] lang/c/msgpack: C++ binding: implemented built-in buffer. git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@55 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile | 3 +- cpp/object.cpp | 10 ++-- cpp/object.hpp | 9 ++-- cpp/test.cpp | 81 +++++++++++++++++++++++++++-- cpp/unpack.cpp | 115 ++++++++++++++++++++++++++++++++++-------- cpp/unpack.hpp | 48 +++++++++++++++--- cpp/unpack_inline.cpp | 7 +-- cpp/zone.cpp | 29 +++++++---- cpp/zone.hpp.erb | 63 +++++++++++++++++++---- 9 files changed, 299 insertions(+), 66 deletions(-) diff --git a/cpp/Makefile b/cpp/Makefile index 5ef7ecd..7cc66b2 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -1,5 +1,6 @@ -CXXFLAGS = -I.. -I. -Wall -g -O4 +CXXFLAGS = -I.. -I. -Wall -g +#CXXFLAGS = -I.. -I. -Wall -g -O4 LDFLAGS = -L. NEED_PREPROCESS = zone.hpp diff --git a/cpp/object.cpp b/cpp/object.cpp index 02e0438..947e8dd 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -320,12 +320,12 @@ const object_class* object_##NAME::inspect(std::ostream& s) const \ { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape -RAW_OBJECT(raw, - raw object_raw::xraw() { return raw(ptr, len); } - const_raw object_raw::xraw() const { return const_raw(ptr, len); } ) +RAW_OBJECT(raw_ref, + raw object_raw_ref::xraw() { return raw(ptr, len); } + const_raw object_raw_ref::xraw() const { return const_raw(ptr, len); } ) -RAW_OBJECT(const_raw, - const_raw object_const_raw::xraw() const { return const_raw(ptr, len); } ) +RAW_OBJECT(const_raw_ref, + const_raw object_const_raw_ref::xraw() const { return const_raw(ptr, len); } ) #undef RAW_OBJECT(NAME, EXTRA) diff --git a/cpp/object.hpp b/cpp/object.hpp index 3f22dfc..f029e32 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -31,8 +31,9 @@ public: }; struct const_raw { - const_raw() : ptr(NULL), len(0) {} - const_raw(const void* p, size_t l) : ptr(p), len(l) {} + explicit const_raw() : ptr(NULL), len(0) {} + explicit const_raw(const void* p, size_t l) : ptr(p), len(l) {} + const_raw(const raw& m) : ptr(m.ptr), len(m.len) {} public: const void* ptr; size_t len; @@ -257,8 +258,8 @@ private: \ uint32_t len; \ }; -RAW_CLASS(raw, void*, raw xraw(); const_raw xraw() const; ) -RAW_CLASS(const_raw, const void*, const_raw xraw() const; ) +RAW_CLASS(raw_ref, void*, raw xraw(); const_raw xraw() const; ) +RAW_CLASS(const_raw_ref, const void*, const_raw xraw() const; ) #undef RAW_CLASS(NAME, TYPE, EXTRA) diff --git a/cpp/test.cpp b/cpp/test.cpp index 12e9150..3756a05 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include class checker { public: @@ -114,14 +116,85 @@ int main(void) }; c.check(d, sizeof(d), z.narray( - z.nraw("", 0), - z.nraw("a", 1), - z.nraw("bc", 2), - z.nraw("def", 3) + z.nraw_ref("", 0), + z.nraw_ref("a", 1), + z.nraw_ref("bc", 2), + z.nraw_ref("def", 3) ) ); } + static const uint16_t TASK_ARRAY = 100; + static char tarray[3]; + static char traw[64]; + + { + memset(traw, 'a', sizeof(traw)); + traw[0] = 0xda; + uint16_t n = htons(sizeof(traw)-3); + traw[1] = ((char*)&n)[0]; + traw[2] = ((char*)&n)[1]; + + msgpack::zone z; + std::cout << msgpack::unpack(traw, sizeof(traw), z) << std::endl;; + } + + { + tarray[0] = 0xdc; + uint16_t n = htons(TASK_ARRAY); + tarray[1] = ((char*)&n)[0]; + tarray[2] = ((char*)&n)[1]; + } + + { + // write message + ssize_t total_bytes = 0; + std::stringstream stream; + for(unsigned q=0; q < 10; ++q) { + stream.write(tarray, sizeof(tarray)); + total_bytes += sizeof(tarray); + for(uint16_t i=0; i < TASK_ARRAY; ++i) { + stream.write(traw, sizeof(traw)); + total_bytes += sizeof(traw); + } + } + + stream.seekg(0); + + // reserive message + unsigned num_msg = 0; + + static const size_t RESERVE_SIZE = 32;//*1024; + + msgpack::unpacker upk; + while(stream.good() && total_bytes > 0) { + + upk.reserve_buffer(RESERVE_SIZE); + size_t sz = stream.readsome( + (char*)upk.buffer(), + upk.buffer_capacity()); + + total_bytes -= sz; + std::cout << "read " << sz << " bytes to capacity " + << upk.buffer_capacity() << " bytes" + << std::endl; + + upk.buffer_consumed(sz); + while( upk.execute() ) { + std::cout << "message parsed" << std::endl; + boost::scoped_ptr pz(upk.release_zone()); + msgpack::object o = upk.data(); + upk.reset(); + std::cout << o << std::endl; + ++num_msg; + } + + } + + std::cout << "stream finished" << std::endl; + std::cout << num_msg << " messages reached" << std::endl; + } + return 0; } diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 0f02d3c..5055008 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -5,10 +5,10 @@ namespace msgpack { struct unpacker::context { - context(zone& z) + context(zone* z) { msgpack_unpacker_init(&m_ctx); - m_ctx.user = &z; + m_ctx.user = z; } ~context() { } @@ -30,6 +30,22 @@ struct unpacker::context { m_ctx.user = z; } + void reset(zone* z) + { + msgpack_unpacker_init(&m_ctx); + m_ctx.user = z; + } + + zone* user() + { + return m_ctx.user; + } + + void user(zone* z) + { + m_ctx.user = z; + } + private: msgpack_unpacker m_ctx; @@ -39,46 +55,105 @@ private: }; -unpacker::unpacker(zone& z) : - m_ctx(new context(z)), - m_zone(z), - m_finished(false) +unpacker::unpacker() : + m_zone(new zone()), + m_ctx(new context(m_zone)), + m_buffer(NULL), + m_used(0), + m_free(0), + m_off(0) { } -unpacker::~unpacker() { delete m_ctx; } - - -size_t unpacker::execute(const void* data, size_t len, size_t off) +unpacker::~unpacker() { - int ret = m_ctx->execute(data, len, &off); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret > 0) { - m_finished = true; - return off; + free(m_buffer); + delete m_ctx; + delete m_zone; +} + + +void unpacker::expand_buffer(size_t len) +{ + if(m_off == 0) { + size_t next_size; + if(m_free != 0) { next_size = m_free * 2; } + else { next_size = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; } + while(next_size < len + m_used) { next_size *= 2; } + + // FIXME realloc? + + void* tmp = malloc(next_size); + if(!tmp) { throw std::bad_alloc(); } + memcpy(tmp, m_buffer, m_used); + + free(m_buffer); + m_buffer = tmp; + m_free = next_size - m_used; + } else { - m_finished = false; - return off; + size_t next_size = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; + while(next_size < len + m_used - m_off) { next_size *= 2; } + + void* tmp = malloc(next_size); + if(!tmp) { throw std::bad_alloc(); } + memcpy(tmp, ((char*)m_buffer)+m_off, m_used-m_off); + + try { + m_zone->push_finalizer(&zone::finalize_free, NULL, m_buffer); + } catch (...) { + free(tmp); + throw; + } + + m_buffer = tmp; + m_used = m_used - m_off; + m_free = next_size - m_used; + m_off = 0; } } +bool unpacker::execute() +{ + int ret = m_ctx->execute(m_buffer, m_used, &m_off); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + return false; + } else { + return true; + } +} + +zone* unpacker::release_zone() +{ + zone* z = m_zone; + m_zone = NULL; + m_zone = new zone(); + m_ctx->user(m_zone); + return z; +} object unpacker::data() { return object(m_ctx->data()); } - void unpacker::reset() { + if(!m_zone->empty()) { + delete m_zone; + m_zone = NULL; + m_zone = new zone(); + } + expand_buffer(0); m_ctx->reset(); } object unpacker::unpack(const void* data, size_t len, zone& z) { - context ctx(z); + context ctx(&z); size_t off = 0; int ret = ctx.execute(data, len, &off); if(ret < 0) { diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 61ba781..df4a636 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -5,6 +5,10 @@ #include "msgpack/zone.hpp" #include +#ifndef MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE +#define MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE 8*1024 +#endif + namespace msgpack { @@ -16,26 +20,58 @@ struct unpack_error : public std::runtime_error { class unpacker { public: - unpacker(zone& z); + unpacker(); ~unpacker(); + public: - size_t execute(const void* data, size_t len, size_t off); - bool is_finished() { return m_finished; } + void reserve_buffer(size_t len); + void* buffer(); + size_t buffer_capacity() const; + void buffer_consumed(size_t len); + bool execute(); + zone* release_zone(); // never throw object data(); void reset(); + private: + zone* m_zone; + struct context; context* m_ctx; - zone& m_zone; - bool m_finished; + + void* m_buffer; + size_t m_used; + size_t m_free; + size_t m_off; + void expand_buffer(size_t len); + private: - unpacker(); unpacker(const unpacker&); + public: static object unpack(const void* data, size_t len, zone& z); }; +inline void unpacker::reserve_buffer(size_t len) +{ + if(m_free >= len) { return; } + expand_buffer(len); +} + +inline void* unpacker::buffer() + { return (void*)(((char*)m_buffer)+m_used); } + +inline size_t unpacker::buffer_capacity() const + { return m_free; } + +inline void unpacker::buffer_consumed(size_t len) +{ + m_used += len; + m_free -= len; +} + + inline object unpack(const void* data, size_t len, zone& z) { return unpacker::unpack(data, len, z); diff --git a/cpp/unpack_inline.cpp b/cpp/unpack_inline.cpp index 82f6e7a..9002327 100644 --- a/cpp/unpack_inline.cpp +++ b/cpp/unpack_inline.cpp @@ -59,11 +59,8 @@ static inline object_class* msgpack_unpack_map_start(zone** z, unsigned int n) static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) { reinterpret_cast(c)->store(k, v); } -static inline object_class* msgpack_unpack_string(zone** z, const void* b, size_t l) -{ return (*z)->nraw(b, l); } - -static inline object_class* msgpack_unpack_raw(zone** z, const void* b, size_t l) -{ return (*z)->nraw(b, l); } +static inline object_class* msgpack_unpack_raw(zone** z, const void* b, const void* p, size_t l) +{ return (*z)->nraw_ref(p, l); } } // extern "C" diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 5031467..de2de22 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -6,27 +6,36 @@ namespace msgpack { void* zone::alloc() { if(m_used >= m_pool.size()*MSGPACK_ZONE_CHUNK_SIZE) { - m_pool.push_back(chunk_t()); + m_pool.push_back(new chunk_t()); } - void* data = m_pool[m_used/MSGPACK_ZONE_CHUNK_SIZE].cells[m_used%MSGPACK_ZONE_CHUNK_SIZE].data; + void* data = m_pool[m_used/MSGPACK_ZONE_CHUNK_SIZE]->cells[m_used%MSGPACK_ZONE_CHUNK_SIZE].data; ++m_used; return data; } void zone::clear() { - for(size_t b=0; b < m_used/MSGPACK_ZONE_CHUNK_SIZE; ++b) { - cell_t* c(m_pool[b].cells); - for(size_t e=0; e < MSGPACK_ZONE_CHUNK_SIZE; ++e) { + if(!m_pool.empty()) { + for(size_t b=0; b < m_used/MSGPACK_ZONE_CHUNK_SIZE; ++b) { + cell_t* c(m_pool[b]->cells); + for(size_t e=0; e < MSGPACK_ZONE_CHUNK_SIZE; ++e) { + reinterpret_cast(c[e].data)->~object_class(); + } + } + cell_t* c(m_pool.back()->cells); + for(size_t e=0; e < m_used%MSGPACK_ZONE_CHUNK_SIZE; ++e) { reinterpret_cast(c[e].data)->~object_class(); } - } - cell_t* c(m_pool.back().cells); - for(size_t e=0; e < m_used%MSGPACK_ZONE_CHUNK_SIZE; ++e) { - reinterpret_cast(c[e].data)->~object_class(); + + for(pool_t::iterator it(m_pool.begin()), it_end(m_pool.end()); + it != it_end; + ++it) { + delete *it; + } + m_pool.clear(); } m_used = 0; - m_pool.resize(1); + for(user_finalizer_t::reverse_iterator it(m_user_finalizer.rbegin()), it_end(m_user_finalizer.rend()); it != it_end; ++it) { diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 1a941af..40ce694 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -1,8 +1,11 @@ #ifndef MSGPACK_ZONE_HPP__ #define MSGPACK_ZONE_HPP__ +#include #include "msgpack/object.hpp" -#include +#include +#include +#include #ifndef MSGPACK_ZONE_CHUNK_SIZE #define MSGPACK_ZONE_CHUNK_SIZE 64 @@ -13,8 +16,8 @@ namespace msgpack { class zone { public: -zone() : m_used(0), m_pool(1) { } -~zone() { clear(); } + zone() : m_used(0) { } + ~zone() { clear(); } public: template @@ -35,11 +38,27 @@ public: object_float* nfloat( float v) { return new (alloc()) object_float(v); } object_double* ndouble( double v) { return new (alloc()) object_double(v); } - object_raw* nraw(void* ptr, uint32_t len) - { return new (alloc()) object_raw(ptr, len); } + object_raw_ref* nraw_ref(void* ptr, uint32_t len) + { return new (alloc()) object_raw_ref(ptr, len); } - object_const_raw* nraw(const void* ptr, uint32_t len) - { return new (alloc()) object_const_raw(ptr, len); } + object_const_raw_ref* nraw_ref(const void* ptr, uint32_t len) + { return new (alloc()) object_const_raw_ref(ptr, len); } + + object_raw_ref* nraw_copy(const void* ptr, uint32_t len) + { + void* copy = malloc(len); + if(!copy) { throw std::bad_alloc(); } + object_raw_ref* o; + try { + o = new (alloc()) object_raw_ref(copy, len); + push_finalizer(&zone::finalize_free, NULL, copy); + } catch (...) { + free(copy); + throw; + } + memcpy(copy, ptr, len); + return o; + } object_array* narray() { return new (alloc()) object_array(); } @@ -67,6 +86,7 @@ public: public: void clear(); + bool empty() const; private: void* alloc(); @@ -75,9 +95,9 @@ private: size_t m_used; static const size_t MAX_OBJECT_SIZE = - sizeof(object_raw) > sizeof(object_array) - ? ( sizeof(object_raw) > sizeof(object_map) - ? sizeof(object_raw) + sizeof(object_raw_ref) > sizeof(object_array) + ? ( sizeof(object_raw_ref) > sizeof(object_map) + ? sizeof(object_raw_ref) : sizeof(object_map) ) : ( sizeof(object_array) > sizeof(object_map) @@ -94,7 +114,7 @@ private: cell_t cells[MSGPACK_ZONE_CHUNK_SIZE]; }; - typedef std::vector pool_t; + typedef std::vector pool_t; pool_t m_pool; @@ -112,11 +132,32 @@ private: typedef std::vector user_finalizer_t; user_finalizer_t m_user_finalizer; +private: + void resize_pool(size_t n); + +public: + static void finalize_free(void* obj, void* user) + { free(user); } + private: zone(const zone&); }; +template +inline void zone::push_finalizer(void (*func)(void* obj, void* user), T* obj, void* user) +{ + m_user_finalizer.push_back( finalizer( + func, reinterpret_cast(obj), + user) ); +} + +inline bool zone::empty() const +{ + return m_used == 0 && m_user_finalizer.empty(); +} + + } // namespace msgpack #endif /* msgpack/zone.hpp */ From 7e6b55a71805dabb2ad82bf6802ea81f36bc911a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0009/1648] lang/c/msgpack: C++ binding: support non-MessagePack message that follows after MessagePack message git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@56 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/test.cpp | 17 +++++++++++++++-- cpp/unpack.cpp | 8 ++++---- cpp/unpack.hpp | 46 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/cpp/test.cpp b/cpp/test.cpp index 3756a05..e236d0f 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -169,7 +169,10 @@ int main(void) msgpack::unpacker upk; while(stream.good() && total_bytes > 0) { + // 1. reserve buffer upk.reserve_buffer(RESERVE_SIZE); + + // 2. read data to buffer() up to buffer_capacity() bytes size_t sz = stream.readsome( (char*)upk.buffer(), upk.buffer_capacity()); @@ -179,14 +182,24 @@ int main(void) << upk.buffer_capacity() << " bytes" << std::endl; + // 3. specify the number of bytes actually copied upk.buffer_consumed(sz); + + // 4. repeat execute() until it returns false while( upk.execute() ) { std::cout << "message parsed" << std::endl; - boost::scoped_ptr pz(upk.release_zone()); + + // 5.1. take out the parsed object msgpack::object o = upk.data(); - upk.reset(); + + // 5.2. the parsed object is valid until the zone is deleted + boost::scoped_ptr pz(upk.release_zone()); + std::cout << o << std::endl; ++num_msg; + + // 5.3 re-initialize unpacker + upk.reset(); } } diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 5055008..d484593 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -82,7 +82,6 @@ void unpacker::expand_buffer(size_t len) while(next_size < len + m_used) { next_size *= 2; } // FIXME realloc? - void* tmp = malloc(next_size); if(!tmp) { throw std::bad_alloc(); } memcpy(tmp, m_buffer, m_used); @@ -121,15 +120,16 @@ bool unpacker::execute() } else if(ret == 0) { return false; } else { + expand_buffer(0); return true; } } zone* unpacker::release_zone() { + zone* nz = new zone(); zone* z = m_zone; - m_zone = NULL; - m_zone = new zone(); + m_zone = nz; m_ctx->user(m_zone); return z; } @@ -141,12 +141,12 @@ object unpacker::data() void unpacker::reset() { + if(m_off != 0) { expand_buffer(0); } if(!m_zone->empty()) { delete m_zone; m_zone = NULL; m_zone = new zone(); } - expand_buffer(0); m_ctx->reset(); } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index df4a636..ae536c9 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -24,15 +24,46 @@ public: ~unpacker(); public: + /*! 1. reserve buffer. at least `len' bytes of capacity will be ready */ void reserve_buffer(size_t len); + + /*! 2. read data to the buffer() up to buffer_capacity() bytes */ void* buffer(); size_t buffer_capacity() const; + + /*! 3. specify the number of bytes actually copied */ void buffer_consumed(size_t len); + + /*! 4. repeat execute() until it retunrs false */ bool execute(); - zone* release_zone(); // never throw + + /*! 5.1. if execute() returns true, take out the parsed object */ object data(); + + /*! 5.2. the parsed object is valid until the zone is deleted */ + // Note that once release_zone() from unpacker, you must delete it + // otherwise the memrory will leak. + zone* release_zone(); + + /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); +public: + // These functions are usable when non-MessagePack message follows after + // MessagePack message. + // Note that there are no parsed buffer when execute() returned true. + + /*! get address of buffer that is not parsed */ + void* nonparsed_buffer(); + size_t nonparsed_size() const; + + /*! get the number of bytes that is already parsed */ + size_t parsed_size() const; + + /*! remove unparsed buffer from unpacker */ + // Note that reset() leaves non-parsed buffer. + void remove_nonparsed_buffer(); + private: zone* m_zone; @@ -72,6 +103,19 @@ inline void unpacker::buffer_consumed(size_t len) } +inline void* unpacker::nonparsed_buffer() + { return (void*)(((char*)m_buffer)+m_off); } + +inline size_t unpacker::nonparsed_size() const + { return m_used - m_off; } + +inline size_t unpacker::parsed_size() const + { return m_off; } + +inline void unpacker::remove_nonparsed_buffer() + { m_used = m_off; } + + inline object unpack(const void* data, size_t len, zone& z) { return unpacker::unpack(data, len, z); From 94cbe54cf377f16bcc18b80d8b6f81eefd50e723 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0010/1648] lang/c/msgpack: C++ binding: added bench.cpp git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@57 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile | 15 ++-- cpp/bench.cpp | 181 +++++++++++++++++++++++++++++++++++++++++++++++++ cpp/unpack.cpp | 2 +- 3 files changed, 191 insertions(+), 7 deletions(-) create mode 100644 cpp/bench.cpp diff --git a/cpp/Makefile b/cpp/Makefile index 7cc66b2..4a3677e 100644 --- a/cpp/Makefile +++ b/cpp/Makefile @@ -1,11 +1,11 @@ -CXXFLAGS = -I.. -I. -Wall -g -#CXXFLAGS = -I.. -I. -Wall -g -O4 -LDFLAGS = -L. +#CXXFLAGS = -I.. -I. -Wall -g +CXXFLAGS = -I.. -I. -Wall -g -O4 +LDFLAGS = -L. $(CXXFLAGS) NEED_PREPROCESS = zone.hpp -all: test +all: test bench %.hpp: %.hpp.erb erb $< > $@ @@ -13,10 +13,13 @@ all: test test: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o test.o object.hpp unpack.hpp pack.hpp $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o test.o -o $@ +bench: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o bench.o object.hpp unpack.hpp pack.hpp + $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o bench.o -o $@ + .PHONY: clean clean: $(RM) unpack.o unpack_inline.o object.o zone.o - $(RM) test.o - $(RM) test + $(RM) test.o test + $(RM) bench.o bench $(RM) $(NEED_PREPROCESS) diff --git a/cpp/bench.cpp b/cpp/bench.cpp new file mode 100644 index 0000000..1a9f6f9 --- /dev/null +++ b/cpp/bench.cpp @@ -0,0 +1,181 @@ +#include +#include +#include +#include +#include +#include +#include + +static const unsigned int TASK_INT_NUM = 1<<24; +static const unsigned int TASK_STR_LEN = 1<<15; +//static const unsigned int TASK_INT_NUM = 1<<23; +//static const unsigned int TASK_STR_LEN = 1<<14; +static const void* TASK_STR_PTR; + + +class simple_timer { +public: + void reset() { gettimeofday(&m_timeval, NULL); } + void show_stat(size_t bufsz) + { + struct timeval endtime; + gettimeofday(&endtime, NULL); + double sec = (endtime.tv_sec - m_timeval.tv_sec) + + (double)(endtime.tv_usec - m_timeval.tv_usec) / 1000 / 1000; + std::cout << sec << " sec" << std::endl; + std::cout << (double(bufsz)/1024/1024) << " MB" << std::endl; + std::cout << (bufsz/sec/1024/1024*8) << " Mbps" << std::endl; + } +private: + timeval m_timeval; +}; + + +class simple_buffer { +public: + static const size_t DEFAULT_INITIAL_SIZE = 32*1024;//512*1024*1024*2; + + simple_buffer(size_t initial_size = DEFAULT_INITIAL_SIZE) : + m_storage((char*)malloc(initial_size)), + m_allocated(initial_size), + m_used(0) + { + if(!m_storage) { throw std::bad_alloc(); } + } + + ~simple_buffer() + { + free(m_storage); + } + +public: + inline simple_buffer& append(const char* buf, size_t len) + { + if(m_allocated - m_used < len) { + expand_buffer(len); + } + memcpy(m_storage + m_used, buf, len); + m_used += len; + return *this; + } + + void clear() + { + m_used = 0; + } + +private: + void expand_buffer(size_t req) + { + size_t nsize = m_allocated * 2; + size_t at_least = m_used + req; + while(nsize < at_least) { nsize *= 2; } + char* tmp = (char*)realloc(m_storage, nsize); + if(!tmp) { throw std::bad_alloc(); } + m_storage = tmp; + m_allocated = nsize; + } + +public: + size_t size() const { return m_used; } + const char* data() const { return m_storage; } + +private: + char* m_storage; + size_t m_allocated; + size_t m_used; +}; + + +void bench_msgpack_int() +{ + simple_buffer buf; + simple_timer timer; + + std::cout << "----" << std::endl; + std::cout << "pack integer" << std::endl; + + timer.reset(); + { + msgpack::packer pk(buf); + pk.pack_array(TASK_INT_NUM); + for(unsigned int i=0; i < TASK_INT_NUM; ++i) { + pk.pack_unsigned_int(i); + } + } + timer.show_stat(buf.size()); + + std::cout << "----" << std::endl; + std::cout << "unpack integer" << std::endl; + + msgpack::zone z; + msgpack::object obj; + + timer.reset(); + { + obj = msgpack::unpack(buf.data(), buf.size(), z); + } + timer.show_stat(buf.size()); + + std::cout << "----" << std::endl; + std::cout << "dynamic pack integer" << std::endl; + + buf.clear(); + + timer.reset(); + msgpack::pack(buf, obj); + timer.show_stat(buf.size()); +} + +void bench_msgpack_str() +{ + simple_buffer buf; + simple_timer timer; + + std::cout << "----" << std::endl; + std::cout << "pack string" << std::endl; + + timer.reset(); + { + msgpack::packer pk(buf); + pk.pack_array(TASK_STR_LEN); + for(unsigned int i=0; i < TASK_STR_LEN; ++i) { + pk.pack_raw(TASK_STR_PTR, i); + } + } + timer.show_stat(buf.size()); + + std::cout << "----" << std::endl; + std::cout << "unpack string" << std::endl; + + msgpack::zone z; + msgpack::object obj; + + timer.reset(); + { + obj = msgpack::unpack(buf.data(), buf.size(), z); + } + timer.show_stat(buf.size()); + + std::cout << "----" << std::endl; + std::cout << "dynamic pack string" << std::endl; + + buf.clear(); + + timer.reset(); + msgpack::pack(buf, obj); + timer.show_stat(buf.size()); +} + +int main(void) +{ + void* str = malloc(TASK_STR_LEN); + memset(str, 'a', TASK_STR_LEN); + TASK_STR_PTR = str; + + bench_msgpack_int(); + bench_msgpack_str(); + + return 0; +} + diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index d484593..807ded9 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -85,8 +85,8 @@ void unpacker::expand_buffer(size_t len) void* tmp = malloc(next_size); if(!tmp) { throw std::bad_alloc(); } memcpy(tmp, m_buffer, m_used); - free(m_buffer); + m_buffer = tmp; m_free = next_size - m_used; From 97bc0441b1204b3eb38f9cfc03f90e5ee675ab9a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0011/1648] lang/c/msgpack: C++ binding: enlarged chunk size of zone git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@58 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/unpack.cpp | 4 ++-- cpp/unpack.hpp | 3 +++ cpp/zone.cpp | 25 ++++++++++++++++--------- cpp/zone.hpp.erb | 11 +++++------ 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 807ded9..6a6a4e8 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -78,7 +78,7 @@ void unpacker::expand_buffer(size_t len) if(m_off == 0) { size_t next_size; if(m_free != 0) { next_size = m_free * 2; } - else { next_size = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; } + else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } // FIXME realloc? @@ -91,7 +91,7 @@ void unpacker::expand_buffer(size_t len) m_free = next_size - m_used; } else { - size_t next_size = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; + size_t next_size = UNPACKER_INITIAL_BUFFER_SIZE; while(next_size < len + m_used - m_off) { next_size *= 2; } void* tmp = malloc(next_size); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index ae536c9..9bda544 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -12,6 +12,9 @@ namespace msgpack { +static const size_t UNPACKER_INITIAL_BUFFER_SIZE = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; + + struct unpack_error : public std::runtime_error { unpack_error(const std::string& msg) : std::runtime_error(msg) { } diff --git a/cpp/zone.cpp b/cpp/zone.cpp index de2de22..ea4c7e4 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -5,10 +5,17 @@ namespace msgpack { void* zone::alloc() { - if(m_used >= m_pool.size()*MSGPACK_ZONE_CHUNK_SIZE) { - m_pool.push_back(new chunk_t()); + if(m_pool.size()*ZONE_CHUNK_SIZE <= m_used) { + cell_t* chunk = (cell_t*)malloc(sizeof(cell_t)*ZONE_CHUNK_SIZE); + if(!chunk) { throw std::bad_alloc(); } + try { + m_pool.push_back(chunk); + } catch (...) { + free(chunk); + throw; + } } - void* data = m_pool[m_used/MSGPACK_ZONE_CHUNK_SIZE]->cells[m_used%MSGPACK_ZONE_CHUNK_SIZE].data; + void* data = m_pool[m_used/ZONE_CHUNK_SIZE][m_used%ZONE_CHUNK_SIZE].data; ++m_used; return data; } @@ -16,21 +23,21 @@ void* zone::alloc() void zone::clear() { if(!m_pool.empty()) { - for(size_t b=0; b < m_used/MSGPACK_ZONE_CHUNK_SIZE; ++b) { - cell_t* c(m_pool[b]->cells); - for(size_t e=0; e < MSGPACK_ZONE_CHUNK_SIZE; ++e) { + for(size_t b=0; b < m_used/ZONE_CHUNK_SIZE; ++b) { + cell_t* c(m_pool[b]); + for(size_t e=0; e < ZONE_CHUNK_SIZE; ++e) { reinterpret_cast(c[e].data)->~object_class(); } } - cell_t* c(m_pool.back()->cells); - for(size_t e=0; e < m_used%MSGPACK_ZONE_CHUNK_SIZE; ++e) { + cell_t* c(m_pool.back()); + for(size_t e=0; e < m_used%ZONE_CHUNK_SIZE; ++e) { reinterpret_cast(c[e].data)->~object_class(); } for(pool_t::iterator it(m_pool.begin()), it_end(m_pool.end()); it != it_end; ++it) { - delete *it; + free(*it); } m_pool.clear(); } diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 40ce694..5c0e0d8 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -8,12 +8,15 @@ #include #ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 64 +#define MSGPACK_ZONE_CHUNK_SIZE 8*1024 #endif namespace msgpack { +static const size_t ZONE_CHUNK_SIZE = MSGPACK_ZONE_CHUNK_SIZE; + + class zone { public: zone() : m_used(0) { } @@ -110,11 +113,7 @@ private: char data[MAX_OBJECT_SIZE]; }; - struct chunk_t { - cell_t cells[MSGPACK_ZONE_CHUNK_SIZE]; - }; - - typedef std::vector pool_t; + typedef std::vector pool_t; pool_t m_pool; From b5f13e7d26e88f4c12675a14b62dda3cd3bc9598 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:56 +0000 Subject: [PATCH 0012/1648] lang/c/msgpack: autotoolized git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@59 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- AUTHORS | 0 ChangeLog | 0 Makefile.am | 7 +++ NEWS | 0 bootstrap | 119 +++++++++++++++++++++++++++++++++++++++++++++++ c/AUTHORS | 0 c/ChangeLog | 0 c/Makefile | 17 ------- c/Makefile.am | 15 ++++++ c/NEWS | 0 c/README | 0 c/bootstrap | 3 ++ c/configure.in | 11 +++++ configure.in | 10 ++++ cpp/AUTHORS | 0 cpp/ChangeLog | 0 cpp/Makefile | 25 ---------- cpp/Makefile.am | 20 ++++++++ cpp/NEWS | 0 cpp/README | 0 cpp/bootstrap | 3 ++ cpp/configure.in | 14 ++++++ ruby/bench.rb | 22 ++++++--- 23 files changed, 218 insertions(+), 48 deletions(-) create mode 100644 AUTHORS create mode 100644 ChangeLog create mode 100644 Makefile.am create mode 100644 NEWS create mode 100755 bootstrap create mode 100644 c/AUTHORS create mode 100644 c/ChangeLog delete mode 100644 c/Makefile create mode 100644 c/Makefile.am create mode 100644 c/NEWS create mode 100644 c/README create mode 100755 c/bootstrap create mode 100644 c/configure.in create mode 100644 configure.in create mode 100644 cpp/AUTHORS create mode 100644 cpp/ChangeLog delete mode 100644 cpp/Makefile create mode 100644 cpp/Makefile.am create mode 100644 cpp/NEWS create mode 100644 cpp/README create mode 100755 cpp/bootstrap create mode 100644 cpp/configure.in diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..e69de29 diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..9d1282d --- /dev/null +++ b/Makefile.am @@ -0,0 +1,7 @@ +SUBDIRS = c cpp + +nobase_include_HEADERS = \ + msgpack/pack/inline_impl.h \ + msgpack/unpack/inline_context.h \ + msgpack/unpack/inline_impl.h + diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap b/bootstrap new file mode 100755 index 0000000..954fa89 --- /dev/null +++ b/bootstrap @@ -0,0 +1,119 @@ +#!/bin/sh +# vim:ts=4:sw=4 +# Calls autotools to build configure script and Makefile.in. +# Generated automatically using bootstrapper 0.2.1 +# http://bootstrapper.sourceforge.net/ +# +# Copyright (C) 2002 Anthony Ventimiglia +# +# This bootstrap script is free software; you can redistribute +# it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# +# Calls proper programs to create configure script and Makefile.in files. +# if run with the --clean option, bootstrap removes files it generates. To +# clean all autogenerated files (eg: for cvs imports) first run +# make distclean, then bootstrap --clean +# see bootstrapper(1) for more infor + + +if test x"$1" = x"--help"; then + echo "$0: automatic bootstrapping utility for GNU Autotools" + echo " cleans up old autogenerated files and runs autoconf," + echo " automake and aclocal on local directory" + echo + echo " --clean clean up auto-generated files without" + echo " creating new scripts" + echo + exit 0 +fi + + +if [ -z "$NO_NEST" ];then + cd c && ./bootstrap $@; cd .. + cd cpp && ./bootstrap $@; cd .. +fi + + +ACLOCAL="aclocal" +ACLOCAL_FILES="aclocal.m4" +ALWAYS_CLEAN="config.status config.log config.cache libtool" +AUTOCONF="autoconf" +AUTOCONF_FILES="configure" +AUTOHEADER="autoheader" +AUTOHEADER_FILES="" +AUTOMAKE="automake --add-missing --copy" +AUTOMAKE_FILES="config.sub stamp-h.in ltmain.sh missing mkinstalldirs install-sh config.guess" +CONFIG_AUX_DIR="." +CONFIG_FILES="stamp-h ltconfig" +CONFIG_HEADER="" +if [ x`uname` = x"Darwin" ]; then + LIBTOOLIZE="glibtoolize --force --copy" +else + LIBTOOLIZE="libtoolize --force --copy" +fi +LIBTOOLIZE_FILES="config.sub ltmain.sh config.guess" +RM="rm" +SUBDIRS="[]" + + +# These are files created by configure, so we'll always clean them +for i in $ALWAYS_CLEAN; do + test -f $i && \ + $RM $i +done + +if test x"$1" = x"--clean"; then + # + #Clean Files left by previous bootstrap run + # + if test -n "$CONFIG_AUX_DIR"; + then CONFIG_AUX_DIR="$CONFIG_AUX_DIR/" + fi + # Clean Libtoolize generated files + for cf in $LIBTOOLIZE_FILES; do + cf="$CONFIG_AUX_DIR$cf" + test -f $cf && \ + $RM $cf + done + #aclocal.m4 created by aclocal + test -f $ACLOCAL_FILES && $RM $ACLOCAL_FILES + #Clean Autoheader Generated files + for cf in $AUTOHEADER_FILES; do + cf=$CONFIG_AUX_DIR$cf + test -f $cf && \ + $RM $cf + done + # remove config header (Usaually config.h) + test -n "$CONFIG_HEADER" && test -f $CONFIG_HEADER && $RM $CONFIG_HEADER + #Clean Automake generated files + for cf in $AUTOMAKE_FILES; do + cf=$CONFIG_AUX_DIR$cf + test -f $cf && \ + $RM $cf + done + for i in $SUBDIRS; do + test -f $i/Makefile.in && \ + $RM $i/Makefile.in + done + #Autoconf generated files + for cf in $AUTOCONF_FILES; do + test -f $cf && \ + $RM $cf + done + for cf in $CONFIG_FILES; do + cf="$CONFIG_AUX_DIR$cf" + test -f $cf && \ + $RM $cf + done +else + $LIBTOOLIZE + $ACLOCAL + $AUTOHEADER + $AUTOMAKE + $AUTOCONF +fi + + diff --git a/c/AUTHORS b/c/AUTHORS new file mode 100644 index 0000000..e69de29 diff --git a/c/ChangeLog b/c/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/c/Makefile b/c/Makefile deleted file mode 100644 index d4b2226..0000000 --- a/c/Makefile +++ /dev/null @@ -1,17 +0,0 @@ - -CFLAGS = -I.. -Wall -g -O4 -LDFLAGS = -L. - -all: bench - -bench: pack.o unpack.o unpack_inline.o bench.o pack.h pack_inline.h unpack.h unpack_context.h - $(CC) $(LDFLAGS) unpack.o unpack_inline.o pack.o bench.o -lyajl_s -o $@ - -bench_inline: pack.o bench_inline.o pack.h pack_inline.h - $(CC) $(LDFLAGS) pack.o bench_inline.o -lyajl_s -o $@ - -.PHONY: clean -clean: - $(RM) unpack.o unpack_inline.o pack.o test.o bench.o bench_inline.o - $(RM) bench bench_inline - diff --git a/c/Makefile.am b/c/Makefile.am new file mode 100644 index 0000000..15cf753 --- /dev/null +++ b/c/Makefile.am @@ -0,0 +1,15 @@ +lib_LTLIBRARIES = libmsgpackc.la + +libmsgpackc_la_SOURCES = \ + pack.c \ + unpack.c \ + unpack_inline.c + +nobase_include_HEADERS = \ + msgpack/pack.h \ + msgpack/unpack.h + +noinst_HEADERS = \ + pack_inline.h \ + unpack_context.h + diff --git a/c/NEWS b/c/NEWS new file mode 100644 index 0000000..e69de29 diff --git a/c/README b/c/README new file mode 100644 index 0000000..e69de29 diff --git a/c/bootstrap b/c/bootstrap new file mode 100755 index 0000000..6a1e814 --- /dev/null +++ b/c/bootstrap @@ -0,0 +1,3 @@ +#!/bin/sh +NO_NEST=1 +source ../bootstrap diff --git a/c/configure.in b/c/configure.in new file mode 100644 index 0000000..0f37460 --- /dev/null +++ b/c/configure.in @@ -0,0 +1,11 @@ +AC_INIT(pack.h) +AM_INIT_AUTOMAKE(msgpackc, 0.1) +AC_CONFIG_HEADER(config.h) + +AC_PROG_CC +AC_PROG_LIBTOOL + +CFLAGS="-O4 $CFLAGS -Wall -I.." + +AC_OUTPUT([Makefile]) + diff --git a/configure.in b/configure.in new file mode 100644 index 0000000..941168c --- /dev/null +++ b/configure.in @@ -0,0 +1,10 @@ +AC_INIT(msgpack/unpack/inline_impl.h) +AM_INIT_AUTOMAKE(msgpack, 0.1) +AC_CONFIG_HEADER(config.h) + +AC_PROG_LIBTOOL +#AC_CHECK_PROG(RUBY, ruby, ruby, [$PATH]) + +AC_CONFIG_SUBDIRS([c cpp]) +AC_OUTPUT([Makefile]) + diff --git a/cpp/AUTHORS b/cpp/AUTHORS new file mode 100644 index 0000000..e69de29 diff --git a/cpp/ChangeLog b/cpp/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/cpp/Makefile b/cpp/Makefile deleted file mode 100644 index 4a3677e..0000000 --- a/cpp/Makefile +++ /dev/null @@ -1,25 +0,0 @@ - -#CXXFLAGS = -I.. -I. -Wall -g -CXXFLAGS = -I.. -I. -Wall -g -O4 -LDFLAGS = -L. $(CXXFLAGS) - -NEED_PREPROCESS = zone.hpp - -all: test bench - -%.hpp: %.hpp.erb - erb $< > $@ - -test: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o test.o object.hpp unpack.hpp pack.hpp - $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o test.o -o $@ - -bench: $(NEED_PREPROCESS) unpack.o unpack_inline.o object.o zone.o bench.o object.hpp unpack.hpp pack.hpp - $(CXX) $(LDFLAGS) unpack.o unpack_inline.o zone.o object.o bench.o -o $@ - -.PHONY: clean -clean: - $(RM) unpack.o unpack_inline.o object.o zone.o - $(RM) test.o test - $(RM) bench.o bench - $(RM) $(NEED_PREPROCESS) - diff --git a/cpp/Makefile.am b/cpp/Makefile.am new file mode 100644 index 0000000..664f1ed --- /dev/null +++ b/cpp/Makefile.am @@ -0,0 +1,20 @@ +lib_LTLIBRARIES = libmsgpack.la + +libmsgpack_la_SOURCES = \ + object.cpp \ + unpack.cpp \ + unpack_inline.cpp \ + zone.cpp + +nobase_include_HEADERS = \ + msgpack/pack.hpp \ + msgpack/unpack.hpp \ + msgpack/object.hpp \ + msgpack/zone.hpp + +noinst_HEADERS = \ + unpack_context.hpp + +zone.hpp: zone.hpp.erb + erb $< > $@ + diff --git a/cpp/NEWS b/cpp/NEWS new file mode 100644 index 0000000..e69de29 diff --git a/cpp/README b/cpp/README new file mode 100644 index 0000000..e69de29 diff --git a/cpp/bootstrap b/cpp/bootstrap new file mode 100755 index 0000000..6a1e814 --- /dev/null +++ b/cpp/bootstrap @@ -0,0 +1,3 @@ +#!/bin/sh +NO_NEST=1 +source ../bootstrap diff --git a/cpp/configure.in b/cpp/configure.in new file mode 100644 index 0000000..a94738a --- /dev/null +++ b/cpp/configure.in @@ -0,0 +1,14 @@ +AC_INIT(object.hpp) +AM_INIT_AUTOMAKE(msgpack, 0.1) +AC_CONFIG_HEADER(config.h) + +AC_PROG_CXX +AC_PROG_LIBTOOL +AC_CHECK_PROG(ERB, erb, erb, [$PATH]) + +AC_CHECK_LIB(stdc++, main) + +CXXFLAGS="-O4 $CXXFLAGS -Wall -I.." + +AC_OUTPUT([Makefile]) + diff --git a/ruby/bench.rb b/ruby/bench.rb index 1e6e27b..3b0b2ae 100644 --- a/ruby/bench.rb +++ b/ruby/bench.rb @@ -9,9 +9,9 @@ end ary = [] i = 0 -while i < (1<<23) - ary << (1<<23) - #ary << i +while i < (1<<24) + #ary << (1<<24) + ary << i i += 1 end @@ -23,10 +23,13 @@ a = Time.now packed = MessagePack::pack(ary) b = Time.now show10(packed) -puts "#{b-a} sec." +sec = b - a +puts "#{sec} sec." +puts "#{packed.length.to_f / sec / 1024 / 1024 * 8} Mbps" GC.start +=begin puts "----" puts "JSON" a = Time.now @@ -37,6 +40,7 @@ puts "#{b-a} sec." ary = nil GC.start +=end puts "----" @@ -44,17 +48,23 @@ puts "MessagePack" a = Time.now ary = MessagePack::unpack(packed) b = Time.now -puts "#{b-a} sec." +sec = b - a +puts "#{sec} sec." +puts "#{packed.length.to_f / sec / 1024 / 1024 * 8} Mbps" + +p ary.size +p (1<<24) ary = nil GC.start +=begin puts "----" puts "JSON" a = Time.now ary = JSON::load(json) b = Time.now puts "#{b-a} sec." - +=end From a721837ed0c1b9783deb72a09d51dfad79411262 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0013/1648] lang/c/msgpack: added license notifications git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@60 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- AUTHORS | 1 + COPYING | 14 ++++++++++++++ c/AUTHORS | 1 + c/COPYING | 14 ++++++++++++++ c/Makefile.am | 3 +++ c/README | 21 +++++++++++++++++++++ c/configure.in | 2 +- c/msgpack.h | 19 +++++++++++++++++++ c/pack.h | 17 +++++++++++++++++ c/pack_inline.h | 17 +++++++++++++++++ c/unpack.c | 2 +- configure.in | 2 +- cpp/AUTHORS | 1 + cpp/COPYING | 14 ++++++++++++++ cpp/Makefile.am | 7 ++++++- cpp/README | 21 +++++++++++++++++++++ cpp/configure.in | 2 +- cpp/msgpack.hpp | 21 +++++++++++++++++++++ cpp/object.cpp | 17 +++++++++++++++++ cpp/object.hpp | 17 +++++++++++++++++ cpp/pack.hpp | 17 +++++++++++++++++ cpp/test.cpp | 5 +++-- cpp/unpack.cpp | 17 +++++++++++++++++ cpp/unpack.hpp | 17 +++++++++++++++++ cpp/unpack_context.hpp | 17 +++++++++++++++++ cpp/unpack_inline.cpp | 17 +++++++++++++++++ cpp/zone.cpp | 17 +++++++++++++++++ cpp/zone.hpp.erb | 17 +++++++++++++++++ ruby/gem/Manifest.txt | 1 - ruby/gem/lib/msgpack/version.rb | 4 ++-- ruby/{gem.sh => gengem.sh} | 1 - 31 files changed, 332 insertions(+), 11 deletions(-) create mode 100644 COPYING create mode 100644 c/COPYING create mode 100644 c/msgpack.h create mode 100644 cpp/COPYING create mode 100644 cpp/msgpack.hpp rename ruby/{gem.sh => gengem.sh} (90%) diff --git a/AUTHORS b/AUTHORS index e69de29..ababacb 100644 --- a/AUTHORS +++ b/AUTHORS @@ -0,0 +1 @@ +FURUHASHI Sadayuki diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..5f100cd --- /dev/null +++ b/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/c/AUTHORS b/c/AUTHORS index e69de29..ababacb 100644 --- a/c/AUTHORS +++ b/c/AUTHORS @@ -0,0 +1 @@ +FURUHASHI Sadayuki diff --git a/c/COPYING b/c/COPYING new file mode 100644 index 0000000..5f100cd --- /dev/null +++ b/c/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/c/Makefile.am b/c/Makefile.am index 15cf753..af8f3ad 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -6,6 +6,7 @@ libmsgpackc_la_SOURCES = \ unpack_inline.c nobase_include_HEADERS = \ + msgpack.h \ msgpack/pack.h \ msgpack/unpack.h @@ -13,3 +14,5 @@ noinst_HEADERS = \ pack_inline.h \ unpack_context.h +libmsgpackc_la_LDFLAGS = -version-info 0:0:0 + diff --git a/c/README b/c/README index e69de29..609b67d 100644 --- a/c/README +++ b/c/README @@ -0,0 +1,21 @@ +MessagePack for C +----------------- +MessagePack is a binary-based efficient data interchange format. + + + +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + diff --git a/c/configure.in b/c/configure.in index 0f37460..4bc7e3e 100644 --- a/c/configure.in +++ b/c/configure.in @@ -1,5 +1,5 @@ AC_INIT(pack.h) -AM_INIT_AUTOMAKE(msgpackc, 0.1) +AM_INIT_AUTOMAKE(msgpackc, 0.1.0) AC_CONFIG_HEADER(config.h) AC_PROG_CC diff --git a/c/msgpack.h b/c/msgpack.h new file mode 100644 index 0000000..7a15f71 --- /dev/null +++ b/c/msgpack.h @@ -0,0 +1,19 @@ +/* + * MessagePack for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/pack.h" +#include "msgpack/unpack.h" diff --git a/c/pack.h b/c/pack.h index 8107d9f..3144f37 100644 --- a/c/pack.h +++ b/c/pack.h @@ -1,3 +1,20 @@ +/* + * MessagePack packing routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef MSGPACK_PACK_H__ #define MSGPACK_PACK_H__ diff --git a/c/pack_inline.h b/c/pack_inline.h index b1f9e63..acd96fa 100644 --- a/c/pack_inline.h +++ b/c/pack_inline.h @@ -1,3 +1,20 @@ +/* + * MessagePack packing routine for C + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #ifndef PACK_INLINE_H__ #define PACK_INLINE_H__ diff --git a/c/unpack.c b/c/unpack.c index 9fe3695..012888e 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -1,5 +1,5 @@ /* - * MessagePack packing routine for C + * MessagePack unpacking routine for C * * Copyright (C) 2008 FURUHASHI Sadayuki * diff --git a/configure.in b/configure.in index 941168c..9b6f6d4 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ AC_INIT(msgpack/unpack/inline_impl.h) -AM_INIT_AUTOMAKE(msgpack, 0.1) +AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL diff --git a/cpp/AUTHORS b/cpp/AUTHORS index e69de29..ababacb 100644 --- a/cpp/AUTHORS +++ b/cpp/AUTHORS @@ -0,0 +1 @@ +FURUHASHI Sadayuki diff --git a/cpp/COPYING b/cpp/COPYING new file mode 100644 index 0000000..5f100cd --- /dev/null +++ b/cpp/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 664f1ed..b9ecb1d 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -7,6 +7,7 @@ libmsgpack_la_SOURCES = \ zone.cpp nobase_include_HEADERS = \ + msgpack.hpp \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ @@ -15,6 +16,10 @@ nobase_include_HEADERS = \ noinst_HEADERS = \ unpack_context.hpp -zone.hpp: zone.hpp.erb +msgpack/zone.hpp: msgpack/zone.hpp.erb erb $< > $@ +MOSTLYCLEANFILES = zone.hpp + +libmsgpack_la_LDFLAGS = -version-info 0:0:0 + diff --git a/cpp/README b/cpp/README index e69de29..eceff1b 100644 --- a/cpp/README +++ b/cpp/README @@ -0,0 +1,21 @@ +MessagePack for C++ +------------------- +MessagePack is a binary-based efficient data interchange format. + + + +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + diff --git a/cpp/configure.in b/cpp/configure.in index a94738a..e9e21ea 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -1,5 +1,5 @@ AC_INIT(object.hpp) -AM_INIT_AUTOMAKE(msgpack, 0.1) +AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) AC_PROG_CXX diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp new file mode 100644 index 0000000..9f635d0 --- /dev/null +++ b/cpp/msgpack.hpp @@ -0,0 +1,21 @@ +// +// MessagePack for C++ +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#include "msgpack/object.hpp" +#include "msgpack/zone.hpp" +#include "msgpack/pack.hpp" +#include "msgpack/unpack.hpp" diff --git a/cpp/object.cpp b/cpp/object.cpp index 947e8dd..d6815ac 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ dynamic typed objects +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #include "msgpack/object.hpp" #include "msgpack/pack.hpp" diff --git a/cpp/object.hpp b/cpp/object.hpp index f029e32..b1f5369 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ dynamic typed objects +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #ifndef MSGPACK_OBJECT_HPP__ #define MSGPACK_OBJECT_HPP__ diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 89f8cee..ac7bb83 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ serializing routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #ifndef MSGPACK_PACK_HPP__ #define MSGPACK_PACK_HPP__ diff --git a/cpp/test.cpp b/cpp/test.cpp index e236d0f..7c91fbd 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -1,7 +1,8 @@ #include #include -#include -#include +//#include +//#include +#include #include #include diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 6a6a4e8..a0128bb 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ deserializing routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #include "msgpack/unpack.hpp" #include "unpack_context.hpp" #include diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 9bda544..452a231 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ deserializing routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #ifndef MSGPACK_UNPACK_HPP__ #define MSGPACK_UNPACK_HPP__ diff --git a/cpp/unpack_context.hpp b/cpp/unpack_context.hpp index 49a93e4..59bd872 100644 --- a/cpp/unpack_context.hpp +++ b/cpp/unpack_context.hpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ deserializing routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #ifndef UNPACK_CONTEXT_HPP__ #define UNPACK_CONTEXT_HPP__ diff --git a/cpp/unpack_inline.cpp b/cpp/unpack_inline.cpp index 9002327..37e0b66 100644 --- a/cpp/unpack_inline.cpp +++ b/cpp/unpack_inline.cpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ deserializing routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #include "unpack_context.hpp" diff --git a/cpp/zone.cpp b/cpp/zone.cpp index ea4c7e4..22be759 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -1,3 +1,20 @@ +// +// MessagePack for C++ memory pool +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #include "zone.hpp" namespace msgpack { diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 5c0e0d8..ac08595 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -1,3 +1,20 @@ +// +// MessagePack for C++ memory pool +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #ifndef MSGPACK_ZONE_HPP__ #define MSGPACK_ZONE_HPP__ #include diff --git a/ruby/gem/Manifest.txt b/ruby/gem/Manifest.txt index fac8555..299a4e0 100644 --- a/ruby/gem/Manifest.txt +++ b/ruby/gem/Manifest.txt @@ -13,7 +13,6 @@ ext/unpack.c ext/unpack.h ext/unpack_context.h ext/unpack_inline.c -msgpack/pack/inline_context.h msgpack/pack/inline_impl.h msgpack/unpack/inline_context.h msgpack/unpack/inline_impl.h diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb index 44d1dc0..c65972f 100644 --- a/ruby/gem/lib/msgpack/version.rb +++ b/ruby/gem/lib/msgpack/version.rb @@ -1,8 +1,8 @@ module MessagePack module VERSION #:nodoc: MAJOR = 0 - MINOR = 0 - TINY = 1 + MINOR = 1 + TINY = 0 STRING = [MAJOR, MINOR, TINY].join('.') end diff --git a/ruby/gem.sh b/ruby/gengem.sh similarity index 90% rename from ruby/gem.sh rename to ruby/gengem.sh index 9d9f429..4f8623c 100755 --- a/ruby/gem.sh +++ b/ruby/gengem.sh @@ -10,7 +10,6 @@ cp unpack.h gem/ext/ cp unpack_context.h gem/ext/ cp unpack_inline.c gem/ext/ cp ../README gem/README.txt -cp ../msgpack/pack/inline_context.h gem/msgpack/pack/ cp ../msgpack/pack/inline_impl.h gem/msgpack/pack/ cp ../msgpack/unpack/inline_context.h gem/msgpack/unpack/ cp ../msgpack/unpack/inline_impl.h gem/msgpack/unpack/ From 4d13f614b6839ac4ca78f5526ab79e911f33ea65 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0014/1648] lang/c/msgpack: optimize zone::alloc() git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@61 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 76 ++++++++++++++++++++++++------------------------ cpp/unpack.cpp | 11 ++++--- cpp/zone.cpp | 32 ++++++++++---------- cpp/zone.hpp.erb | 40 +++++++++++++++---------- 4 files changed, 86 insertions(+), 73 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index b1f5369..959d84e 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -69,28 +69,28 @@ class dynamic_packer; struct object_class { virtual ~object_class() {} - virtual bool isnil() const { return false; } - virtual bool xbool() const { throw cast_error(); } - virtual uint8_t xu8() const { throw cast_error(); } - virtual uint16_t xu16() const { throw cast_error(); } - virtual uint32_t xu32() const { throw cast_error(); } - virtual uint64_t xu64() const { throw cast_error(); } - virtual int8_t xi8() const { throw cast_error(); } - virtual int16_t xi16() const { throw cast_error(); } - virtual int32_t xi32() const { throw cast_error(); } - virtual int64_t xi64() const { throw cast_error(); } - virtual float xfloat() const { throw cast_error(); } - virtual double xdouble() const { throw cast_error(); } - virtual raw xraw() { throw cast_error(); } - virtual array& xarray() { throw cast_error(); } - virtual map& xmap() { throw cast_error(); } - virtual const_raw xraw() const { throw cast_error(); } + virtual bool isnil () const { return false; } + virtual bool xbool () const { throw cast_error(); } + virtual uint8_t xu8 () const { throw cast_error(); } + virtual uint16_t xu16 () const { throw cast_error(); } + virtual uint32_t xu32 () const { throw cast_error(); } + virtual uint64_t xu64 () const { throw cast_error(); } + virtual int8_t xi8 () const { throw cast_error(); } + virtual int16_t xi16 () const { throw cast_error(); } + virtual int32_t xi32 () const { throw cast_error(); } + virtual int64_t xi64 () const { throw cast_error(); } + virtual float xfloat () const { throw cast_error(); } + virtual double xdouble() const { throw cast_error(); } + virtual raw xraw () { throw cast_error(); } + virtual array& xarray () { throw cast_error(); } + virtual map& xmap () { throw cast_error(); } + virtual const_raw xraw () const { throw cast_error(); } virtual const array& xarray() const { throw cast_error(); } - virtual const map& xmap() const { throw cast_error(); } + virtual const map& xmap () const { throw cast_error(); } virtual bool operator== (const object_class* x) const { return false; } - bool operator!= (const object_class* x) const { return !(this->operator==(x)); } virtual bool operator< (const object_class* x) const { throw cast_error(); } virtual bool operator> (const object_class* x) const { throw cast_error(); } + bool operator!= (const object_class* x) const { return !(this->operator==(x)); } virtual void pack(dynamic_packer& p) const = 0; operator bool() const { return xbool(); } // FIXME !isnil(); operator uint8_t() const { return xu8(); } @@ -126,24 +126,24 @@ struct object { object(object_class* v) : val(v) {} //object(object_class& v) : val(&v) {} ~object() {} - bool isnil() const { return val->isnil(); } - bool xbool() const { return val->xbool(); } - uint8_t xu8() const { return val->xu8(); } - uint16_t xu16() const { return val->xu16(); } - uint32_t xu32() const { return val->xu32(); } - uint64_t xu64() const { return val->xu64(); } - int8_t xi8() const { return val->xi8(); } - int16_t xi16() const { return val->xi16(); } - int32_t xi32() const { return val->xi32(); } - int64_t xi64() const { return val->xi64(); } - float xfloat() const { return val->xfloat(); } - double xdouble() const { return val->xdouble(); } - raw xraw() { return val->xraw(); } - array& xarray() { return val->xarray(); } - map& xmap() { return val->xmap(); } - const_raw xraw() const { return const_cast(val)->xraw(); } + bool isnil () const { return val->isnil(); } + bool xbool () const { return val->xbool(); } + uint8_t xu8 () const { return val->xu8(); } + uint16_t xu16 () const { return val->xu16(); } + uint32_t xu32 () const { return val->xu32(); } + uint64_t xu64 () const { return val->xu64(); } + int8_t xi8 () const { return val->xi8(); } + int16_t xi16 () const { return val->xi16(); } + int32_t xi32 () const { return val->xi32(); } + int64_t xi64 () const { return val->xi64(); } + float xfloat () const { return val->xfloat(); } + double xdouble() const { return val->xdouble(); } + raw xraw () { return val->xraw(); } + array& xarray () { return val->xarray(); } + map& xmap () { return val->xmap(); } + const_raw xraw () const { return const_cast(val)->xraw(); } const array& xarray() const { return const_cast(val)->xarray(); } - const map& xmap() const { return const_cast(val)->xmap(); } + const map& xmap () const { return const_cast(val)->xmap(); } bool operator== (object x) const { return val->operator== (x.val); } bool operator!= (object x) const { return val->operator!= (x.val); } bool operator< (object x) const { return val->operator< (x.val); } @@ -163,9 +163,9 @@ struct object { operator raw() { return val->operator raw(); } operator array&() { return val->operator array&(); } operator map&() { return val->operator map&(); } - operator raw() const { return val->operator raw(); } - operator array&() const { return val->operator array&(); } - operator map&() const { return val->operator map&(); } + operator const_raw() const { return val->operator const_raw(); } + operator const array&() const { return val->operator const array&(); } + operator const map&() const { return val->operator const map&(); } const object& inspect(std::ostream& s) const { val->inspect(s); return *this; } private: diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index a0128bb..7a0cd78 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -21,6 +21,7 @@ namespace msgpack { + struct unpacker::context { context(zone* z) { @@ -98,11 +99,13 @@ void unpacker::expand_buffer(size_t len) else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } - // FIXME realloc? - void* tmp = malloc(next_size); + void* tmp = realloc(m_buffer, next_size); if(!tmp) { throw std::bad_alloc(); } - memcpy(tmp, m_buffer, m_used); - free(m_buffer); + m_buffer = tmp; + //void* tmp = malloc(next_size); + //if(!tmp) { throw std::bad_alloc(); } + //memcpy(tmp, m_buffer, m_used); + //free(m_buffer); m_buffer = tmp; m_free = next_size - m_used; diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 22be759..ff22fc3 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -20,34 +20,33 @@ namespace msgpack { -void* zone::alloc() +// FIXME custom allocator? + +void zone::expand_chunk() { - if(m_pool.size()*ZONE_CHUNK_SIZE <= m_used) { - cell_t* chunk = (cell_t*)malloc(sizeof(cell_t)*ZONE_CHUNK_SIZE); - if(!chunk) { throw std::bad_alloc(); } - try { - m_pool.push_back(chunk); - } catch (...) { - free(chunk); - throw; - } + cell_t* chunk = (cell_t*)malloc(sizeof(cell_t)*ZONE_CHUNK_SIZE); + if(!chunk) { throw std::bad_alloc(); } + try { + m_pool.push_back(chunk); + } catch (...) { + free(chunk); + throw; } - void* data = m_pool[m_used/ZONE_CHUNK_SIZE][m_used%ZONE_CHUNK_SIZE].data; - ++m_used; - return data; } void zone::clear() { if(!m_pool.empty()) { - for(size_t b=0; b < m_used/ZONE_CHUNK_SIZE; ++b) { + size_t base_size = m_used / ZONE_CHUNK_SIZE; + size_t extend_size = m_used % ZONE_CHUNK_SIZE; + for(size_t b=0; b < base_size; ++b) { cell_t* c(m_pool[b]); for(size_t e=0; e < ZONE_CHUNK_SIZE; ++e) { reinterpret_cast(c[e].data)->~object_class(); } } cell_t* c(m_pool.back()); - for(size_t e=0; e < m_used%ZONE_CHUNK_SIZE; ++e) { + for(size_t e=0; e < extend_size; ++e) { reinterpret_cast(c[e].data)->~object_class(); } @@ -60,7 +59,8 @@ void zone::clear() } m_used = 0; - for(user_finalizer_t::reverse_iterator it(m_user_finalizer.rbegin()), it_end(m_user_finalizer.rend()); + for(user_finalizer_t::reverse_iterator it(m_user_finalizer.rbegin()), + it_end(m_user_finalizer.rend()); it != it_end; ++it) { it->call(); diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index ac08595..d63fae8 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -25,7 +25,7 @@ #include #ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 8*1024 +#define MSGPACK_ZONE_CHUNK_SIZE 1024 #endif namespace msgpack { @@ -44,19 +44,19 @@ public: void push_finalizer(void (*func)(void* obj, void* user), T* obj, void* user); public: - object_nil* nnil() { return new (alloc()) object_nil(); } - object_true* ntrue() { return new (alloc()) object_true(); } - object_false* nfalse() { return new (alloc()) object_false(); } - object_u8* nu8( uint8_t v) { return new (alloc()) object_u8(v); } - object_u16* nu16(uint16_t v) { return new (alloc()) object_u16(v); } - object_u32* nu32(uint32_t v) { return new (alloc()) object_u32(v); } - object_u64* nu64(uint64_t v) { return new (alloc()) object_u64(v); } - object_i8* ni8( int8_t v) { return new (alloc()) object_i8(v); } - object_i16* ni16( int16_t v) { return new (alloc()) object_i16(v); } - object_i32* ni32( int32_t v) { return new (alloc()) object_i32(v); } - object_i64* ni64( int64_t v) { return new (alloc()) object_i64(v); } - object_float* nfloat( float v) { return new (alloc()) object_float(v); } - object_double* ndouble( double v) { return new (alloc()) object_double(v); } + object_nil* nnil () { return new (alloc()) object_nil(); } + object_true* ntrue () { return new (alloc()) object_true(); } + object_false* nfalse () { return new (alloc()) object_false(); } + object_u8* nu8 (uint8_t v) { return new (alloc()) object_u8(v); } + object_u16* nu16 (uint16_t v) { return new (alloc()) object_u16(v); } + object_u32* nu32 (uint32_t v) { return new (alloc()) object_u32(v); } + object_u64* nu64 (uint64_t v) { return new (alloc()) object_u64(v); } + object_i8* ni8 (int8_t v) { return new (alloc()) object_i8(v); } + object_i16* ni16 (int16_t v) { return new (alloc()) object_i16(v); } + object_i32* ni32 (int32_t v) { return new (alloc()) object_i32(v); } + object_i64* ni64 (int64_t v) { return new (alloc()) object_i64(v); } + object_float* nfloat (float v) { return new (alloc()) object_float(v); } + object_double* ndouble(double v) { return new (alloc()) object_double(v); } object_raw_ref* nraw_ref(void* ptr, uint32_t len) { return new (alloc()) object_raw_ref(ptr, len); } @@ -149,7 +149,7 @@ private: user_finalizer_t m_user_finalizer; private: - void resize_pool(size_t n); + void expand_chunk(); public: static void finalize_free(void* obj, void* user) @@ -173,6 +173,16 @@ inline bool zone::empty() const return m_used == 0 && m_user_finalizer.empty(); } +inline void* zone::alloc() +{ + if(m_pool.size() <= m_used/ZONE_CHUNK_SIZE) { + expand_chunk(); + } + void* data = m_pool[m_used/ZONE_CHUNK_SIZE][m_used%ZONE_CHUNK_SIZE].data; + ++m_used; + return data; +} + } // namespace msgpack From a0a798d79e5c11bae1b0b6a94b25e0dee0c19b77 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0015/1648] lang/c/msgpack: C++ binding: changed calback function of packer from Stream& append(const char*, size_t) to SomeType write(SomePointerType, SomeSizeType) git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@62 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/bench.cpp | 3 +-- cpp/pack.hpp | 47 ++++++++++++++++++++++++++++++++--------------- cpp/test.cpp | 5 +++-- cpp/unpack.cpp | 8 ++++---- cpp/unpack.hpp | 14 +++++++------- 5 files changed, 47 insertions(+), 30 deletions(-) diff --git a/cpp/bench.cpp b/cpp/bench.cpp index 1a9f6f9..649c0cc 100644 --- a/cpp/bench.cpp +++ b/cpp/bench.cpp @@ -49,14 +49,13 @@ public: } public: - inline simple_buffer& append(const char* buf, size_t len) + inline void write(const void* buf, size_t len) { if(m_allocated - m_used < len) { expand_buffer(len); } memcpy(m_storage + m_used, buf, len); m_used += len; - return *this; } void clear() diff --git a/cpp/pack.hpp b/cpp/pack.hpp index ac7bb83..94b5b9a 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -73,7 +73,7 @@ private: static void pack_string_impl(Stream& x, const char* b); static void pack_raw_impl(Stream& x, const void* b, size_t l); static void append_buffer(Stream& x, const unsigned char* buf, unsigned int len) - { x.append((const char*)buf, len); } + { x.write((const char*)buf, len); } private: Stream& m_stream; @@ -100,14 +100,13 @@ public: template dynamic_stream(Stream& s); public: - dynamic_stream& append(const char* buf, size_t len) - { (*m_function)(m_object, buf, len); return *this; } + void write(const char* buf, size_t len) + { (*m_function)(m_object, buf, len); } private: void* m_object; void (*m_function)(void* object, const char* buf, size_t len); private: - template - static void append_trampoline(void* object, const char* buf, size_t len); + struct write_trampoline; }; @@ -137,8 +136,6 @@ public: void pack_string(const char* b) { pack_string_impl(m_stream, b); } void pack_raw(const void* b, size_t l) { pack_raw_impl(m_stream, b, l); } -public: - private: static void pack_int_impl(dynamic_stream& x, int d); static void pack_unsigned_int_impl(dynamic_stream& x, unsigned int d); @@ -160,7 +157,7 @@ private: static void pack_string_impl(dynamic_stream& x, const char* b); static void pack_raw_impl(dynamic_stream& x, const void* b, size_t l); static void append_buffer(dynamic_stream& x, const unsigned char* buf, unsigned int len) - { x.append((const char*)buf, len); } + { x.write((const char*)buf, len); } private: dynamic_stream m_stream; @@ -181,17 +178,37 @@ private: template dynamic_packer::dynamic_packer(Stream& s) : m_stream(s) { } +struct dynamic_stream::write_trampoline { +private: + template + struct ret_type { + typedef R (*type)(void*, const char*, size_t); + }; + + template + static R trampoline(void* obj, const char* buf, size_t len) + { + return (reinterpret_cast(obj)->*MemFun)(buf, len); + } + +public: + template + static typename ret_type::type get(R (Stream::*func)(Ptr*, Sz)) + { + R (*f)(void*, const char*, size_t) = + &trampoline; + return f; + } +}; + template dynamic_stream::dynamic_stream(Stream& s) { m_object = reinterpret_cast(&s); - m_function = &dynamic_stream::append_trampoline; -} - -template -void dynamic_stream::append_trampoline(void* object, const char* buf, size_t len) -{ - (reinterpret_cast(object)->*MemFun)(buf, len); + m_function = reinterpret_cast( + write_trampoline::get(&Stream::write) + ); } diff --git a/cpp/test.cpp b/cpp/test.cpp index 7c91fbd..b63f9a1 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -28,9 +28,10 @@ public: } try { - std::string s; + std::stringstream s; msgpack::pack(s, o); - object ro = msgpack::unpack(s.data(), s.size(), m_zone); + std::string str(s.str()); + object ro = msgpack::unpack(str.data(), str.size(), m_zone); if(ro != o) { throw std::runtime_error("NOT MATCH"); } } catch (std::runtime_error& e) { std::cout << "** REUNPACK FAILED **" << std::endl; diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 7a0cd78..de33051 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -99,10 +99,10 @@ void unpacker::expand_buffer(size_t len) else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } - void* tmp = realloc(m_buffer, next_size); + char* tmp = (char*)realloc(m_buffer, next_size); if(!tmp) { throw std::bad_alloc(); } m_buffer = tmp; - //void* tmp = malloc(next_size); + //char* tmp = (char*)malloc(next_size); //if(!tmp) { throw std::bad_alloc(); } //memcpy(tmp, m_buffer, m_used); //free(m_buffer); @@ -114,9 +114,9 @@ void unpacker::expand_buffer(size_t len) size_t next_size = UNPACKER_INITIAL_BUFFER_SIZE; while(next_size < len + m_used - m_off) { next_size *= 2; } - void* tmp = malloc(next_size); + char* tmp = (char*)malloc(next_size); if(!tmp) { throw std::bad_alloc(); } - memcpy(tmp, ((char*)m_buffer)+m_off, m_used-m_off); + memcpy(tmp, m_buffer+m_off, m_used-m_off); try { m_zone->push_finalizer(&zone::finalize_free, NULL, m_buffer); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 452a231..f5896f3 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -48,7 +48,7 @@ public: void reserve_buffer(size_t len); /*! 2. read data to the buffer() up to buffer_capacity() bytes */ - void* buffer(); + char* buffer(); size_t buffer_capacity() const; /*! 3. specify the number of bytes actually copied */ @@ -74,7 +74,7 @@ public: // Note that there are no parsed buffer when execute() returned true. /*! get address of buffer that is not parsed */ - void* nonparsed_buffer(); + char* nonparsed_buffer(); size_t nonparsed_size() const; /*! get the number of bytes that is already parsed */ @@ -90,7 +90,7 @@ private: struct context; context* m_ctx; - void* m_buffer; + char* m_buffer; size_t m_used; size_t m_free; size_t m_off; @@ -110,8 +110,8 @@ inline void unpacker::reserve_buffer(size_t len) expand_buffer(len); } -inline void* unpacker::buffer() - { return (void*)(((char*)m_buffer)+m_used); } +inline char* unpacker::buffer() + { return m_buffer + m_used; } inline size_t unpacker::buffer_capacity() const { return m_free; } @@ -123,8 +123,8 @@ inline void unpacker::buffer_consumed(size_t len) } -inline void* unpacker::nonparsed_buffer() - { return (void*)(((char*)m_buffer)+m_off); } +inline char* unpacker::nonparsed_buffer() + { return m_buffer + m_off; } inline size_t unpacker::nonparsed_size() const { return m_used - m_off; } From 1278eb3c63e2371283e07c0360b044728d72c704 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0016/1648] lang/c/msgpack: fix types git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@63 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bench.c | 2 +- c/bench_inline.c | 2 -- c/unpack.h | 2 +- c/unpack_inline.c | 2 +- cpp/bench.cpp | 6 ++-- cpp/object.cpp | 19 +++++----- cpp/object.hpp | 61 +++++++++++++++++---------------- cpp/pack.hpp | 4 +-- cpp/test.cpp | 6 ++-- cpp/unpack.cpp | 6 ++-- cpp/unpack.hpp | 4 +-- cpp/unpack_inline.cpp | 4 +-- cpp/zone.hpp.erb | 28 ++++++++++----- msgpack/unpack/callback.h | 2 +- msgpack/unpack/inline_context.h | 2 +- msgpack/unpack/inline_impl.h | 4 +-- ruby/unpack_inline.c | 2 +- 17 files changed, 85 insertions(+), 71 deletions(-) diff --git a/c/bench.c b/c/bench.c index 91f0c21..6b8466f 100644 --- a/c/bench.c +++ b/c/bench.c @@ -55,7 +55,7 @@ static void* unpack_array_start(void* data, unsigned int n) { return NULL; } static void unpack_array_item(void* data, void* c, void* o) { } static void* unpack_map_start(void* data, unsigned int n) { return NULL; } static void unpack_map_item(void* data, void* c, void* k, void* v) { } -static void* unpack_raw(void* data, const void* b, const void* p, size_t l) { /*printf("unpack raw %p %lu\n",p,l);*/ return NULL; } +static void* unpack_raw(void* data, const char* b, const char* p, unsigned int l) { /*printf("unpack raw %p %lu\n",p,l);*/ return NULL; } typedef struct { size_t allocated; diff --git a/c/bench_inline.c b/c/bench_inline.c index 5390319..bb3c431 100644 --- a/c/bench_inline.c +++ b/c/bench_inline.c @@ -56,8 +56,6 @@ static inline void* msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int3 static inline void* msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) { return NULL; } static inline void* msgpack_unpack_float(msgpack_unpack_context* x, float d) { return NULL; } static inline void* msgpack_unpack_double(msgpack_unpack_context* x, double d) { return NULL; } -static inline void* msgpack_unpack_big_int(msgpack_unpack_context* x, const void* b, unsigned int l) { return NULL; } -static inline void* msgpack_unpack_big_float(msgpack_unpack_context* x, const void* b, unsigned int l) { return NULL; } static inline void* msgpack_unpack_nil(msgpack_unpack_context* x) { return NULL; } static inline void* msgpack_unpack_true(msgpack_unpack_context* x) { return NULL; } static inline void* msgpack_unpack_false(msgpack_unpack_context* x) { return NULL; } diff --git a/c/unpack.h b/c/unpack.h index 094328e..6367439 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -39,7 +39,7 @@ typedef struct { void (*unpack_array_item)(void* data, void* c, void* o); void* (*unpack_map_start)(void* data, unsigned int n); void (*unpack_map_item)(void* data, void* c, void* k, void* v); - void* (*unpack_raw)(void* data, const void* b, const void* p, size_t l); + void* (*unpack_raw)(void* data, const char* b, const char* p, unsigned int l); } msgpack_unpack_callback; typedef struct { diff --git a/c/unpack_inline.c b/c/unpack_inline.c index d758d3e..5282282 100644 --- a/c/unpack_inline.c +++ b/c/unpack_inline.c @@ -71,7 +71,7 @@ static inline void* msgpack_unpack_map_start(msgpack_unpack_t* x, unsigned int n static inline void msgpack_unpack_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) { x->callback.unpack_map_item(x->data, c, k, v); } -static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const void* b, const void* p, size_t l) +static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) { return x->callback.unpack_raw(x->data, b, p, l); } diff --git a/cpp/bench.cpp b/cpp/bench.cpp index 649c0cc..4133a2e 100644 --- a/cpp/bench.cpp +++ b/cpp/bench.cpp @@ -10,7 +10,7 @@ static const unsigned int TASK_INT_NUM = 1<<24; static const unsigned int TASK_STR_LEN = 1<<15; //static const unsigned int TASK_INT_NUM = 1<<23; //static const unsigned int TASK_STR_LEN = 1<<14; -static const void* TASK_STR_PTR; +static const char* TASK_STR_PTR; class simple_timer { @@ -49,7 +49,7 @@ public: } public: - inline void write(const void* buf, size_t len) + inline void write(const char* buf, size_t len) { if(m_allocated - m_used < len) { expand_buffer(len); @@ -168,7 +168,7 @@ void bench_msgpack_str() int main(void) { - void* str = malloc(TASK_STR_LEN); + char* str = (char*)malloc(TASK_STR_LEN); memset(str, 'a', TASK_STR_LEN); TASK_STR_PTR = str; diff --git a/cpp/object.cpp b/cpp/object.cpp index d6815ac..4068007 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -320,29 +320,30 @@ FLOAT_OBJECT(double) EXTRA \ bool object_##NAME::operator== (const object_class* x) const \ try { \ - const_raw xr(x->xraw()); \ + raw xr(x->xraw()); \ return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); \ } catch (type_error&) { return false; } \ bool object_##NAME::operator< (const object_class* x) const { \ - const_raw xr(x->xraw()); \ + raw xr(x->xraw()); \ if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } \ else { return len < xr.len; } } \ bool object_##NAME::operator> (const object_class* x) const { \ - const_raw xr(x->xraw()); \ + raw xr(x->xraw()); \ if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } \ else { return len > xr.len; } } \ void object_##NAME::pack(dynamic_packer& p) const \ { p.pack_raw(ptr, len); } \ const object_class* object_##NAME::inspect(std::ostream& s) const \ - { (s << '"').write((const char*)ptr, len) << '"'; return this; } // FIXME escape + { (s << '"').write(ptr, len) << '"'; return this; } // FIXME escape +// FIXME +RAW_OBJECT(mutable_raw_ref, + /*mutable_raw object_mutable_raw_ref::xraw() { return mutable_raw(ptr, len); }*/ + raw object_mutable_raw_ref::xraw() const { return raw(ptr, len); } ) + RAW_OBJECT(raw_ref, - raw object_raw_ref::xraw() { return raw(ptr, len); } - const_raw object_raw_ref::xraw() const { return const_raw(ptr, len); } ) - -RAW_OBJECT(const_raw_ref, - const_raw object_const_raw_ref::xraw() const { return const_raw(ptr, len); } ) + raw object_raw_ref::xraw() const { return raw(ptr, len); } ) #undef RAW_OBJECT(NAME, EXTRA) diff --git a/cpp/object.hpp b/cpp/object.hpp index 959d84e..9f91677 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -37,25 +37,25 @@ class positive_overflow_error : public overflow_error { }; class negative_overflow_error : public overflow_error { }; -struct raw { - explicit raw() : ptr(NULL), len(0) {} - explicit raw(void* p, size_t l) : ptr(p), len(l) {} +struct mutable_raw { + explicit mutable_raw() : ptr(NULL), len(0) {} + explicit mutable_raw(char* p, size_t l) : ptr(p), len(l) {} public: - void* ptr; + char* ptr; size_t len; public: - std::string str() { return std::string((const char*)ptr, len); } + std::string str() { return std::string(ptr, len); } }; -struct const_raw { - explicit const_raw() : ptr(NULL), len(0) {} - explicit const_raw(const void* p, size_t l) : ptr(p), len(l) {} - const_raw(const raw& m) : ptr(m.ptr), len(m.len) {} +struct raw { + explicit raw() : ptr(NULL), len(0) {} + explicit raw(const char* p, size_t l) : ptr(p), len(l) {} + raw(const mutable_raw& m) : ptr(m.ptr), len(m.len) {} public: - const void* ptr; + const char* ptr; size_t len; public: - std::string str() { return std::string((const char*)ptr, len); } + std::string str() { return std::string(ptr, len); } }; @@ -81,10 +81,10 @@ struct object_class { virtual int64_t xi64 () const { throw cast_error(); } virtual float xfloat () const { throw cast_error(); } virtual double xdouble() const { throw cast_error(); } - virtual raw xraw () { throw cast_error(); } - virtual array& xarray () { throw cast_error(); } - virtual map& xmap () { throw cast_error(); } - virtual const_raw xraw () const { throw cast_error(); } + //virtual mutable_raw xraw () { throw cast_error(); } // FIXME + virtual array& xarray() { throw cast_error(); } + virtual map& xmap () { throw cast_error(); } + virtual raw xraw () const { throw cast_error(); } virtual const array& xarray() const { throw cast_error(); } virtual const map& xmap () const { throw cast_error(); } virtual bool operator== (const object_class* x) const { return false; } @@ -103,10 +103,10 @@ struct object_class { operator int64_t() const { return xi64(); } operator float() const { return xfloat(); } operator double() const { return xdouble(); } - operator raw() { return xraw(); } - operator array&() { return xarray(); } - operator map&() { return xmap(); } - operator const_raw() const { return xraw(); } + //operator mutable_raw() { return xraw(); } // FIXME + operator array&() { return xarray(); } + operator map&() { return xmap(); } + operator raw() const { return xraw(); } operator const array&() const { return xarray(); } operator const map&() const { return xmap(); } virtual const object_class* inspect(std::ostream& s) const @@ -138,10 +138,10 @@ struct object { int64_t xi64 () const { return val->xi64(); } float xfloat () const { return val->xfloat(); } double xdouble() const { return val->xdouble(); } - raw xraw () { return val->xraw(); } - array& xarray () { return val->xarray(); } - map& xmap () { return val->xmap(); } - const_raw xraw () const { return const_cast(val)->xraw(); } + //mutable_raw xraw () { return val->xraw(); } // FIXME + array& xarray() { return val->xarray(); } + map& xmap () { return val->xmap(); } + raw xraw () const { return const_cast(val)->xraw(); } const array& xarray() const { return const_cast(val)->xarray(); } const map& xmap () const { return const_cast(val)->xmap(); } bool operator== (object x) const { return val->operator== (x.val); } @@ -149,6 +149,8 @@ struct object { bool operator< (object x) const { return val->operator< (x.val); } bool operator> (object x) const { return val->operator> (x.val); } void pack(dynamic_packer& p) const { val->pack(p); } + template + void pack(Stream& s) const { dynamic_packer p(s); pack(p); } operator bool() const { return val->operator bool(); } operator uint8_t() const { return val->operator uint8_t(); } operator uint16_t() const { return val->operator uint16_t(); } @@ -160,10 +162,10 @@ struct object { operator int64_t() const { return val->operator int64_t(); } operator float() const { return val->operator float(); } operator double() const { return val->operator double(); } - operator raw() { return val->operator raw(); } - operator array&() { return val->operator array&(); } - operator map&() { return val->operator map&(); } - operator const_raw() const { return val->operator const_raw(); } + //operator mutable_raw() { return val->operator mutable_raw(); } // FIXME + operator array&() { return val->operator array&(); } + operator map&() { return val->operator map&(); } + operator raw() const { return val->operator raw(); } operator const array&() const { return val->operator const array&(); } operator const map&() const { return val->operator const map&(); } const object& inspect(std::ostream& s) const @@ -275,8 +277,9 @@ private: \ uint32_t len; \ }; -RAW_CLASS(raw_ref, void*, raw xraw(); const_raw xraw() const; ) -RAW_CLASS(const_raw_ref, const void*, const_raw xraw() const; ) +// FIXME +RAW_CLASS(mutable_raw_ref, char*, /*mutable_raw xraw();*/ raw xraw() const; ) +RAW_CLASS(raw_ref, const char*, raw xraw() const; ) #undef RAW_CLASS(NAME, TYPE, EXTRA) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 94b5b9a..9580679 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -50,7 +50,7 @@ public: void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } void pack_string(const char* b) { pack_string_impl(m_stream, b); } - void pack_raw(const void* b, size_t l) { pack_raw_impl(m_stream, b, l); } + void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } private: static void pack_int_impl(Stream& x, int d); @@ -134,7 +134,7 @@ public: void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } void pack_string(const char* b) { pack_string_impl(m_stream, b); } - void pack_raw(const void* b, size_t l) { pack_raw_impl(m_stream, b, l); } + void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } private: static void pack_int_impl(dynamic_stream& x, int d); diff --git a/cpp/test.cpp b/cpp/test.cpp index b63f9a1..68050a3 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -4,7 +4,7 @@ //#include #include #include -#include +#include class checker { public: @@ -176,7 +176,7 @@ int main(void) // 2. read data to buffer() up to buffer_capacity() bytes size_t sz = stream.readsome( - (char*)upk.buffer(), + upk.buffer(), upk.buffer_capacity()); total_bytes -= sz; @@ -195,7 +195,7 @@ int main(void) msgpack::object o = upk.data(); // 5.2. the parsed object is valid until the zone is deleted - boost::scoped_ptr pz(upk.release_zone()); + std::auto_ptr pz(upk.release_zone()); std::cout << o << std::endl; ++num_msg; diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index de33051..fe7f4b5 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -31,9 +31,9 @@ struct unpacker::context { ~context() { } - int execute(const void* data, size_t len, size_t* off) + int execute(const char* data, size_t len, size_t* off) { - return msgpack_unpacker_execute(&m_ctx, (const char*)data, len, off); + return msgpack_unpacker_execute(&m_ctx, data, len, off); } object_class* data() @@ -171,7 +171,7 @@ void unpacker::reset() } -object unpacker::unpack(const void* data, size_t len, zone& z) +object unpacker::unpack(const char* data, size_t len, zone& z) { context ctx(&z); size_t off = 0; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index f5896f3..473e8e9 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -100,7 +100,7 @@ private: unpacker(const unpacker&); public: - static object unpack(const void* data, size_t len, zone& z); + static object unpack(const char* data, size_t len, zone& z); }; @@ -136,7 +136,7 @@ inline void unpacker::remove_nonparsed_buffer() { m_used = m_off; } -inline object unpack(const void* data, size_t len, zone& z) +inline object unpack(const char* data, size_t len, zone& z) { return unpacker::unpack(data, len, z); } diff --git a/cpp/unpack_inline.cpp b/cpp/unpack_inline.cpp index 37e0b66..40f2769 100644 --- a/cpp/unpack_inline.cpp +++ b/cpp/unpack_inline.cpp @@ -71,12 +71,12 @@ static inline void msgpack_unpack_array_item(zone** z, object_class* c, object_c { reinterpret_cast(c)->push_back(o); } static inline object_class* msgpack_unpack_map_start(zone** z, unsigned int n) -{ return (*z)->narray(); } +{ return (*z)->nmap(); } static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) { reinterpret_cast(c)->store(k, v); } -static inline object_class* msgpack_unpack_raw(zone** z, const void* b, const void* p, size_t l) +static inline object_class* msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) { return (*z)->nraw_ref(p, l); } diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index d63fae8..8af8da2 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -58,19 +58,20 @@ public: object_float* nfloat (float v) { return new (alloc()) object_float(v); } object_double* ndouble(double v) { return new (alloc()) object_double(v); } - object_raw_ref* nraw_ref(void* ptr, uint32_t len) + + object_mutable_raw_ref* nraw_ref(char* ptr, uint32_t len) + { return new (alloc()) object_mutable_raw_ref(ptr, len); } + + object_raw_ref* nraw_ref(const char* ptr, uint32_t len) { return new (alloc()) object_raw_ref(ptr, len); } - object_const_raw_ref* nraw_ref(const void* ptr, uint32_t len) - { return new (alloc()) object_const_raw_ref(ptr, len); } - - object_raw_ref* nraw_copy(const void* ptr, uint32_t len) + object_mutable_raw_ref* nraw_copy(const char* ptr, uint32_t len) { - void* copy = malloc(len); + char* copy = (char*)malloc(len); if(!copy) { throw std::bad_alloc(); } - object_raw_ref* o; + object_mutable_raw_ref* o; try { - o = new (alloc()) object_raw_ref(copy, len); + o = new (alloc()) object_mutable_raw_ref(copy, len); push_finalizer(&zone::finalize_free, NULL, copy); } catch (...) { free(copy); @@ -80,6 +81,17 @@ public: return o; } + + object_mutable_raw_ref* nraw_cstr_ref(char* str) + { return nraw_ref(str, strlen(str)); } + + object_raw_ref* nraw_cstr_ref(const char* str) + { return nraw_ref(str, strlen(str)); } + + object_mutable_raw_ref* nraw_cstr_copy(const char* str) + { return nraw_copy(str, strlen(str)); } + + object_array* narray() { return new (alloc()) object_array(); } diff --git a/msgpack/unpack/callback.h b/msgpack/unpack/callback.h index 51508cc..315bb9e 100644 --- a/msgpack/unpack/callback.h +++ b/msgpack/unpack/callback.h @@ -20,5 +20,5 @@ msgpack_object msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned in msgpack_object msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n); void msgpack_unpack_map_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object k, msgpack_object v); msgpack_object msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l); -msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l); +msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l); diff --git a/msgpack/unpack/inline_context.h b/msgpack/unpack/inline_context.h index 22698d5..e764c09 100644 --- a/msgpack/unpack/inline_context.h +++ b/msgpack/unpack/inline_context.h @@ -42,7 +42,7 @@ typedef struct { typedef struct { msgpack_unpack_context user; // must be first unsigned int cs; - size_t trail; + unsigned int trail; unsigned int top; msgpack_unpacker_stack stack[MSG_STACK_SIZE]; } msgpack_unpacker; diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h index fdbe7f6..443a40c 100644 --- a/msgpack/unpack/inline_impl.h +++ b/msgpack/unpack/inline_impl.h @@ -168,7 +168,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len const unsigned char* const pe = (unsigned char*)data + len; const void* n = NULL; - size_t trail = ctx->trail; + unsigned int trail = ctx->trail; unsigned int cs = ctx->cs; unsigned int top = ctx->top; msgpack_unpacker_stack* stack = ctx->stack; @@ -187,7 +187,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len /*printf("obj %d\n",obj);*/ \ goto _push #define push_variable_value(func, base, pos, len) \ - obj = func(user, (const void*)base, (const void*)pos, len); \ + obj = func(user, (const char*)base, (const char*)pos, len); \ /*printf("obj %d\n",obj);*/ \ goto _push diff --git a/ruby/unpack_inline.c b/ruby/unpack_inline.c index b30754b..849441c 100644 --- a/ruby/unpack_inline.c +++ b/ruby/unpack_inline.c @@ -71,7 +71,7 @@ static inline VALUE msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) { rb_hash_aset(c, k, v); } -static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l) +static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) { return rb_str_new(p, l); } #include "msgpack/unpack/inline_impl.h" From 48e0274505583affa8546cdffe19761033622aa2 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0017/1648] lang/c/msgpack: fixed packaging problem git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@64 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/configure.in | 2 +- c/pack_inline.h | 2 +- c/unpack.c | 2 +- c/unpack_context.h | 2 +- cpp/Makefile.am | 3 ++- cpp/configure.in | 2 +- cpp/object.hpp | 6 +++--- cpp/zone.cpp | 2 +- msgpack/pack/inline_impl.h | 10 +++++----- msgpack/unpack/inline_impl.h | 15 ++++++++------- 10 files changed, 24 insertions(+), 22 deletions(-) diff --git a/c/configure.in b/c/configure.in index 4bc7e3e..1cacbc4 100644 --- a/c/configure.in +++ b/c/configure.in @@ -1,4 +1,4 @@ -AC_INIT(pack.h) +AC_INIT(pack.c) AM_INIT_AUTOMAKE(msgpackc, 0.1.0) AC_CONFIG_HEADER(config.h) diff --git a/c/pack_inline.h b/c/pack_inline.h index acd96fa..dd43a20 100644 --- a/c/pack_inline.h +++ b/c/pack_inline.h @@ -18,7 +18,7 @@ #ifndef PACK_INLINE_H__ #define PACK_INLINE_H__ -#include "pack.h" +#include "msgpack/pack.h" typedef msgpack_pack_t* msgpack_pack_context; diff --git a/c/unpack.c b/c/unpack.c index 012888e..a2fc066 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "unpack.h" +#include "msgpack/unpack.h" #include "unpack_context.h" #include diff --git a/c/unpack_context.h b/c/unpack_context.h index 7337c9e..d7b0388 100644 --- a/c/unpack_context.h +++ b/c/unpack_context.h @@ -18,7 +18,7 @@ #ifndef UNPACK_CONTEXT_H__ #define UNPACK_CONTEXT_H__ -#include "unpack.h" +#include "msgpack/unpack.h" typedef void* msgpack_object; diff --git a/cpp/Makefile.am b/cpp/Makefile.am index b9ecb1d..d1fa826 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -14,7 +14,8 @@ nobase_include_HEADERS = \ msgpack/zone.hpp noinst_HEADERS = \ - unpack_context.hpp + unpack_context.hpp \ + msgpack/zone.hpp.erb msgpack/zone.hpp: msgpack/zone.hpp.erb erb $< > $@ diff --git a/cpp/configure.in b/cpp/configure.in index e9e21ea..5126be4 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -1,4 +1,4 @@ -AC_INIT(object.hpp) +AC_INIT(object.cpp) AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) diff --git a/cpp/object.hpp b/cpp/object.hpp index 9f91677..456210c 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -232,7 +232,7 @@ INTEGER_CLASS(int16_t, i16) INTEGER_CLASS(int32_t, i32) INTEGER_CLASS(int64_t, i64) -#undef INTEGER_CLASS(TYPE, NAME) +#undef INTEGER_CLASS #define FLOAT_CLASS(TYPE, NAME) \ @@ -260,7 +260,7 @@ private: \ FLOAT_CLASS(float, float) FLOAT_CLASS(double, double) -#undef FLOAT_CLASS(TYPE, NAME) +#undef FLOAT_CLASS #define RAW_CLASS(NAME, TYPE, EXTRA) \ @@ -281,7 +281,7 @@ private: \ RAW_CLASS(mutable_raw_ref, char*, /*mutable_raw xraw();*/ raw xraw() const; ) RAW_CLASS(raw_ref, const char*, raw xraw() const; ) -#undef RAW_CLASS(NAME, TYPE, EXTRA) +#undef RAW_CLASS struct object_array : object_class, object_container_mixin { diff --git a/cpp/zone.cpp b/cpp/zone.cpp index ff22fc3..4cc50d7 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -#include "zone.hpp" +#include "msgpack/zone.hpp" namespace msgpack { diff --git a/msgpack/pack/inline_impl.h b/msgpack/pack/inline_impl.h index d4d5a5a..635b697 100644 --- a/msgpack/pack/inline_impl.h +++ b/msgpack/pack/inline_impl.h @@ -52,7 +52,7 @@ #endif -#ifndef msgpack_pack_inline_func(name) +#ifndef msgpack_pack_inline_func #define msgpack_pack_inline_func(name) \ inline void msgpack_pack_##name #endif @@ -293,11 +293,11 @@ msgpack_pack_inline_func(raw)(msgpack_pack_context x, const void* b, size_t l) } -#undef msgpack_pack_inline_func(name) +#undef msgpack_pack_inline_func -#undef STORE_BE16(d) -#undef STORE_BE32(d) -#undef STORE_BE64(d) +#undef STORE_BE16 +#undef STORE_BE32 +#undef STORE_BE64 #endif /* msgpack/pack/inline_impl.h */ diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h index 443a40c..a6557f9 100644 --- a/msgpack/unpack/inline_impl.h +++ b/msgpack/unpack/inline_impl.h @@ -175,6 +175,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len msgpack_unpack_context* user = &ctx->user; msgpack_object obj; + msgpack_unpacker_stack* c = NULL; int ret; @@ -381,7 +382,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len _push: if(top == 0) { goto _finish; } - msgpack_unpacker_stack* c = &stack[top-1]; + c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: msgpack_unpack_array_item(user, c->obj, obj); @@ -444,16 +445,16 @@ _end: } -#ifdef betoh16(x) -#undef betoh16(x) +#ifdef betoh16 +#undef betoh16 #endif -#ifdef betoh32(x) -#undef betoh32(x) +#ifdef betoh32 +#undef betoh32 #endif -#ifdef betoh64(x) -#undef betoh64(x) +#ifdef betoh64 +#undef betoh64 #endif #ifdef __cplusplus From 76dda6d36e5a2edbe21443bd7344c41160373e2e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0018/1648] lang/c/msgpack: fixed cpp/Makefile.am git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@65 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile.am | 6 ++++++ cpp/object.cpp | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index d1fa826..08ea21f 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -17,6 +17,12 @@ noinst_HEADERS = \ unpack_context.hpp \ msgpack/zone.hpp.erb +# FIXME +object.lo: msgpack/zone.hpp +unpack.lo: msgpack/zone.hpp +unpack_context.lo: msgpack/zone.hpp +zone.lo: msgpack/zone.hpp + msgpack/zone.hpp: msgpack/zone.hpp.erb erb $< > $@ diff --git a/cpp/object.cpp b/cpp/object.cpp index 4068007..7bb9841 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -258,7 +258,7 @@ INTEGER_OBJECT(i16) INTEGER_OBJECT(i32) INTEGER_OBJECT(i64) -#undef INTEGER_OBJECT(NAME) +#undef INTEGER_OBJECT #define FLOAT_OBJECT(NAME) \ @@ -313,7 +313,7 @@ const object_class* object_##NAME::inspect(std::ostream& s) const \ FLOAT_OBJECT(float) FLOAT_OBJECT(double) -#undef FLOAT_OBJECT(NAME) +#undef FLOAT_OBJECT #define RAW_OBJECT(NAME, EXTRA) \ @@ -345,7 +345,7 @@ RAW_OBJECT(mutable_raw_ref, RAW_OBJECT(raw_ref, raw object_raw_ref::xraw() const { return raw(ptr, len); } ) -#undef RAW_OBJECT(NAME, EXTRA) +#undef RAW_OBJECT array& object_array::xarray() { return val; } From 1222466a1c52161a3da3c3c5ce552d4b90e32bf6 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:57 +0000 Subject: [PATCH 0019/1648] lang/c/msgpack: c-macro based template git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@66 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- Makefile.am | 6 +- c/Makefile.am | 7 +- c/bench.c | 86 ++-- c/bench.mk | 9 + c/bench_inline.c | 325 ------------ c/pack.c | 21 +- c/pack.h | 33 +- c/pack_inline.h | 32 -- c/unpack.c | 112 ++++- c/unpack.h | 35 +- c/unpack_context.h | 30 -- c/unpack_inline.c | 79 --- cpp/Makefile.am | 3 - cpp/bench.cpp | 10 +- cpp/bench.mk | 9 + cpp/object.cpp | 16 +- cpp/pack.hpp | 80 ++- cpp/unpack.cpp | 116 ++++- cpp/unpack.hpp | 6 +- cpp/unpack_context.hpp | 31 -- cpp/unpack_inline.cpp | 86 ---- .../{pack/inline_impl.h => pack_template.h} | 63 +-- msgpack/unpack/callback.h | 24 - msgpack/unpack/inline_context.h | 59 --- msgpack/unpack/inline_impl.h | 465 ------------------ msgpack/unpack_define.h | 127 +++++ msgpack/unpack_template.h | 360 ++++++++++++++ ruby/gem/Manifest.txt | 9 +- ruby/gem/lib/msgpack/version.rb | 2 +- ruby/gengem.sh | 9 +- ruby/pack.c | 40 +- ruby/pack_inline.h | 35 -- ruby/unpack.c | 103 +++- ruby/unpack_context.h | 35 -- ruby/unpack_inline.c | 78 --- 35 files changed, 1034 insertions(+), 1507 deletions(-) create mode 100644 c/bench.mk delete mode 100644 c/bench_inline.c delete mode 100644 c/pack_inline.h delete mode 100644 c/unpack_context.h delete mode 100644 c/unpack_inline.c create mode 100644 cpp/bench.mk delete mode 100644 cpp/unpack_context.hpp delete mode 100644 cpp/unpack_inline.cpp rename msgpack/{pack/inline_impl.h => pack_template.h} (77%) delete mode 100644 msgpack/unpack/callback.h delete mode 100644 msgpack/unpack/inline_context.h delete mode 100644 msgpack/unpack/inline_impl.h create mode 100644 msgpack/unpack_define.h create mode 100644 msgpack/unpack_template.h delete mode 100644 ruby/pack_inline.h delete mode 100644 ruby/unpack_context.h delete mode 100644 ruby/unpack_inline.c diff --git a/Makefile.am b/Makefile.am index 9d1282d..d79f7eb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,7 @@ SUBDIRS = c cpp nobase_include_HEADERS = \ - msgpack/pack/inline_impl.h \ - msgpack/unpack/inline_context.h \ - msgpack/unpack/inline_impl.h + msgpack/pack_template.h \ + msgpack/unpack_define.h \ + msgpack/unpack_template.h diff --git a/c/Makefile.am b/c/Makefile.am index af8f3ad..f1c57e9 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -2,17 +2,12 @@ lib_LTLIBRARIES = libmsgpackc.la libmsgpackc_la_SOURCES = \ pack.c \ - unpack.c \ - unpack_inline.c + unpack.c nobase_include_HEADERS = \ msgpack.h \ msgpack/pack.h \ msgpack/unpack.h -noinst_HEADERS = \ - pack_inline.h \ - unpack_context.h - libmsgpackc_la_LDFLAGS = -version-info 0:0:0 diff --git a/c/bench.c b/c/bench.c index 6b8466f..fa717c0 100644 --- a/c/bench.c +++ b/c/bench.c @@ -16,14 +16,15 @@ void reset_timer() gettimeofday(&g_timer, NULL); } -double show_timer() +void show_timer(size_t bufsz) { struct timeval endtime; gettimeofday(&endtime, NULL); double sec = (endtime.tv_sec - g_timer.tv_sec) + (double)(endtime.tv_usec - g_timer.tv_usec) / 1000 / 1000; printf("%f sec\n", sec); - return sec; + printf("%f MB\n", ((double)bufsz)/1024/1024); + printf("%f Mbps\n", ((double)bufsz)*8/sec/1000/1000); } @@ -38,32 +39,33 @@ static int reformat_start_array(void * ctx) { return 1; } static int reformat_end_array(void * ctx) { return 1; } -static void* unpack_unsigned_int_8(void* data, uint8_t d) { return NULL; } -static void* unpack_unsigned_int_16(void* data, uint16_t d) { return NULL; } -static void* unpack_unsigned_int_32(void* data, uint32_t d) { return NULL; } -static void* unpack_unsigned_int_64(void* data, uint64_t d) { return NULL; } -static void* unpack_signed_int_8(void* data, int8_t d) { return NULL; } -static void* unpack_signed_int_16(void* data, int16_t d) { return NULL; } -static void* unpack_signed_int_32(void* data, int32_t d) { return NULL; } -static void* unpack_signed_int_64(void* data, int64_t d) { return NULL; } +static void* unpack_uint8(void* data, uint8_t d) { return NULL; } +static void* unpack_uint16(void* data, uint16_t d) { return NULL; } +static void* unpack_uint32(void* data, uint32_t d) { return NULL; } +static void* unpack_uint64(void* data, uint64_t d) { return NULL; } +static void* unpack_int8(void* data, int8_t d) { return NULL; } +static void* unpack_int16(void* data, int16_t d) { return NULL; } +static void* unpack_int32(void* data, int32_t d) { return NULL; } +static void* unpack_int64(void* data, int64_t d) { return NULL; } static void* unpack_float(void* data, float d) { return NULL; } static void* unpack_double(void* data, double d) { return NULL; } static void* unpack_nil(void* data) { return NULL; } static void* unpack_true(void* data) { return NULL; } static void* unpack_false(void* data) { return NULL; } -static void* unpack_array_start(void* data, unsigned int n) { return NULL; } +static void* unpack_array(void* data, unsigned int n) { return NULL; } static void unpack_array_item(void* data, void* c, void* o) { } -static void* unpack_map_start(void* data, unsigned int n) { return NULL; } +static void* unpack_map(void* data, unsigned int n) { return NULL; } static void unpack_map_item(void* data, void* c, void* k, void* v) { } static void* unpack_raw(void* data, const char* b, const char* p, unsigned int l) { /*printf("unpack raw %p %lu\n",p,l);*/ return NULL; } + typedef struct { size_t allocated; size_t length; char* buffer; } pack_buffer; -static const size_t PACK_INITIAL_BUFFER_SIZE = 512; +static const size_t PACK_INITIAL_BUFFER_SIZE = 32*1024; static void pack_buffer_init(pack_buffer* data) { @@ -84,7 +86,7 @@ static void pack_buffer_free(pack_buffer* data) free(data->buffer); } -static void pack_append_buffer(void* user, const unsigned char* b, unsigned int l) +static void pack_append_buffer(void* user, const char* b, unsigned int l) { pack_buffer* data = (pack_buffer*)user; if(data->allocated - data->length < l) { @@ -128,7 +130,6 @@ void bench_json(void) yajl_handle h = yajl_alloc(&callbacks, &hcfg, NULL); - double sec; const unsigned char * buf; unsigned int len; @@ -143,11 +144,9 @@ void bench_json(void) } yajl_gen_array_close(g); } - sec = show_timer(); + show_timer(len); yajl_gen_get_buf(g, &buf, &len); - printf("%u KB\n", len / 1024); - printf("%f MB/s\n", len / sec / 1024 / 1024); puts("----"); puts("parse integer"); @@ -159,9 +158,7 @@ void bench_json(void) fprintf(stderr, (const char *) str); } } - sec = show_timer(); - - printf("%f MB/s\n", len / sec / 1024 / 1024); + show_timer(len); //yajl_gen_clear(g); @@ -182,11 +179,9 @@ void bench_json(void) } yajl_gen_array_close(g); } - sec = show_timer(); + show_timer(len); yajl_gen_get_buf(g, &buf, &len); - printf("%u KB\n", len / 1024); - printf("%f MB/s\n", len / sec / 1024 / 1024); puts("----"); puts("parse string"); @@ -198,15 +193,14 @@ void bench_json(void) fprintf(stderr, (const char *) str); } } - sec = show_timer(); - - printf("%f MB/s\n", len / sec / 1024 / 1024); + show_timer(len); yajl_gen_free(g); yajl_free(h); } + void bench_msgpack(void) { puts("== MessagePack =="); @@ -214,33 +208,33 @@ void bench_msgpack(void) pack_buffer mpkbuf; pack_buffer_init(&mpkbuf); + msgpack_pack_t* mpk = msgpack_pack_new( &mpkbuf, pack_append_buffer); msgpack_unpack_callback cb = { - unpack_unsigned_int_8, - unpack_unsigned_int_16, - unpack_unsigned_int_32, - unpack_unsigned_int_64, - unpack_signed_int_8, - unpack_signed_int_16, - unpack_signed_int_32, - unpack_signed_int_64, + unpack_uint8, + unpack_uint16, + unpack_uint32, + unpack_uint64, + unpack_int8, + unpack_int16, + unpack_int32, + unpack_int64, unpack_float, unpack_double, unpack_nil, unpack_true, unpack_false, - unpack_array_start, + unpack_array, unpack_array_item, - unpack_map_start, + unpack_map, unpack_map_item, unpack_raw, }; msgpack_unpack_t* mupk = msgpack_unpack_new(NULL, &cb); - double sec; size_t len; const char* buf; @@ -254,12 +248,10 @@ void bench_msgpack(void) msgpack_pack_unsigned_int(mpk, i); } } - sec = show_timer(); + show_timer(mpkbuf.length); len = mpkbuf.length; buf = mpkbuf.buffer; - printf("%lu KB\n", len / 1024); - printf("%f MB/s\n", len / sec / 1024 / 1024); puts("----"); puts("unpack integer"); @@ -273,9 +265,7 @@ void bench_msgpack(void) fprintf(stderr, "Not finished.\n"); } } - sec = show_timer(); - - printf("%f MB/s\n", len / sec / 1024 / 1024); + show_timer(mpkbuf.length); pack_buffer_reset(&mpkbuf); @@ -292,12 +282,10 @@ void bench_msgpack(void) msgpack_pack_raw(mpk, TASK_STR_PTR, i); } } - sec = show_timer(); + show_timer(mpkbuf.length); len = mpkbuf.length; buf = mpkbuf.buffer; - printf("%lu KB\n", len / 1024); - printf("%f MB/s\n", len / sec / 1024 / 1024); puts("----"); puts("unpack string"); @@ -311,9 +299,7 @@ void bench_msgpack(void) fprintf(stderr, "Not finished.\n"); } } - sec = show_timer(); - - printf("%f MB/s\n", len / sec / 1024 / 1024); + show_timer(mpkbuf.length); msgpack_unpack_free(mupk); diff --git a/c/bench.mk b/c/bench.mk new file mode 100644 index 0000000..c765e31 --- /dev/null +++ b/c/bench.mk @@ -0,0 +1,9 @@ + +CFLAGS += -Wall -g -I. -I.. -O4 +LDFLAGS += -lyajl + +all: bench + +bench: bench.o pack.o unpack.o pack.h unpack.h + $(CC) bench.o pack.o unpack.o $(CFLAGS) $(LDFLAGS) -o $@ + diff --git a/c/bench_inline.c b/c/bench_inline.c deleted file mode 100644 index bb3c431..0000000 --- a/c/bench_inline.c +++ /dev/null @@ -1,325 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include -#include - - -static struct timeval g_timer; - -void reset_timer() -{ - gettimeofday(&g_timer, NULL); -} - -double show_timer() -{ - struct timeval endtime; - gettimeofday(&endtime, NULL); - double sec = (endtime.tv_sec - g_timer.tv_sec) - + (double)(endtime.tv_usec - g_timer.tv_usec) / 1000 / 1000; - printf("%f sec\n", sec); - return sec; -} - - -static int reformat_null(void * ctx) { return 1; } -static int reformat_boolean(void * ctx, int boolean) { return 1; } -static int reformat_number(void * ctx, const char * s, unsigned int l) { return 1; } -static int reformat_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } -static int reformat_map_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } -static int reformat_start_map(void * ctx) { return 1; } -static int reformat_end_map(void * ctx) { return 1; } -static int reformat_start_array(void * ctx) { return 1; } -static int reformat_end_array(void * ctx) { return 1; } - - -typedef void* msgpack_object; - -typedef struct { -} msgpack_unpack_context; - -#include "msgpack/unpack/inline_context.h" - -static inline void* msgpack_unpack_init(msgpack_unpack_context* x) { return NULL; } -static inline void* msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d) { return NULL; } -static inline void* msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d) { return NULL; } -static inline void* msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d) { return NULL; } -static inline void* msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d) { return NULL; } -static inline void* msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d) { return NULL; } -static inline void* msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d) { return NULL; } -static inline void* msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d) { return NULL; } -static inline void* msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) { return NULL; } -static inline void* msgpack_unpack_float(msgpack_unpack_context* x, float d) { return NULL; } -static inline void* msgpack_unpack_double(msgpack_unpack_context* x, double d) { return NULL; } -static inline void* msgpack_unpack_nil(msgpack_unpack_context* x) { return NULL; } -static inline void* msgpack_unpack_true(msgpack_unpack_context* x) { return NULL; } -static inline void* msgpack_unpack_false(msgpack_unpack_context* x) { return NULL; } -static inline void* msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n) { return NULL; } -static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, void* c, void* o) { } -static inline void* msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) { return NULL; } -static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, void* c, void* k, void* v) { } -static inline void* msgpack_unpack_raw(msgpack_unpack_context* x, const void* b, const void* p, size_t l) { return NULL; } - -#include "msgpack/unpack/inline_impl.h" - -typedef struct { - size_t allocated; - size_t length; - char* buffer; -} pack_buffer; - -static const size_t PACK_INITIAL_BUFFER_SIZE = 512; - -static void pack_buffer_init(pack_buffer* data) -{ - data->buffer = malloc(PACK_INITIAL_BUFFER_SIZE); - data->length = 0; - data->allocated = PACK_INITIAL_BUFFER_SIZE; -} - -static void pack_buffer_reset(pack_buffer* data) -{ - data->buffer = realloc(data->buffer, PACK_INITIAL_BUFFER_SIZE); - data->allocated = PACK_INITIAL_BUFFER_SIZE; - data->length = 0; -} - -static void pack_buffer_free(pack_buffer* data) -{ - free(data->buffer); -} - -static void pack_append_buffer(void* user, const unsigned char* b, unsigned int l) -{ - pack_buffer* data = (pack_buffer*)user; - if(data->allocated - data->length < l) { - data->buffer = realloc(data->buffer, data->allocated*2); - data->allocated *= 2; - } - memcpy(data->buffer + data->length, b, l); - data->length += l; -} - - -static const unsigned int TASK_INT_NUM = 1<<24; -static const unsigned int TASK_STR_LEN = 1<<15; -//static const unsigned int TASK_INT_NUM = 1<<20; -//static const unsigned int TASK_STR_LEN = 1<<12; -static const char* TASK_STR_PTR; - - -void bench_json(void) -{ - puts("== JSON =="); - - - yajl_gen_config gcfg = {0, NULL}; - yajl_gen g = yajl_gen_alloc(&gcfg); - - yajl_parser_config hcfg = { 0, 0 }; - yajl_callbacks callbacks = { - reformat_null, - reformat_boolean, - NULL, - NULL, - reformat_number, - reformat_start_map, - reformat_map_key, - reformat_end_map, - reformat_start_array, - reformat_end_array - }; - yajl_handle h = yajl_alloc(&callbacks, &hcfg, NULL); - - - double sec; - const unsigned char * buf; - unsigned int len; - - - puts("generate integer"); - reset_timer(); - { - unsigned int i; - yajl_gen_array_open(g); - for(i=0; i < TASK_INT_NUM; ++i) { - yajl_gen_integer(g, i); - } - yajl_gen_array_close(g); - } - sec = show_timer(); - - yajl_gen_get_buf(g, &buf, &len); - printf("%u KB\n", len / 1024); - printf("%f Mbps\n", len / sec / 1024 / 1024); - - puts("----"); - puts("parse integer"); - reset_timer(); - { - yajl_status stat = yajl_parse(h, buf, len); - if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { - unsigned char * str = yajl_get_error(h, 1, buf, len); - fprintf(stderr, (const char *) str); - } - } - sec = show_timer(); - - printf("%f Mbps\n", len / sec / 1024 / 1024); - - - //yajl_gen_clear(g); - yajl_gen_free(g); - g = yajl_gen_alloc(&gcfg); - yajl_free(h); - h = yajl_alloc(&callbacks, &hcfg, NULL); - - - puts("----"); - puts("generate string"); - reset_timer(); - { - unsigned int i; - yajl_gen_array_open(g); - for(i=0; i < TASK_STR_LEN; ++i) { - yajl_gen_string(g, (const unsigned char*)TASK_STR_PTR, i); - } - yajl_gen_array_close(g); - } - sec = show_timer(); - - yajl_gen_get_buf(g, &buf, &len); - printf("%u KB\n", len / 1024); - printf("%f Mbps\n", len / sec / 1024 / 1024); - - puts("----"); - puts("parse string"); - reset_timer(); - { - yajl_status stat = yajl_parse(h, buf, len); - if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { - unsigned char * str = yajl_get_error(h, 1, buf, len); - fprintf(stderr, (const char *) str); - } - } - sec = show_timer(); - - printf("%f Mbps\n", len / sec / 1024 / 1024); - - - yajl_gen_free(g); - yajl_free(h); -} - -void bench_msgpack(void) -{ - puts("== MessagePack =="); - - - pack_buffer mpkbuf; - pack_buffer_init(&mpkbuf); - msgpack_pack_t* mpk = msgpack_pack_new( - &mpkbuf, pack_append_buffer); - - msgpack_unpacker mupk; - msgpack_unpacker_init(&mupk); - - double sec; - size_t len; - const char* buf; - - - puts("pack integer"); - reset_timer(); - { - unsigned int i; - msgpack_pack_array(mpk, TASK_INT_NUM); - for(i=0; i < TASK_INT_NUM; ++i) { - msgpack_pack_unsigned_int(mpk, i); - } - } - sec = show_timer(); - - len = mpkbuf.length; - buf = mpkbuf.buffer; - printf("%lu KB\n", len / 1024); - printf("%f Mbps\n", len / sec / 1024 / 1024); - - puts("----"); - puts("unpack integer"); - reset_timer(); - { - size_t off = 0; - int ret = msgpack_unpacker_execute(&mupk, buf, len, &off); - if(ret < 0) { - fprintf(stderr, "Parse error.\n"); - } else if(ret == 0) { - fprintf(stderr, "Not finished.\n"); - } - } - sec = show_timer(); - - printf("%f Mbps\n", len / sec / 1024 / 1024); - - - pack_buffer_reset(&mpkbuf); - msgpack_unpacker_init(&mupk); - - - puts("----"); - puts("pack string"); - reset_timer(); - { - unsigned int i; - msgpack_pack_array(mpk, TASK_STR_LEN); - for(i=0; i < TASK_STR_LEN; ++i) { - msgpack_pack_raw(mpk, TASK_STR_PTR, i); - } - } - sec = show_timer(); - - len = mpkbuf.length; - buf = mpkbuf.buffer; - printf("%lu KB\n", len / 1024); - printf("%f Mbps\n", len / sec / 1024 / 1024); - - puts("----"); - puts("unpack string"); - reset_timer(); - { - size_t off = 0; - int ret = msgpack_unpacker_execute(&mupk, buf, len, &off); - if(ret < 0) { - fprintf(stderr, "Parse error.\n"); - } else if(ret == 0) { - fprintf(stderr, "Not finished.\n"); - } - } - sec = show_timer(); - - printf("%f Mbps\n", len / sec / 1024 / 1024); - - - msgpack_pack_free(mpk); - pack_buffer_free(&mpkbuf); -} - -int main(int argc, char* argv[]) -{ - char* str = malloc(TASK_STR_LEN); - memset(str, 'a', TASK_STR_LEN); - TASK_STR_PTR = str; - - bench_msgpack(); - //bench_json(); - - return 0; -} - - - diff --git a/c/pack.c b/c/pack.c index 05bd38d..766a9d1 100644 --- a/c/pack.c +++ b/c/pack.c @@ -15,11 +15,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "pack_inline.h" +#include "msgpack/pack.h" #include -#include -msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_callback_t callback) + +#define msgpack_pack_inline_func(name) \ + void msgpack_pack_##name + +#define msgpack_pack_user msgpack_pack_t* + +#define msgpack_pack_append_buffer(user, buf, len) \ + (*(user)->callback)((user)->data, (const char*)buf, len) + +#include "msgpack/pack_template.h" + +msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_append_buffer_t callback) { msgpack_pack_t* ctx = calloc(1, sizeof(msgpack_pack_t)); if(!ctx) { return NULL; } @@ -33,8 +43,3 @@ void msgpack_pack_free(msgpack_pack_t* ctx) free(ctx); } -static inline void msgpack_pack_append_buffer(msgpack_pack_t* ctx, const unsigned char* b, unsigned int l) -{ - ctx->callback(ctx->data, b, l); -} - diff --git a/c/pack.h b/c/pack.h index 3144f37..c6cadf4 100644 --- a/c/pack.h +++ b/c/pack.h @@ -21,26 +21,32 @@ #include #include -typedef void (*msgpack_pack_callback_t)(void* data, const unsigned char* b, unsigned int i); +#ifdef __cplusplus +extern "C" { +#endif + + +typedef void (*msgpack_pack_append_buffer_t)(void* data, const char* b, unsigned int i); typedef struct { void* data; - msgpack_pack_callback_t callback; + msgpack_pack_append_buffer_t callback; } msgpack_pack_t; -msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_callback_t callback); +msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_append_buffer_t callback); + void msgpack_pack_free(msgpack_pack_t* ctx); void msgpack_pack_int(msgpack_pack_t* ctx, int d); void msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); -void msgpack_pack_unsigned_int_8(msgpack_pack_t* ctx, uint8_t d); -void msgpack_pack_unsigned_int_16(msgpack_pack_t* ctx, uint16_t d); -void msgpack_pack_unsigned_int_32(msgpack_pack_t* ctx, uint32_t d); -void msgpack_pack_unsigned_int_64(msgpack_pack_t* ctx, uint64_t d); -void msgpack_pack_signed_int_8(msgpack_pack_t* ctx, int8_t d); -void msgpack_pack_signed_int_16(msgpack_pack_t* ctx, int16_t d); -void msgpack_pack_signed_int_32(msgpack_pack_t* ctx, int32_t d); -void msgpack_pack_signed_int_64(msgpack_pack_t* ctx, int64_t d); +void msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); +void msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); +void msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); +void msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); +void msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); +void msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); +void msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); +void msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); void msgpack_pack_float(msgpack_pack_t* ctx, float d); void msgpack_pack_double(msgpack_pack_t* ctx, double d); void msgpack_pack_nil(msgpack_pack_t* ctx); @@ -51,5 +57,10 @@ void msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); void msgpack_pack_string(msgpack_pack_t* ctx, const char* b); void msgpack_pack_raw(msgpack_pack_t* ctx, const void* b, size_t l); + +#ifdef __cplusplus +} +#endif + #endif /* msgpack/pack.h */ diff --git a/c/pack_inline.h b/c/pack_inline.h deleted file mode 100644 index dd43a20..0000000 --- a/c/pack_inline.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * MessagePack packing routine for C - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef PACK_INLINE_H__ -#define PACK_INLINE_H__ - -#include "msgpack/pack.h" - -typedef msgpack_pack_t* msgpack_pack_context; - -static inline void msgpack_pack_append_buffer(msgpack_pack_t* x, const unsigned char* b, unsigned int l); - -#include -#include /* __BYTE_ORDER */ -#include "msgpack/pack/inline_impl.h" - -#endif /* pack_inline.h */ - diff --git a/c/unpack.c b/c/unpack.c index a2fc066..3058427 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -16,15 +16,101 @@ * limitations under the License. */ #include "msgpack/unpack.h" -#include "unpack_context.h" +#include "msgpack/unpack_define.h" #include + +#define msgpack_unpack_struct(name) \ + struct template_##name + +#define msgpack_unpack_func(ret, name) \ + ret template_func_##name + +#define msgpack_unpack_callback(name) \ + template_callback_##name + +#define msgpack_unpack_object void* + +#define msgpack_unpack_user msgpack_unpack_t + + +struct template_context; + +static void template_func_init(struct template_context* ctx); + +static void* template_func_data(struct template_context* ctx); + +static int template_func_execute(struct template_context* ctx, + const char* data, size_t len, size_t* off); + + +static inline void* template_callback_init(msgpack_unpack_t* x) +{ return NULL; } + +static inline void* template_callback_uint8(msgpack_unpack_t* x, uint8_t d) +{ return x->callback.unpack_uint8(x->data, d); } + +static inline void* template_callback_uint16(msgpack_unpack_t* x, uint16_t d) +{ return x->callback.unpack_uint16(x->data, d); } + +static inline void* template_callback_uint32(msgpack_unpack_t* x, uint32_t d) +{ return x->callback.unpack_uint32(x->data, d); } + +static inline void* template_callback_uint64(msgpack_unpack_t* x, uint64_t d) +{ return x->callback.unpack_uint64(x->data, d); } + +static inline void* template_callback_int8(msgpack_unpack_t* x, int8_t d) +{ return x->callback.unpack_int8(x->data, d); } + +static inline void* template_callback_int16(msgpack_unpack_t* x, int16_t d) +{ return x->callback.unpack_int16(x->data, d); } + +static inline void* template_callback_int32(msgpack_unpack_t* x, int32_t d) +{ return x->callback.unpack_int32(x->data, d); } + +static inline void* template_callback_int64(msgpack_unpack_t* x, int64_t d) +{ return x->callback.unpack_int64(x->data, d); } + +static inline void* template_callback_float(msgpack_unpack_t* x, float d) +{ return x->callback.unpack_float(x->data, d); } + +static inline void* template_callback_double(msgpack_unpack_t* x, double d) +{ return x->callback.unpack_double(x->data, d); } + +static inline void* template_callback_nil(msgpack_unpack_t* x) +{ return x->callback.unpack_nil(x->data); } + +static inline void* template_callback_true(msgpack_unpack_t* x) +{ return x->callback.unpack_true(x->data); } + +static inline void* template_callback_false(msgpack_unpack_t* x) +{ return x->callback.unpack_false(x->data); } + +static inline void* template_callback_array(msgpack_unpack_t* x, unsigned int n) +{ return x->callback.unpack_array(x->data, n); } + +static inline void template_callback_array_item(msgpack_unpack_t* x, void* c, void* o) +{ x->callback.unpack_array_item(x->data, c, o); } + +static inline void* template_callback_map(msgpack_unpack_t* x, unsigned int n) +{ return x->callback.unpack_map(x->data, n); } + +static inline void template_callback_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) +{ x->callback.unpack_map_item(x->data, c, k, v); } + +static inline void* template_callback_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) +{ return x->callback.unpack_raw(x->data, b, p, l); } + + +#include "msgpack/unpack_template.h" + + msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback) { - msgpack_unpacker* ctx; - ctx = (msgpack_unpacker*)calloc(1, sizeof(msgpack_unpacker)); + struct template_context* ctx; + ctx = (struct template_context*)calloc(1, sizeof(struct template_context)); if(ctx == NULL) { return NULL; } - msgpack_unpacker_init(ctx); + template_func_init(ctx); ((msgpack_unpack_t*)ctx)->data = data; ((msgpack_unpack_t*)ctx)->callback = *callback; return (msgpack_unpack_t*)ctx; @@ -32,25 +118,27 @@ msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callba void msgpack_unpack_free(msgpack_unpack_t* ctx) { - free((msgpack_unpacker*)ctx); + free((struct template_context*)ctx); } void* msgpack_unpack_data(msgpack_unpack_t* ctx) { - return msgpack_unpacker_data((msgpack_unpacker*)ctx); + return template_func_data((struct template_context*)ctx); } void msgpack_unpack_reset(msgpack_unpack_t* ctx) { - msgpack_unpack_t x = ((msgpack_unpacker*)ctx)->user; - msgpack_unpacker_init((msgpack_unpacker*)ctx); - ((msgpack_unpacker*)ctx)->user = x; + msgpack_unpack_t x = ((struct template_context*)ctx)->user; + template_func_init((struct template_context*)ctx); + ((struct template_context*)ctx)->user = x; } -int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off) +int msgpack_unpack_execute(msgpack_unpack_t* ctx, + const char* data, size_t len, size_t* off) { - return msgpack_unpacker_execute( - (msgpack_unpacker*)ctx, + return template_func_execute( + (struct template_context*)ctx, data, len, off); } + diff --git a/c/unpack.h b/c/unpack.h index 6367439..c1cacab 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -21,24 +21,29 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct { - void* (*unpack_unsigned_int_8)(void* data, uint8_t d); - void* (*unpack_unsigned_int_16)(void* data, uint16_t d); - void* (*unpack_unsigned_int_32)(void* data, uint32_t d); - void* (*unpack_unsigned_int_64)(void* data, uint64_t d); - void* (*unpack_signed_int_8)(void* data, int8_t d); - void* (*unpack_signed_int_16)(void* data, int16_t d); - void* (*unpack_signed_int_32)(void* data, int32_t d); - void* (*unpack_signed_int_64)(void* data, int64_t d); + void* (*unpack_uint8)(void* data, uint8_t d); + void* (*unpack_uint16)(void* data, uint16_t d); + void* (*unpack_uint32)(void* data, uint32_t d); + void* (*unpack_uint64)(void* data, uint64_t d); + void* (*unpack_int8)(void* data, int8_t d); + void* (*unpack_int16)(void* data, int16_t d); + void* (*unpack_int32)(void* data, int32_t d); + void* (*unpack_int64)(void* data, int64_t d); void* (*unpack_float)(void* data, float d); void* (*unpack_double)(void* data, double d); void* (*unpack_nil)(void* data); void* (*unpack_true)(void* data); void* (*unpack_false)(void* data); - void* (*unpack_array_start)(void* data, unsigned int n); + void* (*unpack_array)(void* data, unsigned int n); void (*unpack_array_item)(void* data, void* c, void* o); - void* (*unpack_map_start)(void* data, unsigned int n); - void (*unpack_map_item)(void* data, void* c, void* k, void* v); + void* (*unpack_map)(void* data, unsigned int n); + void (*unpack_map_item)(void* data, void* c, void* k, void* v); void* (*unpack_raw)(void* data, const char* b, const char* p, unsigned int l); } msgpack_unpack_callback; @@ -51,8 +56,14 @@ msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callba void msgpack_unpack_free(msgpack_unpack_t* ctx); void msgpack_unpack_reset(msgpack_unpack_t* ctx); -int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off); +int msgpack_unpack_execute(msgpack_unpack_t* ctx, + const char* data, size_t len, size_t* off); void* msgpack_unpack_data(msgpack_unpack_t* ctx); + +#ifdef __cplusplus +} +#endif + #endif /* msgpack/unpack.h */ diff --git a/c/unpack_context.h b/c/unpack_context.h deleted file mode 100644 index d7b0388..0000000 --- a/c/unpack_context.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * MessagePack unpacking routine for C - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef UNPACK_CONTEXT_H__ -#define UNPACK_CONTEXT_H__ - -#include "msgpack/unpack.h" - -typedef void* msgpack_object; - -typedef msgpack_unpack_t msgpack_unpack_context; - -#include "msgpack/unpack/inline_context.h" - -#endif /* unpack_context.h */ - diff --git a/c/unpack_inline.c b/c/unpack_inline.c deleted file mode 100644 index 5282282..0000000 --- a/c/unpack_inline.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * MessagePack unpacking routine for C - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "unpack_context.h" - -static inline void* msgpack_unpack_init(msgpack_unpack_t* x) -{ return NULL; } - -static inline void* msgpack_unpack_unsigned_int_8(msgpack_unpack_t* x, uint8_t d) -{ return x->callback.unpack_unsigned_int_8(x->data, d); } - -static inline void* msgpack_unpack_unsigned_int_16(msgpack_unpack_t* x, uint16_t d) -{ return x->callback.unpack_unsigned_int_16(x->data, d); } - -static inline void* msgpack_unpack_unsigned_int_32(msgpack_unpack_t* x, uint32_t d) -{ return x->callback.unpack_unsigned_int_32(x->data, d); } - -static inline void* msgpack_unpack_unsigned_int_64(msgpack_unpack_t* x, uint64_t d) -{ return x->callback.unpack_unsigned_int_64(x->data, d); } - -static inline void* msgpack_unpack_signed_int_8(msgpack_unpack_t* x, int8_t d) -{ return x->callback.unpack_signed_int_8(x->data, d); } - -static inline void* msgpack_unpack_signed_int_16(msgpack_unpack_t* x, int16_t d) -{ return x->callback.unpack_signed_int_16(x->data, d); } - -static inline void* msgpack_unpack_signed_int_32(msgpack_unpack_t* x, int32_t d) -{ return x->callback.unpack_signed_int_32(x->data, d); } - -static inline void* msgpack_unpack_signed_int_64(msgpack_unpack_t* x, int64_t d) -{ return x->callback.unpack_signed_int_64(x->data, d); } - -static inline void* msgpack_unpack_float(msgpack_unpack_t* x, float d) -{ return x->callback.unpack_float(x->data, d); } - -static inline void* msgpack_unpack_double(msgpack_unpack_t* x, double d) -{ return x->callback.unpack_double(x->data, d); } - -static inline void* msgpack_unpack_nil(msgpack_unpack_t* x) -{ return x->callback.unpack_nil(x->data); } - -static inline void* msgpack_unpack_true(msgpack_unpack_t* x) -{ return x->callback.unpack_true(x->data); } - -static inline void* msgpack_unpack_false(msgpack_unpack_t* x) -{ return x->callback.unpack_false(x->data); } - -static inline void* msgpack_unpack_array_start(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_array_start(x->data, n); } - -static inline void msgpack_unpack_array_item(msgpack_unpack_t* x, void* c, void* o) -{ x->callback.unpack_array_item(x->data, c, o); } - -static inline void* msgpack_unpack_map_start(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_map_start(x->data, n); } - -static inline void msgpack_unpack_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) -{ x->callback.unpack_map_item(x->data, c, k, v); } - -static inline void* msgpack_unpack_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) -{ return x->callback.unpack_raw(x->data, b, p, l); } - - -#include "msgpack/unpack/inline_impl.h" - diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 08ea21f..aa22cc7 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -3,7 +3,6 @@ lib_LTLIBRARIES = libmsgpack.la libmsgpack_la_SOURCES = \ object.cpp \ unpack.cpp \ - unpack_inline.cpp \ zone.cpp nobase_include_HEADERS = \ @@ -14,13 +13,11 @@ nobase_include_HEADERS = \ msgpack/zone.hpp noinst_HEADERS = \ - unpack_context.hpp \ msgpack/zone.hpp.erb # FIXME object.lo: msgpack/zone.hpp unpack.lo: msgpack/zone.hpp -unpack_context.lo: msgpack/zone.hpp zone.lo: msgpack/zone.hpp msgpack/zone.hpp: msgpack/zone.hpp.erb diff --git a/cpp/bench.cpp b/cpp/bench.cpp index 4133a2e..517a870 100644 --- a/cpp/bench.cpp +++ b/cpp/bench.cpp @@ -6,10 +6,10 @@ #include #include -static const unsigned int TASK_INT_NUM = 1<<24; -static const unsigned int TASK_STR_LEN = 1<<15; -//static const unsigned int TASK_INT_NUM = 1<<23; -//static const unsigned int TASK_STR_LEN = 1<<14; +//static const unsigned int TASK_INT_NUM = 1<<24; +//static const unsigned int TASK_STR_LEN = 1<<15; +static const unsigned int TASK_INT_NUM = 1<<22; +static const unsigned int TASK_STR_LEN = 1<<13; static const char* TASK_STR_PTR; @@ -24,7 +24,7 @@ public: + (double)(endtime.tv_usec - m_timeval.tv_usec) / 1000 / 1000; std::cout << sec << " sec" << std::endl; std::cout << (double(bufsz)/1024/1024) << " MB" << std::endl; - std::cout << (bufsz/sec/1024/1024*8) << " Mbps" << std::endl; + std::cout << (bufsz/sec/1000/1000*8) << " Mbps" << std::endl; } private: timeval m_timeval; diff --git a/cpp/bench.mk b/cpp/bench.mk new file mode 100644 index 0000000..52a4b80 --- /dev/null +++ b/cpp/bench.mk @@ -0,0 +1,9 @@ + +CXXFLAGS += -Wall -g -I. -I.. -O4 +LDFLAGS += + +all: bench + +bench: bench.o unpack.o zone.o object.o pack.hpp unpack.hpp zone.hpp object.hpp + $(CXX) bench.o unpack.o zone.o object.o $(CFLAGS) $(LDFLAGS) -o $@ + diff --git a/cpp/object.cpp b/cpp/object.cpp index 7bb9841..ef2a68c 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -155,35 +155,35 @@ inline void numeric_pack(dynamic_packer& p, V v); template <> inline void numeric_pack(dynamic_packer& p, uint8_t v) - { p.pack_unsigned_int_8(v); } + { p.pack_uint8(v); } template <> inline void numeric_pack(dynamic_packer& p, uint16_t v) - { p.pack_unsigned_int_16(v); } + { p.pack_uint16(v); } template <> inline void numeric_pack(dynamic_packer& p, uint32_t v) - { p.pack_unsigned_int_32(v); } + { p.pack_uint32(v); } template <> inline void numeric_pack(dynamic_packer& p, uint64_t v) - { p.pack_unsigned_int_64(v); } + { p.pack_uint64(v); } template <> inline void numeric_pack(dynamic_packer& p, int8_t v) - { p.pack_unsigned_int_8(v); } + { p.pack_int8(v); } template <> inline void numeric_pack(dynamic_packer& p, int16_t v) - { p.pack_unsigned_int_16(v); } + { p.pack_int16(v); } template <> inline void numeric_pack(dynamic_packer& p, int32_t v) - { p.pack_unsigned_int_32(v); } + { p.pack_int32(v); } template <> inline void numeric_pack(dynamic_packer& p, int64_t v) - { p.pack_unsigned_int_64(v); } + { p.pack_int64(v); } template <> inline void numeric_pack(dynamic_packer& p, float v) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 9580679..f3eeb34 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -34,14 +34,14 @@ public: public: void pack_int(int d) { pack_int_impl(m_stream, d); } void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_unsigned_int_8(uint8_t d) { pack_unsigned_int_8_impl(m_stream, d); } - void pack_unsigned_int_16(uint16_t d) { pack_unsigned_int_16_impl(m_stream, d); } - void pack_unsigned_int_32(uint32_t d) { pack_unsigned_int_32_impl(m_stream, d); } - void pack_unsigned_int_64(uint64_t d) { pack_unsigned_int_64_impl(m_stream, d); } - void pack_signed_int_8(uint8_t d) { pack_signed_int_8_impl(m_stream, d); } - void pack_signed_int_16(uint16_t d) { pack_signed_int_16_impl(m_stream, d); } - void pack_signed_int_32(uint32_t d) { pack_signed_int_32_impl(m_stream, d); } - void pack_signed_int_64(uint64_t d) { pack_signed_int_64_impl(m_stream, d); } + void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } + void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } + void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } + void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } + void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } + void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } + void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } + void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } void pack_float(float d) { pack_float_impl(m_stream, d); } void pack_double(double d) { pack_double_impl(m_stream, d); } void pack_nil() { pack_nil(m_stream); } @@ -49,20 +49,19 @@ public: void pack_false() { pack_false(m_stream); } void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } - void pack_string(const char* b) { pack_string_impl(m_stream, b); } void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } private: static void pack_int_impl(Stream& x, int d); static void pack_unsigned_int_impl(Stream& x, unsigned int d); - static void pack_unsigned_int_8_impl(Stream& x, uint8_t d); - static void pack_unsigned_int_16_impl(Stream& x, uint16_t d); - static void pack_unsigned_int_32_impl(Stream& x, uint32_t d); - static void pack_unsigned_int_64_impl(Stream& x, uint64_t d); - static void pack_signed_int_8_impl(Stream& x, int8_t d); - static void pack_signed_int_16_impl(Stream& x, int16_t d); - static void pack_signed_int_32_impl(Stream& x, int32_t d); - static void pack_signed_int_64_impl(Stream& x, int64_t d); + static void pack_uint8_impl(Stream& x, uint8_t d); + static void pack_uint16_impl(Stream& x, uint16_t d); + static void pack_uint32_impl(Stream& x, uint32_t d); + static void pack_uint64_impl(Stream& x, uint64_t d); + static void pack_int8_impl(Stream& x, int8_t d); + static void pack_int16_impl(Stream& x, int16_t d); + static void pack_int32_impl(Stream& x, int32_t d); + static void pack_int64_impl(Stream& x, int64_t d); static void pack_float_impl(Stream& x, float d); static void pack_double_impl(Stream& x, double d); static void pack_nil_impl(Stream& x); @@ -70,7 +69,6 @@ private: static void pack_false_impl(Stream& x); static void pack_array_impl(Stream& x, unsigned int n); static void pack_map_impl(Stream& x, unsigned int n); - static void pack_string_impl(Stream& x, const char* b); static void pack_raw_impl(Stream& x, const void* b, size_t l); static void append_buffer(Stream& x, const unsigned char* buf, unsigned int len) { x.write((const char*)buf, len); } @@ -85,11 +83,10 @@ private: #define msgpack_pack_inline_func(name) \ template \ inline void packer::pack_ ## name ## _impl -#define msgpack_pack_context Stream& +#define msgpack_pack_user Stream& #define msgpack_pack_append_buffer append_buffer -#include "msgpack/pack/inline_impl.h" -#undef msgpack_pack_context -#undef msgpack_pack_append_buffer +#include "msgpack/pack_template.h" + template packer::packer(Stream& s) : m_stream(s) { } @@ -118,14 +115,14 @@ public: public: void pack_int(int d) { pack_int_impl(m_stream, d); } void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_unsigned_int_8(uint8_t d) { pack_unsigned_int_8_impl(m_stream, d); } - void pack_unsigned_int_16(uint16_t d) { pack_unsigned_int_16_impl(m_stream, d); } - void pack_unsigned_int_32(uint32_t d) { pack_unsigned_int_32_impl(m_stream, d); } - void pack_unsigned_int_64(uint64_t d) { pack_unsigned_int_64_impl(m_stream, d); } - void pack_signed_int_8(uint8_t d) { pack_signed_int_8_impl(m_stream, d); } - void pack_signed_int_16(uint16_t d) { pack_signed_int_16_impl(m_stream, d); } - void pack_signed_int_32(uint32_t d) { pack_signed_int_32_impl(m_stream, d); } - void pack_signed_int_64(uint64_t d) { pack_signed_int_64_impl(m_stream, d); } + void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } + void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } + void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } + void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } + void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } + void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } + void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } + void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } void pack_float(float d) { pack_float_impl(m_stream, d); } void pack_double(double d) { pack_double_impl(m_stream, d); } void pack_nil() { pack_nil_impl(m_stream); } @@ -139,14 +136,14 @@ public: private: static void pack_int_impl(dynamic_stream& x, int d); static void pack_unsigned_int_impl(dynamic_stream& x, unsigned int d); - static void pack_unsigned_int_8_impl(dynamic_stream& x, uint8_t d); - static void pack_unsigned_int_16_impl(dynamic_stream& x, uint16_t d); - static void pack_unsigned_int_32_impl(dynamic_stream& x, uint32_t d); - static void pack_unsigned_int_64_impl(dynamic_stream& x, uint64_t d); - static void pack_signed_int_8_impl(dynamic_stream& x, int8_t d); - static void pack_signed_int_16_impl(dynamic_stream& x, int16_t d); - static void pack_signed_int_32_impl(dynamic_stream& x, int32_t d); - static void pack_signed_int_64_impl(dynamic_stream& x, int64_t d); + static void pack_uint8_impl(dynamic_stream& x, uint8_t d); + static void pack_uint16_impl(dynamic_stream& x, uint16_t d); + static void pack_uint32_impl(dynamic_stream& x, uint32_t d); + static void pack_uint64_impl(dynamic_stream& x, uint64_t d); + static void pack_int8_impl(dynamic_stream& x, int8_t d); + static void pack_int16_impl(dynamic_stream& x, int16_t d); + static void pack_int32_impl(dynamic_stream& x, int32_t d); + static void pack_int64_impl(dynamic_stream& x, int64_t d); static void pack_float_impl(dynamic_stream& x, float d); static void pack_double_impl(dynamic_stream& x, double d); static void pack_nil_impl(dynamic_stream& x); @@ -166,14 +163,11 @@ private: dynamic_packer(); }; -#undef MSGPACK_PACK_INLINE_IMPL_H__ #define msgpack_pack_inline_func(name) \ inline void dynamic_packer::pack_ ## name ## _impl -#define msgpack_pack_context dynamic_stream& +#define msgpack_pack_user dynamic_stream& #define msgpack_pack_append_buffer append_buffer -#include "msgpack/pack/inline_impl.h" -#undef msgpack_pack_context -#undef msgpack_pack_append_buffer +#include "msgpack/pack_template.h" template dynamic_packer::dynamic_packer(Stream& s) : m_stream(s) { } diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index fe7f4b5..873d3da 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -16,12 +16,97 @@ // limitations under the License. // #include "msgpack/unpack.hpp" -#include "unpack_context.hpp" +#include "msgpack/unpack_define.h" #include namespace msgpack { +#define msgpack_unpack_struct(name) \ + struct msgpack_unpacker_##name + +#define msgpack_unpack_func(ret, name) \ + ret msgpack_unpacker_##name + +#define msgpack_unpack_callback(name) \ + msgpack_unpack_##name + +#define msgpack_unpack_object object_class* + +#define msgpack_unpack_user zone* + + +struct msgpack_unpacker_context; + +static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); + +static object_class* msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); + +static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, + const char* data, size_t len, size_t* off); + + +static inline object_class* msgpack_unpack_init(zone** z) +{ return NULL; } + +static inline object_class* msgpack_unpack_uint8(zone** z, uint8_t d) +{ return (*z)->nu8(d); } + +static inline object_class* msgpack_unpack_uint16(zone** z, uint16_t d) +{ return (*z)->nu16(d); } + +static inline object_class* msgpack_unpack_uint32(zone** z, uint32_t d) +{ return (*z)->nu32(d); } + +static inline object_class* msgpack_unpack_uint64(zone** z, uint64_t d) +{ return (*z)->nu64(d); } + +static inline object_class* msgpack_unpack_int8(zone** z, int8_t d) +{ return (*z)->ni8(d); } + +static inline object_class* msgpack_unpack_int16(zone** z, int16_t d) +{ return (*z)->ni16(d); } + +static inline object_class* msgpack_unpack_int32(zone** z, int32_t d) +{ return (*z)->ni32(d); } + +static inline object_class* msgpack_unpack_int64(zone** z, int64_t d) +{ return (*z)->ni64(d); } + +static inline object_class* msgpack_unpack_float(zone** z, float d) +{ return (*z)->nfloat(d); } + +static inline object_class* msgpack_unpack_double(zone** z, double d) +{ return (*z)->ndouble(d); } + +static inline object_class* msgpack_unpack_nil(zone** z) +{ return (*z)->nnil(); } + +static inline object_class* msgpack_unpack_true(zone** z) +{ return (*z)->ntrue(); } + +static inline object_class* msgpack_unpack_false(zone** z) +{ return (*z)->nfalse(); } + +static inline object_class* msgpack_unpack_array(zone** z, unsigned int n) +{ return (*z)->narray(n); } + +static inline void msgpack_unpack_array_item(zone** z, object_class* c, object_class* o) +{ reinterpret_cast(c)->push_back(o); } + +static inline object_class* msgpack_unpack_map(zone** z, unsigned int n) +{ return (*z)->nmap(); } + +static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) +{ reinterpret_cast(c)->store(k, v); } + +static inline object_class* msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) +{ return (*z)->nraw_ref(p, l); } + + +#include "msgpack/unpack_template.h" + + struct unpacker::context { context(zone* z) { @@ -65,7 +150,7 @@ struct unpacker::context { } private: - msgpack_unpacker m_ctx; + msgpack_unpacker_context m_ctx; private: context(); @@ -171,17 +256,26 @@ void unpacker::reset() } -object unpacker::unpack(const char* data, size_t len, zone& z) +object unpacker::unpack(const char* data, size_t len, zone& z, size_t* off) { context ctx(&z); - size_t off = 0; - int ret = ctx.execute(data, len, &off); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - throw unpack_error("insufficient bytes"); - } else if(off < len) { - throw unpack_error("extra bytes"); + if(off) { + int ret = ctx.execute(data, len, off); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + throw unpack_error("insufficient bytes"); + } + } else { + size_t noff = 0; + int ret = ctx.execute(data, len, &noff); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + throw unpack_error("insufficient bytes"); + } else if(noff < len) { + throw unpack_error("extra bytes"); + } } return ctx.data(); } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 473e8e9..86cfb6e 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -100,7 +100,7 @@ private: unpacker(const unpacker&); public: - static object unpack(const char* data, size_t len, zone& z); + static object unpack(const char* data, size_t len, zone& z, size_t* off = NULL); }; @@ -136,9 +136,9 @@ inline void unpacker::remove_nonparsed_buffer() { m_used = m_off; } -inline object unpack(const char* data, size_t len, zone& z) +inline object unpack(const char* data, size_t len, zone& z, size_t* off = NULL) { - return unpacker::unpack(data, len, z); + return unpacker::unpack(data, len, z, off); } diff --git a/cpp/unpack_context.hpp b/cpp/unpack_context.hpp deleted file mode 100644 index 59bd872..0000000 --- a/cpp/unpack_context.hpp +++ /dev/null @@ -1,31 +0,0 @@ -// -// MessagePack for C++ deserializing routine -// -// Copyright (C) 2008 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#ifndef UNPACK_CONTEXT_HPP__ -#define UNPACK_CONTEXT_HPP__ - -#include "msgpack/zone.hpp" -#include "msgpack/object.hpp" - -typedef msgpack::object_class* msgpack_object; - -typedef msgpack::zone* msgpack_unpack_context; - -#include "msgpack/unpack/inline_context.h" - -#endif /* unpack_context.h */ - diff --git a/cpp/unpack_inline.cpp b/cpp/unpack_inline.cpp deleted file mode 100644 index 40f2769..0000000 --- a/cpp/unpack_inline.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// MessagePack for C++ deserializing routine -// -// Copyright (C) 2008 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#include "unpack_context.hpp" - - -extern "C" { -using namespace msgpack; - - -static inline object_class* msgpack_unpack_init(zone** z) -{ return NULL; } - -static inline object_class* msgpack_unpack_unsigned_int_8(zone** z, uint8_t d) -{ return (*z)->nu8(d); } - -static inline object_class* msgpack_unpack_unsigned_int_16(zone** z, uint16_t d) -{ return (*z)->nu16(d); } - -static inline object_class* msgpack_unpack_unsigned_int_32(zone** z, uint32_t d) -{ return (*z)->nu32(d); } - -static inline object_class* msgpack_unpack_unsigned_int_64(zone** z, uint64_t d) -{ return (*z)->nu64(d); } - -static inline object_class* msgpack_unpack_signed_int_8(zone** z, int8_t d) -{ return (*z)->ni8(d); } - -static inline object_class* msgpack_unpack_signed_int_16(zone** z, int16_t d) -{ return (*z)->ni16(d); } - -static inline object_class* msgpack_unpack_signed_int_32(zone** z, int32_t d) -{ return (*z)->ni32(d); } - -static inline object_class* msgpack_unpack_signed_int_64(zone** z, int64_t d) -{ return (*z)->ni64(d); } - -static inline object_class* msgpack_unpack_float(zone** z, float d) -{ return (*z)->nfloat(d); } - -static inline object_class* msgpack_unpack_double(zone** z, double d) -{ return (*z)->ndouble(d); } - -static inline object_class* msgpack_unpack_nil(zone** z) -{ return (*z)->nnil(); } - -static inline object_class* msgpack_unpack_true(zone** z) -{ return (*z)->ntrue(); } - -static inline object_class* msgpack_unpack_false(zone** z) -{ return (*z)->nfalse(); } - -static inline object_class* msgpack_unpack_array_start(zone** z, unsigned int n) -{ return (*z)->narray(n); } - -static inline void msgpack_unpack_array_item(zone** z, object_class* c, object_class* o) -{ reinterpret_cast(c)->push_back(o); } - -static inline object_class* msgpack_unpack_map_start(zone** z, unsigned int n) -{ return (*z)->nmap(); } - -static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) -{ reinterpret_cast(c)->store(k, v); } - -static inline object_class* msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) -{ return (*z)->nraw_ref(p, l); } - - -} // extern "C" - -#include "msgpack/unpack/inline_impl.h" - diff --git a/msgpack/pack/inline_impl.h b/msgpack/pack_template.h similarity index 77% rename from msgpack/pack/inline_impl.h rename to msgpack/pack_template.h index 635b697..22d9911 100644 --- a/msgpack/pack/inline_impl.h +++ b/msgpack/pack_template.h @@ -1,5 +1,5 @@ /* - * MessagePack packing routine + * MessagePack packing routine template * * Copyright (C) 2008 FURUHASHI Sadayuki * @@ -15,8 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MSGPACK_PACK_INLINE_IMPL_H__ -#define MSGPACK_PACK_INLINE_IMPL_H__ #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) #if __BYTE_ORDER == __LITTLE_ENDIAN @@ -52,17 +50,26 @@ #endif + #ifndef msgpack_pack_inline_func -#define msgpack_pack_inline_func(name) \ - inline void msgpack_pack_##name +#error msgpack_pack_inline_func template is not defined #endif +#ifndef msgpack_pack_user +#error msgpack_pack_user type is not defined +#endif + +#ifndef msgpack_pack_append_buffer +#error msgpack_pack_append_buffer callback is not defined +#endif + + /* * Integer */ // wrapper -msgpack_pack_inline_func(int)(msgpack_pack_context x, int d) +msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) { if(d < -32) { if(d < -32768) { // signed 32 @@ -95,7 +102,7 @@ msgpack_pack_inline_func(int)(msgpack_pack_context x, int d) } // wrapper -msgpack_pack_inline_func(unsigned_int)(msgpack_pack_context x, unsigned int d) +msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) { if(d < 128) { // fixnum @@ -115,7 +122,7 @@ msgpack_pack_inline_func(unsigned_int)(msgpack_pack_context x, unsigned int d) } } -msgpack_pack_inline_func(unsigned_int_8)(msgpack_pack_context x, uint8_t d) +msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) { if(d < 128) { msgpack_pack_append_buffer(x, &d, 1); @@ -125,26 +132,26 @@ msgpack_pack_inline_func(unsigned_int_8)(msgpack_pack_context x, uint8_t d) } } -msgpack_pack_inline_func(unsigned_int_16)(msgpack_pack_context x, uint16_t d) +msgpack_pack_inline_func(uint16)(msgpack_pack_user x, uint16_t d) { const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func(unsigned_int_32)(msgpack_pack_context x, uint32_t d) +msgpack_pack_inline_func(uint32)(msgpack_pack_user x, uint32_t d) { const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(unsigned_int_64)(msgpack_pack_context x, uint64_t d) +msgpack_pack_inline_func(uint64)(msgpack_pack_user x, uint64_t d) { // FIXME optimization const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } -msgpack_pack_inline_func(signed_int_8)(msgpack_pack_context x, int8_t d) +msgpack_pack_inline_func(int8)(msgpack_pack_user x, int8_t d) { if(d > 0) { msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); @@ -156,19 +163,19 @@ msgpack_pack_inline_func(signed_int_8)(msgpack_pack_context x, int8_t d) } } -msgpack_pack_inline_func(signed_int_16)(msgpack_pack_context x, int16_t d) +msgpack_pack_inline_func(int16)(msgpack_pack_user x, int16_t d) { const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func(signed_int_32)(msgpack_pack_context x, int32_t d) +msgpack_pack_inline_func(int32)(msgpack_pack_user x, int32_t d) { const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(signed_int_64)(msgpack_pack_context x, int64_t d) +msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) { // FIXME optimization const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; @@ -180,14 +187,14 @@ msgpack_pack_inline_func(signed_int_64)(msgpack_pack_context x, int64_t d) * Float */ -msgpack_pack_inline_func(float)(msgpack_pack_context x, float d) +msgpack_pack_inline_func(float)(msgpack_pack_user x, float d) { uint32_t n = *((uint32_t*)&d); // FIXME const unsigned char buf[5] = {0xca, STORE_BE32(n)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(double)(msgpack_pack_context x, double d) +msgpack_pack_inline_func(double)(msgpack_pack_user x, double d) { uint64_t n = *((uint64_t*)&d); // FIXME const unsigned char buf[9] = {0xcb, STORE_BE64(n)}; @@ -199,7 +206,7 @@ msgpack_pack_inline_func(double)(msgpack_pack_context x, double d) * Nil */ -msgpack_pack_inline_func(nil)(msgpack_pack_context x) +msgpack_pack_inline_func(nil)(msgpack_pack_user x) { static const unsigned char d = 0xc0; msgpack_pack_append_buffer(x, &d, 1); @@ -210,13 +217,13 @@ msgpack_pack_inline_func(nil)(msgpack_pack_context x) * Boolean */ -msgpack_pack_inline_func(true)(msgpack_pack_context x) +msgpack_pack_inline_func(true)(msgpack_pack_user x) { static const unsigned char d = 0xc3; msgpack_pack_append_buffer(x, &d, 1); } -msgpack_pack_inline_func(false)(msgpack_pack_context x) +msgpack_pack_inline_func(false)(msgpack_pack_user x) { static const unsigned char d = 0xc2; msgpack_pack_append_buffer(x, &d, 1); @@ -227,7 +234,7 @@ msgpack_pack_inline_func(false)(msgpack_pack_context x) * Array */ -msgpack_pack_inline_func(array)(msgpack_pack_context x, unsigned int n) +msgpack_pack_inline_func(array)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x90 | n; @@ -248,7 +255,7 @@ msgpack_pack_inline_func(array)(msgpack_pack_context x, unsigned int n) * Map */ -msgpack_pack_inline_func(map)(msgpack_pack_context x, unsigned int n) +msgpack_pack_inline_func(map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; @@ -269,13 +276,7 @@ msgpack_pack_inline_func(map)(msgpack_pack_context x, unsigned int n) * Raw */ -msgpack_pack_inline_func(string)(msgpack_pack_context x, const char* b) -{ - uint32_t l = strlen(b); - msgpack_pack_append_buffer(x, (const unsigned char*)b, l+1); -} - -msgpack_pack_inline_func(raw)(msgpack_pack_context x, const void* b, size_t l) +msgpack_pack_inline_func(raw)(msgpack_pack_user x, const void* b, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; @@ -294,10 +295,10 @@ msgpack_pack_inline_func(raw)(msgpack_pack_context x, const void* b, size_t l) #undef msgpack_pack_inline_func +#undef msgpack_pack_user +#undef msgpack_pack_append_buffer #undef STORE_BE16 #undef STORE_BE32 #undef STORE_BE64 -#endif /* msgpack/pack/inline_impl.h */ - diff --git a/msgpack/unpack/callback.h b/msgpack/unpack/callback.h deleted file mode 100644 index 315bb9e..0000000 --- a/msgpack/unpack/callback.h +++ /dev/null @@ -1,24 +0,0 @@ - -msgpack_object msgpack_unpack_init(msgpack_unpack_context* x); -msgpack_object msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d); -msgpack_object msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d); -msgpack_object msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d); -msgpack_object msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d); -msgpack_object msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d); -msgpack_object msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d); -msgpack_object msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d); -msgpack_object msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d); -msgpack_object msgpack_unpack_float(msgpack_unpack_context* x, float d); -msgpack_object msgpack_unpack_double(msgpack_unpack_context* x, double d); -msgpack_object msgpack_unpack_big_int(msgpack_unpack_context* x, const void* b, unsigned int l); -msgpack_object msgpack_unpack_big_float(msgpack_unpack_context* x, const void* b, unsigned int l); -msgpack_object msgpack_unpack_nil(msgpack_unpack_context* x); -msgpack_object msgpack_unpack_true(msgpack_unpack_context* x); -msgpack_object msgpack_unpack_false(msgpack_unpack_context* x); -msgpack_object msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n); - void msgpack_unpack_array_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object o); -msgpack_object msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n); - void msgpack_unpack_map_item(msgpack_unpack_context* x, msgpack_object c, msgpack_object k, msgpack_object v); -msgpack_object msgpack_unpack_string(msgpack_unpack_context* x, const void* b, size_t l); -msgpack_object msgpack_unpack_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l); - diff --git a/msgpack/unpack/inline_context.h b/msgpack/unpack/inline_context.h deleted file mode 100644 index e764c09..0000000 --- a/msgpack/unpack/inline_context.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * MessagePack unpacking routine - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MSGPACK_UNPACK_INLINE_CONTEXT_H__ -#define MSGPACK_UNPACK_INLINE_CONTEXT_H__ - -#include -#include - -#ifndef MSG_STACK_SIZE -#define MSG_STACK_SIZE 16 -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - msgpack_object obj; - size_t count; - unsigned int ct; - union { - /*const unsigned char* terminal_trail_start;*/ - msgpack_object map_key; - } tmp; -} msgpack_unpacker_stack; - -typedef struct { - msgpack_unpack_context user; // must be first - unsigned int cs; - unsigned int trail; - unsigned int top; - msgpack_unpacker_stack stack[MSG_STACK_SIZE]; -} msgpack_unpacker; - -void msgpack_unpacker_init(msgpack_unpacker* ctx); -int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len, size_t* off); -#define msgpack_unpacker_data(unpacker) (unpacker)->stack[0].obj - -#ifdef __cplusplus -} -#endif - -#endif /* msgpack/unpack/inline_context.h */ - diff --git a/msgpack/unpack/inline_impl.h b/msgpack/unpack/inline_impl.h deleted file mode 100644 index a6557f9..0000000 --- a/msgpack/unpack/inline_impl.h +++ /dev/null @@ -1,465 +0,0 @@ -/* - * MessagePack unpacking routine - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MSGPACK_UNPACK_INLINE_IMPL_H__ -#define MSGPACK_UNPACK_INLINE_IMPL_H__ - -#include -#include -#include -/*#include */ - -#ifdef __cplusplus -extern "C" { -#endif - -// Positive FixNum 0xxxxxxx 0x00 - 0x7f -// Negative FixNum 111xxxxx 0xe0 - 0xff -// Variable 110xxxxx 0xc0 - 0xdf -// nil 00000 0xc0 -// string 00001 0xc1 -// false 00010 0xc2 -// true 00011 0xc3 -// (?) 00100 0xc4 -// (?) 00101 0xc5 -// (?) 00110 0xc6 -// (?) 00111 0xc7 -// (?) 01000 0xc8 -// (?) 01001 0xc9 -// float 01010 0xca -// double 01011 0xcb -// uint 8 01100 0xcc -// uint 16 01101 0xcd -// uint 32 01110 0xce -// uint 64 01111 0xcf -// int 8 10000 0xd0 -// int 16 10001 0xd1 -// int 32 10010 0xd2 -// int 64 10011 0xd3 -// (?) 10100 0xd4 -// (?) 10101 0xd5 -// (big float 16) 10110 0xd6 -// (big float 32) 10111 0xd7 -// (big integer 16) 11000 0xd8 -// (big integer 32) 11001 0xd9 -// raw 16 11010 0xda -// raw 32 11011 0xdb -// array 16 11100 0xdc -// array 32 11101 0xdd -// map 16 11110 0xde -// map 32 11111 0xdf -// FixRaw 101xxxxx 0xa0 - 0xbf -// FixArray 1001xxxx 0x90 - 0x9f -// FixMap 1000xxxx 0x80 - 0x8f - - -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - -#define betoh16(x) ntohs(x) -#define betoh32(x) ntohl(x) - -#ifdef __LITTLE_ENDIAN__ -#if defined(__bswap_64) -# define betoh64(x) __bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define betoh64(x) __DARWIN_OSSwapInt64(x) -#else -static inline uint64_t betoh64(uint64_t x) { - return ((x << 56) & 0xff00000000000000ULL ) | - ((x << 40) & 0x00ff000000000000ULL ) | - ((x << 24) & 0x0000ff0000000000ULL ) | - ((x << 8) & 0x000000ff00000000ULL ) | - ((x >> 8) & 0x00000000ff000000ULL ) | - ((x >> 24) & 0x0000000000ff0000ULL ) | - ((x >> 40) & 0x000000000000ff00ULL ) | - ((x >> 56) & 0x00000000000000ffULL ) ; -} -#endif -#else -#define betoh64(x) (x) -#endif - - -typedef enum { - CS_HEADER = 0x00, // nil - - //CS_STRING = 0x01, - //CS_ = 0x02, // false - //CS_ = 0x03, // true - - //CS_ = 0x04, - //CS_ = 0x05, - //CS_ = 0x06, - //CS_ = 0x07, - - //CS_ = 0x08, - //CS_ = 0x09, - CS_FLOAT = 0x0a, - CS_DOUBLE = 0x0b, - CS_UNSIGNED_INT_8 = 0x0c, - CS_UNSIGNED_INT_16 = 0x0d, - CS_UNSIGNED_INT_32 = 0x0e, - CS_UNSIGNED_INT_64 = 0x0f, - CS_SIGNED_INT_8 = 0x10, - CS_SIGNED_INT_16 = 0x11, - CS_SIGNED_INT_32 = 0x12, - CS_SIGNED_INT_64 = 0x13, - - //CS_ = 0x14, - //CS_ = 0x15, - //CS_BIG_INT_16 = 0x16, - //CS_BIG_INT_32 = 0x17, - //CS_BIG_FLOAT_16 = 0x18, - //CS_BIG_FLOAT_32 = 0x19, - CS_RAW_16 = 0x1a, - CS_RAW_32 = 0x1b, - CS_ARRAY_16 = 0x1c, - CS_ARRAY_32 = 0x1d, - CS_MAP_16 = 0x1e, - CS_MAP_32 = 0x1f, - - //ACS_BIG_INT_VALUE, - //ACS_BIG_FLOAT_VALUE, - ACS_RAW_VALUE, -} current_state_t; - - -typedef enum { - CT_ARRAY_ITEM, - CT_MAP_KEY, - CT_MAP_VALUE, -} container_type_t; - - -void msgpack_unpacker_init(msgpack_unpacker* ctx) -{ - memset(ctx, 0, sizeof(msgpack_unpacker)); // FIXME init ctx->user? - ctx->cs = CS_HEADER; - ctx->trail = 0; - ctx->top = 0; - ctx->stack[0].obj = msgpack_unpack_init(&ctx->user); -} - -int msgpack_unpacker_execute(msgpack_unpacker* ctx, const char* data, size_t len, size_t* off) -{ - assert(len >= *off); - - const unsigned char* p = (unsigned char*)data + *off; - const unsigned char* const pe = (unsigned char*)data + len; - const void* n = NULL; - - unsigned int trail = ctx->trail; - unsigned int cs = ctx->cs; - unsigned int top = ctx->top; - msgpack_unpacker_stack* stack = ctx->stack; - msgpack_unpack_context* user = &ctx->user; - - msgpack_object obj; - msgpack_unpacker_stack* c = NULL; - - int ret; - -#define push_simple_value(func) \ - obj = func(user); \ - /*printf("obj %d\n",obj);*/ \ - goto _push -#define push_fixed_value(func, arg) \ - obj = func(user, arg); \ - /*printf("obj %d\n",obj);*/ \ - goto _push -#define push_variable_value(func, base, pos, len) \ - obj = func(user, (const char*)base, (const char*)pos, len); \ - /*printf("obj %d\n",obj);*/ \ - goto _push - -/* -#define again_terminal_trail(_cs, from) \ - cs = _cs; \ - stack[top].tmp.terminal_trail_start = from; \ - goto _terminal_trail_again -*/ -#define again_fixed_trail(_cs, trail_len) \ - trail = trail_len; \ - cs = _cs; \ - goto _fixed_trail_again -#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ - trail = trail_len; \ - if(trail == 0) { goto ifzero; } \ - cs = _cs; \ - goto _fixed_trail_again - -#define start_container(func, count_, ct_) \ - stack[top].obj = func(user, count_); \ - if((count_) == 0) { obj = stack[top].obj; goto _push; } \ - if(top >= MSG_STACK_SIZE) { goto _failed; } \ - stack[top].ct = ct_; \ - stack[top].count = count_; \ - /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ - /*printf("stack push %d\n", top);*/ \ - ++top; \ - goto _header_again - -#define NEXT_CS(p) \ - ((unsigned int)*p & 0x1f) - -#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) betoh16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) betoh32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) betoh64(*(uint64_t*)ptr) - - if(p == pe) { goto _out; } - do { - switch(cs) { - case CS_HEADER: - switch(*p) { - case 0x00 ... 0x7f: // Positive Fixnum - push_fixed_value(msgpack_unpack_unsigned_int_8, *(uint8_t*)p); - case 0xe0 ... 0xff: // Negative Fixnum - push_fixed_value(msgpack_unpack_signed_int_8, *(int8_t*)p); - case 0xc0 ... 0xdf: // Variable - switch(*p) { - case 0xc0: // nil - push_simple_value(msgpack_unpack_nil); - //case 0xc1: // string - // again_terminal_trail(NEXT_CS(p), p+1); - case 0xc2: // false - push_simple_value(msgpack_unpack_false); - case 0xc3: // true - push_simple_value(msgpack_unpack_true); - //case 0xc4: - //case 0xc5: - //case 0xc6: - //case 0xc7: - //case 0xc8: - //case 0xc9: - case 0xca: // float - case 0xcb: // double - case 0xcc: // unsigned int 8 - case 0xcd: // unsigned int 16 - case 0xce: // unsigned int 32 - case 0xcf: // unsigned int 64 - case 0xd0: // signed int 8 - case 0xd1: // signed int 16 - case 0xd2: // signed int 32 - case 0xd3: // signed int 64 - again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); - //case 0xd4: - //case 0xd5: - //case 0xd6: // big integer 16 - //case 0xd7: // big integer 32 - //case 0xd8: // big float 16 - //case 0xd9: // big float 32 - case 0xda: // raw 16 - case 0xdb: // raw 32 - case 0xdc: // array 16 - case 0xdd: // array 32 - case 0xde: // map 16 - case 0xdf: // map 32 - again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); - default: - goto _failed; - } - case 0xa0 ... 0xbf: // FixRaw - again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - case 0x90 ... 0x9f: // FixArray - start_container(msgpack_unpack_array_start, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - case 0x80 ... 0x8f: // FixMap - start_container(msgpack_unpack_map_start, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - - default: - goto _failed; - } - // end CS_HEADER - - - //_terminal_trail_again: - // ++p; - - //case CS_STRING: - // if(*p == 0) { - // const unsigned char* start = stack[top].tmp.terminal_trail_start; - // obj = msgpack_unpack_string(user, start, p-start); - // goto _push; - // } - // goto _terminal_trail_again; - - - _fixed_trail_again: - ++p; - - default: - if((size_t)(pe - p) < trail) { goto _out; } - n = p; p += trail - 1; - switch(cs) { - //case CS_ - //case CS_ - case CS_FLOAT: { - uint32_t x = PTR_CAST_32(n); // FIXME - push_fixed_value(msgpack_unpack_float, *((float*)&x)); } - case CS_DOUBLE: { - uint64_t x = PTR_CAST_64(n); // FIXME - push_fixed_value(msgpack_unpack_double, *((double*)&x)); } - case CS_UNSIGNED_INT_8: - push_fixed_value(msgpack_unpack_unsigned_int_8, (uint8_t)PTR_CAST_8(n)); - case CS_UNSIGNED_INT_16: - push_fixed_value(msgpack_unpack_unsigned_int_16, (uint16_t)PTR_CAST_16(n)); - case CS_UNSIGNED_INT_32: - push_fixed_value(msgpack_unpack_unsigned_int_32, (uint32_t)PTR_CAST_32(n)); - case CS_UNSIGNED_INT_64: - push_fixed_value(msgpack_unpack_unsigned_int_64, (uint64_t)PTR_CAST_64(n)); - - case CS_SIGNED_INT_8: - push_fixed_value(msgpack_unpack_signed_int_8, (int8_t)PTR_CAST_8(n)); - case CS_SIGNED_INT_16: - push_fixed_value(msgpack_unpack_signed_int_16, (int16_t)PTR_CAST_16(n)); - case CS_SIGNED_INT_32: - push_fixed_value(msgpack_unpack_signed_int_32, (int32_t)PTR_CAST_32(n)); - case CS_SIGNED_INT_64: - push_fixed_value(msgpack_unpack_signed_int_64, (int64_t)PTR_CAST_64(n)); - - //case CS_ - //case CS_ - //case CS_BIG_INT_16: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); - //case CS_BIG_INT_32: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); - //case ACS_BIG_INT_VALUE: - //_big_int_zero: - // // FIXME - // push_variable_value(msgpack_unpack_big_int, data, n, trail); - - //case CS_BIG_FLOAT_16: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); - //case CS_BIG_FLOAT_32: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); - //case ACS_BIG_FLOAT_VALUE: - //_big_float_zero: - // // FIXME - // push_variable_value(msgpack_unpack_big_float, data, n, trail); - - case CS_RAW_16: - again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); - case CS_RAW_32: - again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); - case ACS_RAW_VALUE: - _raw_zero: - push_variable_value(msgpack_unpack_raw, data, n, trail); - - case CS_ARRAY_16: - start_container(msgpack_unpack_array_start, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); - case CS_ARRAY_32: - start_container(msgpack_unpack_array_start, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); - - case CS_MAP_16: - start_container(msgpack_unpack_map_start, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); - case CS_MAP_32: - start_container(msgpack_unpack_map_start, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); - - default: - goto _failed; - } - } - -_push: - if(top == 0) { goto _finish; } - c = &stack[top-1]; - switch(c->ct) { - case CT_ARRAY_ITEM: - msgpack_unpack_array_item(user, c->obj, obj); - if(--c->count == 0) { - obj = c->obj; - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - goto _header_again; - case CT_MAP_KEY: - c->tmp.map_key = obj; - c->ct = CT_MAP_VALUE; - goto _header_again; - case CT_MAP_VALUE: - msgpack_unpack_map_item(user, c->obj, c->tmp.map_key, obj); - if(--c->count == 0) { - obj = c->obj; - --top; - /*printf("stack pop %d\n", top);*/ - goto _push; - } - c->ct = CT_MAP_KEY; - goto _header_again; - - default: - goto _failed; - } - -_header_again: - cs = CS_HEADER; - ++p; - } while(p != pe); - goto _out; - - -_finish: - stack[0].obj = obj; - ++p; - ret = 1; - /*printf("-- finish --\n"); */ - goto _end; - -_failed: - /*printf("** FAILED **\n"); */ - ret = -1; - goto _end; - -_out: - ret = 0; - goto _end; - -_end: - ctx->cs = cs; - ctx->trail = trail; - ctx->top = top; - *off = p - (const unsigned char*)data; - - return ret; -} - - -#ifdef betoh16 -#undef betoh16 -#endif - -#ifdef betoh32 -#undef betoh32 -#endif - -#ifdef betoh64 -#undef betoh64 -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* msgpack/unpack/inline_impl.h */ - diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h new file mode 100644 index 0000000..1d9db19 --- /dev/null +++ b/msgpack/unpack_define.h @@ -0,0 +1,127 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_DEFINE_H__ +#define MSGPACK_UNPACK_DEFINE_H__ + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_MAX_STACK_SIZE +#define MSGPACK_MAX_STACK_SIZE 16 +#endif + + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#define msgpack_betoh16(x) ntohs(x) +#define msgpack_betoh32(x) ntohl(x) + +#ifdef __LITTLE_ENDIAN__ +#if defined(__bswap_64) +# define msgpack_betoh64(x) __bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) +#else +static inline uint64_t msgpack_betoh64(uint64_t x) { + return ((x << 56) & 0xff00000000000000ULL ) | + ((x << 40) & 0x00ff000000000000ULL ) | + ((x << 24) & 0x0000ff0000000000ULL ) | + ((x << 8) & 0x000000ff00000000ULL ) | + ((x >> 8) & 0x00000000ff000000ULL ) | + ((x >> 24) & 0x0000000000ff0000ULL ) | + ((x >> 40) & 0x000000000000ff00ULL ) | + ((x >> 56) & 0x00000000000000ffULL ) ; +} +#endif +#else +#define msgpack_betoh64(x) (x) +#endif + + +typedef enum { + CS_HEADER = 0x00, // nil + + //CS_ = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true + + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, + + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UINT_8 = 0x0c, + CS_UINT_16 = 0x0d, + CS_UINT_32 = 0x0e, + CS_UINT_64 = 0x0f, + CS_INT_8 = 0x10, + CS_INT_16 = 0x11, + CS_INT_32 = 0x12, + CS_INT_64 = 0x13, + + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, + + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, +} msgpack_unpack_state; + + +typedef enum { + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, +} msgpack_container_type; + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/unpack_define.h */ + diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h new file mode 100644 index 0000000..8197fa4 --- /dev/null +++ b/msgpack/unpack_template.h @@ -0,0 +1,360 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef msgpack_unpack_func +#error msgpack_unpack_func template is not defined +#endif + +#ifndef msgpack_unpack_callback +#error msgpack_unpack_callback template is not defined +#endif + +#ifndef msgpack_unpack_struct +#error msgpack_unpack_struct template is not defined +#endif + +#ifndef msgpack_unpack_struct_decl +#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name) +#endif + +#ifndef msgpack_unpack_object +#error msgpack_unpack_object type is not defined +#endif + +#ifndef msgpack_unpack_user +#error msgpack_unpack_user type is not defined +#endif + + +msgpack_unpack_struct_decl(stack) { + msgpack_unpack_object obj; + size_t count; + unsigned int ct; + msgpack_unpack_object map_key; +}; + +msgpack_unpack_struct_decl(context) { + msgpack_unpack_user user; // must be first + unsigned int cs; + unsigned int trail; + unsigned int top; + msgpack_unpack_struct(stack) stack[MSGPACK_MAX_STACK_SIZE]; +}; + + +msgpack_unpack_func(void, init)(msgpack_unpack_struct(context)* ctx) +{ + /*memset(ctx, 0, sizeof( msgpack_unpack_struct(context) )); FIXME needed? */ + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + ctx->stack[0].obj = msgpack_unpack_callback(init)(&ctx->user); +} + +msgpack_unpack_func(msgpack_unpack_object, data)(msgpack_unpack_struct(context)* unpacker) +{ + return (unpacker)->stack[0].obj; +} + + +msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; + + unsigned int trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + msgpack_unpack_struct(stack)* stack = ctx->stack; + msgpack_unpack_user* user = &ctx->user; + + msgpack_unpack_object obj; + msgpack_unpack_struct(stack)* c = NULL; + + int ret; + +#define push_simple_value(func) \ + obj = msgpack_unpack_callback(func)(user); \ + /*printf("obj %d\n",obj);*/ \ + goto _push +#define push_fixed_value(func, arg) \ + obj = msgpack_unpack_callback(func)(user, arg); \ + /*printf("obj %d\n",obj);*/ \ + goto _push +#define push_variable_value(func, base, pos, len) \ + obj = msgpack_unpack_callback(func)(user, (const char*)base, (const char*)pos, len); \ + /*printf("obj %d\n",obj);*/ \ + goto _push + +#define again_fixed_trail(_cs, trail_len) \ + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again +#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again + +#define start_container(func, count_, ct_) \ + stack[top].obj = msgpack_unpack_callback(func)(user, count_); \ + if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ + stack[top].ct = ct_; \ + stack[top].count = count_; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + ++top; \ + goto _header_again + +#define NEXT_CS(p) \ + ((unsigned int)*p & 0x1f) + +#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) + + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + switch(*p) { + case 0x00 ... 0x7f: // Positive Fixnum + push_fixed_value(uint8, *(uint8_t*)p); + case 0xe0 ... 0xff: // Negative Fixnum + push_fixed_value(int8, *(int8_t*)p); + case 0xc0 ... 0xdf: // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(nil); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(false); + case 0xc3: // true + push_simple_value(true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + case 0xa0 ... 0xbf: // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + case 0x90 ... 0x9f: // FixArray + start_container(array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + case 0x80 ... 0x8f: // FixMap + start_container(map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + + default: + goto _failed; + } + // end CS_HEADER + + + _fixed_trail_again: + ++p; + + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + uint32_t x = PTR_CAST_32(n); // FIXME + push_fixed_value(float, *((float*)&x)); } + case CS_DOUBLE: { + uint64_t x = PTR_CAST_64(n); // FIXME + push_fixed_value(double, *((double*)&x)); } + case CS_UINT_8: + push_fixed_value(uint8, (uint8_t)PTR_CAST_8(n)); + case CS_UINT_16: + push_fixed_value(uint16, (uint16_t)PTR_CAST_16(n)); + case CS_UINT_32: + push_fixed_value(uint32, (uint32_t)PTR_CAST_32(n)); + case CS_UINT_64: + push_fixed_value(uint64, (uint64_t)PTR_CAST_64(n)); + + case CS_INT_8: + push_fixed_value(int8, (int8_t)PTR_CAST_8(n)); + case CS_INT_16: + push_fixed_value(int16, (int16_t)PTR_CAST_16(n)); + case CS_INT_32: + push_fixed_value(int32, (int32_t)PTR_CAST_32(n)); + case CS_INT_64: + push_fixed_value(int64, (int64_t)PTR_CAST_64(n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(big_int, data, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(big_float, data, n, trail); + + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(raw, data, n, trail); + + case CS_ARRAY_16: + start_container(array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + start_container(array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + + case CS_MAP_16: + start_container(map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + case CS_MAP_32: + start_container(map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + + default: + goto _failed; + } + } + +_push: + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + msgpack_unpack_callback(array_item)(user, c->obj, obj); + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + msgpack_unpack_callback(map_item)(user, c->obj, c->map_key, obj); + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } + +_header_again: + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; + + +_finish: + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; + +_failed: + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; + +_out: + ret = 0; + goto _end; + +_end: + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; + + return ret; +} + + +#undef msgpack_unpack_func +#undef msgpack_unpack_callback +#undef msgpack_unpack_struct +#undef msgpack_unpack_object +#undef msgpack_unpack_user + +#undef push_simple_value +#undef push_fixed_value +#undef push_variable_value +#undef again_fixed_trail +#undef again_fixed_trail_if_zero +#undef start_container + +#undef NEXT_CS +#undef PTR_CAST_8 +#undef PTR_CAST_16 +#undef PTR_CAST_32 +#undef PTR_CAST_64 + diff --git a/ruby/gem/Manifest.txt b/ruby/gem/Manifest.txt index 299a4e0..388135e 100644 --- a/ruby/gem/Manifest.txt +++ b/ruby/gem/Manifest.txt @@ -7,15 +7,12 @@ config/requirements.rb ext/extconf.rb ext/pack.c ext/pack.h -ext/pack_inline.h ext/rbinit.c ext/unpack.c ext/unpack.h -ext/unpack_context.h -ext/unpack_inline.c -msgpack/pack/inline_impl.h -msgpack/unpack/inline_context.h -msgpack/unpack/inline_impl.h +msgpack/pack_template.h +msgpack/unpack_define.h +msgpack/unpack_template.h lib/msgpack/version.rb script/console script/destroy diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb index c65972f..b2a5db6 100644 --- a/ruby/gem/lib/msgpack/version.rb +++ b/ruby/gem/lib/msgpack/version.rb @@ -1,7 +1,7 @@ module MessagePack module VERSION #:nodoc: MAJOR = 0 - MINOR = 1 + MINOR = 2 TINY = 0 STRING = [MAJOR, MINOR, TINY].join('.') diff --git a/ruby/gengem.sh b/ruby/gengem.sh index 4f8623c..c8abcc2 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -3,16 +3,13 @@ cp extconf.rb gem/ext/ cp pack.c gem/ext/ cp pack.h gem/ext/ -cp pack_inline.h gem/ext/ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ -cp unpack_context.h gem/ext/ -cp unpack_inline.c gem/ext/ cp ../README gem/README.txt -cp ../msgpack/pack/inline_impl.h gem/msgpack/pack/ -cp ../msgpack/unpack/inline_context.h gem/msgpack/unpack/ -cp ../msgpack/unpack/inline_impl.h gem/msgpack/unpack/ +cp ../msgpack/pack_template.h gem/msgpack/ +cp ../msgpack/unpack_define.h gem/msgpack/ +cp ../msgpack/unpack_template.h gem/msgpack/ cd gem && rake --trace package diff --git a/ruby/pack.c b/ruby/pack.c index 3d71776..54f610c 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -15,7 +15,41 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "pack_inline.h" +#include "ruby.h" +#include +#include + +#define msgpack_pack_inline_func(name) \ + static void msgpack_pack_##name + +#define msgpack_pack_user VALUE + +#define msgpack_pack_append_buffer(user, buf, len) \ + rb_str_buf_cat(user, (const void*)buf, len) + +/* +static void msgpack_pack_int(VALUE x, int d); +static void msgpack_pack_unsigned_int(VALUE x, unsigned int d); +static void msgpack_pack_uint8(VALUE x, uint8_t d); +static void msgpack_pack_uint16(VALUE x, uint16_t d); +static void msgpack_pack_uint32(VALUE x, uint32_t d); +static void msgpack_pack_uint64(VALUE x, uint64_t d); +static void msgpack_pack_int8(VALUE x, int8_t d); +static void msgpack_pack_int16(VALUE x, int16_t d); +static void msgpack_pack_int32(VALUE x, int32_t d); +static void msgpack_pack_int64(VALUE x, int64_t d); +static void msgpack_pack_float(VALUE x, float d); +static void msgpack_pack_double(VALUE x, double d); +static void msgpack_pack_nil(VALUE x); +static void msgpack_pack_true(VALUE x); +static void msgpack_pack_false(VALUE x); +static void msgpack_pack_array(VALUE x, unsigned int n); +static void msgpack_pack_map(VALUE x, unsigned int n); +static void msgpack_pack_raw(VALUE x, const void* b, size_t l); +*/ + +#include "msgpack/pack_template.h" + #ifndef RUBY_VM #include "st.h" // ruby hash @@ -72,9 +106,9 @@ static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) ARG_BUFFER(out, argc, argv); // FIXME bignum if(RBIGNUM_SIGN(self)) { // positive - msgpack_pack_unsigned_int_64(out, rb_big2ull(self)); + msgpack_pack_uint64(out, rb_big2ull(self)); } else { // negative - msgpack_pack_signed_int_64(out, rb_big2ll(self)); + msgpack_pack_int64(out, rb_big2ll(self)); } return out; } diff --git a/ruby/pack_inline.h b/ruby/pack_inline.h deleted file mode 100644 index bda74c3..0000000 --- a/ruby/pack_inline.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * MessagePack packing routine for Ruby - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef PACK_INLINE_H__ -#define PACK_INLINE_H__ - -#include "ruby.h" - -typedef VALUE msgpack_pack_context; - -static inline void msgpack_pack_append_buffer(VALUE x, const unsigned char* b, unsigned int l) -{ - rb_str_buf_cat(x, (const void*)b, l); -} - -#include -#include /* __BYTE_ORDER */ -#include "msgpack/pack/inline_impl.h" - -#endif /* pack_inline.h */ - diff --git a/ruby/unpack.c b/ruby/unpack.c index fa2996d..8439c02 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -16,8 +16,99 @@ * limitations under the License. */ #include "ruby.h" -#include "unpack_context.h" -#include +#include "msgpack/unpack_define.h" + + +typedef struct { + int finished; +} msgpack_unpack_context; + + +#define msgpack_unpack_struct(name) \ + struct msgpack_unpacker_##name + +#define msgpack_unpack_func(ret, name) \ + ret msgpack_unpacker_##name + +#define msgpack_unpack_callback(name) \ + template_callback_##name + +#define msgpack_unpack_object VALUE + +#define msgpack_unpack_user msgpack_unpack_context + + +struct msgpack_unpacker_context; +typedef struct msgpack_unpacker_context msgpack_unpacker; + +static void msgpack_unpacker_init(msgpack_unpacker* ctx); + +static VALUE msgpack_unpacker_data(msgpack_unpacker* ctx); + +static int msgpack_unpacker_execute(msgpack_unpacker* ctx, + const char* data, size_t len, size_t* off); + + +static inline VALUE template_callback_init(msgpack_unpack_context* x) +{ return Qnil; } + +static inline VALUE template_callback_uint8(msgpack_unpack_context* x, uint8_t d) +{ return INT2FIX(d); } + +static inline VALUE template_callback_uint16(msgpack_unpack_context* x, uint16_t d) +{ return INT2FIX(d); } + +static inline VALUE template_callback_uint32(msgpack_unpack_context* x, uint32_t d) +{ return UINT2NUM(d); } + +static inline VALUE template_callback_uint64(msgpack_unpack_context* x, uint64_t d) +{ return rb_ull2inum(d); } + +static inline VALUE template_callback_int8(msgpack_unpack_context* x, int8_t d) +{ return INT2FIX((long)d); } + +static inline VALUE template_callback_int16(msgpack_unpack_context* x, int16_t d) +{ return INT2FIX((long)d); } + +static inline VALUE template_callback_int32(msgpack_unpack_context* x, int32_t d) +{ return INT2NUM((long)d); } + +static inline VALUE template_callback_int64(msgpack_unpack_context* x, int64_t d) +{ return rb_ll2inum(d); } + +static inline VALUE template_callback_float(msgpack_unpack_context* x, float d) +{ return rb_float_new(d); } + +static inline VALUE template_callback_double(msgpack_unpack_context* x, double d) +{ return rb_float_new(d); } + +static inline VALUE template_callback_nil(msgpack_unpack_context* x) +{ return Qnil; } + +static inline VALUE template_callback_true(msgpack_unpack_context* x) +{ return Qtrue; } + +static inline VALUE template_callback_false(msgpack_unpack_context* x) +{ return Qfalse; } + +static inline VALUE template_callback_array(msgpack_unpack_context* x, unsigned int n) +{ return rb_ary_new2(n); } + +static inline void template_callback_array_item(msgpack_unpack_context* x, VALUE c, VALUE o) +{ rb_ary_push(c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] + +static inline VALUE template_callback_map(msgpack_unpack_context* x, unsigned int n) +{ return rb_hash_new(); } + +static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) +{ rb_hash_aset(c, k, v); } + +static inline VALUE template_callback_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) +{ return rb_str_new(p, l); } + + +#include "msgpack/unpack_template.h" + #define UNPACKER(from, name) \ msgpack_unpacker *name = NULL; \ @@ -45,7 +136,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) unsigned int i; for(i=0; i < mp->top; ++i) { rb_gc_mark(mp->stack[i].obj); - rb_gc_mark(mp->stack[i].tmp.map_key); + rb_gc_mark(mp->stack[i].map_key); } } @@ -61,7 +152,7 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); - mp->user.finished = false; + mp->user.finished = 0; msgpack_unpacker_init(mp); return self; } @@ -93,10 +184,10 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) if(ret < 0) { rb_raise(eUnpackError, "Parse error."); } else if(ret > 0) { - mp->user.finished = true; + mp->user.finished = 1; return ULONG2NUM(from); } else { - mp->user.finished = false; + mp->user.finished = 0; return ULONG2NUM(from); } } diff --git a/ruby/unpack_context.h b/ruby/unpack_context.h deleted file mode 100644 index 35e0132..0000000 --- a/ruby/unpack_context.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * MessagePack unpacking routine for Ruby - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef UNPACK_CONTEXT_H__ -#define UNPACK_CONTEXT_H__ - -#include "ruby.h" -#include -#include - -typedef VALUE msgpack_object; - -typedef struct { - bool finished; -} msgpack_unpack_context; - - -#include "msgpack/unpack/inline_context.h" - -#endif /* unpack_context.h */ - diff --git a/ruby/unpack_inline.c b/ruby/unpack_inline.c deleted file mode 100644 index 849441c..0000000 --- a/ruby/unpack_inline.c +++ /dev/null @@ -1,78 +0,0 @@ -/* - * MessagePack unpacking routine for Ruby - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "unpack_context.h" - -static inline VALUE msgpack_unpack_init(msgpack_unpack_context* x) -{ return Qnil; } - -static inline VALUE msgpack_unpack_unsigned_int_8(msgpack_unpack_context* x, uint8_t d) -{ return INT2FIX(d); } - -static inline VALUE msgpack_unpack_unsigned_int_16(msgpack_unpack_context* x, uint16_t d) -{ return INT2FIX(d); } - -static inline VALUE msgpack_unpack_unsigned_int_32(msgpack_unpack_context* x, uint32_t d) -{ return UINT2NUM(d); } - -static inline VALUE msgpack_unpack_unsigned_int_64(msgpack_unpack_context* x, uint64_t d) -{ return rb_ull2inum(d); } - -static inline VALUE msgpack_unpack_signed_int_8(msgpack_unpack_context* x, int8_t d) -{ return INT2FIX((long)d); } - -static inline VALUE msgpack_unpack_signed_int_16(msgpack_unpack_context* x, int16_t d) -{ return INT2FIX((long)d); } - -static inline VALUE msgpack_unpack_signed_int_32(msgpack_unpack_context* x, int32_t d) -{ return INT2NUM((long)d); } - -static inline VALUE msgpack_unpack_signed_int_64(msgpack_unpack_context* x, int64_t d) -{ return rb_ll2inum(d); } - -static inline VALUE msgpack_unpack_float(msgpack_unpack_context* x, float d) -{ return rb_float_new(d); } - -static inline VALUE msgpack_unpack_double(msgpack_unpack_context* x, double d) -{ return rb_float_new(d); } - -static inline VALUE msgpack_unpack_nil(msgpack_unpack_context* x) -{ return Qnil; } - -static inline VALUE msgpack_unpack_true(msgpack_unpack_context* x) -{ return Qtrue; } - -static inline VALUE msgpack_unpack_false(msgpack_unpack_context* x) -{ return Qfalse; } - -static inline VALUE msgpack_unpack_array_start(msgpack_unpack_context* x, unsigned int n) -{ return rb_ary_new2(n); } - -static inline void msgpack_unpack_array_item(msgpack_unpack_context* x, VALUE c, VALUE o) -{ rb_ary_push(c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] - -static inline VALUE msgpack_unpack_map_start(msgpack_unpack_context* x, unsigned int n) -{ return rb_hash_new(); } - -static inline void msgpack_unpack_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) -{ rb_hash_aset(c, k, v); } - -static inline VALUE msgpack_unpack_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) -{ return rb_str_new(p, l); } - -#include "msgpack/unpack/inline_impl.h" - From 9923cf4daf631432e389dd0694042b6b405c1288 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0020/1648] lang/c/msgpack: reimplemented C++ binding with template-based static resolution design git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@67 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/unpack.c | 8 +- configure.in | 2 +- cpp/Makefile.am | 21 +- cpp/bench.cpp | 15 +- cpp/bench.mk | 2 +- cpp/object.cpp | 497 ++++++++------------------------------ cpp/object.hpp | 354 ++++++--------------------- cpp/pack.hpp | 10 +- cpp/test.cpp | 156 ++++++------ cpp/type.hpp | 9 + cpp/type/array.hpp | 59 +++++ cpp/type/boolean.hpp | 49 ++++ cpp/type/float.hpp | 67 +++++ cpp/type/integer.hpp | 243 +++++++++++++++++++ cpp/type/map.hpp | 132 ++++++++++ cpp/type/nil.hpp | 47 ++++ cpp/type/raw.hpp | 99 ++++++++ cpp/type/tuple.hpp.erb | 172 +++++++++++++ cpp/unpack.cpp | 149 ++++++------ cpp/unpack.hpp | 16 +- cpp/zone.cpp | 61 ++--- cpp/zone.hpp | 70 ++++++ cpp/zone.hpp.erb | 202 ---------------- msgpack/pack_template.h | 2 - msgpack/unpack_template.h | 4 +- ruby/unpack.c | 8 +- 26 files changed, 1338 insertions(+), 1116 deletions(-) create mode 100644 cpp/type.hpp create mode 100644 cpp/type/array.hpp create mode 100644 cpp/type/boolean.hpp create mode 100644 cpp/type/float.hpp create mode 100644 cpp/type/integer.hpp create mode 100644 cpp/type/map.hpp create mode 100644 cpp/type/nil.hpp create mode 100644 cpp/type/raw.hpp create mode 100644 cpp/type/tuple.hpp.erb create mode 100644 cpp/zone.hpp delete mode 100644 cpp/zone.hpp.erb diff --git a/c/unpack.c b/c/unpack.c index 3058427..c7f25c4 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -89,14 +89,14 @@ static inline void* template_callback_false(msgpack_unpack_t* x) static inline void* template_callback_array(msgpack_unpack_t* x, unsigned int n) { return x->callback.unpack_array(x->data, n); } -static inline void template_callback_array_item(msgpack_unpack_t* x, void* c, void* o) -{ x->callback.unpack_array_item(x->data, c, o); } +static inline void template_callback_array_item(msgpack_unpack_t* x, void** c, void* o) +{ x->callback.unpack_array_item(x->data, *c, o); } static inline void* template_callback_map(msgpack_unpack_t* x, unsigned int n) { return x->callback.unpack_map(x->data, n); } -static inline void template_callback_map_item(msgpack_unpack_t* x, void* c, void* k, void* v) -{ x->callback.unpack_map_item(x->data, c, k, v); } +static inline void template_callback_map_item(msgpack_unpack_t* x, void** c, void* k, void* v) +{ x->callback.unpack_map_item(x->data, *c, k, v); } static inline void* template_callback_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) { return x->callback.unpack_raw(x->data, b, p, l); } diff --git a/configure.in b/configure.in index 9b6f6d4..33acfa6 100644 --- a/configure.in +++ b/configure.in @@ -1,4 +1,4 @@ -AC_INIT(msgpack/unpack/inline_impl.h) +AC_INIT(msgpack/unpack_template.h) AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index aa22cc7..8b13bea 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -10,20 +10,27 @@ nobase_include_HEADERS = \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ - msgpack/zone.hpp + msgpack/zone.hpp \ + msgpack/type.hpp \ + msgpack/type/array.hpp \ + msgpack/type/boolean.hpp \ + msgpack/type/integer.hpp \ + msgpack/type/map.hpp \ + msgpack/type/nil.hpp \ + msgpack/type/tuple.hpp noinst_HEADERS = \ - msgpack/zone.hpp.erb + msgpack/type/tuple.hpp.erb # FIXME -object.lo: msgpack/zone.hpp -unpack.lo: msgpack/zone.hpp -zone.lo: msgpack/zone.hpp +object.lo: msgpack/type/tuple.hpp +unpack.lo: msgpack/type/tuple.hpp +zone.lo: msgpack/type/tuple.hpp -msgpack/zone.hpp: msgpack/zone.hpp.erb +msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb erb $< > $@ -MOSTLYCLEANFILES = zone.hpp +MOSTLYCLEANFILES = msgpack/type/tuple.hpp libmsgpack_la_LDFLAGS = -version-info 0:0:0 diff --git a/cpp/bench.cpp b/cpp/bench.cpp index 517a870..1aad807 100644 --- a/cpp/bench.cpp +++ b/cpp/bench.cpp @@ -6,10 +6,10 @@ #include #include -//static const unsigned int TASK_INT_NUM = 1<<24; -//static const unsigned int TASK_STR_LEN = 1<<15; -static const unsigned int TASK_INT_NUM = 1<<22; -static const unsigned int TASK_STR_LEN = 1<<13; +static const unsigned int TASK_INT_NUM = 1<<24; +static const unsigned int TASK_STR_LEN = 1<<15; +//static const unsigned int TASK_INT_NUM = 1<<22; +//static const unsigned int TASK_STR_LEN = 1<<13; static const char* TASK_STR_PTR; @@ -104,6 +104,7 @@ void bench_msgpack_int() } timer.show_stat(buf.size()); + std::cout << "----" << std::endl; std::cout << "unpack integer" << std::endl; @@ -116,6 +117,7 @@ void bench_msgpack_int() } timer.show_stat(buf.size()); + /* std::cout << "----" << std::endl; std::cout << "dynamic pack integer" << std::endl; @@ -124,6 +126,7 @@ void bench_msgpack_int() timer.reset(); msgpack::pack(buf, obj); timer.show_stat(buf.size()); + */ } void bench_msgpack_str() @@ -144,6 +147,7 @@ void bench_msgpack_str() } timer.show_stat(buf.size()); + std::cout << "----" << std::endl; std::cout << "unpack string" << std::endl; @@ -156,6 +160,8 @@ void bench_msgpack_str() } timer.show_stat(buf.size()); + + /* std::cout << "----" << std::endl; std::cout << "dynamic pack string" << std::endl; @@ -164,6 +170,7 @@ void bench_msgpack_str() timer.reset(); msgpack::pack(buf, obj); timer.show_stat(buf.size()); + */ } int main(void) diff --git a/cpp/bench.mk b/cpp/bench.mk index 52a4b80..8da2b7f 100644 --- a/cpp/bench.mk +++ b/cpp/bench.mk @@ -5,5 +5,5 @@ LDFLAGS += all: bench bench: bench.o unpack.o zone.o object.o pack.hpp unpack.hpp zone.hpp object.hpp - $(CXX) bench.o unpack.o zone.o object.o $(CFLAGS) $(LDFLAGS) -o $@ + $(CXX) bench.o unpack.o zone.o object.o $(CXXFLAGS) $(LDFLAGS) -o $@ diff --git a/cpp/object.cpp b/cpp/object.cpp index ef2a68c..276732c 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -16,417 +16,128 @@ // limitations under the License. // #include "msgpack/object.hpp" -#include "msgpack/pack.hpp" + +#include namespace msgpack { -namespace { -template -struct numeric_overflow_signed_impl; +std::ostream& operator<< (std::ostream& s, const object o) +{ + switch(o.type) { + case type::NIL: + s << "nil"; + break; -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast( std::numeric_limits::max()) < x ) { return 1; } - if( static_cast(-std::numeric_limits::max()) > x ) { return -1; } + case type::BOOLEAN: + s << (o.via.boolean ? "true" : "false"); + break; + + case type::POSITIVE_INTEGER: + s << o.via.u64; + break; + + case type::NEGATIVE_INTEGER: + s << o.via.i64; + break; + + case type::RAW: + (s << '"').write(o.via.ref.ptr, o.via.ref.size) << '"'; + break; + + case type::ARRAY: + s << "["; + if(o.via.container.size != 0) { + object* p(o.via.container.ptr); + s << *p; + ++p; + for(object* const pend(o.via.container.ptr + o.via.container.size); + p < pend; ++p) { + s << ", " << *p; } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } - if( static_cast(-std::numeric_limits::max()) > x) { return -1; } } - return 0; - } -}; + s << "]"; + break; + // FIXME loop optimiziation -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) <= sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + case type::MAP: + s << "{"; + if(o.via.container.size != 0) { + object* p(o.via.container.ptr); + object* const pend(o.via.container.ptr + o.via.container.size*2); + s << *p; ++p; + s << "=>"; + s << *p; ++p; + while(p < pend) { + s << ", "; + s << *p; ++p; + s << "=>"; + s << *p; ++p; } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } } - return 0; - } -}; + s << "}"; + break; + // FIXME loop optimiziation -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( static_cast(0) > x ) { return -1; } - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast( std::numeric_limits::max()) < x) { return 1; } + default: + // FIXME + s << "#" << std::endl; + } + return s; +} + + +bool operator==(const object x, const object y) +{ + if(x.type != y.type) { return false; } + + switch(x.type) { + case type::NIL: + return true; + + case type::BOOLEAN: + return x.via.boolean == y.via.boolean; + + case type::POSITIVE_INTEGER: + return x.via.u64 == y.via.u64; + + case type::NEGATIVE_INTEGER: + return x.via.i64 == y.via.i64; + + case type::RAW: + return x.via.ref.size == y.via.ref.size && + memcmp(x.via.ref.ptr, y.via.ref.ptr, x.via.ref.size) == 0; + + case type::ARRAY: + if(x.via.container.size != y.via.container.size) { return false; } + for(object* px(x.via.container.ptr), + * const pxend(x.via.container.ptr + x.via.container.size), + * py(y.via.container.ptr); + px != pxend; ++px, ++py) { + if(*px != *py) { return false; } } - return 0; - } -}; + return true; + // FIXME loop optimiziation -template -struct numeric_overflow_signed_impl { - static int test(X x) { - if( ( std::numeric_limits::is_integer && std::numeric_limits::is_integer) || - (!std::numeric_limits::is_integer && !std::numeric_limits::is_integer) ) { - if( sizeof(T) < sizeof(X) ) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } - } - } else if(std::numeric_limits::is_integer) { - if( static_cast(std::numeric_limits::max()) < x ) { return 1; } + case type::MAP: + if(x.via.container.size != y.via.container.size) { return false; } + for(object* px(x.via.container.ptr), + * const pxend(x.via.container.ptr + x.via.container.size*2), + * py(y.via.container.ptr); + px != pxend; ++px, ++py) { + if(*px != *py) { return false; } } - return 0; + return true; + + default: + return false; } -}; - -template -struct numeric_overflow { - static int test(X x) { - return numeric_overflow_signed_impl::is_signed, std::numeric_limits::is_signed>::test(x); - } - static void check(X x) { - int r = test(x); - if(r == 1) { throw positive_overflow_error(); } - if(r == -1) { throw negative_overflow_error(); } - } -}; - -template -struct numeric_underflow { - static bool test(X x) { - return static_cast(static_cast(x)) != x; - } - static void check(X x) { - if(test(x)) { throw underflow_error(); } - } -}; - -template -inline T integer_cast(X x) { - numeric_overflow::check(x); - return static_cast(x); } - -template -inline T float_cast(X x) { - numeric_overflow::check(x); - numeric_underflow::check(x); - return static_cast(x); } - -template -inline bool numequal(V v, const object_class* x) - try { return v == static_cast(*x); } - catch (type_error&) { return false; } - -template -inline bool numless(V v, const object_class* x) - try { return v < static_cast(*x); } - catch (positive_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - -template -inline bool numgreater(V v, const object_class* x) - try { return v > static_cast(*x); } - catch (negative_overflow_error&) { return true; } - catch (overflow_error&) { return false; } - -template -inline void numeric_inspect(V v, std::ostream& s) - { s << v; } - -template <> -inline void numeric_inspect(uint8_t v, std::ostream& s) - { s << (uint16_t)v; } - -template <> -inline void numeric_inspect(int8_t v, std::ostream& s) - { s << (int16_t)v; } - -template -inline void numeric_pack(dynamic_packer& p, V v); - -template <> -inline void numeric_pack(dynamic_packer& p, uint8_t v) - { p.pack_uint8(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, uint16_t v) - { p.pack_uint16(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, uint32_t v) - { p.pack_uint32(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, uint64_t v) - { p.pack_uint64(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, int8_t v) - { p.pack_int8(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, int16_t v) - { p.pack_int16(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, int32_t v) - { p.pack_int32(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, int64_t v) - { p.pack_int64(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, float v) - { p.pack_float(v); } - -template <> -inline void numeric_pack(dynamic_packer& p, double v) - { p.pack_double(v); } - -} // noname namespace - - -bool object_nil::isnil() const { return true; } -bool object_nil::operator== (const object_class* x) const - { return typeid(*this) == typeid(*x); } -void object_nil::pack(dynamic_packer& p) const - { p.pack_nil(); } -const object_class* object_nil::inspect(std::ostream& s) const - { s << "nil"; return this; } - -bool object_true::xbool() const { return true; } -bool object_true::operator== (const object_class* x) const - { return typeid(*this) == typeid(*x); } -void object_true::pack(dynamic_packer& p) const - { p.pack_true(); } -const object_class* object_true::inspect(std::ostream& s) const - { s << "true"; return this; } - -bool object_false::xbool() const { return false; } -bool object_false::operator== (const object_class* x) const - { return typeid(*this) == typeid(*x); } -void object_false::pack(dynamic_packer& p) const - { p.pack_false(); } -const object_class* object_false::inspect(std::ostream& s) const - { s << "false"; return this; } - - -#define INTEGER_OBJECT(NAME) \ -uint8_t object_##NAME::xu8 () const { return val; } \ -uint16_t object_##NAME::xu16 () const { return integer_cast(val); } \ -uint32_t object_##NAME::xu32 () const { return integer_cast(val); } \ -uint64_t object_##NAME::xu64 () const { return integer_cast(val); } \ -int8_t object_##NAME::xi8 () const { return integer_cast(val); } \ -int16_t object_##NAME::xi16 () const { return integer_cast(val); } \ -int32_t object_##NAME::xi32 () const { return integer_cast(val); } \ -int64_t object_##NAME::xi64 () const { return integer_cast(val); } \ -float object_##NAME::xfloat () const { return integer_cast(val); } \ -double object_##NAME::xdouble() const { return integer_cast(val); } \ -bool object_##NAME::operator== (const object_class* x) const \ - try { return val == x->x##NAME(); } \ - catch (type_error&) { return false; } \ -bool object_##NAME::operator< (const object_class* x) const \ - try { return val < x->x##NAME(); } \ - catch (positive_overflow_error&) { return true; } \ - catch (overflow_error&) { return false; } \ -bool object_##NAME::operator> (const object_class* x) const \ - try { return val > x->x##NAME(); } \ - catch (negative_overflow_error&) { return true; } \ - catch (overflow_error&) { return false; } \ -void object_##NAME::pack(dynamic_packer& p) const \ - { numeric_pack(p, val); } \ -const object_class* object_##NAME::inspect(std::ostream& s) const \ - { numeric_inspect(val, s); return this; } \ - - -INTEGER_OBJECT(u8) -INTEGER_OBJECT(u16) -INTEGER_OBJECT(u32) -INTEGER_OBJECT(u64) -INTEGER_OBJECT(i8) -INTEGER_OBJECT(i16) -INTEGER_OBJECT(i32) -INTEGER_OBJECT(i64) - -#undef INTEGER_OBJECT - - -#define FLOAT_OBJECT(NAME) \ -uint8_t object_##NAME::xu8 () const { return val; } \ -uint16_t object_##NAME::xu16 () const { return integer_cast(val); } \ -uint32_t object_##NAME::xu32 () const { return integer_cast(val); } \ -uint64_t object_##NAME::xu64 () const { return integer_cast(val); } \ -int8_t object_##NAME::xi8 () const { return integer_cast(val); } \ -int16_t object_##NAME::xi16 () const { return integer_cast(val); } \ -int32_t object_##NAME::xi32 () const { return integer_cast(val); } \ -int64_t object_##NAME::xi64 () const { return integer_cast(val); } \ -float object_##NAME::xfloat () const { return float_cast(val); } \ -double object_##NAME::xdouble() const { return float_cast(val); } \ -bool object_##NAME::operator== (const object_class* x) const \ - try { return val == x->x##NAME(); } \ - catch (type_error&) { return false; } \ -bool object_##NAME::operator< (const object_class* x) const { \ - try { return val < x->xdouble(); } \ - catch (positive_overflow_error&) { return true; } \ - catch (overflow_error&) { return false; } \ - catch (underflow_error&) { \ - if(val < 0.0) { \ - if(numeric_overflow::test(val) == -1) { return true; } \ - try { return static_cast(val) < x->xi64(); } \ - catch (type_error&) { return true; } \ - } else { \ - if(numeric_overflow::test(val) == 1) { return false; } \ - try { return static_cast(val) < x->xu64(); } \ - catch (type_error&) { return false; } \ - } \ - } } \ -bool object_##NAME::operator> (const object_class* x) const { \ - try { return val > x->xdouble(); } \ - catch (negative_overflow_error&) { return true; } \ - catch (overflow_error&) { return false; } \ - catch (underflow_error&) { \ - if(val < 0.0) { \ - if(numeric_overflow::test(val) == -1) { return false; } \ - try { return static_cast(val) > x->xi64(); } \ - catch (type_error&) { return false; } \ - } else { \ - if(numeric_overflow::test(val) == 1) { return true; } \ - try { return static_cast(val) > x->xu64(); } \ - catch (type_error&) { return true; } \ - } \ - } } \ -void object_##NAME::pack(dynamic_packer& p) const \ - { numeric_pack(p, val); } \ -const object_class* object_##NAME::inspect(std::ostream& s) const \ - { s << val; return this; } \ - -FLOAT_OBJECT(float) -FLOAT_OBJECT(double) - -#undef FLOAT_OBJECT - - -#define RAW_OBJECT(NAME, EXTRA) \ -EXTRA \ -bool object_##NAME::operator== (const object_class* x) const \ - try { \ - raw xr(x->xraw()); \ - return len == xr.len && (ptr == xr.ptr || memcmp(ptr, xr.ptr, len) == 0); \ - } catch (type_error&) { return false; } \ -bool object_##NAME::operator< (const object_class* x) const { \ - raw xr(x->xraw()); \ - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) < 0; } \ - else { return len < xr.len; } } \ -bool object_##NAME::operator> (const object_class* x) const { \ - raw xr(x->xraw()); \ - if(len == xr.len) { return ptr != xr.ptr && memcmp(ptr, xr.ptr, len) > 0; } \ - else { return len > xr.len; } } \ -void object_##NAME::pack(dynamic_packer& p) const \ - { p.pack_raw(ptr, len); } \ -const object_class* object_##NAME::inspect(std::ostream& s) const \ - { (s << '"').write(ptr, len) << '"'; return this; } // FIXME escape +} // FIXME -RAW_OBJECT(mutable_raw_ref, - /*mutable_raw object_mutable_raw_ref::xraw() { return mutable_raw(ptr, len); }*/ - raw object_mutable_raw_ref::xraw() const { return raw(ptr, len); } ) - -RAW_OBJECT(raw_ref, - raw object_raw_ref::xraw() const { return raw(ptr, len); } ) - -#undef RAW_OBJECT - - - array& object_array::xarray() { return val; } -const array& object_array::xarray() const { return val; } -bool object_array::operator== (const object_class* x) const - try { - const std::vector& xa(x->xarray()); - if(val.size() != xa.size()) { return false; } - for(std::vector::const_iterator iv(val.begin()), iv_end(val.end()), ix(xa.begin()); - iv != iv_end; - ++iv, ++ix) { - if(*iv != *ix) { return false; } - } - return true; - } catch (type_error&) { return false; } -const object_class* object_array::inspect(std::ostream& s) const -{ - s << '['; - if(!val.empty()) { - std::vector::const_iterator it(val.begin()); - s << *it; - ++it; - for(std::vector::const_iterator it_end(val.end()); - it != it_end; - ++it) { - s << ", " << *it; - } - } - s << ']'; - return this; -} -void object_array::pack(dynamic_packer& p) const -{ - p.pack_array(val.size()); - for(std::vector::const_iterator it(val.begin()), it_end(val.end()); - it != it_end; - ++it) { - it->pack(p); - } -} - - - map& object_map::xmap() { return val; } -const map& object_map::xmap() const { return val; } -bool object_map::operator== (const object_class* x) const - try { - const std::map& xm(x->xmap()); - if(val.size() != xm.size()) { return false; } - for(std::map::const_iterator iv(val.begin()), iv_end(val.end()), ix(xm.begin()); - iv != iv_end; - ++iv, ++ix) { - if(iv->first != ix->first || iv->second != ix->first) { return false; } - } - return true; - } catch (type_error&) { return false; } -const object_class* object_map::inspect(std::ostream& s) const -{ - s << '{'; - if(!val.empty()) { - std::map::const_iterator it(val.begin()); - s << it->first << "=>" << it->second; - ++it; - for(std::map::const_iterator it_end(val.end()); - it != it_end; - ++it) { - s << ", " << it->first << "=>" << it->second; - } - } - s << '}'; - return this; -} -void object_map::pack(dynamic_packer& p) const -{ - p.pack_map(val.size()); - for(std::map::const_iterator it(val.begin()), it_end(val.end()); - it != it_end; - ++it) { - it->first.pack(p); - it->second.pack(p); - } -} +//template +//const object& operator>> (const object& v, packer& o); } // namespace msgpack diff --git a/cpp/object.hpp b/cpp/object.hpp index 456210c..7007b90 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -1,5 +1,5 @@ // -// MessagePack for C++ dynamic typed objects +// MessagePack for C++ static resolution routine // // Copyright (C) 2008 FURUHASHI Sadayuki // @@ -18,305 +18,109 @@ #ifndef MSGPACK_OBJECT_HPP__ #define MSGPACK_OBJECT_HPP__ -#include +#include "msgpack/pack.hpp" +#include #include #include #include -#include -#include -#include namespace msgpack { -class type_error : public std::bad_cast { }; -class cast_error : public type_error { }; -class overflow_error : public type_error { }; -class underflow_error : public type_error { }; -class positive_overflow_error : public overflow_error { }; -class negative_overflow_error : public overflow_error { }; - - -struct mutable_raw { - explicit mutable_raw() : ptr(NULL), len(0) {} - explicit mutable_raw(char* p, size_t l) : ptr(p), len(l) {} -public: - char* ptr; - size_t len; -public: - std::string str() { return std::string(ptr, len); } -}; - -struct raw { - explicit raw() : ptr(NULL), len(0) {} - explicit raw(const char* p, size_t l) : ptr(p), len(l) {} - raw(const mutable_raw& m) : ptr(m.ptr), len(m.len) {} -public: - const char* ptr; - size_t len; -public: - std::string str() { return std::string(ptr, len); } -}; - - -struct object; - -typedef std::map map; -typedef std::vector array; - -class dynamic_packer; - - -struct object_class { - virtual ~object_class() {} - virtual bool isnil () const { return false; } - virtual bool xbool () const { throw cast_error(); } - virtual uint8_t xu8 () const { throw cast_error(); } - virtual uint16_t xu16 () const { throw cast_error(); } - virtual uint32_t xu32 () const { throw cast_error(); } - virtual uint64_t xu64 () const { throw cast_error(); } - virtual int8_t xi8 () const { throw cast_error(); } - virtual int16_t xi16 () const { throw cast_error(); } - virtual int32_t xi32 () const { throw cast_error(); } - virtual int64_t xi64 () const { throw cast_error(); } - virtual float xfloat () const { throw cast_error(); } - virtual double xdouble() const { throw cast_error(); } - //virtual mutable_raw xraw () { throw cast_error(); } // FIXME - virtual array& xarray() { throw cast_error(); } - virtual map& xmap () { throw cast_error(); } - virtual raw xraw () const { throw cast_error(); } - virtual const array& xarray() const { throw cast_error(); } - virtual const map& xmap () const { throw cast_error(); } - virtual bool operator== (const object_class* x) const { return false; } - virtual bool operator< (const object_class* x) const { throw cast_error(); } - virtual bool operator> (const object_class* x) const { throw cast_error(); } - bool operator!= (const object_class* x) const { return !(this->operator==(x)); } - virtual void pack(dynamic_packer& p) const = 0; - operator bool() const { return xbool(); } // FIXME !isnil(); - operator uint8_t() const { return xu8(); } - operator uint16_t() const { return xu16(); } - operator uint32_t() const { return xu32(); } - operator uint64_t() const { return xu64(); } - operator int8_t() const { return xi8(); } - operator int16_t() const { return xi16(); } - operator int32_t() const { return xi32(); } - operator int64_t() const { return xi64(); } - operator float() const { return xfloat(); } - operator double() const { return xdouble(); } - //operator mutable_raw() { return xraw(); } // FIXME - operator array&() { return xarray(); } - operator map&() { return xmap(); } - operator raw() const { return xraw(); } - operator const array&() const { return xarray(); } - operator const map&() const { return xmap(); } - virtual const object_class* inspect(std::ostream& s) const - { s << '<' << typeid(*this).name() << '>'; return this; } -}; - -inline std::ostream& operator<< (std::ostream& s, const object_class* o) - { o->inspect(s); return s; } - - -struct object_container_mixin {}; -struct object_constructor_mixin {}; +class type_error : public std::bad_cast { }; struct object { - explicit object() : val(NULL) {} - object(object_class* v) : val(v) {} - //object(object_class& v) : val(&v) {} - ~object() {} - bool isnil () const { return val->isnil(); } - bool xbool () const { return val->xbool(); } - uint8_t xu8 () const { return val->xu8(); } - uint16_t xu16 () const { return val->xu16(); } - uint32_t xu32 () const { return val->xu32(); } - uint64_t xu64 () const { return val->xu64(); } - int8_t xi8 () const { return val->xi8(); } - int16_t xi16 () const { return val->xi16(); } - int32_t xi32 () const { return val->xi32(); } - int64_t xi64 () const { return val->xi64(); } - float xfloat () const { return val->xfloat(); } - double xdouble() const { return val->xdouble(); } - //mutable_raw xraw () { return val->xraw(); } // FIXME - array& xarray() { return val->xarray(); } - map& xmap () { return val->xmap(); } - raw xraw () const { return const_cast(val)->xraw(); } - const array& xarray() const { return const_cast(val)->xarray(); } - const map& xmap () const { return const_cast(val)->xmap(); } - bool operator== (object x) const { return val->operator== (x.val); } - bool operator!= (object x) const { return val->operator!= (x.val); } - bool operator< (object x) const { return val->operator< (x.val); } - bool operator> (object x) const { return val->operator> (x.val); } - void pack(dynamic_packer& p) const { val->pack(p); } - template - void pack(Stream& s) const { dynamic_packer p(s); pack(p); } - operator bool() const { return val->operator bool(); } - operator uint8_t() const { return val->operator uint8_t(); } - operator uint16_t() const { return val->operator uint16_t(); } - operator uint32_t() const { return val->operator uint32_t(); } - operator uint64_t() const { return val->operator uint64_t(); } - operator int8_t() const { return val->operator int8_t(); } - operator int16_t() const { return val->operator int16_t(); } - operator int32_t() const { return val->operator int32_t(); } - operator int64_t() const { return val->operator int64_t(); } - operator float() const { return val->operator float(); } - operator double() const { return val->operator double(); } - //operator mutable_raw() { return val->operator mutable_raw(); } // FIXME - operator array&() { return val->operator array&(); } - operator map&() { return val->operator map&(); } - operator raw() const { return val->operator raw(); } - operator const array&() const { return val->operator const array&(); } - operator const map&() const { return val->operator const map&(); } - const object& inspect(std::ostream& s) const - { val->inspect(s); return *this; } -private: - friend class object_container_mixin; - friend class object_constructor_mixin; - object_class* val; + unsigned char type; + union { + bool boolean; + uint64_t u64; + int64_t i64; + double dec; + struct { + object* ptr; + uint32_t size; + } container; + struct { + const char* ptr; + uint32_t size; + } ref; + } via; + // FIXME template operator T() { T v; convert(*this, v); return v; }; }; -inline std::ostream& operator<< (std::ostream& s, const object& o) - { o.inspect(s); return s; } +std::ostream& operator<< (std::ostream& s, const object o); + +bool operator==(const object x, const object y); +inline bool operator!=(const object x, const object y) { return !(x == y); } -struct object_nil : object_class { - bool isnil() const; - bool operator== (const object_class* x) const; - void pack(dynamic_packer& p) const; - const object_class* inspect(std::ostream& s) const; -}; - -struct object_true : object_class { - bool xbool() const; - bool operator== (const object_class* x) const; - void pack(dynamic_packer& p) const; - const object_class* inspect(std::ostream& s) const; -}; - -struct object_false : object_class { - bool xbool() const; - bool operator== (const object_class* x) const; - void pack(dynamic_packer& p) const; - const object_class* inspect(std::ostream& s) const; -}; - -#define INTEGER_CLASS(TYPE, NAME) \ -struct object_##NAME : object_class { \ - explicit object_##NAME(TYPE v) : val(v) {} \ - uint8_t xu8 () const; \ - uint16_t xu16 () const; \ - uint32_t xu32 () const; \ - uint64_t xu64 () const; \ - int8_t xi8 () const; \ - int16_t xi16 () const; \ - int32_t xi32 () const; \ - int64_t xi64 () const; \ - float xfloat () const; \ - double xdouble() const; \ - bool operator== (const object_class* x) const; \ - bool operator< (const object_class* x) const; \ - bool operator> (const object_class* x) const; \ - void pack(dynamic_packer& p) const; \ - const object_class* inspect(std::ostream& s) const; \ -private: \ - TYPE val; \ -}; - -INTEGER_CLASS(uint8_t, u8) -INTEGER_CLASS(uint16_t, u16) -INTEGER_CLASS(uint32_t, u32) -INTEGER_CLASS(uint64_t, u64) -INTEGER_CLASS(int8_t, i8) -INTEGER_CLASS(int16_t, i16) -INTEGER_CLASS(int32_t, i32) -INTEGER_CLASS(int64_t, i64) - -#undef INTEGER_CLASS +template +const object& operator>> (const object& v, packer& o); -#define FLOAT_CLASS(TYPE, NAME) \ -struct object_##NAME : object_class { \ - object_##NAME(TYPE v) : val(v) {} \ - uint8_t xu8 () const; \ - uint16_t xu16 () const; \ - uint32_t xu32 () const; \ - uint64_t xu64 () const; \ - int8_t xi8 () const; \ - int16_t xi16 () const; \ - int32_t xi32 () const; \ - int64_t xi64 () const; \ - float xfloat () const; \ - double xdouble() const; \ - bool operator== (const object_class* x) const; \ - bool operator< (const object_class* x) const; \ - bool operator> (const object_class* x) const; \ - void pack(dynamic_packer& p) const; \ - const object_class* inspect(std::ostream& s) const; \ -private: \ - TYPE val; \ -}; - -FLOAT_CLASS(float, float) -FLOAT_CLASS(double, double) - -#undef FLOAT_CLASS +namespace type { + static const unsigned char NIL = 0x01; + static const unsigned char BOOLEAN = 0x02; + static const unsigned char POSITIVE_INTEGER = 0x03; + static const unsigned char NEGATIVE_INTEGER = 0x04; + static const unsigned char DOUBLE = 0x05; + static const unsigned char RAW = 0x06; + static const unsigned char ARRAY = 0x07; + static const unsigned char MAP = 0x08; -#define RAW_CLASS(NAME, TYPE, EXTRA) \ -struct object_##NAME : object_class { \ - explicit object_##NAME(TYPE p, uint32_t l) : ptr(p), len(l) {} \ - EXTRA \ - bool operator== (const object_class* x) const; \ - bool operator< (const object_class* x) const; \ - bool operator> (const object_class* x) const; \ - void pack(dynamic_packer& p) const; \ - const object_class* inspect(std::ostream& s) const; \ -private: \ - TYPE ptr; \ - uint32_t len; \ -}; - -// FIXME -RAW_CLASS(mutable_raw_ref, char*, /*mutable_raw xraw();*/ raw xraw() const; ) -RAW_CLASS(raw_ref, const char*, raw xraw() const; ) - -#undef RAW_CLASS + template + inline T& operator<< (T& v, object o) + { + v = o; + return v; + } -struct object_array : object_class, object_container_mixin { - explicit object_array() {} - explicit object_array(uint32_t n) { val.reserve(n); } - array& xarray(); - const array& xarray() const; - bool operator== (const object_class* x) const; - // FIXME operator<, operator> - void pack(dynamic_packer& p) const; - const object_class* inspect(std::ostream& s) const; -public: - void push_back(object o) { val.push_back(o); } -private: - std::vector val; -}; + namespace detail { + template + inline void pack_copy(T v, packer& o) + { pack(v, o); } + } + + template + inline const T& operator>> (const T& v, packer& o) + { + detail::pack_copy(v.pack(), o); + return v; + } + +} // namespace type -// FIXME hash, operator==: nil, true, false, array, mapを入れられãªã„ -struct object_map : object_class, object_container_mixin { - explicit object_map() {} - map& xmap(); - const map& xmap() const; - bool operator== (const object_class* x) const; - // FIXME operator<, operator> - void pack(dynamic_packer& p) const; - const object_class* inspect(std::ostream& s) const; -public: - void store(object k, object v) { val[k] = v; } -private: - std::map val; -}; +template +inline void convert(T& v, object o) +{ + using namespace type; + v << o; +} + + +template +inline void pack(T& v, packer& o) +{ + using namespace type; + v >> o; +} + + +template +inline void pack(T& v, Stream& s) +{ + packer pk(s); + pack(v, pk); +} } // namespace msgpack +#include "msgpack/type.hpp" + #endif /* msgpack/object.hpp */ diff --git a/cpp/pack.hpp b/cpp/pack.hpp index f3eeb34..fe470b6 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -18,8 +18,6 @@ #ifndef MSGPACK_PACK_HPP__ #define MSGPACK_PACK_HPP__ -#include "msgpack/object.hpp" -#include "msgpack/zone.hpp" #include // __BYTE_ORDER #include @@ -44,9 +42,9 @@ public: void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } void pack_float(float d) { pack_float_impl(m_stream, d); } void pack_double(double d) { pack_double_impl(m_stream, d); } - void pack_nil() { pack_nil(m_stream); } - void pack_true() { pack_true(m_stream); } - void pack_false() { pack_false(m_stream); } + void pack_nil() { pack_nil_impl(m_stream); } + void pack_true() { pack_true_impl(m_stream); } + void pack_false() { pack_false_impl(m_stream); } void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } @@ -92,6 +90,7 @@ template packer::packer(Stream& s) : m_stream(s) { } +/* class dynamic_stream { public: template @@ -212,6 +211,7 @@ inline void pack(Stream& s, object o) dynamic_packer pk(s); o.pack(pk); } +*/ } // namespace msgpack diff --git a/cpp/test.cpp b/cpp/test.cpp index 68050a3..dff9101 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -1,37 +1,35 @@ #include #include -//#include -//#include #include #include #include +using namespace msgpack; + class checker { public: - void check(const char* d, size_t len, msgpack::object should) { - using msgpack::object; + template + void check(const char* d, size_t len, T should) { try { std::cout << "----" << std::endl; object o; try { - o = msgpack::unpack(d, len, m_zone); + o = unpack(d, len, m_zone); } catch (std::runtime_error& e) { - std::cout << should << std::endl; + std::cout << o << std::endl; std::cout << "**" << e.what() << "**" << std::endl; return; } std::cout << o << std::endl; - if(o != should) { - std::cout << "** TEST FAILED **" << std::endl; - } try { std::stringstream s; - msgpack::pack(s, o); + pack(should, s); std::string str(s.str()); - object ro = msgpack::unpack(str.data(), str.size(), m_zone); + object ro = unpack(str.data(), str.size(), m_zone); + std::cout << ro << std::endl; if(ro != o) { throw std::runtime_error("NOT MATCH"); } } catch (std::runtime_error& e) { std::cout << "** REUNPACK FAILED **" << std::endl; @@ -45,7 +43,7 @@ public: m_zone.clear(); } private: - msgpack::zone m_zone; + zone m_zone; }; int main(void) @@ -53,54 +51,48 @@ int main(void) checker c; { // SimpleValue - msgpack::zone z; const char d[] = { 0x93, 0xc0, 0xc2, 0xc3, }; c.check(d, sizeof(d), - z.narray( - z.nnil(), z.nfalse(), z.ntrue() + type::make_tuple( + type::nil(), false, true ) ); } { // Fixnum - msgpack::zone z; const char d[] = { 0x92, 0x93, 0x00, 0x40, 0x7f, 0x93, 0xe0, 0xf0, 0xff, }; c.check(d, sizeof(d), - z.narray( - z.narray( - z.nu8(0), - z.nu8(64), - z.nu8(127) + type::make_tuple( + type::make_tuple( + 0, 64, 127 ), - z.narray( - z.ni8(-32), - z.ni8(-16), - z.ni8(-1) + type::make_tuple( + -32, -16, -1 ) ) ); } { // FixArray - msgpack::zone z; const char d[] = { 0x92, 0x90, 0x91, 0x91, 0xc0, }; + std::vector empty; c.check(d, sizeof(d), - z.narray( - z.narray(), - z.narray( - z.narray( - z.nnil() + type::make_tuple( + empty, + type::make_tuple( + type::make_tuple( + type::nil() ) ) ) @@ -108,7 +100,6 @@ int main(void) } { // FixRaw - msgpack::zone z; const char d[] = { 0x94, 0xa0, @@ -117,91 +108,84 @@ int main(void) 0xa3, 'd', 'e', 'f', }; c.check(d, sizeof(d), - z.narray( - z.nraw_ref("", 0), - z.nraw_ref("a", 1), - z.nraw_ref("bc", 2), - z.nraw_ref("def", 3) + type::make_tuple( + std::string(""), + std::string("a"), + std::string("bc"), + type::raw_ref("def", 3) ) ); } - static const uint16_t TASK_ARRAY = 100; - static char tarray[3]; - static char traw[64]; + static const unsigned TASK_ARRAY = 100; + static const unsigned TASK_REPEAT = 10; + std::vector task; + + // create task { + static char traw[64]; memset(traw, 'a', sizeof(traw)); - traw[0] = 0xda; - uint16_t n = htons(sizeof(traw)-3); - traw[1] = ((char*)&n)[0]; - traw[2] = ((char*)&n)[1]; - - msgpack::zone z; - std::cout << msgpack::unpack(traw, sizeof(traw), z) << std::endl;; - } - - { - tarray[0] = 0xdc; - uint16_t n = htons(TASK_ARRAY); - tarray[1] = ((char*)&n)[0]; - tarray[2] = ((char*)&n)[1]; - } - - { - // write message - ssize_t total_bytes = 0; - std::stringstream stream; - for(unsigned q=0; q < 10; ++q) { - stream.write(tarray, sizeof(tarray)); - total_bytes += sizeof(tarray); - for(uint16_t i=0; i < TASK_ARRAY; ++i) { - stream.write(traw, sizeof(traw)); - total_bytes += sizeof(traw); - } + + task.resize(TASK_ARRAY); + for(unsigned i=0; i < TASK_ARRAY; ++i) { + task[i] = std::string(traw, sizeof(traw)); } + } - stream.seekg(0); - // reserive message + std::stringstream stream; + + // send message + { + for(unsigned i=0; i < TASK_REPEAT; ++i) { + pack(task, stream); + } + std::cout << "send " << stream.str().size() << " bytes" << std::endl; + } + + ssize_t total_bytes = stream.str().size(); + stream.seekg(0); + + // reserive message + { unsigned num_msg = 0; - static const size_t RESERVE_SIZE = 32;//*1024; - msgpack::unpacker upk; + std::auto_ptr pz(new zone()); + + unpacker pac(*pz); + while(stream.good() && total_bytes > 0) { // 1. reserve buffer - upk.reserve_buffer(RESERVE_SIZE); + pac.reserve_buffer(RESERVE_SIZE); // 2. read data to buffer() up to buffer_capacity() bytes size_t sz = stream.readsome( - upk.buffer(), - upk.buffer_capacity()); + pac.buffer(), + pac.buffer_capacity()); total_bytes -= sz; std::cout << "read " << sz << " bytes to capacity " - << upk.buffer_capacity() << " bytes" + << pac.buffer_capacity() << " bytes" << std::endl; // 3. specify the number of bytes actually copied - upk.buffer_consumed(sz); + pac.buffer_consumed(sz); // 4. repeat execute() until it returns false - while( upk.execute() ) { - std::cout << "message parsed" << std::endl; - + while( pac.execute() ) { // 5.1. take out the parsed object - msgpack::object o = upk.data(); + object o = pac.data(); - // 5.2. the parsed object is valid until the zone is deleted - std::auto_ptr pz(upk.release_zone()); - - std::cout << o << std::endl; + // do something using pz and o + std::cout << "message parsed: " << o << std::endl; ++num_msg; - // 5.3 re-initialize unpacker - upk.reset(); + // 5.3 re-initialize unpacker with next zone */ + pz.reset(new zone()); + pac.reset(*pz); } } diff --git a/cpp/type.hpp b/cpp/type.hpp new file mode 100644 index 0000000..1dfd414 --- /dev/null +++ b/cpp/type.hpp @@ -0,0 +1,9 @@ +#include "msgpack/type/array.hpp" +#include "msgpack/type/boolean.hpp" +#include "msgpack/type/float.hpp" +#include "msgpack/type/integer.hpp" +#include "msgpack/type/map.hpp" +#include "msgpack/type/nil.hpp" +#include "msgpack/type/raw.hpp" +#include "msgpack/type/tuple.hpp" + diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp new file mode 100644 index 0000000..7066703 --- /dev/null +++ b/cpp/type/array.hpp @@ -0,0 +1,59 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_ARRAY_HPP__ +#define MSGPACK_TYPE_ARRAY_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { +namespace type { + + +template +inline std::vector operator<< (std::vector& v, object o) +{ + if(o.type != ARRAY) { throw type_error(); } + v.resize(o.via.container.size); + object* p(o.via.container.ptr); + object* const pend(o.via.container.ptr + o.via.container.size); + T* it(&v.front()); + for(; p < pend; ++p, ++it) { + convert(*it, *p); + } + return v; +} + + +template +inline const std::vector& operator>> (const std::vector& v, packer& o) +{ + o.pack_array(v.size()); + for(typename std::vector::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + pack(*it, o); + } + return v; +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/array.hpp */ + diff --git a/cpp/type/boolean.hpp b/cpp/type/boolean.hpp new file mode 100644 index 0000000..0538495 --- /dev/null +++ b/cpp/type/boolean.hpp @@ -0,0 +1,49 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_BOOLEAN_HPP__ +#define MSGPACK_TYPE_BOOLEAN_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { +namespace type { + + +inline bool& operator<< (bool& v, object o) +{ + if(o.type != BOOLEAN) { throw type_error(); } + v = o.via.boolean; + return v; +} + + +template +inline const bool& operator>> (const bool& v, packer o) +{ + if(v) { o.pack_true(); } + else { o.pack_false(); } + return v; +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/bool.hpp */ + diff --git a/cpp/type/float.hpp b/cpp/type/float.hpp new file mode 100644 index 0000000..11fa280 --- /dev/null +++ b/cpp/type/float.hpp @@ -0,0 +1,67 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_FLOAT_HPP__ +#define MSGPACK_TYPE_FLOAT_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { +namespace type { + + +// FIXME check overflow, underflow + + +inline float& operator<< (float& v, object o) +{ + if(o.type != DOUBLE) { throw type_error(); } + v = o.via.dec; + return v; +} + + +template +inline const float& operator>> (const float& v, packer o) +{ + o.pack_float(v); + return v; +} + + +inline double& operator<< (double& v, object o) +{ + if(o.type != DOUBLE) { throw type_error(); } + v = o.via.dec; + return v; +} + + +template +inline const double& operator>> (const double& v, packer o) +{ + o.pack_double(v); + return v; +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/float.hpp */ + diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp new file mode 100644 index 0000000..488c9d4 --- /dev/null +++ b/cpp/type/integer.hpp @@ -0,0 +1,243 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_INTEGER_HPP__ +#define MSGPACK_TYPE_INTEGER_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { +namespace type { + + +namespace detail { + template + struct convert_integer_sign; + + template + struct convert_integer_sign { + static inline T convert(object o) { + if(o.type == POSITIVE_INTEGER) { + if(o.via.u64 > (uint64_t)std::numeric_limits::max()) + { throw type_error(); } + return o.via.u64; + } else if(o.type == NEGATIVE_INTEGER) { + if(o.via.i64 < (int64_t)-std::numeric_limits::max()) + { throw type_error(); } + return o.via.i64; + } + throw type_error(); + } + }; + + template + struct convert_integer_sign { + static inline T convert(object o) { + if(o.type == POSITIVE_INTEGER) { + if(o.via.u64 > (uint64_t)std::numeric_limits::max()) + { throw type_error(); } + return o.via.u64; + } + throw type_error(); + } + }; + + template + static inline T convert_integer(object o) + { + return detail::convert_integer_sign::is_signed>::convert(o); + } + + + + template + struct pack_integer_size_sign; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) + { o.pack_int8(v); } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) + { o.pack_uint8(v); } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (int16_t)v <= (int16_t)std::numeric_limits::max() && + (int16_t)v >= (int16_t)std::numeric_limits::min()) + { o.pack_int8(v); } + else { o.pack_int16(v); } + } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (uint16_t)v <= (uint16_t)std::numeric_limits::max()) + { o.pack_uint8(v); } + else { o.pack_uint16(v); } + } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (int32_t)v <= (int32_t)std::numeric_limits::max() && + (int32_t)v >= (int32_t)std::numeric_limits::min()) + { o.pack_int8(v); } + else if( (int32_t)v <= (int32_t)std::numeric_limits::max() && + (int32_t)v >= (int32_t)std::numeric_limits::min()) + { o.pack_int16(v); } + else { o.pack_int32(v); } + } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) + { o.pack_uint8(v); } + else if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) + { o.pack_uint16(v); } + else { o.pack_uint32(v); } + } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (int64_t)v <= (int64_t)std::numeric_limits::max() && + (int64_t)v >= (int64_t)std::numeric_limits::min()) + { o.pack_int8(v); } + else if( (int64_t)v <= (int64_t)std::numeric_limits::max() && + (int64_t)v >= (int64_t)std::numeric_limits::min()) + { o.pack_int16(v); } + else if( (int64_t)v <= (int64_t)std::numeric_limits::max() && + (int64_t)v >= (int64_t)std::numeric_limits::min()) + { o.pack_int32(v); } + else { o.pack_int64(v); } + } + }; + + template + struct pack_integer_size_sign { + static inline void pack(T v, packer& o) { + if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) + { o.pack_uint8(v); } + else if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) + { o.pack_uint16(v); } + else if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) + { o.pack_uint32(v); } + else { o.pack_uint64(v); } + } + }; + + + template + static inline void pack_integer(T v, packer& o) + { + pack_integer_size_sign::is_signed>::pack(v, o); + } + +} // namespace detail + + +inline signed char& operator<< (signed char& v, object o) + { v = detail::convert_integer(o); return v; } + +inline signed short& operator<< (signed short& v, object o) + { v = detail::convert_integer(o); return v; } + +inline signed int& operator<< (signed int& v, object o) + { v = detail::convert_integer(o); return v; } + +inline signed long& operator<< (signed long& v, object o) + { v = detail::convert_integer(o); return v; } + +inline signed long long& operator<< (signed long long& v, object o) + { v = detail::convert_integer(o); return v; } + + +inline unsigned char& operator<< (unsigned char& v, object o) + { v = detail::convert_integer(o); return v; } + +inline unsigned short& operator<< (unsigned short& v, object o) + { v = detail::convert_integer(o); return v; } + +inline unsigned int& operator<< (unsigned int& v, object o) + { v = detail::convert_integer(o); return v; } + +inline unsigned long& operator<< (unsigned long& v, object o) + { v = detail::convert_integer(o); return v; } + +inline unsigned long long& operator<< (unsigned long long& v, object o) + { v = detail::convert_integer(o); return v; } + + +template +inline const signed char& operator>> (const signed char& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const signed short& operator>> (const signed short& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const signed int& operator>> (const signed int& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const signed long& operator>> (const signed long& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const signed long long& operator>> (const signed long long& v, packer o) + { detail::pack_integer(v, o); return v; } + + +template +inline const unsigned char& operator>> (const unsigned char& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const unsigned short& operator>> (const unsigned short& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const unsigned int& operator>> (const unsigned int& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const unsigned long& operator>> (const unsigned long& v, packer o) + { detail::pack_integer(v, o); return v; } + +template +inline const unsigned long long& operator>> (const unsigned long long& v, packer o) + { detail::pack_integer(v, o); return v; } + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/integer.hpp */ + diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp new file mode 100644 index 0000000..1efecec --- /dev/null +++ b/cpp/type/map.hpp @@ -0,0 +1,132 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_MAP_HPP__ +#define MSGPACK_TYPE_MAP_HPP__ + +#include "msgpack/object.hpp" +#include +#include + +namespace msgpack { +namespace type { + + +template +class assoc_vector : public std::vector< std::pair > {}; + +namespace detail { + template + struct pair_first_less { + bool operator() (const std::pair& x, const std::pair& y) const + { return x.first < y.first; } + }; +} + +template +inline assoc_vector& operator<< (assoc_vector& v, object o) +{ + if(o.type != MAP) { throw type_error(); } + v.resize(o.via.container.size); + object* p(o.via.container.ptr); + object* const pend(o.via.container.ptr + o.via.container.size); + std::pair* it(&v.front()); + for(; p < pend; ++it) { + convert(it->first, *p); ++p; + convert(it->second, *p); ++p; + } + std::sort(v.begin(), v.end(), detail::pair_first_less()); + return v; +} + +template +inline const assoc_vector& operator>> (const assoc_vector& v, packer& o) +{ + o.pack_map(v.size()); + for(typename assoc_vector::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + pack(it->first, o); + pack(it->second, o); + } +} + + +template +inline std::map operator<< (std::map& v, object o) +{ + if(o.type != MAP) { throw type_error(); } + object* p(o.via.container.ptr); + object* const pend(o.via.container.ptr + o.via.container.size*2); + while(p < pend) { + K key; + convert(key, *p); ++p; + typename std::map::iterator it(v.find(key)); + if(it != v.end()) { + V val; + convert(val, *p); ++p; + it->insert( std::pair(key, val) ); + } else { + convert(it->second, *p); ++p; + } + } + return v; +} + +template +inline const std::map& operator>> (const std::map& v, packer& o) +{ + o.pack_map(v.size()); + for(typename std::map::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + pack(it->first, o); + pack(it->second, o); + } +} + + +template +inline std::multimap operator<< (std::multimap& v, object o) +{ + if(o.type != MAP) { throw type_error(); } + object* p(o.via.container.ptr); + object* const pend(o.via.container.ptr + o.via.container.size*2); + while(p < pend) { + std::pair value; + convert(value.first, *p); ++p; + convert(value.second, *p); ++p; + v.insert(value); + } + return v; +} + +template +inline const std::multimap& operator>> (const std::multimap& v, packer& o) +{ + o.pack_multimap(v.size()); + for(typename std::multimap::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + pack(it->first, o); + pack(it->second, o); + } +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/map.hpp */ + diff --git a/cpp/type/nil.hpp b/cpp/type/nil.hpp new file mode 100644 index 0000000..73cedb4 --- /dev/null +++ b/cpp/type/nil.hpp @@ -0,0 +1,47 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_NIL_HPP__ +#define MSGPACK_TYPE_NIL_HPP__ + +#include "msgpack/object.hpp" + +namespace msgpack { +namespace type { + + +struct nil { }; + +inline nil& operator<< (nil& v, object o) +{ + if(o.type != NIL) { throw type_error(); } + return v; +} + +template +inline const nil& operator>> (const nil& v, packer& o) +{ + o.pack_nil(); + return v; +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/nil.hpp */ + diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp new file mode 100644 index 0000000..8767238 --- /dev/null +++ b/cpp/type/raw.hpp @@ -0,0 +1,99 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_RAW_HPP__ +#define MSGPACK_TYPE_RAW_HPP__ + +#include "msgpack/object.hpp" +#include +#include + +namespace msgpack { +namespace type { + + +struct raw_ref { + raw_ref() : ptr(NULL), size(0) {} + raw_ref(const char* p, uint32_t s) : ptr(p), size(s) {} + + const char* ptr; + uint32_t size; + + std::string str() { return std::string(ptr, size); } + + bool operator== (const raw_ref& x) + { + return size == x.size && memcmp(ptr, x.ptr, size) == 0; + } + + bool operator!= (const raw_ref& x) + { + return !(*this != x); + } + + bool operator< (const raw_ref& x) + { + if(size == x.size) { return memcmp(ptr, x.ptr, size) < 0; } + else { return size < x.size; } + } + + bool operator> (const raw_ref& x) + { + if(size == x.size) { return memcmp(ptr, x.ptr, size) > 0; } + else { return size > x.size; } + } +}; + +inline raw_ref& operator<< (raw_ref& v, object o) +{ + if(o.type != RAW) { throw type_error(); } + v.ptr = o.via.ref.ptr; + v.size = o.via.ref.size; + return v; +} + + +inline std::string& operator<< (std::string& v, object o) +{ + raw_ref r; + r << o; + v.assign(r.ptr, r.size); + return v; +} + + +template +inline const raw_ref& operator>> (const raw_ref& v, packer& o) +{ + o.pack_raw(v.ptr, v.size); + return v; +} + + +template +inline const std::string& operator>> (const std::string& v, packer& o) +{ + o.pack_raw(v.data(), v.size()); + return v; +} + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/raw.hpp */ + diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb new file mode 100644 index 0000000..2a8a2dd --- /dev/null +++ b/cpp/type/tuple.hpp.erb @@ -0,0 +1,172 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_TUPLE_HPP__ +#define MSGPACK_TYPE_TUPLE_HPP__ + +#include "msgpack/object.hpp" + +namespace msgpack { +namespace type { + + +// FIXME operator== +// FIXME operator!= + +<% GENERATION_LIMIT = 15 %> + +template < typename A0 <%1.upto(GENERATION_LIMIT+1) {|i|%>, typename A<%=i%> = void<%}%> > +struct tuple; + +template +struct tuple_type; + +template +struct const_tuple_type; + +template +struct tuple_element { + typedef T type; + tuple_element(T& x) : _x(x) {} + type& get() { return _x; } + const type& get() const { return _x; } +private: + type& _x; +}; + +template +struct const_tuple_element { + typedef T type; + const_tuple_element(const T& x) : _x(x) {} + const type& get() const { return _x; } +private: + const type& _x; +}; + + +<%0.upto(GENERATION_LIMIT) {|i|%> +<%0.upto(i) {|j|%> +template < typename A0 <%1.upto(i) {|k|%>, typename A<%=k%> <%}%>> +struct tuple_type, A<%=k%> <%}%>>, <%=j%>> : tuple_element> { + tuple_type(tuple, A<%=k%> <%}%>>& x) : tuple_element>(x.a<%=j%>) {} +}; +<%}%> +<%}%> + + +<%0.upto(GENERATION_LIMIT) {|i|%> +<%0.upto(i) {|j|%> +template < typename A0 <%1.upto(i) {|k|%>, typename A<%=k%> <%}%>> +struct const_tuple_type, A<%=k%> <%}%>>, <%=j%>> : const_tuple_element> { + const_tuple_type(const tuple, A<%=k%> <%}%>>& x) : const_tuple_element>(x.a<%=j%>) {} +}; +<%}%> +<%}%> + + +<%0.upto(GENERATION_LIMIT) {|i|%> +template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> +tuple< A0 <%1.upto(i) {|j|%>, A<%=j%> <%}%>> make_tuple(const A0& a0 <%1.upto(i) {|j|%>, const A<%=j%>& a<%=j%><%}%>) +{ + return tuple< A0 <%1.upto(i) {|j|%>, A<%=j%> <%}%>>(a0 <%1.upto(i) {|j|%>, a<%=j%><%}%>); +} +<%}%> + + +<%0.upto(GENERATION_LIMIT) {|i|%> +template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> +struct tuple, A<%=j%> <%}%>> { + tuple() {} + tuple(const A0& _a0 <%1.upto(i) {|j|%>, const A<%=j%>& _a<%=j%><%}%>) : + a0(_a0) <%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} + tuple(object o) { convert(*this, o); } + template typename tuple_type, A<%=j%> <%}%>>, N>::type& get() + { return tuple_type, A<%=j%> <%}%>>, N>(*this).get(); } + template const typename const_tuple_type, A<%=j%> <%}%>>, N>::type& get() const + { return const_tuple_type, A<%=j%> <%}%>>, N>(*this).get(); } + <%0.upto(i) {|j|%> + A<%=j%> a<%=j%>;<%}%> +}; +<%}%> + + +<%0.upto(GENERATION_LIMIT) {|i|%> +template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> +tuple, A<%=j%> <%}%>>& operator<< ( + tuple, A<%=j%> <%}%>>& v, + object o) { + if(o.type != ARRAY) { throw type_error(); } + if(o.via.container.size < <%=i+1%>) { throw type_error(); } + <%0.upto(i) {|j|%> + convert>(v.template get<<%=j%>>(), o.via.container.ptr[<%=j%>]);<%}%> + return v; +} +<%}%> + + +// FIXME +/* +template +struct tuple_just; + +template +struct tuple_just { + A0 a0; + static inline void convert(object o, tuple_just& v) + { + if(o.type != ARRAY) { throw type_error(); } + if(o.v.container.size != 1) { throw type_error(); } + msgpack::convert(o.v.container.ptr[0], v.a0); + } +}; + +template +struct tuple_just { + A0 a0; + A1 a1; + static inline void convert(object o, tuple_just& v) + { + if(o.type != ARRAY) { throw type_error(); } + if(o.v.container.size != 2) { throw type_error(); } + msgpack::convert(o.v.container.ptr[0], v.a0); + msgpack::convert(o.v.container.ptr[1], v.a1); + } +}; +*/ + + + +<%0.upto(GENERATION_LIMIT) {|i|%> +template < typename Stream , typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> +const tuple, A<%=j%> <%}%>>& operator>> ( + const tuple, A<%=j%> <%}%>>& v, + packer o) { + o.pack_array(<%=i+1%>); + <%0.upto(i) {|j|%> + pack(v.template get<<%=j%>>(), o);<%}%> + return v; +} +<%}%> + + + +} // namespace type +} // namespace msgpack + +#endif /* msgpack/type/tuple.hpp */ + + diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 873d3da..71593a0 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -31,7 +31,7 @@ namespace msgpack { #define msgpack_unpack_callback(name) \ msgpack_unpack_##name -#define msgpack_unpack_object object_class* +#define msgpack_unpack_object object #define msgpack_unpack_user zone* @@ -40,68 +40,88 @@ struct msgpack_unpacker_context; static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); -static object_class* msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); +static object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, const char* data, size_t len, size_t* off); -static inline object_class* msgpack_unpack_init(zone** z) -{ return NULL; } +static inline object msgpack_unpack_init(zone** z) +{ return object(); } -static inline object_class* msgpack_unpack_uint8(zone** z, uint8_t d) -{ return (*z)->nu8(d); } +static inline object msgpack_unpack_uint8(zone** z, uint8_t d) +{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object_class* msgpack_unpack_uint16(zone** z, uint16_t d) -{ return (*z)->nu16(d); } +static inline object msgpack_unpack_uint16(zone** z, uint16_t d) +{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object_class* msgpack_unpack_uint32(zone** z, uint32_t d) -{ return (*z)->nu32(d); } +static inline object msgpack_unpack_uint32(zone** z, uint32_t d) +{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object_class* msgpack_unpack_uint64(zone** z, uint64_t d) -{ return (*z)->nu64(d); } +static inline object msgpack_unpack_uint64(zone** z, uint64_t d) +{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object_class* msgpack_unpack_int8(zone** z, int8_t d) -{ return (*z)->ni8(d); } +static inline object msgpack_unpack_int8(zone** z, int8_t d) +{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object_class* msgpack_unpack_int16(zone** z, int16_t d) -{ return (*z)->ni16(d); } +static inline object msgpack_unpack_int16(zone** z, int16_t d) +{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object_class* msgpack_unpack_int32(zone** z, int32_t d) -{ return (*z)->ni32(d); } +static inline object msgpack_unpack_int32(zone** z, int32_t d) +{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object_class* msgpack_unpack_int64(zone** z, int64_t d) -{ return (*z)->ni64(d); } +static inline object msgpack_unpack_int64(zone** z, int64_t d) +{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object_class* msgpack_unpack_float(zone** z, float d) -{ return (*z)->nfloat(d); } +static inline object msgpack_unpack_float(zone** z, float d) +{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object_class* msgpack_unpack_double(zone** z, double d) -{ return (*z)->ndouble(d); } +static inline object msgpack_unpack_double(zone** z, double d) +{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object_class* msgpack_unpack_nil(zone** z) -{ return (*z)->nnil(); } +static inline object msgpack_unpack_nil(zone** z) +{ object o; o.type = type::NIL; return o; } -static inline object_class* msgpack_unpack_true(zone** z) -{ return (*z)->ntrue(); } +static inline object msgpack_unpack_true(zone** z) +{ object o; o.type = type::BOOLEAN; o.via.boolean = true; return o; } -static inline object_class* msgpack_unpack_false(zone** z) -{ return (*z)->nfalse(); } +static inline object msgpack_unpack_false(zone** z) +{ object o; o.type = type::BOOLEAN; o.via.boolean = false; return o; } -static inline object_class* msgpack_unpack_array(zone** z, unsigned int n) -{ return (*z)->narray(n); } +static inline object msgpack_unpack_array(zone** z, unsigned int n) +{ + object o; + o.type = type::ARRAY; + o.via.container.size = 0; + o.via.container.ptr = (*z)->malloc_container(n); + return o; +} -static inline void msgpack_unpack_array_item(zone** z, object_class* c, object_class* o) -{ reinterpret_cast(c)->push_back(o); } +static inline void msgpack_unpack_array_item(zone** z, object* c, object o) +{ c->via.container.ptr[ c->via.container.size++ ] = o; } -static inline object_class* msgpack_unpack_map(zone** z, unsigned int n) -{ return (*z)->nmap(); } +static inline object msgpack_unpack_map(zone** z, unsigned int n) +{ + object o; + o.type = type::MAP; + o.via.container.size = 0; + o.via.container.ptr = (*z)->malloc_container(n*2); + return o; +} -static inline void msgpack_unpack_map_item(zone** z, object_class* c, object_class* k, object_class* v) -{ reinterpret_cast(c)->store(k, v); } +static inline void msgpack_unpack_map_item(zone** z, object* c, object k, object v) +{ + c->via.container.ptr[ c->via.container.size ] = k; + c->via.container.ptr[ c->via.container.size+1 ] = v; + ++c->via.container.size; +} -static inline object_class* msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) -{ return (*z)->nraw_ref(p, l); } +static inline object msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) +{ object o; o.type = type::RAW; o.via.ref.ptr = p; o.via.ref.size = l; return o; } #include "msgpack/unpack_template.h" @@ -121,7 +141,7 @@ struct unpacker::context { return msgpack_unpacker_execute(&m_ctx, data, len, off); } - object_class* data() + object data() { return msgpack_unpacker_data(&m_ctx); } @@ -158,9 +178,8 @@ private: }; -unpacker::unpacker() : - m_zone(new zone()), - m_ctx(new context(m_zone)), +unpacker::unpacker(zone& z) : + m_ctx(new context(&z)), m_buffer(NULL), m_used(0), m_free(0), @@ -170,9 +189,7 @@ unpacker::unpacker() : unpacker::~unpacker() { - free(m_buffer); delete m_ctx; - delete m_zone; } @@ -184,32 +201,16 @@ void unpacker::expand_buffer(size_t len) else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } - char* tmp = (char*)realloc(m_buffer, next_size); - if(!tmp) { throw std::bad_alloc(); } - m_buffer = tmp; - //char* tmp = (char*)malloc(next_size); - //if(!tmp) { throw std::bad_alloc(); } - //memcpy(tmp, m_buffer, m_used); - //free(m_buffer); - - m_buffer = tmp; + m_buffer = m_ctx->user()->realloc(m_buffer, next_size); m_free = next_size - m_used; } else { size_t next_size = UNPACKER_INITIAL_BUFFER_SIZE; while(next_size < len + m_used - m_off) { next_size *= 2; } - char* tmp = (char*)malloc(next_size); - if(!tmp) { throw std::bad_alloc(); } + char* tmp = m_ctx->user()->malloc(next_size); memcpy(tmp, m_buffer+m_off, m_used-m_off); - try { - m_zone->push_finalizer(&zone::finalize_free, NULL, m_buffer); - } catch (...) { - free(tmp); - throw; - } - m_buffer = tmp; m_used = m_used - m_off; m_free = next_size - m_used; @@ -230,29 +231,15 @@ bool unpacker::execute() } } -zone* unpacker::release_zone() -{ - zone* nz = new zone(); - zone* z = m_zone; - m_zone = nz; - m_ctx->user(m_zone); - return z; -} - object unpacker::data() { - return object(m_ctx->data()); + return m_ctx->data(); } -void unpacker::reset() +void unpacker::reset(zone& z) { if(m_off != 0) { expand_buffer(0); } - if(!m_zone->empty()) { - delete m_zone; - m_zone = NULL; - m_zone = new zone(); - } - m_ctx->reset(); + m_ctx->reset(&z); } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 86cfb6e..2730a08 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -40,7 +40,7 @@ struct unpack_error : public std::runtime_error { class unpacker { public: - unpacker(); + unpacker(zone& z); ~unpacker(); public: @@ -60,13 +60,8 @@ public: /*! 5.1. if execute() returns true, take out the parsed object */ object data(); - /*! 5.2. the parsed object is valid until the zone is deleted */ - // Note that once release_zone() from unpacker, you must delete it - // otherwise the memrory will leak. - zone* release_zone(); - - /*! 5.3. after release_zone(), re-initialize unpacker */ - void reset(); + /*! 5.2. re-initialize unpacker with next zone */ + void reset(zone& z); public: // These functions are usable when non-MessagePack message follows after @@ -85,8 +80,6 @@ public: void remove_nonparsed_buffer(); private: - zone* m_zone; - struct context; context* m_ctx; @@ -94,9 +87,12 @@ private: size_t m_used; size_t m_free; size_t m_off; + +private: void expand_buffer(size_t len); private: + unpacker(); unpacker(const unpacker&); public: diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 4cc50d7..490bc25 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -20,52 +20,35 @@ namespace msgpack { -// FIXME custom allocator? +zone::zone() { } -void zone::expand_chunk() -{ - cell_t* chunk = (cell_t*)malloc(sizeof(cell_t)*ZONE_CHUNK_SIZE); - if(!chunk) { throw std::bad_alloc(); } - try { - m_pool.push_back(chunk); - } catch (...) { - free(chunk); - throw; - } -} +zone::~zone() { clear(); } void zone::clear() { - if(!m_pool.empty()) { - size_t base_size = m_used / ZONE_CHUNK_SIZE; - size_t extend_size = m_used % ZONE_CHUNK_SIZE; - for(size_t b=0; b < base_size; ++b) { - cell_t* c(m_pool[b]); - for(size_t e=0; e < ZONE_CHUNK_SIZE; ++e) { - reinterpret_cast(c[e].data)->~object_class(); + for(std::vector::iterator it(m_ptrs.begin()), it_end(m_ptrs.end()); + it != it_end; ++it) { + free(*it); + } + m_ptrs.clear(); +} + +char* zone::realloc(char* ptr, size_t count) +{ + if(ptr == NULL) { + return zone::malloc(count); + } else { + for(std::vector::reverse_iterator it(m_ptrs.rbegin()), it_end(m_ptrs.rend()); + it != it_end; ++it) { + if(*it == ptr) { + char* tmp = (char*)::realloc(ptr, count); + if(!tmp) { throw std::bad_alloc(); } + *it = tmp; + return tmp; } } - cell_t* c(m_pool.back()); - for(size_t e=0; e < extend_size; ++e) { - reinterpret_cast(c[e].data)->~object_class(); - } - - for(pool_t::iterator it(m_pool.begin()), it_end(m_pool.end()); - it != it_end; - ++it) { - free(*it); - } - m_pool.clear(); + throw std::bad_alloc(); } - m_used = 0; - - for(user_finalizer_t::reverse_iterator it(m_user_finalizer.rbegin()), - it_end(m_user_finalizer.rend()); - it != it_end; - ++it) { - it->call(); - } - m_user_finalizer.clear(); } diff --git a/cpp/zone.hpp b/cpp/zone.hpp new file mode 100644 index 0000000..f8c9cba --- /dev/null +++ b/cpp/zone.hpp @@ -0,0 +1,70 @@ +// +// MessagePack for C++ memory pool +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_ZONE_HPP__ +#define MSGPACK_ZONE_HPP__ + +#include "msgpack/object.hpp" +#include +#include + +namespace msgpack { + + +class zone { +public: + zone(); + ~zone(); + +public: + char* malloc(size_t count); + char* realloc(char* ptr, size_t count); + object* malloc_container(size_t count); + + void clear(); + +private: + std::vector m_ptrs; + +private: + zone(const zone&); +}; + + +inline char* zone::malloc(size_t count) +{ + char* ptr = (char*)::malloc(count); + if(!ptr) { throw std::bad_alloc(); } + try { + m_ptrs.push_back(ptr); + } catch (...) { + free(ptr); + throw; + } + return ptr; +} + +inline object* zone::malloc_container(size_t count) +{ + return (object*)zone::malloc(sizeof(object)*count); +} + + +} // namespace msgpack + +#endif /* msgpack/zone.hpp */ + diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb deleted file mode 100644 index 8af8da2..0000000 --- a/cpp/zone.hpp.erb +++ /dev/null @@ -1,202 +0,0 @@ -// -// MessagePack for C++ memory pool -// -// Copyright (C) 2008 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#ifndef MSGPACK_ZONE_HPP__ -#define MSGPACK_ZONE_HPP__ -#include - -#include "msgpack/object.hpp" -#include -#include -#include - -#ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 1024 -#endif - -namespace msgpack { - - -static const size_t ZONE_CHUNK_SIZE = MSGPACK_ZONE_CHUNK_SIZE; - - -class zone { -public: - zone() : m_used(0) { } - ~zone() { clear(); } - -public: - template - void push_finalizer(void (*func)(void* obj, void* user), T* obj, void* user); - -public: - object_nil* nnil () { return new (alloc()) object_nil(); } - object_true* ntrue () { return new (alloc()) object_true(); } - object_false* nfalse () { return new (alloc()) object_false(); } - object_u8* nu8 (uint8_t v) { return new (alloc()) object_u8(v); } - object_u16* nu16 (uint16_t v) { return new (alloc()) object_u16(v); } - object_u32* nu32 (uint32_t v) { return new (alloc()) object_u32(v); } - object_u64* nu64 (uint64_t v) { return new (alloc()) object_u64(v); } - object_i8* ni8 (int8_t v) { return new (alloc()) object_i8(v); } - object_i16* ni16 (int16_t v) { return new (alloc()) object_i16(v); } - object_i32* ni32 (int32_t v) { return new (alloc()) object_i32(v); } - object_i64* ni64 (int64_t v) { return new (alloc()) object_i64(v); } - object_float* nfloat (float v) { return new (alloc()) object_float(v); } - object_double* ndouble(double v) { return new (alloc()) object_double(v); } - - - object_mutable_raw_ref* nraw_ref(char* ptr, uint32_t len) - { return new (alloc()) object_mutable_raw_ref(ptr, len); } - - object_raw_ref* nraw_ref(const char* ptr, uint32_t len) - { return new (alloc()) object_raw_ref(ptr, len); } - - object_mutable_raw_ref* nraw_copy(const char* ptr, uint32_t len) - { - char* copy = (char*)malloc(len); - if(!copy) { throw std::bad_alloc(); } - object_mutable_raw_ref* o; - try { - o = new (alloc()) object_mutable_raw_ref(copy, len); - push_finalizer(&zone::finalize_free, NULL, copy); - } catch (...) { - free(copy); - throw; - } - memcpy(copy, ptr, len); - return o; - } - - - object_mutable_raw_ref* nraw_cstr_ref(char* str) - { return nraw_ref(str, strlen(str)); } - - object_raw_ref* nraw_cstr_ref(const char* str) - { return nraw_ref(str, strlen(str)); } - - object_mutable_raw_ref* nraw_cstr_copy(const char* str) - { return nraw_copy(str, strlen(str)); } - - - object_array* narray() - { return new (alloc()) object_array(); } - - object_array* narray(size_t reserve_size) - { return new (alloc()) object_array(reserve_size); } - - object_map* nmap() - { return new (alloc()) object_map(); } - -<% GENERATION_SIZE = 16 %> -<% 1.upto(GENERATION_SIZE) {|i| %> - object_array* narray(<% 1.upto(i-1) {|n| %>object o<%=n%>, <% } %>object o<%=i%>) - { object_array* a = new (alloc()) object_array(<%=i%>); - <% 1.upto(i) {|n| %>a->push_back(o<%=n%>); - <% } %>return a; } -<% } %> - -<% 1.upto(GENERATION_SIZE) {|i| %> - object_map* nmap(<% 1.upto(i-1) {|n| %>object k<%=n%>, object v<%=n%>, <% } %>object k<%=i%>, object v<%=i%>) - { object_map* m = new (alloc()) object_map(); - <% 1.upto(i) {|n| %>m->store(k<%=n%>, v<%=n%>); - <% } %>return m; } -<% } %> - -public: - void clear(); - bool empty() const; - -private: - void* alloc(); - -private: - size_t m_used; - - static const size_t MAX_OBJECT_SIZE = - sizeof(object_raw_ref) > sizeof(object_array) - ? ( sizeof(object_raw_ref) > sizeof(object_map) - ? sizeof(object_raw_ref) - : sizeof(object_map) - ) - : ( sizeof(object_array) > sizeof(object_map) - ? sizeof(object_array) - : sizeof(object_map) - ) - ; - - struct cell_t { - char data[MAX_OBJECT_SIZE]; - }; - - typedef std::vector pool_t; - pool_t m_pool; - - - class finalizer { - public: - finalizer(void (*func)(void*, void*), void* obj, void* user) : - m_obj(obj), m_user(user), m_func(func) {} - void call() { (*m_func)(m_obj, m_user); } - private: - void* m_obj; - void* m_user; - void (*m_func)(void*, void*); - }; - - typedef std::vector user_finalizer_t; - user_finalizer_t m_user_finalizer; - -private: - void expand_chunk(); - -public: - static void finalize_free(void* obj, void* user) - { free(user); } - -private: - zone(const zone&); -}; - - -template -inline void zone::push_finalizer(void (*func)(void* obj, void* user), T* obj, void* user) -{ - m_user_finalizer.push_back( finalizer( - func, reinterpret_cast(obj), - user) ); -} - -inline bool zone::empty() const -{ - return m_used == 0 && m_user_finalizer.empty(); -} - -inline void* zone::alloc() -{ - if(m_pool.size() <= m_used/ZONE_CHUNK_SIZE) { - expand_chunk(); - } - void* data = m_pool[m_used/ZONE_CHUNK_SIZE][m_used%ZONE_CHUNK_SIZE].data; - ++m_used; - return data; -} - - -} // namespace msgpack - -#endif /* msgpack/zone.hpp */ - diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 22d9911..928b581 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -146,7 +146,6 @@ msgpack_pack_inline_func(uint32)(msgpack_pack_user x, uint32_t d) msgpack_pack_inline_func(uint64)(msgpack_pack_user x, uint64_t d) { - // FIXME optimization const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -177,7 +176,6 @@ msgpack_pack_inline_func(int32)(msgpack_pack_user x, int32_t d) msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) { - // FIXME optimization const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 8197fa4..a6d7dcb 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -279,7 +279,7 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - msgpack_unpack_callback(array_item)(user, c->obj, obj); + msgpack_unpack_callback(array_item)(user, &c->obj, obj); if(--c->count == 0) { obj = c->obj; --top; @@ -292,7 +292,7 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - msgpack_unpack_callback(map_item)(user, c->obj, c->map_key, obj); + msgpack_unpack_callback(map_item)(user, &c->obj, c->map_key, obj); if(--c->count == 0) { obj = c->obj; --top; diff --git a/ruby/unpack.c b/ruby/unpack.c index 8439c02..b948aa6 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -94,14 +94,14 @@ static inline VALUE template_callback_false(msgpack_unpack_context* x) static inline VALUE template_callback_array(msgpack_unpack_context* x, unsigned int n) { return rb_ary_new2(n); } -static inline void template_callback_array_item(msgpack_unpack_context* x, VALUE c, VALUE o) -{ rb_ary_push(c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] +static inline void template_callback_array_item(msgpack_unpack_context* x, VALUE* c, VALUE o) +{ rb_ary_push(*c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] static inline VALUE template_callback_map(msgpack_unpack_context* x, unsigned int n) { return rb_hash_new(); } -static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE c, VALUE k, VALUE v) -{ rb_hash_aset(c, k, v); } +static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE* c, VALUE k, VALUE v) +{ rb_hash_aset(*c, k, v); } static inline VALUE template_callback_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) { return rb_str_new(p, l); } From b790df530aa28099aa130dafad0a591004b88650 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0021/1648] lang/c/msgpack: C++ binding: safer memory managent git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@68 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile.am | 2 ++ cpp/test.cpp | 16 ++++++------- cpp/unpack.cpp | 48 +++++++++++++++++++++++++++++++-------- cpp/unpack.hpp | 60 +++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 102 insertions(+), 24 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 8b13bea..e244763 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -14,9 +14,11 @@ nobase_include_HEADERS = \ msgpack/type.hpp \ msgpack/type/array.hpp \ msgpack/type/boolean.hpp \ + msgpack/type/float.hpp \ msgpack/type/integer.hpp \ msgpack/type/map.hpp \ msgpack/type/nil.hpp \ + msgpack/type/raw.hpp \ msgpack/type/tuple.hpp noinst_HEADERS = \ diff --git a/cpp/test.cpp b/cpp/test.cpp index dff9101..c632ec5 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -152,9 +152,7 @@ int main(void) unsigned num_msg = 0; static const size_t RESERVE_SIZE = 32;//*1024; - std::auto_ptr pz(new zone()); - - unpacker pac(*pz); + unpacker pac; while(stream.good() && total_bytes > 0) { @@ -179,13 +177,15 @@ int main(void) // 5.1. take out the parsed object object o = pac.data(); - // do something using pz and o + // 5.2 release the zone + std::auto_ptr olife( pac.release_zone() ); + + // 5.3 re-initialize the unpacker */ + pac.reset(); + + // do some with the o and olife std::cout << "message parsed: " << o << std::endl; ++num_msg; - - // 5.3 re-initialize unpacker with next zone */ - pz.reset(new zone()); - pac.reset(*pz); } } diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 71593a0..ce39afd 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -178,12 +178,13 @@ private: }; -unpacker::unpacker(zone& z) : - m_ctx(new context(&z)), +unpacker::unpacker() : m_buffer(NULL), m_used(0), m_free(0), - m_off(0) + m_off(0), + m_zone(new zone()), + m_ctx(new context(&*m_zone)) { } @@ -201,14 +202,14 @@ void unpacker::expand_buffer(size_t len) else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } - m_buffer = m_ctx->user()->realloc(m_buffer, next_size); + m_buffer = m_zone->realloc(m_buffer, next_size); m_free = next_size - m_used; } else { size_t next_size = UNPACKER_INITIAL_BUFFER_SIZE; while(next_size < len + m_used - m_off) { next_size *= 2; } - char* tmp = m_ctx->user()->malloc(next_size); + char* tmp = m_zone->malloc(next_size); memcpy(tmp, m_buffer+m_off, m_used-m_off); m_buffer = tmp; @@ -226,20 +227,49 @@ bool unpacker::execute() } else if(ret == 0) { return false; } else { - expand_buffer(0); return true; } } +zone* unpacker::release_zone() +{ + zone* n = new zone(); + std::auto_ptr old(m_zone.release()); + m_zone.reset(n); + + //std::auto_ptr old(new zone()); + //m_zone.swap(old); + + // move all bytes in m_buffer to new buffer from the new zone + if(m_used <= m_off) { + m_buffer = NULL; + m_used = 0; + m_free = 0; + m_off = 0; + } else { + try { + expand_buffer(0); + } catch (...) { + // m_zone.swap(old); + zone* tmp = old.release(); + old.reset(m_zone.release()); + m_zone.reset(tmp); + throw; + } + } + m_ctx->user(&*m_zone); + return old.release(); +} + object unpacker::data() { return m_ctx->data(); } -void unpacker::reset(zone& z) +void unpacker::reset() { - if(m_off != 0) { expand_buffer(0); } - m_ctx->reset(&z); + if(m_off != 0) { std::auto_ptr old(release_zone()); } + m_ctx->reset(); } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 2730a08..07c52e7 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -20,6 +20,7 @@ #include "msgpack/object.hpp" #include "msgpack/zone.hpp" +#include #include #ifndef MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE @@ -40,7 +41,7 @@ struct unpack_error : public std::runtime_error { class unpacker { public: - unpacker(zone& z); + unpacker(); ~unpacker(); public: @@ -60,8 +61,52 @@ public: /*! 5.1. if execute() returns true, take out the parsed object */ object data(); - /*! 5.2. re-initialize unpacker with next zone */ - void reset(zone& z); + /*! 5.2. the object is valid until the zone is deleted */ + // Note that once release_zone() from unpacker, you must delete it + // otherwise the memrory will leak. + zone* release_zone(); + + /*! 5.3. after release_zone(), re-initialize unpacker */ + void reset(); + + + // Basic usage of the unpacker is as following: + // + // msgpack::unpacker pac; + // + // while( /* readable */ ) { + // + // // 1. + // pac.reserve(1024); + // + // // 2. + // ssize_t bytes = + // read(the_source, pac.buffer, pac.buffer_capacity()); + // + // // error handling ... + // + // // 3. + // pac.buffer_consumed(bytes); + // + // // 4. + // while(pac.execute()) { + // // 5.1 + // object o = pac.data(); + // + // // 5.2 + // std::auto_ptr olife( pac.release_zone() ); + // + // // boost::shared_ptr is also usable: + // // boost::shared_ptr olife( pac.release_zone() ); + // + // // 5.3 + // pac.reset(); + // + // // do some with the object with the old zone. + // do_something(o, olife); + // } + // } + // public: // These functions are usable when non-MessagePack message follows after @@ -80,19 +125,20 @@ public: void remove_nonparsed_buffer(); private: - struct context; - context* m_ctx; - char* m_buffer; size_t m_used; size_t m_free; size_t m_off; + std::auto_ptr m_zone; + + struct context; + context* m_ctx; + private: void expand_buffer(size_t len); private: - unpacker(); unpacker(const unpacker&); public: From a7936ba05b15ec7a19e8dc75667fec2df13b7ea7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0022/1648] lang/c/msgpack: C++ binding: implemented msgpack::object >> packer git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@69 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.cpp | 11 ++---- cpp/object.hpp | 92 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 92 insertions(+), 11 deletions(-) diff --git a/cpp/object.cpp b/cpp/object.cpp index 276732c..45cfb5f 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -81,7 +81,7 @@ std::ostream& operator<< (std::ostream& s, const object o) default: // FIXME - s << "#" << std::endl; + s << "#"; } return s; } @@ -113,7 +113,7 @@ bool operator==(const object x, const object y) for(object* px(x.via.container.ptr), * const pxend(x.via.container.ptr + x.via.container.size), * py(y.via.container.ptr); - px != pxend; ++px, ++py) { + px < pxend; ++px, ++py) { if(*px != *py) { return false; } } return true; @@ -124,7 +124,7 @@ bool operator==(const object x, const object y) for(object* px(x.via.container.ptr), * const pxend(x.via.container.ptr + x.via.container.size*2), * py(y.via.container.ptr); - px != pxend; ++px, ++py) { + px < pxend; ++px, ++py) { if(*px != *py) { return false; } } return true; @@ -135,10 +135,5 @@ bool operator==(const object x, const object y) } -// FIXME -//template -//const object& operator>> (const object& v, packer& o); - - } // namespace msgpack diff --git a/cpp/object.hpp b/cpp/object.hpp index 7007b90..a5e8682 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include namespace msgpack { @@ -46,7 +47,9 @@ struct object { uint32_t size; } ref; } via; - // FIXME template operator T() { T v; convert(*this, v); return v; }; + + template + operator T() { T v; convert(v, *this); return v; }; }; std::ostream& operator<< (std::ostream& s, const object o); @@ -55,6 +58,9 @@ bool operator==(const object x, const object y); inline bool operator!=(const object x, const object y) { return !(x == y); } +inline object& operator<< (object& v, object o) + { v = o; return v; } + template const object& operator>> (const object& v, packer& o); @@ -73,7 +79,7 @@ namespace type { template inline T& operator<< (T& v, object o) { - v = o; + v.msgpack_unpack(o); return v; } @@ -87,7 +93,7 @@ namespace type { template inline const T& operator>> (const T& v, packer& o) { - detail::pack_copy(v.pack(), o); + detail::pack_copy(v.msgpack_pack(), o); return v; } @@ -118,6 +124,86 @@ inline void pack(T& v, Stream& s) } + +template +const object& operator>> (const object& v, packer& o) +{ + switch(v.type) { + case type::NIL: + o.pack_nil(); + return v; + + case type::BOOLEAN: + if(v.via.boolean) { + o.pack_true(); + } else { + o.pack_false(); + } + return v; + + case type::POSITIVE_INTEGER: + if(v.via.u64 <= (uint64_t)std::numeric_limits::max()) { + if(v.via.u64 <= (uint16_t)std::numeric_limits::max()) { + o.pack_uint8(v.via.u64); + } else { + o.pack_uint16(v.via.u64); + } + } else { + if(v.via.u64 <= (uint64_t)std::numeric_limits::max()) { + o.pack_uint32(v.via.u64); + } else { + o.pack_uint64(v.via.u64); + } + } + return v; + + case type::NEGATIVE_INTEGER: + if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { + if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { + o.pack_int8(v.via.i64); + } else { + o.pack_int16(v.via.i64); + } + } else { + if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { + o.pack_int64(v.via.i64); + } else { + o.pack_int64(v.via.i64); + } + } + return v; + + case type::RAW: + o.pack_raw(v.via.ref.ptr, v.via.ref.size); + return v; + + case type::ARRAY: + o.pack_array(v.via.container.size); + for(object* p(v.via.container.ptr), + * const pend(v.via.container.ptr + v.via.container.size); + p < pend; ++p) { + *p >> o; + } + return v; + // FIXME loop optimiziation + + case type::MAP: + o.pack_map(v.via.container.size); + for(object* p(v.via.container.ptr), + * const pend(v.via.container.ptr + v.via.container.size*2); + p < pend; ) { + *p >> o; ++p; + *p >> o; ++p; + } + return v; + // FIXME loop optimiziation + + default: + throw type_error(); + } +} + + } // namespace msgpack #include "msgpack/type.hpp" From 31659738110f7d73d8fd4fe2d6fc62991ac7912f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0023/1648] lang/c/msgpack: fix compile optimization flag git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@70 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/configure.in | 7 ++++++- cpp/configure.in | 7 ++++++- cpp/test.mk | 9 +++++++++ cpp/zone.cpp | 26 +++++++++--------------- cpp/zone.hpp | 16 +++++++++++++++ msgpack/pack_template.h | 44 ++++++++++++++++++++++++----------------- ruby/extconf.rb | 2 +- ruby/test_pack.rb | 2 ++ ruby/unpack.c | 10 +++++----- 9 files changed, 80 insertions(+), 43 deletions(-) create mode 100644 cpp/test.mk diff --git a/c/configure.in b/c/configure.in index 1cacbc4..0017917 100644 --- a/c/configure.in +++ b/c/configure.in @@ -2,10 +2,15 @@ AC_INIT(pack.c) AM_INIT_AUTOMAKE(msgpackc, 0.1.0) AC_CONFIG_HEADER(config.h) +AC_SUBST(CFLAGS) +if test "" = "$CFLAGS"; then + CFLAGS="-g -O4" +fi + AC_PROG_CC AC_PROG_LIBTOOL -CFLAGS="-O4 $CFLAGS -Wall -I.." +CFLAGS="-O4 -Wall $CFLAGS -I.." AC_OUTPUT([Makefile]) diff --git a/cpp/configure.in b/cpp/configure.in index 5126be4..a60a489 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -2,13 +2,18 @@ AC_INIT(object.cpp) AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) +AC_SUBST(CXXFLAGS) +if test "" = "$CXXFLAGS"; then + CXXFLAGS="-g -O4" +fi + AC_PROG_CXX AC_PROG_LIBTOOL AC_CHECK_PROG(ERB, erb, erb, [$PATH]) AC_CHECK_LIB(stdc++, main) -CXXFLAGS="-O4 $CXXFLAGS -Wall -I.." +CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." AC_OUTPUT([Makefile]) diff --git a/cpp/test.mk b/cpp/test.mk new file mode 100644 index 0000000..f1beac5 --- /dev/null +++ b/cpp/test.mk @@ -0,0 +1,9 @@ + +CXXFLAGS += -Wall -g -I. -I.. -O4 +LDFLAGS += + +all: test + +test: test.o unpack.o zone.o object.o pack.hpp unpack.hpp zone.hpp object.hpp + $(CXX) test.o unpack.o zone.o object.o $(CXXFLAGS) $(LDFLAGS) -o $@ + diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 490bc25..527cc9c 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -20,10 +20,6 @@ namespace msgpack { -zone::zone() { } - -zone::~zone() { clear(); } - void zone::clear() { for(std::vector::iterator it(m_ptrs.begin()), it_end(m_ptrs.end()); @@ -33,22 +29,18 @@ void zone::clear() m_ptrs.clear(); } -char* zone::realloc(char* ptr, size_t count) +char* zone::realloc_real(char* ptr, size_t count) { - if(ptr == NULL) { - return zone::malloc(count); - } else { - for(std::vector::reverse_iterator it(m_ptrs.rbegin()), it_end(m_ptrs.rend()); - it != it_end; ++it) { - if(*it == ptr) { - char* tmp = (char*)::realloc(ptr, count); - if(!tmp) { throw std::bad_alloc(); } - *it = tmp; - return tmp; - } + for(std::vector::reverse_iterator it(m_ptrs.rbegin()), it_end(m_ptrs.rend()); + it != it_end; ++it) { + if(*it == ptr) { + char* tmp = (char*)::realloc(ptr, count); + if(!tmp) { throw std::bad_alloc(); } + *it = tmp; + return tmp; } - throw std::bad_alloc(); } + throw std::bad_alloc(); } diff --git a/cpp/zone.hpp b/cpp/zone.hpp index f8c9cba..e7e73e1 100644 --- a/cpp/zone.hpp +++ b/cpp/zone.hpp @@ -40,11 +40,18 @@ public: private: std::vector m_ptrs; +private: + char* realloc_real(char* ptr, size_t count); + private: zone(const zone&); }; +inline zone::zone() { } + +inline zone::~zone() { clear(); } + inline char* zone::malloc(size_t count) { char* ptr = (char*)::malloc(count); @@ -58,6 +65,15 @@ inline char* zone::malloc(size_t count) return ptr; } +inline char* zone::realloc(char* ptr, size_t count) +{ + if(ptr == NULL) { + return zone::malloc(count); + } else { + return realloc_real(ptr, count); + } +} + inline object* zone::malloc_container(size_t count) { return (object*)zone::malloc(sizeof(object)*count); diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 928b581..69c7345 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -72,17 +72,21 @@ msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) { if(d < -32) { - if(d < -32768) { // signed 32 + if(d < -32768) { + // signed 32 const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); - } else if(d < -128) { // signed 16 + } else if(d < -128) { + // signed 16 const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); - } else { // signed 8 + } else { + // signed 8 const unsigned char buf[2] = {0xd0, (uint8_t)d}; msgpack_pack_append_buffer(x, buf, 2); } - } else if(d < 128) { // fixnum + } else if(d < 128) { + // fixnum msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); } else { if(d < 256) { @@ -104,21 +108,25 @@ msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) // wrapper msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) { - if(d < 128) { - // fixnum - msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); - } else if(d < 256) { - // unsigned 8 - const unsigned char buf[2] = {0xcc, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } else if(d < 65536) { - // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); + if(d < 256) { + if(d < 128) { + // fixnum + msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); + } else { + // unsigned 8 + const unsigned char buf[2] = {0xcc, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } } else { - // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); + if(d < 65536) { + // unsigned 16 + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + // unsigned 32 + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } } } diff --git a/ruby/extconf.rb b/ruby/extconf.rb index 88abb55..e6d4bd6 100644 --- a/ruby/extconf.rb +++ b/ruby/extconf.rb @@ -1,4 +1,4 @@ require 'mkmf' -$CFLAGS << " -I.. -Wall -O9" +$CFLAGS << " -I.. -Wall -O4" create_makefile('msgpack') diff --git a/ruby/test_pack.rb b/ruby/test_pack.rb index 16a8ccf..6873d57 100644 --- a/ruby/test_pack.rb +++ b/ruby/test_pack.rb @@ -21,6 +21,8 @@ check 256 check 65535 check 65536 check -1 +check -32 +check -33 check -128 check -129 check -32768 diff --git a/ruby/unpack.c b/ruby/unpack.c index b948aa6..8ab425c 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -176,13 +176,13 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) int ret; if(from >= dlen) { - rb_raise(eUnpackError, "Requested start is after data buffer end."); + rb_raise(eUnpackError, "offset is bigger than data buffer size."); } ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); if(ret < 0) { - rb_raise(eUnpackError, "Parse error."); + rb_raise(eUnpackError, "parse error."); } else if(ret > 0) { mp->user.finished = 1; return ULONG2NUM(from); @@ -242,12 +242,12 @@ static VALUE MessagePack_unpack_impl(VALUE args) ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); if(ret < 0) { - rb_raise(eUnpackError, "Parse error."); + rb_raise(eUnpackError, "parse error."); } else if(ret == 0) { - rb_raise(eUnpackError, "Insufficient bytes."); + rb_raise(eUnpackError, "insufficient bytes."); } else { if(from < dlen) { - rb_raise(eUnpackError, "Extra bytes."); + rb_raise(eUnpackError, "extra bytes."); } return msgpack_unpacker_data(mp); } From f4a6d7faa18eba7fad3e72c8238b0c7463c4294f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0024/1648] lang/c/msgpack: Ruby binding: fix gem warning git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@71 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- ruby/gem/README.txt | 23 +++++++++++++++++++++++ ruby/gengem.sh | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 ruby/gem/README.txt diff --git a/ruby/gem/README.txt b/ruby/gem/README.txt new file mode 100644 index 0000000..7c24b8f --- /dev/null +++ b/ruby/gem/README.txt @@ -0,0 +1,23 @@ +=MessagePack + +== DESCRIPTION: + +Binary-based efficient data interchange format. + + +== LICENSE: + +Copyright (C) 2008 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/ruby/gengem.sh b/ruby/gengem.sh index c8abcc2..000c708 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -6,7 +6,7 @@ cp pack.h gem/ext/ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ -cp ../README gem/README.txt +#cp ../README gem/README.txt cp ../msgpack/pack_template.h gem/msgpack/ cp ../msgpack/unpack_define.h gem/msgpack/ cp ../msgpack/unpack_template.h gem/msgpack/ From 2c7f0b2b1aa78b28916c40171dc8fee07d414d64 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0025/1648] lang/c/msgpack: C++ binding: reexamined global operators git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@72 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 78 +++++++++++--------- cpp/type/array.hpp | 10 +-- cpp/type/boolean.hpp | 6 +- cpp/type/float.hpp | 18 ++--- cpp/type/integer.hpp | 162 ++++++++++++++++++++--------------------- cpp/type/map.hpp | 32 ++++---- cpp/type/nil.hpp | 14 ++-- cpp/type/raw.hpp | 24 +++--- cpp/type/tuple.hpp.erb | 99 ++++++++++++++----------- cpp/unpack.hpp | 7 ++ 10 files changed, 241 insertions(+), 209 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index a5e8682..8cce14c 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -31,6 +31,18 @@ namespace msgpack { class type_error : public std::bad_cast { }; +namespace type { + static const unsigned char NIL = 0x01; + static const unsigned char BOOLEAN = 0x02; + static const unsigned char POSITIVE_INTEGER = 0x03; + static const unsigned char NEGATIVE_INTEGER = 0x04; + static const unsigned char DOUBLE = 0x05; + static const unsigned char RAW = 0x06; + static const unsigned char ARRAY = 0x07; + static const unsigned char MAP = 0x08; +} + + struct object { unsigned char type; union { @@ -50,6 +62,11 @@ struct object { template operator T() { T v; convert(v, *this); return v; }; + + template + T as() { T v; convert(v, *this); return v; } + + bool is_nil() { return type == type::NIL; } }; std::ostream& operator<< (std::ostream& s, const object o); @@ -58,53 +75,36 @@ bool operator==(const object x, const object y); inline bool operator!=(const object x, const object y) { return !(x == y); } -inline object& operator<< (object& v, object o) +inline object& operator>> (object o, object& v) { v = o; return v; } template -const object& operator>> (const object& v, packer& o); +packer& operator<< (packer& o, const object& v); + namespace type { - static const unsigned char NIL = 0x01; - static const unsigned char BOOLEAN = 0x02; - static const unsigned char POSITIVE_INTEGER = 0x03; - static const unsigned char NEGATIVE_INTEGER = 0x04; - static const unsigned char DOUBLE = 0x05; - static const unsigned char RAW = 0x06; - static const unsigned char ARRAY = 0x07; - static const unsigned char MAP = 0x08; - - template - inline T& operator<< (T& v, object o) + inline T& operator>> (object o, T& v) { v.msgpack_unpack(o); return v; } - - - namespace detail { - template - inline void pack_copy(T v, packer& o) - { pack(v, o); } - } - + template - inline const T& operator>> (const T& v, packer& o) + inline packer& operator<< (packer& o, const T& v) { - detail::pack_copy(v.msgpack_pack(), o); - return v; + pack_copy(v.msgpack_pack(), o); + return o; } - -} // namespace type +} template inline void convert(T& v, object o) { using namespace type; - v << o; + o >> v; } @@ -112,7 +112,7 @@ template inline void pack(T& v, packer& o) { using namespace type; - v >> o; + o << v; } @@ -124,14 +124,20 @@ inline void pack(T& v, Stream& s) } +template +inline void pack_copy(T v, packer& o) +{ + pack(v, o); +} + template -const object& operator>> (const object& v, packer& o) +packer& operator<< (packer& o, const object& v) { switch(v.type) { case type::NIL: o.pack_nil(); - return v; + return o; case type::BOOLEAN: if(v.via.boolean) { @@ -139,7 +145,7 @@ const object& operator>> (const object& v, packer& o) } else { o.pack_false(); } - return v; + return o; case type::POSITIVE_INTEGER: if(v.via.u64 <= (uint64_t)std::numeric_limits::max()) { @@ -155,7 +161,7 @@ const object& operator>> (const object& v, packer& o) o.pack_uint64(v.via.u64); } } - return v; + return o; case type::NEGATIVE_INTEGER: if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { @@ -171,11 +177,11 @@ const object& operator>> (const object& v, packer& o) o.pack_int64(v.via.i64); } } - return v; + return o; case type::RAW: o.pack_raw(v.via.ref.ptr, v.via.ref.size); - return v; + return o; case type::ARRAY: o.pack_array(v.via.container.size); @@ -184,7 +190,7 @@ const object& operator>> (const object& v, packer& o) p < pend; ++p) { *p >> o; } - return v; + return o; // FIXME loop optimiziation case type::MAP: @@ -195,7 +201,7 @@ const object& operator>> (const object& v, packer& o) *p >> o; ++p; *p >> o; ++p; } - return v; + return o; // FIXME loop optimiziation default: diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index 7066703..703ac55 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -22,13 +22,12 @@ #include namespace msgpack { -namespace type { template -inline std::vector operator<< (std::vector& v, object o) +inline std::vector operator>> (object o, std::vector& v) { - if(o.type != ARRAY) { throw type_error(); } + if(o.type != type::ARRAY) { throw type_error(); } v.resize(o.via.container.size); object* p(o.via.container.ptr); object* const pend(o.via.container.ptr + o.via.container.size); @@ -41,18 +40,17 @@ inline std::vector operator<< (std::vector& v, object o) template -inline const std::vector& operator>> (const std::vector& v, packer& o) +inline packer& operator<< (packer& o, const std::vector& v) { o.pack_array(v.size()); for(typename std::vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { pack(*it, o); } - return v; + return o; } -} // namespace type } // namespace msgpack #endif /* msgpack/type/array.hpp */ diff --git a/cpp/type/boolean.hpp b/cpp/type/boolean.hpp index 0538495..e958478 100644 --- a/cpp/type/boolean.hpp +++ b/cpp/type/boolean.hpp @@ -25,7 +25,7 @@ namespace msgpack { namespace type { -inline bool& operator<< (bool& v, object o) +inline bool& operator>> (object o, bool& v) { if(o.type != BOOLEAN) { throw type_error(); } v = o.via.boolean; @@ -34,11 +34,11 @@ inline bool& operator<< (bool& v, object o) template -inline const bool& operator>> (const bool& v, packer o) +inline packer& operator<< (packer& o, const bool& v) { if(v) { o.pack_true(); } else { o.pack_false(); } - return v; + return o; } diff --git a/cpp/type/float.hpp b/cpp/type/float.hpp index 11fa280..2178434 100644 --- a/cpp/type/float.hpp +++ b/cpp/type/float.hpp @@ -22,45 +22,43 @@ #include namespace msgpack { -namespace type { // FIXME check overflow, underflow -inline float& operator<< (float& v, object o) +inline float& operator>> (object o, float& v) { - if(o.type != DOUBLE) { throw type_error(); } + if(o.type != type::DOUBLE) { throw type_error(); } v = o.via.dec; return v; } template -inline const float& operator>> (const float& v, packer o) +inline packer& operator<< (packer& o, const float& v) { o.pack_float(v); - return v; + return o; } -inline double& operator<< (double& v, object o) +inline double& operator>> (object o, double& v) { - if(o.type != DOUBLE) { throw type_error(); } + if(o.type != type::DOUBLE) { throw type_error(); } v = o.via.dec; return v; } template -inline const double& operator>> (const double& v, packer o) +inline packer& operator<< (packer& o, const double& v) { o.pack_double(v); - return v; + return o; } -} // namespace type } // namespace msgpack #endif /* msgpack/type/float.hpp */ diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp index 488c9d4..f5841a6 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/integer.hpp @@ -22,9 +22,9 @@ #include namespace msgpack { + + namespace type { - - namespace detail { template struct convert_integer_sign; @@ -32,11 +32,11 @@ namespace detail { template struct convert_integer_sign { static inline T convert(object o) { - if(o.type == POSITIVE_INTEGER) { + if(o.type == type::POSITIVE_INTEGER) { if(o.via.u64 > (uint64_t)std::numeric_limits::max()) { throw type_error(); } return o.via.u64; - } else if(o.type == NEGATIVE_INTEGER) { + } else if(o.type == type::NEGATIVE_INTEGER) { if(o.via.i64 < (int64_t)-std::numeric_limits::max()) { throw type_error(); } return o.via.i64; @@ -48,7 +48,7 @@ namespace detail { template struct convert_integer_sign { static inline T convert(object o) { - if(o.type == POSITIVE_INTEGER) { + if(o.type == type::POSITIVE_INTEGER) { if(o.via.u64 > (uint64_t)std::numeric_limits::max()) { throw type_error(); } return o.via.u64; @@ -160,83 +160,83 @@ namespace detail { } } // namespace detail - - -inline signed char& operator<< (signed char& v, object o) - { v = detail::convert_integer(o); return v; } - -inline signed short& operator<< (signed short& v, object o) - { v = detail::convert_integer(o); return v; } - -inline signed int& operator<< (signed int& v, object o) - { v = detail::convert_integer(o); return v; } - -inline signed long& operator<< (signed long& v, object o) - { v = detail::convert_integer(o); return v; } - -inline signed long long& operator<< (signed long long& v, object o) - { v = detail::convert_integer(o); return v; } - - -inline unsigned char& operator<< (unsigned char& v, object o) - { v = detail::convert_integer(o); return v; } - -inline unsigned short& operator<< (unsigned short& v, object o) - { v = detail::convert_integer(o); return v; } - -inline unsigned int& operator<< (unsigned int& v, object o) - { v = detail::convert_integer(o); return v; } - -inline unsigned long& operator<< (unsigned long& v, object o) - { v = detail::convert_integer(o); return v; } - -inline unsigned long long& operator<< (unsigned long long& v, object o) - { v = detail::convert_integer(o); return v; } - - -template -inline const signed char& operator>> (const signed char& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const signed short& operator>> (const signed short& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const signed int& operator>> (const signed int& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const signed long& operator>> (const signed long& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const signed long long& operator>> (const signed long long& v, packer o) - { detail::pack_integer(v, o); return v; } - - -template -inline const unsigned char& operator>> (const unsigned char& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const unsigned short& operator>> (const unsigned short& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const unsigned int& operator>> (const unsigned int& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const unsigned long& operator>> (const unsigned long& v, packer o) - { detail::pack_integer(v, o); return v; } - -template -inline const unsigned long long& operator>> (const unsigned long long& v, packer o) - { detail::pack_integer(v, o); return v; } - - } // namespace type + + +inline signed char& operator>> (object o, signed char& v) + { v = type::detail::convert_integer(o); return v; } + +inline signed short& operator>> (object o, signed short& v) + { v = type::detail::convert_integer(o); return v; } + +inline signed int& operator>> (object o, signed int& v) + { v = type::detail::convert_integer(o); return v; } + +inline signed long& operator>> (object o, signed long& v) + { v = type::detail::convert_integer(o); return v; } + +inline signed long long& operator>> (object o, signed long long& v) + { v = type::detail::convert_integer(o); return v; } + + +inline unsigned char& operator>> (object o, unsigned char& v) + { v = type::detail::convert_integer(o); return v; } + +inline unsigned short& operator>> (object o, unsigned short& v) + { v = type::detail::convert_integer(o); return v; } + +inline unsigned int& operator>> (object o, unsigned int& v) + { v = type::detail::convert_integer(o); return v; } + +inline unsigned long& operator>> (object o, unsigned long& v) + { v = type::detail::convert_integer(o); return v; } + +inline unsigned long long& operator>> (object o, unsigned long long& v) + { v = type::detail::convert_integer(o); return v; } + + +template +inline packer& operator<< (packer& o, const signed char& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const signed short& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const signed int& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const signed long& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const signed long long& v) + { type::detail::pack_integer(v, o); return o; } + + +template +inline packer& operator<< (packer& o, const unsigned char& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const unsigned short& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const unsigned int& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const unsigned long& v) + { type::detail::pack_integer(v, o); return o; } + +template +inline packer& operator<< (packer& o, const unsigned long long& v) + { type::detail::pack_integer(v, o); return o; } + + } // namespace msgpack #endif /* msgpack/type/integer.hpp */ diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 1efecec..1d5e054 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -23,9 +23,10 @@ #include namespace msgpack { -namespace type { +namespace type { + template class assoc_vector : public std::vector< std::pair > {}; @@ -37,10 +38,13 @@ namespace detail { }; } +} //namespace type + + template -inline assoc_vector& operator<< (assoc_vector& v, object o) +inline type::assoc_vector& operator>> (object o, type::assoc_vector& v) { - if(o.type != MAP) { throw type_error(); } + if(o.type != type::MAP) { throw type_error(); } v.resize(o.via.container.size); object* p(o.via.container.ptr); object* const pend(o.via.container.ptr + o.via.container.size); @@ -49,26 +53,27 @@ inline assoc_vector& operator<< (assoc_vector& v, object o) convert(it->first, *p); ++p; convert(it->second, *p); ++p; } - std::sort(v.begin(), v.end(), detail::pair_first_less()); + std::sort(v.begin(), v.end(), type::detail::pair_first_less()); return v; } template -inline const assoc_vector& operator>> (const assoc_vector& v, packer& o) +inline packer& operator<< (packer& o, const type::assoc_vector& v) { o.pack_map(v.size()); - for(typename assoc_vector::const_iterator it(v.begin()), it_end(v.end()); + for(typename type::assoc_vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { pack(it->first, o); pack(it->second, o); } + return o; } template -inline std::map operator<< (std::map& v, object o) +inline std::map operator>> (object o, std::map& v) { - if(o.type != MAP) { throw type_error(); } + if(o.type != type::MAP) { throw type_error(); } object* p(o.via.container.ptr); object* const pend(o.via.container.ptr + o.via.container.size*2); while(p < pend) { @@ -87,7 +92,7 @@ inline std::map operator<< (std::map& v, object o) } template -inline const std::map& operator>> (const std::map& v, packer& o) +inline packer& operator<< (packer& o, const std::map& v) { o.pack_map(v.size()); for(typename std::map::const_iterator it(v.begin()), it_end(v.end()); @@ -95,13 +100,14 @@ inline const std::map& operator>> (const std::map& v, packer& pack(it->first, o); pack(it->second, o); } + return o; } template -inline std::multimap operator<< (std::multimap& v, object o) +inline std::multimap operator>> (object o, std::multimap& v) { - if(o.type != MAP) { throw type_error(); } + if(o.type != type::MAP) { throw type_error(); } object* p(o.via.container.ptr); object* const pend(o.via.container.ptr + o.via.container.size*2); while(p < pend) { @@ -114,7 +120,7 @@ inline std::multimap operator<< (std::multimap& v, object o) } template -inline const std::multimap& operator>> (const std::multimap& v, packer& o) +inline packer& operator<< (packer& o, const std::multimap& v) { o.pack_multimap(v.size()); for(typename std::multimap::const_iterator it(v.begin()), it_end(v.end()); @@ -122,10 +128,10 @@ inline const std::multimap& operator>> (const std::multimap& v, packer pack(it->first, o); pack(it->second, o); } + return o; } -} // namespace type } // namespace msgpack #endif /* msgpack/type/map.hpp */ diff --git a/cpp/type/nil.hpp b/cpp/type/nil.hpp index 73cedb4..ab0c363 100644 --- a/cpp/type/nil.hpp +++ b/cpp/type/nil.hpp @@ -21,26 +21,28 @@ #include "msgpack/object.hpp" namespace msgpack { -namespace type { +namespace type { struct nil { }; -inline nil& operator<< (nil& v, object o) +} // namespace type + + +inline type::nil& operator>> (object o, type::nil& v) { - if(o.type != NIL) { throw type_error(); } + if(o.type != type::NIL) { throw type_error(); } return v; } template -inline const nil& operator>> (const nil& v, packer& o) +inline packer& operator<< (packer& o, const type::nil& v) { o.pack_nil(); - return v; + return o; } -} // namespace type } // namespace msgpack #endif /* msgpack/type/nil.hpp */ diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index 8767238..14e52f4 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -23,8 +23,8 @@ #include namespace msgpack { -namespace type { +namespace type { struct raw_ref { raw_ref() : ptr(NULL), size(0) {} @@ -58,41 +58,43 @@ struct raw_ref { } }; -inline raw_ref& operator<< (raw_ref& v, object o) +} // namespace type + + +inline type::raw_ref& operator>> (object o, type::raw_ref& v) { - if(o.type != RAW) { throw type_error(); } + if(o.type != type::RAW) { throw type_error(); } v.ptr = o.via.ref.ptr; v.size = o.via.ref.size; return v; } -inline std::string& operator<< (std::string& v, object o) +inline std::string& operator>> (object o, std::string& v) { - raw_ref r; - r << o; + type::raw_ref r; + o >> r; v.assign(r.ptr, r.size); return v; } template -inline const raw_ref& operator>> (const raw_ref& v, packer& o) +inline packer& operator<< (packer& o, const type::raw_ref& v) { o.pack_raw(v.ptr, v.size); - return v; + return o; } template -inline const std::string& operator>> (const std::string& v, packer& o) +inline packer& operator<< (packer& o, const std::string& v) { o.pack_raw(v.data(), v.size()); - return v; + return o; } -} // namespace type } // namespace msgpack #endif /* msgpack/type/raw.hpp */ diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 2a8a2dd..c54739a 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -21,15 +21,14 @@ #include "msgpack/object.hpp" namespace msgpack { -namespace type { +namespace type { // FIXME operator== // FIXME operator!= - <% GENERATION_LIMIT = 15 %> -template < typename A0 <%1.upto(GENERATION_LIMIT+1) {|i|%>, typename A<%=i%> = void<%}%> > +template , typename A<%=i%> = void<%}%>> struct tuple; template @@ -60,9 +59,9 @@ private: <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> -template < typename A0 <%1.upto(i) {|k|%>, typename A<%=k%> <%}%>> -struct tuple_type, A<%=k%> <%}%>>, <%=j%>> : tuple_element> { - tuple_type(tuple, A<%=k%> <%}%>>& x) : tuple_element>(x.a<%=j%>) {} +template , typename A<%=k%><%}%>> +struct tuple_type, A<%=k%><%}%>>, <%=j%>> : tuple_element> { + tuple_type(tuple, A<%=k%> <%}%>>& x) : tuple_element>(x.a<%=j%>) {} }; <%}%> <%}%> @@ -70,46 +69,57 @@ struct tuple_type, A<%=k%> <%}%>>, <%=j%>> : tuple_e <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> -template < typename A0 <%1.upto(i) {|k|%>, typename A<%=k%> <%}%>> -struct const_tuple_type, A<%=k%> <%}%>>, <%=j%>> : const_tuple_element> { - const_tuple_type(const tuple, A<%=k%> <%}%>>& x) : const_tuple_element>(x.a<%=j%>) {} +template , typename A<%=k%><%}%>> +struct const_tuple_type, A<%=k%><%}%>>, <%=j%>> : const_tuple_element> { + const_tuple_type(const tuple, A<%=k%><%}%>>& x) : const_tuple_element>(x.a<%=j%>) {} }; <%}%> <%}%> <%0.upto(GENERATION_LIMIT) {|i|%> -template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> -tuple< A0 <%1.upto(i) {|j|%>, A<%=j%> <%}%>> make_tuple(const A0& a0 <%1.upto(i) {|j|%>, const A<%=j%>& a<%=j%><%}%>) +template , typename A<%=j%><%}%>> +tuple, A<%=j%><%}%>> make_tuple(const A0& a0<%1.upto(i) {|j|%>, const A<%=j%>& a<%=j%><%}%>) { - return tuple< A0 <%1.upto(i) {|j|%>, A<%=j%> <%}%>>(a0 <%1.upto(i) {|j|%>, a<%=j%><%}%>); + return tuple, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>); } <%}%> +template <> +struct tuple<> { +}; <%0.upto(GENERATION_LIMIT) {|i|%> -template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> -struct tuple, A<%=j%> <%}%>> { +template , typename A<%=j%><%}%>> +struct tuple, A<%=j%><%}%>> { tuple() {} - tuple(const A0& _a0 <%1.upto(i) {|j|%>, const A<%=j%>& _a<%=j%><%}%>) : + tuple(const A0& _a0<%1.upto(i) {|j|%>, const A<%=j%>& _a<%=j%><%}%>) : a0(_a0) <%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} tuple(object o) { convert(*this, o); } - template typename tuple_type, A<%=j%> <%}%>>, N>::type& get() - { return tuple_type, A<%=j%> <%}%>>, N>(*this).get(); } - template const typename const_tuple_type, A<%=j%> <%}%>>, N>::type& get() const - { return const_tuple_type, A<%=j%> <%}%>>, N>(*this).get(); } + template typename tuple_type, A<%=j%><%}%>>, N>::type& get() + { return tuple_type, A<%=j%><%}%>>, N>(*this).get(); } + template const typename const_tuple_type, A<%=j%><%}%>>, N>::type& get() const + { return const_tuple_type, A<%=j%><%}%>>, N>(*this).get(); } <%0.upto(i) {|j|%> A<%=j%> a<%=j%>;<%}%> }; <%}%> +} // namespace type + +inline type::tuple<>& operator>> ( + object o, + type::tuple<>& v) { + if(o.type != type::ARRAY) { throw type_error(); } + return v; +} <%0.upto(GENERATION_LIMIT) {|i|%> -template < typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> -tuple, A<%=j%> <%}%>>& operator<< ( - tuple, A<%=j%> <%}%>>& v, - object o) { - if(o.type != ARRAY) { throw type_error(); } +template , typename A<%=j%><%}%>> +type::tuple, A<%=j%><%}%>>& operator>> ( + object o, + type::tuple, A<%=j%><%}%>>& v) { + if(o.type != type::ARRAY) { throw type_error(); } if(o.via.container.size < <%=i+1%>) { throw type_error(); } <%0.upto(i) {|j|%> convert>(v.template get<<%=j%>>(), o.via.container.ptr[<%=j%>]);<%}%> @@ -118,6 +128,26 @@ tuple, A<%=j%> <%}%>>& operator<< ( <%}%> +template +const packer& operator<< ( + packer& o, + const type::tuple<>& v) { + o.pack_array(0); + return o; +} +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +const packer& operator<< ( + packer& o, + const type::tuple, A<%=j%><%}%>>& v) { + o.pack_array(<%=i+1%>); + <%0.upto(i) {|j|%> + pack(v.template get<<%=j%>>(), o);<%}%> + return o; +} +<%}%> + + // FIXME /* template @@ -128,7 +158,7 @@ struct tuple_just { A0 a0; static inline void convert(object o, tuple_just& v) { - if(o.type != ARRAY) { throw type_error(); } + if(o.type != type::ARRAY) { throw type_error(); } if(o.v.container.size != 1) { throw type_error(); } msgpack::convert(o.v.container.ptr[0], v.a0); } @@ -140,7 +170,7 @@ struct tuple_just { A1 a1; static inline void convert(object o, tuple_just& v) { - if(o.type != ARRAY) { throw type_error(); } + if(o.type != type::ARRAY) { throw type_error(); } if(o.v.container.size != 2) { throw type_error(); } msgpack::convert(o.v.container.ptr[0], v.a0); msgpack::convert(o.v.container.ptr[1], v.a1); @@ -149,24 +179,7 @@ struct tuple_just { */ - -<%0.upto(GENERATION_LIMIT) {|i|%> -template < typename Stream , typename A0 <%1.upto(i) {|j|%>, typename A<%=j%> <%}%>> -const tuple, A<%=j%> <%}%>>& operator>> ( - const tuple, A<%=j%> <%}%>>& v, - packer o) { - o.pack_array(<%=i+1%>); - <%0.upto(i) {|j|%> - pack(v.template get<<%=j%>>(), o);<%}%> - return v; -} -<%}%> - - - -} // namespace type } // namespace msgpack #endif /* msgpack/type/tuple.hpp */ - diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 07c52e7..de613a4 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -124,6 +124,10 @@ public: // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); + /*! skip specified size of non-parsed buffer, leaving the buffer */ + // Note the size must be smaller than nonparsed_size() + void skip_nonparsed_buffer(size_t len); + private: char* m_buffer; size_t m_used; @@ -177,6 +181,9 @@ inline size_t unpacker::parsed_size() const inline void unpacker::remove_nonparsed_buffer() { m_used = m_off; } +inline void unpacker::skip_nonparsed_buffer(size_t len) + { m_off += len; } + inline object unpack(const char* data, size_t len, zone& z, size_t* off = NULL) { From 921b0ff62ec99e9339d6cfaa6ba724ae7bb5f9ce Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0026/1648] lang/c/msgpack: C++ binding: pack() git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@73 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 41 +++++++++++++++++------------------------ cpp/test.cpp | 4 ++-- cpp/type/array.hpp | 2 +- cpp/type/boolean.hpp | 4 +--- cpp/type/integer.hpp | 18 +++++++++--------- cpp/type/map.hpp | 12 ++++++------ cpp/type/tuple.hpp.erb | 2 +- cpp/unpack.hpp | 2 +- 8 files changed, 38 insertions(+), 47 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 8cce14c..77d55bb 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -83,51 +83,44 @@ packer& operator<< (packer& o, const object& v); -namespace type { - template - inline T& operator>> (object o, T& v) - { - v.msgpack_unpack(o); - return v; - } - - template - inline packer& operator<< (packer& o, const T& v) - { - pack_copy(v.msgpack_pack(), o); - return o; - } +template +inline T& operator>> (object o, T& v) +{ + v.msgpack_unpack(o); + return v; +} + +template +inline packer& operator<< (packer& o, const T& v) +{ + o << v.msgpack_pack(); + return o; } template inline void convert(T& v, object o) { - using namespace type; o >> v; } - template -inline void pack(T& v, packer& o) +inline void pack(packer& o, const T& v) { - using namespace type; o << v; } - template -inline void pack(T& v, Stream& s) +inline void pack(Stream& s, const T& v) { packer pk(s); - pack(v, pk); + pack(pk, v); } - template -inline void pack_copy(T v, packer& o) +inline void pack_copy(packer& o, T v) { - pack(v, o); + pack(o, v); } diff --git a/cpp/test.cpp b/cpp/test.cpp index c632ec5..02d461c 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -26,7 +26,7 @@ public: try { std::stringstream s; - pack(should, s); + pack(s, should); std::string str(s.str()); object ro = unpack(str.data(), str.size(), m_zone); std::cout << ro << std::endl; @@ -139,7 +139,7 @@ int main(void) // send message { for(unsigned i=0; i < TASK_REPEAT; ++i) { - pack(task, stream); + pack(stream, task); } std::cout << "send " << stream.str().size() << " bytes" << std::endl; } diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index 703ac55..0522d4c 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -45,7 +45,7 @@ inline packer& operator<< (packer& o, const std::vector& v) o.pack_array(v.size()); for(typename std::vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(*it, o); + pack(o, *it); } return o; } diff --git a/cpp/type/boolean.hpp b/cpp/type/boolean.hpp index e958478..60f1714 100644 --- a/cpp/type/boolean.hpp +++ b/cpp/type/boolean.hpp @@ -22,12 +22,11 @@ #include namespace msgpack { -namespace type { inline bool& operator>> (object o, bool& v) { - if(o.type != BOOLEAN) { throw type_error(); } + if(o.type != type::BOOLEAN) { throw type_error(); } v = o.via.boolean; return v; } @@ -42,7 +41,6 @@ inline packer& operator<< (packer& o, const bool& v) } -} // namespace type } // namespace msgpack #endif /* msgpack/type/bool.hpp */ diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp index f5841a6..3fd57b4 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/integer.hpp @@ -70,19 +70,19 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) + static inline void pack(packer& o, T v) { o.pack_int8(v); } }; template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) + static inline void pack(packer& o, T v) { o.pack_uint8(v); } }; template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (int16_t)v <= (int16_t)std::numeric_limits::max() && (int16_t)v >= (int16_t)std::numeric_limits::min()) { o.pack_int8(v); } @@ -92,7 +92,7 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (uint16_t)v <= (uint16_t)std::numeric_limits::max()) { o.pack_uint8(v); } else { o.pack_uint16(v); } @@ -101,7 +101,7 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (int32_t)v <= (int32_t)std::numeric_limits::max() && (int32_t)v >= (int32_t)std::numeric_limits::min()) { o.pack_int8(v); } @@ -114,7 +114,7 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) { o.pack_uint8(v); } else if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) @@ -125,7 +125,7 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (int64_t)v <= (int64_t)std::numeric_limits::max() && (int64_t)v >= (int64_t)std::numeric_limits::min()) { o.pack_int8(v); } @@ -141,7 +141,7 @@ namespace detail { template struct pack_integer_size_sign { - static inline void pack(T v, packer& o) { + static inline void pack(packer& o, T v) { if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) { o.pack_uint8(v); } else if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) @@ -156,7 +156,7 @@ namespace detail { template static inline void pack_integer(T v, packer& o) { - pack_integer_size_sign::is_signed>::pack(v, o); + pack_integer_size_sign::is_signed>::pack(o, v); } } // namespace detail diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 1d5e054..3b544df 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -63,8 +63,8 @@ inline packer& operator<< (packer& o, const type::assoc_vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(it->first, o); - pack(it->second, o); + pack(o, it->first); + pack(o, it->second); } return o; } @@ -97,8 +97,8 @@ inline packer& operator<< (packer& o, const std::map& v) o.pack_map(v.size()); for(typename std::map::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(it->first, o); - pack(it->second, o); + pack(o, it->first); + pack(o, it->second); } return o; } @@ -125,8 +125,8 @@ inline packer& operator<< (packer& o, const std::multimap& o.pack_multimap(v.size()); for(typename std::multimap::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(it->first, o); - pack(it->second, o); + pack(o, it->first); + pack(o, it->second); } return o; } diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index c54739a..397f660 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -142,7 +142,7 @@ const packer& operator<< ( const type::tuple, A<%=j%><%}%>>& v) { o.pack_array(<%=i+1%>); <%0.upto(i) {|j|%> - pack(v.template get<<%=j%>>(), o);<%}%> + pack(o, v.template get<<%=j%>>());<%}%> return o; } <%}%> diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index de613a4..ffee60d 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -81,7 +81,7 @@ public: // // // 2. // ssize_t bytes = - // read(the_source, pac.buffer, pac.buffer_capacity()); + // read(the_source, pac.buffer(), pac.buffer_capacity()); // // // error handling ... // From 1ad04b22d8d3e5267ceec7fc1527651ee0eb0ffe Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:58 +0000 Subject: [PATCH 0027/1648] lang/c/msgpack: divide pack_raw() into pack_raw() and pack_raw_body() git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@74 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bench.c | 3 +- c/pack.h | 4 +- cpp/bench.cpp | 3 +- cpp/object.hpp | 3 +- cpp/pack.hpp | 131 ++-------------------------------------- cpp/type/raw.hpp | 6 +- cpp/unpack.cpp | 4 +- msgpack/pack_template.h | 7 ++- ruby/pack.c | 6 +- 9 files changed, 28 insertions(+), 139 deletions(-) diff --git a/c/bench.c b/c/bench.c index fa717c0..d72a10d 100644 --- a/c/bench.c +++ b/c/bench.c @@ -279,7 +279,8 @@ void bench_msgpack(void) unsigned int i; msgpack_pack_array(mpk, TASK_STR_LEN); for(i=0; i < TASK_STR_LEN; ++i) { - msgpack_pack_raw(mpk, TASK_STR_PTR, i); + msgpack_pack_raw(mpk, i); + msgpack_pack_raw_body(mpk, TASK_STR_PTR, i); } } show_timer(mpkbuf.length); diff --git a/c/pack.h b/c/pack.h index c6cadf4..ec0683e 100644 --- a/c/pack.h +++ b/c/pack.h @@ -54,8 +54,8 @@ void msgpack_pack_true(msgpack_pack_t* ctx); void msgpack_pack_false(msgpack_pack_t* ctx); void msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); void msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); -void msgpack_pack_string(msgpack_pack_t* ctx, const char* b); -void msgpack_pack_raw(msgpack_pack_t* ctx, const void* b, size_t l); +void msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); +void msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); #ifdef __cplusplus diff --git a/cpp/bench.cpp b/cpp/bench.cpp index 1aad807..aa303fa 100644 --- a/cpp/bench.cpp +++ b/cpp/bench.cpp @@ -142,7 +142,8 @@ void bench_msgpack_str() msgpack::packer pk(buf); pk.pack_array(TASK_STR_LEN); for(unsigned int i=0; i < TASK_STR_LEN; ++i) { - pk.pack_raw(TASK_STR_PTR, i); + pk.pack_raw(i); + pk.pack_raw_body(TASK_STR_PTR, i); } } timer.show_stat(buf.size()); diff --git a/cpp/object.hpp b/cpp/object.hpp index 77d55bb..13402cc 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -173,7 +173,8 @@ packer& operator<< (packer& o, const object& v) return o; case type::RAW: - o.pack_raw(v.via.ref.ptr, v.via.ref.size); + o.pack_raw(v.via.ref.size); + o.pack_raw_body(v.via.ref.ptr, v.via.ref.size); return o; case type::ARRAY: diff --git a/cpp/pack.hpp b/cpp/pack.hpp index fe470b6..4fda4ff 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -47,7 +47,8 @@ public: void pack_false() { pack_false_impl(m_stream); } void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } - void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } + void pack_raw(size_t l) { pack_raw_impl(m_stream, l); } + void pack_raw_body(const char* b, size_t l) { pack_raw_body_impl(m_stream, b, l); } private: static void pack_int_impl(Stream& x, int d); @@ -67,7 +68,9 @@ private: static void pack_false_impl(Stream& x); static void pack_array_impl(Stream& x, unsigned int n); static void pack_map_impl(Stream& x, unsigned int n); - static void pack_raw_impl(Stream& x, const void* b, size_t l); + static void pack_raw_impl(Stream& x, size_t l); + static void pack_raw_body_impl(Stream& x, const void* b, size_t l); + static void append_buffer(Stream& x, const unsigned char* buf, unsigned int len) { x.write((const char*)buf, len); } @@ -90,130 +93,6 @@ template packer::packer(Stream& s) : m_stream(s) { } -/* -class dynamic_stream { -public: - template - dynamic_stream(Stream& s); -public: - void write(const char* buf, size_t len) - { (*m_function)(m_object, buf, len); } -private: - void* m_object; - void (*m_function)(void* object, const char* buf, size_t len); -private: - struct write_trampoline; -}; - - -class dynamic_packer { -public: - template - dynamic_packer(Stream& s); - -public: - void pack_int(int d) { pack_int_impl(m_stream, d); } - void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } - void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } - void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } - void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } - void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } - void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } - void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } - void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } - void pack_float(float d) { pack_float_impl(m_stream, d); } - void pack_double(double d) { pack_double_impl(m_stream, d); } - void pack_nil() { pack_nil_impl(m_stream); } - void pack_true() { pack_true_impl(m_stream); } - void pack_false() { pack_false_impl(m_stream); } - void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } - void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } - void pack_string(const char* b) { pack_string_impl(m_stream, b); } - void pack_raw(const char* b, size_t l) { pack_raw_impl(m_stream, (const void*)b, l); } - -private: - static void pack_int_impl(dynamic_stream& x, int d); - static void pack_unsigned_int_impl(dynamic_stream& x, unsigned int d); - static void pack_uint8_impl(dynamic_stream& x, uint8_t d); - static void pack_uint16_impl(dynamic_stream& x, uint16_t d); - static void pack_uint32_impl(dynamic_stream& x, uint32_t d); - static void pack_uint64_impl(dynamic_stream& x, uint64_t d); - static void pack_int8_impl(dynamic_stream& x, int8_t d); - static void pack_int16_impl(dynamic_stream& x, int16_t d); - static void pack_int32_impl(dynamic_stream& x, int32_t d); - static void pack_int64_impl(dynamic_stream& x, int64_t d); - static void pack_float_impl(dynamic_stream& x, float d); - static void pack_double_impl(dynamic_stream& x, double d); - static void pack_nil_impl(dynamic_stream& x); - static void pack_true_impl(dynamic_stream& x); - static void pack_false_impl(dynamic_stream& x); - static void pack_array_impl(dynamic_stream& x, unsigned int n); - static void pack_map_impl(dynamic_stream& x, unsigned int n); - static void pack_string_impl(dynamic_stream& x, const char* b); - static void pack_raw_impl(dynamic_stream& x, const void* b, size_t l); - static void append_buffer(dynamic_stream& x, const unsigned char* buf, unsigned int len) - { x.write((const char*)buf, len); } - -private: - dynamic_stream m_stream; - -private: - dynamic_packer(); -}; - -#define msgpack_pack_inline_func(name) \ - inline void dynamic_packer::pack_ ## name ## _impl -#define msgpack_pack_user dynamic_stream& -#define msgpack_pack_append_buffer append_buffer -#include "msgpack/pack_template.h" - -template -dynamic_packer::dynamic_packer(Stream& s) : m_stream(s) { } - -struct dynamic_stream::write_trampoline { -private: - template - struct ret_type { - typedef R (*type)(void*, const char*, size_t); - }; - - template - static R trampoline(void* obj, const char* buf, size_t len) - { - return (reinterpret_cast(obj)->*MemFun)(buf, len); - } - -public: - template - static typename ret_type::type get(R (Stream::*func)(Ptr*, Sz)) - { - R (*f)(void*, const char*, size_t) = - &trampoline; - return f; - } -}; - -template -dynamic_stream::dynamic_stream(Stream& s) -{ - m_object = reinterpret_cast(&s); - m_function = reinterpret_cast( - write_trampoline::get(&Stream::write) - ); -} - - -template -inline void pack(Stream& s, object o) -{ - dynamic_packer pk(s); - o.pack(pk); -} -*/ - - } // namespace msgpack #endif /* msgpack/pack.hpp */ diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index 14e52f4..e4f9dd3 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -82,7 +82,8 @@ inline std::string& operator>> (object o, std::string& v) template inline packer& operator<< (packer& o, const type::raw_ref& v) { - o.pack_raw(v.ptr, v.size); + o.pack_raw(v.size); + o.pack_raw_body(v.ptr, v.size); return o; } @@ -90,7 +91,8 @@ inline packer& operator<< (packer& o, const type::raw_ref& v) template inline packer& operator<< (packer& o, const std::string& v) { - o.pack_raw(v.data(), v.size()); + o.pack_raw(v.size()); + o.pack_raw_body(v.data(), v.size()); return o; } diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index ce39afd..cbf1356 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -257,7 +257,7 @@ zone* unpacker::release_zone() throw; } } - m_ctx->user(&*m_zone); + m_ctx->user(m_zone.get()); return old.release(); } @@ -268,7 +268,7 @@ object unpacker::data() void unpacker::reset() { - if(m_off != 0) { std::auto_ptr old(release_zone()); } + if(m_off != 0) { delete release_zone(); } m_ctx->reset(); } diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 69c7345..1dbf6fd 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -282,7 +282,7 @@ msgpack_pack_inline_func(map)(msgpack_pack_user x, unsigned int n) * Raw */ -msgpack_pack_inline_func(raw)(msgpack_pack_user x, const void* b, size_t l) +msgpack_pack_inline_func(raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; @@ -296,9 +296,12 @@ msgpack_pack_inline_func(raw)(msgpack_pack_user x, const void* b, size_t l) unsigned char buf[5] = {0xdb, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } - msgpack_pack_append_buffer(x, (const unsigned char*)b, l); } +msgpack_pack_inline_func(raw_body)(msgpack_pack_user x, const void* b, size_t l) +{ + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); +} #undef msgpack_pack_inline_func #undef msgpack_pack_user diff --git a/ruby/pack.c b/ruby/pack.c index 54f610c..e62419d 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -45,7 +45,8 @@ static void msgpack_pack_true(VALUE x); static void msgpack_pack_false(VALUE x); static void msgpack_pack_array(VALUE x, unsigned int n); static void msgpack_pack_map(VALUE x, unsigned int n); -static void msgpack_pack_raw(VALUE x, const void* b, size_t l); +static void msgpack_pack_raw(VALUE x, size_t l); +static void msgpack_pack_raw_body(VALUE x, const void* b, size_t l); */ #include "msgpack/pack_template.h" @@ -123,7 +124,8 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - msgpack_pack_raw(out, RSTRING_PTR(self), RSTRING_LEN(self)); + msgpack_pack_raw(out, RSTRING_LEN(self)); + msgpack_pack_raw_body(out, RSTRING_PTR(self), RSTRING_LEN(self)); return out; } From c3021d3236e1276bb575802597bfaf8279fced72 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0028/1648] lang/c/msgpack: added msgpack::define git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@75 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 53 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 13402cc..5ed1fab 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -81,21 +81,6 @@ inline object& operator>> (object o, object& v) template packer& operator<< (packer& o, const object& v); - - -template -inline T& operator>> (object o, T& v) -{ - v.msgpack_unpack(o); - return v; -} - -template -inline packer& operator<< (packer& o, const T& v) -{ - o << v.msgpack_pack(); - return o; -} template @@ -123,6 +108,44 @@ inline void pack_copy(packer& o, T v) pack(o, v); } + + +template +inline T& operator>> (object o, T& v) +{ + v.msgpack_unpack(o); + return v; +} + +template +inline packer& operator<< (packer& o, const T& v) +{ + o << v.msgpack_pack(); + return o; +} + + +template +class define : public Type { +public: + typedef Type msgpack_type; + typedef define define_type; + + define() {} + define(msgpack_type v) : msgpack_type(v) {} + + msgpack_type msgpack_pack() const + { + return *this; + } + + void msgpack_unpack(object o) + { + convert(static_cast(*this), o); + } +}; + + template packer& operator<< (packer& o, const object& v) From ccdb39ac60f256db36a9a82b3792182f3b339d13 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0029/1648] lang/c/msgpack: msgpack::unpacker: fix buffer reallocation algorithm git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@76 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/unpack.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index cbf1356..ea4d414 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -198,7 +198,7 @@ void unpacker::expand_buffer(size_t len) { if(m_off == 0) { size_t next_size; - if(m_free != 0) { next_size = m_free * 2; } + if(m_used != 0) { next_size = (m_used + m_free) * 2; } else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } while(next_size < len + m_used) { next_size *= 2; } From 40ca91e10110ed7de761a04a96119d14adb41fff Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0030/1648] lang/c/msgpack: ./configure requires erb git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@77 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- configure.in | 1 - cpp/Makefile.am | 2 +- cpp/configure.in | 6 +++++- cpp/type/map.hpp | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 33acfa6..474d6e2 100644 --- a/configure.in +++ b/configure.in @@ -3,7 +3,6 @@ AM_INIT_AUTOMAKE(msgpack, 0.1.0) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL -#AC_CHECK_PROG(RUBY, ruby, ruby, [$PATH]) AC_CONFIG_SUBDIRS([c cpp]) AC_OUTPUT([Makefile]) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index e244763..61a616a 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -30,7 +30,7 @@ unpack.lo: msgpack/type/tuple.hpp zone.lo: msgpack/type/tuple.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb - erb $< > $@ + $(ERB) $< > $@ MOSTLYCLEANFILES = msgpack/type/tuple.hpp diff --git a/cpp/configure.in b/cpp/configure.in index a60a489..2c3e5d0 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -9,7 +9,11 @@ fi AC_PROG_CXX AC_PROG_LIBTOOL -AC_CHECK_PROG(ERB, erb, erb, [$PATH]) + +AC_CHECK_PROG(ERB, erb, erb) +if test "x$ERB" = x; then + AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) +fi AC_CHECK_LIB(stdc++, main) diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 3b544df..c79f31c 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -21,6 +21,7 @@ #include "msgpack/object.hpp" #include #include +#include namespace msgpack { From bb4ea9d17c20007405b0b6e703f5f47bf54e5da0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0031/1648] lang/c/msgpack: improved configure.in git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@78 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile.am | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 61a616a..19c6007 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -30,7 +30,8 @@ unpack.lo: msgpack/type/tuple.hpp zone.lo: msgpack/type/tuple.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb - $(ERB) $< > $@ + $(ERB) $< > $@.tmp + mv $@.tmp $@ MOSTLYCLEANFILES = msgpack/type/tuple.hpp From d930090f130bf363d2330de1a15877ed9c5e9edc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0032/1648] lang/c/msgpack: C++ binding: abolished implicit convertion and added object::convert() git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@79 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 61 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 5ed1fab..5f77d3e 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -44,8 +44,7 @@ namespace type { struct object { - unsigned char type; - union { + union union_type { bool boolean; uint64_t u64; int64_t i64; @@ -58,15 +57,36 @@ struct object { const char* ptr; uint32_t size; } ref; - } via; + }; - template - operator T() { T v; convert(v, *this); return v; }; - - template - T as() { T v; convert(v, *this); return v; } + unsigned char type; + union_type via; bool is_nil() { return type == type::NIL; } + + template + T as(); + + template + void convert(T& v); + +private: + struct implicit_type; + +public: + implicit_type convert(); +}; + + +struct object::implicit_type { + implicit_type(object o) : obj(o) { } + ~implicit_type() { } + + template + operator T() { return obj.as(); } + +private: + object obj; }; std::ostream& operator<< (std::ostream& s, const object o); @@ -74,9 +94,11 @@ std::ostream& operator<< (std::ostream& s, const object o); bool operator==(const object x, const object y); inline bool operator!=(const object x, const object y) { return !(x == y); } - inline object& operator>> (object o, object& v) - { v = o; return v; } +{ + v = o; + return v; +} template packer& operator<< (packer& o, const object& v); @@ -147,6 +169,25 @@ public: +inline object::implicit_type object::convert() +{ + return implicit_type(*this); +} + +template +inline T object::as() +{ + T v; + msgpack::convert(v, *this); + return v; +} + +template +void object::convert(T& v) +{ + msgpack::convert(v, *this); +} + template packer& operator<< (packer& o, const object& v) { From a0fd7c063adf9b249b40429b272178449a96f4ec Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0033/1648] lang/c/msgpack: wrote README git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@80 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- README | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/README b/README index 31a482a..87d9279 100644 --- a/README +++ b/README @@ -3,6 +3,56 @@ MessagePack Binary-based efficient data interchange format. +*Requirements + + MessagePack is only tested on Linux and Mac OS X, but it may run on other + UNIX-like platforms. + + Following programs is required to build: + - gcc >= 4.0 with C++ support + - ruby >= 1.8 (ruby is used as a preprocessor) + + +*Installation + + Simply run ./configure && make && make install to install C and C++ binding. + + $ ./configure --prefix=path/to/prefix + $ make + $ make install + + To install Ruby binding, run ./gengem.sh script in ruby/ directory and install + generated gem package. + + $ cd ruby + $ ./gengem.sh + $ gem install gem/pkg/msgpack-*.gem + + +*Usage + + C++: + #include + // TODO + + + C: + #include + /* TODO */ + + + Ruby: + require 'msgpack' + + # serialize + buf = [1, 2, 3].to_msgpack + + # deserialize + p MessagePack::unpack(buf) + + # TODO + + Copyright (C) 2008 FURUHASHI Sadayuki From 33986868162e0c312fd9547e7c05199d684bd07a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0034/1648] {c,cpp}/bootstrap: improve compatibility git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@81 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/bootstrap | 2 +- cpp/bootstrap | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/c/bootstrap b/c/bootstrap index 6a1e814..7e61b82 100755 --- a/c/bootstrap +++ b/c/bootstrap @@ -1,3 +1,3 @@ #!/bin/sh NO_NEST=1 -source ../bootstrap +. ../bootstrap diff --git a/cpp/bootstrap b/cpp/bootstrap index 6a1e814..7e61b82 100755 --- a/cpp/bootstrap +++ b/cpp/bootstrap @@ -1,3 +1,3 @@ #!/bin/sh NO_NEST=1 -source ../bootstrap +. ../bootstrap From 8be25a7f3216007dd3e9561076bd2e7cf3945b90 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0035/1648] unpacker::unpacker() accepts initial buffer size git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@82 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/unpack.cpp | 9 +++++---- cpp/unpack.hpp | 11 +++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index ea4d414..8edf407 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -178,13 +178,14 @@ private: }; -unpacker::unpacker() : +unpacker::unpacker(size_t initial_buffer_size) : m_buffer(NULL), m_used(0), m_free(0), m_off(0), m_zone(new zone()), - m_ctx(new context(&*m_zone)) + m_ctx(new context(&*m_zone)), + m_initial_buffer_size(initial_buffer_size) { } @@ -199,14 +200,14 @@ void unpacker::expand_buffer(size_t len) if(m_off == 0) { size_t next_size; if(m_used != 0) { next_size = (m_used + m_free) * 2; } - else { next_size = UNPACKER_INITIAL_BUFFER_SIZE; } + else { next_size = m_initial_buffer_size; } while(next_size < len + m_used) { next_size *= 2; } m_buffer = m_zone->realloc(m_buffer, next_size); m_free = next_size - m_used; } else { - size_t next_size = UNPACKER_INITIAL_BUFFER_SIZE; + size_t next_size = m_initial_buffer_size; while(next_size < len + m_used - m_off) { next_size *= 2; } char* tmp = m_zone->malloc(next_size); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index ffee60d..ac20de3 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -23,16 +23,13 @@ #include #include -#ifndef MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE -#define MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE 8*1024 +#ifndef MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE +#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 8*1024 #endif namespace msgpack { -static const size_t UNPACKER_INITIAL_BUFFER_SIZE = MSGPACK_UNPACKER_INITIAL_BUFFER_SIZE; - - struct unpack_error : public std::runtime_error { unpack_error(const std::string& msg) : std::runtime_error(msg) { } @@ -41,7 +38,7 @@ struct unpack_error : public std::runtime_error { class unpacker { public: - unpacker(); + unpacker(size_t initial_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); ~unpacker(); public: @@ -139,6 +136,8 @@ private: struct context; context* m_ctx; + const size_t m_initial_buffer_size; + private: void expand_buffer(size_t len); From 05b63f6c87d45ea2e1aa8a4d9f8bf665a50591a5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:09:59 +0000 Subject: [PATCH 0036/1648] C++: operator<< (std::ostream&, const object&): fix missing double support git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@83 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/object.cpp b/cpp/object.cpp index 45cfb5f..52b6ad0 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -41,6 +41,10 @@ std::ostream& operator<< (std::ostream& s, const object o) s << o.via.i64; break; + case type::DOUBLE: + s << o.via.dec; + break; + case type::RAW: (s << '"').write(o.via.ref.ptr, o.via.ref.size) << '"'; break; From 7e872371a241771489f3c48b7d364e4f6a9a248a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0037/1648] add examples git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@84 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- example/protocol.cc | 86 +++++++++++++++++++++++++++++ example/simple.cc | 31 +++++++++++ example/simple.rb | 5 ++ example/stream.cc | 131 ++++++++++++++++++++++++++++++++++++++++++++ example/stream.rb | 67 ++++++++++++++++++++++ 5 files changed, 320 insertions(+) create mode 100644 example/protocol.cc create mode 100644 example/simple.cc create mode 100644 example/simple.rb create mode 100644 example/stream.cc create mode 100644 example/stream.rb diff --git a/example/protocol.cc b/example/protocol.cc new file mode 100644 index 0000000..e7f2a38 --- /dev/null +++ b/example/protocol.cc @@ -0,0 +1,86 @@ +#include +#include +#include +#include + +namespace myprotocol { + using namespace msgpack::type; + using msgpack::define; + + struct Get : define< tuple > { + Get() { } + Get(uint32_t f, const std::string& k) : + define_type(msgpack_type(f, k)) { } + uint32_t& flags() { return get<0>(); } + std::string& key() { return get<1>(); } + }; + + struct Put : define< tuple > { + Put() { } + Put(uint32_t f, const std::string& k, const char* valref, size_t vallen) : + define_type(msgpack_type( f, k, raw_ref(valref,vallen) )) { } + uint32_t& flags() { return get<0>(); } + std::string& key() { return get<1>(); } + raw_ref& value() { return get<2>(); } + }; + + struct MultiGet : define< std::vector > { + }; +} + + +int main(void) +{ + // send Get request + std::stringstream stream; + { + myprotocol::Get req; + req.flags() = 0; + req.key() = "key0"; + msgpack::pack(stream, req); + } + + stream.seekg(0); + + // receive Get request + { + std::string buffer(stream.str()); + + msgpack::zone mempool; + msgpack::object o = + msgpack::unpack(buffer.data(), buffer.size(), mempool); + + myprotocol::Get req; + msgpack::convert(req, o); + std::cout << "received: " << o << std::endl; + } + + + stream.str(""); + + + // send MultiGet request + { + myprotocol::MultiGet req; + req.push_back( myprotocol::Get(1, "key1") ); + req.push_back( myprotocol::Get(2, "key2") ); + req.push_back( myprotocol::Get(3, "key3") ); + msgpack::pack(stream, req); + } + + stream.seekg(0); + + // receive MultiGet request + { + std::string buffer(stream.str()); + + msgpack::zone mempool; + msgpack::object o = + msgpack::unpack(buffer.data(), buffer.size(), mempool); + + myprotocol::MultiGet req; + msgpack::convert(req, o); + std::cout << "received: " << o << std::endl; + } +} + diff --git a/example/simple.cc b/example/simple.cc new file mode 100644 index 0000000..74beeae --- /dev/null +++ b/example/simple.cc @@ -0,0 +1,31 @@ +#include +#include +#include +#include + +int main(void) +{ + // this is target object + msgpack::type::tuple src(1, true, "example"); + + // any classes that implements write(const char*,size_t) can be a buffer + std::stringstream buffer; + msgpack::pack(buffer, src); + + // send the buffer ... + buffer.seekg(0); + + // deserialize the buffer into msgpack::object type + msgpack::zone mempool; + std::string str(buffer.str()); + msgpack::object deserialized = + msgpack::unpack(str.data(), str.size(), mempool); + + // msgpack::object supports ostream + std::cout << deserialized << std::endl; + + // convert msgpack::object type into the original type + msgpack::type::tuple dst; + msgpack::convert(dst, deserialized); +} + diff --git a/example/simple.rb b/example/simple.rb new file mode 100644 index 0000000..90b4696 --- /dev/null +++ b/example/simple.rb @@ -0,0 +1,5 @@ +require 'msgpack' + +serialized = [1, -1, true, false, nil, {"key" => "value"}].to_msgpack +p MessagePack.unpack(serialized) + diff --git a/example/stream.cc b/example/stream.cc new file mode 100644 index 0000000..49927de --- /dev/null +++ b/example/stream.cc @@ -0,0 +1,131 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +class Server { +public: + Server(int sock) : m_sock(sock) { } + ~Server() { } + + typedef std::auto_ptr auto_zone; + + void socket_readable() + { + m_pac.reserve_buffer(1024); + + ssize_t count = + read(m_sock, m_pac.buffer(), m_pac.buffer_capacity()); + + if(count < 0) { + if(errno == EAGAIN || errno == EINTR) { + return; + } else { + throw std::runtime_error(strerror(errno)); + } + } else if(count == 0) { + throw std::runtime_error("connection closed"); + } + + m_pac.buffer_consumed(count); + + while(m_pac.execute()) { + msgpack::object msg = m_pac.data(); + + auto_zone life( m_pac.release_zone() ); + + m_pac.reset(); + + process_message(msg, life); + } + } + +private: + void process_message(msgpack::object msg, auto_zone& life) + { + std::cout << "message reached: " << msg << std::endl; + } + +private: + int m_sock; + msgpack::unpacker m_pac; +}; + + +static void* run_server(void* arg) +try { + Server* srv = reinterpret_cast(arg); + + while(true) { + srv->socket_readable(); + } + return NULL; + +} catch (std::exception& e) { + std::cerr << "error while processing client packet: " + << e.what() << std::endl; + return NULL; + +} catch (...) { + std::cerr << "error while processing client packet: " + << "unknown error" << std::endl; + return NULL; +} + + +struct fwriter { + fwriter(int fd) : m_fp( fdopen(fd, "w") ) { } + + void write(const char* buf, size_t buflen) + { + size_t count = fwrite(buf, buflen, 1, m_fp); + //if(fwrite(buf, buflen, 1, m_fp) < 1) { + if(count < 1) { + std::cout << buflen << std::endl; + std::cout << count << std::endl; + throw std::runtime_error(strerror(errno)); + } + } + + void flush() { fflush(m_fp); } + + void close() { fclose(m_fp); } + +private: + FILE* m_fp; +}; + + +int main(void) +{ + int pair[2]; + pipe(pair); + + // run server thread + Server srv(pair[0]); + pthread_t thread; + pthread_create(&thread, NULL, + run_server, reinterpret_cast(&srv)); + + // client thread: + fwriter writer(pair[1]); + + typedef msgpack::type::tuple put_t; + typedef msgpack::type::tuple get_t; + + put_t req1("put", "apple", "red"); + put_t req2("put", "lemon", "yellow"); + get_t req3("get", "apple"); + msgpack::pack(writer, req1); + msgpack::pack(writer, req2); + msgpack::pack(writer, req3); + writer.flush(); + writer.close(); + + pthread_join(thread, NULL); +} + diff --git a/example/stream.rb b/example/stream.rb new file mode 100644 index 0000000..e53ce82 --- /dev/null +++ b/example/stream.rb @@ -0,0 +1,67 @@ +require 'msgpack' + +class Server + def initialize(sock) + @sock = sock + @pk = MessagePack::Unpacker.new + @buffer = '' + @nread = 0 + end + + def run + while true + begin + data = @sock.sysread(1024) + rescue + puts "connection closed (#{$!})" + return + end + receive_data(data) + end + end + + private + def receive_data(data) + @buffer << data + + while true + @nread = @pk.execute(@buffer, @nread) + + if @pk.finished? + msg = @pk.data + process_message(msg) + + @pk.reset + @buffer.slice!(0, @nread) + @nread = 0 + next unless @buffer.empty? + end + + break + end + + rescue + puts "error while processing client packet: #{$!}" + end + + def process_message(msg) + puts "message reached: #{msg.inspect}" + end +end + + +rpipe, wpipe = IO.pipe + +# run server thread +thread = Thread.new(Server.new(rpipe)) {|srv| + srv.run +} + +# client thread: +wpipe.write ["put", "apple", "red"].to_msgpack +wpipe.write ["put", "lemon", "yellow"].to_msgpack +wpipe.write ["get", "apple"].to_msgpack +wpipe.close + +thread.join + From 92dd6de59da206571ce488fe1daae80afa5b6b13 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0038/1648] version 0.2.0 git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@85 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 474d6e2..bd34885 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ AC_INIT(msgpack/unpack_template.h) -AM_INIT_AUTOMAKE(msgpack, 0.1.0) +AM_INIT_AUTOMAKE(msgpack, 0.2.0) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL From 512e7600f4977e7d4369af5b0d0f0c540427833a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0039/1648] msgpack::type::tuple supports reference element git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@86 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/type/tuple.hpp.erb | 95 ++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 55 deletions(-) diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 397f660..13d4bd7 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -32,36 +32,48 @@ template , typename A<%=i% struct tuple; template -struct tuple_type; +struct tuple_element; template -struct const_tuple_type; +struct const_tuple_element; template -struct tuple_element { +struct tuple_type { typedef T type; - tuple_element(T& x) : _x(x) {} - type& get() { return _x; } - const type& get() const { return _x; } -private: - type& _x; + typedef T value_type; + typedef T& reference; + typedef const T& const_reference; + typedef const T& transparent_reference; }; template -struct const_tuple_element { +struct tuple_type { typedef T type; - const_tuple_element(const T& x) : _x(x) {} - const type& get() const { return _x; } -private: - const type& _x; + typedef T& value_type; + typedef T& reference; + typedef const T& const_reference; + typedef T& transparent_reference; +}; + +template +struct tuple_type { + typedef T type; + typedef T& value_type; + typedef T& reference; + typedef const T& const_reference; + typedef const T& transparent_reference; }; <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> template , typename A<%=k%><%}%>> -struct tuple_type, A<%=k%><%}%>>, <%=j%>> : tuple_element> { - tuple_type(tuple, A<%=k%> <%}%>>& x) : tuple_element>(x.a<%=j%>) {} +struct tuple_element, A<%=k%><%}%>>, <%=j%>> : tuple_type> { + tuple_element(tuple, A<%=k%> <%}%>>& x) : _x(x.a<%=j%>) {} + typename tuple_type>::reference get() { return _x; } + typename tuple_type>::const_reference get() const { return _x; } +private: + typename tuple_type>::reference _x; }; <%}%> <%}%> @@ -70,8 +82,11 @@ struct tuple_type, A<%=k%><%}%>>, <%=j%>> : tuple_ele <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> template , typename A<%=k%><%}%>> -struct const_tuple_type, A<%=k%><%}%>>, <%=j%>> : const_tuple_element> { - const_tuple_type(const tuple, A<%=k%><%}%>>& x) : const_tuple_element>(x.a<%=j%>) {} +struct const_tuple_element, A<%=k%><%}%>>, <%=j%>> : tuple_type> { + const_tuple_element(const tuple, A<%=k%><%}%>>& x) : _x(x.a<%=j%>) {} + typename tuple_type>::const_reference get() const { return _x; } +private: + typename tuple_type>::const_reference _x; }; <%}%> <%}%> @@ -79,7 +94,7 @@ struct const_tuple_type, A<%=k%><%}%>>, <%=j%>> : con <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> -tuple, A<%=j%><%}%>> make_tuple(const A0& a0<%1.upto(i) {|j|%>, const A<%=j%>& a<%=j%><%}%>) +tuple, A<%=j%><%}%>> make_tuple(typename tuple_type::transparent_reference a0<%1.upto(i) {|j|%>, typename tuple_type>::transparent_reference a<%=j%><%}%>) { return tuple, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>); } @@ -92,14 +107,15 @@ struct tuple<> { <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> struct tuple, A<%=j%><%}%>> { + typedef tuple, A<%=j%><%}%>> value_type; tuple() {} - tuple(const A0& _a0<%1.upto(i) {|j|%>, const A<%=j%>& _a<%=j%><%}%>) : - a0(_a0) <%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} + tuple(typename tuple_type::transparent_reference _a0<%1.upto(i) {|j|%>, typename tuple_type>::transparent_reference _a<%=j%><%}%>) : + a0(_a0)<%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} tuple(object o) { convert(*this, o); } - template typename tuple_type, A<%=j%><%}%>>, N>::type& get() - { return tuple_type, A<%=j%><%}%>>, N>(*this).get(); } - template const typename const_tuple_type, A<%=j%><%}%>>, N>::type& get() const - { return const_tuple_type, A<%=j%><%}%>>, N>(*this).get(); } + template typename tuple_element::reference get() + { return tuple_element(*this).get(); } + template typename const_tuple_element::const_reference get() const + { return const_tuple_element(*this).get(); } <%0.upto(i) {|j|%> A<%=j%> a<%=j%>;<%}%> }; @@ -148,37 +164,6 @@ const packer& operator<< ( <%}%> -// FIXME -/* -template -struct tuple_just; - -template -struct tuple_just { - A0 a0; - static inline void convert(object o, tuple_just& v) - { - if(o.type != type::ARRAY) { throw type_error(); } - if(o.v.container.size != 1) { throw type_error(); } - msgpack::convert(o.v.container.ptr[0], v.a0); - } -}; - -template -struct tuple_just { - A0 a0; - A1 a1; - static inline void convert(object o, tuple_just& v) - { - if(o.type != type::ARRAY) { throw type_error(); } - if(o.v.container.size != 2) { throw type_error(); } - msgpack::convert(o.v.container.ptr[0], v.a0); - msgpack::convert(o.v.container.ptr[1], v.a1); - } -}; -*/ - - } // namespace msgpack #endif /* msgpack/type/tuple.hpp */ From 15837bc3322b6f2fd6191187f4754f7d65c786a4 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0040/1648] add ruby/test_case.rb git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@87 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- ruby/test_case.rb | 142 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 ruby/test_case.rb diff --git a/ruby/test_case.rb b/ruby/test_case.rb new file mode 100644 index 0000000..af6d160 --- /dev/null +++ b/ruby/test_case.rb @@ -0,0 +1,142 @@ +require 'test/unit' +require 'msgpack' + +class MessagePackTestFormat < Test::Unit::TestCase + def self.it(name, &block) + define_method("test_#{name}", &block) + end + + it "nil" do + check 1, nil + end + + it "true" do + check 1, true + end + + it "false" do + check 1, false + end + + it "zero" do + check 1, 0 + end + + it "positive fixnum" do + check 1, 1 + check 1, (1<<6) + check 1, (1<<7)-1 + end + + it "positive int 8" do + check 1, -1 + check 2, (1<<7) + check 2, (1<<8)-1 + end + + it "positive int 16" do + check 3, (1<<8) + check 3, (1<<16)-1 + end + + it "positive int 32" do + check 5, (1<<16) + check 5, (1<<32)-1 + end + + it "positive int 64" do + check 9, (1<<32) + check 9, (1<<64)-1 + end + + it "negative fixnum" do + check 1, -1 + check 1, -((1<<5)-1) + check 1, -(1<<5) + end + + it "negative int 8" do + check 2, -((1<<5)+1) + check 2, -(1<<7) + end + + it "negative int 16" do + check 3, -((1<<7)+1) + check 3, -(1<<15) + end + + it "negative int 32" do + check 5, -((1<<15)+1) + check 5, -(1<<31) + end + + it "negative int 64" do + check 9, -((1<<31)+1) + check 9, -(1<<63) + end + + it "fixraw" do + check_raw 1, 0 + check_raw 1, (1<<5)-1 + end + + it "raw 16" do + check_raw 3, (1<<5) + check_raw 3, (1<<16)-1 + end + + it "raw 32" do + check_raw 5, (1<<16) + #check_raw 5, (1<<32)-1 # memory error + end + + it "fixarray" do + check_array 1, 0 + check_array 1, (1<<4)-1 + end + + it "array 16" do + check_array 3, (1<<4) + check_array 3, (1<<16)-1 + end + + it "array 32" do + check_array 5, (1<<16) + #check_array 5, (1<<32)-1 # memory error + end + +# it "fixmap" do +# check_map 1, 0 +# check_map 1, (1<<4)-1 +# end +# +# it "map 16" do +# check_map 3, (1<<4) +# check_map 3, (1<<16)-1 +# end +# +# it "map 32" do +# check_map 5, (1<<16) +# #check_map 5, (1<<32)-1 # memory error +# end + + private + def check(len, obj) + v = obj.to_msgpack + assert_equal(v.length, len) + assert_equal(MessagePack.unpack(v), obj) + end + + def check_raw(overhead, num) + check num+overhead, " "*num + end + + def check_array(overhead, num) + check num+overhead, Array.new(num) + end + + def check_map(overhead, num) + # FIXME + end +end + From 8f3444c08141520a0977adce643ce0eb7f0324cd Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0041/1648] ruby binding: fix Fixnum serialization bug on x86_64 git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@88 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- configure.in | 2 +- msgpack/pack_template.h | 141 +++++++++++++++++++++++++++++--- ruby/gem/lib/msgpack/version.rb | 2 +- ruby/pack.c | 6 +- 4 files changed, 134 insertions(+), 17 deletions(-) diff --git a/configure.in b/configure.in index bd34885..63762b0 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ AC_INIT(msgpack/unpack_template.h) -AM_INIT_AUTOMAKE(msgpack, 0.2.0) +AM_INIT_AUTOMAKE(msgpack, 0.2.1) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 1dbf6fd..04f8c98 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -68,15 +68,14 @@ * Integer */ -// wrapper -msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) +static inline void msgpack_pack_compress_int32(msgpack_pack_user x, uint32_t d) { - if(d < -32) { - if(d < -32768) { + if(d < -(1<<5)) { + if(d < -(1<<15)) { // signed 32 const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); - } else if(d < -128) { + } else if(d < -(1<<7)) { // signed 16 const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); @@ -85,15 +84,15 @@ msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) const unsigned char buf[2] = {0xd0, (uint8_t)d}; msgpack_pack_append_buffer(x, buf, 2); } - } else if(d < 128) { + } else if(d < (1<<7)) { // fixnum msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); } else { - if(d < 256) { + if(d < (1<<8)) { // unsigned 8 const unsigned char buf[2] = {0xcc, (uint8_t)d}; msgpack_pack_append_buffer(x, buf, 2); - } else if(d < 65536) { + } else if(d < (1<<16)) { // unsigned 16 const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); @@ -105,11 +104,10 @@ msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) } } -// wrapper -msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) +static inline void msgpack_pack_compress_uint32(msgpack_pack_user x, uint32_t d) { - if(d < 256) { - if(d < 128) { + if(d < (1<<8)) { + if(d < (1<<7)) { // fixnum msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); } else { @@ -118,7 +116,7 @@ msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) msgpack_pack_append_buffer(x, buf, 2); } } else { - if(d < 65536) { + if(d < (1<<16)) { // unsigned 16 const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); @@ -130,6 +128,123 @@ msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) } } +static inline void msgpack_pack_compress_int64(msgpack_pack_user x, int64_t d) +{ + if(d < -(1LL<<5)) { + if(d < -(1LL<<15)) { + if(d < -(1LL<<31)) { + // signed 64 + const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); + } else { + // signed 32 + const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } + } else { + if(d < -(1<<7)) { + // signed 16 + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + // signed 8 + const unsigned char buf[2] = {0xd0, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } + } + } else if(d < (1<<7)) { + // fixnum + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); + } else { + if(d < (1LL<<16)) { + if(d < (1<<8)) { + // unsigned 8 + const unsigned char buf[2] = {0xcc, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } else { + // unsigned 16 + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } + } else { + if(d < (1LL<<32)) { + // unsigned 32 + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } else { + // unsigned 64 + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); + } + } + } +} + +static inline void msgpack_pack_compress_uint64(msgpack_pack_user x, uint64_t d) +{ + if(d < (1ULL<<8)) { + if(d < (1<<7)) { + // fixnum + msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); + } else { + // unsigned 8 + const unsigned char buf[2] = {0xcc, (uint8_t)d}; + msgpack_pack_append_buffer(x, buf, 2); + } + } else { + if(d < (1ULL<<16)) { + // signed 16 + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else if(d < (1ULL<<32)) { + // signed 32 + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } else { + // signed 64 + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); + } + } +} + +msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) +{ +#if SIZEOF_INT == 8 + msgpack_pack_compress_int64(x, d); +#else + msgpack_pack_compress_int32(x, d); +#endif +} + +msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) +{ +#if SIZEOF_INT == 8 + msgpack_pack_compress_uint64(x, d); +#else + msgpack_pack_compress_uint32(x, d); +#endif +} + +msgpack_pack_inline_func(long)(msgpack_pack_user x, long d) +{ +#if SIZEOF_LONG == 8 + msgpack_pack_compress_int64(x, d); +#else + msgpack_pack_compress_int32(x, d); +#endif +} + +msgpack_pack_inline_func(unsigned_long)(msgpack_pack_user x, unsigned long d) +{ +#if SIZEOF_LONG == 8 + msgpack_pack_compress_uint64(x, d); +#else + msgpack_pack_compress_uint32(x, d); +#endif +} + + msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) { if(d < 128) { diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb index b2a5db6..229c746 100644 --- a/ruby/gem/lib/msgpack/version.rb +++ b/ruby/gem/lib/msgpack/version.rb @@ -2,7 +2,7 @@ module MessagePack module VERSION #:nodoc: MAJOR = 0 MINOR = 2 - TINY = 0 + TINY = 1 STRING = [MAJOR, MINOR, TINY].join('.') end diff --git a/ruby/pack.c b/ruby/pack.c index e62419d..5262024 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -20,7 +20,7 @@ #include #define msgpack_pack_inline_func(name) \ - static void msgpack_pack_##name + static inline void msgpack_pack_##name #define msgpack_pack_user VALUE @@ -30,6 +30,8 @@ /* static void msgpack_pack_int(VALUE x, int d); static void msgpack_pack_unsigned_int(VALUE x, unsigned int d); +static void msgpack_pack_long(VALUE x, long d); +static void msgpack_pack_unsigned_long(VALUE x, unsigned long d); static void msgpack_pack_uint8(VALUE x, uint8_t d); static void msgpack_pack_uint16(VALUE x, uint16_t d); static void msgpack_pack_uint32(VALUE x, uint32_t d); @@ -93,7 +95,7 @@ static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); - msgpack_pack_int(out, FIX2INT(self)); + msgpack_pack_long(out, FIX2LONG(self)); return out; } From adba617f45f89cacbd23667744d0cc17668ecdda Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0042/1648] c, c++ binding: catch up with ruby binding git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@89 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/pack.h | 2 + cpp/pack.hpp | 42 +++--- msgpack/pack_template.h | 276 ++++++++++++++++++++-------------------- 3 files changed, 165 insertions(+), 155 deletions(-) diff --git a/c/pack.h b/c/pack.h index ec0683e..fac642f 100644 --- a/c/pack.h +++ b/c/pack.h @@ -39,6 +39,8 @@ void msgpack_pack_free(msgpack_pack_t* ctx); void msgpack_pack_int(msgpack_pack_t* ctx, int d); void msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); +void msgpack_pack_long(msgpack_pack_t* ctx, long d); +void msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); void msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); void msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); void msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 4fda4ff..aee4b87 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -30,29 +30,33 @@ public: packer(Stream& s); public: - void pack_int(int d) { pack_int_impl(m_stream, d); } - void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } - void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } - void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } - void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } - void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } - void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } - void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } - void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } - void pack_float(float d) { pack_float_impl(m_stream, d); } - void pack_double(double d) { pack_double_impl(m_stream, d); } - void pack_nil() { pack_nil_impl(m_stream); } - void pack_true() { pack_true_impl(m_stream); } - void pack_false() { pack_false_impl(m_stream); } - void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } - void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } - void pack_raw(size_t l) { pack_raw_impl(m_stream, l); } - void pack_raw_body(const char* b, size_t l) { pack_raw_body_impl(m_stream, b, l); } + void pack_int(int d) { pack_int_impl(m_stream, d); } + void pack_long(long d) { pack_long_impl(m_stream, d); } + void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } + void pack_unsigned_long(unsigned long d) { pack_unsigned_long_impl(m_stream, d); } + void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } + void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } + void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } + void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } + void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } + void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } + void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } + void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } + void pack_float(float d) { pack_float_impl(m_stream, d); } + void pack_double(double d) { pack_double_impl(m_stream, d); } + void pack_nil() { pack_nil_impl(m_stream); } + void pack_true() { pack_true_impl(m_stream); } + void pack_false() { pack_false_impl(m_stream); } + void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } + void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + void pack_raw(size_t l) { pack_raw_impl(m_stream, l); } + void pack_raw_body(const char* b, size_t l) { pack_raw_body_impl(m_stream, b, l); } private: static void pack_int_impl(Stream& x, int d); + static void pack_long_impl(Stream& x, long d); static void pack_unsigned_int_impl(Stream& x, unsigned int d); + static void pack_unsigned_long_impl(Stream& x, unsigned long d); static void pack_uint8_impl(Stream& x, uint8_t d); static void pack_uint16_impl(Stream& x, uint16_t d); static void pack_uint32_impl(Stream& x, uint32_t d); diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 04f8c98..94fbfd4 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -68,145 +68,145 @@ * Integer */ -static inline void msgpack_pack_compress_int32(msgpack_pack_user x, uint32_t d) -{ - if(d < -(1<<5)) { - if(d < -(1<<15)) { - // signed 32 - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } else if(d < -(1<<7)) { - // signed 16 - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } else { - // signed 8 - const unsigned char buf[2] = {0xd0, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } - } else if(d < (1<<7)) { - // fixnum - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); - } else { - if(d < (1<<8)) { - // unsigned 8 - const unsigned char buf[2] = {0xcc, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } else if(d < (1<<16)) { - // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } else { - // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } - } -} +#define msgpack_pack_compress_int32(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<15)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) -static inline void msgpack_pack_compress_uint32(msgpack_pack_user x, uint32_t d) -{ - if(d < (1<<8)) { - if(d < (1<<7)) { - // fixnum - msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); - } else { - // unsigned 8 - const unsigned char buf[2] = {0xcc, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } - } else { - if(d < (1<<16)) { - // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } else { - // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } - } -} +#define msgpack_pack_compress_uint32(x, d) \ +do { \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) -static inline void msgpack_pack_compress_int64(msgpack_pack_user x, int64_t d) -{ - if(d < -(1LL<<5)) { - if(d < -(1LL<<15)) { - if(d < -(1LL<<31)) { - // signed 64 - const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; - msgpack_pack_append_buffer(x, buf, 9); - } else { - // signed 32 - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } - } else { - if(d < -(1<<7)) { - // signed 16 - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } else { - // signed 8 - const unsigned char buf[2] = {0xd0, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } - } - } else if(d < (1<<7)) { - // fixnum - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); - } else { - if(d < (1LL<<16)) { - if(d < (1<<8)) { - // unsigned 8 - const unsigned char buf[2] = {0xcc, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } else { - // unsigned 16 - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } - } else { - if(d < (1LL<<32)) { - // unsigned 32 - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } else { - // unsigned 64 - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; - msgpack_pack_append_buffer(x, buf, 9); - } - } - } -} +#define msgpack_pack_compress_int64(x, d) \ +do { \ + if(d < -(1LL<<5)) { \ + if(d < -(1LL<<15)) { \ + if(d < -(1LL<<31)) { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } else { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } else { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + if(d < (1LL<<16)) { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } else { \ + if(d < (1LL<<32)) { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* unsigned 64 */ \ + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ + } \ +} while(0) -static inline void msgpack_pack_compress_uint64(msgpack_pack_user x, uint64_t d) -{ - if(d < (1ULL<<8)) { - if(d < (1<<7)) { - // fixnum - msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); - } else { - // unsigned 8 - const unsigned char buf[2] = {0xcc, (uint8_t)d}; - msgpack_pack_append_buffer(x, buf, 2); - } - } else { - if(d < (1ULL<<16)) { - // signed 16 - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; - msgpack_pack_append_buffer(x, buf, 3); - } else if(d < (1ULL<<32)) { - // signed 32 - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; - msgpack_pack_append_buffer(x, buf, 5); - } else { - // signed 64 - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; - msgpack_pack_append_buffer(x, buf, 9); - } - } -} +#define msgpack_pack_compress_uint64(x, d) \ +do { \ + if(d < (1ULL<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ +} while(0) msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) { @@ -244,6 +244,10 @@ msgpack_pack_inline_func(unsigned_long)(msgpack_pack_user x, unsigned long d) #endif } +#undef msgpack_pack_compress_int32 +#undef msgpack_pack_compress_uint32 +#undef msgpack_pack_compress_int64 +#undef msgpack_pack_compress_uint64 msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) { From cd973b8483af21d4a3a03497aa223b82b5c1a9a2 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0043/1648] integrate machine-dependent integer serialization routine to msgpack/pack_template.h git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@90 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- Makefile.am | 1 + c/pack.c | 4 + c/pack.h | 6 + cpp/pack.hpp | 74 +++++-- cpp/type/integer.hpp | 116 +--------- msgpack/pack_define.h | 26 +++ msgpack/pack_template.h | 481 ++++++++++++++++++++++++++++++---------- ruby/gem/Manifest.txt | 1 + ruby/gengem.sh | 1 + ruby/pack.c | 30 +-- 10 files changed, 468 insertions(+), 272 deletions(-) create mode 100644 msgpack/pack_define.h diff --git a/Makefile.am b/Makefile.am index d79f7eb..c23320a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,6 +1,7 @@ SUBDIRS = c cpp nobase_include_HEADERS = \ + msgpack/pack_define.h \ msgpack/pack_template.h \ msgpack/unpack_define.h \ msgpack/unpack_template.h diff --git a/c/pack.c b/c/pack.c index 766a9d1..f4787c0 100644 --- a/c/pack.c +++ b/c/pack.c @@ -16,12 +16,16 @@ * limitations under the License. */ #include "msgpack/pack.h" +#include "msgpack/pack_define.h" #include #define msgpack_pack_inline_func(name) \ void msgpack_pack_##name +#define msgpack_pack_inline_func_cint(name) \ + void msgpack_pack_##name + #define msgpack_pack_user msgpack_pack_t* #define msgpack_pack_append_buffer(user, buf, len) \ diff --git a/c/pack.h b/c/pack.h index fac642f..ecf675c 100644 --- a/c/pack.h +++ b/c/pack.h @@ -41,6 +41,7 @@ void msgpack_pack_int(msgpack_pack_t* ctx, int d); void msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); void msgpack_pack_long(msgpack_pack_t* ctx, long d); void msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); + void msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); void msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); void msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); @@ -49,13 +50,18 @@ void msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); void msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); void msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); void msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); + void msgpack_pack_float(msgpack_pack_t* ctx, float d); void msgpack_pack_double(msgpack_pack_t* ctx, double d); + void msgpack_pack_nil(msgpack_pack_t* ctx); void msgpack_pack_true(msgpack_pack_t* ctx); void msgpack_pack_false(msgpack_pack_t* ctx); + void msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); + void msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); + void msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); void msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); diff --git a/cpp/pack.hpp b/cpp/pack.hpp index aee4b87..e860f4c 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -20,6 +20,8 @@ #include // __BYTE_ORDER #include +#include +#include "msgpack/pack_define.h" namespace msgpack { @@ -30,33 +32,38 @@ public: packer(Stream& s); public: - void pack_int(int d) { pack_int_impl(m_stream, d); } - void pack_long(long d) { pack_long_impl(m_stream, d); } - void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_unsigned_long(unsigned long d) { pack_unsigned_long_impl(m_stream, d); } - void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } - void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } - void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } - void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } - void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } - void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } - void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } - void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } - void pack_float(float d) { pack_float_impl(m_stream, d); } - void pack_double(double d) { pack_double_impl(m_stream, d); } - void pack_nil() { pack_nil_impl(m_stream); } - void pack_true() { pack_true_impl(m_stream); } - void pack_false() { pack_false_impl(m_stream); } - void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } - void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } + void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } + void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } + void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } + void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } + void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } + void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } + void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } + + void pack_short(int d) { pack_short_impl(m_stream, d); } + void pack_int(int d) { pack_int_impl(m_stream, d); } + void pack_long(long d) { pack_long_impl(m_stream, d); } + void pack_long_long(long long d) { pack_long_long_impl(m_stream, d); } + void pack_unsigned_short(unsigned short d) { pack_unsigned_short_impl(m_stream, d); } + void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } + void pack_unsigned_long_long(unsigned long long d) { pack_unsigned_long_long_impl(m_stream, d); } + + void pack_float(float d) { pack_float_impl(m_stream, d); } + void pack_double(double d) { pack_double_impl(m_stream, d); } + + void pack_nil() { pack_nil_impl(m_stream); } + void pack_true() { pack_true_impl(m_stream); } + void pack_false() { pack_false_impl(m_stream); } + + void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } + + void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + void pack_raw(size_t l) { pack_raw_impl(m_stream, l); } void pack_raw_body(const char* b, size_t l) { pack_raw_body_impl(m_stream, b, l); } private: - static void pack_int_impl(Stream& x, int d); - static void pack_long_impl(Stream& x, long d); - static void pack_unsigned_int_impl(Stream& x, unsigned int d); - static void pack_unsigned_long_impl(Stream& x, unsigned long d); static void pack_uint8_impl(Stream& x, uint8_t d); static void pack_uint16_impl(Stream& x, uint16_t d); static void pack_uint32_impl(Stream& x, uint32_t d); @@ -65,13 +72,27 @@ private: static void pack_int16_impl(Stream& x, int16_t d); static void pack_int32_impl(Stream& x, int32_t d); static void pack_int64_impl(Stream& x, int64_t d); + + static void pack_short_impl(Stream& x, short d); + static void pack_int_impl(Stream& x, int d); + static void pack_long_impl(Stream& x, long d); + static void pack_long_long_impl(Stream& x, long long d); + static void pack_unsigned_short_impl(Stream& x, unsigned short d); + static void pack_unsigned_int_impl(Stream& x, unsigned int d); + static void pack_unsigned_long_impl(Stream& x, unsigned long d); + static void pack_unsigned_long_long_impl(Stream& x, unsigned long long d); + static void pack_float_impl(Stream& x, float d); static void pack_double_impl(Stream& x, double d); + static void pack_nil_impl(Stream& x); static void pack_true_impl(Stream& x); static void pack_false_impl(Stream& x); + static void pack_array_impl(Stream& x, unsigned int n); + static void pack_map_impl(Stream& x, unsigned int n); + static void pack_raw_impl(Stream& x, size_t l); static void pack_raw_body_impl(Stream& x, const void* b, size_t l); @@ -88,8 +109,15 @@ private: #define msgpack_pack_inline_func(name) \ template \ inline void packer::pack_ ## name ## _impl + +#define msgpack_pack_inline_func_cint(name) \ + template \ + inline void packer::pack_ ## name ## _impl + #define msgpack_pack_user Stream& + #define msgpack_pack_append_buffer append_buffer + #include "msgpack/pack_template.h" diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp index 3fd57b4..5cd10f6 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/integer.hpp @@ -63,102 +63,6 @@ namespace detail { return detail::convert_integer_sign::is_signed>::convert(o); } - - - template - struct pack_integer_size_sign; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) - { o.pack_int8(v); } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) - { o.pack_uint8(v); } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (int16_t)v <= (int16_t)std::numeric_limits::max() && - (int16_t)v >= (int16_t)std::numeric_limits::min()) - { o.pack_int8(v); } - else { o.pack_int16(v); } - } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (uint16_t)v <= (uint16_t)std::numeric_limits::max()) - { o.pack_uint8(v); } - else { o.pack_uint16(v); } - } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (int32_t)v <= (int32_t)std::numeric_limits::max() && - (int32_t)v >= (int32_t)std::numeric_limits::min()) - { o.pack_int8(v); } - else if( (int32_t)v <= (int32_t)std::numeric_limits::max() && - (int32_t)v >= (int32_t)std::numeric_limits::min()) - { o.pack_int16(v); } - else { o.pack_int32(v); } - } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) - { o.pack_uint8(v); } - else if( (uint32_t)v <= (uint32_t)std::numeric_limits::max()) - { o.pack_uint16(v); } - else { o.pack_uint32(v); } - } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (int64_t)v <= (int64_t)std::numeric_limits::max() && - (int64_t)v >= (int64_t)std::numeric_limits::min()) - { o.pack_int8(v); } - else if( (int64_t)v <= (int64_t)std::numeric_limits::max() && - (int64_t)v >= (int64_t)std::numeric_limits::min()) - { o.pack_int16(v); } - else if( (int64_t)v <= (int64_t)std::numeric_limits::max() && - (int64_t)v >= (int64_t)std::numeric_limits::min()) - { o.pack_int32(v); } - else { o.pack_int64(v); } - } - }; - - template - struct pack_integer_size_sign { - static inline void pack(packer& o, T v) { - if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) - { o.pack_uint8(v); } - else if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) - { o.pack_uint16(v); } - else if( (uint64_t)v <= (uint64_t)std::numeric_limits::max()) - { o.pack_uint32(v); } - else { o.pack_uint64(v); } - } - }; - - - template - static inline void pack_integer(T v, packer& o) - { - pack_integer_size_sign::is_signed>::pack(o, v); - } - } // namespace detail } // namespace type @@ -197,44 +101,44 @@ inline unsigned long long& operator>> (object o, unsigned long long& v) template inline packer& operator<< (packer& o, const signed char& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_int8(v); return o; } template inline packer& operator<< (packer& o, const signed short& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_short(v); return o; } template inline packer& operator<< (packer& o, const signed int& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_int(v); return o; } template inline packer& operator<< (packer& o, const signed long& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_long(v); return o; } template inline packer& operator<< (packer& o, const signed long long& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_long_long(v); return o; } template inline packer& operator<< (packer& o, const unsigned char& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_uint8(v); return o; } template inline packer& operator<< (packer& o, const unsigned short& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_unsigned_short(v); return o; } template inline packer& operator<< (packer& o, const unsigned int& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_unsigned_int(v); return o; } template inline packer& operator<< (packer& o, const unsigned long& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_unsigned_long(v); return o; } template inline packer& operator<< (packer& o, const unsigned long long& v) - { type::detail::pack_integer(v, o); return o; } + { o.pack_unsigned_long_long(v); return o; } } // namespace msgpack diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h new file mode 100644 index 0000000..6512d77 --- /dev/null +++ b/msgpack/pack_define.h @@ -0,0 +1,26 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_DEFINE_H__ +#define MSGPACK_PACK_DEFINE_H__ + +#include +#include +#include + +#endif /* msgpack/pack_define.h */ + diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 94fbfd4..dafb0b1 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -50,7 +50,6 @@ #endif - #ifndef msgpack_pack_inline_func #error msgpack_pack_inline_func template is not defined #endif @@ -68,7 +67,127 @@ * Integer */ -#define msgpack_pack_compress_int32(x, d) \ +#define msgpack_pack_real_uint8(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ +} while(0) + +#define msgpack_pack_real_uint16(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ +} while(0) + +#define msgpack_pack_real_uint32(x, d) \ +do { \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_uint64(x, d) \ +do { \ + if(d < (1ULL<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int8(x, d) \ +do { \ + if(d < -(1<<5)) { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } \ +} while(0) + +#define msgpack_pack_real_int16(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int32(x, d) \ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ @@ -104,31 +223,7 @@ do { \ } \ } while(0) -#define msgpack_pack_compress_uint32(x, d) \ -do { \ - if(d < (1<<8)) { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); \ - } else { \ - /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else { \ - if(d < (1<<16)) { \ - /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else { \ - /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ - msgpack_pack_append_buffer(x, buf, 5); \ - } \ - } \ -} while(0) - -#define msgpack_pack_compress_int64(x, d) \ +#define msgpack_pack_real_int64(x, d) \ do { \ if(d < -(1LL<<5)) { \ if(d < -(1LL<<15)) { \ @@ -180,133 +275,276 @@ do { \ } \ } while(0) -#define msgpack_pack_compress_uint64(x, d) \ -do { \ - if(d < (1ULL<<8)) { \ - if(d < (1<<7)) { \ - /* fixnum */ \ - msgpack_pack_append_buffer(x, (unsigned char*)&d, 1); \ - } else { \ - /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ - msgpack_pack_append_buffer(x, buf, 2); \ - } \ - } else { \ - if(d < (1ULL<<16)) { \ - /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ - msgpack_pack_append_buffer(x, buf, 3); \ - } else if(d < (1ULL<<32)) { \ - /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ - msgpack_pack_append_buffer(x, buf, 5); \ - } else { \ - /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ - msgpack_pack_append_buffer(x, buf, 9); \ - } \ - } \ -} while(0) -msgpack_pack_inline_func(int)(msgpack_pack_user x, int d) +#ifdef msgpack_pack_inline_func_fastint + +msgpack_pack_inline_func_fastint(uint8)(msgpack_pack_user x, uint8_t d) { -#if SIZEOF_INT == 8 - msgpack_pack_compress_int64(x, d); -#else - msgpack_pack_compress_int32(x, d); -#endif + const unsigned char buf[2] = {0xcc, d}; + msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func(unsigned_int)(msgpack_pack_user x, unsigned int d) -{ -#if SIZEOF_INT == 8 - msgpack_pack_compress_uint64(x, d); -#else - msgpack_pack_compress_uint32(x, d); -#endif -} - -msgpack_pack_inline_func(long)(msgpack_pack_user x, long d) -{ -#if SIZEOF_LONG == 8 - msgpack_pack_compress_int64(x, d); -#else - msgpack_pack_compress_int32(x, d); -#endif -} - -msgpack_pack_inline_func(unsigned_long)(msgpack_pack_user x, unsigned long d) -{ -#if SIZEOF_LONG == 8 - msgpack_pack_compress_uint64(x, d); -#else - msgpack_pack_compress_uint32(x, d); -#endif -} - -#undef msgpack_pack_compress_int32 -#undef msgpack_pack_compress_uint32 -#undef msgpack_pack_compress_int64 -#undef msgpack_pack_compress_uint64 - -msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) -{ - if(d < 128) { - msgpack_pack_append_buffer(x, &d, 1); - } else { - const unsigned char buf[2] = {0xcc, d}; - msgpack_pack_append_buffer(x, buf, 2); - } -} - -msgpack_pack_inline_func(uint16)(msgpack_pack_user x, uint16_t d) +msgpack_pack_inline_func_fastint(uint16)(msgpack_pack_user x, uint16_t d) { const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func(uint32)(msgpack_pack_user x, uint32_t d) +msgpack_pack_inline_func_fastint(uint32)(msgpack_pack_user x, uint32_t d) { const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(uint64)(msgpack_pack_user x, uint64_t d) +msgpack_pack_inline_func_fastint(uint64)(msgpack_pack_user x, uint64_t d) { const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } -msgpack_pack_inline_func(int8)(msgpack_pack_user x, int8_t d) +msgpack_pack_inline_func_fastint(int8)(msgpack_pack_user x, int8_t d) { - if(d > 0) { - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); - } else if(d >= -32) { - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); - } else { - const unsigned char buf[2] = {0xd0, d}; - msgpack_pack_append_buffer(x, buf, 2); - } + const unsigned char buf[2] = {0xd0, d}; + msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func(int16)(msgpack_pack_user x, int16_t d) +msgpack_pack_inline_func_fastint(int16)(msgpack_pack_user x, int16_t d) { const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func(int32)(msgpack_pack_user x, int32_t d) +msgpack_pack_inline_func_fastint(int32)(msgpack_pack_user x, int32_t d) { const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) +msgpack_pack_inline_func_fastint(int64)(msgpack_pack_user x, int64_t d) { const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } +#undef msgpack_pack_inline_func_fastint +#endif + + +msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) +{ + msgpack_pack_real_uint8(x, d); +} + +msgpack_pack_inline_func(uint16)(msgpack_pack_user x, uint16_t d) +{ + msgpack_pack_real_uint16(x, d); +} + +msgpack_pack_inline_func(uint32)(msgpack_pack_user x, uint32_t d) +{ + msgpack_pack_real_uint32(x, d); +} + +msgpack_pack_inline_func(uint64)(msgpack_pack_user x, uint64_t d) +{ + msgpack_pack_real_uint64(x, d); +} + +msgpack_pack_inline_func(int8)(msgpack_pack_user x, int8_t d) +{ + msgpack_pack_real_int8(x, d); +} + +msgpack_pack_inline_func(int16)(msgpack_pack_user x, int16_t d) +{ + msgpack_pack_real_int16(x, d); +} + +msgpack_pack_inline_func(int32)(msgpack_pack_user x, int32_t d) +{ + msgpack_pack_real_int32(x, d); +} + +msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) +{ + msgpack_pack_real_int64(x, d); +} + + +#ifdef msgpack_pack_inline_func_cint + +msgpack_pack_inline_func_cint(short)(msgpack_pack_user x, short d) +{ +#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) +#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(short) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(short) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(int)(msgpack_pack_user x, int d) +{ +#if defined(SIZEOF_INT) || defined(INT_MAX) +#if SIZEOF_INT == 2 || INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(int) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(int) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(long)(msgpack_pack_user x, long d) +{ +#if defined(SIZEOF_LONG) || defined(LONG_MAX) +#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(long_long)(msgpack_pack_user x, long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(unsigned_short)(msgpack_pack_user x, unsigned short d) +{ +#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) +#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned short) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned short) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(unsigned_int)(msgpack_pack_user x, unsigned int d) +{ +#if defined(SIZEOF_INT) || defined(UINT_MAX) +#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(unsigned_long)(msgpack_pack_user x, unsigned long d) +{ +#if defined(SIZEOF_LONG) || defined(ULONG_MAX) +#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned long long) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned long long) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +#undef msgpack_pack_inline_func_cint +#endif + + /* * Float @@ -430,3 +668,12 @@ msgpack_pack_inline_func(raw_body)(msgpack_pack_user x, const void* b, size_t l) #undef STORE_BE32 #undef STORE_BE64 +#undef msgpack_pack_real_uint8 +#undef msgpack_pack_real_uint16 +#undef msgpack_pack_real_uint32 +#undef msgpack_pack_real_uint64 +#undef msgpack_pack_real_int8 +#undef msgpack_pack_real_int16 +#undef msgpack_pack_real_int32 +#undef msgpack_pack_real_int64 + diff --git a/ruby/gem/Manifest.txt b/ruby/gem/Manifest.txt index 388135e..17e2b78 100644 --- a/ruby/gem/Manifest.txt +++ b/ruby/gem/Manifest.txt @@ -10,6 +10,7 @@ ext/pack.h ext/rbinit.c ext/unpack.c ext/unpack.h +msgpack/pack_define.h msgpack/pack_template.h msgpack/unpack_define.h msgpack/unpack_template.h diff --git a/ruby/gengem.sh b/ruby/gengem.sh index 000c708..de9f4a5 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -7,6 +7,7 @@ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ #cp ../README gem/README.txt +cp ../msgpack/pack_define.h gem/msgpack/ cp ../msgpack/pack_template.h gem/msgpack/ cp ../msgpack/unpack_define.h gem/msgpack/ cp ../msgpack/unpack_template.h gem/msgpack/ diff --git a/ruby/pack.c b/ruby/pack.c index 5262024..c399c25 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -16,41 +16,19 @@ * limitations under the License. */ #include "ruby.h" -#include -#include +#include "msgpack/pack_define.h" #define msgpack_pack_inline_func(name) \ static inline void msgpack_pack_##name +#define msgpack_pack_inline_func_cint(name) \ + static inline void msgpack_pack_##name + #define msgpack_pack_user VALUE #define msgpack_pack_append_buffer(user, buf, len) \ rb_str_buf_cat(user, (const void*)buf, len) -/* -static void msgpack_pack_int(VALUE x, int d); -static void msgpack_pack_unsigned_int(VALUE x, unsigned int d); -static void msgpack_pack_long(VALUE x, long d); -static void msgpack_pack_unsigned_long(VALUE x, unsigned long d); -static void msgpack_pack_uint8(VALUE x, uint8_t d); -static void msgpack_pack_uint16(VALUE x, uint16_t d); -static void msgpack_pack_uint32(VALUE x, uint32_t d); -static void msgpack_pack_uint64(VALUE x, uint64_t d); -static void msgpack_pack_int8(VALUE x, int8_t d); -static void msgpack_pack_int16(VALUE x, int16_t d); -static void msgpack_pack_int32(VALUE x, int32_t d); -static void msgpack_pack_int64(VALUE x, int64_t d); -static void msgpack_pack_float(VALUE x, float d); -static void msgpack_pack_double(VALUE x, double d); -static void msgpack_pack_nil(VALUE x); -static void msgpack_pack_true(VALUE x); -static void msgpack_pack_false(VALUE x); -static void msgpack_pack_array(VALUE x, unsigned int n); -static void msgpack_pack_map(VALUE x, unsigned int n); -static void msgpack_pack_raw(VALUE x, size_t l); -static void msgpack_pack_raw_body(VALUE x, const void* b, size_t l); -*/ - #include "msgpack/pack_template.h" From c93d45371b2b2a085fa2fe84d6ceb79decf778d2 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0044/1648] c++ binding: fix missing packer::pack_unsigned_long git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@91 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/pack.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index e860f4c..c07bcc3 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -47,6 +47,7 @@ public: void pack_long_long(long long d) { pack_long_long_impl(m_stream, d); } void pack_unsigned_short(unsigned short d) { pack_unsigned_short_impl(m_stream, d); } void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } + void pack_unsigned_long(unsigned long d) { pack_unsigned_long_impl(m_stream, d); } void pack_unsigned_long_long(unsigned long long d) { pack_unsigned_long_long_impl(m_stream, d); } void pack_float(float d) { pack_float_impl(m_stream, d); } From e582fa34c7f1246d27cce5a2333e0f954cbed251 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:00 +0000 Subject: [PATCH 0045/1648] c++ binding: improve msgpack::zone, zero-copy stream deserializer git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@92 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/Makefile.am | 13 ++- cpp/unpack.cpp | 233 +++++++++++++++++++++++++++++++---------------- cpp/unpack.hpp | 8 +- cpp/zone.cpp | 82 ++++++++++++++--- cpp/zone.hpp | 86 ----------------- cpp/zone.hpp.erb | 103 +++++++++++++++++++++ 6 files changed, 336 insertions(+), 189 deletions(-) delete mode 100644 cpp/zone.hpp create mode 100644 cpp/zone.hpp.erb diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 19c6007..45c75e0 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -26,14 +26,21 @@ noinst_HEADERS = \ # FIXME object.lo: msgpack/type/tuple.hpp -unpack.lo: msgpack/type/tuple.hpp -zone.lo: msgpack/type/tuple.hpp +unpack.lo: msgpack/type/tuple.hpp msgpack/zone.hpp +zone.lo: msgpack/type/tuple.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp mv $@.tmp $@ -MOSTLYCLEANFILES = msgpack/type/tuple.hpp +msgpack/zone.hpp: msgpack/zone.hpp.erb + $(ERB) $< > $@.tmp + mv $@.tmp $@ +MOSTLYCLEANFILES = \ + msgpack/type/tuple.hpp \ + msgpack/zone.hpp + +# FIXME libmsgpack_la_LDFLAGS = -version-info 0:0:0 diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 8edf407..f2ef330 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -22,6 +22,19 @@ namespace msgpack { +//namespace { +struct allocator { + zone* z; + bool referenced; + + inline object* malloc_object(size_t n) + { + return (object*)z->malloc(sizeof(object)*n); + } +}; +//} // noname namespace + + #define msgpack_unpack_struct(name) \ struct msgpack_unpacker_##name @@ -33,7 +46,7 @@ namespace msgpack { #define msgpack_unpack_object object -#define msgpack_unpack_user zone* +#define msgpack_unpack_user allocator struct msgpack_unpacker_context; @@ -46,92 +59,100 @@ static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, const char* data, size_t len, size_t* off); -static inline object msgpack_unpack_init(zone** z) +static inline object msgpack_unpack_init(allocator* a) { return object(); } -static inline object msgpack_unpack_uint8(zone** z, uint8_t d) +static inline object msgpack_unpack_uint8(allocator* a, uint8_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint16(zone** z, uint16_t d) +static inline object msgpack_unpack_uint16(allocator* a, uint16_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint32(zone** z, uint32_t d) +static inline object msgpack_unpack_uint32(allocator* a, uint32_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint64(zone** z, uint64_t d) +static inline object msgpack_unpack_uint64(allocator* a, uint64_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_int8(zone** z, int8_t d) +static inline object msgpack_unpack_int8(allocator* a, int8_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int16(zone** z, int16_t d) +static inline object msgpack_unpack_int16(allocator* a, int16_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int32(zone** z, int32_t d) +static inline object msgpack_unpack_int32(allocator* a, int32_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int64(zone** z, int64_t d) +static inline object msgpack_unpack_int64(allocator* a, int64_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_float(zone** z, float d) +static inline object msgpack_unpack_float(allocator* a, float d) { object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object msgpack_unpack_double(zone** z, double d) +static inline object msgpack_unpack_double(allocator* a, double d) { object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object msgpack_unpack_nil(zone** z) +static inline object msgpack_unpack_nil(allocator* a) { object o; o.type = type::NIL; return o; } -static inline object msgpack_unpack_true(zone** z) +static inline object msgpack_unpack_true(allocator* a) { object o; o.type = type::BOOLEAN; o.via.boolean = true; return o; } -static inline object msgpack_unpack_false(zone** z) +static inline object msgpack_unpack_false(allocator* a) { object o; o.type = type::BOOLEAN; o.via.boolean = false; return o; } -static inline object msgpack_unpack_array(zone** z, unsigned int n) +static inline object msgpack_unpack_array(allocator* a, unsigned int n) { object o; o.type = type::ARRAY; o.via.container.size = 0; - o.via.container.ptr = (*z)->malloc_container(n); + o.via.container.ptr = a->malloc_object(n); return o; } -static inline void msgpack_unpack_array_item(zone** z, object* c, object o) +static inline void msgpack_unpack_array_item(allocator* a, object* c, object o) { c->via.container.ptr[ c->via.container.size++ ] = o; } -static inline object msgpack_unpack_map(zone** z, unsigned int n) +static inline object msgpack_unpack_map(allocator* a, unsigned int n) { object o; o.type = type::MAP; o.via.container.size = 0; - o.via.container.ptr = (*z)->malloc_container(n*2); + o.via.container.ptr = a->malloc_object(n*2); return o; } -static inline void msgpack_unpack_map_item(zone** z, object* c, object k, object v) +static inline void msgpack_unpack_map_item(allocator* a, object* c, object k, object v) { c->via.container.ptr[ c->via.container.size ] = k; c->via.container.ptr[ c->via.container.size+1 ] = v; ++c->via.container.size; } -static inline object msgpack_unpack_raw(zone** z, const char* b, const char* p, unsigned int l) -{ object o; o.type = type::RAW; o.via.ref.ptr = p; o.via.ref.size = l; return o; } - +static inline object msgpack_unpack_raw(allocator* a, const char* b, const char* p, unsigned int l) +{ + object o; + o.type = type::RAW; + o.via.ref.ptr = p; + o.via.ref.size = l; + a->referenced = true; + return o; +} #include "msgpack/unpack_template.h" -struct unpacker::context { - context(zone* z) +namespace { +struct context { + context() { msgpack_unpacker_init(&m_ctx); - m_ctx.user = z; + allocator a = {NULL, false}; + m_ctx.user = a; } ~context() { } @@ -148,35 +169,64 @@ struct unpacker::context { void reset() { - zone* z = m_ctx.user; + zone* z = m_ctx.user.z; msgpack_unpacker_init(&m_ctx); - m_ctx.user = z; + allocator a = {z, false}; + m_ctx.user = a; } - void reset(zone* z) + void set_zone(zone* z) { - msgpack_unpacker_init(&m_ctx); - m_ctx.user = z; + m_ctx.user.z = z; } - zone* user() + bool is_referenced() const { - return m_ctx.user; - } - - void user(zone* z) - { - m_ctx.user = z; + return m_ctx.user.referenced; } private: msgpack_unpacker_context m_ctx; + zone* m_zone; private: - context(); context(const context&); }; +context* as_ctx(void* m) +{ + return reinterpret_cast(m); +} + + +static const size_t COUNTER_SIZE = sizeof(unsigned int); + +static inline void init_count(void* buffer) +{ + *(volatile unsigned int*)buffer = 1; +} + +static inline void decl_count(void* buffer) +{ + //if(--*(unsigned int*)buffer == 0) { + if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + free(buffer); + } +} + +static inline void incr_count(void* buffer) +{ + //++*(unsigned int*)buffer; + __sync_add_and_fetch((unsigned int*)buffer, 1); +} + +static inline unsigned int get_count(void* buffer) +{ + return *(volatile unsigned int*)buffer; +} + +} // noname namespace + unpacker::unpacker(size_t initial_buffer_size) : m_buffer(NULL), @@ -184,45 +234,77 @@ unpacker::unpacker(size_t initial_buffer_size) : m_free(0), m_off(0), m_zone(new zone()), - m_ctx(new context(&*m_zone)), + m_ctx(new context()), m_initial_buffer_size(initial_buffer_size) -{ } +{ + if(m_initial_buffer_size < COUNTER_SIZE) { + m_initial_buffer_size = COUNTER_SIZE; + } + + as_ctx(m_ctx)->set_zone(m_zone.get()); + + m_buffer = (char*)::malloc(m_initial_buffer_size); + if(!m_buffer) { throw std::bad_alloc(); } + init_count(m_buffer); + + m_used = COUNTER_SIZE; + m_free = m_initial_buffer_size - m_used; + m_off = COUNTER_SIZE; +} unpacker::~unpacker() { - delete m_ctx; + delete as_ctx(m_ctx); + decl_count(m_buffer); } - void unpacker::expand_buffer(size_t len) { - if(m_off == 0) { - size_t next_size; - if(m_used != 0) { next_size = (m_used + m_free) * 2; } - else { next_size = m_initial_buffer_size; } + if(m_used == m_off && get_count(m_buffer) == 1 && + !as_ctx(m_ctx)->is_referenced()) { + // rewind buffer + m_free += m_used - COUNTER_SIZE; + m_used = COUNTER_SIZE; + m_off = COUNTER_SIZE; + if(m_free >= len) { return; } + } + + if(m_off == COUNTER_SIZE) { + size_t next_size = (m_used + m_free) * 2; while(next_size < len + m_used) { next_size *= 2; } - m_buffer = m_zone->realloc(m_buffer, next_size); + char* tmp = (char*)::realloc(m_buffer, next_size); + if(!tmp) { throw std::bad_alloc(); } + + m_buffer = tmp; m_free = next_size - m_used; } else { - size_t next_size = m_initial_buffer_size; - while(next_size < len + m_used - m_off) { next_size *= 2; } + size_t next_size = m_initial_buffer_size; // include COUNTER_SIZE + size_t not_parsed = m_used - m_off; + while(next_size < len + not_parsed + COUNTER_SIZE) { next_size *= 2; } - char* tmp = m_zone->malloc(next_size); - memcpy(tmp, m_buffer+m_off, m_used-m_off); + char* tmp = (char*)::malloc(next_size); + if(!tmp) { throw std::bad_alloc(); } + init_count(tmp); + + try { + m_zone->push_finalizer(decl_count, m_buffer); + } catch (...) { free(tmp); throw; } + + memcpy(tmp+COUNTER_SIZE, m_buffer+m_off, not_parsed); m_buffer = tmp; - m_used = m_used - m_off; + m_used = not_parsed + COUNTER_SIZE; m_free = next_size - m_used; - m_off = 0; + m_off = COUNTER_SIZE; } } bool unpacker::execute() { - int ret = m_ctx->execute(m_buffer, m_used, &m_off); + int ret = as_ctx(m_ctx)->execute(m_buffer, m_used, &m_off); if(ret < 0) { throw unpack_error("parse error"); } else if(ret == 0) { @@ -234,56 +316,45 @@ bool unpacker::execute() zone* unpacker::release_zone() { + m_zone->push_finalizer(decl_count, m_buffer); + incr_count(m_buffer); + + //std::auto_ptr old(new zone()); + //m_zone.swap(old); zone* n = new zone(); std::auto_ptr old(m_zone.release()); m_zone.reset(n); - //std::auto_ptr old(new zone()); - //m_zone.swap(old); + as_ctx(m_ctx)->set_zone(m_zone.get()); - // move all bytes in m_buffer to new buffer from the new zone - if(m_used <= m_off) { - m_buffer = NULL; - m_used = 0; - m_free = 0; - m_off = 0; - } else { - try { - expand_buffer(0); - } catch (...) { - // m_zone.swap(old); - zone* tmp = old.release(); - old.reset(m_zone.release()); - m_zone.reset(tmp); - throw; - } - } - m_ctx->user(m_zone.get()); return old.release(); } object unpacker::data() { - return m_ctx->data(); + return as_ctx(m_ctx)->data(); } void unpacker::reset() { - if(m_off != 0) { delete release_zone(); } - m_ctx->reset(); + if(!m_zone->empty()) { delete release_zone(); } + as_ctx(m_ctx)->reset(); } object unpacker::unpack(const char* data, size_t len, zone& z, size_t* off) { - context ctx(&z); + context ctx; + ctx.set_zone(&z); if(off) { - int ret = ctx.execute(data, len, off); + size_t noff = *off; + int ret = ctx.execute(data, len, &noff); if(ret < 0) { throw unpack_error("parse error"); } else if(ret == 0) { throw unpack_error("insufficient bytes"); } + *off = noff; } else { size_t noff = 0; int ret = ctx.execute(data, len, &noff); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index ac20de3..cba1963 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -24,7 +24,8 @@ #include #ifndef MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE -#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 8*1024 +#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 16 +//#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 8*1024 #endif namespace msgpack { @@ -133,10 +134,9 @@ private: std::auto_ptr m_zone; - struct context; - context* m_ctx; + void* m_ctx; - const size_t m_initial_buffer_size; + size_t m_initial_buffer_size; private: void expand_buffer(size_t len); diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 527cc9c..7c5a1c9 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -16,31 +16,83 @@ // limitations under the License. // #include "msgpack/zone.hpp" +#include namespace msgpack { -void zone::clear() +zone::zone(size_t chunk_size) : + m_chunk_size(chunk_size) { - for(std::vector::iterator it(m_ptrs.begin()), it_end(m_ptrs.end()); - it != it_end; ++it) { - free(*it); - } - m_ptrs.clear(); + chunk dummy = {0, NULL, NULL}; + m_chunk_array.push_back(dummy); } -char* zone::realloc_real(char* ptr, size_t count) +zone::~zone() { - for(std::vector::reverse_iterator it(m_ptrs.rbegin()), it_end(m_ptrs.rend()); - it != it_end; ++it) { - if(*it == ptr) { - char* tmp = (char*)::realloc(ptr, count); - if(!tmp) { throw std::bad_alloc(); } - *it = tmp; - return tmp; + clear(); +} + +namespace { + template + struct zone_finalize { + void operator() (Private& f) { + (*f.func)(f.obj); } + }; + + template + struct zone_free { + void operator() (Private& c) { + ::free(c.alloc); + } + }; +} + +void zone::clear() +{ + std::for_each(m_finalizers.rbegin(), m_finalizers.rend(), + zone_finalize()); + m_finalizers.clear(); + + std::for_each(m_chunk_array.begin(), m_chunk_array.end(), + zone_free()); + m_chunk_array.resize(1); + m_chunk_array[0].ptr = NULL; + m_chunk_array[0].free = 0; +} + +bool zone::empty() const +{ + return m_chunk_array.back().alloc == NULL && + m_finalizers.empty(); +} + +void* zone::malloc(size_t size) +{ + if(m_chunk_array.back().free > size) { + char* p = (char*)m_chunk_array.back().ptr; + m_chunk_array.back().ptr = p + size; + m_chunk_array.back().free -= size; + return p; } - throw std::bad_alloc(); + + size_t sz = m_chunk_size; + while(sz < size) { sz *= 2; } + + chunk dummy = {0, NULL, NULL}; + m_chunk_array.push_back(dummy); + + char* p = (char*)::malloc(sz); + if(!p) { + m_chunk_array.pop_back(); + throw std::bad_alloc(); + } + + m_chunk_array.back().free = sz - size; + m_chunk_array.back().ptr = p + size; + m_chunk_array.back().alloc = p; + return p; } diff --git a/cpp/zone.hpp b/cpp/zone.hpp deleted file mode 100644 index e7e73e1..0000000 --- a/cpp/zone.hpp +++ /dev/null @@ -1,86 +0,0 @@ -// -// MessagePack for C++ memory pool -// -// Copyright (C) 2008 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#ifndef MSGPACK_ZONE_HPP__ -#define MSGPACK_ZONE_HPP__ - -#include "msgpack/object.hpp" -#include -#include - -namespace msgpack { - - -class zone { -public: - zone(); - ~zone(); - -public: - char* malloc(size_t count); - char* realloc(char* ptr, size_t count); - object* malloc_container(size_t count); - - void clear(); - -private: - std::vector m_ptrs; - -private: - char* realloc_real(char* ptr, size_t count); - -private: - zone(const zone&); -}; - - -inline zone::zone() { } - -inline zone::~zone() { clear(); } - -inline char* zone::malloc(size_t count) -{ - char* ptr = (char*)::malloc(count); - if(!ptr) { throw std::bad_alloc(); } - try { - m_ptrs.push_back(ptr); - } catch (...) { - free(ptr); - throw; - } - return ptr; -} - -inline char* zone::realloc(char* ptr, size_t count) -{ - if(ptr == NULL) { - return zone::malloc(count); - } else { - return realloc_real(ptr, count); - } -} - -inline object* zone::malloc_container(size_t count) -{ - return (object*)zone::malloc(sizeof(object)*count); -} - - -} // namespace msgpack - -#endif /* msgpack/zone.hpp */ - diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb new file mode 100644 index 0000000..9a8c81a --- /dev/null +++ b/cpp/zone.hpp.erb @@ -0,0 +1,103 @@ +// +// MessagePack for C++ memory pool +// +// Copyright (C) 2008 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_ZONE_HPP__ +#define MSGPACK_ZONE_HPP__ + +#include "msgpack/object.hpp" +#include +#include + +#ifndef MSGPACK_ZONE_CHUNK_SIZE +#define MSGPACK_ZONE_CHUNK_SIZE 2048 +#endif +<% GENERATION_LIMIT = 15 %> +namespace msgpack { + + +class zone { +public: + zone(size_t chunk_size = MSGPACK_ZONE_CHUNK_SIZE); + ~zone(); + +public: + void* malloc(size_t size); + + void push_finalizer(void (*func)(void*), void* obj); + + void clear(); + + bool empty() const; + + <%0.upto(GENERATION_LIMIT) {|i|%> + template , typename A<%=j%><%}%>> + T* allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>); + <%}%> + +private: + struct chunk { + size_t free; + void* ptr; + void* alloc; + }; + + std::vector m_chunk_array; + + struct finalizer { + void (*func)(void*); + void* obj; + }; + + std::vector m_finalizers; + + template + static void object_destructor(void* obj); + + size_t m_chunk_size; + +private: + zone(const zone&); +}; + + +inline void zone::push_finalizer(void (*func)(void*), void* obj) +{ + finalizer f = {func, obj}; + m_finalizers.push_back(f); +} + +template +void zone::object_destructor(void* obj) +{ + reinterpret_cast(obj)->~T(); +} + +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) +{ + void* x = malloc(sizeof(T)); + push_finalizer(&zone::object_destructor, x); + try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } + catch (...) { m_finalizers.pop_back(); throw; } +} +<%}%> + +} // namespace msgpack + +#endif /* msgpack/zone.hpp */ + From cbf2be8db4f6bc18e1ec3002534da06774561a4d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0046/1648] c++ binding: remove unpacker::parsed_size() API git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@93 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/unpack.hpp | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index cba1963..fe20fc7 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -109,23 +109,19 @@ public: public: // These functions are usable when non-MessagePack message follows after // MessagePack message. - // Note that there are no parsed buffer when execute() returned true. - /*! get address of buffer that is not parsed */ + /*! get address of the buffer that is not parsed */ char* nonparsed_buffer(); size_t nonparsed_size() const; - /*! get the number of bytes that is already parsed */ - size_t parsed_size() const; + /*! skip specified size of non-parsed buffer, leaving the buffer */ + // Note that the `len' argument must be smaller than nonparsed_size() + void skip_nonparsed_buffer(size_t len); /*! remove unparsed buffer from unpacker */ // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); - /*! skip specified size of non-parsed buffer, leaving the buffer */ - // Note the size must be smaller than nonparsed_size() - void skip_nonparsed_buffer(size_t len); - private: char* m_buffer; size_t m_used; @@ -174,15 +170,12 @@ inline char* unpacker::nonparsed_buffer() inline size_t unpacker::nonparsed_size() const { return m_used - m_off; } -inline size_t unpacker::parsed_size() const - { return m_off; } +inline void unpacker::skip_nonparsed_buffer(size_t len) + { m_off += len; } inline void unpacker::remove_nonparsed_buffer() { m_used = m_off; } -inline void unpacker::skip_nonparsed_buffer(size_t len) - { m_off += len; } - inline object unpack(const char* data, size_t len, zone& z, size_t* off = NULL) { From 823add403e5f0d905fc0003023c2b270adb22897 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0047/1648] merge 0.2.2 git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@94 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- configure.in | 2 +- msgpack/unpack_define.h | 4 +++- ruby/gem/lib/msgpack/version.rb | 2 +- ruby/msgpack.gemspec | 11 +++++++++++ ruby/unpack.c | 13 +++++++++++-- 5 files changed, 27 insertions(+), 5 deletions(-) create mode 100755 ruby/msgpack.gemspec diff --git a/configure.in b/configure.in index 63762b0..3a67081 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ AC_INIT(msgpack/unpack_template.h) -AM_INIT_AUTOMAKE(msgpack, 0.2.1) +AM_INIT_AUTOMAKE(msgpack, 0.2.2) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 1d9db19..f639841 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -22,8 +22,10 @@ #include #include #include -#include #include +#ifndef __WIN32__ +#include +#endif #ifdef __cplusplus extern "C" { diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb index 229c746..488e0b2 100644 --- a/ruby/gem/lib/msgpack/version.rb +++ b/ruby/gem/lib/msgpack/version.rb @@ -2,7 +2,7 @@ module MessagePack module VERSION #:nodoc: MAJOR = 0 MINOR = 2 - TINY = 1 + TINY = 2 STRING = [MAJOR, MINOR, TINY].join('.') end diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec new file mode 100755 index 0000000..db2fcd5 --- /dev/null +++ b/ruby/msgpack.gemspec @@ -0,0 +1,11 @@ +Gem::Specification.new do |s| + s.platform = Gem::Platform::CURRENT + s.name = "msgpack" + s.version = "0.2.2" + s.summary = "MessagePack" + s.author = "FURUHASHI Sadayuki" + s.email = "frsyuki _at_ users.sourceforge.jp" + s.homepage = "https://launchpad.net/msgpack/" + s.require_paths = ["lib", "ext"] + s.files = ["lib/**/*", "ext/**/*"].map {|g| Dir.glob(g) }.flatten +end diff --git a/ruby/unpack.c b/ruby/unpack.c index 8ab425c..df72246 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -21,6 +21,7 @@ typedef struct { int finished; + VALUE origstr; } msgpack_unpack_context; @@ -104,7 +105,7 @@ static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE* { rb_hash_aset(*c, k, v); } static inline VALUE template_callback_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) -{ return rb_str_new(p, l); } +{ return l == 0 ? rb_str_new(0,0) : rb_str_substr(x->origstr, p - b, l); } #include "msgpack/unpack_template.h" @@ -152,8 +153,9 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); - mp->user.finished = 0; msgpack_unpacker_init(mp); + msgpack_unpack_context ctx = {0, Qnil}; + mp->user = ctx; return self; } @@ -179,7 +181,9 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) rb_raise(eUnpackError, "offset is bigger than data buffer size."); } + mp->user.origstr = data; ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); + mp->user.origstr = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -239,7 +243,9 @@ static VALUE MessagePack_unpack_impl(VALUE args) long dlen = RSTRING_LEN(data); int ret; + mp->user.origstr = data; ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); + mp->user.origstr = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -268,6 +274,9 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) CHECK_STRING_TYPE(data); msgpack_unpacker mp; msgpack_unpacker_init(&mp); + msgpack_unpack_context ctx = {0, Qnil}; + mp.user = ctx; + rb_gc_disable(); VALUE args[2] = {(VALUE)&mp, data}; VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, From e893dde57e60e8e75b30b0e6a539ec233e112a4a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0048/1648] C++ binding: efficient serializing interface git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@95 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/object.hpp | 11 ++++++----- cpp/test.cpp | 4 +++- cpp/unpack.hpp | 3 +-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 5f77d3e..1f0dcdf 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -135,14 +135,14 @@ inline void pack_copy(packer& o, T v) template inline T& operator>> (object o, T& v) { - v.msgpack_unpack(o); + v.msgpack_unpack(o.convert()); return v; } template inline packer& operator<< (packer& o, const T& v) { - o << v.msgpack_pack(); + v.msgpack_pack(o); return o; } @@ -156,14 +156,15 @@ public: define() {} define(msgpack_type v) : msgpack_type(v) {} - msgpack_type msgpack_pack() const + template + void msgpack_pack(Packer& o) const { - return *this; + o << static_cast(*this); } void msgpack_unpack(object o) { - convert(static_cast(*this), o); + o >> static_cast(*this); } }; diff --git a/cpp/test.cpp b/cpp/test.cpp index 02d461c..dd0b1fd 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -50,6 +50,7 @@ int main(void) { checker c; +#if 0 { // SimpleValue const char d[] = { 0x93, 0xc0, 0xc2, 0xc3, @@ -116,9 +117,10 @@ int main(void) ) ); } +#endif - static const unsigned TASK_ARRAY = 100; + static const unsigned TASK_ARRAY = 1000; static const unsigned TASK_REPEAT = 10; std::vector task; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index fe20fc7..293a5d3 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -24,8 +24,7 @@ #include #ifndef MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE -#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 16 -//#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE 8*1024 +#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE (32*1024) #endif namespace msgpack { From 2c7cdd5f40832b94c20ea578729ee00e3660d180 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0049/1648] Ruby binding: add MessagePack::unpack_limit, MessagePack::Unpacker#execute_limit git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@96 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- configure.in | 2 +- ruby/gem/lib/msgpack/version.rb | 4 ++-- ruby/unpack.c | 30 ++++++++++++++++++++++-------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/configure.in b/configure.in index 3a67081..a464fb9 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ AC_INIT(msgpack/unpack_template.h) -AM_INIT_AUTOMAKE(msgpack, 0.2.2) +AM_INIT_AUTOMAKE(msgpack, 0.3.0) AC_CONFIG_HEADER(config.h) AC_PROG_LIBTOOL diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb index 488e0b2..433e6fc 100644 --- a/ruby/gem/lib/msgpack/version.rb +++ b/ruby/gem/lib/msgpack/version.rb @@ -1,8 +1,8 @@ module MessagePack module VERSION #:nodoc: MAJOR = 0 - MINOR = 2 - TINY = 2 + MINOR = 3 + TINY = 0 STRING = [MAJOR, MINOR, TINY].join('.') end diff --git a/ruby/unpack.c b/ruby/unpack.c index df72246..2920240 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -169,12 +169,11 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) { VALUE self = ((VALUE*)args)[0]; VALUE data = ((VALUE*)args)[1]; - VALUE off = ((VALUE*)args)[2]; UNPACKER(self, mp); - size_t from = NUM2UINT(off); + size_t from = NUM2UINT(((VALUE*)args)[2]); char* dptr = RSTRING_PTR(data); - long dlen = RSTRING_LEN(data); + long dlen = FIX2LONG(((VALUE*)args)[3]); int ret; if(from >= dlen) { @@ -206,17 +205,24 @@ static VALUE MessagePack_Unpacker_execute_rescue(VALUE nouse) #endif } -static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) +static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, + VALUE off, VALUE limit) { // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ rb_gc_disable(); - VALUE args[3] = {self, data, off}; + VALUE args[4] = {self, data, off, limit}; VALUE ret = rb_rescue(MessagePack_Unpacker_execute_impl, (VALUE)args, MessagePack_Unpacker_execute_rescue, Qnil); rb_gc_enable(); return ret; } +static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) +{ + return MessagePack_Unpacker_execute_limit(self, data, off, + LONG2FIX(RSTRING_LEN(data))); +} + static VALUE MessagePack_Unpacker_finished_p(VALUE self) { UNPACKER(self, mp); @@ -240,7 +246,7 @@ static VALUE MessagePack_unpack_impl(VALUE args) size_t from = 0; char* dptr = RSTRING_PTR(data); - long dlen = RSTRING_LEN(data); + long dlen = FIX2LONG(((VALUE*)args)[2]); int ret; mp->user.origstr = data; @@ -269,7 +275,7 @@ static VALUE MessagePack_unpack_rescue(VALUE args) #endif } -static VALUE MessagePack_unpack(VALUE self, VALUE data) +static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); msgpack_unpacker mp; @@ -278,13 +284,19 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) mp.user = ctx; rb_gc_disable(); - VALUE args[2] = {(VALUE)&mp, data}; + VALUE args[3] = {(VALUE)&mp, data, limit}; VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, MessagePack_unpack_rescue, Qnil); rb_gc_enable(); return ret; } +static VALUE MessagePack_unpack(VALUE self, VALUE data) +{ + return MessagePack_unpack_limit(self, data, + LONG2FIX(RSTRING_LEN(data))); +} + void Init_msgpack_unpack(VALUE mMessagePack) { @@ -293,10 +305,12 @@ void Init_msgpack_unpack(VALUE mMessagePack) rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, 0); rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); + rb_define_method(cUnpacker, "execute_limit", MessagePack_Unpacker_execute_limit, 3); rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); + rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2); } From 79c02cb2180809d4c2d484d534a3a2dfa014b3ba Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0050/1648] c binding: add msgpack_object and msgpack_zone git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@97 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/object.c | 167 +++++++++++++++++++++++++++++++++++++++++++++++++++++ c/object.h | 82 ++++++++++++++++++++++++++ c/zone.c | 103 +++++++++++++++++++++++++++++++++ c/zone.h | 42 ++++++++++++++ 4 files changed, 394 insertions(+) create mode 100644 c/object.c create mode 100644 c/object.h create mode 100644 c/zone.c create mode 100644 c/zone.h diff --git a/c/object.c b/c/object.c new file mode 100644 index 0000000..7a41064 --- /dev/null +++ b/c/object.c @@ -0,0 +1,167 @@ +/* + * MessagePack for C dynamic typing routine + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/unpack.h" +#include "msgpack/unpack_define.h" +#include "msgpack/object.h" + +typedef struct { + msgpack_zone* z; + bool referenced; + bool failed; +} unpack_user; + +#define msgpack_unpack_struct(name) \ + struct msgpack_unpacker ## name + +#define msgpack_unpack_func(ret, name) \ + ret msgpack_unpacker ## name + +#define msgpack_unpack_callback(name) \ + msgpack_unpack ## name + +#define msgpack_unpack_object msgpack_object + +#define msgpack_unpack_user unpack_user + + +struct msgpack_unpacker_context; + +static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); + +static msgpack_object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); + +static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, + const char* data, size_t len, size_t* off); + +static inline msgpack_object msgpack_unpack_init(unpack_user* u) +{ msgpack_object o; return o; } + +static inline msgpack_object msgpack_unpack_uint8(unpack_user* u, uint8_t d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + +static inline msgpack_object msgpack_unpack_uint16(unpack_user* u, uint16_t d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + +static inline msgpack_object msgpack_unpack_uint32(unpack_user* u, uint32_t d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + +static inline msgpack_object msgpack_unpack_uint64(unpack_user* u, uint64_t d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + +static inline msgpack_object msgpack_unpack_int8(unpack_user* u, int8_t d) +{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } + +static inline msgpack_object msgpack_unpack_int16(unpack_user* u, int16_t d) +{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } + +static inline msgpack_object msgpack_unpack_int32(unpack_user* u, int32_t d) +{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } + +static inline msgpack_object msgpack_unpack_int64(unpack_user* u, int64_t d) +{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } + else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } + +static inline msgpack_object msgpack_unpack_float(unpack_user* u, float d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } + +static inline msgpack_object msgpack_unpack_double(unpack_user* u, double d) +{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } + +static inline msgpack_object msgpack_unpack_nil(unpack_user* u) +{ msgpack_object o; o.type = MSGPACK_OBJECT_NIL; return o; } + +static inline msgpack_object msgpack_unpack_true(unpack_user* u) +{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = true; return o; } + +static inline msgpack_object msgpack_unpack_false(unpack_user* u) +{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = false; return o; } + +static inline msgpack_object msgpack_unpack_array(unpack_user* u, unsigned int n) +{ + msgpack_object o; + o.type = MSGPACK_OBJECT_ARRAY; + o.via.container.size = 0; + o.via.container.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); + if(o.via.container.ptr == NULL) { u->failed = true; } + return o; +} + +static inline void msgpack_unpack_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) +{ + if(u->failed) { return; } + c->via.container.ptr[ c->via.container.size++ ] = o; +} + +static inline msgpack_object msgpack_unpack_map(unpack_user* u, unsigned int n) +{ + msgpack_object o; + o.type = MSGPACK_OBJECT_MAP; + o.via.container.size = 0; + o.via.container.ptr = msgpack_zone_malloc(u->z, n*2*sizeof(msgpack_object)); + if(o.via.container.ptr == NULL) { u->failed = true; } + return o; +} + +static inline void msgpack_unpack_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) +{ + if(u->failed) { return; } + c->via.container.ptr[ c->via.container.size ] = k; + c->via.container.ptr[ c->via.container.size+1 ] = v; + ++c->via.container.size; +} + +static inline msgpack_object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) +{ + msgpack_object o; + o.type = MSGPACK_OBJECT_RAW; + o.via.ref.ptr = p; + o.via.ref.size = l; + u->referenced = true; + return o; +} + +#include "msgpack/unpack_template.h" + +msgpack_object_unpack_return +msgpack_object_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result) +{ + struct msgpack_unpacker_context ctx; + msgpack_unpacker_init(&ctx); + unpack_user u = {z, false, false}; + ctx.user = u; + + size_t noff = (off ? *off : 0); + int ret = msgpack_unpacker_execute(&ctx, data, len, &noff); + if(ret < 0 || ctx.user.failed) { + return MSGPACK_OBJECT_PARSE_ERROR; + } else if(ret == 0) { + return MSGPACK_OBJECT_INSUFFICIENT_BYTES; + } + *result = msgpack_unpacker_data(&ctx); + if(off) { *off = noff; } + if(ret == 0) { + return MSGPACK_OBJECT_EXTRA_BYTES; + } + return MSGPACK_OBJECT_PARSE_SUCCESS; +} + + diff --git a/c/object.h b/c/object.h new file mode 100644 index 0000000..1b60c57 --- /dev/null +++ b/c/object.h @@ -0,0 +1,82 @@ +/* + * MessagePack for C dynamic typing routine + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_OBJECT_H__ +#define MSGPACK_OBJECT_H__ + +#include "msgpack/zone.h" +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef enum { + MSGPACK_OBJECT_NIL = 0x01, + MSGPACK_OBJECT_BOOLEAN = 0x02, + MSGPACK_OBJECT_POSITIVE_INTEGER = 0x03, + MSGPACK_OBJECT_NEGATIVE_INTEGER = 0x04, + MSGPACK_OBJECT_DOUBLE = 0x05, + MSGPACK_OBJECT_RAW = 0x06, + MSGPACK_OBJECT_ARRAY = 0x07, + MSGPACK_OBJECT_MAP = 0x08, +} msgpack_object_type; + +struct _msgpack_object; + +typedef union { + bool boolean; + uint64_t u64; + int64_t i64; + double dec; + struct { + struct _msgpack_object* ptr; + uint32_t size; + } container; + struct { + const char* ptr; + uint32_t size; + } ref; +} msgpack_object_union; + +typedef struct _msgpack_object { + msgpack_object_type type; + msgpack_object_union via; +} msgpack_object; + + +typedef enum { + MSGPACK_OBJECT_PARSE_SUCCESS = 0, + MSGPACK_OBJECT_EXTRA_BYTES = 1, + MSGPACK_OBJECT_INSUFFICIENT_BYTES = -1, + MSGPACK_OBJECT_PARSE_ERROR = -2, +} msgpack_object_unpack_return; + +msgpack_object_unpack_return +msgpack_object_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result); + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/object.h */ + diff --git a/c/zone.c b/c/zone.c new file mode 100644 index 0000000..a41bb44 --- /dev/null +++ b/c/zone.c @@ -0,0 +1,103 @@ +/* + * MessagePack for C memory pool implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/zone.h" +#include +#include + +typedef struct { + size_t free; + void* ptr; + void* alloc; +} msgpack_zone_chunk; + +struct _msgpack_zone { + msgpack_zone_chunk* array; + size_t ntail; + size_t usable; +}; + +msgpack_zone* msgpack_zone_new() +{ + return calloc(1, sizeof(msgpack_zone)); +} + +void msgpack_zone_free(msgpack_zone* z) +{ + if(z->array) { + size_t i; + for(i=0; i <= z->ntail; ++i) { + free(z->array[i].ptr); + } + } + free(z); +} + + +void* msgpack_zone_malloc(msgpack_zone* z, size_t size) +{ + if(!z->array) { + const size_t n = (sizeof(msgpack_zone_chunk) < 72/2) ? + 72 / sizeof(msgpack_zone_chunk) : 8; + msgpack_zone_chunk* array = + (msgpack_zone_chunk*)malloc(sizeof(msgpack_zone_chunk) * n); + if(!array) { return NULL; } + + size_t sz = 2048; /* FIXME chunk_size */ + while(sz < size) { sz *= 2; } + char* p = (char*)malloc(sz); + if(!p) { + free(array); + return NULL; + } + + z->array = array; + z->usable = n - 1; + array[0].free = sz - size; + array[0].ptr = p + size; + array[0].alloc = p; + return p; + } + + if(z->array[z->ntail].free > size) { + char* p = (char*)z->array[z->ntail].ptr; + z->array[z->ntail].ptr = p + size; + z->array[z->ntail].free -= size; + return p; + } + + if(z->usable <= z->ntail) { + const size_t n = (z->usable + 1) * 2; + msgpack_zone_chunk* tmp = + (msgpack_zone_chunk*)realloc(z->array, sizeof(msgpack_zone_chunk) * n); + if(!tmp) { return NULL; } + z->array = tmp; + z->usable = n - 1; + } + + size_t sz = 2048; /* FIXME chunk_size */ + while(sz < size) { sz *= 2; } + char* p = (char*)malloc(sz); + if(!p) { return NULL; } + + ++z->ntail; + z->array[z->ntail].free = sz - size; + z->array[z->ntail].ptr = p + size; + z->array[z->ntail].alloc = p; + return p; +} + diff --git a/c/zone.h b/c/zone.h new file mode 100644 index 0000000..ff6ca62 --- /dev/null +++ b/c/zone.h @@ -0,0 +1,42 @@ +/* + * MessagePack for C memory pool implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_ZONE_H__ +#define MSGPACK_ZONE_H__ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +struct _msgpack_zone; +typedef struct _msgpack_zone msgpack_zone; + +msgpack_zone* msgpack_zone_new(); +void msgpack_zone_free(msgpack_zone* z); + +void* msgpack_zone_malloc(msgpack_zone* z, size_t size); + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/zone.h */ + From e3f889c9582372efec963bd7e2bd7fb618a55596 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0051/1648] c binding: add msgpack_sbuffer git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@98 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/sbuffer.h | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 c/sbuffer.h diff --git a/c/sbuffer.h b/c/sbuffer.h new file mode 100644 index 0000000..3694dbd --- /dev/null +++ b/c/sbuffer.h @@ -0,0 +1,68 @@ +/* + * MessagePack for C simple buffer implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_SBUFFER_H__ +#define MSGPACK_SBUFFER_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + char* ptr; + size_t size; + size_t capacity; +} msgpack_sbuffer; + +static inline void msgpack_sbuffer_init(msgpack_sbuffer* sbuf) +{ + memset(sbuf, 0, sizeof(msgpack_sbuffer)); +} + +static inline void msgpack_sbuffer_destroy(msgpack_sbuffer* sbuf) +{ + free(sbuf->ptr); +} + +static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned int len) +{ + msgpack_sbuffer* sbuf = (msgpack_sbuffer*)data; + if(sbuf->capacity - sbuf->size < len) { + size_t nsize = (sbuf->capacity ? sbuf->capacity*2 : 2048); + while(nsize < sbuf->size + len) { nsize *= 2; } + + void* tmp = realloc(sbuf->ptr, nsize); + if(!tmp) { return -1; } + + sbuf->ptr = (char*)tmp; + sbuf->capacity = nsize; + } + memcpy(sbuf->ptr + sbuf->size, buf, len); + sbuf->size += len; + return 0; +} + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/sbuffer.h */ + From 81a771094d2c54437ca20c3c09a5c8f05f81d0c5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:01 +0000 Subject: [PATCH 0052/1648] c binding: add msgpack::sbuffer git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@99 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- cpp/sbuffer.hpp | 86 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 cpp/sbuffer.hpp diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp new file mode 100644 index 0000000..1dac6da --- /dev/null +++ b/cpp/sbuffer.hpp @@ -0,0 +1,86 @@ +// +// MessagePack for C++ simple buffer implementation +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_SBUFFER_HPP__ +#define MSGPACK_SBUFFER_HPP__ + +#include +#include +#include + +namespace msgpack { + + +class sbuffer { +public: + sbuffer() : m_capacity(0), m_size(0), m_ptr(NULL) { } + + ~sbuffer() + { + free(m_ptr); + } + +public: + void write(const char* buf, size_t len) + { + if(m_capacity - m_size < len) { + size_t nsize = (m_capacity ? m_capacity*2 : 2048); + while(nsize < m_size + len) { nsize *= 2; } + + char* tmp = (char*)realloc(m_ptr, nsize); + if(!tmp) { throw std::bad_alloc(); } + + m_ptr = tmp; + m_capacity = nsize; + } + memcpy(m_ptr + m_size, buf, len); + m_size += len; + } + + char* data() + { + return m_ptr; + } + + size_t size() const + { + return m_size; + } + + char* release() + { + char* tmp = m_ptr; + m_capacity = 0; + m_size = 0; + m_ptr = NULL; + return tmp; + } + +private: + size_t m_capacity; + size_t m_size; + char* m_ptr; + +private: + sbuffer(const sbuffer&); +}; + + +} // namespace msgpack + +#endif /* msgpack/sbuffer.hpp */ + From a114f4a5a5136a24f46bb52ffef1d1f588ba94e1 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:02 +0000 Subject: [PATCH 0053/1648] update pack/unpack routines git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@100 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/COPYING | 2 +- c/Makefile.am | 13 ++- c/README | 2 +- c/msgpack.h | 5 +- c/pack.c | 49 --------- c/pack.h | 92 +++++++++++----- c/unpack.c | 10 +- c/unpack.h | 6 +- cpp/COPYING | 2 +- cpp/Makefile.am | 5 +- cpp/README | 2 +- cpp/msgpack.hpp | 3 +- cpp/object.cpp | 2 +- cpp/object.hpp | 136 ++++++++++++++---------- cpp/pack.hpp | 216 ++++++++++++++++++++++++++++---------- cpp/type/array.hpp | 10 +- cpp/type/map.hpp | 34 +++--- cpp/type/tuple.hpp.erb | 6 +- cpp/unpack.cpp | 71 ++++++------- cpp/unpack.hpp | 2 +- cpp/zone.cpp | 2 +- cpp/zone.hpp.erb | 2 +- msgpack/pack_define.h | 2 +- msgpack/pack_template.h | 78 +++++++------- msgpack/unpack_define.h | 2 +- msgpack/unpack_template.h | 80 +++++++------- ruby/pack.c | 8 +- ruby/pack.h | 4 +- ruby/rbinit.c | 2 +- ruby/test_case.rb | 7 ++ ruby/unpack.c | 10 +- ruby/unpack.h | 4 +- 32 files changed, 503 insertions(+), 366 deletions(-) delete mode 100644 c/pack.c diff --git a/c/COPYING b/c/COPYING index 5f100cd..6f5f220 100644 --- a/c/COPYING +++ b/c/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/c/Makefile.am b/c/Makefile.am index f1c57e9..e7cdc10 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -1,13 +1,18 @@ lib_LTLIBRARIES = libmsgpackc.la libmsgpackc_la_SOURCES = \ - pack.c \ - unpack.c + unpack.c \ + object.c \ + zone.c nobase_include_HEADERS = \ msgpack.h \ + msgpack/sbuffer.h \ msgpack/pack.h \ - msgpack/unpack.h + msgpack/unpack.h \ + msgpack/object.h \ + msgpack/zone.h -libmsgpackc_la_LDFLAGS = -version-info 0:0:0 +# -version-info CURRENT:REVISION:AGE +libmsgpackc_la_LDFLAGS = -version-info 1:0:0 diff --git a/c/README b/c/README index 609b67d..76dc221 100644 --- a/c/README +++ b/c/README @@ -4,7 +4,7 @@ MessagePack is a binary-based efficient data interchange format. -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/c/msgpack.h b/c/msgpack.h index 7a15f71..a59ef03 100644 --- a/c/msgpack.h +++ b/c/msgpack.h @@ -1,7 +1,7 @@ /* * MessagePack for C * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,5 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "msgpack/object.h" +#include "msgpack/zone.h" #include "msgpack/pack.h" #include "msgpack/unpack.h" +#include "msgpack/sbuffer.h" diff --git a/c/pack.c b/c/pack.c deleted file mode 100644 index f4787c0..0000000 --- a/c/pack.c +++ /dev/null @@ -1,49 +0,0 @@ -/* - * MessagePack packing routine for C - * - * Copyright (C) 2008 FURUHASHI Sadayuki - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "msgpack/pack.h" -#include "msgpack/pack_define.h" -#include - - -#define msgpack_pack_inline_func(name) \ - void msgpack_pack_##name - -#define msgpack_pack_inline_func_cint(name) \ - void msgpack_pack_##name - -#define msgpack_pack_user msgpack_pack_t* - -#define msgpack_pack_append_buffer(user, buf, len) \ - (*(user)->callback)((user)->data, (const char*)buf, len) - -#include "msgpack/pack_template.h" - -msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_append_buffer_t callback) -{ - msgpack_pack_t* ctx = calloc(1, sizeof(msgpack_pack_t)); - if(!ctx) { return NULL; } - ctx->data = data; - ctx->callback = callback; - return ctx; -} - -void msgpack_pack_free(msgpack_pack_t* ctx) -{ - free(ctx); -} - diff --git a/c/pack.h b/c/pack.h index ecf675c..752cbc4 100644 --- a/c/pack.h +++ b/c/pack.h @@ -1,7 +1,7 @@ /* - * MessagePack packing routine for C + * MessagePack for C packing routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,50 +20,90 @@ #include #include +#include #ifdef __cplusplus extern "C" { #endif -typedef void (*msgpack_pack_append_buffer_t)(void* data, const char* b, unsigned int i); +typedef int (*msgpack_pack_write_t)(void* data, const char* buf, unsigned int len); typedef struct { void* data; - msgpack_pack_append_buffer_t callback; + msgpack_pack_write_t callback; } msgpack_pack_t; -msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_append_buffer_t callback); +void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback); +msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback); void msgpack_pack_free(msgpack_pack_t* ctx); -void msgpack_pack_int(msgpack_pack_t* ctx, int d); -void msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); -void msgpack_pack_long(msgpack_pack_t* ctx, long d); -void msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); +int msgpack_pack_short(msgpack_pack_t* ctx, short d); +int msgpack_pack_int(msgpack_pack_t* ctx, int d); +int msgpack_pack_long(msgpack_pack_t* ctx, long d); +int msgpack_pack_long_long(msgpack_pack_t* ctx, long long d); +int msgpack_pack_unsigned_short(msgpack_pack_t* ctx, unsigned short d); +int msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); +int msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); +int msgpack_pack_unsigned_long_long(msgpack_pack_t* ctx, unsigned long long d); -void msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); -void msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); -void msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); -void msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); -void msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); -void msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); -void msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); -void msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); +int msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); +int msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); +int msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); +int msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); +int msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); +int msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); +int msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); +int msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); -void msgpack_pack_float(msgpack_pack_t* ctx, float d); -void msgpack_pack_double(msgpack_pack_t* ctx, double d); +int msgpack_pack_float(msgpack_pack_t* ctx, float d); +int msgpack_pack_double(msgpack_pack_t* ctx, double d); -void msgpack_pack_nil(msgpack_pack_t* ctx); -void msgpack_pack_true(msgpack_pack_t* ctx); -void msgpack_pack_false(msgpack_pack_t* ctx); +int msgpack_pack_nil(msgpack_pack_t* ctx); +int msgpack_pack_true(msgpack_pack_t* ctx); +int msgpack_pack_false(msgpack_pack_t* ctx); -void msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); +int msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); -void msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); +int msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); -void msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); -void msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); +int msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); +int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); + + + +#define msgpack_pack_inline_func(name) \ + inline int msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + inline int msgpack_pack ## name + +#define msgpack_pack_user msgpack_pack_t* + +#define msgpack_pack_append_buffer(user, buf, len) \ + return (*(user)->callback)((user)->data, (const char*)buf, len) + +#include "msgpack/pack_template.h" + +inline void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback) +{ + ctx->data = data; + ctx->callback = callback; +} + +inline msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback) +{ + msgpack_pack_t* ctx = (msgpack_pack_t*)calloc(1, sizeof(msgpack_pack_t)); + if(!ctx) { return NULL; } + msgpack_pack_init(ctx, data, callback); + return ctx; +} + +inline void msgpack_pack_free(msgpack_pack_t* ctx) +{ + free(ctx); +} #ifdef __cplusplus diff --git a/c/unpack.c b/c/unpack.c index c7f25c4..03c67be 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -1,7 +1,7 @@ /* - * MessagePack unpacking routine for C + * MessagePack for C unpacking routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,13 +21,13 @@ #define msgpack_unpack_struct(name) \ - struct template_##name + struct template ## name #define msgpack_unpack_func(ret, name) \ - ret template_func_##name + ret template_func ## name #define msgpack_unpack_callback(name) \ - template_callback_##name + template_callback ## name #define msgpack_unpack_object void* diff --git a/c/unpack.h b/c/unpack.h index c1cacab..4977f51 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -1,7 +1,7 @@ /* - * MessagePack unpacking routine for C + * MessagePack for C unpacking routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,11 +54,11 @@ typedef struct { msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback); void msgpack_unpack_free(msgpack_unpack_t* ctx); -void msgpack_unpack_reset(msgpack_unpack_t* ctx); int msgpack_unpack_execute(msgpack_unpack_t* ctx, const char* data, size_t len, size_t* off); void* msgpack_unpack_data(msgpack_unpack_t* ctx); +void msgpack_unpack_reset(msgpack_unpack_t* ctx); #ifdef __cplusplus diff --git a/cpp/COPYING b/cpp/COPYING index 5f100cd..6f5f220 100644 --- a/cpp/COPYING +++ b/cpp/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 45c75e0..c7ddf4b 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -7,6 +7,7 @@ libmsgpack_la_SOURCES = \ nobase_include_HEADERS = \ msgpack.hpp \ + msgpack/sbuffer.hpp \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ @@ -41,6 +42,6 @@ MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ msgpack/zone.hpp -# FIXME -libmsgpack_la_LDFLAGS = -version-info 0:0:0 +# -version-info CURRENT:REVISION:AGE +libmsgpack_la_LDFLAGS = -version-info 1:0:0 diff --git a/cpp/README b/cpp/README index eceff1b..96f18b1 100644 --- a/cpp/README +++ b/cpp/README @@ -4,7 +4,7 @@ MessagePack is a binary-based efficient data interchange format. -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp index 9f635d0..f00da06 100644 --- a/cpp/msgpack.hpp +++ b/cpp/msgpack.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,3 +19,4 @@ #include "msgpack/zone.hpp" #include "msgpack/pack.hpp" #include "msgpack/unpack.hpp" +#include "msgpack/sbuffer.hpp" diff --git a/cpp/object.cpp b/cpp/object.cpp index 52b6ad0..d217c06 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -1,7 +1,7 @@ // // MessagePack for C++ dynamic typed objects // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/object.hpp b/cpp/object.hpp index 1f0dcdf..c6f1e4a 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -32,14 +32,16 @@ class type_error : public std::bad_cast { }; namespace type { - static const unsigned char NIL = 0x01; - static const unsigned char BOOLEAN = 0x02; - static const unsigned char POSITIVE_INTEGER = 0x03; - static const unsigned char NEGATIVE_INTEGER = 0x04; - static const unsigned char DOUBLE = 0x05; - static const unsigned char RAW = 0x06; - static const unsigned char ARRAY = 0x07; - static const unsigned char MAP = 0x08; + enum object_type { + NIL = 0x01, + BOOLEAN = 0x02, + POSITIVE_INTEGER = 0x03, + NEGATIVE_INTEGER = 0x04, + DOUBLE = 0x05, + RAW = 0x06, + ARRAY = 0x07, + MAP = 0x08, + }; } @@ -59,7 +61,7 @@ struct object { } ref; }; - unsigned char type; + type::object_type type; union_type via; bool is_nil() { return type == type::NIL; } @@ -68,7 +70,7 @@ struct object { T as(); template - void convert(T& v); + void convert(T* v); private: struct implicit_type; @@ -77,6 +79,21 @@ public: implicit_type convert(); }; +bool operator==(const object x, const object y); +bool operator!=(const object x, const object y); + +std::ostream& operator<< (std::ostream& s, const object o); + + +template +inline void pack(Stream& s, const T& v); + +template +packer& operator<< (packer& o, const T& v); + +template +T& operator>> (object o, T& v); + struct object::implicit_type { implicit_type(object o) : obj(o) { } @@ -89,39 +106,41 @@ private: object obj; }; -std::ostream& operator<< (std::ostream& s, const object o); -bool operator==(const object x, const object y); -inline bool operator!=(const object x, const object y) { return !(x == y); } +template +class define : public Type { +public: + typedef Type msgpack_type; + typedef define define_type; + + define() {} + define(const msgpack_type& v) : msgpack_type(v) {} + + template + void msgpack_pack(Packer& o) const + { + o << static_cast(*this); + } + + void msgpack_unpack(object o) + { + o >> static_cast(*this); + } +}; -inline object& operator>> (object o, object& v) -{ - v = o; - return v; -} template -packer& operator<< (packer& o, const object& v); - - - template -inline void convert(T& v, object o) +inline packer& packer::pack(const T& v) { - o >> v; -} - -template -inline void pack(packer& o, const T& v) -{ - o << v; + *this << v; + return *this; } template inline void pack(Stream& s, const T& v) { - packer pk(s); - pack(pk, v); + packer(s).pack(v); } template @@ -130,7 +149,11 @@ inline void pack_copy(packer& o, T v) pack(o, v); } - +inline object& operator>> (object o, object& v) +{ + v = o; + return v; +} template inline T& operator>> (object o, T& v) @@ -147,27 +170,8 @@ inline packer& operator<< (packer& o, const T& v) } -template -class define : public Type { -public: - typedef Type msgpack_type; - typedef define define_type; - - define() {} - define(msgpack_type v) : msgpack_type(v) {} - - template - void msgpack_pack(Packer& o) const - { - o << static_cast(*this); - } - - void msgpack_unpack(object o) - { - o >> static_cast(*this); - } -}; - +inline bool operator!=(const object x, const object y) +{ return !(x == y); } inline object::implicit_type object::convert() @@ -179,16 +183,32 @@ template inline T object::as() { T v; - msgpack::convert(v, *this); + convert(&v); return v; } template -void object::convert(T& v) +inline void object::convert(T* v) { - msgpack::convert(v, *this); + *this >> *v; } + +// obsolete +template +inline void convert(T& v, object o) +{ + o.convert(&v); +} + +// obsolete +template +inline void pack(packer& o, const T& v) +{ + o.pack(v); +} + + template packer& operator<< (packer& o, const object& v) { diff --git a/cpp/pack.hpp b/cpp/pack.hpp index c07bcc3..52854ac 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ serializing routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -32,70 +32,73 @@ public: packer(Stream& s); public: - void pack_uint8(uint8_t d) { pack_uint8_impl(m_stream, d); } - void pack_uint16(uint16_t d) { pack_uint16_impl(m_stream, d); } - void pack_uint32(uint32_t d) { pack_uint32_impl(m_stream, d); } - void pack_uint64(uint64_t d) { pack_uint64_impl(m_stream, d); } - void pack_int8(uint8_t d) { pack_int8_impl(m_stream, d); } - void pack_int16(uint16_t d) { pack_int16_impl(m_stream, d); } - void pack_int32(uint32_t d) { pack_int32_impl(m_stream, d); } - void pack_int64(uint64_t d) { pack_int64_impl(m_stream, d); } + template + packer& pack(const T& v); - void pack_short(int d) { pack_short_impl(m_stream, d); } - void pack_int(int d) { pack_int_impl(m_stream, d); } - void pack_long(long d) { pack_long_impl(m_stream, d); } - void pack_long_long(long long d) { pack_long_long_impl(m_stream, d); } - void pack_unsigned_short(unsigned short d) { pack_unsigned_short_impl(m_stream, d); } - void pack_unsigned_int(unsigned int d) { pack_unsigned_int_impl(m_stream, d); } - void pack_unsigned_long(unsigned long d) { pack_unsigned_long_impl(m_stream, d); } - void pack_unsigned_long_long(unsigned long long d) { pack_unsigned_long_long_impl(m_stream, d); } + packer& pack_uint8(uint8_t d); + packer& pack_uint16(uint16_t d); + packer& pack_uint32(uint32_t d); + packer& pack_uint64(uint64_t d); + packer& pack_int8(uint8_t d); + packer& pack_int16(uint16_t d); + packer& pack_int32(uint32_t d); + packer& pack_int64(uint64_t d); - void pack_float(float d) { pack_float_impl(m_stream, d); } - void pack_double(double d) { pack_double_impl(m_stream, d); } + packer& pack_short(int d); + packer& pack_int(int d); + packer& pack_long(long d); + packer& pack_long_long(long long d); + packer& pack_unsigned_short(unsigned short d); + packer& pack_unsigned_int(unsigned int d); + packer& pack_unsigned_long(unsigned long d); + packer& pack_unsigned_long_long(unsigned long long d); - void pack_nil() { pack_nil_impl(m_stream); } - void pack_true() { pack_true_impl(m_stream); } - void pack_false() { pack_false_impl(m_stream); } + packer& pack_float(float d); + packer& pack_double(double d); - void pack_array(unsigned int n) { pack_array_impl(m_stream, n); } + packer& pack_nil(); + packer& pack_true(); + packer& pack_false(); - void pack_map(unsigned int n) { pack_map_impl(m_stream, n); } + packer& pack_array(unsigned int n); - void pack_raw(size_t l) { pack_raw_impl(m_stream, l); } - void pack_raw_body(const char* b, size_t l) { pack_raw_body_impl(m_stream, b, l); } + packer& pack_map(unsigned int n); + + packer& pack_raw(size_t l); + packer& pack_raw_body(const char* b, size_t l); private: - static void pack_uint8_impl(Stream& x, uint8_t d); - static void pack_uint16_impl(Stream& x, uint16_t d); - static void pack_uint32_impl(Stream& x, uint32_t d); - static void pack_uint64_impl(Stream& x, uint64_t d); - static void pack_int8_impl(Stream& x, int8_t d); - static void pack_int16_impl(Stream& x, int16_t d); - static void pack_int32_impl(Stream& x, int32_t d); - static void pack_int64_impl(Stream& x, int64_t d); + static void _pack_uint8(Stream& x, uint8_t d); + static void _pack_uint16(Stream& x, uint16_t d); + static void _pack_uint32(Stream& x, uint32_t d); + static void _pack_uint64(Stream& x, uint64_t d); + static void _pack_int8(Stream& x, int8_t d); + static void _pack_int16(Stream& x, int16_t d); + static void _pack_int32(Stream& x, int32_t d); + static void _pack_int64(Stream& x, int64_t d); - static void pack_short_impl(Stream& x, short d); - static void pack_int_impl(Stream& x, int d); - static void pack_long_impl(Stream& x, long d); - static void pack_long_long_impl(Stream& x, long long d); - static void pack_unsigned_short_impl(Stream& x, unsigned short d); - static void pack_unsigned_int_impl(Stream& x, unsigned int d); - static void pack_unsigned_long_impl(Stream& x, unsigned long d); - static void pack_unsigned_long_long_impl(Stream& x, unsigned long long d); + static void _pack_short(Stream& x, short d); + static void _pack_int(Stream& x, int d); + static void _pack_long(Stream& x, long d); + static void _pack_long_long(Stream& x, long long d); + static void _pack_unsigned_short(Stream& x, unsigned short d); + static void _pack_unsigned_int(Stream& x, unsigned int d); + static void _pack_unsigned_long(Stream& x, unsigned long d); + static void _pack_unsigned_long_long(Stream& x, unsigned long long d); - static void pack_float_impl(Stream& x, float d); - static void pack_double_impl(Stream& x, double d); + static void _pack_float(Stream& x, float d); + static void _pack_double(Stream& x, double d); - static void pack_nil_impl(Stream& x); - static void pack_true_impl(Stream& x); - static void pack_false_impl(Stream& x); + static void _pack_nil(Stream& x); + static void _pack_true(Stream& x); + static void _pack_false(Stream& x); - static void pack_array_impl(Stream& x, unsigned int n); + static void _pack_array(Stream& x, unsigned int n); - static void pack_map_impl(Stream& x, unsigned int n); + static void _pack_map(Stream& x, unsigned int n); - static void pack_raw_impl(Stream& x, size_t l); - static void pack_raw_body_impl(Stream& x, const void* b, size_t l); + static void _pack_raw(Stream& x, size_t l); + static void _pack_raw_body(Stream& x, const void* b, size_t l); static void append_buffer(Stream& x, const unsigned char* buf, unsigned int len) { x.write((const char*)buf, len); } @@ -107,13 +110,14 @@ private: packer(); }; + #define msgpack_pack_inline_func(name) \ template \ - inline void packer::pack_ ## name ## _impl + inline void packer::_pack ## name #define msgpack_pack_inline_func_cint(name) \ template \ - inline void packer::pack_ ## name ## _impl + inline void packer::_pack ## name #define msgpack_pack_user Stream& @@ -125,6 +129,112 @@ private: template packer::packer(Stream& s) : m_stream(s) { } +template +inline packer& packer::pack_uint8(uint8_t d) +{ _pack_uint8(m_stream, d); return *this; } + +template +inline packer& packer::pack_uint16(uint16_t d) +{ _pack_uint16(m_stream, d); return *this; } + +template +inline packer& packer::pack_uint32(uint32_t d) +{ _pack_uint32(m_stream, d); return *this; } + +template +inline packer& packer::pack_uint64(uint64_t d) +{ _pack_uint64(m_stream, d); return *this; } + +template +inline packer& packer::pack_int8(uint8_t d) +{ _pack_int8(m_stream, d); return *this; } + +template +inline packer& packer::pack_int16(uint16_t d) +{ _pack_int16(m_stream, d); return *this; } + +template +inline packer& packer::pack_int32(uint32_t d) +{ _pack_int32(m_stream, d); return *this; } + +template +inline packer& packer::pack_int64(uint64_t d) +{ _pack_int64(m_stream, d); return *this;} + + +template +inline packer& packer::pack_short(int d) +{ _pack_short(m_stream, d); return *this; } + +template +inline packer& packer::pack_int(int d) +{ _pack_int(m_stream, d); return *this; } + +template +inline packer& packer::pack_long(long d) +{ _pack_long(m_stream, d); return *this; } + +template +inline packer& packer::pack_long_long(long long d) +{ _pack_long_long(m_stream, d); return *this; } + +template +inline packer& packer::pack_unsigned_short(unsigned short d) +{ _pack_unsigned_short(m_stream, d); return *this; } + +template +inline packer& packer::pack_unsigned_int(unsigned int d) +{ _pack_unsigned_int(m_stream, d); return *this; } + +template +inline packer& packer::pack_unsigned_long(unsigned long d) +{ _pack_unsigned_long(m_stream, d); return *this; } + +template +inline packer& packer::pack_unsigned_long_long(unsigned long long d) +{ _pack_unsigned_long_long(m_stream, d); return *this; } + + +template +inline packer& packer::pack_float(float d) +{ _pack_float(m_stream, d); return *this; } + +template +inline packer& packer::pack_double(double d) +{ _pack_double(m_stream, d); return *this; } + + +template +inline packer& packer::pack_nil() +{ _pack_nil(m_stream); return *this; } + +template +inline packer& packer::pack_true() +{ _pack_true(m_stream); return *this; } + +template +inline packer& packer::pack_false() +{ _pack_false(m_stream); return *this; } + + +template +inline packer& packer::pack_array(unsigned int n) +{ _pack_array(m_stream, n); return *this; } + + +template +inline packer& packer::pack_map(unsigned int n) +{ _pack_map(m_stream, n); return *this; } + + +template +inline packer& packer::pack_raw(size_t l) +{ _pack_raw(m_stream, l); return *this; } + +template +inline packer& packer::pack_raw_body(const char* b, size_t l) +{ _pack_raw_body(m_stream, b, l); return *this; } + } // namespace msgpack diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index 0522d4c..f9f8038 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -29,11 +29,11 @@ inline std::vector operator>> (object o, std::vector& v) { if(o.type != type::ARRAY) { throw type_error(); } v.resize(o.via.container.size); - object* p(o.via.container.ptr); - object* const pend(o.via.container.ptr + o.via.container.size); - T* it(&v.front()); + object* p = o.via.container.ptr; + object* const pend = o.via.container.ptr + o.via.container.size; + T* it = &v.front(); for(; p < pend; ++p, ++it) { - convert(*it, *p); + p->convert(it); } return v; } @@ -45,7 +45,7 @@ inline packer& operator<< (packer& o, const std::vector& v) o.pack_array(v.size()); for(typename std::vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(o, *it); + o.pack(*it); } return o; } diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index c79f31c..619c36e 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -47,12 +47,12 @@ inline type::assoc_vector& operator>> (object o, type::assoc_vector& v { if(o.type != type::MAP) { throw type_error(); } v.resize(o.via.container.size); - object* p(o.via.container.ptr); - object* const pend(o.via.container.ptr + o.via.container.size); + object* p = o.via.container.ptr; + object* const pend = o.via.container.ptr + o.via.container.size; std::pair* it(&v.front()); for(; p < pend; ++it) { - convert(it->first, *p); ++p; - convert(it->second, *p); ++p; + p->convert(&it->first); ++p; + p->convert(&it->second); ++p; } std::sort(v.begin(), v.end(), type::detail::pair_first_less()); return v; @@ -64,8 +64,8 @@ inline packer& operator<< (packer& o, const type::assoc_vector::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(o, it->first); - pack(o, it->second); + o.pack(it->first); + o.pack(it->second); } return o; } @@ -79,14 +79,14 @@ inline std::map operator>> (object o, std::map& v) object* const pend(o.via.container.ptr + o.via.container.size*2); while(p < pend) { K key; - convert(key, *p); ++p; + p->convert(&key); ++p; typename std::map::iterator it(v.find(key)); if(it != v.end()) { V val; - convert(val, *p); ++p; + p->convert(&val); ++p; it->insert( std::pair(key, val) ); } else { - convert(it->second, *p); ++p; + p->convert(&it->second); ++p; } } return v; @@ -98,8 +98,8 @@ inline packer& operator<< (packer& o, const std::map& v) o.pack_map(v.size()); for(typename std::map::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(o, it->first); - pack(o, it->second); + o.pack(it->first); + o.pack(it->second); } return o; } @@ -109,12 +109,12 @@ template inline std::multimap operator>> (object o, std::multimap& v) { if(o.type != type::MAP) { throw type_error(); } - object* p(o.via.container.ptr); - object* const pend(o.via.container.ptr + o.via.container.size*2); + object* p = o.via.container.ptr; + object* const pend = o.via.container.ptr + o.via.container.size*2; while(p < pend) { std::pair value; - convert(value.first, *p); ++p; - convert(value.second, *p); ++p; + p->convert(&value.first); ++p; + p->convert(&value.second); ++p; v.insert(value); } return v; @@ -126,8 +126,8 @@ inline packer& operator<< (packer& o, const std::multimap& o.pack_multimap(v.size()); for(typename std::multimap::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { - pack(o, it->first); - pack(o, it->second); + o.pack(it->first); + o.pack(it->second); } return o; } diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 13d4bd7..a66c57e 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -111,7 +111,7 @@ struct tuple, A<%=j%><%}%>> { tuple() {} tuple(typename tuple_type::transparent_reference _a0<%1.upto(i) {|j|%>, typename tuple_type>::transparent_reference _a<%=j%><%}%>) : a0(_a0)<%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} - tuple(object o) { convert(*this, o); } + tuple(object o) { o.convert(this); } template typename tuple_element::reference get() { return tuple_element(*this).get(); } template typename const_tuple_element::const_reference get() const @@ -138,7 +138,7 @@ type::tuple, A<%=j%><%}%>>& operator>> ( if(o.type != type::ARRAY) { throw type_error(); } if(o.via.container.size < <%=i+1%>) { throw type_error(); } <%0.upto(i) {|j|%> - convert>(v.template get<<%=j%>>(), o.via.container.ptr[<%=j%>]);<%}%> + o.via.container.ptr[<%=j%>].convert>(&v.template get<<%=j%>>());<%}%> return v; } <%}%> @@ -158,7 +158,7 @@ const packer& operator<< ( const type::tuple, A<%=j%><%}%>>& v) { o.pack_array(<%=i+1%>); <%0.upto(i) {|j|%> - pack(o, v.template get<<%=j%>>());<%}%> + o.pack(v.template get<<%=j%>>());<%}%> return o; } <%}%> diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index f2ef330..62088d6 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -1,7 +1,7 @@ // // MessagePack for C++ deserializing routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,30 +23,25 @@ namespace msgpack { //namespace { -struct allocator { +struct unpack_user { zone* z; bool referenced; - - inline object* malloc_object(size_t n) - { - return (object*)z->malloc(sizeof(object)*n); - } }; //} // noname namespace #define msgpack_unpack_struct(name) \ - struct msgpack_unpacker_##name + struct msgpack_unpacker ## name #define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker_##name + ret msgpack_unpacker ## name #define msgpack_unpack_callback(name) \ - msgpack_unpack_##name + msgpack_unpack ## name #define msgpack_unpack_object object -#define msgpack_unpack_user allocator +#define msgpack_unpack_user unpack_user struct msgpack_unpacker_context; @@ -59,87 +54,87 @@ static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, const char* data, size_t len, size_t* off); -static inline object msgpack_unpack_init(allocator* a) +static inline object msgpack_unpack_init(unpack_user* u) { return object(); } -static inline object msgpack_unpack_uint8(allocator* a, uint8_t d) +static inline object msgpack_unpack_uint8(unpack_user* u, uint8_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint16(allocator* a, uint16_t d) +static inline object msgpack_unpack_uint16(unpack_user* u, uint16_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint32(allocator* a, uint32_t d) +static inline object msgpack_unpack_uint32(unpack_user* u, uint32_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_uint64(allocator* a, uint64_t d) +static inline object msgpack_unpack_uint64(unpack_user* u, uint64_t d) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } -static inline object msgpack_unpack_int8(allocator* a, int8_t d) +static inline object msgpack_unpack_int8(unpack_user* u, int8_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int16(allocator* a, int16_t d) +static inline object msgpack_unpack_int16(unpack_user* u, int16_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int32(allocator* a, int32_t d) +static inline object msgpack_unpack_int32(unpack_user* u, int32_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_int64(allocator* a, int64_t d) +static inline object msgpack_unpack_int64(unpack_user* u, int64_t d) { if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } -static inline object msgpack_unpack_float(allocator* a, float d) +static inline object msgpack_unpack_float(unpack_user* u, float d) { object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object msgpack_unpack_double(allocator* a, double d) +static inline object msgpack_unpack_double(unpack_user* u, double d) { object o; o.type = type::DOUBLE; o.via.dec = d; return o; } -static inline object msgpack_unpack_nil(allocator* a) +static inline object msgpack_unpack_nil(unpack_user* u) { object o; o.type = type::NIL; return o; } -static inline object msgpack_unpack_true(allocator* a) +static inline object msgpack_unpack_true(unpack_user* u) { object o; o.type = type::BOOLEAN; o.via.boolean = true; return o; } -static inline object msgpack_unpack_false(allocator* a) +static inline object msgpack_unpack_false(unpack_user* u) { object o; o.type = type::BOOLEAN; o.via.boolean = false; return o; } -static inline object msgpack_unpack_array(allocator* a, unsigned int n) +static inline object msgpack_unpack_array(unpack_user* u, unsigned int n) { object o; o.type = type::ARRAY; o.via.container.size = 0; - o.via.container.ptr = a->malloc_object(n); + o.via.container.ptr = (object*)u->z->malloc(n*sizeof(object)); return o; } -static inline void msgpack_unpack_array_item(allocator* a, object* c, object o) +static inline void msgpack_unpack_array_item(unpack_user* u, object* c, object o) { c->via.container.ptr[ c->via.container.size++ ] = o; } -static inline object msgpack_unpack_map(allocator* a, unsigned int n) +static inline object msgpack_unpack_map(unpack_user* u, unsigned int n) { object o; o.type = type::MAP; o.via.container.size = 0; - o.via.container.ptr = a->malloc_object(n*2); + o.via.container.ptr = (object*)u->z->malloc(n*2*sizeof(object)); return o; } -static inline void msgpack_unpack_map_item(allocator* a, object* c, object k, object v) +static inline void msgpack_unpack_map_item(unpack_user* u, object* c, object k, object v) { c->via.container.ptr[ c->via.container.size ] = k; c->via.container.ptr[ c->via.container.size+1 ] = v; ++c->via.container.size; } -static inline object msgpack_unpack_raw(allocator* a, const char* b, const char* p, unsigned int l) +static inline object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) { object o; o.type = type::RAW; o.via.ref.ptr = p; o.via.ref.size = l; - a->referenced = true; + u->referenced = true; return o; } @@ -151,8 +146,8 @@ struct context { context() { msgpack_unpacker_init(&m_ctx); - allocator a = {NULL, false}; - m_ctx.user = a; + unpack_user u = {NULL, false}; + m_ctx.user = u; } ~context() { } @@ -171,8 +166,8 @@ struct context { { zone* z = m_ctx.user.z; msgpack_unpacker_init(&m_ctx); - allocator a = {z, false}; - m_ctx.user = a; + unpack_user u = {z, false}; + m_ctx.user = u; } void set_zone(zone* z) @@ -193,7 +188,7 @@ private: context(const context&); }; -context* as_ctx(void* m) +static inline context* as_ctx(void* m) { return reinterpret_cast(m); } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 293a5d3..cde45e7 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ deserializing routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/zone.cpp b/cpp/zone.cpp index 7c5a1c9..f765266 100644 --- a/cpp/zone.cpp +++ b/cpp/zone.cpp @@ -1,7 +1,7 @@ // // MessagePack for C++ memory pool // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 9a8c81a..c0eb2e3 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -1,7 +1,7 @@ // // MessagePack for C++ memory pool // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index 6512d77..33408e5 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index dafb0b1..212c0e6 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -1,7 +1,7 @@ /* * MessagePack packing routine template * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -278,49 +278,49 @@ do { \ #ifdef msgpack_pack_inline_func_fastint -msgpack_pack_inline_func_fastint(uint8)(msgpack_pack_user x, uint8_t d) +msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { const unsigned char buf[2] = {0xcc, d}; msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func_fastint(uint16)(msgpack_pack_user x, uint16_t d) +msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func_fastint(uint32)(msgpack_pack_user x, uint32_t d) +msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { const unsigned char buf[5] = {0xce, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func_fastint(uint64)(msgpack_pack_user x, uint64_t d) +msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } -msgpack_pack_inline_func_fastint(int8)(msgpack_pack_user x, int8_t d) +msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { const unsigned char buf[2] = {0xd0, d}; msgpack_pack_append_buffer(x, buf, 2); } -msgpack_pack_inline_func_fastint(int16)(msgpack_pack_user x, int16_t d) +msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } -msgpack_pack_inline_func_fastint(int32)(msgpack_pack_user x, int32_t d) +msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func_fastint(int64)(msgpack_pack_user x, int64_t d) +msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); @@ -330,42 +330,42 @@ msgpack_pack_inline_func_fastint(int64)(msgpack_pack_user x, int64_t d) #endif -msgpack_pack_inline_func(uint8)(msgpack_pack_user x, uint8_t d) +msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d) { msgpack_pack_real_uint8(x, d); } -msgpack_pack_inline_func(uint16)(msgpack_pack_user x, uint16_t d) +msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d) { msgpack_pack_real_uint16(x, d); } -msgpack_pack_inline_func(uint32)(msgpack_pack_user x, uint32_t d) +msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d) { msgpack_pack_real_uint32(x, d); } -msgpack_pack_inline_func(uint64)(msgpack_pack_user x, uint64_t d) +msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d) { msgpack_pack_real_uint64(x, d); } -msgpack_pack_inline_func(int8)(msgpack_pack_user x, int8_t d) +msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d) { msgpack_pack_real_int8(x, d); } -msgpack_pack_inline_func(int16)(msgpack_pack_user x, int16_t d) +msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d) { msgpack_pack_real_int16(x, d); } -msgpack_pack_inline_func(int32)(msgpack_pack_user x, int32_t d) +msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d) { msgpack_pack_real_int32(x, d); } -msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) +msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) { msgpack_pack_real_int64(x, d); } @@ -373,7 +373,7 @@ msgpack_pack_inline_func(int64)(msgpack_pack_user x, int64_t d) #ifdef msgpack_pack_inline_func_cint -msgpack_pack_inline_func_cint(short)(msgpack_pack_user x, short d) +msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) { #if defined(SIZEOF_SHORT) || defined(SHRT_MAX) #if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff @@ -394,7 +394,7 @@ if(sizeof(short) == 2) { #endif } -msgpack_pack_inline_func_cint(int)(msgpack_pack_user x, int d) +msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) { #if defined(SIZEOF_INT) || defined(INT_MAX) #if SIZEOF_INT == 2 || INT_MAX == 0x7fff @@ -415,7 +415,7 @@ if(sizeof(int) == 2) { #endif } -msgpack_pack_inline_func_cint(long)(msgpack_pack_user x, long d) +msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) { #if defined(SIZEOF_LONG) || defined(LONG_MAX) #if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL @@ -436,7 +436,7 @@ if(sizeof(long) == 2) { #endif } -msgpack_pack_inline_func_cint(long_long)(msgpack_pack_user x, long long d) +msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) { #if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) #if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL @@ -457,7 +457,7 @@ if(sizeof(long long) == 2) { #endif } -msgpack_pack_inline_func_cint(unsigned_short)(msgpack_pack_user x, unsigned short d) +msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) { #if defined(SIZEOF_SHORT) || defined(USHRT_MAX) #if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU @@ -478,7 +478,7 @@ if(sizeof(unsigned short) == 2) { #endif } -msgpack_pack_inline_func_cint(unsigned_int)(msgpack_pack_user x, unsigned int d) +msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) { #if defined(SIZEOF_INT) || defined(UINT_MAX) #if SIZEOF_INT == 2 || UINT_MAX == 0xffffU @@ -499,7 +499,7 @@ if(sizeof(unsigned int) == 2) { #endif } -msgpack_pack_inline_func_cint(unsigned_long)(msgpack_pack_user x, unsigned long d) +msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) { #if defined(SIZEOF_LONG) || defined(ULONG_MAX) #if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL @@ -520,7 +520,7 @@ if(sizeof(unsigned int) == 2) { #endif } -msgpack_pack_inline_func_cint(unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) { #if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) #if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL @@ -550,17 +550,19 @@ if(sizeof(unsigned long long) == 2) { * Float */ -msgpack_pack_inline_func(float)(msgpack_pack_user x, float d) +msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { - uint32_t n = *((uint32_t*)&d); // FIXME - const unsigned char buf[5] = {0xca, STORE_BE32(n)}; + union { char buf[4]; uint32_t num; } f; + *((float*)&f.buf) = d; // FIXME + const unsigned char buf[5] = {0xca, STORE_BE32(f.num)}; msgpack_pack_append_buffer(x, buf, 5); } -msgpack_pack_inline_func(double)(msgpack_pack_user x, double d) +msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { - uint64_t n = *((uint64_t*)&d); // FIXME - const unsigned char buf[9] = {0xcb, STORE_BE64(n)}; + union { char buf[8]; uint64_t num; } f; + *((double*)&f.buf) = d; // FIXME + const unsigned char buf[9] = {0xcb, STORE_BE64(f.num)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -569,7 +571,7 @@ msgpack_pack_inline_func(double)(msgpack_pack_user x, double d) * Nil */ -msgpack_pack_inline_func(nil)(msgpack_pack_user x) +msgpack_pack_inline_func(_nil)(msgpack_pack_user x) { static const unsigned char d = 0xc0; msgpack_pack_append_buffer(x, &d, 1); @@ -580,13 +582,13 @@ msgpack_pack_inline_func(nil)(msgpack_pack_user x) * Boolean */ -msgpack_pack_inline_func(true)(msgpack_pack_user x) +msgpack_pack_inline_func(_true)(msgpack_pack_user x) { static const unsigned char d = 0xc3; msgpack_pack_append_buffer(x, &d, 1); } -msgpack_pack_inline_func(false)(msgpack_pack_user x) +msgpack_pack_inline_func(_false)(msgpack_pack_user x) { static const unsigned char d = 0xc2; msgpack_pack_append_buffer(x, &d, 1); @@ -597,7 +599,7 @@ msgpack_pack_inline_func(false)(msgpack_pack_user x) * Array */ -msgpack_pack_inline_func(array)(msgpack_pack_user x, unsigned int n) +msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x90 | n; @@ -618,7 +620,7 @@ msgpack_pack_inline_func(array)(msgpack_pack_user x, unsigned int n) * Map */ -msgpack_pack_inline_func(map)(msgpack_pack_user x, unsigned int n) +msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; @@ -639,7 +641,7 @@ msgpack_pack_inline_func(map)(msgpack_pack_user x, unsigned int n) * Raw */ -msgpack_pack_inline_func(raw)(msgpack_pack_user x, size_t l) +msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; @@ -655,7 +657,7 @@ msgpack_pack_inline_func(raw)(msgpack_pack_user x, size_t l) } } -msgpack_pack_inline_func(raw_body)(msgpack_pack_user x, const void* b, size_t l) +msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l) { msgpack_pack_append_buffer(x, (const unsigned char*)b, l); } diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index f639841..63668c2 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index a6d7dcb..a45369b 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,38 +41,38 @@ #endif -msgpack_unpack_struct_decl(stack) { +msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; size_t count; unsigned int ct; msgpack_unpack_object map_key; }; -msgpack_unpack_struct_decl(context) { +msgpack_unpack_struct_decl(_context) { msgpack_unpack_user user; // must be first unsigned int cs; unsigned int trail; unsigned int top; - msgpack_unpack_struct(stack) stack[MSGPACK_MAX_STACK_SIZE]; + msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; }; -msgpack_unpack_func(void, init)(msgpack_unpack_struct(context)* ctx) +msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) { - /*memset(ctx, 0, sizeof( msgpack_unpack_struct(context) )); FIXME needed? */ + /*memset(ctx, 0, sizeof( msgpack_unpack_struct(_context) )); FIXME needed? */ ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; - ctx->stack[0].obj = msgpack_unpack_callback(init)(&ctx->user); + ctx->stack[0].obj = msgpack_unpack_callback(_init)(&ctx->user); } -msgpack_unpack_func(msgpack_unpack_object, data)(msgpack_unpack_struct(context)* unpacker) +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* unpacker) { return (unpacker)->stack[0].obj; } -msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const char* data, size_t len, size_t* off) +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) { assert(len >= *off); @@ -83,11 +83,11 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha unsigned int trail = ctx->trail; unsigned int cs = ctx->cs; unsigned int top = ctx->top; - msgpack_unpack_struct(stack)* stack = ctx->stack; + msgpack_unpack_struct(_stack)* stack = ctx->stack; msgpack_unpack_user* user = &ctx->user; msgpack_unpack_object obj; - msgpack_unpack_struct(stack)* c = NULL; + msgpack_unpack_struct(_stack)* c = NULL; int ret; @@ -139,19 +139,19 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha case CS_HEADER: switch(*p) { case 0x00 ... 0x7f: // Positive Fixnum - push_fixed_value(uint8, *(uint8_t*)p); + push_fixed_value(_uint8, *(uint8_t*)p); case 0xe0 ... 0xff: // Negative Fixnum - push_fixed_value(int8, *(int8_t*)p); + push_fixed_value(_int8, *(int8_t*)p); case 0xc0 ... 0xdf: // Variable switch(*p) { case 0xc0: // nil - push_simple_value(nil); + push_simple_value(_nil); //case 0xc1: // string // again_terminal_trail(NEXT_CS(p), p+1); case 0xc2: // false - push_simple_value(false); + push_simple_value(_false); case 0xc3: // true - push_simple_value(true); + push_simple_value(_true); //case 0xc4: //case 0xc5: //case 0xc6: @@ -188,9 +188,9 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha case 0xa0 ... 0xbf: // FixRaw again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); case 0x90 ... 0x9f: // FixArray - start_container(array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); case 0x80 ... 0x8f: // FixMap - start_container(map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); default: goto _failed; @@ -208,28 +208,30 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha //case CS_ //case CS_ case CS_FLOAT: { - uint32_t x = PTR_CAST_32(n); // FIXME - push_fixed_value(float, *((float*)&x)); } + union { uint32_t num; char buf[4]; } f; + f.num = PTR_CAST_32(n); // FIXME + push_fixed_value(_float, *((float*)f.buf)); } case CS_DOUBLE: { - uint64_t x = PTR_CAST_64(n); // FIXME - push_fixed_value(double, *((double*)&x)); } + union { uint64_t num; char buf[8]; } f; + f.num = PTR_CAST_64(n); // FIXME + push_fixed_value(_double, *((double*)f.buf)); } case CS_UINT_8: - push_fixed_value(uint8, (uint8_t)PTR_CAST_8(n)); + push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); case CS_UINT_16: - push_fixed_value(uint16, (uint16_t)PTR_CAST_16(n)); + push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); case CS_UINT_32: - push_fixed_value(uint32, (uint32_t)PTR_CAST_32(n)); + push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); case CS_UINT_64: - push_fixed_value(uint64, (uint64_t)PTR_CAST_64(n)); + push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); case CS_INT_8: - push_fixed_value(int8, (int8_t)PTR_CAST_8(n)); + push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); case CS_INT_16: - push_fixed_value(int16, (int16_t)PTR_CAST_16(n)); + push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); case CS_INT_32: - push_fixed_value(int32, (int32_t)PTR_CAST_32(n)); + push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); case CS_INT_64: - push_fixed_value(int64, (int64_t)PTR_CAST_64(n)); + push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); //case CS_ //case CS_ @@ -240,7 +242,7 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha //case ACS_BIG_INT_VALUE: //_big_int_zero: // // FIXME - // push_variable_value(big_int, data, n, trail); + // push_variable_value(_big_int, data, n, trail); //case CS_BIG_FLOAT_16: // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); @@ -249,7 +251,7 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha //case ACS_BIG_FLOAT_VALUE: //_big_float_zero: // // FIXME - // push_variable_value(big_float, data, n, trail); + // push_variable_value(_big_float, data, n, trail); case CS_RAW_16: again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); @@ -257,17 +259,17 @@ msgpack_unpack_func(int, execute)(msgpack_unpack_struct(context)* ctx, const cha again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); case ACS_RAW_VALUE: _raw_zero: - push_variable_value(raw, data, n, trail); + push_variable_value(_raw, data, n, trail); case CS_ARRAY_16: - start_container(array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); case CS_ARRAY_32: - start_container(array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); case CS_MAP_16: - start_container(map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); case CS_MAP_32: - start_container(map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); default: goto _failed; @@ -279,7 +281,7 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - msgpack_unpack_callback(array_item)(user, &c->obj, obj); + msgpack_unpack_callback(_array_item)(user, &c->obj, obj); if(--c->count == 0) { obj = c->obj; --top; @@ -292,7 +294,7 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - msgpack_unpack_callback(map_item)(user, &c->obj, c->map_key, obj); + msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj); if(--c->count == 0) { obj = c->obj; --top; diff --git a/ruby/pack.c b/ruby/pack.c index c399c25..0a3711f 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -1,7 +1,7 @@ /* - * MessagePack packing routine for Ruby + * MessagePack for Ruby packing routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,10 @@ #include "msgpack/pack_define.h" #define msgpack_pack_inline_func(name) \ - static inline void msgpack_pack_##name + static inline void msgpack_pack ## name #define msgpack_pack_inline_func_cint(name) \ - static inline void msgpack_pack_##name + static inline void msgpack_pack ## name #define msgpack_pack_user VALUE diff --git a/ruby/pack.h b/ruby/pack.h index c38ac48..c9b08a4 100644 --- a/ruby/pack.h +++ b/ruby/pack.h @@ -1,7 +1,7 @@ /* - * MessagePack packing routine for Ruby + * MessagePack for Ruby packing routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 7ef92fb..80d1d8c 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -1,7 +1,7 @@ /* * MessagePack for Ruby * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ruby/test_case.rb b/ruby/test_case.rb index af6d160..1e2d474 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -75,6 +75,13 @@ class MessagePackTestFormat < Test::Unit::TestCase check 9, -(1<<63) end + it "double" do + check 9, 1.0 + check 9, 0.1 + check 9, -0.1 + check 9, -1.0 + end + it "fixraw" do check_raw 1, 0 check_raw 1, (1<<5)-1 diff --git a/ruby/unpack.c b/ruby/unpack.c index 2920240..4650ed1 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -1,7 +1,7 @@ /* - * MessagePack unpacking routine for Ruby + * MessagePack for Ruby unpacking routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,13 +26,13 @@ typedef struct { #define msgpack_unpack_struct(name) \ - struct msgpack_unpacker_##name + struct msgpack_unpacker ## name #define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker_##name + ret msgpack_unpacker ## name #define msgpack_unpack_callback(name) \ - template_callback_##name + template_callback ## name #define msgpack_unpack_object VALUE diff --git a/ruby/unpack.h b/ruby/unpack.h index 0fe01ec..ce2a8de 100644 --- a/ruby/unpack.h +++ b/ruby/unpack.h @@ -1,7 +1,7 @@ /* - * MessagePack unpacking routine for Ruby + * MessagePack for Ruby unpacking routine * - * Copyright (C) 2008 FURUHASHI Sadayuki + * Copyright (C) 2008-2009 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 01d40bbb765a30d32c24298a2b666152de8a283f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:02 +0000 Subject: [PATCH 0054/1648] fix c/configure.in git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@101 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- c/configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c/configure.in b/c/configure.in index 0017917..47eb8aa 100644 --- a/c/configure.in +++ b/c/configure.in @@ -1,4 +1,4 @@ -AC_INIT(pack.c) +AC_INIT(unpack.c) AM_INIT_AUTOMAKE(msgpackc, 0.1.0) AC_CONFIG_HEADER(config.h) From 6083300ea949855e1b2cd3cc4e92bf340c4e90e7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 09:10:02 +0000 Subject: [PATCH 0055/1648] ruby binding: simplify gem package skeleton git-svn-id: file:///Users/frsyuki/project/msgpack-git/svn/x@102 5a5092ae-2292-43ba-b2d5-dcab9c1a2731 --- ruby/gem/{History.txt => ChangeLog} | 0 ruby/gem/License.txt | 13 - ruby/gem/Manifest.txt | 23 - ruby/gem/PostInstall.txt | 1 - ruby/gem/README | 29 + ruby/gem/README.txt | 23 - ruby/gem/Rakefile | 137 ++- ruby/gem/config/hoe.rb | 75 -- ruby/gem/config/requirements.rb | 15 - ruby/gem/lib/msgpack/version.rb | 9 - ruby/gem/script/console | 10 - ruby/gem/script/destroy | 14 - ruby/gem/script/generate | 14 - ruby/gem/script/txt2html | 82 -- ruby/gem/setup.rb | 1585 --------------------------- ruby/gem/tasks/deployment.rake | 34 - ruby/gem/tasks/environment.rake | 7 - ruby/gem/test/test_helper.rb | 3 + ruby/gengem.sh | 1 + ruby/test_case.rb | 3 +- 20 files changed, 168 insertions(+), 1910 deletions(-) rename ruby/gem/{History.txt => ChangeLog} (100%) delete mode 100644 ruby/gem/License.txt delete mode 100644 ruby/gem/Manifest.txt delete mode 100644 ruby/gem/PostInstall.txt create mode 100644 ruby/gem/README delete mode 100644 ruby/gem/README.txt delete mode 100644 ruby/gem/config/hoe.rb delete mode 100644 ruby/gem/config/requirements.rb delete mode 100644 ruby/gem/lib/msgpack/version.rb delete mode 100755 ruby/gem/script/console delete mode 100755 ruby/gem/script/destroy delete mode 100755 ruby/gem/script/generate delete mode 100755 ruby/gem/script/txt2html delete mode 100644 ruby/gem/setup.rb delete mode 100644 ruby/gem/tasks/deployment.rake delete mode 100644 ruby/gem/tasks/environment.rake create mode 100644 ruby/gem/test/test_helper.rb diff --git a/ruby/gem/History.txt b/ruby/gem/ChangeLog similarity index 100% rename from ruby/gem/History.txt rename to ruby/gem/ChangeLog diff --git a/ruby/gem/License.txt b/ruby/gem/License.txt deleted file mode 100644 index f4000b7..0000000 --- a/ruby/gem/License.txt +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2008 Furuhashi Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/ruby/gem/Manifest.txt b/ruby/gem/Manifest.txt deleted file mode 100644 index 17e2b78..0000000 --- a/ruby/gem/Manifest.txt +++ /dev/null @@ -1,23 +0,0 @@ -License.txt -Manifest.txt -README.txt -Rakefile -config/hoe.rb -config/requirements.rb -ext/extconf.rb -ext/pack.c -ext/pack.h -ext/rbinit.c -ext/unpack.c -ext/unpack.h -msgpack/pack_define.h -msgpack/pack_template.h -msgpack/unpack_define.h -msgpack/unpack_template.h -lib/msgpack/version.rb -script/console -script/destroy -script/generate -setup.rb -tasks/deployment.rake -tasks/environment.rake diff --git a/ruby/gem/PostInstall.txt b/ruby/gem/PostInstall.txt deleted file mode 100644 index 8b13789..0000000 --- a/ruby/gem/PostInstall.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ruby/gem/README b/ruby/gem/README new file mode 100644 index 0000000..78fe6cb --- /dev/null +++ b/ruby/gem/README @@ -0,0 +1,29 @@ + += MessagePack + + +== Description + + +== Installation + +=== Archive Installation + + rake install + +=== Gem Installation + + gem install msgpack + + +== Features/Problems + + +== Synopsis + + +== Copyright + +Author:: frsyuki +Copyright:: Copyright (c) 2009 frsyuki +License:: Apache License, Version 2.0 diff --git a/ruby/gem/README.txt b/ruby/gem/README.txt deleted file mode 100644 index 7c24b8f..0000000 --- a/ruby/gem/README.txt +++ /dev/null @@ -1,23 +0,0 @@ -=MessagePack - -== DESCRIPTION: - -Binary-based efficient data interchange format. - - -== LICENSE: - -Copyright (C) 2008 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile index e469154..87635cb 100644 --- a/ruby/gem/Rakefile +++ b/ruby/gem/Rakefile @@ -1,4 +1,133 @@ -require 'config/requirements' -require 'config/hoe' # setup Hoe + all gem configuration - -Dir['tasks/**/*.rake'].each { |rake| load rake } \ No newline at end of file +require 'rubygems' +require 'rake' +require 'rake/clean' +require 'rake/testtask' +require 'rake/packagetask' +require 'rake/gempackagetask' +require 'rake/rdoctask' +require 'rake/contrib/rubyforgepublisher' +require 'rake/contrib/sshpublisher' +require 'fileutils' +include FileUtils + +NAME = "msgpack" +AUTHOR = "FURUHASHI Sadayuki" +EMAIL = "frsyuki _at_ users.sourceforge.jp" +DESCRIPTION = "Binary-based efficient data interchange format." +RUBYFORGE_PROJECT = "msgpack" +HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" +BIN_FILES = %w( ) +VERS = "0.3.0" + +#REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil +REV = nil +CLEAN.include ['**/.*.sw?', '*.gem', '.config'] +RDOC_OPTS = [ + '--title', "#{NAME} documentation", + "--charset", "utf-8", + "--opname", "index.html", + "--line-numbers", + "--main", "README", + "--inline-source", +] + +task :default => [:test] +task :package => [:clean] + +Rake::TestTask.new("test") do |t| + t.libs << "test" + t.pattern = "test/**/*_test.rb" + t.verbose = true +end + +spec = Gem::Specification.new do |s| + s.name = NAME + s.version = VERS + s.platform = Gem::Platform::RUBY + s.has_rdoc = true + s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] + s.rdoc_options += RDOC_OPTS + ['--exclude', '^(examples|extras)/'] + s.summary = DESCRIPTION + s.description = DESCRIPTION + s.author = AUTHOR + s.email = EMAIL + s.homepage = HOMEPATH + s.executables = BIN_FILES + s.rubyforge_project = RUBYFORGE_PROJECT + s.bindir = "bin" + s.require_path = "ext" + s.autorequire = "" + s.test_files = Dir["test/test_*.rb"] + + #s.add_dependency('activesupport', '>=1.3.1') + #s.required_ruby_version = '>= 1.8.2' + + s.files = %w(README ChangeLog Rakefile) + + Dir.glob("{bin,doc,test,lib,templates,generator,extras,website,script}/**/*") + + Dir.glob("ext/**/*.{h,c,rb}") + + Dir.glob("examples/**/*.rb") + + Dir.glob("tools/*.rb") + + Dir.glob("msgpack/*.h") + + s.extensions = FileList["ext/**/extconf.rb"].to_a +end + +Rake::GemPackageTask.new(spec) do |p| + p.need_tar = true + p.gem_spec = spec +end + +task :install do + name = "#{NAME}-#{VERS}.gem" + sh %{rake package} + sh %{sudo gem install pkg/#{name}} +end + +task :uninstall => [:clean] do + sh %{sudo gem uninstall #{NAME}} +end + + +#Rake::RDocTask.new do |rdoc| +# rdoc.rdoc_dir = 'html' +# rdoc.options += RDOC_OPTS +# rdoc.template = "resh" +# #rdoc.template = "#{ENV['template']}.rb" if ENV['template'] +# if ENV['DOC_FILES'] +# rdoc.rdoc_files.include(ENV['DOC_FILES'].split(/,\s*/)) +# else +# rdoc.rdoc_files.include('README', 'ChangeLog') +# rdoc.rdoc_files.include('lib/**/*.rb') +# rdoc.rdoc_files.include('ext/**/*.c') +# end +#end + +desc "Publish to RubyForge" +task :rubyforge => [:rdoc, :package] do + require 'rubyforge' + Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, 'frsyuki').upload +end + +desc 'Package and upload the release to rubyforge.' +task :release => [:clean, :package] do |t| + v = ENV["VERSION"] or abort "Must supply VERSION=x.y.z" + abort "Versions don't match #{v} vs #{VERS}" unless v == VERS + pkg = "pkg/#{NAME}-#{VERS}" + + rf = RubyForge.new + puts "Logging in" + rf.login + + c = rf.userconfig +# c["release_notes"] = description if description +# c["release_changes"] = changes if changes + c["preformatted"] = true + + files = [ + "#{pkg}.tgz", + "#{pkg}.gem" + ].compact + + puts "Releasing #{NAME} v. #{VERS}" + rf.add_release RUBYFORGE_PROJECT, NAME, VERS, *files +end diff --git a/ruby/gem/config/hoe.rb b/ruby/gem/config/hoe.rb deleted file mode 100644 index 8500acf..0000000 --- a/ruby/gem/config/hoe.rb +++ /dev/null @@ -1,75 +0,0 @@ -require 'msgpack/version' - -AUTHOR = 'FURUHASHI Sadayuki' # can also be an array of Authors -EMAIL = "fr _at_ syuki.skr.jp" -DESCRIPTION = "An object-oriented parser generator based on Parser Expression Grammar" -GEM_NAME = 'msgpack' # what ppl will type to install your gem -RUBYFORGE_PROJECT = 'msgpack' # The unix name for your project -HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" -DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}" -EXTRA_DEPENDENCIES = [ -# ['activesupport', '>= 1.3.1'] -] # An array of rubygem dependencies [name, version] - -@config_file = "~/.rubyforge/user-config.yml" -@config = nil -RUBYFORGE_USERNAME = "unknown" -def rubyforge_username - unless @config - begin - @config = YAML.load(File.read(File.expand_path(@config_file))) - rescue - puts <<-EOS -ERROR: No rubyforge config file found: #{@config_file} -Run 'rubyforge setup' to prepare your env for access to Rubyforge - - See http://newgem.rubyforge.org/rubyforge.html for more details - EOS - exit - end - end - RUBYFORGE_USERNAME.replace @config["username"] -end - - -REV = nil -# UNCOMMENT IF REQUIRED: -# REV = YAML.load(`svn info`)['Revision'] -VERS = MessagePack::VERSION::STRING + (REV ? ".#{REV}" : "") -RDOC_OPTS = ['--quiet', '--title', 'msgpack documentation', - "--opname", "index.html", - "--line-numbers", - "--main", "README", - "--inline-source"] - -class Hoe - def extra_deps - @extra_deps.reject! { |x| Array(x).first == 'hoe' } - @extra_deps - end -end - -# Generate all the Rake tasks -# Run 'rake -T' to see list of generated tasks (from gem root directory) -$hoe = Hoe.new(GEM_NAME, VERS) do |p| - p.developer(AUTHOR, EMAIL) - p.description = DESCRIPTION - p.summary = DESCRIPTION - p.url = HOMEPATH - p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT - p.test_globs = ["test/**/test_*.rb"] - p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store'] #An array of file patterns to delete on clean. - - # == Optional - p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n") - #p.extra_deps = EXTRA_DEPENDENCIES - - p.spec_extras = { # A hash of extra values to set in the gemspec. - :extensions => %w[ext/extconf.rb] - } -end - -CHANGES = $hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n") -PATH = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}" -$hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc') -$hoe.rsync_args = '-av --delete --ignore-errors' -$hoe.spec.post_install_message = File.open(File.dirname(__FILE__) + "/../PostInstall.txt").read rescue "" diff --git a/ruby/gem/config/requirements.rb b/ruby/gem/config/requirements.rb deleted file mode 100644 index 9292b69..0000000 --- a/ruby/gem/config/requirements.rb +++ /dev/null @@ -1,15 +0,0 @@ -require 'fileutils' -include FileUtils - -require 'rubygems' -%w[rake hoe newgem rubigen].each do |req_gem| - begin - require req_gem - rescue LoadError - puts "This Rakefile requires the '#{req_gem}' RubyGem." - puts "Installation: gem install #{req_gem} -y" - exit - end -end - -$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib])) diff --git a/ruby/gem/lib/msgpack/version.rb b/ruby/gem/lib/msgpack/version.rb deleted file mode 100644 index 433e6fc..0000000 --- a/ruby/gem/lib/msgpack/version.rb +++ /dev/null @@ -1,9 +0,0 @@ -module MessagePack - module VERSION #:nodoc: - MAJOR = 0 - MINOR = 3 - TINY = 0 - - STRING = [MAJOR, MINOR, TINY].join('.') - end -end diff --git a/ruby/gem/script/console b/ruby/gem/script/console deleted file mode 100755 index 76f32a0..0000000 --- a/ruby/gem/script/console +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby -# File: script/console -irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb' - -libs = " -r irb/completion" -# Perhaps use a console_lib to store any extra methods I may want available in the cosole -# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}" -libs << " -r #{File.dirname(__FILE__) + '/../lib/msgpack.rb'}" -puts "Loading msgpack gem" -exec "#{irb} #{libs} --simple-prompt" diff --git a/ruby/gem/script/destroy b/ruby/gem/script/destroy deleted file mode 100755 index e48464d..0000000 --- a/ruby/gem/script/destroy +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env ruby -APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) - -begin - require 'rubigen' -rescue LoadError - require 'rubygems' - require 'rubigen' -end -require 'rubigen/scripts/destroy' - -ARGV.shift if ['--help', '-h'].include?(ARGV[0]) -RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit] -RubiGen::Scripts::Destroy.new.run(ARGV) diff --git a/ruby/gem/script/generate b/ruby/gem/script/generate deleted file mode 100755 index c27f655..0000000 --- a/ruby/gem/script/generate +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env ruby -APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..')) - -begin - require 'rubigen' -rescue LoadError - require 'rubygems' - require 'rubigen' -end -require 'rubigen/scripts/generate' - -ARGV.shift if ['--help', '-h'].include?(ARGV[0]) -RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit] -RubiGen::Scripts::Generate.new.run(ARGV) diff --git a/ruby/gem/script/txt2html b/ruby/gem/script/txt2html deleted file mode 100755 index 09c583f..0000000 --- a/ruby/gem/script/txt2html +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env ruby - -GEM_NAME = 'msgpack' # what ppl will type to install your gem -RUBYFORGE_PROJECT = 'msgpack' - -require 'rubygems' -begin - require 'newgem' - require 'rubyforge' -rescue LoadError - puts "\n\nGenerating the website requires the newgem RubyGem" - puts "Install: gem install newgem\n\n" - exit(1) -end -require 'redcloth' -require 'syntax/convertors/html' -require 'erb' -require File.dirname(__FILE__) + "/../lib/#{GEM_NAME}/version.rb" - -version = MessagePack::VERSION::STRING -download = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}" - -def rubyforge_project_id - RubyForge.new.autoconfig["group_ids"][RUBYFORGE_PROJECT] -end - -class Fixnum - def ordinal - # teens - return 'th' if (10..19).include?(self % 100) - # others - case self % 10 - when 1: return 'st' - when 2: return 'nd' - when 3: return 'rd' - else return 'th' - end - end -end - -class Time - def pretty - return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}" - end -end - -def convert_syntax(syntax, source) - return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^
|
$!,'') -end - -if ARGV.length >= 1 - src, template = ARGV - template ||= File.join(File.dirname(__FILE__), '/../website/template.html.erb') -else - puts("Usage: #{File.split($0).last} source.txt [template.html.erb] > output.html") - exit! -end - -template = ERB.new(File.open(template).read) - -title = nil -body = nil -File.open(src) do |fsrc| - title_text = fsrc.readline - body_text_template = fsrc.read - body_text = ERB.new(body_text_template).result(binding) - syntax_items = [] - body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)!m){ - ident = syntax_items.length - element, syntax, source = $1, $2, $3 - syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}" - "syntax-temp-#{ident}" - } - title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip - body = RedCloth.new(body_text).to_html - body.gsub!(%r!(?:
)?syntax-temp-(\d+)(?:
)?!){ syntax_items[$1.to_i] } -end -stat = File.stat(src) -created = stat.ctime -modified = stat.mtime - -$stdout << template.result(binding) diff --git a/ruby/gem/setup.rb b/ruby/gem/setup.rb deleted file mode 100644 index 424a5f3..0000000 --- a/ruby/gem/setup.rb +++ /dev/null @@ -1,1585 +0,0 @@ -# -# setup.rb -# -# Copyright (c) 2000-2005 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the terms of -# the GNU LGPL, Lesser General Public License version 2.1. -# - -unless Enumerable.method_defined?(:map) # Ruby 1.4.6 - module Enumerable - alias map collect - end -end - -unless File.respond_to?(:read) # Ruby 1.6 - def File.read(fname) - open(fname) {|f| - return f.read - } - end -end - -unless Errno.const_defined?(:ENOTEMPTY) # Windows? - module Errno - class ENOTEMPTY - # We do not raise this exception, implementation is not needed. - end - end -end - -def File.binread(fname) - open(fname, 'rb') {|f| - return f.read - } -end - -# for corrupted Windows' stat(2) -def File.dir?(path) - File.directory?((path[-1,1] == '/') ? path : path + '/') -end - - -class ConfigTable - - include Enumerable - - def initialize(rbconfig) - @rbconfig = rbconfig - @items = [] - @table = {} - # options - @install_prefix = nil - @config_opt = nil - @verbose = true - @no_harm = false - end - - attr_accessor :install_prefix - attr_accessor :config_opt - - attr_writer :verbose - - def verbose? - @verbose - end - - attr_writer :no_harm - - def no_harm? - @no_harm - end - - def [](key) - lookup(key).resolve(self) - end - - def []=(key, val) - lookup(key).set val - end - - def names - @items.map {|i| i.name } - end - - def each(&block) - @items.each(&block) - end - - def key?(name) - @table.key?(name) - end - - def lookup(name) - @table[name] or setup_rb_error "no such config item: #{name}" - end - - def add(item) - @items.push item - @table[item.name] = item - end - - def remove(name) - item = lookup(name) - @items.delete_if {|i| i.name == name } - @table.delete_if {|name, i| i.name == name } - item - end - - def load_script(path, inst = nil) - if File.file?(path) - MetaConfigEnvironment.new(self, inst).instance_eval File.read(path), path - end - end - - def savefile - '.config' - end - - def load_savefile - begin - File.foreach(savefile()) do |line| - k, v = *line.split(/=/, 2) - self[k] = v.strip - end - rescue Errno::ENOENT - setup_rb_error $!.message + "\n#{File.basename($0)} config first" - end - end - - def save - @items.each {|i| i.value } - File.open(savefile(), 'w') {|f| - @items.each do |i| - f.printf "%s=%s\n", i.name, i.value if i.value? and i.value - end - } - end - - def load_standard_entries - standard_entries(@rbconfig).each do |ent| - add ent - end - end - - def standard_entries(rbconfig) - c = rbconfig - - rubypath = File.join(c['bindir'], c['ruby_install_name'] + c['EXEEXT']) - - major = c['MAJOR'].to_i - minor = c['MINOR'].to_i - teeny = c['TEENY'].to_i - version = "#{major}.#{minor}" - - # ruby ver. >= 1.4.4? - newpath_p = ((major >= 2) or - ((major == 1) and - ((minor >= 5) or - ((minor == 4) and (teeny >= 4))))) - - if c['rubylibdir'] - # V > 1.6.3 - libruby = "#{c['prefix']}/lib/ruby" - librubyver = c['rubylibdir'] - librubyverarch = c['archdir'] - siteruby = c['sitedir'] - siterubyver = c['sitelibdir'] - siterubyverarch = c['sitearchdir'] - elsif newpath_p - # 1.4.4 <= V <= 1.6.3 - libruby = "#{c['prefix']}/lib/ruby" - librubyver = "#{c['prefix']}/lib/ruby/#{version}" - librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" - siteruby = c['sitedir'] - siterubyver = "$siteruby/#{version}" - siterubyverarch = "$siterubyver/#{c['arch']}" - else - # V < 1.4.4 - libruby = "#{c['prefix']}/lib/ruby" - librubyver = "#{c['prefix']}/lib/ruby/#{version}" - librubyverarch = "#{c['prefix']}/lib/ruby/#{version}/#{c['arch']}" - siteruby = "#{c['prefix']}/lib/ruby/#{version}/site_ruby" - siterubyver = siteruby - siterubyverarch = "$siterubyver/#{c['arch']}" - end - parameterize = lambda {|path| - path.sub(/\A#{Regexp.quote(c['prefix'])}/, '$prefix') - } - - if arg = c['configure_args'].split.detect {|arg| /--with-make-prog=/ =~ arg } - makeprog = arg.sub(/'/, '').split(/=/, 2)[1] - else - makeprog = 'make' - end - - [ - ExecItem.new('installdirs', 'std/site/home', - 'std: install under libruby; site: install under site_ruby; home: install under $HOME')\ - {|val, table| - case val - when 'std' - table['rbdir'] = '$librubyver' - table['sodir'] = '$librubyverarch' - when 'site' - table['rbdir'] = '$siterubyver' - table['sodir'] = '$siterubyverarch' - when 'home' - setup_rb_error '$HOME was not set' unless ENV['HOME'] - table['prefix'] = ENV['HOME'] - table['rbdir'] = '$libdir/ruby' - table['sodir'] = '$libdir/ruby' - end - }, - PathItem.new('prefix', 'path', c['prefix'], - 'path prefix of target environment'), - PathItem.new('bindir', 'path', parameterize.call(c['bindir']), - 'the directory for commands'), - PathItem.new('libdir', 'path', parameterize.call(c['libdir']), - 'the directory for libraries'), - PathItem.new('datadir', 'path', parameterize.call(c['datadir']), - 'the directory for shared data'), - PathItem.new('mandir', 'path', parameterize.call(c['mandir']), - 'the directory for man pages'), - PathItem.new('sysconfdir', 'path', parameterize.call(c['sysconfdir']), - 'the directory for system configuration files'), - PathItem.new('localstatedir', 'path', parameterize.call(c['localstatedir']), - 'the directory for local state data'), - PathItem.new('libruby', 'path', libruby, - 'the directory for ruby libraries'), - PathItem.new('librubyver', 'path', librubyver, - 'the directory for standard ruby libraries'), - PathItem.new('librubyverarch', 'path', librubyverarch, - 'the directory for standard ruby extensions'), - PathItem.new('siteruby', 'path', siteruby, - 'the directory for version-independent aux ruby libraries'), - PathItem.new('siterubyver', 'path', siterubyver, - 'the directory for aux ruby libraries'), - PathItem.new('siterubyverarch', 'path', siterubyverarch, - 'the directory for aux ruby binaries'), - PathItem.new('rbdir', 'path', '$siterubyver', - 'the directory for ruby scripts'), - PathItem.new('sodir', 'path', '$siterubyverarch', - 'the directory for ruby extentions'), - PathItem.new('rubypath', 'path', rubypath, - 'the path to set to #! line'), - ProgramItem.new('rubyprog', 'name', rubypath, - 'the ruby program using for installation'), - ProgramItem.new('makeprog', 'name', makeprog, - 'the make program to compile ruby extentions'), - SelectItem.new('shebang', 'all/ruby/never', 'ruby', - 'shebang line (#!) editing mode'), - BoolItem.new('without-ext', 'yes/no', 'no', - 'does not compile/install ruby extentions') - ] - end - private :standard_entries - - def load_multipackage_entries - multipackage_entries().each do |ent| - add ent - end - end - - def multipackage_entries - [ - PackageSelectionItem.new('with', 'name,name...', '', 'ALL', - 'package names that you want to install'), - PackageSelectionItem.new('without', 'name,name...', '', 'NONE', - 'package names that you do not want to install') - ] - end - private :multipackage_entries - - ALIASES = { - 'std-ruby' => 'librubyver', - 'stdruby' => 'librubyver', - 'rubylibdir' => 'librubyver', - 'archdir' => 'librubyverarch', - 'site-ruby-common' => 'siteruby', # For backward compatibility - 'site-ruby' => 'siterubyver', # For backward compatibility - 'bin-dir' => 'bindir', - 'bin-dir' => 'bindir', - 'rb-dir' => 'rbdir', - 'so-dir' => 'sodir', - 'data-dir' => 'datadir', - 'ruby-path' => 'rubypath', - 'ruby-prog' => 'rubyprog', - 'ruby' => 'rubyprog', - 'make-prog' => 'makeprog', - 'make' => 'makeprog' - } - - def fixup - ALIASES.each do |ali, name| - @table[ali] = @table[name] - end - @items.freeze - @table.freeze - @options_re = /\A--(#{@table.keys.join('|')})(?:=(.*))?\z/ - end - - def parse_opt(opt) - m = @options_re.match(opt) or setup_rb_error "config: unknown option #{opt}" - m.to_a[1,2] - end - - def dllext - @rbconfig['DLEXT'] - end - - def value_config?(name) - lookup(name).value? - end - - class Item - def initialize(name, template, default, desc) - @name = name.freeze - @template = template - @value = default - @default = default - @description = desc - end - - attr_reader :name - attr_reader :description - - attr_accessor :default - alias help_default default - - def help_opt - "--#{@name}=#{@template}" - end - - def value? - true - end - - def value - @value - end - - def resolve(table) - @value.gsub(%r<\$([^/]+)>) { table[$1] } - end - - def set(val) - @value = check(val) - end - - private - - def check(val) - setup_rb_error "config: --#{name} requires argument" unless val - val - end - end - - class BoolItem < Item - def config_type - 'bool' - end - - def help_opt - "--#{@name}" - end - - private - - def check(val) - return 'yes' unless val - case val - when /\Ay(es)?\z/i, /\At(rue)?\z/i then 'yes' - when /\An(o)?\z/i, /\Af(alse)\z/i then 'no' - else - setup_rb_error "config: --#{@name} accepts only yes/no for argument" - end - end - end - - class PathItem < Item - def config_type - 'path' - end - - private - - def check(path) - setup_rb_error "config: --#{@name} requires argument" unless path - path[0,1] == '$' ? path : File.expand_path(path) - end - end - - class ProgramItem < Item - def config_type - 'program' - end - end - - class SelectItem < Item - def initialize(name, selection, default, desc) - super - @ok = selection.split('/') - end - - def config_type - 'select' - end - - private - - def check(val) - unless @ok.include?(val.strip) - setup_rb_error "config: use --#{@name}=#{@template} (#{val})" - end - val.strip - end - end - - class ExecItem < Item - def initialize(name, selection, desc, &block) - super name, selection, nil, desc - @ok = selection.split('/') - @action = block - end - - def config_type - 'exec' - end - - def value? - false - end - - def resolve(table) - setup_rb_error "$#{name()} wrongly used as option value" - end - - undef set - - def evaluate(val, table) - v = val.strip.downcase - unless @ok.include?(v) - setup_rb_error "invalid option --#{@name}=#{val} (use #{@template})" - end - @action.call v, table - end - end - - class PackageSelectionItem < Item - def initialize(name, template, default, help_default, desc) - super name, template, default, desc - @help_default = help_default - end - - attr_reader :help_default - - def config_type - 'package' - end - - private - - def check(val) - unless File.dir?("packages/#{val}") - setup_rb_error "config: no such package: #{val}" - end - val - end - end - - class MetaConfigEnvironment - def initialize(config, installer) - @config = config - @installer = installer - end - - def config_names - @config.names - end - - def config?(name) - @config.key?(name) - end - - def bool_config?(name) - @config.lookup(name).config_type == 'bool' - end - - def path_config?(name) - @config.lookup(name).config_type == 'path' - end - - def value_config?(name) - @config.lookup(name).config_type != 'exec' - end - - def add_config(item) - @config.add item - end - - def add_bool_config(name, default, desc) - @config.add BoolItem.new(name, 'yes/no', default ? 'yes' : 'no', desc) - end - - def add_path_config(name, default, desc) - @config.add PathItem.new(name, 'path', default, desc) - end - - def set_config_default(name, default) - @config.lookup(name).default = default - end - - def remove_config(name) - @config.remove(name) - end - - # For only multipackage - def packages - raise '[setup.rb fatal] multi-package metaconfig API packages() called for single-package; contact application package vendor' unless @installer - @installer.packages - end - - # For only multipackage - def declare_packages(list) - raise '[setup.rb fatal] multi-package metaconfig API declare_packages() called for single-package; contact application package vendor' unless @installer - @installer.packages = list - end - end - -end # class ConfigTable - - -# This module requires: #verbose?, #no_harm? -module FileOperations - - def mkdir_p(dirname, prefix = nil) - dirname = prefix + File.expand_path(dirname) if prefix - $stderr.puts "mkdir -p #{dirname}" if verbose? - return if no_harm? - - # Does not check '/', it's too abnormal. - dirs = File.expand_path(dirname).split(%r<(?=/)>) - if /\A[a-z]:\z/i =~ dirs[0] - disk = dirs.shift - dirs[0] = disk + dirs[0] - end - dirs.each_index do |idx| - path = dirs[0..idx].join('') - Dir.mkdir path unless File.dir?(path) - end - end - - def rm_f(path) - $stderr.puts "rm -f #{path}" if verbose? - return if no_harm? - force_remove_file path - end - - def rm_rf(path) - $stderr.puts "rm -rf #{path}" if verbose? - return if no_harm? - remove_tree path - end - - def remove_tree(path) - if File.symlink?(path) - remove_file path - elsif File.dir?(path) - remove_tree0 path - else - force_remove_file path - end - end - - def remove_tree0(path) - Dir.foreach(path) do |ent| - next if ent == '.' - next if ent == '..' - entpath = "#{path}/#{ent}" - if File.symlink?(entpath) - remove_file entpath - elsif File.dir?(entpath) - remove_tree0 entpath - else - force_remove_file entpath - end - end - begin - Dir.rmdir path - rescue Errno::ENOTEMPTY - # directory may not be empty - end - end - - def move_file(src, dest) - force_remove_file dest - begin - File.rename src, dest - rescue - File.open(dest, 'wb') {|f| - f.write File.binread(src) - } - File.chmod File.stat(src).mode, dest - File.unlink src - end - end - - def force_remove_file(path) - begin - remove_file path - rescue - end - end - - def remove_file(path) - File.chmod 0777, path - File.unlink path - end - - def install(from, dest, mode, prefix = nil) - $stderr.puts "install #{from} #{dest}" if verbose? - return if no_harm? - - realdest = prefix ? prefix + File.expand_path(dest) : dest - realdest = File.join(realdest, File.basename(from)) if File.dir?(realdest) - str = File.binread(from) - if diff?(str, realdest) - verbose_off { - rm_f realdest if File.exist?(realdest) - } - File.open(realdest, 'wb') {|f| - f.write str - } - File.chmod mode, realdest - - File.open("#{objdir_root()}/InstalledFiles", 'a') {|f| - if prefix - f.puts realdest.sub(prefix, '') - else - f.puts realdest - end - } - end - end - - def diff?(new_content, path) - return true unless File.exist?(path) - new_content != File.binread(path) - end - - def command(*args) - $stderr.puts args.join(' ') if verbose? - system(*args) or raise RuntimeError, - "system(#{args.map{|a| a.inspect }.join(' ')}) failed" - end - - def ruby(*args) - command config('rubyprog'), *args - end - - def make(task = nil) - command(*[config('makeprog'), task].compact) - end - - def extdir?(dir) - File.exist?("#{dir}/MANIFEST") or File.exist?("#{dir}/extconf.rb") - end - - def files_of(dir) - Dir.open(dir) {|d| - return d.select {|ent| File.file?("#{dir}/#{ent}") } - } - end - - DIR_REJECT = %w( . .. CVS SCCS RCS CVS.adm .svn ) - - def directories_of(dir) - Dir.open(dir) {|d| - return d.select {|ent| File.dir?("#{dir}/#{ent}") } - DIR_REJECT - } - end - -end - - -# This module requires: #srcdir_root, #objdir_root, #relpath -module HookScriptAPI - - def get_config(key) - @config[key] - end - - alias config get_config - - # obsolete: use metaconfig to change configuration - def set_config(key, val) - @config[key] = val - end - - # - # srcdir/objdir (works only in the package directory) - # - - def curr_srcdir - "#{srcdir_root()}/#{relpath()}" - end - - def curr_objdir - "#{objdir_root()}/#{relpath()}" - end - - def srcfile(path) - "#{curr_srcdir()}/#{path}" - end - - def srcexist?(path) - File.exist?(srcfile(path)) - end - - def srcdirectory?(path) - File.dir?(srcfile(path)) - end - - def srcfile?(path) - File.file?(srcfile(path)) - end - - def srcentries(path = '.') - Dir.open("#{curr_srcdir()}/#{path}") {|d| - return d.to_a - %w(. ..) - } - end - - def srcfiles(path = '.') - srcentries(path).select {|fname| - File.file?(File.join(curr_srcdir(), path, fname)) - } - end - - def srcdirectories(path = '.') - srcentries(path).select {|fname| - File.dir?(File.join(curr_srcdir(), path, fname)) - } - end - -end - - -class ToplevelInstaller - - Version = '3.4.1' - Copyright = 'Copyright (c) 2000-2005 Minero Aoki' - - TASKS = [ - [ 'all', 'do config, setup, then install' ], - [ 'config', 'saves your configurations' ], - [ 'show', 'shows current configuration' ], - [ 'setup', 'compiles ruby extentions and others' ], - [ 'install', 'installs files' ], - [ 'test', 'run all tests in test/' ], - [ 'clean', "does `make clean' for each extention" ], - [ 'distclean',"does `make distclean' for each extention" ] - ] - - def ToplevelInstaller.invoke - config = ConfigTable.new(load_rbconfig()) - config.load_standard_entries - config.load_multipackage_entries if multipackage? - config.fixup - klass = (multipackage?() ? ToplevelInstallerMulti : ToplevelInstaller) - klass.new(File.dirname($0), config).invoke - end - - def ToplevelInstaller.multipackage? - File.dir?(File.dirname($0) + '/packages') - end - - def ToplevelInstaller.load_rbconfig - if arg = ARGV.detect {|arg| /\A--rbconfig=/ =~ arg } - ARGV.delete(arg) - load File.expand_path(arg.split(/=/, 2)[1]) - $".push 'rbconfig.rb' - else - require 'rbconfig' - end - ::Config::CONFIG - end - - def initialize(ardir_root, config) - @ardir = File.expand_path(ardir_root) - @config = config - # cache - @valid_task_re = nil - end - - def config(key) - @config[key] - end - - def inspect - "#<#{self.class} #{__id__()}>" - end - - def invoke - run_metaconfigs - case task = parsearg_global() - when nil, 'all' - parsearg_config - init_installers - exec_config - exec_setup - exec_install - else - case task - when 'config', 'test' - ; - when 'clean', 'distclean' - @config.load_savefile if File.exist?(@config.savefile) - else - @config.load_savefile - end - __send__ "parsearg_#{task}" - init_installers - __send__ "exec_#{task}" - end - end - - def run_metaconfigs - @config.load_script "#{@ardir}/metaconfig" - end - - def init_installers - @installer = Installer.new(@config, @ardir, File.expand_path('.')) - end - - # - # Hook Script API bases - # - - def srcdir_root - @ardir - end - - def objdir_root - '.' - end - - def relpath - '.' - end - - # - # Option Parsing - # - - def parsearg_global - while arg = ARGV.shift - case arg - when /\A\w+\z/ - setup_rb_error "invalid task: #{arg}" unless valid_task?(arg) - return arg - when '-q', '--quiet' - @config.verbose = false - when '--verbose' - @config.verbose = true - when '--help' - print_usage $stdout - exit 0 - when '--version' - puts "#{File.basename($0)} version #{Version}" - exit 0 - when '--copyright' - puts Copyright - exit 0 - else - setup_rb_error "unknown global option '#{arg}'" - end - end - nil - end - - def valid_task?(t) - valid_task_re() =~ t - end - - def valid_task_re - @valid_task_re ||= /\A(?:#{TASKS.map {|task,desc| task }.join('|')})\z/ - end - - def parsearg_no_options - unless ARGV.empty? - task = caller(0).first.slice(%r<`parsearg_(\w+)'>, 1) - setup_rb_error "#{task}: unknown options: #{ARGV.join(' ')}" - end - end - - alias parsearg_show parsearg_no_options - alias parsearg_setup parsearg_no_options - alias parsearg_test parsearg_no_options - alias parsearg_clean parsearg_no_options - alias parsearg_distclean parsearg_no_options - - def parsearg_config - evalopt = [] - set = [] - @config.config_opt = [] - while i = ARGV.shift - if /\A--?\z/ =~ i - @config.config_opt = ARGV.dup - break - end - name, value = *@config.parse_opt(i) - if @config.value_config?(name) - @config[name] = value - else - evalopt.push [name, value] - end - set.push name - end - evalopt.each do |name, value| - @config.lookup(name).evaluate value, @config - end - # Check if configuration is valid - set.each do |n| - @config[n] if @config.value_config?(n) - end - end - - def parsearg_install - @config.no_harm = false - @config.install_prefix = '' - while a = ARGV.shift - case a - when '--no-harm' - @config.no_harm = true - when /\A--prefix=/ - path = a.split(/=/, 2)[1] - path = File.expand_path(path) unless path[0,1] == '/' - @config.install_prefix = path - else - setup_rb_error "install: unknown option #{a}" - end - end - end - - def print_usage(out) - out.puts 'Typical Installation Procedure:' - out.puts " $ ruby #{File.basename $0} config" - out.puts " $ ruby #{File.basename $0} setup" - out.puts " # ruby #{File.basename $0} install (may require root privilege)" - out.puts - out.puts 'Detailed Usage:' - out.puts " ruby #{File.basename $0} " - out.puts " ruby #{File.basename $0} [] []" - - fmt = " %-24s %s\n" - out.puts - out.puts 'Global options:' - out.printf fmt, '-q,--quiet', 'suppress message outputs' - out.printf fmt, ' --verbose', 'output messages verbosely' - out.printf fmt, ' --help', 'print this message' - out.printf fmt, ' --version', 'print version and quit' - out.printf fmt, ' --copyright', 'print copyright and quit' - out.puts - out.puts 'Tasks:' - TASKS.each do |name, desc| - out.printf fmt, name, desc - end - - fmt = " %-24s %s [%s]\n" - out.puts - out.puts 'Options for CONFIG or ALL:' - @config.each do |item| - out.printf fmt, item.help_opt, item.description, item.help_default - end - out.printf fmt, '--rbconfig=path', 'rbconfig.rb to load',"running ruby's" - out.puts - out.puts 'Options for INSTALL:' - out.printf fmt, '--no-harm', 'only display what to do if given', 'off' - out.printf fmt, '--prefix=path', 'install path prefix', '' - out.puts - end - - # - # Task Handlers - # - - def exec_config - @installer.exec_config - @config.save # must be final - end - - def exec_setup - @installer.exec_setup - end - - def exec_install - @installer.exec_install - end - - def exec_test - @installer.exec_test - end - - def exec_show - @config.each do |i| - printf "%-20s %s\n", i.name, i.value if i.value? - end - end - - def exec_clean - @installer.exec_clean - end - - def exec_distclean - @installer.exec_distclean - end - -end # class ToplevelInstaller - - -class ToplevelInstallerMulti < ToplevelInstaller - - include FileOperations - - def initialize(ardir_root, config) - super - @packages = directories_of("#{@ardir}/packages") - raise 'no package exists' if @packages.empty? - @root_installer = Installer.new(@config, @ardir, File.expand_path('.')) - end - - def run_metaconfigs - @config.load_script "#{@ardir}/metaconfig", self - @packages.each do |name| - @config.load_script "#{@ardir}/packages/#{name}/metaconfig" - end - end - - attr_reader :packages - - def packages=(list) - raise 'package list is empty' if list.empty? - list.each do |name| - raise "directory packages/#{name} does not exist"\ - unless File.dir?("#{@ardir}/packages/#{name}") - end - @packages = list - end - - def init_installers - @installers = {} - @packages.each do |pack| - @installers[pack] = Installer.new(@config, - "#{@ardir}/packages/#{pack}", - "packages/#{pack}") - end - with = extract_selection(config('with')) - without = extract_selection(config('without')) - @selected = @installers.keys.select {|name| - (with.empty? or with.include?(name)) \ - and not without.include?(name) - } - end - - def extract_selection(list) - a = list.split(/,/) - a.each do |name| - setup_rb_error "no such package: #{name}" unless @installers.key?(name) - end - a - end - - def print_usage(f) - super - f.puts 'Inluded packages:' - f.puts ' ' + @packages.sort.join(' ') - f.puts - end - - # - # Task Handlers - # - - def exec_config - run_hook 'pre-config' - each_selected_installers {|inst| inst.exec_config } - run_hook 'post-config' - @config.save # must be final - end - - def exec_setup - run_hook 'pre-setup' - each_selected_installers {|inst| inst.exec_setup } - run_hook 'post-setup' - end - - def exec_install - run_hook 'pre-install' - each_selected_installers {|inst| inst.exec_install } - run_hook 'post-install' - end - - def exec_test - run_hook 'pre-test' - each_selected_installers {|inst| inst.exec_test } - run_hook 'post-test' - end - - def exec_clean - rm_f @config.savefile - run_hook 'pre-clean' - each_selected_installers {|inst| inst.exec_clean } - run_hook 'post-clean' - end - - def exec_distclean - rm_f @config.savefile - run_hook 'pre-distclean' - each_selected_installers {|inst| inst.exec_distclean } - run_hook 'post-distclean' - end - - # - # lib - # - - def each_selected_installers - Dir.mkdir 'packages' unless File.dir?('packages') - @selected.each do |pack| - $stderr.puts "Processing the package `#{pack}' ..." if verbose? - Dir.mkdir "packages/#{pack}" unless File.dir?("packages/#{pack}") - Dir.chdir "packages/#{pack}" - yield @installers[pack] - Dir.chdir '../..' - end - end - - def run_hook(id) - @root_installer.run_hook id - end - - # module FileOperations requires this - def verbose? - @config.verbose? - end - - # module FileOperations requires this - def no_harm? - @config.no_harm? - end - -end # class ToplevelInstallerMulti - - -class Installer - - FILETYPES = %w( bin lib ext data conf man ) - - include FileOperations - include HookScriptAPI - - def initialize(config, srcroot, objroot) - @config = config - @srcdir = File.expand_path(srcroot) - @objdir = File.expand_path(objroot) - @currdir = '.' - end - - def inspect - "#<#{self.class} #{File.basename(@srcdir)}>" - end - - def noop(rel) - end - - # - # Hook Script API base methods - # - - def srcdir_root - @srcdir - end - - def objdir_root - @objdir - end - - def relpath - @currdir - end - - # - # Config Access - # - - # module FileOperations requires this - def verbose? - @config.verbose? - end - - # module FileOperations requires this - def no_harm? - @config.no_harm? - end - - def verbose_off - begin - save, @config.verbose = @config.verbose?, false - yield - ensure - @config.verbose = save - end - end - - # - # TASK config - # - - def exec_config - exec_task_traverse 'config' - end - - alias config_dir_bin noop - alias config_dir_lib noop - - def config_dir_ext(rel) - extconf if extdir?(curr_srcdir()) - end - - alias config_dir_data noop - alias config_dir_conf noop - alias config_dir_man noop - - def extconf - ruby "#{curr_srcdir()}/extconf.rb", *@config.config_opt - end - - # - # TASK setup - # - - def exec_setup - exec_task_traverse 'setup' - end - - def setup_dir_bin(rel) - files_of(curr_srcdir()).each do |fname| - update_shebang_line "#{curr_srcdir()}/#{fname}" - end - end - - alias setup_dir_lib noop - - def setup_dir_ext(rel) - make if extdir?(curr_srcdir()) - end - - alias setup_dir_data noop - alias setup_dir_conf noop - alias setup_dir_man noop - - def update_shebang_line(path) - return if no_harm? - return if config('shebang') == 'never' - old = Shebang.load(path) - if old - $stderr.puts "warning: #{path}: Shebang line includes too many args. It is not portable and your program may not work." if old.args.size > 1 - new = new_shebang(old) - return if new.to_s == old.to_s - else - return unless config('shebang') == 'all' - new = Shebang.new(config('rubypath')) - end - $stderr.puts "updating shebang: #{File.basename(path)}" if verbose? - open_atomic_writer(path) {|output| - File.open(path, 'rb') {|f| - f.gets if old # discard - output.puts new.to_s - output.print f.read - } - } - end - - def new_shebang(old) - if /\Aruby/ =~ File.basename(old.cmd) - Shebang.new(config('rubypath'), old.args) - elsif File.basename(old.cmd) == 'env' and old.args.first == 'ruby' - Shebang.new(config('rubypath'), old.args[1..-1]) - else - return old unless config('shebang') == 'all' - Shebang.new(config('rubypath')) - end - end - - def open_atomic_writer(path, &block) - tmpfile = File.basename(path) + '.tmp' - begin - File.open(tmpfile, 'wb', &block) - File.rename tmpfile, File.basename(path) - ensure - File.unlink tmpfile if File.exist?(tmpfile) - end - end - - class Shebang - def Shebang.load(path) - line = nil - File.open(path) {|f| - line = f.gets - } - return nil unless /\A#!/ =~ line - parse(line) - end - - def Shebang.parse(line) - cmd, *args = *line.strip.sub(/\A\#!/, '').split(' ') - new(cmd, args) - end - - def initialize(cmd, args = []) - @cmd = cmd - @args = args - end - - attr_reader :cmd - attr_reader :args - - def to_s - "#! #{@cmd}" + (@args.empty? ? '' : " #{@args.join(' ')}") - end - end - - # - # TASK install - # - - def exec_install - rm_f 'InstalledFiles' - exec_task_traverse 'install' - end - - def install_dir_bin(rel) - install_files targetfiles(), "#{config('bindir')}/#{rel}", 0755 - end - - def install_dir_lib(rel) - install_files libfiles(), "#{config('rbdir')}/#{rel}", 0644 - end - - def install_dir_ext(rel) - return unless extdir?(curr_srcdir()) - install_files rubyextentions('.'), - "#{config('sodir')}/#{File.dirname(rel)}", - 0555 - end - - def install_dir_data(rel) - install_files targetfiles(), "#{config('datadir')}/#{rel}", 0644 - end - - def install_dir_conf(rel) - # FIXME: should not remove current config files - # (rename previous file to .old/.org) - install_files targetfiles(), "#{config('sysconfdir')}/#{rel}", 0644 - end - - def install_dir_man(rel) - install_files targetfiles(), "#{config('mandir')}/#{rel}", 0644 - end - - def install_files(list, dest, mode) - mkdir_p dest, @config.install_prefix - list.each do |fname| - install fname, dest, mode, @config.install_prefix - end - end - - def libfiles - glob_reject(%w(*.y *.output), targetfiles()) - end - - def rubyextentions(dir) - ents = glob_select("*.#{@config.dllext}", targetfiles()) - if ents.empty? - setup_rb_error "no ruby extention exists: 'ruby #{$0} setup' first" - end - ents - end - - def targetfiles - mapdir(existfiles() - hookfiles()) - end - - def mapdir(ents) - ents.map {|ent| - if File.exist?(ent) - then ent # objdir - else "#{curr_srcdir()}/#{ent}" # srcdir - end - } - end - - # picked up many entries from cvs-1.11.1/src/ignore.c - JUNK_FILES = %w( - core RCSLOG tags TAGS .make.state - .nse_depinfo #* .#* cvslog.* ,* .del-* *.olb - *~ *.old *.bak *.BAK *.orig *.rej _$* *$ - - *.org *.in .* - ) - - def existfiles - glob_reject(JUNK_FILES, (files_of(curr_srcdir()) | files_of('.'))) - end - - def hookfiles - %w( pre-%s post-%s pre-%s.rb post-%s.rb ).map {|fmt| - %w( config setup install clean ).map {|t| sprintf(fmt, t) } - }.flatten - end - - def glob_select(pat, ents) - re = globs2re([pat]) - ents.select {|ent| re =~ ent } - end - - def glob_reject(pats, ents) - re = globs2re(pats) - ents.reject {|ent| re =~ ent } - end - - GLOB2REGEX = { - '.' => '\.', - '$' => '\$', - '#' => '\#', - '*' => '.*' - } - - def globs2re(pats) - /\A(?:#{ - pats.map {|pat| pat.gsub(/[\.\$\#\*]/) {|ch| GLOB2REGEX[ch] } }.join('|') - })\z/ - end - - # - # TASK test - # - - TESTDIR = 'test' - - def exec_test - unless File.directory?('test') - $stderr.puts 'no test in this package' if verbose? - return - end - $stderr.puts 'Running tests...' if verbose? - begin - require 'test/unit' - rescue LoadError - setup_rb_error 'test/unit cannot loaded. You need Ruby 1.8 or later to invoke this task.' - end - runner = Test::Unit::AutoRunner.new(true) - runner.to_run << TESTDIR - runner.run - end - - # - # TASK clean - # - - def exec_clean - exec_task_traverse 'clean' - rm_f @config.savefile - rm_f 'InstalledFiles' - end - - alias clean_dir_bin noop - alias clean_dir_lib noop - alias clean_dir_data noop - alias clean_dir_conf noop - alias clean_dir_man noop - - def clean_dir_ext(rel) - return unless extdir?(curr_srcdir()) - make 'clean' if File.file?('Makefile') - end - - # - # TASK distclean - # - - def exec_distclean - exec_task_traverse 'distclean' - rm_f @config.savefile - rm_f 'InstalledFiles' - end - - alias distclean_dir_bin noop - alias distclean_dir_lib noop - - def distclean_dir_ext(rel) - return unless extdir?(curr_srcdir()) - make 'distclean' if File.file?('Makefile') - end - - alias distclean_dir_data noop - alias distclean_dir_conf noop - alias distclean_dir_man noop - - # - # Traversing - # - - def exec_task_traverse(task) - run_hook "pre-#{task}" - FILETYPES.each do |type| - if type == 'ext' and config('without-ext') == 'yes' - $stderr.puts 'skipping ext/* by user option' if verbose? - next - end - traverse task, type, "#{task}_dir_#{type}" - end - run_hook "post-#{task}" - end - - def traverse(task, rel, mid) - dive_into(rel) { - run_hook "pre-#{task}" - __send__ mid, rel.sub(%r[\A.*?(?:/|\z)], '') - directories_of(curr_srcdir()).each do |d| - traverse task, "#{rel}/#{d}", mid - end - run_hook "post-#{task}" - } - end - - def dive_into(rel) - return unless File.dir?("#{@srcdir}/#{rel}") - - dir = File.basename(rel) - Dir.mkdir dir unless File.dir?(dir) - prevdir = Dir.pwd - Dir.chdir dir - $stderr.puts '---> ' + rel if verbose? - @currdir = rel - yield - Dir.chdir prevdir - $stderr.puts '<--- ' + rel if verbose? - @currdir = File.dirname(rel) - end - - def run_hook(id) - path = [ "#{curr_srcdir()}/#{id}", - "#{curr_srcdir()}/#{id}.rb" ].detect {|cand| File.file?(cand) } - return unless path - begin - instance_eval File.read(path), path, 1 - rescue - raise if $DEBUG - setup_rb_error "hook #{path} failed:\n" + $!.message - end - end - -end # class Installer - - -class SetupError < StandardError; end - -def setup_rb_error(msg) - raise SetupError, msg -end - -if $0 == __FILE__ - begin - ToplevelInstaller.invoke - rescue SetupError - raise if $DEBUG - $stderr.puts $!.message - $stderr.puts "Try 'ruby #{$0} --help' for detailed usage." - exit 1 - end -end diff --git a/ruby/gem/tasks/deployment.rake b/ruby/gem/tasks/deployment.rake deleted file mode 100644 index 2f43742..0000000 --- a/ruby/gem/tasks/deployment.rake +++ /dev/null @@ -1,34 +0,0 @@ -desc 'Release the website and new gem version' -task :deploy => [:check_version, :website, :release] do - puts "Remember to create SVN tag:" - puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " + - "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} " - puts "Suggested comment:" - puts "Tagging release #{CHANGES}" -end - -desc 'Runs tasks website_generate and install_gem as a local deployment of the gem' -task :local_deploy => [:website_generate, :install_gem] - -task :check_version do - unless ENV['VERSION'] - puts 'Must pass a VERSION=x.y.z release version' - exit - end - unless ENV['VERSION'] == VERS - puts "Please update your version.rb to match the release version, currently #{VERS}" - exit - end -end - -desc 'Install the package as a gem, without generating documentation(ri/rdoc)' -task :install_gem_no_doc => [:clean, :package] do - sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri" -end - -namespace :manifest do - desc 'Recreate Manifest.txt to include ALL files' - task :refresh do - `rake check_manifest | patch -p0 > Manifest.txt` - end -end \ No newline at end of file diff --git a/ruby/gem/tasks/environment.rake b/ruby/gem/tasks/environment.rake deleted file mode 100644 index 691ed3b..0000000 --- a/ruby/gem/tasks/environment.rake +++ /dev/null @@ -1,7 +0,0 @@ -task :ruby_env do - RUBY_APP = if RUBY_PLATFORM =~ /java/ - "jruby" - else - "ruby" - end unless defined? RUBY_APP -end diff --git a/ruby/gem/test/test_helper.rb b/ruby/gem/test/test_helper.rb new file mode 100644 index 0000000..3088faa --- /dev/null +++ b/ruby/gem/test/test_helper.rb @@ -0,0 +1,3 @@ +require 'test/unit' +require File.dirname(__FILE__) + '/../ext/msgpack' + diff --git a/ruby/gengem.sh b/ruby/gengem.sh index de9f4a5..be9d14b 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -6,6 +6,7 @@ cp pack.h gem/ext/ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ +cat test_case.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > gem/test/msgpack_test.rb #cp ../README gem/README.txt cp ../msgpack/pack_define.h gem/msgpack/ cp ../msgpack/pack_template.h gem/msgpack/ diff --git a/ruby/test_case.rb b/ruby/test_case.rb index 1e2d474..1a06b10 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -1,5 +1,6 @@ -require 'test/unit' +#!/usr/bin/env ruby require 'msgpack' +require 'test/unit' class MessagePackTestFormat < Test::Unit::TestCase def self.it(name, &block) From 761b3980834e4644ce0ce57f6db13d1c93883380 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 15 Feb 2009 18:39:30 +0900 Subject: [PATCH 0056/1648] rewrite msgpack::object::via --- c/object.c | 24 ++++++++--------- c/object.h | 32 +++++++++++++++++------ c/pack.h | 58 ++++++++++++++++++++++-------------------- c/zone.c | 2 +- cpp/object.cpp | 48 ++++++++++++++++------------------ cpp/object.hpp | 56 ++++++++++++++++++++++++++-------------- cpp/type/array.hpp | 6 ++--- cpp/type/map.hpp | 36 +++++++++++++------------- cpp/type/raw.hpp | 10 ++++---- cpp/type/tuple.hpp.erb | 4 +-- cpp/unpack.cpp | 22 ++++++++-------- 11 files changed, 165 insertions(+), 133 deletions(-) diff --git a/c/object.c b/c/object.c index 7a41064..6376438 100644 --- a/c/object.c +++ b/c/object.c @@ -98,42 +98,42 @@ static inline msgpack_object msgpack_unpack_array(unpack_user* u, unsigned int n { msgpack_object o; o.type = MSGPACK_OBJECT_ARRAY; - o.via.container.size = 0; - o.via.container.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); - if(o.via.container.ptr == NULL) { u->failed = true; } + o.via.array.size = 0; + o.via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); + if(o.via.array.ptr == NULL) { u->failed = true; } return o; } static inline void msgpack_unpack_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) { if(u->failed) { return; } - c->via.container.ptr[ c->via.container.size++ ] = o; + c->via.array.ptr[ c->via.array.size++ ] = o; } static inline msgpack_object msgpack_unpack_map(unpack_user* u, unsigned int n) { msgpack_object o; o.type = MSGPACK_OBJECT_MAP; - o.via.container.size = 0; - o.via.container.ptr = msgpack_zone_malloc(u->z, n*2*sizeof(msgpack_object)); - if(o.via.container.ptr == NULL) { u->failed = true; } + o.via.map.size = 0; + o.via.map.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object_kv)); + if(o.via.map.ptr == NULL) { u->failed = true; } return o; } static inline void msgpack_unpack_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) { if(u->failed) { return; } - c->via.container.ptr[ c->via.container.size ] = k; - c->via.container.ptr[ c->via.container.size+1 ] = v; - ++c->via.container.size; + c->via.map.ptr[c->via.map.size].key = k; + c->via.map.ptr[c->via.map.size].val = v; + ++c->via.map.size; } static inline msgpack_object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) { msgpack_object o; o.type = MSGPACK_OBJECT_RAW; - o.via.ref.ptr = p; - o.via.ref.size = l; + o.via.raw.ptr = p; + o.via.raw.size = l; u->referenced = true; return o; } diff --git a/c/object.h b/c/object.h index 1b60c57..b590ebe 100644 --- a/c/object.h +++ b/c/object.h @@ -41,19 +41,31 @@ typedef enum { struct _msgpack_object; +struct _msgpack_object_kv; + +typedef struct { + uint32_t size; + struct _msgpack_object* ptr; +} msgpack_object_array; + +typedef struct { + uint32_t size; + struct _msgpack_object_kv* ptr; +} msgpack_object_map; + +typedef struct { + uint32_t size; + const char* ptr; +} msgpack_object_raw; + typedef union { bool boolean; uint64_t u64; int64_t i64; double dec; - struct { - struct _msgpack_object* ptr; - uint32_t size; - } container; - struct { - const char* ptr; - uint32_t size; - } ref; + msgpack_object_array array; + msgpack_object_map map; + msgpack_object_raw raw; } msgpack_object_union; typedef struct _msgpack_object { @@ -61,6 +73,10 @@ typedef struct _msgpack_object { msgpack_object_union via; } msgpack_object; +typedef struct _msgpack_object_kv { + msgpack_object key; + msgpack_object val; +} msgpack_object_kv; typedef enum { MSGPACK_OBJECT_PARSE_SUCCESS = 0, diff --git a/c/pack.h b/c/pack.h index 752cbc4..a510eff 100644 --- a/c/pack.h +++ b/c/pack.h @@ -21,6 +21,7 @@ #include #include #include +#include "msgpack/pack_define.h" #ifdef __cplusplus extern "C" { @@ -34,42 +35,43 @@ typedef struct { msgpack_pack_write_t callback; } msgpack_pack_t; -void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback); +static void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback); -msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback); -void msgpack_pack_free(msgpack_pack_t* ctx); +static msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback); -int msgpack_pack_short(msgpack_pack_t* ctx, short d); -int msgpack_pack_int(msgpack_pack_t* ctx, int d); -int msgpack_pack_long(msgpack_pack_t* ctx, long d); -int msgpack_pack_long_long(msgpack_pack_t* ctx, long long d); -int msgpack_pack_unsigned_short(msgpack_pack_t* ctx, unsigned short d); -int msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); -int msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); -int msgpack_pack_unsigned_long_long(msgpack_pack_t* ctx, unsigned long long d); +static void msgpack_pack_free(msgpack_pack_t* ctx); -int msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); -int msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); -int msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); -int msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); -int msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); -int msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); -int msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); -int msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); +static int msgpack_pack_short(msgpack_pack_t* ctx, short d); +static int msgpack_pack_int(msgpack_pack_t* ctx, int d); +static int msgpack_pack_long(msgpack_pack_t* ctx, long d); +static int msgpack_pack_long_long(msgpack_pack_t* ctx, long long d); +static int msgpack_pack_unsigned_short(msgpack_pack_t* ctx, unsigned short d); +static int msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); +static int msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); +static int msgpack_pack_unsigned_long_long(msgpack_pack_t* ctx, unsigned long long d); -int msgpack_pack_float(msgpack_pack_t* ctx, float d); -int msgpack_pack_double(msgpack_pack_t* ctx, double d); +static int msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); +static int msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); +static int msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); +static int msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); +static int msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); +static int msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); +static int msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); +static int msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); -int msgpack_pack_nil(msgpack_pack_t* ctx); -int msgpack_pack_true(msgpack_pack_t* ctx); -int msgpack_pack_false(msgpack_pack_t* ctx); +static int msgpack_pack_float(msgpack_pack_t* ctx, float d); +static int msgpack_pack_double(msgpack_pack_t* ctx, double d); -int msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); +static int msgpack_pack_nil(msgpack_pack_t* ctx); +static int msgpack_pack_true(msgpack_pack_t* ctx); +static int msgpack_pack_false(msgpack_pack_t* ctx); -int msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); +static int msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); -int msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); -int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); +static int msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); + +static int msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); +static int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); diff --git a/c/zone.c b/c/zone.c index a41bb44..ccd702d 100644 --- a/c/zone.c +++ b/c/zone.c @@ -41,7 +41,7 @@ void msgpack_zone_free(msgpack_zone* z) if(z->array) { size_t i; for(i=0; i <= z->ntail; ++i) { - free(z->array[i].ptr); + free(z->array[i].alloc); } } free(z); diff --git a/cpp/object.cpp b/cpp/object.cpp index d217c06..eb45f77 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -46,16 +46,16 @@ std::ostream& operator<< (std::ostream& s, const object o) break; case type::RAW: - (s << '"').write(o.via.ref.ptr, o.via.ref.size) << '"'; + (s << '"').write(o.via.raw.ptr, o.via.raw.size) << '"'; break; case type::ARRAY: s << "["; - if(o.via.container.size != 0) { - object* p(o.via.container.ptr); + if(o.via.array.size != 0) { + object* p(o.via.array.ptr); s << *p; ++p; - for(object* const pend(o.via.container.ptr + o.via.container.size); + for(object* const pend(o.via.array.ptr + o.via.array.size); p < pend; ++p) { s << ", " << *p; } @@ -66,17 +66,13 @@ std::ostream& operator<< (std::ostream& s, const object o) case type::MAP: s << "{"; - if(o.via.container.size != 0) { - object* p(o.via.container.ptr); - object* const pend(o.via.container.ptr + o.via.container.size*2); - s << *p; ++p; - s << "=>"; - s << *p; ++p; - while(p < pend) { - s << ", "; - s << *p; ++p; - s << "=>"; - s << *p; ++p; + if(o.via.map.size != 0) { + object_kv* p(o.via.map.ptr); + s << p->key << "=>" << p->val; + ++p; + for(object_kv* const pend(o.via.map.ptr + o.via.map.size); + p < pend; ++p) { + s << ", " << p->key << "=>" << p->val; } } s << "}"; @@ -109,14 +105,14 @@ bool operator==(const object x, const object y) return x.via.i64 == y.via.i64; case type::RAW: - return x.via.ref.size == y.via.ref.size && - memcmp(x.via.ref.ptr, y.via.ref.ptr, x.via.ref.size) == 0; + return x.via.raw.size == y.via.raw.size && + memcmp(x.via.raw.ptr, y.via.raw.ptr, x.via.raw.size) == 0; case type::ARRAY: - if(x.via.container.size != y.via.container.size) { return false; } - for(object* px(x.via.container.ptr), - * const pxend(x.via.container.ptr + x.via.container.size), - * py(y.via.container.ptr); + if(x.via.array.size != y.via.array.size) { return false; } + for(object* px(x.via.array.ptr), + * const pxend(x.via.array.ptr + x.via.array.size), + * py(y.via.array.ptr); px < pxend; ++px, ++py) { if(*px != *py) { return false; } } @@ -124,12 +120,12 @@ bool operator==(const object x, const object y) // FIXME loop optimiziation case type::MAP: - if(x.via.container.size != y.via.container.size) { return false; } - for(object* px(x.via.container.ptr), - * const pxend(x.via.container.ptr + x.via.container.size*2), - * py(y.via.container.ptr); + if(x.via.map.size != y.via.map.size) { return false; } + for(object_kv* px(x.via.map.ptr), + * const pxend(x.via.map.ptr + x.via.map.size), + * py(y.via.map.ptr); px < pxend; ++px, ++py) { - if(*px != *py) { return false; } + if(px->key != py->key || px->val != py->val) { return false; } } return true; diff --git a/cpp/object.hpp b/cpp/object.hpp index c6f1e4a..08e715d 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -45,20 +45,33 @@ namespace type { } +struct object; +struct object_kv; + +struct object_array { + uint32_t size; + object* ptr; +}; + +struct object_map { + uint32_t size; + object_kv* ptr; +}; + +struct object_raw { + uint32_t size; + const char* ptr; +}; + struct object { union union_type { bool boolean; uint64_t u64; int64_t i64; double dec; - struct { - object* ptr; - uint32_t size; - } container; - struct { - const char* ptr; - uint32_t size; - } ref; + object_array array; + object_map map; + object_raw raw; }; type::object_type type; @@ -79,6 +92,11 @@ public: implicit_type convert(); }; +struct object_kv { + object key; + object val; +}; + bool operator==(const object x, const object y); bool operator!=(const object x, const object y); @@ -258,14 +276,14 @@ packer& operator<< (packer& o, const object& v) return o; case type::RAW: - o.pack_raw(v.via.ref.size); - o.pack_raw_body(v.via.ref.ptr, v.via.ref.size); + o.pack_raw(v.via.raw.size); + o.pack_raw_body(v.via.raw.ptr, v.via.raw.size); return o; case type::ARRAY: - o.pack_array(v.via.container.size); - for(object* p(v.via.container.ptr), - * const pend(v.via.container.ptr + v.via.container.size); + o.pack_array(v.via.array.size); + for(object* p(v.via.array.ptr), + * const pend(v.via.array.ptr + v.via.array.size); p < pend; ++p) { *p >> o; } @@ -273,12 +291,12 @@ packer& operator<< (packer& o, const object& v) // FIXME loop optimiziation case type::MAP: - o.pack_map(v.via.container.size); - for(object* p(v.via.container.ptr), - * const pend(v.via.container.ptr + v.via.container.size*2); - p < pend; ) { - *p >> o; ++p; - *p >> o; ++p; + o.pack_map(v.via.map.size); + for(object_kv* p(v.via.map.ptr), + * const pend(v.via.map.ptr + v.via.map.size); + p < pend; ++p) { + p->key >> o; + p->val >> o; } return o; // FIXME loop optimiziation diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index f9f8038..b2a81ef 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -28,9 +28,9 @@ template inline std::vector operator>> (object o, std::vector& v) { if(o.type != type::ARRAY) { throw type_error(); } - v.resize(o.via.container.size); - object* p = o.via.container.ptr; - object* const pend = o.via.container.ptr + o.via.container.size; + v.resize(o.via.array.size); + object* p = o.via.array.ptr; + object* const pend = o.via.array.ptr + o.via.array.size; T* it = &v.front(); for(; p < pend; ++p, ++it) { p->convert(it); diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 619c36e..4585d66 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -46,13 +46,13 @@ template inline type::assoc_vector& operator>> (object o, type::assoc_vector& v) { if(o.type != type::MAP) { throw type_error(); } - v.resize(o.via.container.size); - object* p = o.via.container.ptr; - object* const pend = o.via.container.ptr + o.via.container.size; + v.resize(o.via.map.size); + object_kv* p = o.via.map.ptr; + object_kv* const pend = o.via.map.ptr + o.via.map.size; std::pair* it(&v.front()); - for(; p < pend; ++it) { - p->convert(&it->first); ++p; - p->convert(&it->second); ++p; + for(; p < pend; ++p, ++it) { + p->key.convert(&it->first); + p->val.convert(&it->second); } std::sort(v.begin(), v.end(), type::detail::pair_first_less()); return v; @@ -75,18 +75,18 @@ template inline std::map operator>> (object o, std::map& v) { if(o.type != type::MAP) { throw type_error(); } - object* p(o.via.container.ptr); - object* const pend(o.via.container.ptr + o.via.container.size*2); - while(p < pend) { + object_kv* p(o.via.map.ptr); + object_kv* const pend(o.via.map.ptr + o.via.map.size); + for(; p != pend; ++p) { K key; - p->convert(&key); ++p; + p->key.convert(&key); typename std::map::iterator it(v.find(key)); if(it != v.end()) { V val; - p->convert(&val); ++p; + p->val.convert(&val); it->insert( std::pair(key, val) ); } else { - p->convert(&it->second); ++p; + p->val.convert(&it->second); } } return v; @@ -109,12 +109,12 @@ template inline std::multimap operator>> (object o, std::multimap& v) { if(o.type != type::MAP) { throw type_error(); } - object* p = o.via.container.ptr; - object* const pend = o.via.container.ptr + o.via.container.size*2; - while(p < pend) { + object_kv* p(o.via.map.ptr); + object_kv* const pend(o.via.map.ptr + o.via.map.size); + for(; p != pend; ++p) { std::pair value; - p->convert(&value.first); ++p; - p->convert(&value.second); ++p; + p->key.convert(&value.first); + p->val.convert(&value.second); v.insert(value); } return v; @@ -123,7 +123,7 @@ inline std::multimap operator>> (object o, std::multimap& v) template inline packer& operator<< (packer& o, const std::multimap& v) { - o.pack_multimap(v.size()); + o.pack_map(v.size()); for(typename std::multimap::const_iterator it(v.begin()), it_end(v.end()); it != it_end; ++it) { o.pack(it->first); diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index e4f9dd3..b102ee8 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -27,11 +27,11 @@ namespace msgpack { namespace type { struct raw_ref { - raw_ref() : ptr(NULL), size(0) {} - raw_ref(const char* p, uint32_t s) : ptr(p), size(s) {} + raw_ref() : size(0), ptr(NULL) {} + raw_ref(const char* p, uint32_t s) : size(s), ptr(p) {} - const char* ptr; uint32_t size; + const char* ptr; std::string str() { return std::string(ptr, size); } @@ -64,8 +64,8 @@ struct raw_ref { inline type::raw_ref& operator>> (object o, type::raw_ref& v) { if(o.type != type::RAW) { throw type_error(); } - v.ptr = o.via.ref.ptr; - v.size = o.via.ref.size; + v.ptr = o.via.raw.ptr; + v.size = o.via.raw.size; return v; } diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index a66c57e..a20f5d9 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -136,9 +136,9 @@ type::tuple, A<%=j%><%}%>>& operator>> ( object o, type::tuple, A<%=j%><%}%>>& v) { if(o.type != type::ARRAY) { throw type_error(); } - if(o.via.container.size < <%=i+1%>) { throw type_error(); } + if(o.via.array.size < <%=i+1%>) { throw type_error(); } <%0.upto(i) {|j|%> - o.via.container.ptr[<%=j%>].convert>(&v.template get<<%=j%>>());<%}%> + o.via.array.ptr[<%=j%>].convert>(&v.template get<<%=j%>>());<%}%> return v; } <%}%> diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp index 62088d6..0b9a476 100644 --- a/cpp/unpack.cpp +++ b/cpp/unpack.cpp @@ -104,36 +104,36 @@ static inline object msgpack_unpack_array(unpack_user* u, unsigned int n) { object o; o.type = type::ARRAY; - o.via.container.size = 0; - o.via.container.ptr = (object*)u->z->malloc(n*sizeof(object)); + o.via.array.size = 0; + o.via.array.ptr = (object*)u->z->malloc(n*sizeof(object)); return o; } static inline void msgpack_unpack_array_item(unpack_user* u, object* c, object o) -{ c->via.container.ptr[ c->via.container.size++ ] = o; } +{ c->via.array.ptr[c->via.array.size++] = o; } static inline object msgpack_unpack_map(unpack_user* u, unsigned int n) { object o; o.type = type::MAP; - o.via.container.size = 0; - o.via.container.ptr = (object*)u->z->malloc(n*2*sizeof(object)); + o.via.map.size = 0; + o.via.map.ptr = (object_kv*)u->z->malloc(n*sizeof(object_kv)); return o; } static inline void msgpack_unpack_map_item(unpack_user* u, object* c, object k, object v) { - c->via.container.ptr[ c->via.container.size ] = k; - c->via.container.ptr[ c->via.container.size+1 ] = v; - ++c->via.container.size; + c->via.map.ptr[c->via.map.size].key = k; + c->via.map.ptr[c->via.map.size].val = v; + ++c->via.map.size; } static inline object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) { object o; o.type = type::RAW; - o.via.ref.ptr = p; - o.via.ref.size = l; + o.via.raw.ptr = p; + o.via.raw.size = l; u->referenced = true; return o; } @@ -332,7 +332,7 @@ object unpacker::data() void unpacker::reset() { - if(!m_zone->empty()) { delete release_zone(); } + //if(!m_zone->empty()) { delete release_zone(); } as_ctx(m_ctx)->reset(); } From cbf7afc1ccbbabb8230a1a076eb2251aff7919e0 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Sun, 22 Feb 2009 15:14:21 +0900 Subject: [PATCH 0057/1648] c and c++: rewritten and integrated --- COPYING | 2 +- Makefile.am | 5 + README | 2 +- bootstrap | 8 +- c/AUTHORS | 1 - c/COPYING | 14 -- c/ChangeLog | 0 c/NEWS | 0 c/README | 21 --- c/bootstrap | 3 - c/configure.in | 16 -- c/object.c | 199 ++++++-------------- c/object.h | 23 +-- c/pack.h | 87 +++++---- c/unpack.c | 384 +++++++++++++++++++++++++++++++------- c/unpack.h | 103 ++++++---- c/zone.c | 253 +++++++++++++++++++------ c/zone.h | 49 ++++- configure.in | 38 +++- cpp/AUTHORS | 1 - cpp/COPYING | 14 -- cpp/ChangeLog | 0 cpp/Makefile.am | 7 +- cpp/NEWS | 0 cpp/README | 21 --- cpp/bootstrap | 3 - cpp/configure.in | 23 --- cpp/type/array.hpp | 2 +- cpp/type/boolean.hpp | 2 +- cpp/type/float.hpp | 2 +- cpp/type/integer.hpp | 2 +- cpp/type/map.hpp | 2 +- cpp/type/nil.hpp | 2 +- cpp/type/raw.hpp | 2 +- cpp/type/tuple.hpp.erb | 2 +- cpp/unpack.cpp | 369 ------------------------------------ cpp/unpack.hpp | 171 ++++++++++++----- cpp/zone.cpp | 100 ---------- cpp/zone.hpp.erb | 59 +++--- example/stream.cc | 1 + msgpack/unpack_template.h | 26 +-- ruby/gem/README | 4 +- ruby/gengem.sh | 2 + ruby/unpack.c | 134 ++++++------- 44 files changed, 1035 insertions(+), 1124 deletions(-) delete mode 100644 c/AUTHORS delete mode 100644 c/COPYING delete mode 100644 c/ChangeLog delete mode 100644 c/NEWS delete mode 100644 c/README delete mode 100755 c/bootstrap delete mode 100644 c/configure.in delete mode 100644 cpp/AUTHORS delete mode 100644 cpp/COPYING delete mode 100644 cpp/ChangeLog delete mode 100644 cpp/NEWS delete mode 100644 cpp/README delete mode 100755 cpp/bootstrap delete mode 100644 cpp/configure.in delete mode 100644 cpp/unpack.cpp delete mode 100644 cpp/zone.cpp diff --git a/COPYING b/COPYING index 5f100cd..6f5f220 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile.am b/Makefile.am index c23320a..3144972 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,9 @@ +if ENABLE_CXX +export ERB SUBDIRS = c cpp +else +SUBDIRS = c +endif nobase_include_HEADERS = \ msgpack/pack_define.h \ diff --git a/README b/README index 87d9279..fd5fa8d 100644 --- a/README +++ b/README @@ -54,7 +54,7 @@ Binary-based efficient data interchange format. -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/bootstrap b/bootstrap index 954fa89..eb16c35 100755 --- a/bootstrap +++ b/bootstrap @@ -31,10 +31,10 @@ if test x"$1" = x"--help"; then fi -if [ -z "$NO_NEST" ];then - cd c && ./bootstrap $@; cd .. - cd cpp && ./bootstrap $@; cd .. -fi +#if [ -z "$NO_NEST" ];then +# cd c && ./bootstrap $@; cd .. +# cd cpp && ./bootstrap $@; cd .. +#fi ACLOCAL="aclocal" diff --git a/c/AUTHORS b/c/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/c/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/c/COPYING b/c/COPYING deleted file mode 100644 index 6f5f220..0000000 --- a/c/COPYING +++ /dev/null @@ -1,14 +0,0 @@ -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/c/ChangeLog b/c/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/c/NEWS b/c/NEWS deleted file mode 100644 index e69de29..0000000 diff --git a/c/README b/c/README deleted file mode 100644 index 76dc221..0000000 --- a/c/README +++ /dev/null @@ -1,21 +0,0 @@ -MessagePack for C ------------------ -MessagePack is a binary-based efficient data interchange format. - - - -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - diff --git a/c/bootstrap b/c/bootstrap deleted file mode 100755 index 7e61b82..0000000 --- a/c/bootstrap +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -NO_NEST=1 -. ../bootstrap diff --git a/c/configure.in b/c/configure.in deleted file mode 100644 index 47eb8aa..0000000 --- a/c/configure.in +++ /dev/null @@ -1,16 +0,0 @@ -AC_INIT(unpack.c) -AM_INIT_AUTOMAKE(msgpackc, 0.1.0) -AC_CONFIG_HEADER(config.h) - -AC_SUBST(CFLAGS) -if test "" = "$CFLAGS"; then - CFLAGS="-g -O4" -fi - -AC_PROG_CC -AC_PROG_LIBTOOL - -CFLAGS="-O4 -Wall $CFLAGS -I.." - -AC_OUTPUT([Makefile]) - diff --git a/c/object.c b/c/object.c index 6376438..871153d 100644 --- a/c/object.c +++ b/c/object.c @@ -15,153 +15,78 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "msgpack/unpack.h" -#include "msgpack/unpack_define.h" #include "msgpack/object.h" +#include +#include -typedef struct { - msgpack_zone* z; - bool referenced; - bool failed; -} unpack_user; - -#define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name - -#define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name - -#define msgpack_unpack_callback(name) \ - msgpack_unpack ## name - -#define msgpack_unpack_object msgpack_object - -#define msgpack_unpack_user unpack_user - - -struct msgpack_unpacker_context; - -static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); - -static msgpack_object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); - -static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, - const char* data, size_t len, size_t* off); - -static inline msgpack_object msgpack_unpack_init(unpack_user* u) -{ msgpack_object o; return o; } - -static inline msgpack_object msgpack_unpack_uint8(unpack_user* u, uint8_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint16(unpack_user* u, uint16_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint32(unpack_user* u, uint32_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint64(unpack_user* u, uint64_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_int8(unpack_user* u, int8_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int16(unpack_user* u, int16_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int32(unpack_user* u, int32_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int64(unpack_user* u, int64_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_float(unpack_user* u, float d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } - -static inline msgpack_object msgpack_unpack_double(unpack_user* u, double d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } - -static inline msgpack_object msgpack_unpack_nil(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_NIL; return o; } - -static inline msgpack_object msgpack_unpack_true(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = true; return o; } - -static inline msgpack_object msgpack_unpack_false(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = false; return o; } - -static inline msgpack_object msgpack_unpack_array(unpack_user* u, unsigned int n) +void msgpack_object_print(FILE* out, msgpack_object o) { - msgpack_object o; - o.type = MSGPACK_OBJECT_ARRAY; - o.via.array.size = 0; - o.via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); - if(o.via.array.ptr == NULL) { u->failed = true; } - return o; -} + switch(o.type) { + case MSGPACK_OBJECT_NIL: + fprintf(out, "nil"); + break; -static inline void msgpack_unpack_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) -{ - if(u->failed) { return; } - c->via.array.ptr[ c->via.array.size++ ] = o; -} + case MSGPACK_OBJECT_BOOLEAN: + fprintf(out, (o.via.boolean ? "true" : "false")); + break; -static inline msgpack_object msgpack_unpack_map(unpack_user* u, unsigned int n) -{ - msgpack_object o; - o.type = MSGPACK_OBJECT_MAP; - o.via.map.size = 0; - o.via.map.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object_kv)); - if(o.via.map.ptr == NULL) { u->failed = true; } - return o; -} + case MSGPACK_OBJECT_POSITIVE_INTEGER: + fprintf(out, "%"PRIu64, o.via.u64); + break; -static inline void msgpack_unpack_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) -{ - if(u->failed) { return; } - c->via.map.ptr[c->via.map.size].key = k; - c->via.map.ptr[c->via.map.size].val = v; - ++c->via.map.size; -} + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + fprintf(out, "%"PRIi64, o.via.i64); + break; -static inline msgpack_object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) -{ - msgpack_object o; - o.type = MSGPACK_OBJECT_RAW; - o.via.raw.ptr = p; - o.via.raw.size = l; - u->referenced = true; - return o; -} + case MSGPACK_OBJECT_DOUBLE: + fprintf(out, "%f", o.via.dec); + break; -#include "msgpack/unpack_template.h" + case MSGPACK_OBJECT_RAW: + fprintf(out, "\""); + fwrite(o.via.raw.ptr, o.via.raw.size, 1, out); + fprintf(out, "\""); + break; -msgpack_object_unpack_return -msgpack_object_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_object* result) -{ - struct msgpack_unpacker_context ctx; - msgpack_unpacker_init(&ctx); - unpack_user u = {z, false, false}; - ctx.user = u; + case MSGPACK_OBJECT_ARRAY: + fprintf(out, "["); + if(o.via.array.size != 0) { + msgpack_object* p = o.via.array.ptr; + msgpack_object_print(out, *p); + ++p; + msgpack_object* const pend = o.via.array.ptr + o.via.array.size; + for(; p < pend; ++p) { + fprintf(out, ", "); + msgpack_object_print(out, *p); + } + } + fprintf(out, "]"); + break; + // FIXME loop optimiziation - size_t noff = (off ? *off : 0); - int ret = msgpack_unpacker_execute(&ctx, data, len, &noff); - if(ret < 0 || ctx.user.failed) { - return MSGPACK_OBJECT_PARSE_ERROR; - } else if(ret == 0) { - return MSGPACK_OBJECT_INSUFFICIENT_BYTES; + case MSGPACK_OBJECT_MAP: + fprintf(out, "{"); + if(o.via.map.size != 0) { + msgpack_object_kv* p = o.via.map.ptr; + msgpack_object_print(out, p->key); + fprintf(out, "=>"); + msgpack_object_print(out, p->val); + ++p; + msgpack_object_kv* const pend = o.via.map.ptr + o.via.map.size; + for(; p < pend; ++p) { + fprintf(out, ", "); + msgpack_object_print(out, p->key); + fprintf(out, "=>"); + msgpack_object_print(out, p->val); + } + } + fprintf(out, "}"); + break; + // FIXME loop optimiziation + + default: + // FIXME + fprintf(out, "#", o.type, o.via.u64); } - *result = msgpack_unpacker_data(&ctx); - if(off) { *off = noff; } - if(ret == 0) { - return MSGPACK_OBJECT_EXTRA_BYTES; - } - return MSGPACK_OBJECT_PARSE_SUCCESS; } - diff --git a/c/object.h b/c/object.h index b590ebe..7c603b3 100644 --- a/c/object.h +++ b/c/object.h @@ -22,6 +22,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -39,18 +40,18 @@ typedef enum { MSGPACK_OBJECT_MAP = 0x08, } msgpack_object_type; -struct _msgpack_object; -struct _msgpack_object_kv; +struct msgpack_object; +struct msgpack_object_kv; typedef struct { uint32_t size; - struct _msgpack_object* ptr; + struct msgpack_object* ptr; } msgpack_object_array; typedef struct { uint32_t size; - struct _msgpack_object_kv* ptr; + struct msgpack_object_kv* ptr; } msgpack_object_map; typedef struct { @@ -68,26 +69,18 @@ typedef union { msgpack_object_raw raw; } msgpack_object_union; -typedef struct _msgpack_object { +typedef struct msgpack_object { msgpack_object_type type; msgpack_object_union via; } msgpack_object; -typedef struct _msgpack_object_kv { +typedef struct msgpack_object_kv { msgpack_object key; msgpack_object val; } msgpack_object_kv; -typedef enum { - MSGPACK_OBJECT_PARSE_SUCCESS = 0, - MSGPACK_OBJECT_EXTRA_BYTES = 1, - MSGPACK_OBJECT_INSUFFICIENT_BYTES = -1, - MSGPACK_OBJECT_PARSE_ERROR = -2, -} msgpack_object_unpack_return; -msgpack_object_unpack_return -msgpack_object_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_object* result); +void msgpack_object_print(FILE* out, msgpack_object o); #ifdef __cplusplus diff --git a/c/pack.h b/c/pack.h index a510eff..46de722 100644 --- a/c/pack.h +++ b/c/pack.h @@ -28,50 +28,49 @@ extern "C" { #endif -typedef int (*msgpack_pack_write_t)(void* data, const char* buf, unsigned int len); +typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); -typedef struct { +typedef struct msgpack_packer { void* data; - msgpack_pack_write_t callback; -} msgpack_pack_t; + msgpack_packer_write callback; +} msgpack_packer; -static void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback); +static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback); +static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static void msgpack_packer_free(msgpack_packer* pk); -static void msgpack_pack_free(msgpack_pack_t* ctx); +static int msgpack_pack_short(msgpack_packer* pk, short d); +static int msgpack_pack_int(msgpack_packer* pk, int d); +static int msgpack_pack_long(msgpack_packer* pk, long d); +static int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); -static int msgpack_pack_short(msgpack_pack_t* ctx, short d); -static int msgpack_pack_int(msgpack_pack_t* ctx, int d); -static int msgpack_pack_long(msgpack_pack_t* ctx, long d); -static int msgpack_pack_long_long(msgpack_pack_t* ctx, long long d); -static int msgpack_pack_unsigned_short(msgpack_pack_t* ctx, unsigned short d); -static int msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); -static int msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); -static int msgpack_pack_unsigned_long_long(msgpack_pack_t* ctx, unsigned long long d); +static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); -static int msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); -static int msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); -static int msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); -static int msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); -static int msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); -static int msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); -static int msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); -static int msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); +static int msgpack_pack_float(msgpack_packer* pk, float d); +static int msgpack_pack_double(msgpack_packer* pk, double d); -static int msgpack_pack_float(msgpack_pack_t* ctx, float d); -static int msgpack_pack_double(msgpack_pack_t* ctx, double d); +static int msgpack_pack_nil(msgpack_packer* pk); +static int msgpack_pack_true(msgpack_packer* pk); +static int msgpack_pack_false(msgpack_packer* pk); -static int msgpack_pack_nil(msgpack_pack_t* ctx); -static int msgpack_pack_true(msgpack_pack_t* ctx); -static int msgpack_pack_false(msgpack_pack_t* ctx); +static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); +static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); - -static int msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); -static int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); +static int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); @@ -81,30 +80,30 @@ static int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); #define msgpack_pack_inline_func_cint(name) \ inline int msgpack_pack ## name -#define msgpack_pack_user msgpack_pack_t* +#define msgpack_pack_user msgpack_packer* #define msgpack_pack_append_buffer(user, buf, len) \ return (*(user)->callback)((user)->data, (const char*)buf, len) #include "msgpack/pack_template.h" -inline void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback) +inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) { - ctx->data = data; - ctx->callback = callback; + pk->data = data; + pk->callback = callback; } -inline msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback) +inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) { - msgpack_pack_t* ctx = (msgpack_pack_t*)calloc(1, sizeof(msgpack_pack_t)); - if(!ctx) { return NULL; } - msgpack_pack_init(ctx, data, callback); - return ctx; + msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); + if(!pk) { return NULL; } + msgpack_packer_init(pk, data, callback); + return pk; } -inline void msgpack_pack_free(msgpack_pack_t* ctx) +inline void msgpack_packer_free(msgpack_packer* pk) { - free(ctx); + free(pk); } diff --git a/c/unpack.c b/c/unpack.c index 03c67be..e06d97c 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -20,125 +20,371 @@ #include +typedef struct { + msgpack_zone* z; + bool* referenced; +} unpack_user; + + #define msgpack_unpack_struct(name) \ struct template ## name #define msgpack_unpack_func(ret, name) \ - ret template_func ## name + ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name -#define msgpack_unpack_object void* +#define msgpack_unpack_object msgpack_object -#define msgpack_unpack_user msgpack_unpack_t +#define msgpack_unpack_user unpack_user struct template_context; +typedef struct template_context template_context; -static void template_func_init(struct template_context* ctx); +static void template_init(template_context* ctx); -static void* template_func_data(struct template_context* ctx); +static msgpack_object template_data(template_context* ctx); -static int template_func_execute(struct template_context* ctx, +static int template_execute(template_context* ctx, const char* data, size_t len, size_t* off); -static inline void* template_callback_init(msgpack_unpack_t* x) -{ return NULL; } +static inline msgpack_object template_callback_root(unpack_user* u) +{ msgpack_object o; return o; } -static inline void* template_callback_uint8(msgpack_unpack_t* x, uint8_t d) -{ return x->callback.unpack_uint8(x->data, d); } +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint16(msgpack_unpack_t* x, uint16_t d) -{ return x->callback.unpack_uint16(x->data, d); } +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint32(msgpack_unpack_t* x, uint32_t d) -{ return x->callback.unpack_uint32(x->data, d); } +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint64(msgpack_unpack_t* x, uint64_t d) -{ return x->callback.unpack_uint64(x->data, d); } +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_int8(msgpack_unpack_t* x, int8_t d) -{ return x->callback.unpack_int8(x->data, d); } +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int16(msgpack_unpack_t* x, int16_t d) -{ return x->callback.unpack_int16(x->data, d); } +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int32(msgpack_unpack_t* x, int32_t d) -{ return x->callback.unpack_int32(x->data, d); } +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int64(msgpack_unpack_t* x, int64_t d) -{ return x->callback.unpack_int64(x->data, d); } +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_float(msgpack_unpack_t* x, float d) -{ return x->callback.unpack_float(x->data, d); } +static inline int template_callback_float(unpack_user* u, float d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_DOUBLE; o->via.dec = d; return 0; } -static inline void* template_callback_double(msgpack_unpack_t* x, double d) -{ return x->callback.unpack_double(x->data, d); } +static inline int template_callback_double(unpack_user* u, double d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_DOUBLE; o->via.dec = d; return 0; } -static inline void* template_callback_nil(msgpack_unpack_t* x) -{ return x->callback.unpack_nil(x->data); } +static inline int template_callback_nil(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_NIL; return 0; } -static inline void* template_callback_true(msgpack_unpack_t* x) -{ return x->callback.unpack_true(x->data); } +static inline int template_callback_true(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_BOOLEAN; o->via.boolean = true; return 0; } -static inline void* template_callback_false(msgpack_unpack_t* x) -{ return x->callback.unpack_false(x->data); } +static inline int template_callback_false(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_BOOLEAN; o->via.boolean = false; return 0; } -static inline void* template_callback_array(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_array(x->data, n); } +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_ARRAY; + o->via.array.size = 0; + o->via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); + if(o->via.array.ptr == NULL) { return -1; } + return 0; +} -static inline void template_callback_array_item(msgpack_unpack_t* x, void** c, void* o) -{ x->callback.unpack_array_item(x->data, *c, o); } +static inline int template_callback_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) +{ c->via.array.ptr[c->via.array.size++] = o; return 0; } -static inline void* template_callback_map(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_map(x->data, n); } +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_MAP; + o->via.map.size = 0; + o->via.map.ptr = (msgpack_object_kv*)msgpack_zone_malloc(u->z, n*sizeof(msgpack_object_kv)); + if(o->via.map.ptr == NULL) { return -1; } + return 0; +} -static inline void template_callback_map_item(msgpack_unpack_t* x, void** c, void* k, void* v) -{ x->callback.unpack_map_item(x->data, *c, k, v); } - -static inline void* template_callback_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) -{ return x->callback.unpack_raw(x->data, b, p, l); } +static inline int template_callback_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) +{ + c->via.map.ptr[c->via.map.size].key = k; + c->via.map.ptr[c->via.map.size].val = v; + ++c->via.map.size; + return 0; +} +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_RAW; + o->via.raw.ptr = p; + o->via.raw.size = l; + *u->referenced = true; + return 0; +} #include "msgpack/unpack_template.h" -msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback) +#define CTX_CAST(m) ((template_context*)(m)) + + +static const size_t COUNTER_SIZE = sizeof(unsigned int); + +static inline void init_count(void* buf) { - struct template_context* ctx; - ctx = (struct template_context*)calloc(1, sizeof(struct template_context)); - if(ctx == NULL) { return NULL; } - template_func_init(ctx); - ((msgpack_unpack_t*)ctx)->data = data; - ((msgpack_unpack_t*)ctx)->callback = *callback; - return (msgpack_unpack_t*)ctx; + *(volatile unsigned int*)buf = 1; } -void msgpack_unpack_free(msgpack_unpack_t* ctx) +static inline void decl_count(void* buf) { - free((struct template_context*)ctx); + //if(--*(unsigned int*)buf == 0) { + if(__sync_sub_and_fetch((unsigned int*)buf, 1) == 0) { + free(buf); + } } -void* msgpack_unpack_data(msgpack_unpack_t* ctx) +static inline void incr_count(void* buf) { - return template_func_data((struct template_context*)ctx); + //++*(unsigned int*)buf; + __sync_add_and_fetch((unsigned int*)buf, 1); } -void msgpack_unpack_reset(msgpack_unpack_t* ctx) +static inline unsigned int get_count(void* buf) { - msgpack_unpack_t x = ((struct template_context*)ctx)->user; - template_func_init((struct template_context*)ctx); - ((struct template_context*)ctx)->user = x; -} - -int msgpack_unpack_execute(msgpack_unpack_t* ctx, - const char* data, size_t len, size_t* off) -{ - return template_func_execute( - (struct template_context*)ctx, - data, len, off); + return *(volatile unsigned int*)buf; } + +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) +{ + if(initial_buffer_size < COUNTER_SIZE) { + initial_buffer_size = COUNTER_SIZE; + } + + char* buf = (char*)malloc(initial_buffer_size); + if(buf == NULL) { + return false; + } + + void* ctx = malloc(sizeof(template_context)); + if(ctx == NULL) { + free(buf); + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + free(ctx); + free(buf); + return false; + } + + mpac->buf = buf; + mpac->used = COUNTER_SIZE; + mpac->free = initial_buffer_size - mpac->used; + mpac->off = COUNTER_SIZE; + mpac->initial_buffer_size = initial_buffer_size; + mpac->z = z; + mpac->referenced = false; + mpac->ctx = ctx; + + init_count(mpac->buf); + + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = mpac->z; + CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + + return true; +} + +void msgpack_unpacker_destroy(msgpack_unpacker* mpac) +{ + msgpack_zone_free(mpac->z); + free(mpac->ctx); + decl_count(mpac->buf); +} + + +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) +{ + msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); + if(mpac == NULL) { + return NULL; + } + + if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { + free(mpac); + return NULL; + } + + return mpac; +} + +void msgpack_unpacker_free(msgpack_unpacker* mpac) +{ + msgpack_unpacker_destroy(mpac); + free(mpac); +} + + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->used == mpac->off && get_count(mpac->buf) == 1 && !mpac->referenced) { + // rewind buffer + mpac->free += mpac->used - COUNTER_SIZE; + mpac->used = COUNTER_SIZE; + mpac->off = COUNTER_SIZE; + + if(mpac->free >= size) { + return true; + } + } + + if(mpac->off == COUNTER_SIZE) { + size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE + while(next_size < size + mpac->used) { + next_size *= 2; + } + + char* tmp = (char*)realloc(mpac->buf, next_size); + if(tmp == NULL) { + return false; + } + + mpac->buf = tmp; + mpac->free = next_size - mpac->used; + + } else { + size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE + size_t not_parsed = mpac->used - mpac->off; + while(next_size < size + not_parsed + COUNTER_SIZE) { + next_size *= 2; + } + + char* tmp = (char*)malloc(next_size); + if(tmp == NULL) { + return false; + } + + init_count(tmp); + + if(mpac->referenced) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + free(tmp); + return false; + } + mpac->referenced = false; + } else { + decl_count(mpac->buf); + } + + memcpy(tmp+COUNTER_SIZE, mpac->buf+mpac->off, not_parsed); + + mpac->buf = tmp; + mpac->used = not_parsed + COUNTER_SIZE; + mpac->free = next_size - mpac->used; + mpac->off = COUNTER_SIZE; + } + + return true; +} + +int msgpack_unpacker_execute(msgpack_unpacker* mpac) +{ + return template_execute(CTX_CAST(mpac->ctx), + mpac->buf, mpac->used, &mpac->off); +} + +msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) +{ + return template_data(CTX_CAST(mpac->ctx)); +} + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) +{ + if(!msgpack_unpacker_flush_zone(mpac)) { + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + return NULL; + } + + msgpack_zone* old = mpac->z; + mpac->z = z; + + return old; +} + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) +{ + if(mpac->referenced) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + return false; + } + mpac->referenced = false; + + incr_count(mpac->buf); + } + + return true; +} + +void msgpack_unpacker_reset(msgpack_unpacker* mpac) +{ + msgpack_zone* z = mpac->z; + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = z; + CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; +} + + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result) +{ + template_context ctx; + template_init(&ctx); + + bool referenced = false; + ctx.user.z = z; + ctx.user.referenced = &referenced; + + size_t noff = 0; + if(off != NULL) { noff = *off; } + + int ret = template_execute(&ctx, data, len, &noff); + if(ret < 0) { + return MSGPACK_UNPACK_PARSE_ERROR; + } + + if(off != NULL) { *off = noff; } + + if(ret == 0) { + return MSGPACK_UNPACK_CONTINUE; + } + + *result = template_data(&ctx); + + if(noff < len) { + return MSGPACK_UNPACK_EXTRA_BYTES; + } + + return MSGPACK_UNPACK_SUCCESS; +} + diff --git a/c/unpack.h b/c/unpack.h index 4977f51..ab202a1 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -15,9 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MSGPACK_UNPACK_H__ -#define MSGPACK_UNPACK_H__ +#ifndef msgpack_unpacker_H__ +#define msgpack_unpacker_H__ +#include "msgpack/zone.h" +#include "msgpack/object.h" #include #include @@ -26,39 +28,76 @@ extern "C" { #endif -typedef struct { - void* (*unpack_uint8)(void* data, uint8_t d); - void* (*unpack_uint16)(void* data, uint16_t d); - void* (*unpack_uint32)(void* data, uint32_t d); - void* (*unpack_uint64)(void* data, uint64_t d); - void* (*unpack_int8)(void* data, int8_t d); - void* (*unpack_int16)(void* data, int16_t d); - void* (*unpack_int32)(void* data, int32_t d); - void* (*unpack_int64)(void* data, int64_t d); - void* (*unpack_float)(void* data, float d); - void* (*unpack_double)(void* data, double d); - void* (*unpack_nil)(void* data); - void* (*unpack_true)(void* data); - void* (*unpack_false)(void* data); - void* (*unpack_array)(void* data, unsigned int n); - void (*unpack_array_item)(void* data, void* c, void* o); - void* (*unpack_map)(void* data, unsigned int n); - void (*unpack_map_item)(void* data, void* c, void* k, void* v); - void* (*unpack_raw)(void* data, const char* b, const char* p, unsigned int l); -} msgpack_unpack_callback; +typedef struct msgpack_unpacker { + char* buf; + size_t used; + size_t free; + size_t off; + msgpack_zone* z; + bool referenced; + size_t initial_buffer_size; + void* ctx; +} msgpack_unpacker; -typedef struct { - void* data; - msgpack_unpack_callback callback; -} msgpack_unpack_t; -msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback); -void msgpack_unpack_free(msgpack_unpack_t* ctx); +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size); +void msgpack_unpacker_destroy(msgpack_unpacker* mpac); -int msgpack_unpack_execute(msgpack_unpack_t* ctx, - const char* data, size_t len, size_t* off); -void* msgpack_unpack_data(msgpack_unpack_t* ctx); -void msgpack_unpack_reset(msgpack_unpack_t* ctx); +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); +void msgpack_unpacker_free(msgpack_unpacker* mpac); + +static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); +static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); +static inline void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size); + + +int msgpack_unpacker_execute(msgpack_unpacker* mpac); + +msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac); + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); + +void msgpack_unpacker_reset(msgpack_unpacker* mpac); + + +typedef enum { + MSGPACK_UNPACK_SUCCESS = 2, + MSGPACK_UNPACK_EXTRA_BYTES = 1, + MSGPACK_UNPACK_CONTINUE = 0, + MSGPACK_UNPACK_PARSE_ERROR = -1, +} msgpack_unpack_return; + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result); + + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); + +bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->free >= size) { return true; } + return msgpack_unpacker_expand_buffer(mpac, size); +} + +char* msgpack_unpacker_buffer(msgpack_unpacker* mpac) +{ + return mpac->buf + mpac->used; +} + +size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac) +{ + return mpac->free; +} + +void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) +{ + mpac->used += size; + mpac->free -= size; +} #ifdef __cplusplus diff --git a/c/zone.c b/c/zone.c index ccd702d..e891c82 100644 --- a/c/zone.c +++ b/c/zone.c @@ -19,85 +19,214 @@ #include #include -typedef struct { - size_t free; - void* ptr; - void* alloc; -} msgpack_zone_chunk; -struct _msgpack_zone { - msgpack_zone_chunk* array; - size_t ntail; - size_t usable; -}; - -msgpack_zone* msgpack_zone_new() +static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_size) { - return calloc(1, sizeof(msgpack_zone)); -} + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + const size_t nfirst = (sizeof(msgpack_zone_chunk) < 72/2) ? + 72 / sizeof(msgpack_zone_chunk) : 8; -void msgpack_zone_free(msgpack_zone* z) -{ - if(z->array) { - size_t i; - for(i=0; i <= z->ntail; ++i) { - free(z->array[i].alloc); - } + msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc( + sizeof(msgpack_zone_chunk) * nfirst); + if(!array) { + return false; } - free(z); + + const size_t sz = chunk_size; + + char* ptr = (char*)malloc(sz); + if(!ptr) { + free(array); + return NULL; + } + + ca->tail = array; + ca->end = array + nfirst; + ca->array = array; + + array[0].free = sz; + array[0].ptr = ptr; + array[0].alloc = ptr; + + return true; } - -void* msgpack_zone_malloc(msgpack_zone* z, size_t size) +static inline void destroy_chunk_array(msgpack_zone_chunk_array* ca) { - if(!z->array) { - const size_t n = (sizeof(msgpack_zone_chunk) < 72/2) ? - 72 / sizeof(msgpack_zone_chunk) : 8; - msgpack_zone_chunk* array = - (msgpack_zone_chunk*)malloc(sizeof(msgpack_zone_chunk) * n); - if(!array) { return NULL; } + msgpack_zone_chunk* chunk = ca->array; + for(; chunk != ca->tail+1; ++chunk) { + free(chunk->alloc); + } + free(ca->array); +} - size_t sz = 2048; /* FIXME chunk_size */ - while(sz < size) { sz *= 2; } - char* p = (char*)malloc(sz); - if(!p) { - free(array); +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + msgpack_zone_chunk_array* const ca = &zone->chunk_array; + + msgpack_zone_chunk* chunk = ca->tail; + + if(chunk->free > size) { + // chunkã«ç©ºã容é‡ãŒã‚ã‚‹ + // 空ã容é‡ã‚’消費ã—ã¦è¿”ã™ + + char* ptr = chunk->ptr; + + chunk->ptr += size; + chunk->free -= size; + + return ptr; + } + + chunk = ++ca->tail; + + if(chunk == ca->end) { + // ca->arrayã«ç©ºããŒãªã„ + // ca->arrayã‚’æ‹¡å¼µã™ã‚‹ + + const size_t nused = ca->end - ca->array; + const size_t nnext = (ca->end - ca->array) * 2; + + chunk = (msgpack_zone_chunk*)realloc(ca->array, + sizeof(msgpack_zone_chunk) * nnext); + if(!chunk) { return NULL; } - z->array = array; - z->usable = n - 1; - array[0].free = sz - size; - array[0].ptr = p + size; - array[0].alloc = p; - return p; + ca->array = chunk; + ca->end = chunk + nnext; + chunk = ca->tail = chunk + nused; } - if(z->array[z->ntail].free > size) { - char* p = (char*)z->array[z->ntail].ptr; - z->array[z->ntail].ptr = p + size; - z->array[z->ntail].free -= size; - return p; + size_t sz = zone->chunk_size; + + while(sz < size) { + sz *= 2; } - if(z->usable <= z->ntail) { - const size_t n = (z->usable + 1) * 2; - msgpack_zone_chunk* tmp = - (msgpack_zone_chunk*)realloc(z->array, sizeof(msgpack_zone_chunk) * n); - if(!tmp) { return NULL; } - z->array = tmp; - z->usable = n - 1; + char* ptr = (char*)malloc(sz); + if(!ptr) { + return NULL; } - size_t sz = 2048; /* FIXME chunk_size */ - while(sz < size) { sz *= 2; } - char* p = (char*)malloc(sz); - if(!p) { return NULL; } + chunk->free = sz - size; + chunk->ptr = ptr + size; + chunk->alloc = ptr; - ++z->ntail; - z->array[z->ntail].free = sz - size; - z->array[z->ntail].ptr = p + size; - z->array[z->ntail].alloc = p; - return p; + return ptr; +} + + +static inline void init_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + fa->tail = NULL; + fa->end = NULL; + fa->array = NULL; +} + +static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + // 逆順ã«å‘¼ã³å‡ºã— + msgpack_zone_finalizer* fin = fa->tail; + for(; fin != fa->array; --fin) { + (*(fin-1)->func)((fin-1)->data); + } + free(fa->array); +} + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data) +{ + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + + msgpack_zone_finalizer* fin = fa->tail; + + if(fin == fa->end) { + // fa->arrayã«ç©ºããŒãªã„ + // fa->arrayã‚’æ‹¡å¼µã™ã‚‹ + + size_t nnext; + const size_t nused = fa->end - fa->array; + + if(nused == 0) { + // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL + + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? + 72 / sizeof(msgpack_zone_finalizer) : 8; + + } else { + nnext = (fa->end - fa->array) * 2; + } + + fin = (msgpack_zone_finalizer*)realloc(fa->array, + sizeof(msgpack_zone_finalizer) * nnext); + if(!fin) { + return false; + } + + fa->array = fin; + fa->end = fin + nnext; + fin = fa->tail = fin + nused; + } + + fin->func = func; + fin->data = data; + + ++fa->tail; + + return true; +} + + +bool msgpack_zone_is_empty(msgpack_zone* zone) +{ + msgpack_zone_chunk_array* const ca = &zone->chunk_array; + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + return ca->array[0].ptr == ca->array[0].alloc && + ca->tail == ca->array && + fa->tail == fa->array; +} + + +bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size) +{ + zone->chunk_size = chunk_size; + + if(!init_chunk_array(&zone->chunk_array, chunk_size)) { + return false; + } + + init_finalizer_array(&zone->finalizer_array); + + return true; +} + +void msgpack_zone_destroy(msgpack_zone* zone) +{ + destroy_finalizer_array(&zone->finalizer_array); + destroy_chunk_array(&zone->chunk_array); +} + + +msgpack_zone* msgpack_zone_new(size_t chunk_size) +{ + msgpack_zone* zone = (msgpack_zone*)malloc(sizeof(msgpack_zone)); + if(zone == NULL) { + return NULL; + } + + if(!msgpack_zone_init(zone, chunk_size)) { + free(zone); + return NULL; + } + + return zone; +} + +void msgpack_zone_free(msgpack_zone* zone) +{ + msgpack_zone_destroy(zone); + free(zone); } diff --git a/c/zone.h b/c/zone.h index ff6ca62..3dc9f52 100644 --- a/c/zone.h +++ b/c/zone.h @@ -19,19 +19,58 @@ #define MSGPACK_ZONE_H__ #include +#include #ifdef __cplusplus extern "C" { #endif -struct _msgpack_zone; -typedef struct _msgpack_zone msgpack_zone; +typedef struct msgpack_zone_chunk { + size_t free; + char* ptr; + void* alloc; +} msgpack_zone_chunk; -msgpack_zone* msgpack_zone_new(); -void msgpack_zone_free(msgpack_zone* z); +typedef struct msgpack_zone_finalizer { + void (*func)(void* data); + void* data; +} msgpack_zone_finalizer; -void* msgpack_zone_malloc(msgpack_zone* z, size_t size); +typedef struct msgpack_zone_chunk_array { + msgpack_zone_chunk* tail; + msgpack_zone_chunk* end; + msgpack_zone_chunk* array; +} msgpack_zone_chunk_array; + +typedef struct msgpack_zone_finalizer_array { + msgpack_zone_finalizer* tail; + msgpack_zone_finalizer* end; + msgpack_zone_finalizer* array; +} msgpack_zone_finalizer_array; + +typedef struct msgpack_zone { + msgpack_zone_chunk_array chunk_array; + msgpack_zone_finalizer_array finalizer_array; + size_t chunk_size; +} msgpack_zone; + +#ifndef MSGPACK_ZONE_CHUNK_SIZE +#define MSGPACK_ZONE_CHUNK_SIZE 2048 +#endif + +bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size); +void msgpack_zone_destroy(msgpack_zone* zone); + +msgpack_zone* msgpack_zone_new(size_t chunk_size); +void msgpack_zone_free(msgpack_zone* zone); + +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data); + +bool msgpack_zone_is_empty(msgpack_zone* zone); #ifdef __cplusplus diff --git a/configure.in b/configure.in index a464fb9..9fa6ac5 100644 --- a/configure.in +++ b/configure.in @@ -2,8 +2,42 @@ AC_INIT(msgpack/unpack_template.h) AM_INIT_AUTOMAKE(msgpack, 0.3.0) AC_CONFIG_HEADER(config.h) +AC_SUBST(CFLAGS) +if test "" = "$CFLAGS"; then + CFLAGS="-g -O4" +fi + +AC_PROG_CC + +CFLAGS="-O4 -Wall $CFLAGS -I.." + + +AC_MSG_CHECKING([if c++ api is enabled]) +AC_ARG_ENABLE(cxx, + AS_HELP_STRING([--disable-cxx], + [don't build c++ api.]) ) +AC_MSG_RESULT($enable_cxx) +if test "$enable_cxx" != "no"; then + AC_SUBST(CXXFLAGS) + if test "" = "$CXXFLAGS"; then + CXXFLAGS="-g -O4" + fi + + AC_CHECK_PROG(ERB, erb, erb) + if test "x$ERB" = x; then + AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) + fi +fi + +# FIXME +AC_PROG_CXX + +CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." + + +AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") + AC_PROG_LIBTOOL -AC_CONFIG_SUBDIRS([c cpp]) -AC_OUTPUT([Makefile]) +AC_OUTPUT([Makefile c/Makefile cpp/Makefile]) diff --git a/cpp/AUTHORS b/cpp/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/cpp/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/cpp/COPYING b/cpp/COPYING deleted file mode 100644 index 6f5f220..0000000 --- a/cpp/COPYING +++ /dev/null @@ -1,14 +0,0 @@ -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/cpp/ChangeLog b/cpp/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/cpp/Makefile.am b/cpp/Makefile.am index c7ddf4b..406c57b 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -1,9 +1,7 @@ lib_LTLIBRARIES = libmsgpack.la libmsgpack_la_SOURCES = \ - object.cpp \ - unpack.cpp \ - zone.cpp + object.cpp nobase_include_HEADERS = \ msgpack.hpp \ @@ -28,7 +26,6 @@ noinst_HEADERS = \ # FIXME object.lo: msgpack/type/tuple.hpp unpack.lo: msgpack/type/tuple.hpp msgpack/zone.hpp -zone.lo: msgpack/type/tuple.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp @@ -42,6 +39,8 @@ MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ msgpack/zone.hpp +libmsgpack_la_LIBADD = -L../c -lmsgpackc + # -version-info CURRENT:REVISION:AGE libmsgpack_la_LDFLAGS = -version-info 1:0:0 diff --git a/cpp/NEWS b/cpp/NEWS deleted file mode 100644 index e69de29..0000000 diff --git a/cpp/README b/cpp/README deleted file mode 100644 index 96f18b1..0000000 --- a/cpp/README +++ /dev/null @@ -1,21 +0,0 @@ -MessagePack for C++ -------------------- -MessagePack is a binary-based efficient data interchange format. - - - -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - diff --git a/cpp/bootstrap b/cpp/bootstrap deleted file mode 100755 index 7e61b82..0000000 --- a/cpp/bootstrap +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -NO_NEST=1 -. ../bootstrap diff --git a/cpp/configure.in b/cpp/configure.in deleted file mode 100644 index 2c3e5d0..0000000 --- a/cpp/configure.in +++ /dev/null @@ -1,23 +0,0 @@ -AC_INIT(object.cpp) -AM_INIT_AUTOMAKE(msgpack, 0.1.0) -AC_CONFIG_HEADER(config.h) - -AC_SUBST(CXXFLAGS) -if test "" = "$CXXFLAGS"; then - CXXFLAGS="-g -O4" -fi - -AC_PROG_CXX -AC_PROG_LIBTOOL - -AC_CHECK_PROG(ERB, erb, erb) -if test "x$ERB" = x; then - AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) -fi - -AC_CHECK_LIB(stdc++, main) - -CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." - -AC_OUTPUT([Makefile]) - diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index b2a81ef..6027251 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/boolean.hpp b/cpp/type/boolean.hpp index 60f1714..86bd697 100644 --- a/cpp/type/boolean.hpp +++ b/cpp/type/boolean.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/float.hpp b/cpp/type/float.hpp index 2178434..108709d 100644 --- a/cpp/type/float.hpp +++ b/cpp/type/float.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp index 5cd10f6..ecb7b89 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/integer.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 4585d66..c136d53 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/nil.hpp b/cpp/type/nil.hpp index ab0c363..93e66ff 100644 --- a/cpp/type/nil.hpp +++ b/cpp/type/nil.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index b102ee8..b6ace3f 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index a20f5d9..586d84c 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp deleted file mode 100644 index 0b9a476..0000000 --- a/cpp/unpack.cpp +++ /dev/null @@ -1,369 +0,0 @@ -// -// MessagePack for C++ deserializing routine -// -// Copyright (C) 2008-2009 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#include "msgpack/unpack.hpp" -#include "msgpack/unpack_define.h" -#include - -namespace msgpack { - - -//namespace { -struct unpack_user { - zone* z; - bool referenced; -}; -//} // noname namespace - - -#define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name - -#define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name - -#define msgpack_unpack_callback(name) \ - msgpack_unpack ## name - -#define msgpack_unpack_object object - -#define msgpack_unpack_user unpack_user - - -struct msgpack_unpacker_context; - -static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); - -static object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); - -static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, - const char* data, size_t len, size_t* off); - - -static inline object msgpack_unpack_init(unpack_user* u) -{ return object(); } - -static inline object msgpack_unpack_uint8(unpack_user* u, uint8_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint16(unpack_user* u, uint16_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint32(unpack_user* u, uint32_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint64(unpack_user* u, uint64_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_int8(unpack_user* u, int8_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int16(unpack_user* u, int16_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int32(unpack_user* u, int32_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int64(unpack_user* u, int64_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_float(unpack_user* u, float d) -{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } - -static inline object msgpack_unpack_double(unpack_user* u, double d) -{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } - -static inline object msgpack_unpack_nil(unpack_user* u) -{ object o; o.type = type::NIL; return o; } - -static inline object msgpack_unpack_true(unpack_user* u) -{ object o; o.type = type::BOOLEAN; o.via.boolean = true; return o; } - -static inline object msgpack_unpack_false(unpack_user* u) -{ object o; o.type = type::BOOLEAN; o.via.boolean = false; return o; } - -static inline object msgpack_unpack_array(unpack_user* u, unsigned int n) -{ - object o; - o.type = type::ARRAY; - o.via.array.size = 0; - o.via.array.ptr = (object*)u->z->malloc(n*sizeof(object)); - return o; -} - -static inline void msgpack_unpack_array_item(unpack_user* u, object* c, object o) -{ c->via.array.ptr[c->via.array.size++] = o; } - -static inline object msgpack_unpack_map(unpack_user* u, unsigned int n) -{ - object o; - o.type = type::MAP; - o.via.map.size = 0; - o.via.map.ptr = (object_kv*)u->z->malloc(n*sizeof(object_kv)); - return o; -} - -static inline void msgpack_unpack_map_item(unpack_user* u, object* c, object k, object v) -{ - c->via.map.ptr[c->via.map.size].key = k; - c->via.map.ptr[c->via.map.size].val = v; - ++c->via.map.size; -} - -static inline object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) -{ - object o; - o.type = type::RAW; - o.via.raw.ptr = p; - o.via.raw.size = l; - u->referenced = true; - return o; -} - -#include "msgpack/unpack_template.h" - - -namespace { -struct context { - context() - { - msgpack_unpacker_init(&m_ctx); - unpack_user u = {NULL, false}; - m_ctx.user = u; - } - - ~context() { } - - int execute(const char* data, size_t len, size_t* off) - { - return msgpack_unpacker_execute(&m_ctx, data, len, off); - } - - object data() - { - return msgpack_unpacker_data(&m_ctx); - } - - void reset() - { - zone* z = m_ctx.user.z; - msgpack_unpacker_init(&m_ctx); - unpack_user u = {z, false}; - m_ctx.user = u; - } - - void set_zone(zone* z) - { - m_ctx.user.z = z; - } - - bool is_referenced() const - { - return m_ctx.user.referenced; - } - -private: - msgpack_unpacker_context m_ctx; - zone* m_zone; - -private: - context(const context&); -}; - -static inline context* as_ctx(void* m) -{ - return reinterpret_cast(m); -} - - -static const size_t COUNTER_SIZE = sizeof(unsigned int); - -static inline void init_count(void* buffer) -{ - *(volatile unsigned int*)buffer = 1; -} - -static inline void decl_count(void* buffer) -{ - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { - free(buffer); - } -} - -static inline void incr_count(void* buffer) -{ - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); -} - -static inline unsigned int get_count(void* buffer) -{ - return *(volatile unsigned int*)buffer; -} - -} // noname namespace - - -unpacker::unpacker(size_t initial_buffer_size) : - m_buffer(NULL), - m_used(0), - m_free(0), - m_off(0), - m_zone(new zone()), - m_ctx(new context()), - m_initial_buffer_size(initial_buffer_size) -{ - if(m_initial_buffer_size < COUNTER_SIZE) { - m_initial_buffer_size = COUNTER_SIZE; - } - - as_ctx(m_ctx)->set_zone(m_zone.get()); - - m_buffer = (char*)::malloc(m_initial_buffer_size); - if(!m_buffer) { throw std::bad_alloc(); } - init_count(m_buffer); - - m_used = COUNTER_SIZE; - m_free = m_initial_buffer_size - m_used; - m_off = COUNTER_SIZE; -} - - -unpacker::~unpacker() -{ - delete as_ctx(m_ctx); - decl_count(m_buffer); -} - -void unpacker::expand_buffer(size_t len) -{ - if(m_used == m_off && get_count(m_buffer) == 1 && - !as_ctx(m_ctx)->is_referenced()) { - // rewind buffer - m_free += m_used - COUNTER_SIZE; - m_used = COUNTER_SIZE; - m_off = COUNTER_SIZE; - if(m_free >= len) { return; } - } - - if(m_off == COUNTER_SIZE) { - size_t next_size = (m_used + m_free) * 2; - while(next_size < len + m_used) { next_size *= 2; } - - char* tmp = (char*)::realloc(m_buffer, next_size); - if(!tmp) { throw std::bad_alloc(); } - - m_buffer = tmp; - m_free = next_size - m_used; - - } else { - size_t next_size = m_initial_buffer_size; // include COUNTER_SIZE - size_t not_parsed = m_used - m_off; - while(next_size < len + not_parsed + COUNTER_SIZE) { next_size *= 2; } - - char* tmp = (char*)::malloc(next_size); - if(!tmp) { throw std::bad_alloc(); } - init_count(tmp); - - try { - m_zone->push_finalizer(decl_count, m_buffer); - } catch (...) { free(tmp); throw; } - - memcpy(tmp+COUNTER_SIZE, m_buffer+m_off, not_parsed); - - m_buffer = tmp; - m_used = not_parsed + COUNTER_SIZE; - m_free = next_size - m_used; - m_off = COUNTER_SIZE; - } -} - -bool unpacker::execute() -{ - int ret = as_ctx(m_ctx)->execute(m_buffer, m_used, &m_off); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - return false; - } else { - return true; - } -} - -zone* unpacker::release_zone() -{ - m_zone->push_finalizer(decl_count, m_buffer); - incr_count(m_buffer); - - //std::auto_ptr old(new zone()); - //m_zone.swap(old); - zone* n = new zone(); - std::auto_ptr old(m_zone.release()); - m_zone.reset(n); - - as_ctx(m_ctx)->set_zone(m_zone.get()); - - return old.release(); -} - -object unpacker::data() -{ - return as_ctx(m_ctx)->data(); -} - -void unpacker::reset() -{ - //if(!m_zone->empty()) { delete release_zone(); } - as_ctx(m_ctx)->reset(); -} - - -object unpacker::unpack(const char* data, size_t len, zone& z, size_t* off) -{ - context ctx; - ctx.set_zone(&z); - if(off) { - size_t noff = *off; - int ret = ctx.execute(data, len, &noff); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - throw unpack_error("insufficient bytes"); - } - *off = noff; - } else { - size_t noff = 0; - int ret = ctx.execute(data, len, &noff); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - throw unpack_error("insufficient bytes"); - } else if(noff < len) { - throw unpack_error("extra bytes"); - } - } - return ctx.data(); -} - - -} // namespace msgpack - diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index cde45e7..8c77726 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -18,6 +18,7 @@ #ifndef MSGPACK_UNPACK_HPP__ #define MSGPACK_UNPACK_HPP__ +#include "msgpack/unpack.h" #include "msgpack/object.hpp" #include "msgpack/zone.hpp" #include @@ -36,21 +37,21 @@ struct unpack_error : public std::runtime_error { }; -class unpacker { +class unpacker : public msgpack_unpacker { public: unpacker(size_t initial_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); ~unpacker(); public: - /*! 1. reserve buffer. at least `len' bytes of capacity will be ready */ - void reserve_buffer(size_t len); + /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ + void reserve_buffer(size_t size); /*! 2. read data to the buffer() up to buffer_capacity() bytes */ char* buffer(); size_t buffer_capacity() const; /*! 3. specify the number of bytes actually copied */ - void buffer_consumed(size_t len); + void buffer_consumed(size_t size); /*! 4. repeat execute() until it retunrs false */ bool execute(); @@ -114,71 +115,157 @@ public: size_t nonparsed_size() const; /*! skip specified size of non-parsed buffer, leaving the buffer */ - // Note that the `len' argument must be smaller than nonparsed_size() - void skip_nonparsed_buffer(size_t len); + // Note that the `size' argument must be smaller than nonparsed_size() + void skip_nonparsed_buffer(size_t size); /*! remove unparsed buffer from unpacker */ // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); -private: - char* m_buffer; - size_t m_used; - size_t m_free; - size_t m_off; - - std::auto_ptr m_zone; - - void* m_ctx; - - size_t m_initial_buffer_size; - -private: - void expand_buffer(size_t len); - private: unpacker(const unpacker&); - -public: - static object unpack(const char* data, size_t len, zone& z, size_t* off = NULL); }; -inline void unpacker::reserve_buffer(size_t len) +typedef enum { + MSGPACK_UNPACK_SUCCESS = 2, + MSGPACK_UNPACK_EXTRA_BYTES = 1, + MSGPACK_UNPACK_CONTINUE = 0, + MSGPACK_UNPACK_PARSE_ERROR = -1, +} unpack_return; + +static unpack_return unpack(const char* data, size_t len, size_t* off, + zone* z, object* result); + + +// obsolete +static object unpack(const char* data, size_t len, zone& z, size_t* off = NULL); + + +inline unpacker::unpacker(size_t initial_buffer_size) { - if(m_free >= len) { return; } - expand_buffer(len); + if(!msgpack_unpacker_init(this, initial_buffer_size)) { + throw std::bad_alloc(); + } +} + +inline unpacker::~unpacker() +{ + msgpack_unpacker_destroy(this); +} + +inline void unpacker::reserve_buffer(size_t size) +{ + if(!msgpack_unpacker_reserve_buffer(this, size)) { + throw std::bad_alloc(); + } } inline char* unpacker::buffer() - { return m_buffer + m_used; } +{ + return msgpack_unpacker_buffer(this); +} inline size_t unpacker::buffer_capacity() const - { return m_free; } - -inline void unpacker::buffer_consumed(size_t len) { - m_used += len; - m_free -= len; + return msgpack_unpacker_buffer_capacity(this); +} + +inline void unpacker::buffer_consumed(size_t size) +{ + return msgpack_unpacker_buffer_consumed(this, size); +} + + +inline bool unpacker::execute() +{ + int ret = msgpack_unpacker_execute(this); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + return false; + } else { + return true; + } +} + +inline object unpacker::data() +{ + msgpack_object obj = msgpack_unpacker_data(this); + return *reinterpret_cast(&obj); +} + +inline zone* unpacker::release_zone() +{ + if(!msgpack_unpacker_flush_zone(this)) { + throw std::bad_alloc(); + } + + zone* r = new zone(); + + msgpack_zone old = *this->z; + *this->z = *z; + *z = old; + + return r; +} + +inline void unpacker::reset() +{ + msgpack_unpacker_reset(this); } inline char* unpacker::nonparsed_buffer() - { return m_buffer + m_off; } +{ + return buf + off; +} inline size_t unpacker::nonparsed_size() const - { return m_used - m_off; } +{ + return used - off; +} -inline void unpacker::skip_nonparsed_buffer(size_t len) - { m_off += len; } +inline void unpacker::skip_nonparsed_buffer(size_t size) +{ + off += size; +} inline void unpacker::remove_nonparsed_buffer() - { m_used = m_off; } - - -inline object unpack(const char* data, size_t len, zone& z, size_t* off = NULL) { - return unpacker::unpack(data, len, z, off); + used = off; +} + + +inline unpack_return unpack(const char* data, size_t len, size_t* off, + zone* z, object* result) +{ + return (unpack_return)msgpack_unpack(data, len, off, + z, reinterpret_cast(result)); +} + +inline object unpack(const char* data, size_t len, zone& z, size_t* off) +{ + object result; + + switch( msgpack::unpack(data, len, off, &z, &result) ) { + case MSGPACK_UNPACK_SUCCESS: + return result; + + case MSGPACK_UNPACK_EXTRA_BYTES: + if(off) { + return result; + } else { + throw unpack_error("extra bytes"); + } + + case MSGPACK_UNPACK_CONTINUE: + throw unpack_error("insufficient bytes"); + + case MSGPACK_UNPACK_PARSE_ERROR: + default: + throw unpack_error("parse error"); + } } diff --git a/cpp/zone.cpp b/cpp/zone.cpp deleted file mode 100644 index f765266..0000000 --- a/cpp/zone.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// -// MessagePack for C++ memory pool -// -// Copyright (C) 2008-2009 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#include "msgpack/zone.hpp" -#include - -namespace msgpack { - - -zone::zone(size_t chunk_size) : - m_chunk_size(chunk_size) -{ - chunk dummy = {0, NULL, NULL}; - m_chunk_array.push_back(dummy); -} - -zone::~zone() -{ - clear(); -} - -namespace { - template - struct zone_finalize { - void operator() (Private& f) { - (*f.func)(f.obj); - } - }; - - template - struct zone_free { - void operator() (Private& c) { - ::free(c.alloc); - } - }; -} - -void zone::clear() -{ - std::for_each(m_finalizers.rbegin(), m_finalizers.rend(), - zone_finalize()); - m_finalizers.clear(); - - std::for_each(m_chunk_array.begin(), m_chunk_array.end(), - zone_free()); - m_chunk_array.resize(1); - m_chunk_array[0].ptr = NULL; - m_chunk_array[0].free = 0; -} - -bool zone::empty() const -{ - return m_chunk_array.back().alloc == NULL && - m_finalizers.empty(); -} - -void* zone::malloc(size_t size) -{ - if(m_chunk_array.back().free > size) { - char* p = (char*)m_chunk_array.back().ptr; - m_chunk_array.back().ptr = p + size; - m_chunk_array.back().free -= size; - return p; - } - - size_t sz = m_chunk_size; - while(sz < size) { sz *= 2; } - - chunk dummy = {0, NULL, NULL}; - m_chunk_array.push_back(dummy); - - char* p = (char*)::malloc(sz); - if(!p) { - m_chunk_array.pop_back(); - throw std::bad_alloc(); - } - - m_chunk_array.back().free = sz - size; - m_chunk_array.back().ptr = p + size; - m_chunk_array.back().alloc = p; - return p; -} - - -} // namespace msgpack - diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index c0eb2e3..06cb9d3 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -19,17 +19,15 @@ #define MSGPACK_ZONE_HPP__ #include "msgpack/object.hpp" +#include "msgpack/zone.h" #include #include -#ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 2048 -#endif <% GENERATION_LIMIT = 15 %> namespace msgpack { -class zone { +class zone : public msgpack_zone { public: zone(size_t chunk_size = MSGPACK_ZONE_CHUNK_SIZE); ~zone(); @@ -37,11 +35,7 @@ public: public: void* malloc(size_t size); - void push_finalizer(void (*func)(void*), void* obj); - - void clear(); - - bool empty() const; + void push_finalizer(void (*func)(void*), void* data); <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> @@ -49,35 +43,39 @@ public: <%}%> private: - struct chunk { - size_t free; - void* ptr; - void* alloc; - }; - - std::vector m_chunk_array; - - struct finalizer { - void (*func)(void*); - void* obj; - }; - - std::vector m_finalizers; - template static void object_destructor(void* obj); - size_t m_chunk_size; - private: zone(const zone&); }; -inline void zone::push_finalizer(void (*func)(void*), void* obj) + +inline zone::zone(size_t chunk_size) { - finalizer f = {func, obj}; - m_finalizers.push_back(f); + msgpack_zone_init(this, chunk_size); +} + +inline zone::~zone() +{ + msgpack_zone_destroy(this); +} + +inline void* zone::malloc(size_t size) +{ + void* ptr = msgpack_zone_malloc(this, size); + if(!ptr) { + throw std::bad_alloc(); + } + return ptr; +} + +inline void zone::push_finalizer(void (*func)(void*), void* data) +{ + if(!msgpack_zone_push_finalizer(this, func, data)) { + throw std::bad_alloc(); + } } template @@ -93,7 +91,7 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) void* x = malloc(sizeof(T)); push_finalizer(&zone::object_destructor, x); try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } - catch (...) { m_finalizers.pop_back(); throw; } + catch (...) { --finalizer_array.tail; throw; } } <%}%> @@ -101,3 +99,4 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) #endif /* msgpack/zone.hpp */ +// vim: ft=cpp ts=4 sw=4 softtabstop=4 noexpandtab smarttab diff --git a/example/stream.cc b/example/stream.cc index 49927de..aef4ec8 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -10,6 +10,7 @@ class Server { public: Server(int sock) : m_sock(sock) { } + ~Server() { } typedef std::auto_ptr auto_zone; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index a45369b..ff30955 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -49,7 +49,7 @@ msgpack_unpack_struct_decl(_stack) { }; msgpack_unpack_struct_decl(_context) { - msgpack_unpack_user user; // must be first + msgpack_unpack_user user; unsigned int cs; unsigned int trail; unsigned int top; @@ -63,12 +63,12 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; - ctx->stack[0].obj = msgpack_unpack_callback(_init)(&ctx->user); + ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } -msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* unpacker) +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { - return (unpacker)->stack[0].obj; + return (ctx)->stack[0].obj; } @@ -92,16 +92,14 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c int ret; #define push_simple_value(func) \ - obj = msgpack_unpack_callback(func)(user); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ goto _push #define push_fixed_value(func, arg) \ - obj = msgpack_unpack_callback(func)(user, arg); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ goto _push #define push_variable_value(func, base, pos, len) \ - obj = msgpack_unpack_callback(func)(user, (const char*)base, (const char*)pos, len); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ goto _push #define again_fixed_trail(_cs, trail_len) \ @@ -115,7 +113,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c goto _fixed_trail_again #define start_container(func, count_, ct_) \ - stack[top].obj = msgpack_unpack_callback(func)(user, count_); \ + if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; goto _push; } \ if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ stack[top].ct = ct_; \ @@ -264,11 +262,13 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c case CS_ARRAY_16: start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); case CS_ARRAY_32: + /* FIXME security guard */ start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); case CS_MAP_16: start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); case CS_MAP_32: + /* FIXME security guard */ start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); default: @@ -281,7 +281,7 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - msgpack_unpack_callback(_array_item)(user, &c->obj, obj); + if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; } if(--c->count == 0) { obj = c->obj; --top; @@ -294,7 +294,7 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj); + if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(--c->count == 0) { obj = c->obj; --top; diff --git a/ruby/gem/README b/ruby/gem/README index 78fe6cb..859ae2b 100644 --- a/ruby/gem/README +++ b/ruby/gem/README @@ -24,6 +24,6 @@ == Copyright -Author:: frsyuki -Copyright:: Copyright (c) 2009 frsyuki +Author:: frsyuki +Copyright:: Copyright (c) 2008-2009 frsyuki License:: Apache License, Version 2.0 diff --git a/ruby/gengem.sh b/ruby/gengem.sh index be9d14b..6c07be3 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -1,5 +1,7 @@ #!/bin/sh +mkdir -p gem/ext +mkdir -p gem/msgpack cp extconf.rb gem/ext/ cp pack.c gem/ext/ cp pack.h gem/ext/ diff --git a/ruby/unpack.c b/ruby/unpack.c index 4650ed1..4de4955 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -21,99 +21,99 @@ typedef struct { int finished; - VALUE origstr; -} msgpack_unpack_context; + VALUE source; +} unpack_user; #define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name + struct template ## name #define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name + ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name #define msgpack_unpack_object VALUE -#define msgpack_unpack_user msgpack_unpack_context +#define msgpack_unpack_user unpack_user -struct msgpack_unpacker_context; -typedef struct msgpack_unpacker_context msgpack_unpacker; +struct template_context; +typedef struct template_context msgpack_unpack_t; -static void msgpack_unpacker_init(msgpack_unpacker* ctx); +static void template_init(msgpack_unpack_t* u); -static VALUE msgpack_unpacker_data(msgpack_unpacker* ctx); +static VALUE template_data(msgpack_unpack_t* u); -static int msgpack_unpacker_execute(msgpack_unpacker* ctx, +static int template_execute(msgpack_unpack_t* u, const char* data, size_t len, size_t* off); -static inline VALUE template_callback_init(msgpack_unpack_context* x) +static inline VALUE template_callback_root(unpack_user* u) { return Qnil; } -static inline VALUE template_callback_uint8(msgpack_unpack_context* x, uint8_t d) -{ return INT2FIX(d); } +static inline int template_callback_uint8(unpack_user* u, uint8_t d, VALUE* o) +{ *o = INT2FIX(d); return 0; } -static inline VALUE template_callback_uint16(msgpack_unpack_context* x, uint16_t d) -{ return INT2FIX(d); } +static inline int template_callback_uint16(unpack_user* u, uint16_t d, VALUE* o) +{ *o = INT2FIX(d); return 0; } -static inline VALUE template_callback_uint32(msgpack_unpack_context* x, uint32_t d) -{ return UINT2NUM(d); } +static inline int template_callback_uint32(unpack_user* u, uint32_t d, VALUE* o) +{ *o = UINT2NUM(d); return 0; } -static inline VALUE template_callback_uint64(msgpack_unpack_context* x, uint64_t d) -{ return rb_ull2inum(d); } +static inline int template_callback_uint64(unpack_user* u, uint64_t d, VALUE* o) +{ *o = rb_ull2inum(d); return 0; } -static inline VALUE template_callback_int8(msgpack_unpack_context* x, int8_t d) -{ return INT2FIX((long)d); } +static inline int template_callback_int8(unpack_user* u, int8_t d, VALUE* o) +{ *o = INT2FIX((long)d); return 0; } -static inline VALUE template_callback_int16(msgpack_unpack_context* x, int16_t d) -{ return INT2FIX((long)d); } +static inline int template_callback_int16(unpack_user* u, int16_t d, VALUE* o) +{ *o = INT2FIX((long)d); return 0; } -static inline VALUE template_callback_int32(msgpack_unpack_context* x, int32_t d) -{ return INT2NUM((long)d); } +static inline int template_callback_int32(unpack_user* u, int32_t d, VALUE* o) +{ *o = INT2NUM((long)d); return 0; } -static inline VALUE template_callback_int64(msgpack_unpack_context* x, int64_t d) -{ return rb_ll2inum(d); } +static inline int template_callback_int64(unpack_user* u, int64_t d, VALUE* o) +{ *o = rb_ll2inum(d); return 0; } -static inline VALUE template_callback_float(msgpack_unpack_context* x, float d) -{ return rb_float_new(d); } +static inline int template_callback_float(unpack_user* u, float d, VALUE* o) +{ *o = rb_float_new(d); return 0; } -static inline VALUE template_callback_double(msgpack_unpack_context* x, double d) -{ return rb_float_new(d); } +static inline int template_callback_double(unpack_user* u, double d, VALUE* o) +{ *o = rb_float_new(d); return 0; } -static inline VALUE template_callback_nil(msgpack_unpack_context* x) -{ return Qnil; } +static inline int template_callback_nil(unpack_user* u, VALUE* o) +{ *o = Qnil; return 0; } -static inline VALUE template_callback_true(msgpack_unpack_context* x) -{ return Qtrue; } +static inline int template_callback_true(unpack_user* u, VALUE* o) +{ *o = Qtrue; return 0; } -static inline VALUE template_callback_false(msgpack_unpack_context* x) -{ return Qfalse; } +static inline int template_callback_false(unpack_user* u, VALUE* o) +{ *o = Qfalse; return 0;} -static inline VALUE template_callback_array(msgpack_unpack_context* x, unsigned int n) -{ return rb_ary_new2(n); } +static inline int template_callback_array(unpack_user* u, unsigned int n, VALUE* o) +{ *o = rb_ary_new2(n); return 0; } -static inline void template_callback_array_item(msgpack_unpack_context* x, VALUE* c, VALUE o) -{ rb_ary_push(*c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] +static inline int template_callback_array_item(unpack_user* u, VALUE* c, VALUE o) +{ rb_ary_push(*c, o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] -static inline VALUE template_callback_map(msgpack_unpack_context* x, unsigned int n) -{ return rb_hash_new(); } +static inline int template_callback_map(unpack_user* u, unsigned int n, VALUE* o) +{ *o = rb_hash_new(); return 0; } -static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE* c, VALUE k, VALUE v) -{ rb_hash_aset(*c, k, v); } +static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, VALUE v) +{ rb_hash_aset(*c, k, v); return 0; } -static inline VALUE template_callback_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) -{ return l == 0 ? rb_str_new(0,0) : rb_str_substr(x->origstr, p - b, l); } +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) +{ *o = (l == 0) ? rb_str_new(0,0) : rb_str_substr(u->source, p - b, l); return 0; } #include "msgpack/unpack_template.h" #define UNPACKER(from, name) \ - msgpack_unpacker *name = NULL; \ - Data_Get_Struct(from, msgpack_unpacker, name); \ + msgpack_unpack_t *name = NULL; \ + Data_Get_Struct(from, msgpack_unpack_t, name); \ if(name == NULL) { \ rb_raise(rb_eArgError, "NULL found for " # name " when shouldn't be."); \ } @@ -132,7 +132,7 @@ static void MessagePack_Unpacker_free(void* data) if(data) { free(data); } } -static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) +static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) { unsigned int i; for(i=0; i < mp->top; ++i) { @@ -144,7 +144,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) static VALUE MessagePack_Unpacker_alloc(VALUE klass) { VALUE obj; - msgpack_unpacker* mp = ALLOC_N(msgpack_unpacker, 1); + msgpack_unpack_t* mp = ALLOC_N(msgpack_unpack_t, 1); obj = Data_Wrap_Struct(klass, MessagePack_Unpacker_mark, MessagePack_Unpacker_free, mp); return obj; @@ -153,9 +153,9 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); - msgpack_unpacker_init(mp); - msgpack_unpack_context ctx = {0, Qnil}; - mp->user = ctx; + template_init(mp); + unpack_user u = {0, Qnil}; + mp->user = u; return self; } @@ -180,9 +180,9 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) rb_raise(eUnpackError, "offset is bigger than data buffer size."); } - mp->user.origstr = data; - ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); - mp->user.origstr = Qnil; + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -235,13 +235,13 @@ static VALUE MessagePack_Unpacker_finished_p(VALUE self) static VALUE MessagePack_Unpacker_data(VALUE self) { UNPACKER(self, mp); - return msgpack_unpacker_data(mp); + return template_data(mp); } static VALUE MessagePack_unpack_impl(VALUE args) { - msgpack_unpacker* mp = (msgpack_unpacker*)((VALUE*)args)[0]; + msgpack_unpack_t* mp = (msgpack_unpack_t*)((VALUE*)args)[0]; VALUE data = ((VALUE*)args)[1]; size_t from = 0; @@ -249,9 +249,9 @@ static VALUE MessagePack_unpack_impl(VALUE args) long dlen = FIX2LONG(((VALUE*)args)[2]); int ret; - mp->user.origstr = data; - ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); - mp->user.origstr = Qnil; + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -261,7 +261,7 @@ static VALUE MessagePack_unpack_impl(VALUE args) if(from < dlen) { rb_raise(eUnpackError, "extra bytes."); } - return msgpack_unpacker_data(mp); + return template_data(mp); } } @@ -278,10 +278,10 @@ static VALUE MessagePack_unpack_rescue(VALUE args) static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); - msgpack_unpacker mp; - msgpack_unpacker_init(&mp); - msgpack_unpack_context ctx = {0, Qnil}; - mp.user = ctx; + msgpack_unpack_t mp; + template_init(&mp); + unpack_user u = {0, Qnil}; + mp.user = u; rb_gc_disable(); VALUE args[3] = {(VALUE)&mp, data, limit}; From 11abec1093cf8c9914b5aeeac383d30abc1cad72 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 22 Feb 2009 15:14:21 +0900 Subject: [PATCH 0058/1648] c and c++: rewritten and integrated --- COPYING | 2 +- Makefile.am | 5 + README | 2 +- bootstrap | 8 +- c/AUTHORS | 1 - c/COPYING | 14 -- c/ChangeLog | 0 c/NEWS | 0 c/README | 21 --- c/bootstrap | 3 - c/configure.in | 16 -- c/object.c | 199 ++++++-------------- c/object.h | 23 +-- c/pack.h | 87 +++++---- c/unpack.c | 384 +++++++++++++++++++++++++++++++------- c/unpack.h | 103 ++++++---- c/zone.c | 253 +++++++++++++++++++------ c/zone.h | 49 ++++- configure.in | 38 +++- cpp/AUTHORS | 1 - cpp/COPYING | 14 -- cpp/ChangeLog | 0 cpp/Makefile.am | 7 +- cpp/NEWS | 0 cpp/README | 21 --- cpp/bootstrap | 3 - cpp/configure.in | 23 --- cpp/type/array.hpp | 2 +- cpp/type/boolean.hpp | 2 +- cpp/type/float.hpp | 2 +- cpp/type/integer.hpp | 2 +- cpp/type/map.hpp | 2 +- cpp/type/nil.hpp | 2 +- cpp/type/raw.hpp | 2 +- cpp/type/tuple.hpp.erb | 2 +- cpp/unpack.cpp | 369 ------------------------------------ cpp/unpack.hpp | 171 ++++++++++++----- cpp/zone.cpp | 100 ---------- cpp/zone.hpp.erb | 59 +++--- example/stream.cc | 1 + msgpack/unpack_template.h | 26 +-- ruby/gem/README | 4 +- ruby/gengem.sh | 2 + ruby/unpack.c | 134 ++++++------- 44 files changed, 1035 insertions(+), 1124 deletions(-) delete mode 100644 c/AUTHORS delete mode 100644 c/COPYING delete mode 100644 c/ChangeLog delete mode 100644 c/NEWS delete mode 100644 c/README delete mode 100755 c/bootstrap delete mode 100644 c/configure.in delete mode 100644 cpp/AUTHORS delete mode 100644 cpp/COPYING delete mode 100644 cpp/ChangeLog delete mode 100644 cpp/NEWS delete mode 100644 cpp/README delete mode 100755 cpp/bootstrap delete mode 100644 cpp/configure.in delete mode 100644 cpp/unpack.cpp delete mode 100644 cpp/zone.cpp diff --git a/COPYING b/COPYING index 5f100cd..6f5f220 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile.am b/Makefile.am index c23320a..3144972 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,4 +1,9 @@ +if ENABLE_CXX +export ERB SUBDIRS = c cpp +else +SUBDIRS = c +endif nobase_include_HEADERS = \ msgpack/pack_define.h \ diff --git a/README b/README index 87d9279..fd5fa8d 100644 --- a/README +++ b/README @@ -54,7 +54,7 @@ Binary-based efficient data interchange format. -Copyright (C) 2008 FURUHASHI Sadayuki +Copyright (C) 2008-2009 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/bootstrap b/bootstrap index 954fa89..eb16c35 100755 --- a/bootstrap +++ b/bootstrap @@ -31,10 +31,10 @@ if test x"$1" = x"--help"; then fi -if [ -z "$NO_NEST" ];then - cd c && ./bootstrap $@; cd .. - cd cpp && ./bootstrap $@; cd .. -fi +#if [ -z "$NO_NEST" ];then +# cd c && ./bootstrap $@; cd .. +# cd cpp && ./bootstrap $@; cd .. +#fi ACLOCAL="aclocal" diff --git a/c/AUTHORS b/c/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/c/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/c/COPYING b/c/COPYING deleted file mode 100644 index 6f5f220..0000000 --- a/c/COPYING +++ /dev/null @@ -1,14 +0,0 @@ -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/c/ChangeLog b/c/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/c/NEWS b/c/NEWS deleted file mode 100644 index e69de29..0000000 diff --git a/c/README b/c/README deleted file mode 100644 index 76dc221..0000000 --- a/c/README +++ /dev/null @@ -1,21 +0,0 @@ -MessagePack for C ------------------ -MessagePack is a binary-based efficient data interchange format. - - - -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - diff --git a/c/bootstrap b/c/bootstrap deleted file mode 100755 index 7e61b82..0000000 --- a/c/bootstrap +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -NO_NEST=1 -. ../bootstrap diff --git a/c/configure.in b/c/configure.in deleted file mode 100644 index 47eb8aa..0000000 --- a/c/configure.in +++ /dev/null @@ -1,16 +0,0 @@ -AC_INIT(unpack.c) -AM_INIT_AUTOMAKE(msgpackc, 0.1.0) -AC_CONFIG_HEADER(config.h) - -AC_SUBST(CFLAGS) -if test "" = "$CFLAGS"; then - CFLAGS="-g -O4" -fi - -AC_PROG_CC -AC_PROG_LIBTOOL - -CFLAGS="-O4 -Wall $CFLAGS -I.." - -AC_OUTPUT([Makefile]) - diff --git a/c/object.c b/c/object.c index 6376438..871153d 100644 --- a/c/object.c +++ b/c/object.c @@ -15,153 +15,78 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "msgpack/unpack.h" -#include "msgpack/unpack_define.h" #include "msgpack/object.h" +#include +#include -typedef struct { - msgpack_zone* z; - bool referenced; - bool failed; -} unpack_user; - -#define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name - -#define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name - -#define msgpack_unpack_callback(name) \ - msgpack_unpack ## name - -#define msgpack_unpack_object msgpack_object - -#define msgpack_unpack_user unpack_user - - -struct msgpack_unpacker_context; - -static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); - -static msgpack_object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); - -static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, - const char* data, size_t len, size_t* off); - -static inline msgpack_object msgpack_unpack_init(unpack_user* u) -{ msgpack_object o; return o; } - -static inline msgpack_object msgpack_unpack_uint8(unpack_user* u, uint8_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint16(unpack_user* u, uint16_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint32(unpack_user* u, uint32_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_uint64(unpack_user* u, uint64_t d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline msgpack_object msgpack_unpack_int8(unpack_user* u, int8_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int16(unpack_user* u, int16_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int32(unpack_user* u, int32_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_int64(unpack_user* u, int64_t d) -{ if(d >= 0) { msgpack_object o; o.type = MSGPACK_OBJECT_POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { msgpack_object o; o.type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline msgpack_object msgpack_unpack_float(unpack_user* u, float d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } - -static inline msgpack_object msgpack_unpack_double(unpack_user* u, double d) -{ msgpack_object o; o.type = MSGPACK_OBJECT_DOUBLE; o.via.dec = d; return o; } - -static inline msgpack_object msgpack_unpack_nil(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_NIL; return o; } - -static inline msgpack_object msgpack_unpack_true(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = true; return o; } - -static inline msgpack_object msgpack_unpack_false(unpack_user* u) -{ msgpack_object o; o.type = MSGPACK_OBJECT_BOOLEAN; o.via.boolean = false; return o; } - -static inline msgpack_object msgpack_unpack_array(unpack_user* u, unsigned int n) +void msgpack_object_print(FILE* out, msgpack_object o) { - msgpack_object o; - o.type = MSGPACK_OBJECT_ARRAY; - o.via.array.size = 0; - o.via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); - if(o.via.array.ptr == NULL) { u->failed = true; } - return o; -} + switch(o.type) { + case MSGPACK_OBJECT_NIL: + fprintf(out, "nil"); + break; -static inline void msgpack_unpack_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) -{ - if(u->failed) { return; } - c->via.array.ptr[ c->via.array.size++ ] = o; -} + case MSGPACK_OBJECT_BOOLEAN: + fprintf(out, (o.via.boolean ? "true" : "false")); + break; -static inline msgpack_object msgpack_unpack_map(unpack_user* u, unsigned int n) -{ - msgpack_object o; - o.type = MSGPACK_OBJECT_MAP; - o.via.map.size = 0; - o.via.map.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object_kv)); - if(o.via.map.ptr == NULL) { u->failed = true; } - return o; -} + case MSGPACK_OBJECT_POSITIVE_INTEGER: + fprintf(out, "%"PRIu64, o.via.u64); + break; -static inline void msgpack_unpack_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) -{ - if(u->failed) { return; } - c->via.map.ptr[c->via.map.size].key = k; - c->via.map.ptr[c->via.map.size].val = v; - ++c->via.map.size; -} + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + fprintf(out, "%"PRIi64, o.via.i64); + break; -static inline msgpack_object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) -{ - msgpack_object o; - o.type = MSGPACK_OBJECT_RAW; - o.via.raw.ptr = p; - o.via.raw.size = l; - u->referenced = true; - return o; -} + case MSGPACK_OBJECT_DOUBLE: + fprintf(out, "%f", o.via.dec); + break; -#include "msgpack/unpack_template.h" + case MSGPACK_OBJECT_RAW: + fprintf(out, "\""); + fwrite(o.via.raw.ptr, o.via.raw.size, 1, out); + fprintf(out, "\""); + break; -msgpack_object_unpack_return -msgpack_object_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_object* result) -{ - struct msgpack_unpacker_context ctx; - msgpack_unpacker_init(&ctx); - unpack_user u = {z, false, false}; - ctx.user = u; + case MSGPACK_OBJECT_ARRAY: + fprintf(out, "["); + if(o.via.array.size != 0) { + msgpack_object* p = o.via.array.ptr; + msgpack_object_print(out, *p); + ++p; + msgpack_object* const pend = o.via.array.ptr + o.via.array.size; + for(; p < pend; ++p) { + fprintf(out, ", "); + msgpack_object_print(out, *p); + } + } + fprintf(out, "]"); + break; + // FIXME loop optimiziation - size_t noff = (off ? *off : 0); - int ret = msgpack_unpacker_execute(&ctx, data, len, &noff); - if(ret < 0 || ctx.user.failed) { - return MSGPACK_OBJECT_PARSE_ERROR; - } else if(ret == 0) { - return MSGPACK_OBJECT_INSUFFICIENT_BYTES; + case MSGPACK_OBJECT_MAP: + fprintf(out, "{"); + if(o.via.map.size != 0) { + msgpack_object_kv* p = o.via.map.ptr; + msgpack_object_print(out, p->key); + fprintf(out, "=>"); + msgpack_object_print(out, p->val); + ++p; + msgpack_object_kv* const pend = o.via.map.ptr + o.via.map.size; + for(; p < pend; ++p) { + fprintf(out, ", "); + msgpack_object_print(out, p->key); + fprintf(out, "=>"); + msgpack_object_print(out, p->val); + } + } + fprintf(out, "}"); + break; + // FIXME loop optimiziation + + default: + // FIXME + fprintf(out, "#", o.type, o.via.u64); } - *result = msgpack_unpacker_data(&ctx); - if(off) { *off = noff; } - if(ret == 0) { - return MSGPACK_OBJECT_EXTRA_BYTES; - } - return MSGPACK_OBJECT_PARSE_SUCCESS; } - diff --git a/c/object.h b/c/object.h index b590ebe..7c603b3 100644 --- a/c/object.h +++ b/c/object.h @@ -22,6 +22,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -39,18 +40,18 @@ typedef enum { MSGPACK_OBJECT_MAP = 0x08, } msgpack_object_type; -struct _msgpack_object; -struct _msgpack_object_kv; +struct msgpack_object; +struct msgpack_object_kv; typedef struct { uint32_t size; - struct _msgpack_object* ptr; + struct msgpack_object* ptr; } msgpack_object_array; typedef struct { uint32_t size; - struct _msgpack_object_kv* ptr; + struct msgpack_object_kv* ptr; } msgpack_object_map; typedef struct { @@ -68,26 +69,18 @@ typedef union { msgpack_object_raw raw; } msgpack_object_union; -typedef struct _msgpack_object { +typedef struct msgpack_object { msgpack_object_type type; msgpack_object_union via; } msgpack_object; -typedef struct _msgpack_object_kv { +typedef struct msgpack_object_kv { msgpack_object key; msgpack_object val; } msgpack_object_kv; -typedef enum { - MSGPACK_OBJECT_PARSE_SUCCESS = 0, - MSGPACK_OBJECT_EXTRA_BYTES = 1, - MSGPACK_OBJECT_INSUFFICIENT_BYTES = -1, - MSGPACK_OBJECT_PARSE_ERROR = -2, -} msgpack_object_unpack_return; -msgpack_object_unpack_return -msgpack_object_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_object* result); +void msgpack_object_print(FILE* out, msgpack_object o); #ifdef __cplusplus diff --git a/c/pack.h b/c/pack.h index a510eff..46de722 100644 --- a/c/pack.h +++ b/c/pack.h @@ -28,50 +28,49 @@ extern "C" { #endif -typedef int (*msgpack_pack_write_t)(void* data, const char* buf, unsigned int len); +typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); -typedef struct { +typedef struct msgpack_packer { void* data; - msgpack_pack_write_t callback; -} msgpack_pack_t; + msgpack_packer_write callback; +} msgpack_packer; -static void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback); +static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback); +static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static void msgpack_packer_free(msgpack_packer* pk); -static void msgpack_pack_free(msgpack_pack_t* ctx); +static int msgpack_pack_short(msgpack_packer* pk, short d); +static int msgpack_pack_int(msgpack_packer* pk, int d); +static int msgpack_pack_long(msgpack_packer* pk, long d); +static int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); -static int msgpack_pack_short(msgpack_pack_t* ctx, short d); -static int msgpack_pack_int(msgpack_pack_t* ctx, int d); -static int msgpack_pack_long(msgpack_pack_t* ctx, long d); -static int msgpack_pack_long_long(msgpack_pack_t* ctx, long long d); -static int msgpack_pack_unsigned_short(msgpack_pack_t* ctx, unsigned short d); -static int msgpack_pack_unsigned_int(msgpack_pack_t* ctx, unsigned int d); -static int msgpack_pack_unsigned_long(msgpack_pack_t* ctx, unsigned long d); -static int msgpack_pack_unsigned_long_long(msgpack_pack_t* ctx, unsigned long long d); +static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); -static int msgpack_pack_uint8(msgpack_pack_t* ctx, uint8_t d); -static int msgpack_pack_uint16(msgpack_pack_t* ctx, uint16_t d); -static int msgpack_pack_uint32(msgpack_pack_t* ctx, uint32_t d); -static int msgpack_pack_uint64(msgpack_pack_t* ctx, uint64_t d); -static int msgpack_pack_int8(msgpack_pack_t* ctx, int8_t d); -static int msgpack_pack_int16(msgpack_pack_t* ctx, int16_t d); -static int msgpack_pack_int32(msgpack_pack_t* ctx, int32_t d); -static int msgpack_pack_int64(msgpack_pack_t* ctx, int64_t d); +static int msgpack_pack_float(msgpack_packer* pk, float d); +static int msgpack_pack_double(msgpack_packer* pk, double d); -static int msgpack_pack_float(msgpack_pack_t* ctx, float d); -static int msgpack_pack_double(msgpack_pack_t* ctx, double d); +static int msgpack_pack_nil(msgpack_packer* pk); +static int msgpack_pack_true(msgpack_packer* pk); +static int msgpack_pack_false(msgpack_packer* pk); -static int msgpack_pack_nil(msgpack_pack_t* ctx); -static int msgpack_pack_true(msgpack_pack_t* ctx); -static int msgpack_pack_false(msgpack_pack_t* ctx); +static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_array(msgpack_pack_t* ctx, unsigned int n); +static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_map(msgpack_pack_t* ctx, unsigned int n); - -static int msgpack_pack_raw(msgpack_pack_t* ctx, size_t l); -static int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); +static int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); @@ -81,30 +80,30 @@ static int msgpack_pack_raw_body(msgpack_pack_t* ctx, const void* b, size_t l); #define msgpack_pack_inline_func_cint(name) \ inline int msgpack_pack ## name -#define msgpack_pack_user msgpack_pack_t* +#define msgpack_pack_user msgpack_packer* #define msgpack_pack_append_buffer(user, buf, len) \ return (*(user)->callback)((user)->data, (const char*)buf, len) #include "msgpack/pack_template.h" -inline void msgpack_pack_init(msgpack_pack_t* ctx, void* data, msgpack_pack_write_t callback) +inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) { - ctx->data = data; - ctx->callback = callback; + pk->data = data; + pk->callback = callback; } -inline msgpack_pack_t* msgpack_pack_new(void* data, msgpack_pack_write_t callback) +inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) { - msgpack_pack_t* ctx = (msgpack_pack_t*)calloc(1, sizeof(msgpack_pack_t)); - if(!ctx) { return NULL; } - msgpack_pack_init(ctx, data, callback); - return ctx; + msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); + if(!pk) { return NULL; } + msgpack_packer_init(pk, data, callback); + return pk; } -inline void msgpack_pack_free(msgpack_pack_t* ctx) +inline void msgpack_packer_free(msgpack_packer* pk) { - free(ctx); + free(pk); } diff --git a/c/unpack.c b/c/unpack.c index 03c67be..e06d97c 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -20,125 +20,371 @@ #include +typedef struct { + msgpack_zone* z; + bool* referenced; +} unpack_user; + + #define msgpack_unpack_struct(name) \ struct template ## name #define msgpack_unpack_func(ret, name) \ - ret template_func ## name + ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name -#define msgpack_unpack_object void* +#define msgpack_unpack_object msgpack_object -#define msgpack_unpack_user msgpack_unpack_t +#define msgpack_unpack_user unpack_user struct template_context; +typedef struct template_context template_context; -static void template_func_init(struct template_context* ctx); +static void template_init(template_context* ctx); -static void* template_func_data(struct template_context* ctx); +static msgpack_object template_data(template_context* ctx); -static int template_func_execute(struct template_context* ctx, +static int template_execute(template_context* ctx, const char* data, size_t len, size_t* off); -static inline void* template_callback_init(msgpack_unpack_t* x) -{ return NULL; } +static inline msgpack_object template_callback_root(unpack_user* u) +{ msgpack_object o; return o; } -static inline void* template_callback_uint8(msgpack_unpack_t* x, uint8_t d) -{ return x->callback.unpack_uint8(x->data, d); } +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint16(msgpack_unpack_t* x, uint16_t d) -{ return x->callback.unpack_uint16(x->data, d); } +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint32(msgpack_unpack_t* x, uint32_t d) -{ return x->callback.unpack_uint32(x->data, d); } +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_uint64(msgpack_unpack_t* x, uint64_t d) -{ return x->callback.unpack_uint64(x->data, d); } +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } -static inline void* template_callback_int8(msgpack_unpack_t* x, int8_t d) -{ return x->callback.unpack_int8(x->data, d); } +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int16(msgpack_unpack_t* x, int16_t d) -{ return x->callback.unpack_int16(x->data, d); } +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int32(msgpack_unpack_t* x, int32_t d) -{ return x->callback.unpack_int32(x->data, d); } +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_int64(msgpack_unpack_t* x, int64_t d) -{ return x->callback.unpack_int64(x->data, d); } +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_object* o) +{ if(d >= 0) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } + else { o->type = MSGPACK_OBJECT_NEGATIVE_INTEGER; o->via.i64 = d; return 0; } } -static inline void* template_callback_float(msgpack_unpack_t* x, float d) -{ return x->callback.unpack_float(x->data, d); } +static inline int template_callback_float(unpack_user* u, float d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_DOUBLE; o->via.dec = d; return 0; } -static inline void* template_callback_double(msgpack_unpack_t* x, double d) -{ return x->callback.unpack_double(x->data, d); } +static inline int template_callback_double(unpack_user* u, double d, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_DOUBLE; o->via.dec = d; return 0; } -static inline void* template_callback_nil(msgpack_unpack_t* x) -{ return x->callback.unpack_nil(x->data); } +static inline int template_callback_nil(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_NIL; return 0; } -static inline void* template_callback_true(msgpack_unpack_t* x) -{ return x->callback.unpack_true(x->data); } +static inline int template_callback_true(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_BOOLEAN; o->via.boolean = true; return 0; } -static inline void* template_callback_false(msgpack_unpack_t* x) -{ return x->callback.unpack_false(x->data); } +static inline int template_callback_false(unpack_user* u, msgpack_object* o) +{ o->type = MSGPACK_OBJECT_BOOLEAN; o->via.boolean = false; return 0; } -static inline void* template_callback_array(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_array(x->data, n); } +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_ARRAY; + o->via.array.size = 0; + o->via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); + if(o->via.array.ptr == NULL) { return -1; } + return 0; +} -static inline void template_callback_array_item(msgpack_unpack_t* x, void** c, void* o) -{ x->callback.unpack_array_item(x->data, *c, o); } +static inline int template_callback_array_item(unpack_user* u, msgpack_object* c, msgpack_object o) +{ c->via.array.ptr[c->via.array.size++] = o; return 0; } -static inline void* template_callback_map(msgpack_unpack_t* x, unsigned int n) -{ return x->callback.unpack_map(x->data, n); } +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_MAP; + o->via.map.size = 0; + o->via.map.ptr = (msgpack_object_kv*)msgpack_zone_malloc(u->z, n*sizeof(msgpack_object_kv)); + if(o->via.map.ptr == NULL) { return -1; } + return 0; +} -static inline void template_callback_map_item(msgpack_unpack_t* x, void** c, void* k, void* v) -{ x->callback.unpack_map_item(x->data, *c, k, v); } - -static inline void* template_callback_raw(msgpack_unpack_t* x, const char* b, const char* p, unsigned int l) -{ return x->callback.unpack_raw(x->data, b, p, l); } +static inline int template_callback_map_item(unpack_user* u, msgpack_object* c, msgpack_object k, msgpack_object v) +{ + c->via.map.ptr[c->via.map.size].key = k; + c->via.map.ptr[c->via.map.size].val = v; + ++c->via.map.size; + return 0; +} +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_object* o) +{ + o->type = MSGPACK_OBJECT_RAW; + o->via.raw.ptr = p; + o->via.raw.size = l; + *u->referenced = true; + return 0; +} #include "msgpack/unpack_template.h" -msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback) +#define CTX_CAST(m) ((template_context*)(m)) + + +static const size_t COUNTER_SIZE = sizeof(unsigned int); + +static inline void init_count(void* buf) { - struct template_context* ctx; - ctx = (struct template_context*)calloc(1, sizeof(struct template_context)); - if(ctx == NULL) { return NULL; } - template_func_init(ctx); - ((msgpack_unpack_t*)ctx)->data = data; - ((msgpack_unpack_t*)ctx)->callback = *callback; - return (msgpack_unpack_t*)ctx; + *(volatile unsigned int*)buf = 1; } -void msgpack_unpack_free(msgpack_unpack_t* ctx) +static inline void decl_count(void* buf) { - free((struct template_context*)ctx); + //if(--*(unsigned int*)buf == 0) { + if(__sync_sub_and_fetch((unsigned int*)buf, 1) == 0) { + free(buf); + } } -void* msgpack_unpack_data(msgpack_unpack_t* ctx) +static inline void incr_count(void* buf) { - return template_func_data((struct template_context*)ctx); + //++*(unsigned int*)buf; + __sync_add_and_fetch((unsigned int*)buf, 1); } -void msgpack_unpack_reset(msgpack_unpack_t* ctx) +static inline unsigned int get_count(void* buf) { - msgpack_unpack_t x = ((struct template_context*)ctx)->user; - template_func_init((struct template_context*)ctx); - ((struct template_context*)ctx)->user = x; -} - -int msgpack_unpack_execute(msgpack_unpack_t* ctx, - const char* data, size_t len, size_t* off) -{ - return template_func_execute( - (struct template_context*)ctx, - data, len, off); + return *(volatile unsigned int*)buf; } + +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) +{ + if(initial_buffer_size < COUNTER_SIZE) { + initial_buffer_size = COUNTER_SIZE; + } + + char* buf = (char*)malloc(initial_buffer_size); + if(buf == NULL) { + return false; + } + + void* ctx = malloc(sizeof(template_context)); + if(ctx == NULL) { + free(buf); + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + free(ctx); + free(buf); + return false; + } + + mpac->buf = buf; + mpac->used = COUNTER_SIZE; + mpac->free = initial_buffer_size - mpac->used; + mpac->off = COUNTER_SIZE; + mpac->initial_buffer_size = initial_buffer_size; + mpac->z = z; + mpac->referenced = false; + mpac->ctx = ctx; + + init_count(mpac->buf); + + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = mpac->z; + CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + + return true; +} + +void msgpack_unpacker_destroy(msgpack_unpacker* mpac) +{ + msgpack_zone_free(mpac->z); + free(mpac->ctx); + decl_count(mpac->buf); +} + + +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) +{ + msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); + if(mpac == NULL) { + return NULL; + } + + if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { + free(mpac); + return NULL; + } + + return mpac; +} + +void msgpack_unpacker_free(msgpack_unpacker* mpac) +{ + msgpack_unpacker_destroy(mpac); + free(mpac); +} + + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->used == mpac->off && get_count(mpac->buf) == 1 && !mpac->referenced) { + // rewind buffer + mpac->free += mpac->used - COUNTER_SIZE; + mpac->used = COUNTER_SIZE; + mpac->off = COUNTER_SIZE; + + if(mpac->free >= size) { + return true; + } + } + + if(mpac->off == COUNTER_SIZE) { + size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE + while(next_size < size + mpac->used) { + next_size *= 2; + } + + char* tmp = (char*)realloc(mpac->buf, next_size); + if(tmp == NULL) { + return false; + } + + mpac->buf = tmp; + mpac->free = next_size - mpac->used; + + } else { + size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE + size_t not_parsed = mpac->used - mpac->off; + while(next_size < size + not_parsed + COUNTER_SIZE) { + next_size *= 2; + } + + char* tmp = (char*)malloc(next_size); + if(tmp == NULL) { + return false; + } + + init_count(tmp); + + if(mpac->referenced) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + free(tmp); + return false; + } + mpac->referenced = false; + } else { + decl_count(mpac->buf); + } + + memcpy(tmp+COUNTER_SIZE, mpac->buf+mpac->off, not_parsed); + + mpac->buf = tmp; + mpac->used = not_parsed + COUNTER_SIZE; + mpac->free = next_size - mpac->used; + mpac->off = COUNTER_SIZE; + } + + return true; +} + +int msgpack_unpacker_execute(msgpack_unpacker* mpac) +{ + return template_execute(CTX_CAST(mpac->ctx), + mpac->buf, mpac->used, &mpac->off); +} + +msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) +{ + return template_data(CTX_CAST(mpac->ctx)); +} + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) +{ + if(!msgpack_unpacker_flush_zone(mpac)) { + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + return NULL; + } + + msgpack_zone* old = mpac->z; + mpac->z = z; + + return old; +} + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) +{ + if(mpac->referenced) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + return false; + } + mpac->referenced = false; + + incr_count(mpac->buf); + } + + return true; +} + +void msgpack_unpacker_reset(msgpack_unpacker* mpac) +{ + msgpack_zone* z = mpac->z; + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = z; + CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; +} + + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result) +{ + template_context ctx; + template_init(&ctx); + + bool referenced = false; + ctx.user.z = z; + ctx.user.referenced = &referenced; + + size_t noff = 0; + if(off != NULL) { noff = *off; } + + int ret = template_execute(&ctx, data, len, &noff); + if(ret < 0) { + return MSGPACK_UNPACK_PARSE_ERROR; + } + + if(off != NULL) { *off = noff; } + + if(ret == 0) { + return MSGPACK_UNPACK_CONTINUE; + } + + *result = template_data(&ctx); + + if(noff < len) { + return MSGPACK_UNPACK_EXTRA_BYTES; + } + + return MSGPACK_UNPACK_SUCCESS; +} + diff --git a/c/unpack.h b/c/unpack.h index 4977f51..ab202a1 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -15,9 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MSGPACK_UNPACK_H__ -#define MSGPACK_UNPACK_H__ +#ifndef msgpack_unpacker_H__ +#define msgpack_unpacker_H__ +#include "msgpack/zone.h" +#include "msgpack/object.h" #include #include @@ -26,39 +28,76 @@ extern "C" { #endif -typedef struct { - void* (*unpack_uint8)(void* data, uint8_t d); - void* (*unpack_uint16)(void* data, uint16_t d); - void* (*unpack_uint32)(void* data, uint32_t d); - void* (*unpack_uint64)(void* data, uint64_t d); - void* (*unpack_int8)(void* data, int8_t d); - void* (*unpack_int16)(void* data, int16_t d); - void* (*unpack_int32)(void* data, int32_t d); - void* (*unpack_int64)(void* data, int64_t d); - void* (*unpack_float)(void* data, float d); - void* (*unpack_double)(void* data, double d); - void* (*unpack_nil)(void* data); - void* (*unpack_true)(void* data); - void* (*unpack_false)(void* data); - void* (*unpack_array)(void* data, unsigned int n); - void (*unpack_array_item)(void* data, void* c, void* o); - void* (*unpack_map)(void* data, unsigned int n); - void (*unpack_map_item)(void* data, void* c, void* k, void* v); - void* (*unpack_raw)(void* data, const char* b, const char* p, unsigned int l); -} msgpack_unpack_callback; +typedef struct msgpack_unpacker { + char* buf; + size_t used; + size_t free; + size_t off; + msgpack_zone* z; + bool referenced; + size_t initial_buffer_size; + void* ctx; +} msgpack_unpacker; -typedef struct { - void* data; - msgpack_unpack_callback callback; -} msgpack_unpack_t; -msgpack_unpack_t* msgpack_unpack_new(void* data, msgpack_unpack_callback* callback); -void msgpack_unpack_free(msgpack_unpack_t* ctx); +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size); +void msgpack_unpacker_destroy(msgpack_unpacker* mpac); -int msgpack_unpack_execute(msgpack_unpack_t* ctx, - const char* data, size_t len, size_t* off); -void* msgpack_unpack_data(msgpack_unpack_t* ctx); -void msgpack_unpack_reset(msgpack_unpack_t* ctx); +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); +void msgpack_unpacker_free(msgpack_unpacker* mpac); + +static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); +static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); +static inline void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size); + + +int msgpack_unpacker_execute(msgpack_unpacker* mpac); + +msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac); + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); + +void msgpack_unpacker_reset(msgpack_unpacker* mpac); + + +typedef enum { + MSGPACK_UNPACK_SUCCESS = 2, + MSGPACK_UNPACK_EXTRA_BYTES = 1, + MSGPACK_UNPACK_CONTINUE = 0, + MSGPACK_UNPACK_PARSE_ERROR = -1, +} msgpack_unpack_return; + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_object* result); + + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); + +bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->free >= size) { return true; } + return msgpack_unpacker_expand_buffer(mpac, size); +} + +char* msgpack_unpacker_buffer(msgpack_unpacker* mpac) +{ + return mpac->buf + mpac->used; +} + +size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac) +{ + return mpac->free; +} + +void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) +{ + mpac->used += size; + mpac->free -= size; +} #ifdef __cplusplus diff --git a/c/zone.c b/c/zone.c index ccd702d..e891c82 100644 --- a/c/zone.c +++ b/c/zone.c @@ -19,85 +19,214 @@ #include #include -typedef struct { - size_t free; - void* ptr; - void* alloc; -} msgpack_zone_chunk; -struct _msgpack_zone { - msgpack_zone_chunk* array; - size_t ntail; - size_t usable; -}; - -msgpack_zone* msgpack_zone_new() +static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_size) { - return calloc(1, sizeof(msgpack_zone)); -} + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + const size_t nfirst = (sizeof(msgpack_zone_chunk) < 72/2) ? + 72 / sizeof(msgpack_zone_chunk) : 8; -void msgpack_zone_free(msgpack_zone* z) -{ - if(z->array) { - size_t i; - for(i=0; i <= z->ntail; ++i) { - free(z->array[i].alloc); - } + msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc( + sizeof(msgpack_zone_chunk) * nfirst); + if(!array) { + return false; } - free(z); + + const size_t sz = chunk_size; + + char* ptr = (char*)malloc(sz); + if(!ptr) { + free(array); + return NULL; + } + + ca->tail = array; + ca->end = array + nfirst; + ca->array = array; + + array[0].free = sz; + array[0].ptr = ptr; + array[0].alloc = ptr; + + return true; } - -void* msgpack_zone_malloc(msgpack_zone* z, size_t size) +static inline void destroy_chunk_array(msgpack_zone_chunk_array* ca) { - if(!z->array) { - const size_t n = (sizeof(msgpack_zone_chunk) < 72/2) ? - 72 / sizeof(msgpack_zone_chunk) : 8; - msgpack_zone_chunk* array = - (msgpack_zone_chunk*)malloc(sizeof(msgpack_zone_chunk) * n); - if(!array) { return NULL; } + msgpack_zone_chunk* chunk = ca->array; + for(; chunk != ca->tail+1; ++chunk) { + free(chunk->alloc); + } + free(ca->array); +} - size_t sz = 2048; /* FIXME chunk_size */ - while(sz < size) { sz *= 2; } - char* p = (char*)malloc(sz); - if(!p) { - free(array); +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + msgpack_zone_chunk_array* const ca = &zone->chunk_array; + + msgpack_zone_chunk* chunk = ca->tail; + + if(chunk->free > size) { + // chunkã«ç©ºã容é‡ãŒã‚ã‚‹ + // 空ã容é‡ã‚’消費ã—ã¦è¿”ã™ + + char* ptr = chunk->ptr; + + chunk->ptr += size; + chunk->free -= size; + + return ptr; + } + + chunk = ++ca->tail; + + if(chunk == ca->end) { + // ca->arrayã«ç©ºããŒãªã„ + // ca->arrayã‚’æ‹¡å¼µã™ã‚‹ + + const size_t nused = ca->end - ca->array; + const size_t nnext = (ca->end - ca->array) * 2; + + chunk = (msgpack_zone_chunk*)realloc(ca->array, + sizeof(msgpack_zone_chunk) * nnext); + if(!chunk) { return NULL; } - z->array = array; - z->usable = n - 1; - array[0].free = sz - size; - array[0].ptr = p + size; - array[0].alloc = p; - return p; + ca->array = chunk; + ca->end = chunk + nnext; + chunk = ca->tail = chunk + nused; } - if(z->array[z->ntail].free > size) { - char* p = (char*)z->array[z->ntail].ptr; - z->array[z->ntail].ptr = p + size; - z->array[z->ntail].free -= size; - return p; + size_t sz = zone->chunk_size; + + while(sz < size) { + sz *= 2; } - if(z->usable <= z->ntail) { - const size_t n = (z->usable + 1) * 2; - msgpack_zone_chunk* tmp = - (msgpack_zone_chunk*)realloc(z->array, sizeof(msgpack_zone_chunk) * n); - if(!tmp) { return NULL; } - z->array = tmp; - z->usable = n - 1; + char* ptr = (char*)malloc(sz); + if(!ptr) { + return NULL; } - size_t sz = 2048; /* FIXME chunk_size */ - while(sz < size) { sz *= 2; } - char* p = (char*)malloc(sz); - if(!p) { return NULL; } + chunk->free = sz - size; + chunk->ptr = ptr + size; + chunk->alloc = ptr; - ++z->ntail; - z->array[z->ntail].free = sz - size; - z->array[z->ntail].ptr = p + size; - z->array[z->ntail].alloc = p; - return p; + return ptr; +} + + +static inline void init_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + fa->tail = NULL; + fa->end = NULL; + fa->array = NULL; +} + +static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + // 逆順ã«å‘¼ã³å‡ºã— + msgpack_zone_finalizer* fin = fa->tail; + for(; fin != fa->array; --fin) { + (*(fin-1)->func)((fin-1)->data); + } + free(fa->array); +} + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data) +{ + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + + msgpack_zone_finalizer* fin = fa->tail; + + if(fin == fa->end) { + // fa->arrayã«ç©ºããŒãªã„ + // fa->arrayã‚’æ‹¡å¼µã™ã‚‹ + + size_t nnext; + const size_t nused = fa->end - fa->array; + + if(nused == 0) { + // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL + + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? + 72 / sizeof(msgpack_zone_finalizer) : 8; + + } else { + nnext = (fa->end - fa->array) * 2; + } + + fin = (msgpack_zone_finalizer*)realloc(fa->array, + sizeof(msgpack_zone_finalizer) * nnext); + if(!fin) { + return false; + } + + fa->array = fin; + fa->end = fin + nnext; + fin = fa->tail = fin + nused; + } + + fin->func = func; + fin->data = data; + + ++fa->tail; + + return true; +} + + +bool msgpack_zone_is_empty(msgpack_zone* zone) +{ + msgpack_zone_chunk_array* const ca = &zone->chunk_array; + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + return ca->array[0].ptr == ca->array[0].alloc && + ca->tail == ca->array && + fa->tail == fa->array; +} + + +bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size) +{ + zone->chunk_size = chunk_size; + + if(!init_chunk_array(&zone->chunk_array, chunk_size)) { + return false; + } + + init_finalizer_array(&zone->finalizer_array); + + return true; +} + +void msgpack_zone_destroy(msgpack_zone* zone) +{ + destroy_finalizer_array(&zone->finalizer_array); + destroy_chunk_array(&zone->chunk_array); +} + + +msgpack_zone* msgpack_zone_new(size_t chunk_size) +{ + msgpack_zone* zone = (msgpack_zone*)malloc(sizeof(msgpack_zone)); + if(zone == NULL) { + return NULL; + } + + if(!msgpack_zone_init(zone, chunk_size)) { + free(zone); + return NULL; + } + + return zone; +} + +void msgpack_zone_free(msgpack_zone* zone) +{ + msgpack_zone_destroy(zone); + free(zone); } diff --git a/c/zone.h b/c/zone.h index ff6ca62..3dc9f52 100644 --- a/c/zone.h +++ b/c/zone.h @@ -19,19 +19,58 @@ #define MSGPACK_ZONE_H__ #include +#include #ifdef __cplusplus extern "C" { #endif -struct _msgpack_zone; -typedef struct _msgpack_zone msgpack_zone; +typedef struct msgpack_zone_chunk { + size_t free; + char* ptr; + void* alloc; +} msgpack_zone_chunk; -msgpack_zone* msgpack_zone_new(); -void msgpack_zone_free(msgpack_zone* z); +typedef struct msgpack_zone_finalizer { + void (*func)(void* data); + void* data; +} msgpack_zone_finalizer; -void* msgpack_zone_malloc(msgpack_zone* z, size_t size); +typedef struct msgpack_zone_chunk_array { + msgpack_zone_chunk* tail; + msgpack_zone_chunk* end; + msgpack_zone_chunk* array; +} msgpack_zone_chunk_array; + +typedef struct msgpack_zone_finalizer_array { + msgpack_zone_finalizer* tail; + msgpack_zone_finalizer* end; + msgpack_zone_finalizer* array; +} msgpack_zone_finalizer_array; + +typedef struct msgpack_zone { + msgpack_zone_chunk_array chunk_array; + msgpack_zone_finalizer_array finalizer_array; + size_t chunk_size; +} msgpack_zone; + +#ifndef MSGPACK_ZONE_CHUNK_SIZE +#define MSGPACK_ZONE_CHUNK_SIZE 2048 +#endif + +bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size); +void msgpack_zone_destroy(msgpack_zone* zone); + +msgpack_zone* msgpack_zone_new(size_t chunk_size); +void msgpack_zone_free(msgpack_zone* zone); + +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data); + +bool msgpack_zone_is_empty(msgpack_zone* zone); #ifdef __cplusplus diff --git a/configure.in b/configure.in index a464fb9..9fa6ac5 100644 --- a/configure.in +++ b/configure.in @@ -2,8 +2,42 @@ AC_INIT(msgpack/unpack_template.h) AM_INIT_AUTOMAKE(msgpack, 0.3.0) AC_CONFIG_HEADER(config.h) +AC_SUBST(CFLAGS) +if test "" = "$CFLAGS"; then + CFLAGS="-g -O4" +fi + +AC_PROG_CC + +CFLAGS="-O4 -Wall $CFLAGS -I.." + + +AC_MSG_CHECKING([if c++ api is enabled]) +AC_ARG_ENABLE(cxx, + AS_HELP_STRING([--disable-cxx], + [don't build c++ api.]) ) +AC_MSG_RESULT($enable_cxx) +if test "$enable_cxx" != "no"; then + AC_SUBST(CXXFLAGS) + if test "" = "$CXXFLAGS"; then + CXXFLAGS="-g -O4" + fi + + AC_CHECK_PROG(ERB, erb, erb) + if test "x$ERB" = x; then + AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) + fi +fi + +# FIXME +AC_PROG_CXX + +CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." + + +AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") + AC_PROG_LIBTOOL -AC_CONFIG_SUBDIRS([c cpp]) -AC_OUTPUT([Makefile]) +AC_OUTPUT([Makefile c/Makefile cpp/Makefile]) diff --git a/cpp/AUTHORS b/cpp/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/cpp/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/cpp/COPYING b/cpp/COPYING deleted file mode 100644 index 6f5f220..0000000 --- a/cpp/COPYING +++ /dev/null @@ -1,14 +0,0 @@ -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/cpp/ChangeLog b/cpp/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/cpp/Makefile.am b/cpp/Makefile.am index c7ddf4b..406c57b 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -1,9 +1,7 @@ lib_LTLIBRARIES = libmsgpack.la libmsgpack_la_SOURCES = \ - object.cpp \ - unpack.cpp \ - zone.cpp + object.cpp nobase_include_HEADERS = \ msgpack.hpp \ @@ -28,7 +26,6 @@ noinst_HEADERS = \ # FIXME object.lo: msgpack/type/tuple.hpp unpack.lo: msgpack/type/tuple.hpp msgpack/zone.hpp -zone.lo: msgpack/type/tuple.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp @@ -42,6 +39,8 @@ MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ msgpack/zone.hpp +libmsgpack_la_LIBADD = -L../c -lmsgpackc + # -version-info CURRENT:REVISION:AGE libmsgpack_la_LDFLAGS = -version-info 1:0:0 diff --git a/cpp/NEWS b/cpp/NEWS deleted file mode 100644 index e69de29..0000000 diff --git a/cpp/README b/cpp/README deleted file mode 100644 index 96f18b1..0000000 --- a/cpp/README +++ /dev/null @@ -1,21 +0,0 @@ -MessagePack for C++ -------------------- -MessagePack is a binary-based efficient data interchange format. - - - -Copyright (C) 2008-2009 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - diff --git a/cpp/bootstrap b/cpp/bootstrap deleted file mode 100755 index 7e61b82..0000000 --- a/cpp/bootstrap +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -NO_NEST=1 -. ../bootstrap diff --git a/cpp/configure.in b/cpp/configure.in deleted file mode 100644 index 2c3e5d0..0000000 --- a/cpp/configure.in +++ /dev/null @@ -1,23 +0,0 @@ -AC_INIT(object.cpp) -AM_INIT_AUTOMAKE(msgpack, 0.1.0) -AC_CONFIG_HEADER(config.h) - -AC_SUBST(CXXFLAGS) -if test "" = "$CXXFLAGS"; then - CXXFLAGS="-g -O4" -fi - -AC_PROG_CXX -AC_PROG_LIBTOOL - -AC_CHECK_PROG(ERB, erb, erb) -if test "x$ERB" = x; then - AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) -fi - -AC_CHECK_LIB(stdc++, main) - -CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." - -AC_OUTPUT([Makefile]) - diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index b2a81ef..6027251 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/boolean.hpp b/cpp/type/boolean.hpp index 60f1714..86bd697 100644 --- a/cpp/type/boolean.hpp +++ b/cpp/type/boolean.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/float.hpp b/cpp/type/float.hpp index 2178434..108709d 100644 --- a/cpp/type/float.hpp +++ b/cpp/type/float.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/integer.hpp b/cpp/type/integer.hpp index 5cd10f6..ecb7b89 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/integer.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index 4585d66..c136d53 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/nil.hpp b/cpp/type/nil.hpp index ab0c363..93e66ff 100644 --- a/cpp/type/nil.hpp +++ b/cpp/type/nil.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index b102ee8..b6ace3f 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index a20f5d9..586d84c 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -1,7 +1,7 @@ // // MessagePack for C++ static resolution routine // -// Copyright (C) 2008 FURUHASHI Sadayuki +// Copyright (C) 2008-2009 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cpp/unpack.cpp b/cpp/unpack.cpp deleted file mode 100644 index 0b9a476..0000000 --- a/cpp/unpack.cpp +++ /dev/null @@ -1,369 +0,0 @@ -// -// MessagePack for C++ deserializing routine -// -// Copyright (C) 2008-2009 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#include "msgpack/unpack.hpp" -#include "msgpack/unpack_define.h" -#include - -namespace msgpack { - - -//namespace { -struct unpack_user { - zone* z; - bool referenced; -}; -//} // noname namespace - - -#define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name - -#define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name - -#define msgpack_unpack_callback(name) \ - msgpack_unpack ## name - -#define msgpack_unpack_object object - -#define msgpack_unpack_user unpack_user - - -struct msgpack_unpacker_context; - -static void msgpack_unpacker_init(struct msgpack_unpacker_context* ctx); - -static object msgpack_unpacker_data(struct msgpack_unpacker_context* ctx); - -static int msgpack_unpacker_execute(struct msgpack_unpacker_context* ctx, - const char* data, size_t len, size_t* off); - - -static inline object msgpack_unpack_init(unpack_user* u) -{ return object(); } - -static inline object msgpack_unpack_uint8(unpack_user* u, uint8_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint16(unpack_user* u, uint16_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint32(unpack_user* u, uint32_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_uint64(unpack_user* u, uint64_t d) -{ object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - -static inline object msgpack_unpack_int8(unpack_user* u, int8_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int16(unpack_user* u, int16_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int32(unpack_user* u, int32_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_int64(unpack_user* u, int64_t d) -{ if(d >= 0) { object o; o.type = type::POSITIVE_INTEGER; o.via.u64 = d; return o; } - else { object o; o.type = type::NEGATIVE_INTEGER; o.via.i64 = d; return o; } } - -static inline object msgpack_unpack_float(unpack_user* u, float d) -{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } - -static inline object msgpack_unpack_double(unpack_user* u, double d) -{ object o; o.type = type::DOUBLE; o.via.dec = d; return o; } - -static inline object msgpack_unpack_nil(unpack_user* u) -{ object o; o.type = type::NIL; return o; } - -static inline object msgpack_unpack_true(unpack_user* u) -{ object o; o.type = type::BOOLEAN; o.via.boolean = true; return o; } - -static inline object msgpack_unpack_false(unpack_user* u) -{ object o; o.type = type::BOOLEAN; o.via.boolean = false; return o; } - -static inline object msgpack_unpack_array(unpack_user* u, unsigned int n) -{ - object o; - o.type = type::ARRAY; - o.via.array.size = 0; - o.via.array.ptr = (object*)u->z->malloc(n*sizeof(object)); - return o; -} - -static inline void msgpack_unpack_array_item(unpack_user* u, object* c, object o) -{ c->via.array.ptr[c->via.array.size++] = o; } - -static inline object msgpack_unpack_map(unpack_user* u, unsigned int n) -{ - object o; - o.type = type::MAP; - o.via.map.size = 0; - o.via.map.ptr = (object_kv*)u->z->malloc(n*sizeof(object_kv)); - return o; -} - -static inline void msgpack_unpack_map_item(unpack_user* u, object* c, object k, object v) -{ - c->via.map.ptr[c->via.map.size].key = k; - c->via.map.ptr[c->via.map.size].val = v; - ++c->via.map.size; -} - -static inline object msgpack_unpack_raw(unpack_user* u, const char* b, const char* p, unsigned int l) -{ - object o; - o.type = type::RAW; - o.via.raw.ptr = p; - o.via.raw.size = l; - u->referenced = true; - return o; -} - -#include "msgpack/unpack_template.h" - - -namespace { -struct context { - context() - { - msgpack_unpacker_init(&m_ctx); - unpack_user u = {NULL, false}; - m_ctx.user = u; - } - - ~context() { } - - int execute(const char* data, size_t len, size_t* off) - { - return msgpack_unpacker_execute(&m_ctx, data, len, off); - } - - object data() - { - return msgpack_unpacker_data(&m_ctx); - } - - void reset() - { - zone* z = m_ctx.user.z; - msgpack_unpacker_init(&m_ctx); - unpack_user u = {z, false}; - m_ctx.user = u; - } - - void set_zone(zone* z) - { - m_ctx.user.z = z; - } - - bool is_referenced() const - { - return m_ctx.user.referenced; - } - -private: - msgpack_unpacker_context m_ctx; - zone* m_zone; - -private: - context(const context&); -}; - -static inline context* as_ctx(void* m) -{ - return reinterpret_cast(m); -} - - -static const size_t COUNTER_SIZE = sizeof(unsigned int); - -static inline void init_count(void* buffer) -{ - *(volatile unsigned int*)buffer = 1; -} - -static inline void decl_count(void* buffer) -{ - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { - free(buffer); - } -} - -static inline void incr_count(void* buffer) -{ - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); -} - -static inline unsigned int get_count(void* buffer) -{ - return *(volatile unsigned int*)buffer; -} - -} // noname namespace - - -unpacker::unpacker(size_t initial_buffer_size) : - m_buffer(NULL), - m_used(0), - m_free(0), - m_off(0), - m_zone(new zone()), - m_ctx(new context()), - m_initial_buffer_size(initial_buffer_size) -{ - if(m_initial_buffer_size < COUNTER_SIZE) { - m_initial_buffer_size = COUNTER_SIZE; - } - - as_ctx(m_ctx)->set_zone(m_zone.get()); - - m_buffer = (char*)::malloc(m_initial_buffer_size); - if(!m_buffer) { throw std::bad_alloc(); } - init_count(m_buffer); - - m_used = COUNTER_SIZE; - m_free = m_initial_buffer_size - m_used; - m_off = COUNTER_SIZE; -} - - -unpacker::~unpacker() -{ - delete as_ctx(m_ctx); - decl_count(m_buffer); -} - -void unpacker::expand_buffer(size_t len) -{ - if(m_used == m_off && get_count(m_buffer) == 1 && - !as_ctx(m_ctx)->is_referenced()) { - // rewind buffer - m_free += m_used - COUNTER_SIZE; - m_used = COUNTER_SIZE; - m_off = COUNTER_SIZE; - if(m_free >= len) { return; } - } - - if(m_off == COUNTER_SIZE) { - size_t next_size = (m_used + m_free) * 2; - while(next_size < len + m_used) { next_size *= 2; } - - char* tmp = (char*)::realloc(m_buffer, next_size); - if(!tmp) { throw std::bad_alloc(); } - - m_buffer = tmp; - m_free = next_size - m_used; - - } else { - size_t next_size = m_initial_buffer_size; // include COUNTER_SIZE - size_t not_parsed = m_used - m_off; - while(next_size < len + not_parsed + COUNTER_SIZE) { next_size *= 2; } - - char* tmp = (char*)::malloc(next_size); - if(!tmp) { throw std::bad_alloc(); } - init_count(tmp); - - try { - m_zone->push_finalizer(decl_count, m_buffer); - } catch (...) { free(tmp); throw; } - - memcpy(tmp+COUNTER_SIZE, m_buffer+m_off, not_parsed); - - m_buffer = tmp; - m_used = not_parsed + COUNTER_SIZE; - m_free = next_size - m_used; - m_off = COUNTER_SIZE; - } -} - -bool unpacker::execute() -{ - int ret = as_ctx(m_ctx)->execute(m_buffer, m_used, &m_off); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - return false; - } else { - return true; - } -} - -zone* unpacker::release_zone() -{ - m_zone->push_finalizer(decl_count, m_buffer); - incr_count(m_buffer); - - //std::auto_ptr old(new zone()); - //m_zone.swap(old); - zone* n = new zone(); - std::auto_ptr old(m_zone.release()); - m_zone.reset(n); - - as_ctx(m_ctx)->set_zone(m_zone.get()); - - return old.release(); -} - -object unpacker::data() -{ - return as_ctx(m_ctx)->data(); -} - -void unpacker::reset() -{ - //if(!m_zone->empty()) { delete release_zone(); } - as_ctx(m_ctx)->reset(); -} - - -object unpacker::unpack(const char* data, size_t len, zone& z, size_t* off) -{ - context ctx; - ctx.set_zone(&z); - if(off) { - size_t noff = *off; - int ret = ctx.execute(data, len, &noff); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - throw unpack_error("insufficient bytes"); - } - *off = noff; - } else { - size_t noff = 0; - int ret = ctx.execute(data, len, &noff); - if(ret < 0) { - throw unpack_error("parse error"); - } else if(ret == 0) { - throw unpack_error("insufficient bytes"); - } else if(noff < len) { - throw unpack_error("extra bytes"); - } - } - return ctx.data(); -} - - -} // namespace msgpack - diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index cde45e7..8c77726 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -18,6 +18,7 @@ #ifndef MSGPACK_UNPACK_HPP__ #define MSGPACK_UNPACK_HPP__ +#include "msgpack/unpack.h" #include "msgpack/object.hpp" #include "msgpack/zone.hpp" #include @@ -36,21 +37,21 @@ struct unpack_error : public std::runtime_error { }; -class unpacker { +class unpacker : public msgpack_unpacker { public: unpacker(size_t initial_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); ~unpacker(); public: - /*! 1. reserve buffer. at least `len' bytes of capacity will be ready */ - void reserve_buffer(size_t len); + /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ + void reserve_buffer(size_t size); /*! 2. read data to the buffer() up to buffer_capacity() bytes */ char* buffer(); size_t buffer_capacity() const; /*! 3. specify the number of bytes actually copied */ - void buffer_consumed(size_t len); + void buffer_consumed(size_t size); /*! 4. repeat execute() until it retunrs false */ bool execute(); @@ -114,71 +115,157 @@ public: size_t nonparsed_size() const; /*! skip specified size of non-parsed buffer, leaving the buffer */ - // Note that the `len' argument must be smaller than nonparsed_size() - void skip_nonparsed_buffer(size_t len); + // Note that the `size' argument must be smaller than nonparsed_size() + void skip_nonparsed_buffer(size_t size); /*! remove unparsed buffer from unpacker */ // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); -private: - char* m_buffer; - size_t m_used; - size_t m_free; - size_t m_off; - - std::auto_ptr m_zone; - - void* m_ctx; - - size_t m_initial_buffer_size; - -private: - void expand_buffer(size_t len); - private: unpacker(const unpacker&); - -public: - static object unpack(const char* data, size_t len, zone& z, size_t* off = NULL); }; -inline void unpacker::reserve_buffer(size_t len) +typedef enum { + MSGPACK_UNPACK_SUCCESS = 2, + MSGPACK_UNPACK_EXTRA_BYTES = 1, + MSGPACK_UNPACK_CONTINUE = 0, + MSGPACK_UNPACK_PARSE_ERROR = -1, +} unpack_return; + +static unpack_return unpack(const char* data, size_t len, size_t* off, + zone* z, object* result); + + +// obsolete +static object unpack(const char* data, size_t len, zone& z, size_t* off = NULL); + + +inline unpacker::unpacker(size_t initial_buffer_size) { - if(m_free >= len) { return; } - expand_buffer(len); + if(!msgpack_unpacker_init(this, initial_buffer_size)) { + throw std::bad_alloc(); + } +} + +inline unpacker::~unpacker() +{ + msgpack_unpacker_destroy(this); +} + +inline void unpacker::reserve_buffer(size_t size) +{ + if(!msgpack_unpacker_reserve_buffer(this, size)) { + throw std::bad_alloc(); + } } inline char* unpacker::buffer() - { return m_buffer + m_used; } +{ + return msgpack_unpacker_buffer(this); +} inline size_t unpacker::buffer_capacity() const - { return m_free; } - -inline void unpacker::buffer_consumed(size_t len) { - m_used += len; - m_free -= len; + return msgpack_unpacker_buffer_capacity(this); +} + +inline void unpacker::buffer_consumed(size_t size) +{ + return msgpack_unpacker_buffer_consumed(this, size); +} + + +inline bool unpacker::execute() +{ + int ret = msgpack_unpacker_execute(this); + if(ret < 0) { + throw unpack_error("parse error"); + } else if(ret == 0) { + return false; + } else { + return true; + } +} + +inline object unpacker::data() +{ + msgpack_object obj = msgpack_unpacker_data(this); + return *reinterpret_cast(&obj); +} + +inline zone* unpacker::release_zone() +{ + if(!msgpack_unpacker_flush_zone(this)) { + throw std::bad_alloc(); + } + + zone* r = new zone(); + + msgpack_zone old = *this->z; + *this->z = *z; + *z = old; + + return r; +} + +inline void unpacker::reset() +{ + msgpack_unpacker_reset(this); } inline char* unpacker::nonparsed_buffer() - { return m_buffer + m_off; } +{ + return buf + off; +} inline size_t unpacker::nonparsed_size() const - { return m_used - m_off; } +{ + return used - off; +} -inline void unpacker::skip_nonparsed_buffer(size_t len) - { m_off += len; } +inline void unpacker::skip_nonparsed_buffer(size_t size) +{ + off += size; +} inline void unpacker::remove_nonparsed_buffer() - { m_used = m_off; } - - -inline object unpack(const char* data, size_t len, zone& z, size_t* off = NULL) { - return unpacker::unpack(data, len, z, off); + used = off; +} + + +inline unpack_return unpack(const char* data, size_t len, size_t* off, + zone* z, object* result) +{ + return (unpack_return)msgpack_unpack(data, len, off, + z, reinterpret_cast(result)); +} + +inline object unpack(const char* data, size_t len, zone& z, size_t* off) +{ + object result; + + switch( msgpack::unpack(data, len, off, &z, &result) ) { + case MSGPACK_UNPACK_SUCCESS: + return result; + + case MSGPACK_UNPACK_EXTRA_BYTES: + if(off) { + return result; + } else { + throw unpack_error("extra bytes"); + } + + case MSGPACK_UNPACK_CONTINUE: + throw unpack_error("insufficient bytes"); + + case MSGPACK_UNPACK_PARSE_ERROR: + default: + throw unpack_error("parse error"); + } } diff --git a/cpp/zone.cpp b/cpp/zone.cpp deleted file mode 100644 index f765266..0000000 --- a/cpp/zone.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// -// MessagePack for C++ memory pool -// -// Copyright (C) 2008-2009 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -#include "msgpack/zone.hpp" -#include - -namespace msgpack { - - -zone::zone(size_t chunk_size) : - m_chunk_size(chunk_size) -{ - chunk dummy = {0, NULL, NULL}; - m_chunk_array.push_back(dummy); -} - -zone::~zone() -{ - clear(); -} - -namespace { - template - struct zone_finalize { - void operator() (Private& f) { - (*f.func)(f.obj); - } - }; - - template - struct zone_free { - void operator() (Private& c) { - ::free(c.alloc); - } - }; -} - -void zone::clear() -{ - std::for_each(m_finalizers.rbegin(), m_finalizers.rend(), - zone_finalize()); - m_finalizers.clear(); - - std::for_each(m_chunk_array.begin(), m_chunk_array.end(), - zone_free()); - m_chunk_array.resize(1); - m_chunk_array[0].ptr = NULL; - m_chunk_array[0].free = 0; -} - -bool zone::empty() const -{ - return m_chunk_array.back().alloc == NULL && - m_finalizers.empty(); -} - -void* zone::malloc(size_t size) -{ - if(m_chunk_array.back().free > size) { - char* p = (char*)m_chunk_array.back().ptr; - m_chunk_array.back().ptr = p + size; - m_chunk_array.back().free -= size; - return p; - } - - size_t sz = m_chunk_size; - while(sz < size) { sz *= 2; } - - chunk dummy = {0, NULL, NULL}; - m_chunk_array.push_back(dummy); - - char* p = (char*)::malloc(sz); - if(!p) { - m_chunk_array.pop_back(); - throw std::bad_alloc(); - } - - m_chunk_array.back().free = sz - size; - m_chunk_array.back().ptr = p + size; - m_chunk_array.back().alloc = p; - return p; -} - - -} // namespace msgpack - diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index c0eb2e3..06cb9d3 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -19,17 +19,15 @@ #define MSGPACK_ZONE_HPP__ #include "msgpack/object.hpp" +#include "msgpack/zone.h" #include #include -#ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 2048 -#endif <% GENERATION_LIMIT = 15 %> namespace msgpack { -class zone { +class zone : public msgpack_zone { public: zone(size_t chunk_size = MSGPACK_ZONE_CHUNK_SIZE); ~zone(); @@ -37,11 +35,7 @@ public: public: void* malloc(size_t size); - void push_finalizer(void (*func)(void*), void* obj); - - void clear(); - - bool empty() const; + void push_finalizer(void (*func)(void*), void* data); <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> @@ -49,35 +43,39 @@ public: <%}%> private: - struct chunk { - size_t free; - void* ptr; - void* alloc; - }; - - std::vector m_chunk_array; - - struct finalizer { - void (*func)(void*); - void* obj; - }; - - std::vector m_finalizers; - template static void object_destructor(void* obj); - size_t m_chunk_size; - private: zone(const zone&); }; -inline void zone::push_finalizer(void (*func)(void*), void* obj) + +inline zone::zone(size_t chunk_size) { - finalizer f = {func, obj}; - m_finalizers.push_back(f); + msgpack_zone_init(this, chunk_size); +} + +inline zone::~zone() +{ + msgpack_zone_destroy(this); +} + +inline void* zone::malloc(size_t size) +{ + void* ptr = msgpack_zone_malloc(this, size); + if(!ptr) { + throw std::bad_alloc(); + } + return ptr; +} + +inline void zone::push_finalizer(void (*func)(void*), void* data) +{ + if(!msgpack_zone_push_finalizer(this, func, data)) { + throw std::bad_alloc(); + } } template @@ -93,7 +91,7 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) void* x = malloc(sizeof(T)); push_finalizer(&zone::object_destructor, x); try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } - catch (...) { m_finalizers.pop_back(); throw; } + catch (...) { --finalizer_array.tail; throw; } } <%}%> @@ -101,3 +99,4 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) #endif /* msgpack/zone.hpp */ +// vim: ft=cpp ts=4 sw=4 softtabstop=4 noexpandtab smarttab diff --git a/example/stream.cc b/example/stream.cc index 49927de..aef4ec8 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -10,6 +10,7 @@ class Server { public: Server(int sock) : m_sock(sock) { } + ~Server() { } typedef std::auto_ptr auto_zone; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index a45369b..ff30955 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -49,7 +49,7 @@ msgpack_unpack_struct_decl(_stack) { }; msgpack_unpack_struct_decl(_context) { - msgpack_unpack_user user; // must be first + msgpack_unpack_user user; unsigned int cs; unsigned int trail; unsigned int top; @@ -63,12 +63,12 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; - ctx->stack[0].obj = msgpack_unpack_callback(_init)(&ctx->user); + ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } -msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* unpacker) +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { - return (unpacker)->stack[0].obj; + return (ctx)->stack[0].obj; } @@ -92,16 +92,14 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c int ret; #define push_simple_value(func) \ - obj = msgpack_unpack_callback(func)(user); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ goto _push #define push_fixed_value(func, arg) \ - obj = msgpack_unpack_callback(func)(user, arg); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ goto _push #define push_variable_value(func, base, pos, len) \ - obj = msgpack_unpack_callback(func)(user, (const char*)base, (const char*)pos, len); \ - /*printf("obj %d\n",obj);*/ \ + if(msgpack_unpack_callback(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ goto _push #define again_fixed_trail(_cs, trail_len) \ @@ -115,7 +113,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c goto _fixed_trail_again #define start_container(func, count_, ct_) \ - stack[top].obj = msgpack_unpack_callback(func)(user, count_); \ + if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; goto _push; } \ if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ stack[top].ct = ct_; \ @@ -264,11 +262,13 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c case CS_ARRAY_16: start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); case CS_ARRAY_32: + /* FIXME security guard */ start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); case CS_MAP_16: start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); case CS_MAP_32: + /* FIXME security guard */ start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); default: @@ -281,7 +281,7 @@ _push: c = &stack[top-1]; switch(c->ct) { case CT_ARRAY_ITEM: - msgpack_unpack_callback(_array_item)(user, &c->obj, obj); + if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; } if(--c->count == 0) { obj = c->obj; --top; @@ -294,7 +294,7 @@ _push: c->ct = CT_MAP_VALUE; goto _header_again; case CT_MAP_VALUE: - msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj); + if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } if(--c->count == 0) { obj = c->obj; --top; diff --git a/ruby/gem/README b/ruby/gem/README index 78fe6cb..859ae2b 100644 --- a/ruby/gem/README +++ b/ruby/gem/README @@ -24,6 +24,6 @@ == Copyright -Author:: frsyuki -Copyright:: Copyright (c) 2009 frsyuki +Author:: frsyuki +Copyright:: Copyright (c) 2008-2009 frsyuki License:: Apache License, Version 2.0 diff --git a/ruby/gengem.sh b/ruby/gengem.sh index be9d14b..6c07be3 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -1,5 +1,7 @@ #!/bin/sh +mkdir -p gem/ext +mkdir -p gem/msgpack cp extconf.rb gem/ext/ cp pack.c gem/ext/ cp pack.h gem/ext/ diff --git a/ruby/unpack.c b/ruby/unpack.c index 4650ed1..4de4955 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -21,99 +21,99 @@ typedef struct { int finished; - VALUE origstr; -} msgpack_unpack_context; + VALUE source; +} unpack_user; #define msgpack_unpack_struct(name) \ - struct msgpack_unpacker ## name + struct template ## name #define msgpack_unpack_func(ret, name) \ - ret msgpack_unpacker ## name + ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name #define msgpack_unpack_object VALUE -#define msgpack_unpack_user msgpack_unpack_context +#define msgpack_unpack_user unpack_user -struct msgpack_unpacker_context; -typedef struct msgpack_unpacker_context msgpack_unpacker; +struct template_context; +typedef struct template_context msgpack_unpack_t; -static void msgpack_unpacker_init(msgpack_unpacker* ctx); +static void template_init(msgpack_unpack_t* u); -static VALUE msgpack_unpacker_data(msgpack_unpacker* ctx); +static VALUE template_data(msgpack_unpack_t* u); -static int msgpack_unpacker_execute(msgpack_unpacker* ctx, +static int template_execute(msgpack_unpack_t* u, const char* data, size_t len, size_t* off); -static inline VALUE template_callback_init(msgpack_unpack_context* x) +static inline VALUE template_callback_root(unpack_user* u) { return Qnil; } -static inline VALUE template_callback_uint8(msgpack_unpack_context* x, uint8_t d) -{ return INT2FIX(d); } +static inline int template_callback_uint8(unpack_user* u, uint8_t d, VALUE* o) +{ *o = INT2FIX(d); return 0; } -static inline VALUE template_callback_uint16(msgpack_unpack_context* x, uint16_t d) -{ return INT2FIX(d); } +static inline int template_callback_uint16(unpack_user* u, uint16_t d, VALUE* o) +{ *o = INT2FIX(d); return 0; } -static inline VALUE template_callback_uint32(msgpack_unpack_context* x, uint32_t d) -{ return UINT2NUM(d); } +static inline int template_callback_uint32(unpack_user* u, uint32_t d, VALUE* o) +{ *o = UINT2NUM(d); return 0; } -static inline VALUE template_callback_uint64(msgpack_unpack_context* x, uint64_t d) -{ return rb_ull2inum(d); } +static inline int template_callback_uint64(unpack_user* u, uint64_t d, VALUE* o) +{ *o = rb_ull2inum(d); return 0; } -static inline VALUE template_callback_int8(msgpack_unpack_context* x, int8_t d) -{ return INT2FIX((long)d); } +static inline int template_callback_int8(unpack_user* u, int8_t d, VALUE* o) +{ *o = INT2FIX((long)d); return 0; } -static inline VALUE template_callback_int16(msgpack_unpack_context* x, int16_t d) -{ return INT2FIX((long)d); } +static inline int template_callback_int16(unpack_user* u, int16_t d, VALUE* o) +{ *o = INT2FIX((long)d); return 0; } -static inline VALUE template_callback_int32(msgpack_unpack_context* x, int32_t d) -{ return INT2NUM((long)d); } +static inline int template_callback_int32(unpack_user* u, int32_t d, VALUE* o) +{ *o = INT2NUM((long)d); return 0; } -static inline VALUE template_callback_int64(msgpack_unpack_context* x, int64_t d) -{ return rb_ll2inum(d); } +static inline int template_callback_int64(unpack_user* u, int64_t d, VALUE* o) +{ *o = rb_ll2inum(d); return 0; } -static inline VALUE template_callback_float(msgpack_unpack_context* x, float d) -{ return rb_float_new(d); } +static inline int template_callback_float(unpack_user* u, float d, VALUE* o) +{ *o = rb_float_new(d); return 0; } -static inline VALUE template_callback_double(msgpack_unpack_context* x, double d) -{ return rb_float_new(d); } +static inline int template_callback_double(unpack_user* u, double d, VALUE* o) +{ *o = rb_float_new(d); return 0; } -static inline VALUE template_callback_nil(msgpack_unpack_context* x) -{ return Qnil; } +static inline int template_callback_nil(unpack_user* u, VALUE* o) +{ *o = Qnil; return 0; } -static inline VALUE template_callback_true(msgpack_unpack_context* x) -{ return Qtrue; } +static inline int template_callback_true(unpack_user* u, VALUE* o) +{ *o = Qtrue; return 0; } -static inline VALUE template_callback_false(msgpack_unpack_context* x) -{ return Qfalse; } +static inline int template_callback_false(unpack_user* u, VALUE* o) +{ *o = Qfalse; return 0;} -static inline VALUE template_callback_array(msgpack_unpack_context* x, unsigned int n) -{ return rb_ary_new2(n); } +static inline int template_callback_array(unpack_user* u, unsigned int n, VALUE* o) +{ *o = rb_ary_new2(n); return 0; } -static inline void template_callback_array_item(msgpack_unpack_context* x, VALUE* c, VALUE o) -{ rb_ary_push(*c, o); } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] +static inline int template_callback_array_item(unpack_user* u, VALUE* c, VALUE o) +{ rb_ary_push(*c, o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] -static inline VALUE template_callback_map(msgpack_unpack_context* x, unsigned int n) -{ return rb_hash_new(); } +static inline int template_callback_map(unpack_user* u, unsigned int n, VALUE* o) +{ *o = rb_hash_new(); return 0; } -static inline void template_callback_map_item(msgpack_unpack_context* x, VALUE* c, VALUE k, VALUE v) -{ rb_hash_aset(*c, k, v); } +static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, VALUE v) +{ rb_hash_aset(*c, k, v); return 0; } -static inline VALUE template_callback_raw(msgpack_unpack_context* x, const char* b, const char* p, unsigned int l) -{ return l == 0 ? rb_str_new(0,0) : rb_str_substr(x->origstr, p - b, l); } +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) +{ *o = (l == 0) ? rb_str_new(0,0) : rb_str_substr(u->source, p - b, l); return 0; } #include "msgpack/unpack_template.h" #define UNPACKER(from, name) \ - msgpack_unpacker *name = NULL; \ - Data_Get_Struct(from, msgpack_unpacker, name); \ + msgpack_unpack_t *name = NULL; \ + Data_Get_Struct(from, msgpack_unpack_t, name); \ if(name == NULL) { \ rb_raise(rb_eArgError, "NULL found for " # name " when shouldn't be."); \ } @@ -132,7 +132,7 @@ static void MessagePack_Unpacker_free(void* data) if(data) { free(data); } } -static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) +static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) { unsigned int i; for(i=0; i < mp->top; ++i) { @@ -144,7 +144,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpacker *mp) static VALUE MessagePack_Unpacker_alloc(VALUE klass) { VALUE obj; - msgpack_unpacker* mp = ALLOC_N(msgpack_unpacker, 1); + msgpack_unpack_t* mp = ALLOC_N(msgpack_unpack_t, 1); obj = Data_Wrap_Struct(klass, MessagePack_Unpacker_mark, MessagePack_Unpacker_free, mp); return obj; @@ -153,9 +153,9 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); - msgpack_unpacker_init(mp); - msgpack_unpack_context ctx = {0, Qnil}; - mp->user = ctx; + template_init(mp); + unpack_user u = {0, Qnil}; + mp->user = u; return self; } @@ -180,9 +180,9 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) rb_raise(eUnpackError, "offset is bigger than data buffer size."); } - mp->user.origstr = data; - ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); - mp->user.origstr = Qnil; + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -235,13 +235,13 @@ static VALUE MessagePack_Unpacker_finished_p(VALUE self) static VALUE MessagePack_Unpacker_data(VALUE self) { UNPACKER(self, mp); - return msgpack_unpacker_data(mp); + return template_data(mp); } static VALUE MessagePack_unpack_impl(VALUE args) { - msgpack_unpacker* mp = (msgpack_unpacker*)((VALUE*)args)[0]; + msgpack_unpack_t* mp = (msgpack_unpack_t*)((VALUE*)args)[0]; VALUE data = ((VALUE*)args)[1]; size_t from = 0; @@ -249,9 +249,9 @@ static VALUE MessagePack_unpack_impl(VALUE args) long dlen = FIX2LONG(((VALUE*)args)[2]); int ret; - mp->user.origstr = data; - ret = msgpack_unpacker_execute(mp, dptr, (size_t)dlen, &from); - mp->user.origstr = Qnil; + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = Qnil; if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -261,7 +261,7 @@ static VALUE MessagePack_unpack_impl(VALUE args) if(from < dlen) { rb_raise(eUnpackError, "extra bytes."); } - return msgpack_unpacker_data(mp); + return template_data(mp); } } @@ -278,10 +278,10 @@ static VALUE MessagePack_unpack_rescue(VALUE args) static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); - msgpack_unpacker mp; - msgpack_unpacker_init(&mp); - msgpack_unpack_context ctx = {0, Qnil}; - mp.user = ctx; + msgpack_unpack_t mp; + template_init(&mp); + unpack_user u = {0, Qnil}; + mp.user = u; rb_gc_disable(); VALUE args[3] = {(VALUE)&mp, data, limit}; From 96423688359d5211d11c12fe1c5d17ba01c760e6 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Sun, 22 Feb 2009 15:36:02 +0900 Subject: [PATCH 0059/1648] type conversion operator msgpack_object <-> msgpack::object --- configure.in | 2 +- cpp/object.hpp | 24 ++++++++++++++++++++++++ cpp/unpack.hpp | 3 +-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 9fa6ac5..d7bb926 100644 --- a/configure.in +++ b/configure.in @@ -32,7 +32,7 @@ fi # FIXME AC_PROG_CXX -CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." +CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") diff --git a/cpp/object.hpp b/cpp/object.hpp index 08e715d..c3f2872 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -18,8 +18,10 @@ #ifndef MSGPACK_OBJECT_HPP__ #define MSGPACK_OBJECT_HPP__ +#include "msgpack/object.h" #include "msgpack/pack.hpp" #include +#include #include #include #include @@ -72,6 +74,7 @@ struct object { object_array array; object_map map; object_raw raw; + object_raw ref; // obsolete }; type::object_type type; @@ -85,6 +88,10 @@ struct object { template void convert(T* v); + object(); + object(msgpack_object obj); + operator msgpack_object(); + private: struct implicit_type; @@ -192,6 +199,23 @@ inline bool operator!=(const object x, const object y) { return !(x == y); } +inline object::object() { } + +inline object::object(msgpack_object obj) +{ + // FIXME beter way? + ::memcpy(this, &obj, sizeof(obj)); +} + +inline object::operator msgpack_object() +{ + // FIXME beter way? + msgpack_object obj; + ::memcpy(&obj, this, sizeof(obj)); + return obj; +} + + inline object::implicit_type object::convert() { return implicit_type(*this); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 8c77726..57623f2 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -191,8 +191,7 @@ inline bool unpacker::execute() inline object unpacker::data() { - msgpack_object obj = msgpack_unpacker_data(this); - return *reinterpret_cast(&obj); + return msgpack_unpacker_data(this); } inline zone* unpacker::release_zone() From d078eb0ad5b2b1c7749942a4618f6cfedd1067d5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 22 Feb 2009 15:36:02 +0900 Subject: [PATCH 0060/1648] type conversion operator msgpack_object <-> msgpack::object --- configure.in | 2 +- cpp/object.hpp | 24 ++++++++++++++++++++++++ cpp/unpack.hpp | 3 +-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 9fa6ac5..d7bb926 100644 --- a/configure.in +++ b/configure.in @@ -32,7 +32,7 @@ fi # FIXME AC_PROG_CXX -CXXFLAGS="-O4 -Wall $CXXFLAGS -I.." +CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") diff --git a/cpp/object.hpp b/cpp/object.hpp index 08e715d..c3f2872 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -18,8 +18,10 @@ #ifndef MSGPACK_OBJECT_HPP__ #define MSGPACK_OBJECT_HPP__ +#include "msgpack/object.h" #include "msgpack/pack.hpp" #include +#include #include #include #include @@ -72,6 +74,7 @@ struct object { object_array array; object_map map; object_raw raw; + object_raw ref; // obsolete }; type::object_type type; @@ -85,6 +88,10 @@ struct object { template void convert(T* v); + object(); + object(msgpack_object obj); + operator msgpack_object(); + private: struct implicit_type; @@ -192,6 +199,23 @@ inline bool operator!=(const object x, const object y) { return !(x == y); } +inline object::object() { } + +inline object::object(msgpack_object obj) +{ + // FIXME beter way? + ::memcpy(this, &obj, sizeof(obj)); +} + +inline object::operator msgpack_object() +{ + // FIXME beter way? + msgpack_object obj; + ::memcpy(&obj, this, sizeof(obj)); + return obj; +} + + inline object::implicit_type object::convert() { return implicit_type(*this); diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 8c77726..57623f2 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -191,8 +191,7 @@ inline bool unpacker::execute() inline object unpacker::data() { - msgpack_object obj = msgpack_unpacker_data(this); - return *reinterpret_cast(&obj); + return msgpack_unpacker_data(this); } inline zone* unpacker::release_zone() From a0b82e39e11b4cfcbadc7e81b8927d77c1e1a1f5 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Sun, 22 Feb 2009 15:47:06 +0900 Subject: [PATCH 0061/1648] AC_CONFIG_AUX_DIR(ac) --- README | 2 +- bootstrap | 1 + configure.in | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README b/README index fd5fa8d..6e6ace4 100644 --- a/README +++ b/README @@ -9,7 +9,7 @@ Binary-based efficient data interchange format. UNIX-like platforms. Following programs is required to build: - - gcc >= 4.0 with C++ support + - gcc >= 4.1 with C++ support - ruby >= 1.8 (ruby is used as a preprocessor) diff --git a/bootstrap b/bootstrap index eb16c35..9ddb3da 100755 --- a/bootstrap +++ b/bootstrap @@ -35,6 +35,7 @@ fi # cd c && ./bootstrap $@; cd .. # cd cpp && ./bootstrap $@; cd .. #fi +mkdir -p ac ACLOCAL="aclocal" diff --git a/configure.in b/configure.in index d7bb926..2c1bee7 100644 --- a/configure.in +++ b/configure.in @@ -1,4 +1,5 @@ AC_INIT(msgpack/unpack_template.h) +AC_CONFIG_AUX_DIR(ac) AM_INIT_AUTOMAKE(msgpack, 0.3.0) AC_CONFIG_HEADER(config.h) From 0698304071f3cfb4f86135fcc0423609d8288d6b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 22 Feb 2009 15:47:06 +0900 Subject: [PATCH 0062/1648] AC_CONFIG_AUX_DIR(ac) --- README | 2 +- bootstrap | 1 + configure.in | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README b/README index fd5fa8d..6e6ace4 100644 --- a/README +++ b/README @@ -9,7 +9,7 @@ Binary-based efficient data interchange format. UNIX-like platforms. Following programs is required to build: - - gcc >= 4.0 with C++ support + - gcc >= 4.1 with C++ support - ruby >= 1.8 (ruby is used as a preprocessor) diff --git a/bootstrap b/bootstrap index eb16c35..9ddb3da 100755 --- a/bootstrap +++ b/bootstrap @@ -35,6 +35,7 @@ fi # cd c && ./bootstrap $@; cd .. # cd cpp && ./bootstrap $@; cd .. #fi +mkdir -p ac ACLOCAL="aclocal" diff --git a/configure.in b/configure.in index d7bb926..2c1bee7 100644 --- a/configure.in +++ b/configure.in @@ -1,4 +1,5 @@ AC_INIT(msgpack/unpack_template.h) +AC_CONFIG_AUX_DIR(ac) AM_INIT_AUTOMAKE(msgpack, 0.3.0) AC_CONFIG_HEADER(config.h) From 5d51129e65b54c32876aaf27db3a465ef89a6687 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Tue, 24 Feb 2009 16:37:01 +0900 Subject: [PATCH 0063/1648] add example/simple.c --- example/simple.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 example/simple.c diff --git a/example/simple.c b/example/simple.c new file mode 100644 index 0000000..df60424 --- /dev/null +++ b/example/simple.c @@ -0,0 +1,37 @@ +#include +#include + +int main(void) +{ + /* msgpack::sbuffer is a simple buffer implementation. */ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + + /* serialize values into the buffer using msgpack_sbuffer_write callback function. */ + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + msgpack_pack_array(&pk, 3); + msgpack_pack_int(&pk, 1); + msgpack_pack_true(&pk); + msgpack_pack_raw(&pk, 7); + msgpack_pack_raw_body(&pk, "example", 7); + + /* deserialize the buffer into msgpack_object instance. */ + /* deserialized object is valid during the msgpack_zone instance alive. */ + msgpack_zone mempool; + msgpack_zone_init(&mempool, 2048); + + msgpack_object deserialized; + msgpack_unpack(sbuf.ptr, sbuf.size, NULL, &mempool, &deserialized); + + /* print the deserialized object. */ + msgpack_object_print(stdout, deserialized); + puts(""); + + msgpack_zone_destroy(&mempool); + msgpack_sbuffer_destroy(&sbuf); + + return 0; +} + From aaaaecb8bad862afc66ad5a2772adb78b9082df8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 24 Feb 2009 16:37:01 +0900 Subject: [PATCH 0064/1648] add example/simple.c --- example/simple.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 example/simple.c diff --git a/example/simple.c b/example/simple.c new file mode 100644 index 0000000..df60424 --- /dev/null +++ b/example/simple.c @@ -0,0 +1,37 @@ +#include +#include + +int main(void) +{ + /* msgpack::sbuffer is a simple buffer implementation. */ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + + /* serialize values into the buffer using msgpack_sbuffer_write callback function. */ + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + msgpack_pack_array(&pk, 3); + msgpack_pack_int(&pk, 1); + msgpack_pack_true(&pk); + msgpack_pack_raw(&pk, 7); + msgpack_pack_raw_body(&pk, "example", 7); + + /* deserialize the buffer into msgpack_object instance. */ + /* deserialized object is valid during the msgpack_zone instance alive. */ + msgpack_zone mempool; + msgpack_zone_init(&mempool, 2048); + + msgpack_object deserialized; + msgpack_unpack(sbuf.ptr, sbuf.size, NULL, &mempool, &deserialized); + + /* print the deserialized object. */ + msgpack_object_print(stdout, deserialized); + puts(""); + + msgpack_zone_destroy(&mempool); + msgpack_sbuffer_destroy(&sbuf); + + return 0; +} + From 4d708aa4da36759c5e2ca83e579d8f36e1c53fb0 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Tue, 24 Feb 2009 16:37:47 +0900 Subject: [PATCH 0065/1648] c: msgpack_pack_object --- c/object.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++ c/pack.h | 3 +++ cpp/msgpack.hpp | 1 + cpp/object.hpp | 32 +++++++++++----------- cpp/unpack.hpp | 1 + example/simple.cc | 24 ++++++++++------- example/stream.cc | 1 - 7 files changed, 106 insertions(+), 25 deletions(-) diff --git a/c/object.c b/c/object.c index 871153d..bcb2537 100644 --- a/c/object.c +++ b/c/object.c @@ -16,9 +16,78 @@ * limitations under the License. */ #include "msgpack/object.h" +#include "msgpack/pack.h" #include #include + +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d) +{ + switch(d.type) { + case MSGPACK_OBJECT_NIL: + return msgpack_pack_nil(pk); + + case MSGPACK_OBJECT_BOOLEAN: + if(d.via.boolean) { + return msgpack_pack_true(pk); + } else { + return msgpack_pack_false(pk); + } + + case MSGPACK_OBJECT_POSITIVE_INTEGER: + return msgpack_pack_uint64(pk, d.via.u64); + + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + return msgpack_pack_int64(pk, d.via.i64); + + case MSGPACK_OBJECT_DOUBLE: + return msgpack_pack_double(pk, d.via.dec); + + case MSGPACK_OBJECT_RAW: + { + int ret = msgpack_pack_raw(pk, d.via.raw.size); + if(ret < 0) { return ret; } + return msgpack_pack_raw_body(pk, d.via.raw.ptr, d.via.raw.size); + } + + case MSGPACK_OBJECT_ARRAY: + { + int ret = msgpack_pack_array(pk, d.via.array.size); + if(ret < 0) { return ret; } + + msgpack_object* o = d.via.array.ptr; + msgpack_object* const oend = d.via.array.ptr + d.via.array.size; + for(; o != oend; ++o) { + ret = msgpack_pack_object(pk, *o); + if(ret < 0) { return ret; } + } + + return 0; + } + + case MSGPACK_OBJECT_MAP: + { + int ret = msgpack_pack_map(pk, d.via.map.size); + if(ret < 0) { return ret; } + + msgpack_object_kv* kv = d.via.map.ptr; + msgpack_object_kv* const kvend = d.via.map.ptr + d.via.map.size; + for(; kv != kvend; ++kv) { + ret = msgpack_pack_object(pk, kv->key); + if(ret < 0) { return ret; } + ret = msgpack_pack_object(pk, kv->val); + if(ret < 0) { return ret; } + } + + return 0; + } + + default: + return -1; + } +} + + void msgpack_object_print(FILE* out, msgpack_object o) { switch(o.type) { diff --git a/c/pack.h b/c/pack.h index 46de722..1a57ea4 100644 --- a/c/pack.h +++ b/c/pack.h @@ -22,6 +22,7 @@ #include #include #include "msgpack/pack_define.h" +#include "msgpack/object.h" #ifdef __cplusplus extern "C" { @@ -72,6 +73,8 @@ static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static int msgpack_pack_raw(msgpack_packer* pk, size_t l); static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); + #define msgpack_pack_inline_func(name) \ diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp index f00da06..58b40ac 100644 --- a/cpp/msgpack.hpp +++ b/cpp/msgpack.hpp @@ -20,3 +20,4 @@ #include "msgpack/pack.hpp" #include "msgpack/unpack.hpp" #include "msgpack/sbuffer.hpp" +#include "msgpack.h" diff --git a/cpp/object.hpp b/cpp/object.hpp index c3f2872..b4c21f1 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -162,18 +162,6 @@ inline packer& packer::pack(const T& v) return *this; } -template -inline void pack(Stream& s, const T& v) -{ - packer(s).pack(v); -} - -template -inline void pack_copy(packer& o, T v) -{ - pack(o, v); -} - inline object& operator>> (object o, object& v) { v = o; @@ -250,6 +238,20 @@ inline void pack(packer& o, const T& v) o.pack(v); } +// obsolete +template +inline void pack(Stream& s, const T& v) +{ + packer(s).pack(v); +} + +// obsolete +template +inline void pack_copy(packer& o, T v) +{ + pack(o, v); +} + template packer& operator<< (packer& o, const object& v) @@ -309,7 +311,7 @@ packer& operator<< (packer& o, const object& v) for(object* p(v.via.array.ptr), * const pend(v.via.array.ptr + v.via.array.size); p < pend; ++p) { - *p >> o; + o << *p; } return o; // FIXME loop optimiziation @@ -319,8 +321,8 @@ packer& operator<< (packer& o, const object& v) for(object_kv* p(v.via.map.ptr), * const pend(v.via.map.ptr + v.via.map.size); p < pend; ++p) { - p->key >> o; - p->val >> o; + o << p->key; + o << p->val; } return o; // FIXME loop optimiziation diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 57623f2..4e7ccf5 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -243,6 +243,7 @@ inline unpack_return unpack(const char* data, size_t len, size_t* off, z, reinterpret_cast(result)); } +// obsolete inline object unpack(const char* data, size_t len, zone& z, size_t* off) { object result; diff --git a/example/simple.cc b/example/simple.cc index 74beeae..55ecdf9 100644 --- a/example/simple.cc +++ b/example/simple.cc @@ -5,27 +5,33 @@ int main(void) { - // this is target object msgpack::type::tuple src(1, true, "example"); - // any classes that implements write(const char*,size_t) can be a buffer + // serialize the object into the buffer. + // any classes that implements write(const char*,size_t) can be a buffer. std::stringstream buffer; msgpack::pack(buffer, src); // send the buffer ... buffer.seekg(0); - // deserialize the buffer into msgpack::object type - msgpack::zone mempool; + // deserialize the buffer into msgpack::object instance. std::string str(buffer.str()); - msgpack::object deserialized = - msgpack::unpack(str.data(), str.size(), mempool); - // msgpack::object supports ostream + // deserialized object is valid during the msgpack::zone instance alive. + msgpack::zone mempool; + + msgpack::object deserialized; + msgpack::unpack(str.data(), str.size(), NULL, &mempool, &deserialized); + + // msgpack::object supports ostream. std::cout << deserialized << std::endl; - // convert msgpack::object type into the original type + // convert msgpack::object instance into the original type. + // if the type is mismatched, it throws msgpack::type_error exception. msgpack::type::tuple dst; - msgpack::convert(dst, deserialized); + deserialized.convert(&dst); + + return 0; } diff --git a/example/stream.cc b/example/stream.cc index aef4ec8..e57318f 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -84,7 +84,6 @@ struct fwriter { void write(const char* buf, size_t buflen) { size_t count = fwrite(buf, buflen, 1, m_fp); - //if(fwrite(buf, buflen, 1, m_fp) < 1) { if(count < 1) { std::cout << buflen << std::endl; std::cout << count << std::endl; From bdd13859b60d06ec6fabdbaccea81cbee189b8bc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 24 Feb 2009 16:37:47 +0900 Subject: [PATCH 0066/1648] c: msgpack_pack_object --- c/object.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++ c/pack.h | 3 +++ cpp/msgpack.hpp | 1 + cpp/object.hpp | 32 +++++++++++----------- cpp/unpack.hpp | 1 + example/simple.cc | 24 ++++++++++------- example/stream.cc | 1 - 7 files changed, 106 insertions(+), 25 deletions(-) diff --git a/c/object.c b/c/object.c index 871153d..bcb2537 100644 --- a/c/object.c +++ b/c/object.c @@ -16,9 +16,78 @@ * limitations under the License. */ #include "msgpack/object.h" +#include "msgpack/pack.h" #include #include + +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d) +{ + switch(d.type) { + case MSGPACK_OBJECT_NIL: + return msgpack_pack_nil(pk); + + case MSGPACK_OBJECT_BOOLEAN: + if(d.via.boolean) { + return msgpack_pack_true(pk); + } else { + return msgpack_pack_false(pk); + } + + case MSGPACK_OBJECT_POSITIVE_INTEGER: + return msgpack_pack_uint64(pk, d.via.u64); + + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + return msgpack_pack_int64(pk, d.via.i64); + + case MSGPACK_OBJECT_DOUBLE: + return msgpack_pack_double(pk, d.via.dec); + + case MSGPACK_OBJECT_RAW: + { + int ret = msgpack_pack_raw(pk, d.via.raw.size); + if(ret < 0) { return ret; } + return msgpack_pack_raw_body(pk, d.via.raw.ptr, d.via.raw.size); + } + + case MSGPACK_OBJECT_ARRAY: + { + int ret = msgpack_pack_array(pk, d.via.array.size); + if(ret < 0) { return ret; } + + msgpack_object* o = d.via.array.ptr; + msgpack_object* const oend = d.via.array.ptr + d.via.array.size; + for(; o != oend; ++o) { + ret = msgpack_pack_object(pk, *o); + if(ret < 0) { return ret; } + } + + return 0; + } + + case MSGPACK_OBJECT_MAP: + { + int ret = msgpack_pack_map(pk, d.via.map.size); + if(ret < 0) { return ret; } + + msgpack_object_kv* kv = d.via.map.ptr; + msgpack_object_kv* const kvend = d.via.map.ptr + d.via.map.size; + for(; kv != kvend; ++kv) { + ret = msgpack_pack_object(pk, kv->key); + if(ret < 0) { return ret; } + ret = msgpack_pack_object(pk, kv->val); + if(ret < 0) { return ret; } + } + + return 0; + } + + default: + return -1; + } +} + + void msgpack_object_print(FILE* out, msgpack_object o) { switch(o.type) { diff --git a/c/pack.h b/c/pack.h index 46de722..1a57ea4 100644 --- a/c/pack.h +++ b/c/pack.h @@ -22,6 +22,7 @@ #include #include #include "msgpack/pack_define.h" +#include "msgpack/object.h" #ifdef __cplusplus extern "C" { @@ -72,6 +73,8 @@ static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static int msgpack_pack_raw(msgpack_packer* pk, size_t l); static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); + #define msgpack_pack_inline_func(name) \ diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp index f00da06..58b40ac 100644 --- a/cpp/msgpack.hpp +++ b/cpp/msgpack.hpp @@ -20,3 +20,4 @@ #include "msgpack/pack.hpp" #include "msgpack/unpack.hpp" #include "msgpack/sbuffer.hpp" +#include "msgpack.h" diff --git a/cpp/object.hpp b/cpp/object.hpp index c3f2872..b4c21f1 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -162,18 +162,6 @@ inline packer& packer::pack(const T& v) return *this; } -template -inline void pack(Stream& s, const T& v) -{ - packer(s).pack(v); -} - -template -inline void pack_copy(packer& o, T v) -{ - pack(o, v); -} - inline object& operator>> (object o, object& v) { v = o; @@ -250,6 +238,20 @@ inline void pack(packer& o, const T& v) o.pack(v); } +// obsolete +template +inline void pack(Stream& s, const T& v) +{ + packer(s).pack(v); +} + +// obsolete +template +inline void pack_copy(packer& o, T v) +{ + pack(o, v); +} + template packer& operator<< (packer& o, const object& v) @@ -309,7 +311,7 @@ packer& operator<< (packer& o, const object& v) for(object* p(v.via.array.ptr), * const pend(v.via.array.ptr + v.via.array.size); p < pend; ++p) { - *p >> o; + o << *p; } return o; // FIXME loop optimiziation @@ -319,8 +321,8 @@ packer& operator<< (packer& o, const object& v) for(object_kv* p(v.via.map.ptr), * const pend(v.via.map.ptr + v.via.map.size); p < pend; ++p) { - p->key >> o; - p->val >> o; + o << p->key; + o << p->val; } return o; // FIXME loop optimiziation diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 57623f2..4e7ccf5 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -243,6 +243,7 @@ inline unpack_return unpack(const char* data, size_t len, size_t* off, z, reinterpret_cast(result)); } +// obsolete inline object unpack(const char* data, size_t len, zone& z, size_t* off) { object result; diff --git a/example/simple.cc b/example/simple.cc index 74beeae..55ecdf9 100644 --- a/example/simple.cc +++ b/example/simple.cc @@ -5,27 +5,33 @@ int main(void) { - // this is target object msgpack::type::tuple src(1, true, "example"); - // any classes that implements write(const char*,size_t) can be a buffer + // serialize the object into the buffer. + // any classes that implements write(const char*,size_t) can be a buffer. std::stringstream buffer; msgpack::pack(buffer, src); // send the buffer ... buffer.seekg(0); - // deserialize the buffer into msgpack::object type - msgpack::zone mempool; + // deserialize the buffer into msgpack::object instance. std::string str(buffer.str()); - msgpack::object deserialized = - msgpack::unpack(str.data(), str.size(), mempool); - // msgpack::object supports ostream + // deserialized object is valid during the msgpack::zone instance alive. + msgpack::zone mempool; + + msgpack::object deserialized; + msgpack::unpack(str.data(), str.size(), NULL, &mempool, &deserialized); + + // msgpack::object supports ostream. std::cout << deserialized << std::endl; - // convert msgpack::object type into the original type + // convert msgpack::object instance into the original type. + // if the type is mismatched, it throws msgpack::type_error exception. msgpack::type::tuple dst; - msgpack::convert(dst, deserialized); + deserialized.convert(&dst); + + return 0; } diff --git a/example/stream.cc b/example/stream.cc index aef4ec8..e57318f 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -84,7 +84,6 @@ struct fwriter { void write(const char* buf, size_t buflen) { size_t count = fwrite(buf, buflen, 1, m_fp); - //if(fwrite(buf, buflen, 1, m_fp) < 1) { if(count < 1) { std::cout << buflen << std::endl; std::cout << count << std::endl; From 43360e378611ae550ece16554311e0de6e3cc046 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Wed, 25 Feb 2009 18:55:56 +0900 Subject: [PATCH 0067/1648] fix msgpack::unpack_return --- cpp/pack.hpp | 4 ++++ cpp/unpack.hpp | 16 ++++++++-------- example/stream.cc | 7 ++++--- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 52854ac..257ccb6 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -30,6 +30,7 @@ template class packer { public: packer(Stream& s); + ~packer(); public: template @@ -129,6 +130,9 @@ private: template packer::packer(Stream& s) : m_stream(s) { } +template +packer::~packer() { } + template inline packer& packer::pack_uint8(uint8_t d) { _pack_uint8(m_stream, d); return *this; } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 4e7ccf5..e6d35da 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -128,10 +128,10 @@ private: typedef enum { - MSGPACK_UNPACK_SUCCESS = 2, - MSGPACK_UNPACK_EXTRA_BYTES = 1, - MSGPACK_UNPACK_CONTINUE = 0, - MSGPACK_UNPACK_PARSE_ERROR = -1, + UNPACK_SUCCESS = 2, + UNPACK_EXTRA_BYTES = 1, + UNPACK_CONTINUE = 0, + UNPACK_PARSE_ERROR = -1, } unpack_return; static unpack_return unpack(const char* data, size_t len, size_t* off, @@ -249,20 +249,20 @@ inline object unpack(const char* data, size_t len, zone& z, size_t* off) object result; switch( msgpack::unpack(data, len, off, &z, &result) ) { - case MSGPACK_UNPACK_SUCCESS: + case UNPACK_SUCCESS: return result; - case MSGPACK_UNPACK_EXTRA_BYTES: + case UNPACK_EXTRA_BYTES: if(off) { return result; } else { throw unpack_error("extra bytes"); } - case MSGPACK_UNPACK_CONTINUE: + case UNPACK_CONTINUE: throw unpack_error("insufficient bytes"); - case MSGPACK_UNPACK_PARSE_ERROR: + case UNPACK_PARSE_ERROR: default: throw unpack_error("parse error"); } diff --git a/example/stream.cc b/example/stream.cc index e57318f..8fd92f9 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -113,6 +113,7 @@ int main(void) // client thread: fwriter writer(pair[1]); + msgpack::packer pk(writer); typedef msgpack::type::tuple put_t; typedef msgpack::type::tuple get_t; @@ -120,9 +121,9 @@ int main(void) put_t req1("put", "apple", "red"); put_t req2("put", "lemon", "yellow"); get_t req3("get", "apple"); - msgpack::pack(writer, req1); - msgpack::pack(writer, req2); - msgpack::pack(writer, req3); + pk.pack(req1); + pk.pack(req2); + pk.pack(req3); writer.flush(); writer.close(); From a89dfc70b997201cea3587de922924dec2bb28e7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 25 Feb 2009 18:55:56 +0900 Subject: [PATCH 0068/1648] fix msgpack::unpack_return --- cpp/pack.hpp | 4 ++++ cpp/unpack.hpp | 16 ++++++++-------- example/stream.cc | 7 ++++--- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 52854ac..257ccb6 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -30,6 +30,7 @@ template class packer { public: packer(Stream& s); + ~packer(); public: template @@ -129,6 +130,9 @@ private: template packer::packer(Stream& s) : m_stream(s) { } +template +packer::~packer() { } + template inline packer& packer::pack_uint8(uint8_t d) { _pack_uint8(m_stream, d); return *this; } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 4e7ccf5..e6d35da 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -128,10 +128,10 @@ private: typedef enum { - MSGPACK_UNPACK_SUCCESS = 2, - MSGPACK_UNPACK_EXTRA_BYTES = 1, - MSGPACK_UNPACK_CONTINUE = 0, - MSGPACK_UNPACK_PARSE_ERROR = -1, + UNPACK_SUCCESS = 2, + UNPACK_EXTRA_BYTES = 1, + UNPACK_CONTINUE = 0, + UNPACK_PARSE_ERROR = -1, } unpack_return; static unpack_return unpack(const char* data, size_t len, size_t* off, @@ -249,20 +249,20 @@ inline object unpack(const char* data, size_t len, zone& z, size_t* off) object result; switch( msgpack::unpack(data, len, off, &z, &result) ) { - case MSGPACK_UNPACK_SUCCESS: + case UNPACK_SUCCESS: return result; - case MSGPACK_UNPACK_EXTRA_BYTES: + case UNPACK_EXTRA_BYTES: if(off) { return result; } else { throw unpack_error("extra bytes"); } - case MSGPACK_UNPACK_CONTINUE: + case UNPACK_CONTINUE: throw unpack_error("insufficient bytes"); - case MSGPACK_UNPACK_PARSE_ERROR: + case UNPACK_PARSE_ERROR: default: throw unpack_error("parse error"); } diff --git a/example/stream.cc b/example/stream.cc index e57318f..8fd92f9 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -113,6 +113,7 @@ int main(void) // client thread: fwriter writer(pair[1]); + msgpack::packer pk(writer); typedef msgpack::type::tuple put_t; typedef msgpack::type::tuple get_t; @@ -120,9 +121,9 @@ int main(void) put_t req1("put", "apple", "red"); put_t req2("put", "lemon", "yellow"); get_t req3("get", "apple"); - msgpack::pack(writer, req1); - msgpack::pack(writer, req2); - msgpack::pack(writer, req3); + pk.pack(req1); + pk.pack(req2); + pk.pack(req3); writer.flush(); writer.close(); From 6ffee9e54abc54e84a2c81450bb1f607f0637a72 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Wed, 25 Feb 2009 23:31:53 +0900 Subject: [PATCH 0069/1648] c: msgpack_unpacker_buffered_size; c++: unpacker::buffered_size --- c/unpack.h | 6 ++++++ cpp/unpack.hpp | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/c/unpack.h b/c/unpack.h index ab202a1..46777b9 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -46,6 +46,7 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac); msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); void msgpack_unpacker_free(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac); static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); @@ -77,6 +78,11 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); +size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac) +{ + return mpac->used; +} + bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) { if(mpac->free >= size) { return true; } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index e6d35da..6e9b641 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -43,6 +43,9 @@ public: ~unpacker(); public: + /*! 0. check if the buffered size is not exceed the assumption. */ + size_t buffered_size() const; + /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ void reserve_buffer(size_t size); @@ -154,6 +157,12 @@ inline unpacker::~unpacker() msgpack_unpacker_destroy(this); } + +inline size_t unpacker::buffered_size() const +{ + return msgpack_unpacker_buffered_size(this); +} + inline void unpacker::reserve_buffer(size_t size) { if(!msgpack_unpacker_reserve_buffer(this, size)) { From 8893523776e5258e2817711da75f00a6074a13ae Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 25 Feb 2009 23:31:53 +0900 Subject: [PATCH 0070/1648] c: msgpack_unpacker_buffered_size; c++: unpacker::buffered_size --- c/unpack.h | 6 ++++++ cpp/unpack.hpp | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/c/unpack.h b/c/unpack.h index ab202a1..46777b9 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -46,6 +46,7 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac); msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); void msgpack_unpacker_free(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac); static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); @@ -77,6 +78,11 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); +size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac) +{ + return mpac->used; +} + bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) { if(mpac->free >= size) { return true; } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index e6d35da..6e9b641 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -43,6 +43,9 @@ public: ~unpacker(); public: + /*! 0. check if the buffered size is not exceed the assumption. */ + size_t buffered_size() const; + /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ void reserve_buffer(size_t size); @@ -154,6 +157,12 @@ inline unpacker::~unpacker() msgpack_unpacker_destroy(this); } + +inline size_t unpacker::buffered_size() const +{ + return msgpack_unpacker_buffered_size(this); +} + inline void unpacker::reserve_buffer(size_t size) { if(!msgpack_unpacker_reserve_buffer(this, size)) { From 65ef9cab66c39914da1292fb886c4c93c0a7312e Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Thu, 26 Feb 2009 01:15:14 +0900 Subject: [PATCH 0071/1648] remove msgpack_unpacker_buffered_size, add msgpack_unpacker_parsed_size --- c/unpack.c | 12 ++++++++---- c/unpack.h | 14 ++++++++------ cpp/unpack.hpp | 16 ++++++++-------- msgpack/unpack_template.h | 1 - ruby/unpack.c | 2 ++ 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index e06d97c..865e0d4 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -197,6 +197,7 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) mpac->used = COUNTER_SIZE; mpac->free = initial_buffer_size - mpac->used; mpac->off = COUNTER_SIZE; + mpac->parsed = 0; mpac->initial_buffer_size = initial_buffer_size; mpac->z = z; mpac->referenced = false; @@ -305,8 +306,13 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) int msgpack_unpacker_execute(msgpack_unpacker* mpac) { - return template_execute(CTX_CAST(mpac->ctx), + size_t off = mpac->off; + int ret = template_execute(CTX_CAST(mpac->ctx), mpac->buf, mpac->used, &mpac->off); + if(mpac->off > off) { + mpac->parsed += mpac->off - off; + } + return ret; } msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) @@ -347,10 +353,8 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) void msgpack_unpacker_reset(msgpack_unpacker* mpac) { - msgpack_zone* z = mpac->z; template_init(CTX_CAST(mpac->ctx)); - CTX_CAST(mpac->ctx)->user.z = z; - CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + mpac->parsed = 0; } diff --git a/c/unpack.h b/c/unpack.h index 46777b9..77fbd54 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -33,6 +33,7 @@ typedef struct msgpack_unpacker { size_t used; size_t free; size_t off; + size_t parsed; msgpack_zone* z; bool referenced; size_t initial_buffer_size; @@ -46,7 +47,6 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac); msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); void msgpack_unpacker_free(msgpack_unpacker* mpac); -static inline size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac); static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); @@ -61,6 +61,8 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); void msgpack_unpacker_reset(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); + typedef enum { MSGPACK_UNPACK_SUCCESS = 2, @@ -78,11 +80,6 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); -size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac) -{ - return mpac->used; -} - bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) { if(mpac->free >= size) { return true; } @@ -105,6 +102,11 @@ void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) mpac->free -= size; } +size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac) +{ + return mpac->parsed; +} + #ifdef __cplusplus } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 6e9b641..0b7fe57 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -43,9 +43,6 @@ public: ~unpacker(); public: - /*! 0. check if the buffered size is not exceed the assumption. */ - size_t buffered_size() const; - /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ void reserve_buffer(size_t size); @@ -70,6 +67,9 @@ public: /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); + /*! 6. check if the parsed message size doesn't exceed assumption. */ + size_t parsed_size() const; + // Basic usage of the unpacker is as following: // @@ -158,11 +158,6 @@ inline unpacker::~unpacker() } -inline size_t unpacker::buffered_size() const -{ - return msgpack_unpacker_buffered_size(this); -} - inline void unpacker::reserve_buffer(size_t size) { if(!msgpack_unpacker_reserve_buffer(this, size)) { @@ -223,6 +218,11 @@ inline void unpacker::reset() msgpack_unpacker_reset(this); } +inline size_t unpacker::parsed_size() const +{ + return msgpack_unpacker_parsed_size(this); +} + inline char* unpacker::nonparsed_buffer() { diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index ff30955..d67fd1e 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -59,7 +59,6 @@ msgpack_unpack_struct_decl(_context) { msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) { - /*memset(ctx, 0, sizeof( msgpack_unpack_struct(_context) )); FIXME needed? */ ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; diff --git a/ruby/unpack.c b/ruby/unpack.c index 4de4955..411a94d 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -278,6 +278,7 @@ static VALUE MessagePack_unpack_rescue(VALUE args) static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); + msgpack_unpack_t mp; template_init(&mp); unpack_user u = {0, Qnil}; @@ -288,6 +289,7 @@ static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, MessagePack_unpack_rescue, Qnil); rb_gc_enable(); + return ret; } From 5697b9a15d89a397a53174b9eca2d96bc7ebaef0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 26 Feb 2009 01:15:14 +0900 Subject: [PATCH 0072/1648] remove msgpack_unpacker_buffered_size, add msgpack_unpacker_parsed_size --- c/unpack.c | 12 ++++++++---- c/unpack.h | 14 ++++++++------ cpp/unpack.hpp | 16 ++++++++-------- msgpack/unpack_template.h | 1 - ruby/unpack.c | 2 ++ 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index e06d97c..865e0d4 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -197,6 +197,7 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) mpac->used = COUNTER_SIZE; mpac->free = initial_buffer_size - mpac->used; mpac->off = COUNTER_SIZE; + mpac->parsed = 0; mpac->initial_buffer_size = initial_buffer_size; mpac->z = z; mpac->referenced = false; @@ -305,8 +306,13 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) int msgpack_unpacker_execute(msgpack_unpacker* mpac) { - return template_execute(CTX_CAST(mpac->ctx), + size_t off = mpac->off; + int ret = template_execute(CTX_CAST(mpac->ctx), mpac->buf, mpac->used, &mpac->off); + if(mpac->off > off) { + mpac->parsed += mpac->off - off; + } + return ret; } msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) @@ -347,10 +353,8 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) void msgpack_unpacker_reset(msgpack_unpacker* mpac) { - msgpack_zone* z = mpac->z; template_init(CTX_CAST(mpac->ctx)); - CTX_CAST(mpac->ctx)->user.z = z; - CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + mpac->parsed = 0; } diff --git a/c/unpack.h b/c/unpack.h index 46777b9..77fbd54 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -33,6 +33,7 @@ typedef struct msgpack_unpacker { size_t used; size_t free; size_t off; + size_t parsed; msgpack_zone* z; bool referenced; size_t initial_buffer_size; @@ -46,7 +47,6 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac); msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size); void msgpack_unpacker_free(msgpack_unpacker* mpac); -static inline size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac); static inline bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size); static inline char* msgpack_unpacker_buffer(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac); @@ -61,6 +61,8 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); void msgpack_unpacker_reset(msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); + typedef enum { MSGPACK_UNPACK_SUCCESS = 2, @@ -78,11 +80,6 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); -size_t msgpack_unpacker_buffered_size(const msgpack_unpacker* mpac) -{ - return mpac->used; -} - bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) { if(mpac->free >= size) { return true; } @@ -105,6 +102,11 @@ void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) mpac->free -= size; } +size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac) +{ + return mpac->parsed; +} + #ifdef __cplusplus } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 6e9b641..0b7fe57 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -43,9 +43,6 @@ public: ~unpacker(); public: - /*! 0. check if the buffered size is not exceed the assumption. */ - size_t buffered_size() const; - /*! 1. reserve buffer. at least `size' bytes of capacity will be ready */ void reserve_buffer(size_t size); @@ -70,6 +67,9 @@ public: /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); + /*! 6. check if the parsed message size doesn't exceed assumption. */ + size_t parsed_size() const; + // Basic usage of the unpacker is as following: // @@ -158,11 +158,6 @@ inline unpacker::~unpacker() } -inline size_t unpacker::buffered_size() const -{ - return msgpack_unpacker_buffered_size(this); -} - inline void unpacker::reserve_buffer(size_t size) { if(!msgpack_unpacker_reserve_buffer(this, size)) { @@ -223,6 +218,11 @@ inline void unpacker::reset() msgpack_unpacker_reset(this); } +inline size_t unpacker::parsed_size() const +{ + return msgpack_unpacker_parsed_size(this); +} + inline char* unpacker::nonparsed_buffer() { diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index ff30955..d67fd1e 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -59,7 +59,6 @@ msgpack_unpack_struct_decl(_context) { msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) { - /*memset(ctx, 0, sizeof( msgpack_unpack_struct(_context) )); FIXME needed? */ ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; diff --git a/ruby/unpack.c b/ruby/unpack.c index 4de4955..411a94d 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -278,6 +278,7 @@ static VALUE MessagePack_unpack_rescue(VALUE args) static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); + msgpack_unpack_t mp; template_init(&mp); unpack_user u = {0, Qnil}; @@ -288,6 +289,7 @@ static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, MessagePack_unpack_rescue, Qnil); rb_gc_enable(); + return ret; } From c0baf9b87349a0ba250aa8ecf0df420db506dcdd Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Thu, 26 Feb 2009 01:27:00 +0900 Subject: [PATCH 0073/1648] add msgpack_unpacker_message_size --- c/unpack.h | 10 +++++++++- cpp/unpack.hpp | 12 +++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/c/unpack.h b/c/unpack.h index 77fbd54..779e256 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -61,7 +61,8 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); void msgpack_unpacker_reset(msgpack_unpacker* mpac); -static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac); + typedef enum { @@ -76,6 +77,8 @@ msgpack_unpack(const char* data, size_t len, size_t* off, msgpack_zone* z, msgpack_object* result); +static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); + bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); @@ -102,6 +105,11 @@ void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) mpac->free -= size; } +size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac) +{ + return mpac->parsed - mpac->off + mpac->used; +} + size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac) { return mpac->parsed; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 0b7fe57..e140b36 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -67,8 +67,8 @@ public: /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); - /*! 6. check if the parsed message size doesn't exceed assumption. */ - size_t parsed_size() const; + /*! 6. check if the size of message doesn't exceed assumption. */ + size_t message_size() const; // Basic usage of the unpacker is as following: @@ -112,6 +112,7 @@ public: public: // These functions are usable when non-MessagePack message follows after // MessagePack message. + size_t parsed_size() const; /*! get address of the buffer that is not parsed */ char* nonparsed_buffer(); @@ -218,12 +219,17 @@ inline void unpacker::reset() msgpack_unpacker_reset(this); } +inline size_t unpacker::message_size() const +{ + return msgpack_unpacker_message_size(this); +} + + inline size_t unpacker::parsed_size() const { return msgpack_unpacker_parsed_size(this); } - inline char* unpacker::nonparsed_buffer() { return buf + off; From 980bb9cdd08bd3e5a0ff6021a45a1d880691c013 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 26 Feb 2009 01:27:00 +0900 Subject: [PATCH 0074/1648] add msgpack_unpacker_message_size --- c/unpack.h | 10 +++++++++- cpp/unpack.hpp | 12 +++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/c/unpack.h b/c/unpack.h index 77fbd54..779e256 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -61,7 +61,8 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); void msgpack_unpacker_reset(msgpack_unpacker* mpac); -static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); +static inline size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac); + typedef enum { @@ -76,6 +77,8 @@ msgpack_unpack(const char* data, size_t len, size_t* off, msgpack_zone* z, msgpack_object* result); +static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac); + bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac); bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size); @@ -102,6 +105,11 @@ void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size) mpac->free -= size; } +size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac) +{ + return mpac->parsed - mpac->off + mpac->used; +} + size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac) { return mpac->parsed; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 0b7fe57..e140b36 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -67,8 +67,8 @@ public: /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); - /*! 6. check if the parsed message size doesn't exceed assumption. */ - size_t parsed_size() const; + /*! 6. check if the size of message doesn't exceed assumption. */ + size_t message_size() const; // Basic usage of the unpacker is as following: @@ -112,6 +112,7 @@ public: public: // These functions are usable when non-MessagePack message follows after // MessagePack message. + size_t parsed_size() const; /*! get address of the buffer that is not parsed */ char* nonparsed_buffer(); @@ -218,12 +219,17 @@ inline void unpacker::reset() msgpack_unpacker_reset(this); } +inline size_t unpacker::message_size() const +{ + return msgpack_unpacker_message_size(this); +} + + inline size_t unpacker::parsed_size() const { return msgpack_unpacker_parsed_size(this); } - inline char* unpacker::nonparsed_buffer() { return buf + off; From 9b4b49a6a85f9044ebaffff71b2498a3cf6c8ae4 Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Thu, 26 Feb 2009 01:33:34 +0900 Subject: [PATCH 0075/1648] example: limit message size --- example/stream.cc | 4 ++++ example/stream.rb | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/example/stream.cc b/example/stream.cc index 8fd92f9..1e2e733 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -43,6 +43,10 @@ public: process_message(msg, life); } + + if(m_pac.message_size() > 10*1024*1024) { + throw std::runtime_error("message is too large"); + } } private: diff --git a/example/stream.rb b/example/stream.rb index e53ce82..a72f5b9 100644 --- a/example/stream.rb +++ b/example/stream.rb @@ -34,12 +34,17 @@ class Server @pk.reset @buffer.slice!(0, @nread) @nread = 0 + next unless @buffer.empty? end break end + if @buffer.length > 10*1024*1024 + raise "message is too large" + end + rescue puts "error while processing client packet: #{$!}" end From e7fe3be34dbae518af91eb671e4fee79bc32fb4e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 26 Feb 2009 01:33:34 +0900 Subject: [PATCH 0076/1648] example: limit message size --- example/stream.cc | 4 ++++ example/stream.rb | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/example/stream.cc b/example/stream.cc index 8fd92f9..1e2e733 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -43,6 +43,10 @@ public: process_message(msg, life); } + + if(m_pac.message_size() > 10*1024*1024) { + throw std::runtime_error("message is too large"); + } } private: diff --git a/example/stream.rb b/example/stream.rb index e53ce82..a72f5b9 100644 --- a/example/stream.rb +++ b/example/stream.rb @@ -34,12 +34,17 @@ class Server @pk.reset @buffer.slice!(0, @nread) @nread = 0 + next unless @buffer.empty? end break end + if @buffer.length > 10*1024*1024 + raise "message is too large" + end + rescue puts "error while processing client packet: #{$!}" end From c60b6be5488df5b8218c1bbd0d8c3711d8dc73aa Mon Sep 17 00:00:00 2001 From: "frsyuki (none)" Date: Thu, 26 Feb 2009 18:37:13 +0900 Subject: [PATCH 0077/1648] add NOTICE file --- LICENSE | 202 +++++++++++++++++++++++++++++++++++++++++++++++++ NOTICE | 4 + README | 38 +++++----- cpp/object.hpp | 2 - 4 files changed, 226 insertions(+), 20 deletions(-) create mode 100644 LICENSE create mode 100644 NOTICE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..e706f2a --- /dev/null +++ b/NOTICE @@ -0,0 +1,4 @@ +MessagePack is developed by FURUHASHI Sadayuki, licensed under Apache License, +Version 2.0. The original software and related information is available at +http://msgpack.sourceforge.jp/. + diff --git a/README b/README index 6e6ace4..0c1d440 100644 --- a/README +++ b/README @@ -17,41 +17,43 @@ Binary-based efficient data interchange format. Simply run ./configure && make && make install to install C and C++ binding. - $ ./configure --prefix=path/to/prefix - $ make - $ make install + $ ./configure + $ make + $ sudo make install - To install Ruby binding, run ./gengem.sh script in ruby/ directory and install + To install Ruby binding, run ./gengem.sh script on ruby/ directory and install generated gem package. - $ cd ruby - $ ./gengem.sh - $ gem install gem/pkg/msgpack-*.gem + $ cd ruby + $ ./gengem.sh + $ gem install gem/pkg/msgpack-*.gem *Usage C++: - #include - // TODO + include msgpack.hpp header and link libmsgpack library. + see example/simple.cc for example. + + g++ simple.cc -lmsgpack + g++ stream.cc -lmsgpack -lpthread C: - #include - /* TODO */ + include msgpack.h header and link libmsgpackc library. + see example/simple.c for example. + + gcc simple.c -lmsgpackc Ruby: - require 'msgpack' + require msgpack library. + see example/simple.rb for example. - # serialize - buf = [1, 2, 3].to_msgpack + ruby -rubygems simple.rb - # deserialize - p MessagePack::unpack(buf) - - # TODO + API Document is available at http://msgpack.sourceforge.jp/. Copyright (C) 2008-2009 FURUHASHI Sadayuki diff --git a/cpp/object.hpp b/cpp/object.hpp index b4c21f1..860b569 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -314,7 +314,6 @@ packer& operator<< (packer& o, const object& v) o << *p; } return o; - // FIXME loop optimiziation case type::MAP: o.pack_map(v.via.map.size); @@ -325,7 +324,6 @@ packer& operator<< (packer& o, const object& v) o << p->val; } return o; - // FIXME loop optimiziation default: throw type_error(); From 00dcad17b915129f3e4f91f885cd946451fd8e31 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 26 Feb 2009 18:37:13 +0900 Subject: [PATCH 0078/1648] add NOTICE file --- LICENSE | 202 +++++++++++++++++++++++++++++++++++++++++++++++++ NOTICE | 4 + README | 38 +++++----- cpp/object.hpp | 2 - 4 files changed, 226 insertions(+), 20 deletions(-) create mode 100644 LICENSE create mode 100644 NOTICE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..e706f2a --- /dev/null +++ b/NOTICE @@ -0,0 +1,4 @@ +MessagePack is developed by FURUHASHI Sadayuki, licensed under Apache License, +Version 2.0. The original software and related information is available at +http://msgpack.sourceforge.jp/. + diff --git a/README b/README index 6e6ace4..0c1d440 100644 --- a/README +++ b/README @@ -17,41 +17,43 @@ Binary-based efficient data interchange format. Simply run ./configure && make && make install to install C and C++ binding. - $ ./configure --prefix=path/to/prefix - $ make - $ make install + $ ./configure + $ make + $ sudo make install - To install Ruby binding, run ./gengem.sh script in ruby/ directory and install + To install Ruby binding, run ./gengem.sh script on ruby/ directory and install generated gem package. - $ cd ruby - $ ./gengem.sh - $ gem install gem/pkg/msgpack-*.gem + $ cd ruby + $ ./gengem.sh + $ gem install gem/pkg/msgpack-*.gem *Usage C++: - #include - // TODO + include msgpack.hpp header and link libmsgpack library. + see example/simple.cc for example. + + g++ simple.cc -lmsgpack + g++ stream.cc -lmsgpack -lpthread C: - #include - /* TODO */ + include msgpack.h header and link libmsgpackc library. + see example/simple.c for example. + + gcc simple.c -lmsgpackc Ruby: - require 'msgpack' + require msgpack library. + see example/simple.rb for example. - # serialize - buf = [1, 2, 3].to_msgpack + ruby -rubygems simple.rb - # deserialize - p MessagePack::unpack(buf) - - # TODO + API Document is available at http://msgpack.sourceforge.jp/. Copyright (C) 2008-2009 FURUHASHI Sadayuki diff --git a/cpp/object.hpp b/cpp/object.hpp index b4c21f1..860b569 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -314,7 +314,6 @@ packer& operator<< (packer& o, const object& v) o << *p; } return o; - // FIXME loop optimiziation case type::MAP: o.pack_map(v.via.map.size); @@ -325,7 +324,6 @@ packer& operator<< (packer& o, const object& v) o << p->val; } return o; - // FIXME loop optimiziation default: throw type_error(); From 4f4fa39cd569f4f4c329c55f1d84d33c53ee8dd4 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 00:59:15 +0900 Subject: [PATCH 0079/1648] c: msgpack_sbuffer; cpp: msgpack::sbuffer --- c/sbuffer.h | 37 ++++++++++++++++++------- cpp/sbuffer.hpp | 72 ++++++++++++++++++++++++++++++------------------ cpp/zone.hpp.erb | 1 - example/simple.c | 2 +- 4 files changed, 73 insertions(+), 39 deletions(-) diff --git a/c/sbuffer.h b/c/sbuffer.h index 3694dbd..8dae103 100644 --- a/c/sbuffer.h +++ b/c/sbuffer.h @@ -21,14 +21,18 @@ #include #include +#ifndef MSGPACK_SBUFFER_INIT_SIZE +#define MSGPACK_SBUFFER_INIT_SIZE 2048 +#endif + #ifdef __cplusplus extern "C" { #endif -typedef struct { - char* ptr; +typedef struct msgpack_sbuffer { size_t size; - size_t capacity; + char* data; + size_t alloc; } msgpack_sbuffer; static inline void msgpack_sbuffer_init(msgpack_sbuffer* sbuf) @@ -38,27 +42,40 @@ static inline void msgpack_sbuffer_init(msgpack_sbuffer* sbuf) static inline void msgpack_sbuffer_destroy(msgpack_sbuffer* sbuf) { - free(sbuf->ptr); + free(sbuf->data); } static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned int len) { msgpack_sbuffer* sbuf = (msgpack_sbuffer*)data; - if(sbuf->capacity - sbuf->size < len) { - size_t nsize = (sbuf->capacity ? sbuf->capacity*2 : 2048); + + if(sbuf->alloc - sbuf->size < len) { + size_t nsize = (sbuf->alloc) ? + sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + while(nsize < sbuf->size + len) { nsize *= 2; } - void* tmp = realloc(sbuf->ptr, nsize); + void* tmp = realloc(sbuf->data, nsize); if(!tmp) { return -1; } - sbuf->ptr = (char*)tmp; - sbuf->capacity = nsize; + sbuf->data = (char*)tmp; + sbuf->alloc = nsize; } - memcpy(sbuf->ptr + sbuf->size, buf, len); + + memcpy(sbuf->data + sbuf->size, buf, len); sbuf->size += len; return 0; } +static inline char* msgpack_sbuffer_release(msgpack_sbuffer* sbuf) +{ + char* tmp = sbuf->data; + sbuf->data = NULL; + sbuf->size = 0; + sbuf->alloc = NULL; + return tmp; +} + #ifdef __cplusplus } diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 1dac6da..37ede49 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -18,62 +18,80 @@ #ifndef MSGPACK_SBUFFER_HPP__ #define MSGPACK_SBUFFER_HPP__ -#include -#include +#include "msgpack/sbuffer.h" #include namespace msgpack { -class sbuffer { +class sbuffer : public msgpack_sbuffer { public: - sbuffer() : m_capacity(0), m_size(0), m_ptr(NULL) { } + sbuffer(size_t initsz = MSGPACK_SBUFFER_INIT_SIZE) + { + msgpack_sbuffer* sbuf = static_cast(this); + + sbuf->data = (char*)::malloc(initsz); + if(!sbuf->data) { + throw std::bad_alloc(); + } + + sbuf->size = 0; + sbuf->alloc = initsz; + } ~sbuffer() { - free(m_ptr); + msgpack_sbuffer* sbuf = static_cast(this); + ::free(sbuf->data); } public: - void write(const char* buf, size_t len) + void write(const char* buf, unsigned int len) { - if(m_capacity - m_size < len) { - size_t nsize = (m_capacity ? m_capacity*2 : 2048); - while(nsize < m_size + len) { nsize *= 2; } - - char* tmp = (char*)realloc(m_ptr, nsize); - if(!tmp) { throw std::bad_alloc(); } - - m_ptr = tmp; - m_capacity = nsize; + msgpack_sbuffer* sbuf = static_cast(this); + if(sbuf->alloc - sbuf->size < len) { + expand_buffer(len); } - memcpy(m_ptr + m_size, buf, len); - m_size += len; + memcpy(sbuf->data + sbuf->size, buf, len); + sbuf->size += len; } char* data() { - return m_ptr; + msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->data; } size_t size() const { - return m_size; + const msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->size; } char* release() { - char* tmp = m_ptr; - m_capacity = 0; - m_size = 0; - m_ptr = NULL; - return tmp; + msgpack_sbuffer* sbuf = static_cast(this); + return msgpack_sbuffer_release(sbuf); } private: - size_t m_capacity; - size_t m_size; - char* m_ptr; + void expand_buffer(size_t len) + { + msgpack_sbuffer* sbuf = static_cast(this); + + size_t nsize = (sbuf->alloc) ? + sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + + while(nsize < sbuf->size + len) { nsize *= 2; } + + void* tmp = realloc(sbuf->data, nsize); + if(!tmp) { + throw std::bad_alloc(); + } + + sbuf->data = (char*)tmp; + sbuf->alloc = nsize; + } private: sbuffer(const sbuffer&); diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 06cb9d3..a253627 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -99,4 +99,3 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) #endif /* msgpack/zone.hpp */ -// vim: ft=cpp ts=4 sw=4 softtabstop=4 noexpandtab smarttab diff --git a/example/simple.c b/example/simple.c index df60424..41d8bb7 100644 --- a/example/simple.c +++ b/example/simple.c @@ -23,7 +23,7 @@ int main(void) msgpack_zone_init(&mempool, 2048); msgpack_object deserialized; - msgpack_unpack(sbuf.ptr, sbuf.size, NULL, &mempool, &deserialized); + msgpack_unpack(sbuf.data, sbuf.size, NULL, &mempool, &deserialized); /* print the deserialized object. */ msgpack_object_print(stdout, deserialized); From 76f18a0ea679af7f964ec455d93acc9b2c25b942 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 00:59:15 +0900 Subject: [PATCH 0080/1648] c: msgpack_sbuffer; cpp: msgpack::sbuffer --- c/sbuffer.h | 37 ++++++++++++++++++------- cpp/sbuffer.hpp | 72 ++++++++++++++++++++++++++++++------------------ cpp/zone.hpp.erb | 1 - example/simple.c | 2 +- 4 files changed, 73 insertions(+), 39 deletions(-) diff --git a/c/sbuffer.h b/c/sbuffer.h index 3694dbd..8dae103 100644 --- a/c/sbuffer.h +++ b/c/sbuffer.h @@ -21,14 +21,18 @@ #include #include +#ifndef MSGPACK_SBUFFER_INIT_SIZE +#define MSGPACK_SBUFFER_INIT_SIZE 2048 +#endif + #ifdef __cplusplus extern "C" { #endif -typedef struct { - char* ptr; +typedef struct msgpack_sbuffer { size_t size; - size_t capacity; + char* data; + size_t alloc; } msgpack_sbuffer; static inline void msgpack_sbuffer_init(msgpack_sbuffer* sbuf) @@ -38,27 +42,40 @@ static inline void msgpack_sbuffer_init(msgpack_sbuffer* sbuf) static inline void msgpack_sbuffer_destroy(msgpack_sbuffer* sbuf) { - free(sbuf->ptr); + free(sbuf->data); } static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned int len) { msgpack_sbuffer* sbuf = (msgpack_sbuffer*)data; - if(sbuf->capacity - sbuf->size < len) { - size_t nsize = (sbuf->capacity ? sbuf->capacity*2 : 2048); + + if(sbuf->alloc - sbuf->size < len) { + size_t nsize = (sbuf->alloc) ? + sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + while(nsize < sbuf->size + len) { nsize *= 2; } - void* tmp = realloc(sbuf->ptr, nsize); + void* tmp = realloc(sbuf->data, nsize); if(!tmp) { return -1; } - sbuf->ptr = (char*)tmp; - sbuf->capacity = nsize; + sbuf->data = (char*)tmp; + sbuf->alloc = nsize; } - memcpy(sbuf->ptr + sbuf->size, buf, len); + + memcpy(sbuf->data + sbuf->size, buf, len); sbuf->size += len; return 0; } +static inline char* msgpack_sbuffer_release(msgpack_sbuffer* sbuf) +{ + char* tmp = sbuf->data; + sbuf->data = NULL; + sbuf->size = 0; + sbuf->alloc = NULL; + return tmp; +} + #ifdef __cplusplus } diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 1dac6da..37ede49 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -18,62 +18,80 @@ #ifndef MSGPACK_SBUFFER_HPP__ #define MSGPACK_SBUFFER_HPP__ -#include -#include +#include "msgpack/sbuffer.h" #include namespace msgpack { -class sbuffer { +class sbuffer : public msgpack_sbuffer { public: - sbuffer() : m_capacity(0), m_size(0), m_ptr(NULL) { } + sbuffer(size_t initsz = MSGPACK_SBUFFER_INIT_SIZE) + { + msgpack_sbuffer* sbuf = static_cast(this); + + sbuf->data = (char*)::malloc(initsz); + if(!sbuf->data) { + throw std::bad_alloc(); + } + + sbuf->size = 0; + sbuf->alloc = initsz; + } ~sbuffer() { - free(m_ptr); + msgpack_sbuffer* sbuf = static_cast(this); + ::free(sbuf->data); } public: - void write(const char* buf, size_t len) + void write(const char* buf, unsigned int len) { - if(m_capacity - m_size < len) { - size_t nsize = (m_capacity ? m_capacity*2 : 2048); - while(nsize < m_size + len) { nsize *= 2; } - - char* tmp = (char*)realloc(m_ptr, nsize); - if(!tmp) { throw std::bad_alloc(); } - - m_ptr = tmp; - m_capacity = nsize; + msgpack_sbuffer* sbuf = static_cast(this); + if(sbuf->alloc - sbuf->size < len) { + expand_buffer(len); } - memcpy(m_ptr + m_size, buf, len); - m_size += len; + memcpy(sbuf->data + sbuf->size, buf, len); + sbuf->size += len; } char* data() { - return m_ptr; + msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->data; } size_t size() const { - return m_size; + const msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->size; } char* release() { - char* tmp = m_ptr; - m_capacity = 0; - m_size = 0; - m_ptr = NULL; - return tmp; + msgpack_sbuffer* sbuf = static_cast(this); + return msgpack_sbuffer_release(sbuf); } private: - size_t m_capacity; - size_t m_size; - char* m_ptr; + void expand_buffer(size_t len) + { + msgpack_sbuffer* sbuf = static_cast(this); + + size_t nsize = (sbuf->alloc) ? + sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + + while(nsize < sbuf->size + len) { nsize *= 2; } + + void* tmp = realloc(sbuf->data, nsize); + if(!tmp) { + throw std::bad_alloc(); + } + + sbuf->data = (char*)tmp; + sbuf->alloc = nsize; + } private: sbuffer(const sbuffer&); diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 06cb9d3..a253627 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -99,4 +99,3 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) #endif /* msgpack/zone.hpp */ -// vim: ft=cpp ts=4 sw=4 softtabstop=4 noexpandtab smarttab diff --git a/example/simple.c b/example/simple.c index df60424..41d8bb7 100644 --- a/example/simple.c +++ b/example/simple.c @@ -23,7 +23,7 @@ int main(void) msgpack_zone_init(&mempool, 2048); msgpack_object deserialized; - msgpack_unpack(sbuf.ptr, sbuf.size, NULL, &mempool, &deserialized); + msgpack_unpack(sbuf.data, sbuf.size, NULL, &mempool, &deserialized); /* print the deserialized object. */ msgpack_object_print(stdout, deserialized); From 6fda01111e45bd17aa46a8e1ae862eb4b298c8da Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:06:16 +0900 Subject: [PATCH 0081/1648] msgpack::sbuffer::data() const --- cpp/sbuffer.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 37ede49..2651b58 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -62,6 +62,12 @@ public: return sbuf->data; } + const char* data() const + { + const msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->data; + } + size_t size() const { const msgpack_sbuffer* sbuf = static_cast(this); From 17c2fa5a4ed436e9b9c5b6d186eea40875b68345 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:06:16 +0900 Subject: [PATCH 0082/1648] msgpack::sbuffer::data() const --- cpp/sbuffer.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 37ede49..2651b58 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -62,6 +62,12 @@ public: return sbuf->data; } + const char* data() const + { + const msgpack_sbuffer* sbuf = static_cast(this); + return sbuf->data; + } + size_t size() const { const msgpack_sbuffer* sbuf = static_cast(this); From 879c70f93a1d624b8f3f1646a593eeaf75a33613 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:13:31 +0900 Subject: [PATCH 0083/1648] fix msgpack_sbuffer_release --- c/sbuffer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c/sbuffer.h b/c/sbuffer.h index 8dae103..b0fab46 100644 --- a/c/sbuffer.h +++ b/c/sbuffer.h @@ -70,9 +70,9 @@ static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned in static inline char* msgpack_sbuffer_release(msgpack_sbuffer* sbuf) { char* tmp = sbuf->data; - sbuf->data = NULL; sbuf->size = 0; - sbuf->alloc = NULL; + sbuf->data = NULL; + sbuf->alloc = 0; return tmp; } From 4f2755366faef06ebf9bff17ea15e2a53ec15649 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:13:31 +0900 Subject: [PATCH 0084/1648] fix msgpack_sbuffer_release --- c/sbuffer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c/sbuffer.h b/c/sbuffer.h index 8dae103..b0fab46 100644 --- a/c/sbuffer.h +++ b/c/sbuffer.h @@ -70,9 +70,9 @@ static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned in static inline char* msgpack_sbuffer_release(msgpack_sbuffer* sbuf) { char* tmp = sbuf->data; - sbuf->data = NULL; sbuf->size = 0; - sbuf->alloc = NULL; + sbuf->data = NULL; + sbuf->alloc = 0; return tmp; } From 6fc38d1669a9bdd5f14c0d6a261ce0b6c04d0ecc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:31:12 +0900 Subject: [PATCH 0085/1648] msgpack::pack is not obsolete --- cpp/object.hpp | 10 ---------- cpp/pack.hpp | 7 +++++++ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 860b569..09ddb89 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -110,9 +110,6 @@ bool operator!=(const object x, const object y); std::ostream& operator<< (std::ostream& s, const object o); -template -inline void pack(Stream& s, const T& v); - template packer& operator<< (packer& o, const T& v); @@ -238,13 +235,6 @@ inline void pack(packer& o, const T& v) o.pack(v); } -// obsolete -template -inline void pack(Stream& s, const T& v) -{ - packer(s).pack(v); -} - // obsolete template inline void pack_copy(packer& o, T v) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 257ccb6..c8e37eb 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -112,6 +112,13 @@ private: }; +template +inline void pack(Stream& s, const T& v) +{ + packer(s).pack(v); +} + + #define msgpack_pack_inline_func(name) \ template \ inline void packer::_pack ## name From a8545b49c9da6ca75374973751e519d4da0bbf20 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 01:31:12 +0900 Subject: [PATCH 0086/1648] msgpack::pack is not obsolete --- cpp/object.hpp | 10 ---------- cpp/pack.hpp | 7 +++++++ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 860b569..09ddb89 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -110,9 +110,6 @@ bool operator!=(const object x, const object y); std::ostream& operator<< (std::ostream& s, const object o); -template -inline void pack(Stream& s, const T& v); - template packer& operator<< (packer& o, const T& v); @@ -238,13 +235,6 @@ inline void pack(packer& o, const T& v) o.pack(v); } -// obsolete -template -inline void pack(Stream& s, const T& v) -{ - packer(s).pack(v); -} - // obsolete template inline void pack_copy(packer& o, T v) diff --git a/cpp/pack.hpp b/cpp/pack.hpp index 257ccb6..c8e37eb 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -112,6 +112,13 @@ private: }; +template +inline void pack(Stream& s, const T& v) +{ + packer(s).pack(v); +} + + #define msgpack_pack_inline_func(name) \ template \ inline void packer::_pack ## name From bf13ba72b599e78ae4aa70f88fee7346e6d76dac Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 02:27:04 +0900 Subject: [PATCH 0087/1648] fix unpacker --- c/unpack.c | 84 ++++++++++++++++++++++++------------------------- c/unpack.h | 5 ++- cpp/sbuffer.hpp | 49 +++++++++++++---------------- cpp/unpack.hpp | 19 ++++++----- 4 files changed, 76 insertions(+), 81 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index 865e0d4..e90a29a 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -22,7 +22,7 @@ typedef struct { msgpack_zone* z; - bool* referenced; + bool referenced; } unpack_user; @@ -131,7 +131,7 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha o->type = MSGPACK_OBJECT_RAW; o->via.raw.ptr = p; o->via.raw.size = l; - *u->referenced = true; + u->referenced = true; return 0; } @@ -139,32 +139,33 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #define CTX_CAST(m) ((template_context*)(m)) +#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced static const size_t COUNTER_SIZE = sizeof(unsigned int); -static inline void init_count(void* buf) +static inline void init_count(void* buffer) { - *(volatile unsigned int*)buf = 1; + *(volatile unsigned int*)buffer = 1; } -static inline void decl_count(void* buf) +static inline void decl_count(void* buffer) { - //if(--*(unsigned int*)buf == 0) { - if(__sync_sub_and_fetch((unsigned int*)buf, 1) == 0) { - free(buf); + //if(--*(unsigned int*)buffer == 0) { + if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + free(buffer); } } -static inline void incr_count(void* buf) +static inline void incr_count(void* buffer) { - //++*(unsigned int*)buf; - __sync_add_and_fetch((unsigned int*)buf, 1); + //++*(unsigned int*)buffer; + __sync_add_and_fetch((unsigned int*)buffer, 1); } -static inline unsigned int get_count(void* buf) +static inline unsigned int get_count(void* buffer) { - return *(volatile unsigned int*)buf; + return *(volatile unsigned int*)buffer; } @@ -175,39 +176,38 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) initial_buffer_size = COUNTER_SIZE; } - char* buf = (char*)malloc(initial_buffer_size); - if(buf == NULL) { + char* buffer = (char*)malloc(initial_buffer_size); + if(buffer == NULL) { return false; } void* ctx = malloc(sizeof(template_context)); if(ctx == NULL) { - free(buf); + free(buffer); return false; } msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); if(z == NULL) { free(ctx); - free(buf); + free(buffer); return false; } - mpac->buf = buf; + mpac->buffer = buffer; mpac->used = COUNTER_SIZE; mpac->free = initial_buffer_size - mpac->used; mpac->off = COUNTER_SIZE; mpac->parsed = 0; mpac->initial_buffer_size = initial_buffer_size; mpac->z = z; - mpac->referenced = false; mpac->ctx = ctx; - init_count(mpac->buf); + init_count(mpac->buffer); template_init(CTX_CAST(mpac->ctx)); CTX_CAST(mpac->ctx)->user.z = mpac->z; - CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + CTX_CAST(mpac->ctx)->user.referenced = false; return true; } @@ -216,7 +216,7 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac) { msgpack_zone_free(mpac->z); free(mpac->ctx); - decl_count(mpac->buf); + decl_count(mpac->buffer); } @@ -241,10 +241,10 @@ void msgpack_unpacker_free(msgpack_unpacker* mpac) free(mpac); } - bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) { - if(mpac->used == mpac->off && get_count(mpac->buf) == 1 && !mpac->referenced) { + if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 + && !CTX_REFERENCED(mpac)) { // rewind buffer mpac->free += mpac->used - COUNTER_SIZE; mpac->used = COUNTER_SIZE; @@ -261,12 +261,12 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) next_size *= 2; } - char* tmp = (char*)realloc(mpac->buf, next_size); + char* tmp = (char*)realloc(mpac->buffer, next_size); if(tmp == NULL) { return false; } - mpac->buf = tmp; + mpac->buffer = tmp; mpac->free = next_size - mpac->used; } else { @@ -283,19 +283,19 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) init_count(tmp); - if(mpac->referenced) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { free(tmp); return false; } - mpac->referenced = false; + CTX_REFERENCED(mpac) = false; } else { - decl_count(mpac->buf); + decl_count(mpac->buffer); } - memcpy(tmp+COUNTER_SIZE, mpac->buf+mpac->off, not_parsed); + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - mpac->buf = tmp; + mpac->buffer = tmp; mpac->used = not_parsed + COUNTER_SIZE; mpac->free = next_size - mpac->used; mpac->off = COUNTER_SIZE; @@ -308,7 +308,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* mpac) { size_t off = mpac->off; int ret = template_execute(CTX_CAST(mpac->ctx), - mpac->buf, mpac->used, &mpac->off); + mpac->buffer, mpac->used, &mpac->off); if(mpac->off > off) { mpac->parsed += mpac->off - off; } @@ -326,26 +326,26 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) return false; } - msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(z == NULL) { + msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(r == NULL) { return NULL; } msgpack_zone* old = mpac->z; - mpac->z = z; + mpac->z = r; return old; } bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) { - if(mpac->referenced) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { return false; } - mpac->referenced = false; + CTX_REFERENCED(mpac) = false; - incr_count(mpac->buf); + incr_count(mpac->buffer); } return true; @@ -354,6 +354,7 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) void msgpack_unpacker_reset(msgpack_unpacker* mpac) { template_init(CTX_CAST(mpac->ctx)); + // don't reset referenced flag mpac->parsed = 0; } @@ -365,9 +366,8 @@ msgpack_unpack(const char* data, size_t len, size_t* off, template_context ctx; template_init(&ctx); - bool referenced = false; ctx.user.z = z; - ctx.user.referenced = &referenced; + ctx.user.referenced = false; size_t noff = 0; if(off != NULL) { noff = *off; } diff --git a/c/unpack.h b/c/unpack.h index 779e256..a9caf07 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -29,13 +29,12 @@ extern "C" { typedef struct msgpack_unpacker { - char* buf; + char* buffer; size_t used; size_t free; size_t off; size_t parsed; msgpack_zone* z; - bool referenced; size_t initial_buffer_size; void* ctx; } msgpack_unpacker; @@ -91,7 +90,7 @@ bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) char* msgpack_unpacker_buffer(msgpack_unpacker* mpac) { - return mpac->buf + mpac->used; + return mpac->buffer + mpac->used; } size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac) diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 2651b58..ca06884e 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -28,77 +28,70 @@ class sbuffer : public msgpack_sbuffer { public: sbuffer(size_t initsz = MSGPACK_SBUFFER_INIT_SIZE) { - msgpack_sbuffer* sbuf = static_cast(this); - - sbuf->data = (char*)::malloc(initsz); - if(!sbuf->data) { + base::data = (char*)::malloc(initsz); + if(!base::data) { throw std::bad_alloc(); } - sbuf->size = 0; - sbuf->alloc = initsz; + base::size = 0; + base::alloc = initsz; } ~sbuffer() { - msgpack_sbuffer* sbuf = static_cast(this); - ::free(sbuf->data); + ::free(base::data); } public: void write(const char* buf, unsigned int len) { - msgpack_sbuffer* sbuf = static_cast(this); - if(sbuf->alloc - sbuf->size < len) { + if(base::alloc - base::size < len) { expand_buffer(len); } - memcpy(sbuf->data + sbuf->size, buf, len); - sbuf->size += len; + memcpy(base::data + base::size, buf, len); + base::size += len; } char* data() { - msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->data; + return base::data; } const char* data() const { - const msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->data; + return base::data; } size_t size() const { - const msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->size; + return base::size; } char* release() { - msgpack_sbuffer* sbuf = static_cast(this); - return msgpack_sbuffer_release(sbuf); + return msgpack_sbuffer_release(this); } private: void expand_buffer(size_t len) { - msgpack_sbuffer* sbuf = static_cast(this); - - size_t nsize = (sbuf->alloc) ? - sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + size_t nsize = (base::alloc) ? + base::alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; - while(nsize < sbuf->size + len) { nsize *= 2; } + while(nsize < base::size + len) { nsize *= 2; } - void* tmp = realloc(sbuf->data, nsize); + void* tmp = realloc(base::data, nsize); if(!tmp) { throw std::bad_alloc(); } - sbuf->data = (char*)tmp; - sbuf->alloc = nsize; + base::data = (char*)tmp; + base::alloc = nsize; } +private: + typedef msgpack_sbuffer base; + private: sbuffer(const sbuffer&); }; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index e140b36..38ac7ac 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -39,7 +39,7 @@ struct unpack_error : public std::runtime_error { class unpacker : public msgpack_unpacker { public: - unpacker(size_t initial_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); + unpacker(size_t init_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); ~unpacker(); public: @@ -126,6 +126,9 @@ public: // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); +private: + typedef msgpack_unpacker base; + private: unpacker(const unpacker&); }; @@ -207,9 +210,9 @@ inline zone* unpacker::release_zone() zone* r = new zone(); - msgpack_zone old = *this->z; - *this->z = *z; - *z = old; + msgpack_zone old = *base::z; + *base::z = *r; + *static_cast(r) = old; return r; } @@ -232,22 +235,22 @@ inline size_t unpacker::parsed_size() const inline char* unpacker::nonparsed_buffer() { - return buf + off; + return base::buffer + base::off; } inline size_t unpacker::nonparsed_size() const { - return used - off; + return base::used - base::off; } inline void unpacker::skip_nonparsed_buffer(size_t size) { - off += size; + base::off += size; } inline void unpacker::remove_nonparsed_buffer() { - used = off; + base::used = base::off; } From ef1c4f82b27e2690694e1995828ca374b5cd225e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 02:27:04 +0900 Subject: [PATCH 0088/1648] fix unpacker --- c/unpack.c | 84 ++++++++++++++++++++++++------------------------- c/unpack.h | 5 ++- cpp/sbuffer.hpp | 49 +++++++++++++---------------- cpp/unpack.hpp | 19 ++++++----- 4 files changed, 76 insertions(+), 81 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index 865e0d4..e90a29a 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -22,7 +22,7 @@ typedef struct { msgpack_zone* z; - bool* referenced; + bool referenced; } unpack_user; @@ -131,7 +131,7 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha o->type = MSGPACK_OBJECT_RAW; o->via.raw.ptr = p; o->via.raw.size = l; - *u->referenced = true; + u->referenced = true; return 0; } @@ -139,32 +139,33 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #define CTX_CAST(m) ((template_context*)(m)) +#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced static const size_t COUNTER_SIZE = sizeof(unsigned int); -static inline void init_count(void* buf) +static inline void init_count(void* buffer) { - *(volatile unsigned int*)buf = 1; + *(volatile unsigned int*)buffer = 1; } -static inline void decl_count(void* buf) +static inline void decl_count(void* buffer) { - //if(--*(unsigned int*)buf == 0) { - if(__sync_sub_and_fetch((unsigned int*)buf, 1) == 0) { - free(buf); + //if(--*(unsigned int*)buffer == 0) { + if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + free(buffer); } } -static inline void incr_count(void* buf) +static inline void incr_count(void* buffer) { - //++*(unsigned int*)buf; - __sync_add_and_fetch((unsigned int*)buf, 1); + //++*(unsigned int*)buffer; + __sync_add_and_fetch((unsigned int*)buffer, 1); } -static inline unsigned int get_count(void* buf) +static inline unsigned int get_count(void* buffer) { - return *(volatile unsigned int*)buf; + return *(volatile unsigned int*)buffer; } @@ -175,39 +176,38 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) initial_buffer_size = COUNTER_SIZE; } - char* buf = (char*)malloc(initial_buffer_size); - if(buf == NULL) { + char* buffer = (char*)malloc(initial_buffer_size); + if(buffer == NULL) { return false; } void* ctx = malloc(sizeof(template_context)); if(ctx == NULL) { - free(buf); + free(buffer); return false; } msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); if(z == NULL) { free(ctx); - free(buf); + free(buffer); return false; } - mpac->buf = buf; + mpac->buffer = buffer; mpac->used = COUNTER_SIZE; mpac->free = initial_buffer_size - mpac->used; mpac->off = COUNTER_SIZE; mpac->parsed = 0; mpac->initial_buffer_size = initial_buffer_size; mpac->z = z; - mpac->referenced = false; mpac->ctx = ctx; - init_count(mpac->buf); + init_count(mpac->buffer); template_init(CTX_CAST(mpac->ctx)); CTX_CAST(mpac->ctx)->user.z = mpac->z; - CTX_CAST(mpac->ctx)->user.referenced = &mpac->referenced; + CTX_CAST(mpac->ctx)->user.referenced = false; return true; } @@ -216,7 +216,7 @@ void msgpack_unpacker_destroy(msgpack_unpacker* mpac) { msgpack_zone_free(mpac->z); free(mpac->ctx); - decl_count(mpac->buf); + decl_count(mpac->buffer); } @@ -241,10 +241,10 @@ void msgpack_unpacker_free(msgpack_unpacker* mpac) free(mpac); } - bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) { - if(mpac->used == mpac->off && get_count(mpac->buf) == 1 && !mpac->referenced) { + if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 + && !CTX_REFERENCED(mpac)) { // rewind buffer mpac->free += mpac->used - COUNTER_SIZE; mpac->used = COUNTER_SIZE; @@ -261,12 +261,12 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) next_size *= 2; } - char* tmp = (char*)realloc(mpac->buf, next_size); + char* tmp = (char*)realloc(mpac->buffer, next_size); if(tmp == NULL) { return false; } - mpac->buf = tmp; + mpac->buffer = tmp; mpac->free = next_size - mpac->used; } else { @@ -283,19 +283,19 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) init_count(tmp); - if(mpac->referenced) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { free(tmp); return false; } - mpac->referenced = false; + CTX_REFERENCED(mpac) = false; } else { - decl_count(mpac->buf); + decl_count(mpac->buffer); } - memcpy(tmp+COUNTER_SIZE, mpac->buf+mpac->off, not_parsed); + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - mpac->buf = tmp; + mpac->buffer = tmp; mpac->used = not_parsed + COUNTER_SIZE; mpac->free = next_size - mpac->used; mpac->off = COUNTER_SIZE; @@ -308,7 +308,7 @@ int msgpack_unpacker_execute(msgpack_unpacker* mpac) { size_t off = mpac->off; int ret = template_execute(CTX_CAST(mpac->ctx), - mpac->buf, mpac->used, &mpac->off); + mpac->buffer, mpac->used, &mpac->off); if(mpac->off > off) { mpac->parsed += mpac->off - off; } @@ -326,26 +326,26 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) return false; } - msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(z == NULL) { + msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(r == NULL) { return NULL; } msgpack_zone* old = mpac->z; - mpac->z = z; + mpac->z = r; return old; } bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) { - if(mpac->referenced) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buf)) { + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { return false; } - mpac->referenced = false; + CTX_REFERENCED(mpac) = false; - incr_count(mpac->buf); + incr_count(mpac->buffer); } return true; @@ -354,6 +354,7 @@ bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) void msgpack_unpacker_reset(msgpack_unpacker* mpac) { template_init(CTX_CAST(mpac->ctx)); + // don't reset referenced flag mpac->parsed = 0; } @@ -365,9 +366,8 @@ msgpack_unpack(const char* data, size_t len, size_t* off, template_context ctx; template_init(&ctx); - bool referenced = false; ctx.user.z = z; - ctx.user.referenced = &referenced; + ctx.user.referenced = false; size_t noff = 0; if(off != NULL) { noff = *off; } diff --git a/c/unpack.h b/c/unpack.h index 779e256..a9caf07 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -29,13 +29,12 @@ extern "C" { typedef struct msgpack_unpacker { - char* buf; + char* buffer; size_t used; size_t free; size_t off; size_t parsed; msgpack_zone* z; - bool referenced; size_t initial_buffer_size; void* ctx; } msgpack_unpacker; @@ -91,7 +90,7 @@ bool msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size) char* msgpack_unpacker_buffer(msgpack_unpacker* mpac) { - return mpac->buf + mpac->used; + return mpac->buffer + mpac->used; } size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac) diff --git a/cpp/sbuffer.hpp b/cpp/sbuffer.hpp index 2651b58..ca06884e 100644 --- a/cpp/sbuffer.hpp +++ b/cpp/sbuffer.hpp @@ -28,77 +28,70 @@ class sbuffer : public msgpack_sbuffer { public: sbuffer(size_t initsz = MSGPACK_SBUFFER_INIT_SIZE) { - msgpack_sbuffer* sbuf = static_cast(this); - - sbuf->data = (char*)::malloc(initsz); - if(!sbuf->data) { + base::data = (char*)::malloc(initsz); + if(!base::data) { throw std::bad_alloc(); } - sbuf->size = 0; - sbuf->alloc = initsz; + base::size = 0; + base::alloc = initsz; } ~sbuffer() { - msgpack_sbuffer* sbuf = static_cast(this); - ::free(sbuf->data); + ::free(base::data); } public: void write(const char* buf, unsigned int len) { - msgpack_sbuffer* sbuf = static_cast(this); - if(sbuf->alloc - sbuf->size < len) { + if(base::alloc - base::size < len) { expand_buffer(len); } - memcpy(sbuf->data + sbuf->size, buf, len); - sbuf->size += len; + memcpy(base::data + base::size, buf, len); + base::size += len; } char* data() { - msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->data; + return base::data; } const char* data() const { - const msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->data; + return base::data; } size_t size() const { - const msgpack_sbuffer* sbuf = static_cast(this); - return sbuf->size; + return base::size; } char* release() { - msgpack_sbuffer* sbuf = static_cast(this); - return msgpack_sbuffer_release(sbuf); + return msgpack_sbuffer_release(this); } private: void expand_buffer(size_t len) { - msgpack_sbuffer* sbuf = static_cast(this); - - size_t nsize = (sbuf->alloc) ? - sbuf->alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; + size_t nsize = (base::alloc) ? + base::alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE; - while(nsize < sbuf->size + len) { nsize *= 2; } + while(nsize < base::size + len) { nsize *= 2; } - void* tmp = realloc(sbuf->data, nsize); + void* tmp = realloc(base::data, nsize); if(!tmp) { throw std::bad_alloc(); } - sbuf->data = (char*)tmp; - sbuf->alloc = nsize; + base::data = (char*)tmp; + base::alloc = nsize; } +private: + typedef msgpack_sbuffer base; + private: sbuffer(const sbuffer&); }; diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index e140b36..38ac7ac 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -39,7 +39,7 @@ struct unpack_error : public std::runtime_error { class unpacker : public msgpack_unpacker { public: - unpacker(size_t initial_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); + unpacker(size_t init_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); ~unpacker(); public: @@ -126,6 +126,9 @@ public: // Note that reset() leaves non-parsed buffer. void remove_nonparsed_buffer(); +private: + typedef msgpack_unpacker base; + private: unpacker(const unpacker&); }; @@ -207,9 +210,9 @@ inline zone* unpacker::release_zone() zone* r = new zone(); - msgpack_zone old = *this->z; - *this->z = *z; - *z = old; + msgpack_zone old = *base::z; + *base::z = *r; + *static_cast(r) = old; return r; } @@ -232,22 +235,22 @@ inline size_t unpacker::parsed_size() const inline char* unpacker::nonparsed_buffer() { - return buf + off; + return base::buffer + base::off; } inline size_t unpacker::nonparsed_size() const { - return used - off; + return base::used - base::off; } inline void unpacker::skip_nonparsed_buffer(size_t size) { - off += size; + base::off += size; } inline void unpacker::remove_nonparsed_buffer() { - used = off; + base::used = base::off; } From 0efb8160b97c78b67329436a45203e53e0f5cf2d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 02:55:45 +0900 Subject: [PATCH 0089/1648] msgpack_zone_clear, msgpack::zone::clear --- c/unpack.c | 5 +++++ c/unpack.h | 2 ++ c/zone.c | 32 +++++++++++++++++++++++++++++++- c/zone.h | 2 ++ cpp/unpack.hpp | 8 ++++++++ cpp/zone.hpp.erb | 7 +++++++ 6 files changed, 55 insertions(+), 1 deletion(-) diff --git a/c/unpack.c b/c/unpack.c index e90a29a..f1b3bb1 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -337,6 +337,11 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) return old; } +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) +{ + msgpack_zone_clear(mpac->z); +} + bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) { if(CTX_REFERENCED(mpac)) { diff --git a/c/unpack.h b/c/unpack.h index a9caf07..ef63774 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -58,6 +58,8 @@ msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac); msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac); + void msgpack_unpacker_reset(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac); diff --git a/c/zone.c b/c/zone.c index e891c82..7a275f6 100644 --- a/c/zone.c +++ b/c/zone.c @@ -57,9 +57,23 @@ static inline void destroy_chunk_array(msgpack_zone_chunk_array* ca) for(; chunk != ca->tail+1; ++chunk) { free(chunk->alloc); } + free(ca->array); } +static inline void clear_chunk_array(msgpack_zone_chunk_array* ca) +{ + msgpack_zone_chunk* chunk = ca->array + 1; + for(; chunk != ca->tail+1; ++chunk) { + free(chunk->alloc); + } + + ca->tail = ca->array; + + ca->array[0].free += ca->array[0].ptr - (char*)ca->array[0].alloc; + ca->array[0].ptr = (char*)ca->array[0].alloc; +} + void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) { msgpack_zone_chunk_array* const ca = &zone->chunk_array; @@ -124,16 +138,27 @@ static inline void init_finalizer_array(msgpack_zone_finalizer_array* fa) fa->array = NULL; } -static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +static inline void call_finalizer_array(msgpack_zone_finalizer_array* fa) { // 逆順ã«å‘¼ã³å‡ºã— msgpack_zone_finalizer* fin = fa->tail; for(; fin != fa->array; --fin) { (*(fin-1)->func)((fin-1)->data); } +} + +static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + call_finalizer_array(fa); free(fa->array); } +static inline void clear_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + call_finalizer_array(fa); + fa->tail = fa->array; +} + bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data) { @@ -208,6 +233,11 @@ void msgpack_zone_destroy(msgpack_zone* zone) destroy_chunk_array(&zone->chunk_array); } +void msgpack_zone_clear(msgpack_zone* zone) +{ + clear_finalizer_array(&zone->finalizer_array); + clear_chunk_array(&zone->chunk_array); +} msgpack_zone* msgpack_zone_new(size_t chunk_size) { diff --git a/c/zone.h b/c/zone.h index 3dc9f52..3c1188d 100644 --- a/c/zone.h +++ b/c/zone.h @@ -72,6 +72,8 @@ bool msgpack_zone_push_finalizer(msgpack_zone* zone, bool msgpack_zone_is_empty(msgpack_zone* zone); +void msgpack_zone_clear(msgpack_zone* zone); + #ifdef __cplusplus } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 38ac7ac..324111a 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -64,6 +64,9 @@ public: // otherwise the memrory will leak. zone* release_zone(); + /*! 5.2. this method is equivalence to `delete release_zone()` */ + void reset_zone(); + /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); @@ -217,6 +220,11 @@ inline zone* unpacker::release_zone() return r; } +inline void unpacker::reset_zone() +{ + msgpack_unpacker_reset_zone(this); +} + inline void unpacker::reset() { msgpack_unpacker_reset(this); diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index a253627..930c8e8 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -37,6 +37,8 @@ public: void push_finalizer(void (*func)(void*), void* data); + void clear(); + <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> T* allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>); @@ -78,6 +80,11 @@ inline void zone::push_finalizer(void (*func)(void*), void* data) } } +inline void zone::clear() +{ + msgpack_zone_clear(this); +} + template void zone::object_destructor(void* obj) { From b33ecbd92baa866803e34e9d7f1148007b16f180 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 02:55:45 +0900 Subject: [PATCH 0090/1648] msgpack_zone_clear, msgpack::zone::clear --- c/unpack.c | 5 +++++ c/unpack.h | 2 ++ c/zone.c | 32 +++++++++++++++++++++++++++++++- c/zone.h | 2 ++ cpp/unpack.hpp | 8 ++++++++ cpp/zone.hpp.erb | 7 +++++++ 6 files changed, 55 insertions(+), 1 deletion(-) diff --git a/c/unpack.c b/c/unpack.c index e90a29a..f1b3bb1 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -337,6 +337,11 @@ msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) return old; } +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) +{ + msgpack_zone_clear(mpac->z); +} + bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) { if(CTX_REFERENCED(mpac)) { diff --git a/c/unpack.h b/c/unpack.h index a9caf07..ef63774 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -58,6 +58,8 @@ msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac); msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac); +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac); + void msgpack_unpacker_reset(msgpack_unpacker* mpac); static inline size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac); diff --git a/c/zone.c b/c/zone.c index e891c82..7a275f6 100644 --- a/c/zone.c +++ b/c/zone.c @@ -57,9 +57,23 @@ static inline void destroy_chunk_array(msgpack_zone_chunk_array* ca) for(; chunk != ca->tail+1; ++chunk) { free(chunk->alloc); } + free(ca->array); } +static inline void clear_chunk_array(msgpack_zone_chunk_array* ca) +{ + msgpack_zone_chunk* chunk = ca->array + 1; + for(; chunk != ca->tail+1; ++chunk) { + free(chunk->alloc); + } + + ca->tail = ca->array; + + ca->array[0].free += ca->array[0].ptr - (char*)ca->array[0].alloc; + ca->array[0].ptr = (char*)ca->array[0].alloc; +} + void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) { msgpack_zone_chunk_array* const ca = &zone->chunk_array; @@ -124,16 +138,27 @@ static inline void init_finalizer_array(msgpack_zone_finalizer_array* fa) fa->array = NULL; } -static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +static inline void call_finalizer_array(msgpack_zone_finalizer_array* fa) { // 逆順ã«å‘¼ã³å‡ºã— msgpack_zone_finalizer* fin = fa->tail; for(; fin != fa->array; --fin) { (*(fin-1)->func)((fin-1)->data); } +} + +static inline void destroy_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + call_finalizer_array(fa); free(fa->array); } +static inline void clear_finalizer_array(msgpack_zone_finalizer_array* fa) +{ + call_finalizer_array(fa); + fa->tail = fa->array; +} + bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data) { @@ -208,6 +233,11 @@ void msgpack_zone_destroy(msgpack_zone* zone) destroy_chunk_array(&zone->chunk_array); } +void msgpack_zone_clear(msgpack_zone* zone) +{ + clear_finalizer_array(&zone->finalizer_array); + clear_chunk_array(&zone->chunk_array); +} msgpack_zone* msgpack_zone_new(size_t chunk_size) { diff --git a/c/zone.h b/c/zone.h index 3dc9f52..3c1188d 100644 --- a/c/zone.h +++ b/c/zone.h @@ -72,6 +72,8 @@ bool msgpack_zone_push_finalizer(msgpack_zone* zone, bool msgpack_zone_is_empty(msgpack_zone* zone); +void msgpack_zone_clear(msgpack_zone* zone); + #ifdef __cplusplus } diff --git a/cpp/unpack.hpp b/cpp/unpack.hpp index 38ac7ac..324111a 100644 --- a/cpp/unpack.hpp +++ b/cpp/unpack.hpp @@ -64,6 +64,9 @@ public: // otherwise the memrory will leak. zone* release_zone(); + /*! 5.2. this method is equivalence to `delete release_zone()` */ + void reset_zone(); + /*! 5.3. after release_zone(), re-initialize unpacker */ void reset(); @@ -217,6 +220,11 @@ inline zone* unpacker::release_zone() return r; } +inline void unpacker::reset_zone() +{ + msgpack_unpacker_reset_zone(this); +} + inline void unpacker::reset() { msgpack_unpacker_reset(this); diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index a253627..930c8e8 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -37,6 +37,8 @@ public: void push_finalizer(void (*func)(void*), void* data); + void clear(); + <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> T* allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>); @@ -78,6 +80,11 @@ inline void zone::push_finalizer(void (*func)(void*), void* data) } } +inline void zone::clear() +{ + msgpack_zone_clear(this); +} + template void zone::object_destructor(void* obj) { From cb24a0c24f79adc6d3646cbd5b63005029e8fa0c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 03:39:34 +0900 Subject: [PATCH 0091/1648] fix msgpack_unpacker_expand_buffer --- c/unpack.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index f1b3bb1..08fd6cb 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -283,6 +283,8 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) init_count(tmp); + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); + if(CTX_REFERENCED(mpac)) { if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { free(tmp); @@ -293,8 +295,6 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) decl_count(mpac->buffer); } - memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - mpac->buffer = tmp; mpac->used = not_parsed + COUNTER_SIZE; mpac->free = next_size - mpac->used; From 87724d5b51b33968dc7db0bdc520c6405f50a5a7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 03:39:34 +0900 Subject: [PATCH 0092/1648] fix msgpack_unpacker_expand_buffer --- c/unpack.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index f1b3bb1..08fd6cb 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -283,6 +283,8 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) init_count(tmp); + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); + if(CTX_REFERENCED(mpac)) { if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { free(tmp); @@ -293,8 +295,6 @@ bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) decl_count(mpac->buffer); } - memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - mpac->buffer = tmp; mpac->used = not_parsed + COUNTER_SIZE; mpac->free = next_size - mpac->used; From c3f9696268e94bea44427c31c592402d54f02358 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 13:46:20 +0900 Subject: [PATCH 0093/1648] fix packaging problem --- Makefile.am | 17 +++++++++++++++++ cpp/Makefile.am | 10 +++++----- ruby/gem/AUTHORS | 1 - ruby/gem/ChangeLog | 0 ruby/gengem.sh | 3 ++- 5 files changed, 24 insertions(+), 7 deletions(-) delete mode 100644 ruby/gem/AUTHORS delete mode 100644 ruby/gem/ChangeLog diff --git a/Makefile.am b/Makefile.am index 3144972..d067fe0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -11,3 +11,20 @@ nobase_include_HEADERS = \ msgpack/unpack_define.h \ msgpack/unpack_template.h +EXTRA_DIST = \ + ruby/bench.rb \ + ruby/extconf.rb \ + ruby/gem/README \ + ruby/gem/Rakefile \ + ruby/gem/test/test_helper.rb \ + ruby/gengem.sh \ + ruby/msgpack.gemspec \ + ruby/pack.c \ + ruby/pack.h \ + ruby/rbinit.c \ + ruby/test_case.rb \ + ruby/test_format.rb \ + ruby/test_pack.rb \ + ruby/unpack.c \ + ruby/unpack.h + diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 406c57b..76770f4 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -20,12 +20,8 @@ nobase_include_HEADERS = \ msgpack/type/raw.hpp \ msgpack/type/tuple.hpp -noinst_HEADERS = \ - msgpack/type/tuple.hpp.erb - # FIXME -object.lo: msgpack/type/tuple.hpp -unpack.lo: msgpack/type/tuple.hpp msgpack/zone.hpp +object.lo: msgpack/type/tuple.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp @@ -39,6 +35,10 @@ MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ msgpack/zone.hpp +EXTRA_DIST = \ + msgpack/type/tuple.hpp.erb \ + msgpack/zone.hpp.erb + libmsgpack_la_LIBADD = -L../c -lmsgpackc # -version-info CURRENT:REVISION:AGE diff --git a/ruby/gem/AUTHORS b/ruby/gem/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/ruby/gem/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/ruby/gem/ChangeLog b/ruby/gem/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/ruby/gengem.sh b/ruby/gengem.sh index 6c07be3..0afb8f5 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -9,7 +9,8 @@ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ cat test_case.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > gem/test/msgpack_test.rb -#cp ../README gem/README.txt +cp ../AUTHORS gem/AUTHORS +cp ../ChangeLog gem/ChangeLog cp ../msgpack/pack_define.h gem/msgpack/ cp ../msgpack/pack_template.h gem/msgpack/ cp ../msgpack/unpack_define.h gem/msgpack/ From c398d5d82820aaab89ed7e2e407db742376f5d3f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 13:46:20 +0900 Subject: [PATCH 0094/1648] fix packaging problem --- Makefile.am | 17 +++++++++++++++++ cpp/Makefile.am | 10 +++++----- ruby/gem/AUTHORS | 1 - ruby/gem/ChangeLog | 0 ruby/gengem.sh | 3 ++- 5 files changed, 24 insertions(+), 7 deletions(-) delete mode 100644 ruby/gem/AUTHORS delete mode 100644 ruby/gem/ChangeLog diff --git a/Makefile.am b/Makefile.am index 3144972..d067fe0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -11,3 +11,20 @@ nobase_include_HEADERS = \ msgpack/unpack_define.h \ msgpack/unpack_template.h +EXTRA_DIST = \ + ruby/bench.rb \ + ruby/extconf.rb \ + ruby/gem/README \ + ruby/gem/Rakefile \ + ruby/gem/test/test_helper.rb \ + ruby/gengem.sh \ + ruby/msgpack.gemspec \ + ruby/pack.c \ + ruby/pack.h \ + ruby/rbinit.c \ + ruby/test_case.rb \ + ruby/test_format.rb \ + ruby/test_pack.rb \ + ruby/unpack.c \ + ruby/unpack.h + diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 406c57b..76770f4 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -20,12 +20,8 @@ nobase_include_HEADERS = \ msgpack/type/raw.hpp \ msgpack/type/tuple.hpp -noinst_HEADERS = \ - msgpack/type/tuple.hpp.erb - # FIXME -object.lo: msgpack/type/tuple.hpp -unpack.lo: msgpack/type/tuple.hpp msgpack/zone.hpp +object.lo: msgpack/type/tuple.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp @@ -39,6 +35,10 @@ MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ msgpack/zone.hpp +EXTRA_DIST = \ + msgpack/type/tuple.hpp.erb \ + msgpack/zone.hpp.erb + libmsgpack_la_LIBADD = -L../c -lmsgpackc # -version-info CURRENT:REVISION:AGE diff --git a/ruby/gem/AUTHORS b/ruby/gem/AUTHORS deleted file mode 100644 index ababacb..0000000 --- a/ruby/gem/AUTHORS +++ /dev/null @@ -1 +0,0 @@ -FURUHASHI Sadayuki diff --git a/ruby/gem/ChangeLog b/ruby/gem/ChangeLog deleted file mode 100644 index e69de29..0000000 diff --git a/ruby/gengem.sh b/ruby/gengem.sh index 6c07be3..0afb8f5 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -9,7 +9,8 @@ cp rbinit.c gem/ext/ cp unpack.c gem/ext/ cp unpack.h gem/ext/ cat test_case.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > gem/test/msgpack_test.rb -#cp ../README gem/README.txt +cp ../AUTHORS gem/AUTHORS +cp ../ChangeLog gem/ChangeLog cp ../msgpack/pack_define.h gem/msgpack/ cp ../msgpack/pack_template.h gem/msgpack/ cp ../msgpack/unpack_define.h gem/msgpack/ From e7403013e5ca7d16d3610f81cb780ce4e0f548dc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 14:59:13 +0900 Subject: [PATCH 0095/1648] msgpack_zone: sizeof(int) bytes alignment --- c/zone.c | 19 ++----------------- c/zone.h | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/c/zone.c b/c/zone.c index 7a275f6..877f4ed 100644 --- a/c/zone.c +++ b/c/zone.c @@ -19,7 +19,6 @@ #include #include - static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_size) { // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ @@ -74,25 +73,11 @@ static inline void clear_chunk_array(msgpack_zone_chunk_array* ca) ca->array[0].ptr = (char*)ca->array[0].alloc; } -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) { msgpack_zone_chunk_array* const ca = &zone->chunk_array; - msgpack_zone_chunk* chunk = ca->tail; - - if(chunk->free > size) { - // chunkã«ç©ºã容é‡ãŒã‚ã‚‹ - // 空ã容é‡ã‚’消費ã—ã¦è¿”ã™ - - char* ptr = chunk->ptr; - - chunk->ptr += size; - chunk->free -= size; - - return ptr; - } - - chunk = ++ca->tail; + msgpack_zone_chunk* chunk = ++ca->tail; if(chunk == ca->end) { // ca->arrayã«ç©ºããŒãªã„ diff --git a/c/zone.h b/c/zone.h index 3c1188d..93a4642 100644 --- a/c/zone.h +++ b/c/zone.h @@ -65,7 +65,7 @@ void msgpack_zone_destroy(msgpack_zone* zone); msgpack_zone* msgpack_zone_new(size_t chunk_size); void msgpack_zone_free(msgpack_zone* zone); -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); +static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); @@ -75,6 +75,30 @@ bool msgpack_zone_is_empty(msgpack_zone* zone); void msgpack_zone_clear(msgpack_zone* zone); + +#ifndef MSGPACK_ZONE_ALIGN +#define MSGPACK_ZONE_ALIGN sizeof(int) +#endif + +void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size); + +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + size = ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1); + + msgpack_zone_chunk* chunk = zone->chunk_array.tail; + + if(chunk->free > size) { + char* ptr = chunk->ptr; + chunk->ptr += size; + chunk->free -= size; + return ptr; + } + + return msgpack_zone_malloc_expand(zone, size); +} + + #ifdef __cplusplus } #endif From 319838e51757b59a9aa9e017491139602bcc1e68 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 14:59:13 +0900 Subject: [PATCH 0096/1648] msgpack_zone: sizeof(int) bytes alignment --- c/zone.c | 19 ++----------------- c/zone.h | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/c/zone.c b/c/zone.c index 7a275f6..877f4ed 100644 --- a/c/zone.c +++ b/c/zone.c @@ -19,7 +19,6 @@ #include #include - static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_size) { // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ @@ -74,25 +73,11 @@ static inline void clear_chunk_array(msgpack_zone_chunk_array* ca) ca->array[0].ptr = (char*)ca->array[0].alloc; } -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) { msgpack_zone_chunk_array* const ca = &zone->chunk_array; - msgpack_zone_chunk* chunk = ca->tail; - - if(chunk->free > size) { - // chunkã«ç©ºã容é‡ãŒã‚ã‚‹ - // 空ã容é‡ã‚’消費ã—ã¦è¿”ã™ - - char* ptr = chunk->ptr; - - chunk->ptr += size; - chunk->free -= size; - - return ptr; - } - - chunk = ++ca->tail; + msgpack_zone_chunk* chunk = ++ca->tail; if(chunk == ca->end) { // ca->arrayã«ç©ºããŒãªã„ diff --git a/c/zone.h b/c/zone.h index 3c1188d..93a4642 100644 --- a/c/zone.h +++ b/c/zone.h @@ -65,7 +65,7 @@ void msgpack_zone_destroy(msgpack_zone* zone); msgpack_zone* msgpack_zone_new(size_t chunk_size); void msgpack_zone_free(msgpack_zone* zone); -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); +static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); @@ -75,6 +75,30 @@ bool msgpack_zone_is_empty(msgpack_zone* zone); void msgpack_zone_clear(msgpack_zone* zone); + +#ifndef MSGPACK_ZONE_ALIGN +#define MSGPACK_ZONE_ALIGN sizeof(int) +#endif + +void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size); + +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + size = ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1); + + msgpack_zone_chunk* chunk = zone->chunk_array.tail; + + if(chunk->free > size) { + char* ptr = chunk->ptr; + chunk->ptr += size; + chunk->free -= size; + return ptr; + } + + return msgpack_zone_malloc_expand(zone, size); +} + + #ifdef __cplusplus } #endif From e707b7a6000b34a80470638cfd66a6192578a97c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 15:49:24 +0900 Subject: [PATCH 0097/1648] zone::push_finalizer reverts memory allocation on exception --- c/zone.c | 58 ++++++++++++++++++++++-------------------------- c/zone.h | 37 ++++++++++++++++++++++++------ cpp/zone.hpp.erb | 23 ++++++++++++++++--- 3 files changed, 76 insertions(+), 42 deletions(-) diff --git a/c/zone.c b/c/zone.c index 877f4ed..1aaad9f 100644 --- a/c/zone.c +++ b/c/zone.c @@ -34,7 +34,7 @@ static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_s const size_t sz = chunk_size; char* ptr = (char*)malloc(sz); - if(!ptr) { + if(ptr == NULL) { free(array); return NULL; } @@ -88,7 +88,7 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) chunk = (msgpack_zone_chunk*)realloc(ca->array, sizeof(msgpack_zone_chunk) * nnext); - if(!chunk) { + if(chunk == NULL) { return NULL; } @@ -104,7 +104,7 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) } char* ptr = (char*)malloc(sz); - if(!ptr) { + if(ptr == NULL) { return NULL; } @@ -144,44 +144,38 @@ static inline void clear_finalizer_array(msgpack_zone_finalizer_array* fa) fa->tail = fa->array; } -bool msgpack_zone_push_finalizer(msgpack_zone* zone, +bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, void (*func)(void* data), void* data) { msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; - msgpack_zone_finalizer* fin = fa->tail; + const size_t nused = fa->end - fa->array; - if(fin == fa->end) { - // fa->arrayã«ç©ºããŒãªã„ - // fa->arrayã‚’æ‹¡å¼µã™ã‚‹ + size_t nnext; + if(nused == 0) { + // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL - size_t nnext; - const size_t nused = fa->end - fa->array; + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? + 72 / sizeof(msgpack_zone_finalizer) : 8; - if(nused == 0) { - // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL - - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ - nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? - 72 / sizeof(msgpack_zone_finalizer) : 8; - - } else { - nnext = (fa->end - fa->array) * 2; - } - - fin = (msgpack_zone_finalizer*)realloc(fa->array, - sizeof(msgpack_zone_finalizer) * nnext); - if(!fin) { - return false; - } - - fa->array = fin; - fa->end = fin + nnext; - fin = fa->tail = fin + nused; + } else { + nnext = nused * 2; } - fin->func = func; - fin->data = data; + msgpack_zone_finalizer* tmp = + (msgpack_zone_finalizer*)realloc(fa->array, + sizeof(msgpack_zone_finalizer) * nnext); + if(tmp == NULL) { + return false; + } + + fa->array = tmp; + fa->end = tmp + nnext; + fa->tail = tmp + nused; + + fa->tail->func = func; + fa->tail->data = data; ++fa->tail; diff --git a/c/zone.h b/c/zone.h index 93a4642..a3dfe41 100644 --- a/c/zone.h +++ b/c/zone.h @@ -67,7 +67,7 @@ void msgpack_zone_free(msgpack_zone* zone); static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); -bool msgpack_zone_push_finalizer(msgpack_zone* zone, +static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); bool msgpack_zone_is_empty(msgpack_zone* zone); @@ -88,14 +88,37 @@ void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) msgpack_zone_chunk* chunk = zone->chunk_array.tail; - if(chunk->free > size) { - char* ptr = chunk->ptr; - chunk->ptr += size; - chunk->free -= size; - return ptr; + if(chunk->free < size) { + return msgpack_zone_malloc_expand(zone, size); } - return msgpack_zone_malloc_expand(zone, size); + char* ptr = chunk->ptr; + chunk->ptr += size; + chunk->free -= size; + + return ptr; +} + + +bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, + void (*func)(void* data), void* data); + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data) +{ + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + msgpack_zone_finalizer* fin = fa->tail; + + if(fin == fa->end) { + return msgpack_zone_push_finalizer_expand(zone, func, data); + } + + fin->func = func; + fin->data = data; + + ++fa->tail; + + return true; } diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 930c8e8..874c900 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -45,6 +45,8 @@ public: <%}%> private: + void undo_malloc(size_t s); + template static void object_destructor(void* obj); @@ -91,14 +93,29 @@ void zone::object_destructor(void* obj) reinterpret_cast(obj)->~T(); } +inline void zone::undo_malloc(size_t s) +{ + msgpack_zone_chunk* chunk = chunk_array.tail; + chunk->ptr -= s; + chunk->free += s; +} + <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) { void* x = malloc(sizeof(T)); - push_finalizer(&zone::object_destructor, x); - try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } - catch (...) { --finalizer_array.tail; throw; } + if(!msgpack_zone_push_finalizer(this, &zone::object_destructor, x)) { + undo_malloc(sizeof(T)); + throw std::bad_alloc(); + } + try { + return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); + } catch (...) { + --finalizer_array.tail; + undo_malloc(sizeof(T)); + throw; + } } <%}%> From 21040d9cd3b03ee27248b39a6b657948fdac5f9d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 15:49:24 +0900 Subject: [PATCH 0098/1648] zone::push_finalizer reverts memory allocation on exception --- c/zone.c | 58 ++++++++++++++++++++++-------------------------- c/zone.h | 37 ++++++++++++++++++++++++------ cpp/zone.hpp.erb | 23 ++++++++++++++++--- 3 files changed, 76 insertions(+), 42 deletions(-) diff --git a/c/zone.c b/c/zone.c index 877f4ed..1aaad9f 100644 --- a/c/zone.c +++ b/c/zone.c @@ -34,7 +34,7 @@ static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_s const size_t sz = chunk_size; char* ptr = (char*)malloc(sz); - if(!ptr) { + if(ptr == NULL) { free(array); return NULL; } @@ -88,7 +88,7 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) chunk = (msgpack_zone_chunk*)realloc(ca->array, sizeof(msgpack_zone_chunk) * nnext); - if(!chunk) { + if(chunk == NULL) { return NULL; } @@ -104,7 +104,7 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) } char* ptr = (char*)malloc(sz); - if(!ptr) { + if(ptr == NULL) { return NULL; } @@ -144,44 +144,38 @@ static inline void clear_finalizer_array(msgpack_zone_finalizer_array* fa) fa->tail = fa->array; } -bool msgpack_zone_push_finalizer(msgpack_zone* zone, +bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, void (*func)(void* data), void* data) { msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; - msgpack_zone_finalizer* fin = fa->tail; + const size_t nused = fa->end - fa->array; - if(fin == fa->end) { - // fa->arrayã«ç©ºããŒãªã„ - // fa->arrayã‚’æ‹¡å¼µã™ã‚‹ + size_t nnext; + if(nused == 0) { + // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL - size_t nnext; - const size_t nused = fa->end - fa->array; + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? + 72 / sizeof(msgpack_zone_finalizer) : 8; - if(nused == 0) { - // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL - - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ - nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? - 72 / sizeof(msgpack_zone_finalizer) : 8; - - } else { - nnext = (fa->end - fa->array) * 2; - } - - fin = (msgpack_zone_finalizer*)realloc(fa->array, - sizeof(msgpack_zone_finalizer) * nnext); - if(!fin) { - return false; - } - - fa->array = fin; - fa->end = fin + nnext; - fin = fa->tail = fin + nused; + } else { + nnext = nused * 2; } - fin->func = func; - fin->data = data; + msgpack_zone_finalizer* tmp = + (msgpack_zone_finalizer*)realloc(fa->array, + sizeof(msgpack_zone_finalizer) * nnext); + if(tmp == NULL) { + return false; + } + + fa->array = tmp; + fa->end = tmp + nnext; + fa->tail = tmp + nused; + + fa->tail->func = func; + fa->tail->data = data; ++fa->tail; diff --git a/c/zone.h b/c/zone.h index 93a4642..a3dfe41 100644 --- a/c/zone.h +++ b/c/zone.h @@ -67,7 +67,7 @@ void msgpack_zone_free(msgpack_zone* zone); static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); -bool msgpack_zone_push_finalizer(msgpack_zone* zone, +static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); bool msgpack_zone_is_empty(msgpack_zone* zone); @@ -88,14 +88,37 @@ void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) msgpack_zone_chunk* chunk = zone->chunk_array.tail; - if(chunk->free > size) { - char* ptr = chunk->ptr; - chunk->ptr += size; - chunk->free -= size; - return ptr; + if(chunk->free < size) { + return msgpack_zone_malloc_expand(zone, size); } - return msgpack_zone_malloc_expand(zone, size); + char* ptr = chunk->ptr; + chunk->ptr += size; + chunk->free -= size; + + return ptr; +} + + +bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, + void (*func)(void* data), void* data); + +bool msgpack_zone_push_finalizer(msgpack_zone* zone, + void (*func)(void* data), void* data) +{ + msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; + msgpack_zone_finalizer* fin = fa->tail; + + if(fin == fa->end) { + return msgpack_zone_push_finalizer_expand(zone, func, data); + } + + fin->func = func; + fin->data = data; + + ++fa->tail; + + return true; } diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 930c8e8..874c900 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -45,6 +45,8 @@ public: <%}%> private: + void undo_malloc(size_t s); + template static void object_destructor(void* obj); @@ -91,14 +93,29 @@ void zone::object_destructor(void* obj) reinterpret_cast(obj)->~T(); } +inline void zone::undo_malloc(size_t s) +{ + msgpack_zone_chunk* chunk = chunk_array.tail; + chunk->ptr -= s; + chunk->free += s; +} + <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) { void* x = malloc(sizeof(T)); - push_finalizer(&zone::object_destructor, x); - try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } - catch (...) { --finalizer_array.tail; throw; } + if(!msgpack_zone_push_finalizer(this, &zone::object_destructor, x)) { + undo_malloc(sizeof(T)); + throw std::bad_alloc(); + } + try { + return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); + } catch (...) { + --finalizer_array.tail; + undo_malloc(sizeof(T)); + throw; + } } <%}%> From 2fad10ca3fb289c4aa20e1045fa89d2178ac1fde Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 16:40:02 +0900 Subject: [PATCH 0099/1648] zone::base --- cpp/zone.hpp.erb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 874c900..9e85080 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -45,11 +45,13 @@ public: <%}%> private: - void undo_malloc(size_t s); + void undo_malloc(size_t size); template static void object_destructor(void* obj); + typedef msgpack_zone base; + private: zone(const zone&); }; @@ -93,11 +95,11 @@ void zone::object_destructor(void* obj) reinterpret_cast(obj)->~T(); } -inline void zone::undo_malloc(size_t s) +inline void zone::undo_malloc(size_t size) { - msgpack_zone_chunk* chunk = chunk_array.tail; - chunk->ptr -= s; - chunk->free += s; + msgpack_zone_chunk* chunk = base::chunk_array.tail; + chunk->ptr -= size; + chunk->free += size; } <%0.upto(GENERATION_LIMIT) {|i|%> @@ -112,7 +114,7 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } catch (...) { - --finalizer_array.tail; + --base::finalizer_array.tail; undo_malloc(sizeof(T)); throw; } From 5fcf1d590ba3eff80b2a1f49db1641d058e1e3e5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 16:40:02 +0900 Subject: [PATCH 0100/1648] zone::base --- cpp/zone.hpp.erb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 874c900..9e85080 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -45,11 +45,13 @@ public: <%}%> private: - void undo_malloc(size_t s); + void undo_malloc(size_t size); template static void object_destructor(void* obj); + typedef msgpack_zone base; + private: zone(const zone&); }; @@ -93,11 +95,11 @@ void zone::object_destructor(void* obj) reinterpret_cast(obj)->~T(); } -inline void zone::undo_malloc(size_t s) +inline void zone::undo_malloc(size_t size) { - msgpack_zone_chunk* chunk = chunk_array.tail; - chunk->ptr -= s; - chunk->free += s; + msgpack_zone_chunk* chunk = base::chunk_array.tail; + chunk->ptr -= size; + chunk->free += size; } <%0.upto(GENERATION_LIMIT) {|i|%> @@ -112,7 +114,7 @@ T* zone::allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>) try { return new (x) T(<%=(1..i).map{|j|"a#{j}"}.join(', ')%>); } catch (...) { - --finalizer_array.tail; + --base::finalizer_array.tail; undo_malloc(sizeof(T)); throw; } From 09978e500a180e321b76328f4bb7e4f0dd12166f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 18:32:50 +0900 Subject: [PATCH 0101/1648] update ruby/msgpack.gemspec --- ruby/msgpack.gemspec | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index db2fcd5..bb5d9d0 100755 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,11 +1,12 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.2.2" + s.version = "0.3.0" s.summary = "MessagePack" s.author = "FURUHASHI Sadayuki" - s.email = "frsyuki _at_ users.sourceforge.jp" - s.homepage = "https://launchpad.net/msgpack/" + s.email = "frsyuki@users.sourceforge.jp" + s.homepage = "http://msgpack.sourceforge.jp/" + s.rubyforge_project = "msgpack" s.require_paths = ["lib", "ext"] s.files = ["lib/**/*", "ext/**/*"].map {|g| Dir.glob(g) }.flatten end From d58d3ed2b0447709af70310a175336a560133573 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 1 Mar 2009 18:32:50 +0900 Subject: [PATCH 0102/1648] update ruby/msgpack.gemspec --- ruby/msgpack.gemspec | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index db2fcd5..bb5d9d0 100755 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,11 +1,12 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.2.2" + s.version = "0.3.0" s.summary = "MessagePack" s.author = "FURUHASHI Sadayuki" - s.email = "frsyuki _at_ users.sourceforge.jp" - s.homepage = "https://launchpad.net/msgpack/" + s.email = "frsyuki@users.sourceforge.jp" + s.homepage = "http://msgpack.sourceforge.jp/" + s.rubyforge_project = "msgpack" s.require_paths = ["lib", "ext"] s.files = ["lib/**/*", "ext/**/*"].map {|g| Dir.glob(g) }.flatten end From c7f193e4417cb253021bb902e4723d6bcf374c54 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:04:43 +0900 Subject: [PATCH 0103/1648] fix serialization bug on BigEndian environment --- msgpack/pack_template.h | 179 ++++++++++++++++++++++++++-------------- 1 file changed, 117 insertions(+), 62 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 212c0e6..14e3292 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -24,29 +24,79 @@ #endif #endif + #ifdef __LITTLE_ENDIAN__ -#define STORE_BE16(d) \ - ((char*)&d)[1], ((char*)&d)[0] +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] -#define STORE_BE32(d) \ - ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] -#define STORE_BE64(d) \ - ((char*)&d)[7], ((char*)&d)[6], ((char*)&d)[5], ((char*)&d)[4], \ - ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + #elif __BIG_ENDIAN__ -#define STORE_BE16(d) \ - ((char*)&d)[0], ((char*)&d)[1] +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] -#define STORE_BE32(d) \ - ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3] -#define STORE_BE64(d) \ - ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3], \ - ((char*)&d)[4], ((char*)&d)[5], ((char*)&d)[6], ((char*)&d)[7] +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[1] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[3] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[7] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] #endif @@ -71,10 +121,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -83,14 +133,14 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -100,20 +150,20 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -124,24 +174,24 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -151,11 +201,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, d}; \ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ } \ } while(0) @@ -164,24 +214,24 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -192,32 +242,32 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -229,46 +279,46 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -554,7 +604,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE_BE32(f.num)}; + const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; msgpack_pack_append_buffer(x, buf, 5); } @@ -562,7 +612,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE_BE64(f.num)}; + const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -606,11 +656,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE_BE16(d)}; + unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE_BE32(d)}; + unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -627,11 +677,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE_BE16(d)}; + unsigned char buf[3] = {0xde, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE_BE32(d)}; + unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -648,11 +698,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) msgpack_pack_append_buffer(x, &d, 1); } else if(l < 65536) { uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE_BE16(d)}; + unsigned char buf[3] = {0xda, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE_BE32(d)}; + unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -666,9 +716,14 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE_BE16 -#undef STORE_BE32 -#undef STORE_BE64 +#undef STORE16_BE16 + +#undef STORE32_BE16 +#undef STORE32_BE32 + +#undef STORE64_BE16 +#undef STORE64_BE32 +#undef STORE64_BE64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 From 44cdc5f9763d2fd119774bbd026ba635eb04e2e5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:04:43 +0900 Subject: [PATCH 0104/1648] fix serialization bug on BigEndian environment --- msgpack/pack_template.h | 179 ++++++++++++++++++++++++++-------------- 1 file changed, 117 insertions(+), 62 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 212c0e6..14e3292 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -24,29 +24,79 @@ #endif #endif + #ifdef __LITTLE_ENDIAN__ -#define STORE_BE16(d) \ - ((char*)&d)[1], ((char*)&d)[0] +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] -#define STORE_BE32(d) \ - ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] -#define STORE_BE64(d) \ - ((char*)&d)[7], ((char*)&d)[6], ((char*)&d)[5], ((char*)&d)[4], \ - ((char*)&d)[3], ((char*)&d)[2], ((char*)&d)[1], ((char*)&d)[0] +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + #elif __BIG_ENDIAN__ -#define STORE_BE16(d) \ - ((char*)&d)[0], ((char*)&d)[1] +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] -#define STORE_BE32(d) \ - ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3] -#define STORE_BE64(d) \ - ((char*)&d)[0], ((char*)&d)[1], ((char*)&d)[2], ((char*)&d)[3], \ - ((char*)&d)[4], ((char*)&d)[5], ((char*)&d)[6], ((char*)&d)[7] +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[1] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[3] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[7] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] #endif @@ -71,10 +121,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -83,14 +133,14 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -100,20 +150,20 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -124,24 +174,24 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -151,11 +201,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, d}; \ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ } \ } while(0) @@ -164,24 +214,24 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -192,32 +242,32 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -229,46 +279,46 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, (uint8_t)d}; \ + const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, (uint8_t*)&d, 1); \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, (uint8_t)d}; \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -554,7 +604,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE_BE32(f.num)}; + const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; msgpack_pack_append_buffer(x, buf, 5); } @@ -562,7 +612,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE_BE64(f.num)}; + const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -606,11 +656,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE_BE16(d)}; + unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE_BE32(d)}; + unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -627,11 +677,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE_BE16(d)}; + unsigned char buf[3] = {0xde, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE_BE32(d)}; + unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -648,11 +698,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) msgpack_pack_append_buffer(x, &d, 1); } else if(l < 65536) { uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE_BE16(d)}; + unsigned char buf[3] = {0xda, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } else { uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE_BE32(d)}; + unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } } @@ -666,9 +716,14 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE_BE16 -#undef STORE_BE32 -#undef STORE_BE64 +#undef STORE16_BE16 + +#undef STORE32_BE16 +#undef STORE32_BE32 + +#undef STORE64_BE16 +#undef STORE64_BE32 +#undef STORE64_BE64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 From 163fc6f589b9e0de89ae091b3a5cb0b0bffdf71e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:15:22 +0900 Subject: [PATCH 0105/1648] fix serialization bug on BigEndian environment 3 --- msgpack/pack_template.h | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 14e3292..aa620f5 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -330,49 +330,49 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, d}; + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, d}; + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -674,7 +674,7 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &d, 1); + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; unsigned char buf[3] = {0xde, STORE16_BE16(d)}; @@ -695,7 +695,7 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &d, 1); + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); } else if(l < 65536) { uint16_t d = (uint16_t)l; unsigned char buf[3] = {0xda, STORE16_BE16(d)}; @@ -716,11 +716,16 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer +#undef STORE8_BE8 + +#undef STORE16_BE8 #undef STORE16_BE16 +#undef STORE32_BE8 #undef STORE32_BE16 #undef STORE32_BE32 +#undef STORE64_BE8 #undef STORE64_BE16 #undef STORE64_BE32 #undef STORE64_BE64 From c612a177ccf7efe5ad3a7f67269b1399d4e8f96c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:15:22 +0900 Subject: [PATCH 0106/1648] fix serialization bug on BigEndian environment 3 --- msgpack/pack_template.h | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 14e3292..aa620f5 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -330,49 +330,49 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, d}; + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE_BE16(d)}; + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE_BE32(d)}; + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE_BE64(d)}; + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, d}; + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE_BE16(d)}; + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE_BE32(d)}; + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE_BE64(d)}; + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; msgpack_pack_append_buffer(x, buf, 9); } @@ -674,7 +674,7 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &d, 1); + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); } else if(n < 65536) { uint16_t d = (uint16_t)n; unsigned char buf[3] = {0xde, STORE16_BE16(d)}; @@ -695,7 +695,7 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &d, 1); + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); } else if(l < 65536) { uint16_t d = (uint16_t)l; unsigned char buf[3] = {0xda, STORE16_BE16(d)}; @@ -716,11 +716,16 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer +#undef STORE8_BE8 + +#undef STORE16_BE8 #undef STORE16_BE16 +#undef STORE32_BE8 #undef STORE32_BE16 #undef STORE32_BE32 +#undef STORE64_BE8 #undef STORE64_BE16 #undef STORE64_BE32 #undef STORE64_BE64 From cc7379d532b5bd9671e05860207dde0970d6e270 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:36:56 +0900 Subject: [PATCH 0107/1648] more test caces --- ruby/test_case.rb | 78 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/ruby/test_case.rb b/ruby/test_case.rb index 1a06b10..2d897df 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -113,6 +113,80 @@ class MessagePackTestFormat < Test::Unit::TestCase #check_array 5, (1<<32)-1 # memory error end + it "nil" do + match nil, "\xc0" + end + + it "false" do + match false, "\xc2" + end + + it "true" do + match true, "\xc3" + end + + it "0" do + match 0, "\x00" + end + + it "127" do + match 127, "\x7f" + end + + it "128" do + match 128, "\xcc\x80" + end + + it "256" do + match 256, "\xcd\x01\x00" + end + + it "-1" do + match -1, "\xff" + end + + it "-33" do + match -33, "\xd0\xdf" + end + + it "-129" do + match -129, "\xd1\xff\x7f" + end + + it "{1=>1}" do + match ({1=>1}), "\x81\x01\x01" + end + + it "1.0" do + match 1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00" + end + + it "[]" do + match [], "\x90" + end + + it "[0, 1, ..., 14]" do + match (0..14).to_a, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" + end + + it "[0, 1, ..., 15]" do + match (0..15).to_a, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + end + + it "{}" do + match ({}), "\x80" + end + + it "{0=>0, 1=>1, ..., 14=>14}" do + a = (0..14).to_a; + match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a" + end + + it "{0=>0, 1=>1, ..., 15=>15}" do + a = (0..15).to_a; + match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a" + end + # it "fixmap" do # check_map 1, 0 # check_map 1, (1<<4)-1 @@ -143,8 +217,8 @@ class MessagePackTestFormat < Test::Unit::TestCase check num+overhead, Array.new(num) end - def check_map(overhead, num) - # FIXME + def match(obj, buf) + assert_equal(obj.to_msgpack, buf) end end From 840388720ef19e160e970e25329e176328b74fd0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:36:56 +0900 Subject: [PATCH 0108/1648] more test caces --- ruby/test_case.rb | 78 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/ruby/test_case.rb b/ruby/test_case.rb index 1a06b10..2d897df 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -113,6 +113,80 @@ class MessagePackTestFormat < Test::Unit::TestCase #check_array 5, (1<<32)-1 # memory error end + it "nil" do + match nil, "\xc0" + end + + it "false" do + match false, "\xc2" + end + + it "true" do + match true, "\xc3" + end + + it "0" do + match 0, "\x00" + end + + it "127" do + match 127, "\x7f" + end + + it "128" do + match 128, "\xcc\x80" + end + + it "256" do + match 256, "\xcd\x01\x00" + end + + it "-1" do + match -1, "\xff" + end + + it "-33" do + match -33, "\xd0\xdf" + end + + it "-129" do + match -129, "\xd1\xff\x7f" + end + + it "{1=>1}" do + match ({1=>1}), "\x81\x01\x01" + end + + it "1.0" do + match 1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00" + end + + it "[]" do + match [], "\x90" + end + + it "[0, 1, ..., 14]" do + match (0..14).to_a, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" + end + + it "[0, 1, ..., 15]" do + match (0..15).to_a, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + end + + it "{}" do + match ({}), "\x80" + end + + it "{0=>0, 1=>1, ..., 14=>14}" do + a = (0..14).to_a; + match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a" + end + + it "{0=>0, 1=>1, ..., 15=>15}" do + a = (0..15).to_a; + match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a" + end + # it "fixmap" do # check_map 1, 0 # check_map 1, (1<<4)-1 @@ -143,8 +217,8 @@ class MessagePackTestFormat < Test::Unit::TestCase check num+overhead, Array.new(num) end - def check_map(overhead, num) - # FIXME + def match(obj, buf) + assert_equal(obj.to_msgpack, buf) end end From b8cc8b72bd896066d02aa869405161375b528b33 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:54:07 +0900 Subject: [PATCH 0109/1648] version 0.3.1 --- configure.in | 2 +- ruby/gem/Rakefile | 2 +- ruby/msgpack.gemspec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 2c1bee7..3569244 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.0) +AM_INIT_AUTOMAKE(msgpack, 0.3.1) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile index 87635cb..5445906 100644 --- a/ruby/gem/Rakefile +++ b/ruby/gem/Rakefile @@ -17,7 +17,7 @@ DESCRIPTION = "Binary-based efficient data interchange format." RUBYFORGE_PROJECT = "msgpack" HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" BIN_FILES = %w( ) -VERS = "0.3.0" +VERS = "0.3.1" #REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil REV = nil diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index bb5d9d0..f0b5c44 100755 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.3.0" + s.version = "0.3.1" s.summary = "MessagePack" s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From 5bd53f018f5317b96e635bbfec6a42b7950d1ef0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 4 Mar 2009 01:54:07 +0900 Subject: [PATCH 0110/1648] version 0.3.1 --- configure.in | 2 +- ruby/gem/Rakefile | 2 +- ruby/msgpack.gemspec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 2c1bee7..3569244 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.0) +AM_INIT_AUTOMAKE(msgpack, 0.3.1) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile index 87635cb..5445906 100644 --- a/ruby/gem/Rakefile +++ b/ruby/gem/Rakefile @@ -17,7 +17,7 @@ DESCRIPTION = "Binary-based efficient data interchange format." RUBYFORGE_PROJECT = "msgpack" HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" BIN_FILES = %w( ) -VERS = "0.3.0" +VERS = "0.3.1" #REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil REV = nil diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index bb5d9d0..f0b5c44 100755 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.3.0" + s.version = "0.3.1" s.summary = "MessagePack" s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From 8c62d93bcad27621a02fcc9923cb8e6ce46f7c9a Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 12:55:41 +0900 Subject: [PATCH 0111/1648] added perl support --- perl/Makefile.PL | 34 + perl/MessagePack.c | 39 + perl/benchmark/p1.pl | 16 + perl/lib/Data/MessagePack.pm | 33 + perl/lib/Data/MessagePack/Unpacker.pod | 52 + perl/pack.c | 122 + perl/ppport.h | 6984 ++++++++++++++++++++++++ perl/t/00_compile.t | 6 + perl/t/01_pack.t | 61 + perl/t/02_unpack.t | 24 + perl/t/03_stream_unpack.t | 32 + perl/t/04_invert.t | 24 + perl/t/Util.pm | 20 + perl/t/data.pl | 15 + perl/unpack.c | 268 + perl/xt/99_pod.t | 4 + 16 files changed, 7734 insertions(+) create mode 100644 perl/Makefile.PL create mode 100644 perl/MessagePack.c create mode 100644 perl/benchmark/p1.pl create mode 100644 perl/lib/Data/MessagePack.pm create mode 100644 perl/lib/Data/MessagePack/Unpacker.pod create mode 100644 perl/pack.c create mode 100644 perl/ppport.h create mode 100644 perl/t/00_compile.t create mode 100644 perl/t/01_pack.t create mode 100644 perl/t/02_unpack.t create mode 100644 perl/t/03_stream_unpack.t create mode 100644 perl/t/04_invert.t create mode 100644 perl/t/Util.pm create mode 100644 perl/t/data.pl create mode 100644 perl/unpack.c create mode 100644 perl/xt/99_pod.t diff --git a/perl/Makefile.PL b/perl/Makefile.PL new file mode 100644 index 0000000..f60e125 --- /dev/null +++ b/perl/Makefile.PL @@ -0,0 +1,34 @@ +use inc::Module::Install; +name 'Data-MessagePack'; +all_from 'lib/Data/MessagePack.pm'; + +perl_version '5.008005'; +license 'perl'; +can_cc or die "This module requires a C compiler"; + +my $ccflags = '-I../ '; + +makemaker_args( + OBJECT => '$(O_FILES)', + LIBS => [''], + CCFLAGS => $ccflags, + clean => { + FILES => q{ + *.stackdump + *.gcov *.gcda *.gcno + *.out + nytprof + cover_db + }, + }, +); + +tests 't/*.t'; +author_tests('xt'); + +auto_set_repository; +build_requires 'Test::More'; +use_test_base; +auto_include; +WriteAll; + diff --git a/perl/MessagePack.c b/perl/MessagePack.c new file mode 100644 index 0000000..c40e46a --- /dev/null +++ b/perl/MessagePack.c @@ -0,0 +1,39 @@ +#ifdef __cplusplus +extern "C" { +#endif +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" +#ifdef __cplusplus +}; +#endif + +XS(xs_pack); +XS(xs_unpack); +XS(xs_unpacker_new); +XS(xs_unpacker_execute); +XS(xs_unpacker_execute_limit); +XS(xs_unpacker_is_finished); +XS(xs_unpacker_data); +XS(xs_unpacker_reset); + +XS(boot_Data__MessagePack) { + dXSARGS; + HV * stash; + + newXS("Data::MessagePack::pack", xs_pack, __FILE__); + newXS("Data::MessagePack::unpack", xs_unpack, __FILE__); + stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE); + newCONSTSUB(stash, "true", &PL_sv_yes); + newCONSTSUB(stash, "false", &PL_sv_no); + + newXS("Data::MessagePack::Unpacker::new", xs_unpacker_new, __FILE__); + newXS("Data::MessagePack::Unpacker::execute", xs_unpacker_execute, __FILE__); + newXS("Data::MessagePack::Unpacker::execute_limit", xs_unpacker_execute_limit, __FILE__); + newXS("Data::MessagePack::Unpacker::is_finished", xs_unpacker_is_finished, __FILE__); + newXS("Data::MessagePack::Unpacker::data", xs_unpacker_data, __FILE__); + newXS("Data::MessagePack::Unpacker::reset", xs_unpacker_reset, __FILE__); + +} + diff --git a/perl/benchmark/p1.pl b/perl/benchmark/p1.pl new file mode 100644 index 0000000..257932e --- /dev/null +++ b/perl/benchmark/p1.pl @@ -0,0 +1,16 @@ +use strict; +use warnings; +use Data::MessagePack; +use JSON::XS; +use Benchmark ':all'; + +my $a = [0..2**24]; + +print "-- serialize\n"; +cmpthese( + -1 => { + json => sub { JSON::XS::encode_json($a) }, + mp => sub { Data::MessagePack->pack($a) }, + } +); + diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm new file mode 100644 index 0000000..d26bf9e --- /dev/null +++ b/perl/lib/Data/MessagePack.pm @@ -0,0 +1,33 @@ +package Data::MessagePack; +use strict; +use warnings; +use XSLoader; + +our $VERSION = 0.01; + +XSLoader::load(__PACKAGE__, $VERSION); + +1; +__END__ + +=head1 NAME + +Data::MessagePack - messagepack + +=head1 SYNOPSIS + + my $packed = Data::MessagePack->pack($dat); + my $unpacked = Data::MessagePack->unpack($dat); + +=head1 DESCRIPTION + +Data::MessagePack is a binary packer for perl. + +=head1 AUTHORS + +Tokuhiro Matsuno + +=head1 SEE ALSO + +L + diff --git a/perl/lib/Data/MessagePack/Unpacker.pod b/perl/lib/Data/MessagePack/Unpacker.pod new file mode 100644 index 0000000..61cbd21 --- /dev/null +++ b/perl/lib/Data/MessagePack/Unpacker.pod @@ -0,0 +1,52 @@ +=head1 NAME + +Data::MessagePack::Unpacker - messagepack streaming deserializer + +=head1 SYNOPSIS + + use Data::Dumper; + my $up = Data::MessagePack::Unpacker->new; + my $ret = $up->execute($v, 0); + if ($ret != length($v)) { + fail "extra bytes"; + } + return Dumper($up->data); + +=head1 DESCRIPTION + +This is an streaming deserializer for messagepack. + +=head1 METHODS + +=over 4 + +=item my $up = Data::MessagePack::Unpacker->new() + +create new stream deserializer + +=item $up->execute() + +=item $up->execute_limit() + +=item $up->is_finished() + +is this deserializer finished? + +=item $up->data() + +returns deserialized object. + +=item $up->reset() + +reset the stream deserializer, without memory zone. + +=back + +=head1 AUTHORS + +Tokuhiro Matsuno + +=head1 SEE ALSO + +L + diff --git a/perl/pack.c b/perl/pack.c new file mode 100644 index 0000000..5bb667b --- /dev/null +++ b/perl/pack.c @@ -0,0 +1,122 @@ +#ifdef __cplusplus +extern "C" { +#endif +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" +#ifdef __cplusplus +}; +#endif + +#include "msgpack/pack_define.h" + +#define msgpack_pack_inline_func(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_user SV* + +#define msgpack_pack_append_buffer(user, buf, len) \ + sv_catpvn(user, (const char*)(buf), len); + +#include "msgpack/pack_template.h" + +#define _PACK_WRAPPER(t) msgpack_pack_##t +#define PACK_WRAPPER(t) _PACK_WRAPPER(t) + +// move to pack.c +static void _msgpack_pack_sv(SV* buf, SV* val) { + if (val==NULL) { + msgpack_pack_nil(buf); + return; + } + + switch (SvTYPE(val)) { + case SVt_NULL: + msgpack_pack_nil(buf); + break; + case SVt_IV: + if (SvIOK_UV(val)) { + msgpack_pack_uint32(buf, SvUV(val)); + } else { + PACK_WRAPPER(IVTYPE)(buf, SvIV(val)); + } + break; + case SVt_PVNV: + { + STRLEN len = 0; + char *pv = SvPV(val, len); + if (len == 1 && *pv == '1') { + msgpack_pack_true(buf); + } else if (len == 0 && *pv==0) { + msgpack_pack_false(buf); + } else { + msgpack_pack_nil(buf); + } + } + break; + case SVt_PV: + { + STRLEN len; + char * cval = SvPV(val, len); + msgpack_pack_raw(buf, len); + msgpack_pack_raw_body(buf, cval, len); + } + break; + case SVt_NV: + PACK_WRAPPER(NVTYPE)(buf, SvNV(val)); + break; + case SVt_PVAV: + { + AV* ary = (AV*)val; + int len = av_len(ary) + 1; + int i; + msgpack_pack_array(buf, len); + for (i=0; i is designed to support operation with Perl +installations back to 5.003, and has been tested up to 5.10.0. + +=head1 OPTIONS + +=head2 --help + +Display a brief usage summary. + +=head2 --version + +Display the version of F. + +=head2 --patch=I + +If this option is given, a single patch file will be created if +any changes are suggested. This requires a working diff program +to be installed on your system. + +=head2 --copy=I + +If this option is given, a copy of each file will be saved with +the given suffix that contains the suggested changes. This does +not require any external programs. Note that this does not +automagially add a dot between the original filename and the +suffix. If you want the dot, you have to include it in the option +argument. + +If neither C<--patch> or C<--copy> are given, the default is to +simply print the diffs for each file. This requires either +C or a C program to be installed. + +=head2 --diff=I + +Manually set the diff program and options to use. The default +is to use C, when installed, and output unified +context diffs. + +=head2 --compat-version=I + +Tell F to check for compatibility with the given +Perl version. The default is to check for compatibility with Perl +version 5.003. You can use this option to reduce the output +of F if you intend to be backward compatible only +down to a certain Perl version. + +=head2 --cplusplus + +Usually, F will detect C++ style comments and +replace them with C style comments for portability reasons. +Using this option instructs F to leave C++ +comments untouched. + +=head2 --quiet + +Be quiet. Don't print anything except fatal errors. + +=head2 --nodiag + +Don't output any diagnostic messages. Only portability +alerts will be printed. + +=head2 --nohints + +Don't output any hints. Hints often contain useful portability +notes. Warnings will still be displayed. + +=head2 --nochanges + +Don't suggest any changes. Only give diagnostic output and hints +unless these are also deactivated. + +=head2 --nofilter + +Don't filter the list of input files. By default, files not looking +like source code (i.e. not *.xs, *.c, *.cc, *.cpp or *.h) are skipped. + +=head2 --strip + +Strip all script and documentation functionality from F. +This reduces the size of F dramatically and may be useful +if you want to include F in smaller modules without +increasing their distribution size too much. + +The stripped F will have a C<--unstrip> option that allows +you to undo the stripping, but only if an appropriate C +module is installed. + +=head2 --list-provided + +Lists the API elements for which compatibility is provided by +F. Also lists if it must be explicitly requested, +if it has dependencies, and if there are hints or warnings for it. + +=head2 --list-unsupported + +Lists the API elements that are known not to be supported by +F and below which version of Perl they probably +won't be available or work. + +=head2 --api-info=I + +Show portability information for API elements matching I. +If I is surrounded by slashes, it is interpreted as a regular +expression. + +=head1 DESCRIPTION + +In order for a Perl extension (XS) module to be as portable as possible +across differing versions of Perl itself, certain steps need to be taken. + +=over 4 + +=item * + +Including this header is the first major one. This alone will give you +access to a large part of the Perl API that hasn't been available in +earlier Perl releases. Use + + perl ppport.h --list-provided + +to see which API elements are provided by ppport.h. + +=item * + +You should avoid using deprecated parts of the API. For example, using +global Perl variables without the C prefix is deprecated. Also, +some API functions used to have a C prefix. Using this form is +also deprecated. You can safely use the supported API, as F +will provide wrappers for older Perl versions. + +=item * + +If you use one of a few functions or variables that were not present in +earlier versions of Perl, and that can't be provided using a macro, you +have to explicitly request support for these functions by adding one or +more C<#define>s in your source code before the inclusion of F. + +These functions or variables will be marked C in the list shown +by C<--list-provided>. + +Depending on whether you module has a single or multiple files that +use such functions or variables, you want either C or global +variants. + +For a C function or variable (used only in a single source +file), use: + + #define NEED_function + #define NEED_variable + +For a global function or variable (used in multiple source files), +use: + + #define NEED_function_GLOBAL + #define NEED_variable_GLOBAL + +Note that you mustn't have more than one global request for the +same function or variable in your project. + + Function / Variable Static Request Global Request + ----------------------------------------------------------------------------------------- + PL_parser NEED_PL_parser NEED_PL_parser_GLOBAL + PL_signals NEED_PL_signals NEED_PL_signals_GLOBAL + eval_pv() NEED_eval_pv NEED_eval_pv_GLOBAL + grok_bin() NEED_grok_bin NEED_grok_bin_GLOBAL + grok_hex() NEED_grok_hex NEED_grok_hex_GLOBAL + grok_number() NEED_grok_number NEED_grok_number_GLOBAL + grok_numeric_radix() NEED_grok_numeric_radix NEED_grok_numeric_radix_GLOBAL + grok_oct() NEED_grok_oct NEED_grok_oct_GLOBAL + load_module() NEED_load_module NEED_load_module_GLOBAL + my_snprintf() NEED_my_snprintf NEED_my_snprintf_GLOBAL + my_sprintf() NEED_my_sprintf NEED_my_sprintf_GLOBAL + my_strlcat() NEED_my_strlcat NEED_my_strlcat_GLOBAL + my_strlcpy() NEED_my_strlcpy NEED_my_strlcpy_GLOBAL + newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL + newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL + newSVpvn_flags() NEED_newSVpvn_flags NEED_newSVpvn_flags_GLOBAL + newSVpvn_share() NEED_newSVpvn_share NEED_newSVpvn_share_GLOBAL + pv_display() NEED_pv_display NEED_pv_display_GLOBAL + pv_escape() NEED_pv_escape NEED_pv_escape_GLOBAL + pv_pretty() NEED_pv_pretty NEED_pv_pretty_GLOBAL + sv_2pv_flags() NEED_sv_2pv_flags NEED_sv_2pv_flags_GLOBAL + sv_2pvbyte() NEED_sv_2pvbyte NEED_sv_2pvbyte_GLOBAL + sv_catpvf_mg() NEED_sv_catpvf_mg NEED_sv_catpvf_mg_GLOBAL + sv_catpvf_mg_nocontext() NEED_sv_catpvf_mg_nocontext NEED_sv_catpvf_mg_nocontext_GLOBAL + sv_pvn_force_flags() NEED_sv_pvn_force_flags NEED_sv_pvn_force_flags_GLOBAL + sv_setpvf_mg() NEED_sv_setpvf_mg NEED_sv_setpvf_mg_GLOBAL + sv_setpvf_mg_nocontext() NEED_sv_setpvf_mg_nocontext NEED_sv_setpvf_mg_nocontext_GLOBAL + vload_module() NEED_vload_module NEED_vload_module_GLOBAL + vnewSVpvf() NEED_vnewSVpvf NEED_vnewSVpvf_GLOBAL + warner() NEED_warner NEED_warner_GLOBAL + +To avoid namespace conflicts, you can change the namespace of the +explicitly exported functions / variables using the C +macro. Just C<#define> the macro before including C: + + #define DPPP_NAMESPACE MyOwnNamespace_ + #include "ppport.h" + +The default namespace is C. + +=back + +The good thing is that most of the above can be checked by running +F on your source code. See the next section for +details. + +=head1 EXAMPLES + +To verify whether F is needed for your module, whether you +should make any changes to your code, and whether any special defines +should be used, F can be run as a Perl script to check your +source code. Simply say: + + perl ppport.h + +The result will usually be a list of patches suggesting changes +that should at least be acceptable, if not necessarily the most +efficient solution, or a fix for all possible problems. + +If you know that your XS module uses features only available in +newer Perl releases, if you're aware that it uses C++ comments, +and if you want all suggestions as a single patch file, you could +use something like this: + + perl ppport.h --compat-version=5.6.0 --cplusplus --patch=test.diff + +If you only want your code to be scanned without any suggestions +for changes, use: + + perl ppport.h --nochanges + +You can specify a different C program or options, using +the C<--diff> option: + + perl ppport.h --diff='diff -C 10' + +This would output context diffs with 10 lines of context. + +If you want to create patched copies of your files instead, use: + + perl ppport.h --copy=.new + +To display portability information for the C function, +use: + + perl ppport.h --api-info=newSVpvn + +Since the argument to C<--api-info> can be a regular expression, +you can use + + perl ppport.h --api-info=/_nomg$/ + +to display portability information for all C<_nomg> functions or + + perl ppport.h --api-info=/./ + +to display information for all known API elements. + +=head1 BUGS + +If this version of F is causing failure during +the compilation of this module, please check if newer versions +of either this module or C are available on CPAN +before sending a bug report. + +If F was generated using the latest version of +C and is causing failure of this module, please +file a bug report using the CPAN Request Tracker at L. + +Please include the following information: + +=over 4 + +=item 1. + +The complete output from running "perl -V" + +=item 2. + +This file. + +=item 3. + +The name and version of the module you were trying to build. + +=item 4. + +A full log of the build that failed. + +=item 5. + +Any other information that you think could be relevant. + +=back + +For the latest version of this code, please get the C +module from CPAN. + +=head1 COPYRIGHT + +Version 3.x, Copyright (c) 2004-2009, Marcus Holland-Moritz. + +Version 2.x, Copyright (C) 2001, Paul Marquess. + +Version 1.x, Copyright (C) 1999, Kenneth Albanowski. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +=head1 SEE ALSO + +See L. + +=cut + +use strict; + +# Disable broken TRIE-optimization +BEGIN { eval '${^RE_TRIE_MAXBUF} = -1' if $] >= 5.009004 && $] <= 5.009005 } + +my $VERSION = 3.17; + +my %opt = ( + quiet => 0, + diag => 1, + hints => 1, + changes => 1, + cplusplus => 0, + filter => 1, + strip => 0, + version => 0, +); + +my($ppport) = $0 =~ /([\w.]+)$/; +my $LF = '(?:\r\n|[\r\n])'; # line feed +my $HS = "[ \t]"; # horizontal whitespace + +# Never use C comments in this file! +my $ccs = '/'.'*'; +my $cce = '*'.'/'; +my $rccs = quotemeta $ccs; +my $rcce = quotemeta $cce; + +eval { + require Getopt::Long; + Getopt::Long::GetOptions(\%opt, qw( + help quiet diag! filter! hints! changes! cplusplus strip version + patch=s copy=s diff=s compat-version=s + list-provided list-unsupported api-info=s + )) or usage(); +}; + +if ($@ and grep /^-/, @ARGV) { + usage() if "@ARGV" =~ /^--?h(?:elp)?$/; + die "Getopt::Long not found. Please don't use any options.\n"; +} + +if ($opt{version}) { + print "This is $0 $VERSION.\n"; + exit 0; +} + +usage() if $opt{help}; +strip() if $opt{strip}; + +if (exists $opt{'compat-version'}) { + my($r,$v,$s) = eval { parse_version($opt{'compat-version'}) }; + if ($@) { + die "Invalid version number format: '$opt{'compat-version'}'\n"; + } + die "Only Perl 5 is supported\n" if $r != 5; + die "Invalid version number: $opt{'compat-version'}\n" if $v >= 1000 || $s >= 1000; + $opt{'compat-version'} = sprintf "%d.%03d%03d", $r, $v, $s; +} +else { + $opt{'compat-version'} = 5; +} + +my %API = map { /^(\w+)\|([^|]*)\|([^|]*)\|(\w*)$/ + ? ( $1 => { + ($2 ? ( base => $2 ) : ()), + ($3 ? ( todo => $3 ) : ()), + (index($4, 'v') >= 0 ? ( varargs => 1 ) : ()), + (index($4, 'p') >= 0 ? ( provided => 1 ) : ()), + (index($4, 'n') >= 0 ? ( nothxarg => 1 ) : ()), + } ) + : die "invalid spec: $_" } qw( +AvFILLp|5.004050||p +AvFILL||| +CLASS|||n +CPERLscope|5.005000||p +CX_CURPAD_SAVE||| +CX_CURPAD_SV||| +CopFILEAV|5.006000||p +CopFILEGV_set|5.006000||p +CopFILEGV|5.006000||p +CopFILESV|5.006000||p +CopFILE_set|5.006000||p +CopFILE|5.006000||p +CopSTASHPV_set|5.006000||p +CopSTASHPV|5.006000||p +CopSTASH_eq|5.006000||p +CopSTASH_set|5.006000||p +CopSTASH|5.006000||p +CopyD|5.009002||p +Copy||| +CvPADLIST||| +CvSTASH||| +CvWEAKOUTSIDE||| +DEFSV_set|5.011000||p +DEFSV|5.004050||p +END_EXTERN_C|5.005000||p +ENTER||| +ERRSV|5.004050||p +EXTEND||| +EXTERN_C|5.005000||p +F0convert|||n +FREETMPS||| +GIMME_V||5.004000|n +GIMME|||n +GROK_NUMERIC_RADIX|5.007002||p +G_ARRAY||| +G_DISCARD||| +G_EVAL||| +G_METHOD|5.006001||p +G_NOARGS||| +G_SCALAR||| +G_VOID||5.004000| +GetVars||| +GvSV||| +Gv_AMupdate||| +HEf_SVKEY||5.004000| +HeHASH||5.004000| +HeKEY||5.004000| +HeKLEN||5.004000| +HePV||5.004000| +HeSVKEY_force||5.004000| +HeSVKEY_set||5.004000| +HeSVKEY||5.004000| +HeUTF8||5.011000| +HeVAL||5.004000| +HvNAME||| +INT2PTR|5.006000||p +IN_LOCALE_COMPILETIME|5.007002||p +IN_LOCALE_RUNTIME|5.007002||p +IN_LOCALE|5.007002||p +IN_PERL_COMPILETIME|5.008001||p +IS_NUMBER_GREATER_THAN_UV_MAX|5.007002||p +IS_NUMBER_INFINITY|5.007002||p +IS_NUMBER_IN_UV|5.007002||p +IS_NUMBER_NAN|5.007003||p +IS_NUMBER_NEG|5.007002||p +IS_NUMBER_NOT_INT|5.007002||p +IVSIZE|5.006000||p +IVTYPE|5.006000||p +IVdf|5.006000||p +LEAVE||| +LVRET||| +MARK||| +MULTICALL||5.011000| +MY_CXT_CLONE|5.009002||p +MY_CXT_INIT|5.007003||p +MY_CXT|5.007003||p +MoveD|5.009002||p +Move||| +NOOP|5.005000||p +NUM2PTR|5.006000||p +NVTYPE|5.006000||p +NVef|5.006001||p +NVff|5.006001||p +NVgf|5.006001||p +Newxc|5.009003||p +Newxz|5.009003||p +Newx|5.009003||p +Nullav||| +Nullch||| +Nullcv||| +Nullhv||| +Nullsv||| +ORIGMARK||| +PAD_BASE_SV||| +PAD_CLONE_VARS||| +PAD_COMPNAME_FLAGS||| +PAD_COMPNAME_GEN_set||| +PAD_COMPNAME_GEN||| +PAD_COMPNAME_OURSTASH||| +PAD_COMPNAME_PV||| +PAD_COMPNAME_TYPE||| +PAD_DUP||| +PAD_RESTORE_LOCAL||| +PAD_SAVE_LOCAL||| +PAD_SAVE_SETNULLPAD||| +PAD_SETSV||| +PAD_SET_CUR_NOSAVE||| +PAD_SET_CUR||| +PAD_SVl||| +PAD_SV||| +PERLIO_FUNCS_CAST|5.009003||p +PERLIO_FUNCS_DECL|5.009003||p +PERL_ABS|5.008001||p +PERL_BCDVERSION|5.011000||p +PERL_GCC_BRACE_GROUPS_FORBIDDEN|5.008001||p +PERL_HASH|5.004000||p +PERL_INT_MAX|5.004000||p +PERL_INT_MIN|5.004000||p +PERL_LONG_MAX|5.004000||p +PERL_LONG_MIN|5.004000||p +PERL_MAGIC_arylen|5.007002||p +PERL_MAGIC_backref|5.007002||p +PERL_MAGIC_bm|5.007002||p +PERL_MAGIC_collxfrm|5.007002||p +PERL_MAGIC_dbfile|5.007002||p +PERL_MAGIC_dbline|5.007002||p +PERL_MAGIC_defelem|5.007002||p +PERL_MAGIC_envelem|5.007002||p +PERL_MAGIC_env|5.007002||p +PERL_MAGIC_ext|5.007002||p +PERL_MAGIC_fm|5.007002||p +PERL_MAGIC_glob|5.011000||p +PERL_MAGIC_isaelem|5.007002||p +PERL_MAGIC_isa|5.007002||p +PERL_MAGIC_mutex|5.011000||p +PERL_MAGIC_nkeys|5.007002||p +PERL_MAGIC_overload_elem|5.007002||p +PERL_MAGIC_overload_table|5.007002||p +PERL_MAGIC_overload|5.007002||p +PERL_MAGIC_pos|5.007002||p +PERL_MAGIC_qr|5.007002||p +PERL_MAGIC_regdata|5.007002||p +PERL_MAGIC_regdatum|5.007002||p +PERL_MAGIC_regex_global|5.007002||p +PERL_MAGIC_shared_scalar|5.007003||p +PERL_MAGIC_shared|5.007003||p +PERL_MAGIC_sigelem|5.007002||p +PERL_MAGIC_sig|5.007002||p +PERL_MAGIC_substr|5.007002||p +PERL_MAGIC_sv|5.007002||p +PERL_MAGIC_taint|5.007002||p +PERL_MAGIC_tiedelem|5.007002||p +PERL_MAGIC_tiedscalar|5.007002||p +PERL_MAGIC_tied|5.007002||p +PERL_MAGIC_utf8|5.008001||p +PERL_MAGIC_uvar_elem|5.007003||p +PERL_MAGIC_uvar|5.007002||p +PERL_MAGIC_vec|5.007002||p +PERL_MAGIC_vstring|5.008001||p +PERL_PV_ESCAPE_ALL|5.009004||p +PERL_PV_ESCAPE_FIRSTCHAR|5.009004||p +PERL_PV_ESCAPE_NOBACKSLASH|5.009004||p +PERL_PV_ESCAPE_NOCLEAR|5.009004||p +PERL_PV_ESCAPE_QUOTE|5.009004||p +PERL_PV_ESCAPE_RE|5.009005||p +PERL_PV_ESCAPE_UNI_DETECT|5.009004||p +PERL_PV_ESCAPE_UNI|5.009004||p +PERL_PV_PRETTY_DUMP|5.009004||p +PERL_PV_PRETTY_ELLIPSES|5.010000||p +PERL_PV_PRETTY_LTGT|5.009004||p +PERL_PV_PRETTY_NOCLEAR|5.010000||p +PERL_PV_PRETTY_QUOTE|5.009004||p +PERL_PV_PRETTY_REGPROP|5.009004||p +PERL_QUAD_MAX|5.004000||p +PERL_QUAD_MIN|5.004000||p +PERL_REVISION|5.006000||p +PERL_SCAN_ALLOW_UNDERSCORES|5.007003||p +PERL_SCAN_DISALLOW_PREFIX|5.007003||p +PERL_SCAN_GREATER_THAN_UV_MAX|5.007003||p +PERL_SCAN_SILENT_ILLDIGIT|5.008001||p +PERL_SHORT_MAX|5.004000||p +PERL_SHORT_MIN|5.004000||p +PERL_SIGNALS_UNSAFE_FLAG|5.008001||p +PERL_SUBVERSION|5.006000||p +PERL_UCHAR_MAX|5.004000||p +PERL_UCHAR_MIN|5.004000||p +PERL_UINT_MAX|5.004000||p +PERL_UINT_MIN|5.004000||p +PERL_ULONG_MAX|5.004000||p +PERL_ULONG_MIN|5.004000||p +PERL_UNUSED_ARG|5.009003||p +PERL_UNUSED_CONTEXT|5.009004||p +PERL_UNUSED_DECL|5.007002||p +PERL_UNUSED_VAR|5.007002||p +PERL_UQUAD_MAX|5.004000||p +PERL_UQUAD_MIN|5.004000||p +PERL_USE_GCC_BRACE_GROUPS|5.009004||p +PERL_USHORT_MAX|5.004000||p +PERL_USHORT_MIN|5.004000||p +PERL_VERSION|5.006000||p +PL_DBsignal|5.005000||p +PL_DBsingle|||pn +PL_DBsub|||pn +PL_DBtrace|||pn +PL_Sv|5.005000||p +PL_bufend|5.011000||p +PL_bufptr|5.011000||p +PL_compiling|5.004050||p +PL_copline|5.011000||p +PL_curcop|5.004050||p +PL_curstash|5.004050||p +PL_debstash|5.004050||p +PL_defgv|5.004050||p +PL_diehook|5.004050||p +PL_dirty|5.004050||p +PL_dowarn|||pn +PL_errgv|5.004050||p +PL_expect|5.011000||p +PL_hexdigit|5.005000||p +PL_hints|5.005000||p +PL_last_in_gv|||n +PL_laststatval|5.005000||p +PL_lex_state|5.011000||p +PL_lex_stuff|5.011000||p +PL_linestr|5.011000||p +PL_modglobal||5.005000|n +PL_na|5.004050||pn +PL_no_modify|5.006000||p +PL_ofsgv|||n +PL_parser|5.009005||p +PL_perl_destruct_level|5.004050||p +PL_perldb|5.004050||p +PL_ppaddr|5.006000||p +PL_rsfp_filters|5.004050||p +PL_rsfp|5.004050||p +PL_rs|||n +PL_signals|5.008001||p +PL_stack_base|5.004050||p +PL_stack_sp|5.004050||p +PL_statcache|5.005000||p +PL_stdingv|5.004050||p +PL_sv_arenaroot|5.004050||p +PL_sv_no|5.004050||pn +PL_sv_undef|5.004050||pn +PL_sv_yes|5.004050||pn +PL_tainted|5.004050||p +PL_tainting|5.004050||p +PL_tokenbuf|5.011000||p +POP_MULTICALL||5.011000| +POPi|||n +POPl|||n +POPn|||n +POPpbytex||5.007001|n +POPpx||5.005030|n +POPp|||n +POPs|||n +PTR2IV|5.006000||p +PTR2NV|5.006000||p +PTR2UV|5.006000||p +PTR2nat|5.009003||p +PTR2ul|5.007001||p +PTRV|5.006000||p +PUSHMARK||| +PUSH_MULTICALL||5.011000| +PUSHi||| +PUSHmortal|5.009002||p +PUSHn||| +PUSHp||| +PUSHs||| +PUSHu|5.004000||p +PUTBACK||| +PerlIO_clearerr||5.007003| +PerlIO_close||5.007003| +PerlIO_context_layers||5.009004| +PerlIO_eof||5.007003| +PerlIO_error||5.007003| +PerlIO_fileno||5.007003| +PerlIO_fill||5.007003| +PerlIO_flush||5.007003| +PerlIO_get_base||5.007003| +PerlIO_get_bufsiz||5.007003| +PerlIO_get_cnt||5.007003| +PerlIO_get_ptr||5.007003| +PerlIO_read||5.007003| +PerlIO_seek||5.007003| +PerlIO_set_cnt||5.007003| +PerlIO_set_ptrcnt||5.007003| +PerlIO_setlinebuf||5.007003| +PerlIO_stderr||5.007003| +PerlIO_stdin||5.007003| +PerlIO_stdout||5.007003| +PerlIO_tell||5.007003| +PerlIO_unread||5.007003| +PerlIO_write||5.007003| +Perl_signbit||5.009005|n +PoisonFree|5.009004||p +PoisonNew|5.009004||p +PoisonWith|5.009004||p +Poison|5.008000||p +RETVAL|||n +Renewc||| +Renew||| +SAVECLEARSV||| +SAVECOMPPAD||| +SAVEPADSV||| +SAVETMPS||| +SAVE_DEFSV|5.004050||p +SPAGAIN||| +SP||| +START_EXTERN_C|5.005000||p +START_MY_CXT|5.007003||p +STMT_END|||p +STMT_START|||p +STR_WITH_LEN|5.009003||p +ST||| +SV_CONST_RETURN|5.009003||p +SV_COW_DROP_PV|5.008001||p +SV_COW_SHARED_HASH_KEYS|5.009005||p +SV_GMAGIC|5.007002||p +SV_HAS_TRAILING_NUL|5.009004||p +SV_IMMEDIATE_UNREF|5.007001||p +SV_MUTABLE_RETURN|5.009003||p +SV_NOSTEAL|5.009002||p +SV_SMAGIC|5.009003||p +SV_UTF8_NO_ENCODING|5.008001||p +SVf_UTF8|5.006000||p +SVf|5.006000||p +SVt_IV||| +SVt_NV||| +SVt_PVAV||| +SVt_PVCV||| +SVt_PVHV||| +SVt_PVMG||| +SVt_PV||| +Safefree||| +Slab_Alloc||| +Slab_Free||| +Slab_to_rw||| +StructCopy||| +SvCUR_set||| +SvCUR||| +SvEND||| +SvGAMAGIC||5.006001| +SvGETMAGIC|5.004050||p +SvGROW||| +SvIOK_UV||5.006000| +SvIOK_notUV||5.006000| +SvIOK_off||| +SvIOK_only_UV||5.006000| +SvIOK_only||| +SvIOK_on||| +SvIOKp||| +SvIOK||| +SvIVX||| +SvIV_nomg|5.009001||p +SvIV_set||| +SvIVx||| +SvIV||| +SvIsCOW_shared_hash||5.008003| +SvIsCOW||5.008003| +SvLEN_set||| +SvLEN||| +SvLOCK||5.007003| +SvMAGIC_set|5.009003||p +SvNIOK_off||| +SvNIOKp||| +SvNIOK||| +SvNOK_off||| +SvNOK_only||| +SvNOK_on||| +SvNOKp||| +SvNOK||| +SvNVX||| +SvNV_set||| +SvNVx||| +SvNV||| +SvOK||| +SvOOK_offset||5.011000| +SvOOK||| +SvPOK_off||| +SvPOK_only_UTF8||5.006000| +SvPOK_only||| +SvPOK_on||| +SvPOKp||| +SvPOK||| +SvPVX_const|5.009003||p +SvPVX_mutable|5.009003||p +SvPVX||| +SvPV_const|5.009003||p +SvPV_flags_const_nolen|5.009003||p +SvPV_flags_const|5.009003||p +SvPV_flags_mutable|5.009003||p +SvPV_flags|5.007002||p +SvPV_force_flags_mutable|5.009003||p +SvPV_force_flags_nolen|5.009003||p +SvPV_force_flags|5.007002||p +SvPV_force_mutable|5.009003||p +SvPV_force_nolen|5.009003||p +SvPV_force_nomg_nolen|5.009003||p +SvPV_force_nomg|5.007002||p +SvPV_force|||p +SvPV_mutable|5.009003||p +SvPV_nolen_const|5.009003||p +SvPV_nolen|5.006000||p +SvPV_nomg_const_nolen|5.009003||p +SvPV_nomg_const|5.009003||p +SvPV_nomg|5.007002||p +SvPV_renew|5.009003||p +SvPV_set||| +SvPVbyte_force||5.009002| +SvPVbyte_nolen||5.006000| +SvPVbytex_force||5.006000| +SvPVbytex||5.006000| +SvPVbyte|5.006000||p +SvPVutf8_force||5.006000| +SvPVutf8_nolen||5.006000| +SvPVutf8x_force||5.006000| +SvPVutf8x||5.006000| +SvPVutf8||5.006000| +SvPVx||| +SvPV||| +SvREFCNT_dec||| +SvREFCNT_inc_NN|5.009004||p +SvREFCNT_inc_simple_NN|5.009004||p +SvREFCNT_inc_simple_void_NN|5.009004||p +SvREFCNT_inc_simple_void|5.009004||p +SvREFCNT_inc_simple|5.009004||p +SvREFCNT_inc_void_NN|5.009004||p +SvREFCNT_inc_void|5.009004||p +SvREFCNT_inc|||p +SvREFCNT||| +SvROK_off||| +SvROK_on||| +SvROK||| +SvRV_set|5.009003||p +SvRV||| +SvRXOK||5.009005| +SvRX||5.009005| +SvSETMAGIC||| +SvSHARED_HASH|5.009003||p +SvSHARE||5.007003| +SvSTASH_set|5.009003||p +SvSTASH||| +SvSetMagicSV_nosteal||5.004000| +SvSetMagicSV||5.004000| +SvSetSV_nosteal||5.004000| +SvSetSV||| +SvTAINTED_off||5.004000| +SvTAINTED_on||5.004000| +SvTAINTED||5.004000| +SvTAINT||| +SvTRUE||| +SvTYPE||| +SvUNLOCK||5.007003| +SvUOK|5.007001|5.006000|p +SvUPGRADE||| +SvUTF8_off||5.006000| +SvUTF8_on||5.006000| +SvUTF8||5.006000| +SvUVXx|5.004000||p +SvUVX|5.004000||p +SvUV_nomg|5.009001||p +SvUV_set|5.009003||p +SvUVx|5.004000||p +SvUV|5.004000||p +SvVOK||5.008001| +SvVSTRING_mg|5.009004||p +THIS|||n +UNDERBAR|5.009002||p +UTF8_MAXBYTES|5.009002||p +UVSIZE|5.006000||p +UVTYPE|5.006000||p +UVXf|5.007001||p +UVof|5.006000||p +UVuf|5.006000||p +UVxf|5.006000||p +WARN_ALL|5.006000||p +WARN_AMBIGUOUS|5.006000||p +WARN_ASSERTIONS|5.011000||p +WARN_BAREWORD|5.006000||p +WARN_CLOSED|5.006000||p +WARN_CLOSURE|5.006000||p +WARN_DEBUGGING|5.006000||p +WARN_DEPRECATED|5.006000||p +WARN_DIGIT|5.006000||p +WARN_EXEC|5.006000||p +WARN_EXITING|5.006000||p +WARN_GLOB|5.006000||p +WARN_INPLACE|5.006000||p +WARN_INTERNAL|5.006000||p +WARN_IO|5.006000||p +WARN_LAYER|5.008000||p +WARN_MALLOC|5.006000||p +WARN_MISC|5.006000||p +WARN_NEWLINE|5.006000||p +WARN_NUMERIC|5.006000||p +WARN_ONCE|5.006000||p +WARN_OVERFLOW|5.006000||p +WARN_PACK|5.006000||p +WARN_PARENTHESIS|5.006000||p +WARN_PIPE|5.006000||p +WARN_PORTABLE|5.006000||p +WARN_PRECEDENCE|5.006000||p +WARN_PRINTF|5.006000||p +WARN_PROTOTYPE|5.006000||p +WARN_QW|5.006000||p +WARN_RECURSION|5.006000||p +WARN_REDEFINE|5.006000||p +WARN_REGEXP|5.006000||p +WARN_RESERVED|5.006000||p +WARN_SEMICOLON|5.006000||p +WARN_SEVERE|5.006000||p +WARN_SIGNAL|5.006000||p +WARN_SUBSTR|5.006000||p +WARN_SYNTAX|5.006000||p +WARN_TAINT|5.006000||p +WARN_THREADS|5.008000||p +WARN_UNINITIALIZED|5.006000||p +WARN_UNOPENED|5.006000||p +WARN_UNPACK|5.006000||p +WARN_UNTIE|5.006000||p +WARN_UTF8|5.006000||p +WARN_VOID|5.006000||p +XCPT_CATCH|5.009002||p +XCPT_RETHROW|5.009002||p +XCPT_TRY_END|5.009002||p +XCPT_TRY_START|5.009002||p +XPUSHi||| +XPUSHmortal|5.009002||p +XPUSHn||| +XPUSHp||| +XPUSHs||| +XPUSHu|5.004000||p +XSRETURN_EMPTY||| +XSRETURN_IV||| +XSRETURN_NO||| +XSRETURN_NV||| +XSRETURN_PV||| +XSRETURN_UNDEF||| +XSRETURN_UV|5.008001||p +XSRETURN_YES||| +XSRETURN|||p +XST_mIV||| +XST_mNO||| +XST_mNV||| +XST_mPV||| +XST_mUNDEF||| +XST_mUV|5.008001||p +XST_mYES||| +XS_VERSION_BOOTCHECK||| +XS_VERSION||| +XSprePUSH|5.006000||p +XS||| +ZeroD|5.009002||p +Zero||| +_aMY_CXT|5.007003||p +_pMY_CXT|5.007003||p +aMY_CXT_|5.007003||p +aMY_CXT|5.007003||p +aTHXR_|5.011000||p +aTHXR|5.011000||p +aTHX_|5.006000||p +aTHX|5.006000||p +add_data|||n +addmad||| +allocmy||| +amagic_call||| +amagic_cmp_locale||| +amagic_cmp||| +amagic_i_ncmp||| +amagic_ncmp||| +any_dup||| +ao||| +append_elem||| +append_list||| +append_madprops||| +apply_attrs_my||| +apply_attrs_string||5.006001| +apply_attrs||| +apply||| +atfork_lock||5.007003|n +atfork_unlock||5.007003|n +av_arylen_p||5.009003| +av_clear||| +av_create_and_push||5.009005| +av_create_and_unshift_one||5.009005| +av_delete||5.006000| +av_exists||5.006000| +av_extend||| +av_fetch||| +av_fill||| +av_iter_p||5.011000| +av_len||| +av_make||| +av_pop||| +av_push||| +av_reify||| +av_shift||| +av_store||| +av_undef||| +av_unshift||| +ax|||n +bad_type||| +bind_match||| +block_end||| +block_gimme||5.004000| +block_start||| +boolSV|5.004000||p +boot_core_PerlIO||| +boot_core_UNIVERSAL||| +boot_core_mro||| +boot_core_xsutils||| +bytes_from_utf8||5.007001| +bytes_to_uni|||n +bytes_to_utf8||5.006001| +call_argv|5.006000||p +call_atexit||5.006000| +call_list||5.004000| +call_method|5.006000||p +call_pv|5.006000||p +call_sv|5.006000||p +calloc||5.007002|n +cando||| +cast_i32||5.006000| +cast_iv||5.006000| +cast_ulong||5.006000| +cast_uv||5.006000| +check_type_and_open||| +check_uni||| +checkcomma||| +checkposixcc||| +ckWARN|5.006000||p +ck_anoncode||| +ck_bitop||| +ck_concat||| +ck_defined||| +ck_delete||| +ck_die||| +ck_each||| +ck_eof||| +ck_eval||| +ck_exec||| +ck_exists||| +ck_exit||| +ck_ftst||| +ck_fun||| +ck_glob||| +ck_grep||| +ck_index||| +ck_join||| +ck_lfun||| +ck_listiob||| +ck_match||| +ck_method||| +ck_null||| +ck_open||| +ck_readline||| +ck_repeat||| +ck_require||| +ck_return||| +ck_rfun||| +ck_rvconst||| +ck_sassign||| +ck_select||| +ck_shift||| +ck_sort||| +ck_spair||| +ck_split||| +ck_subr||| +ck_substr||| +ck_svconst||| +ck_trunc||| +ck_unpack||| +ckwarn_d||5.009003| +ckwarn||5.009003| +cl_and|||n +cl_anything|||n +cl_init_zero|||n +cl_init|||n +cl_is_anything|||n +cl_or|||n +clear_placeholders||| +closest_cop||| +convert||| +cop_free||| +cr_textfilter||| +create_eval_scope||| +croak_nocontext|||vn +croak_xs_usage||5.011000| +croak|||v +csighandler||5.009003|n +curmad||| +custom_op_desc||5.007003| +custom_op_name||5.007003| +cv_ckproto_len||| +cv_clone||| +cv_const_sv||5.004000| +cv_dump||| +cv_undef||| +cx_dump||5.005000| +cx_dup||| +cxinc||| +dAXMARK|5.009003||p +dAX|5.007002||p +dITEMS|5.007002||p +dMARK||| +dMULTICALL||5.009003| +dMY_CXT_SV|5.007003||p +dMY_CXT|5.007003||p +dNOOP|5.006000||p +dORIGMARK||| +dSP||| +dTHR|5.004050||p +dTHXR|5.011000||p +dTHXa|5.006000||p +dTHXoa|5.006000||p +dTHX|5.006000||p +dUNDERBAR|5.009002||p +dVAR|5.009003||p +dXCPT|5.009002||p +dXSARGS||| +dXSI32||| +dXSTARG|5.006000||p +deb_curcv||| +deb_nocontext|||vn +deb_stack_all||| +deb_stack_n||| +debop||5.005000| +debprofdump||5.005000| +debprof||| +debstackptrs||5.007003| +debstack||5.007003| +debug_start_match||| +deb||5.007003|v +del_sv||| +delete_eval_scope||| +delimcpy||5.004000| +deprecate_old||| +deprecate||| +despatch_signals||5.007001| +destroy_matcher||| +die_nocontext|||vn +die_where||| +die|||v +dirp_dup||| +div128||| +djSP||| +do_aexec5||| +do_aexec||| +do_aspawn||| +do_binmode||5.004050| +do_chomp||| +do_chop||| +do_close||| +do_dump_pad||| +do_eof||| +do_exec3||| +do_execfree||| +do_exec||| +do_gv_dump||5.006000| +do_gvgv_dump||5.006000| +do_hv_dump||5.006000| +do_ipcctl||| +do_ipcget||| +do_join||| +do_kv||| +do_magic_dump||5.006000| +do_msgrcv||| +do_msgsnd||| +do_oddball||| +do_op_dump||5.006000| +do_op_xmldump||| +do_open9||5.006000| +do_openn||5.007001| +do_open||5.004000| +do_pmop_dump||5.006000| +do_pmop_xmldump||| +do_print||| +do_readline||| +do_seek||| +do_semop||| +do_shmio||| +do_smartmatch||| +do_spawn_nowait||| +do_spawn||| +do_sprintf||| +do_sv_dump||5.006000| +do_sysseek||| +do_tell||| +do_trans_complex_utf8||| +do_trans_complex||| +do_trans_count_utf8||| +do_trans_count||| +do_trans_simple_utf8||| +do_trans_simple||| +do_trans||| +do_vecget||| +do_vecset||| +do_vop||| +docatch||| +doeval||| +dofile||| +dofindlabel||| +doform||| +doing_taint||5.008001|n +dooneliner||| +doopen_pm||| +doparseform||| +dopoptoeval||| +dopoptogiven||| +dopoptolabel||| +dopoptoloop||| +dopoptosub_at||| +dopoptowhen||| +doref||5.009003| +dounwind||| +dowantarray||| +dump_all||5.006000| +dump_eval||5.006000| +dump_exec_pos||| +dump_fds||| +dump_form||5.006000| +dump_indent||5.006000|v +dump_mstats||| +dump_packsubs||5.006000| +dump_sub||5.006000| +dump_sv_child||| +dump_trie_interim_list||| +dump_trie_interim_table||| +dump_trie||| +dump_vindent||5.006000| +dumpuntil||| +dup_attrlist||| +emulate_cop_io||| +eval_pv|5.006000||p +eval_sv|5.006000||p +exec_failed||| +expect_number||| +fbm_compile||5.005000| +fbm_instr||5.005000| +feature_is_enabled||| +fetch_cop_label||5.011000| +filter_add||| +filter_del||| +filter_gets||| +filter_read||| +find_and_forget_pmops||| +find_array_subscript||| +find_beginning||| +find_byclass||| +find_hash_subscript||| +find_in_my_stash||| +find_runcv||5.008001| +find_rundefsvoffset||5.009002| +find_script||| +find_uninit_var||| +first_symbol|||n +fold_constants||| +forbid_setid||| +force_ident||| +force_list||| +force_next||| +force_version||| +force_word||| +forget_pmop||| +form_nocontext|||vn +form||5.004000|v +fp_dup||| +fprintf_nocontext|||vn +free_global_struct||| +free_tied_hv_pool||| +free_tmps||| +gen_constant_list||| +get_arena||| +get_aux_mg||| +get_av|5.006000||p +get_context||5.006000|n +get_cvn_flags||5.009005| +get_cv|5.006000||p +get_db_sub||| +get_debug_opts||| +get_hash_seed||| +get_hv|5.006000||p +get_isa_hash||| +get_mstats||| +get_no_modify||| +get_num||| +get_op_descs||5.005000| +get_op_names||5.005000| +get_opargs||| +get_ppaddr||5.006000| +get_re_arg||| +get_sv|5.006000||p +get_vtbl||5.005030| +getcwd_sv||5.007002| +getenv_len||| +glob_2number||| +glob_2pv||| +glob_assign_glob||| +glob_assign_ref||| +gp_dup||| +gp_free||| +gp_ref||| +grok_bin|5.007003||p +grok_hex|5.007003||p +grok_number|5.007002||p +grok_numeric_radix|5.007002||p +grok_oct|5.007003||p +group_end||| +gv_AVadd||| +gv_HVadd||| +gv_IOadd||| +gv_SVadd||| +gv_autoload4||5.004000| +gv_check||| +gv_const_sv||5.009003| +gv_dump||5.006000| +gv_efullname3||5.004000| +gv_efullname4||5.006001| +gv_efullname||| +gv_ename||| +gv_fetchfile_flags||5.009005| +gv_fetchfile||| +gv_fetchmeth_autoload||5.007003| +gv_fetchmethod_autoload||5.004000| +gv_fetchmethod_flags||5.011000| +gv_fetchmethod||| +gv_fetchmeth||| +gv_fetchpvn_flags||5.009002| +gv_fetchpv||| +gv_fetchsv||5.009002| +gv_fullname3||5.004000| +gv_fullname4||5.006001| +gv_fullname||| +gv_get_super_pkg||| +gv_handler||5.007001| +gv_init_sv||| +gv_init||| +gv_name_set||5.009004| +gv_stashpvn|5.004000||p +gv_stashpvs||5.009003| +gv_stashpv||| +gv_stashsv||| +he_dup||| +hek_dup||| +hfreeentries||| +hsplit||| +hv_assert||5.011000| +hv_auxinit|||n +hv_backreferences_p||| +hv_clear_placeholders||5.009001| +hv_clear||| +hv_common_key_len||5.010000| +hv_common||5.010000| +hv_copy_hints_hv||| +hv_delayfree_ent||5.004000| +hv_delete_common||| +hv_delete_ent||5.004000| +hv_delete||| +hv_eiter_p||5.009003| +hv_eiter_set||5.009003| +hv_exists_ent||5.004000| +hv_exists||| +hv_fetch_ent||5.004000| +hv_fetchs|5.009003||p +hv_fetch||| +hv_free_ent||5.004000| +hv_iterinit||| +hv_iterkeysv||5.004000| +hv_iterkey||| +hv_iternext_flags||5.008000| +hv_iternextsv||| +hv_iternext||| +hv_iterval||| +hv_kill_backrefs||| +hv_ksplit||5.004000| +hv_magic_check|||n +hv_magic||| +hv_name_set||5.009003| +hv_notallowed||| +hv_placeholders_get||5.009003| +hv_placeholders_p||5.009003| +hv_placeholders_set||5.009003| +hv_riter_p||5.009003| +hv_riter_set||5.009003| +hv_scalar||5.009001| +hv_store_ent||5.004000| +hv_store_flags||5.008000| +hv_stores|5.009004||p +hv_store||| +hv_undef||| +ibcmp_locale||5.004000| +ibcmp_utf8||5.007003| +ibcmp||| +incline||| +incpush_if_exists||| +incpush_use_sep||| +incpush||| +ingroup||| +init_argv_symbols||| +init_debugger||| +init_global_struct||| +init_i18nl10n||5.006000| +init_i18nl14n||5.006000| +init_ids||| +init_interp||| +init_main_stash||| +init_perllib||| +init_postdump_symbols||| +init_predump_symbols||| +init_stacks||5.005000| +init_tm||5.007002| +instr||| +intro_my||| +intuit_method||| +intuit_more||| +invert||| +io_close||| +isALNUMC|5.006000||p +isALNUM||| +isALPHA||| +isASCII|5.006000||p +isBLANK|5.006001||p +isCNTRL|5.006000||p +isDIGIT||| +isGRAPH|5.006000||p +isLOWER||| +isPRINT|5.004000||p +isPSXSPC|5.006001||p +isPUNCT|5.006000||p +isSPACE||| +isUPPER||| +isXDIGIT|5.006000||p +is_an_int||| +is_gv_magical_sv||| +is_handle_constructor|||n +is_list_assignment||| +is_lvalue_sub||5.007001| +is_uni_alnum_lc||5.006000| +is_uni_alnumc_lc||5.006000| +is_uni_alnumc||5.006000| +is_uni_alnum||5.006000| +is_uni_alpha_lc||5.006000| +is_uni_alpha||5.006000| +is_uni_ascii_lc||5.006000| +is_uni_ascii||5.006000| +is_uni_cntrl_lc||5.006000| +is_uni_cntrl||5.006000| +is_uni_digit_lc||5.006000| +is_uni_digit||5.006000| +is_uni_graph_lc||5.006000| +is_uni_graph||5.006000| +is_uni_idfirst_lc||5.006000| +is_uni_idfirst||5.006000| +is_uni_lower_lc||5.006000| +is_uni_lower||5.006000| +is_uni_print_lc||5.006000| +is_uni_print||5.006000| +is_uni_punct_lc||5.006000| +is_uni_punct||5.006000| +is_uni_space_lc||5.006000| +is_uni_space||5.006000| +is_uni_upper_lc||5.006000| +is_uni_upper||5.006000| +is_uni_xdigit_lc||5.006000| +is_uni_xdigit||5.006000| +is_utf8_alnumc||5.006000| +is_utf8_alnum||5.006000| +is_utf8_alpha||5.006000| +is_utf8_ascii||5.006000| +is_utf8_char_slow|||n +is_utf8_char||5.006000| +is_utf8_cntrl||5.006000| +is_utf8_common||| +is_utf8_digit||5.006000| +is_utf8_graph||5.006000| +is_utf8_idcont||5.008000| +is_utf8_idfirst||5.006000| +is_utf8_lower||5.006000| +is_utf8_mark||5.006000| +is_utf8_print||5.006000| +is_utf8_punct||5.006000| +is_utf8_space||5.006000| +is_utf8_string_loclen||5.009003| +is_utf8_string_loc||5.008001| +is_utf8_string||5.006001| +is_utf8_upper||5.006000| +is_utf8_xdigit||5.006000| +isa_lookup||| +items|||n +ix|||n +jmaybe||| +join_exact||| +keyword||| +leave_scope||| +lex_end||| +lex_start||| +linklist||| +listkids||| +list||| +load_module_nocontext|||vn +load_module|5.006000||pv +localize||| +looks_like_bool||| +looks_like_number||| +lop||| +mPUSHi|5.009002||p +mPUSHn|5.009002||p +mPUSHp|5.009002||p +mPUSHs|5.011000||p +mPUSHu|5.009002||p +mXPUSHi|5.009002||p +mXPUSHn|5.009002||p +mXPUSHp|5.009002||p +mXPUSHs|5.011000||p +mXPUSHu|5.009002||p +mad_free||| +madlex||| +madparse||| +magic_clear_all_env||| +magic_clearenv||| +magic_clearhint||| +magic_clearisa||| +magic_clearpack||| +magic_clearsig||| +magic_dump||5.006000| +magic_existspack||| +magic_freearylen_p||| +magic_freeovrld||| +magic_getarylen||| +magic_getdefelem||| +magic_getnkeys||| +magic_getpack||| +magic_getpos||| +magic_getsig||| +magic_getsubstr||| +magic_gettaint||| +magic_getuvar||| +magic_getvec||| +magic_get||| +magic_killbackrefs||| +magic_len||| +magic_methcall||| +magic_methpack||| +magic_nextpack||| +magic_regdata_cnt||| +magic_regdatum_get||| +magic_regdatum_set||| +magic_scalarpack||| +magic_set_all_env||| +magic_setamagic||| +magic_setarylen||| +magic_setcollxfrm||| +magic_setdbline||| +magic_setdefelem||| +magic_setenv||| +magic_sethint||| +magic_setisa||| +magic_setmglob||| +magic_setnkeys||| +magic_setpack||| +magic_setpos||| +magic_setregexp||| +magic_setsig||| +magic_setsubstr||| +magic_settaint||| +magic_setutf8||| +magic_setuvar||| +magic_setvec||| +magic_set||| +magic_sizepack||| +magic_wipepack||| +make_matcher||| +make_trie_failtable||| +make_trie||| +malloc_good_size|||n +malloced_size|||n +malloc||5.007002|n +markstack_grow||| +matcher_matches_sv||| +measure_struct||| +memEQ|5.004000||p +memNE|5.004000||p +mem_collxfrm||| +mem_log_common|||n +mess_alloc||| +mess_nocontext|||vn +mess||5.006000|v +method_common||| +mfree||5.007002|n +mg_clear||| +mg_copy||| +mg_dup||| +mg_find||| +mg_free||| +mg_get||| +mg_length||5.005000| +mg_localize||| +mg_magical||| +mg_set||| +mg_size||5.005000| +mini_mktime||5.007002| +missingterm||| +mode_from_discipline||| +modkids||| +mod||| +more_bodies||| +more_sv||| +moreswitches||| +mro_get_from_name||5.011000| +mro_get_linear_isa_dfs||| +mro_get_linear_isa||5.009005| +mro_get_private_data||5.011000| +mro_isa_changed_in||| +mro_meta_dup||| +mro_meta_init||| +mro_method_changed_in||5.009005| +mro_register||5.011000| +mro_set_mro||5.011000| +mro_set_private_data||5.011000| +mul128||| +mulexp10|||n +my_atof2||5.007002| +my_atof||5.006000| +my_attrs||| +my_bcopy|||n +my_betoh16|||n +my_betoh32|||n +my_betoh64|||n +my_betohi|||n +my_betohl|||n +my_betohs|||n +my_bzero|||n +my_chsize||| +my_clearenv||| +my_cxt_index||| +my_cxt_init||| +my_dirfd||5.009005| +my_exit_jump||| +my_exit||| +my_failure_exit||5.004000| +my_fflush_all||5.006000| +my_fork||5.007003|n +my_htobe16|||n +my_htobe32|||n +my_htobe64|||n +my_htobei|||n +my_htobel|||n +my_htobes|||n +my_htole16|||n +my_htole32|||n +my_htole64|||n +my_htolei|||n +my_htolel|||n +my_htoles|||n +my_htonl||| +my_kid||| +my_letoh16|||n +my_letoh32|||n +my_letoh64|||n +my_letohi|||n +my_letohl|||n +my_letohs|||n +my_lstat||| +my_memcmp||5.004000|n +my_memset|||n +my_ntohl||| +my_pclose||5.004000| +my_popen_list||5.007001| +my_popen||5.004000| +my_setenv||| +my_snprintf|5.009004||pvn +my_socketpair||5.007003|n +my_sprintf|5.009003||pvn +my_stat||| +my_strftime||5.007002| +my_strlcat|5.009004||pn +my_strlcpy|5.009004||pn +my_swabn|||n +my_swap||| +my_unexec||| +my_vsnprintf||5.009004|n +need_utf8|||n +newANONATTRSUB||5.006000| +newANONHASH||| +newANONLIST||| +newANONSUB||| +newASSIGNOP||| +newATTRSUB||5.006000| +newAVREF||| +newAV||| +newBINOP||| +newCONDOP||| +newCONSTSUB|5.004050||p +newCVREF||| +newDEFSVOP||| +newFORM||| +newFOROP||| +newGIVENOP||5.009003| +newGIVWHENOP||| +newGP||| +newGVOP||| +newGVREF||| +newGVgen||| +newHVREF||| +newHVhv||5.005000| +newHV||| +newIO||| +newLISTOP||| +newLOGOP||| +newLOOPEX||| +newLOOPOP||| +newMADPROP||| +newMADsv||| +newMYSUB||| +newNULLLIST||| +newOP||| +newPADOP||| +newPMOP||| +newPROG||| +newPVOP||| +newRANGE||| +newRV_inc|5.004000||p +newRV_noinc|5.004000||p +newRV||| +newSLICEOP||| +newSTATEOP||| +newSUB||| +newSVOP||| +newSVREF||| +newSV_type||5.009005| +newSVhek||5.009003| +newSViv||| +newSVnv||| +newSVpvf_nocontext|||vn +newSVpvf||5.004000|v +newSVpvn_flags|5.011000||p +newSVpvn_share|5.007001||p +newSVpvn_utf8|5.011000||p +newSVpvn|5.004050||p +newSVpvs_flags|5.011000||p +newSVpvs_share||5.009003| +newSVpvs|5.009003||p +newSVpv||| +newSVrv||| +newSVsv||| +newSVuv|5.006000||p +newSV||| +newTOKEN||| +newUNOP||| +newWHENOP||5.009003| +newWHILEOP||5.009003| +newXS_flags||5.009004| +newXSproto||5.006000| +newXS||5.006000| +new_collate||5.006000| +new_constant||| +new_ctype||5.006000| +new_he||| +new_logop||| +new_numeric||5.006000| +new_stackinfo||5.005000| +new_version||5.009000| +new_warnings_bitfield||| +next_symbol||| +nextargv||| +nextchar||| +ninstr||| +no_bareword_allowed||| +no_fh_allowed||| +no_op||| +not_a_number||| +nothreadhook||5.008000| +nuke_stacks||| +num_overflow|||n +offer_nice_chunk||| +oopsAV||| +oopsHV||| +op_clear||| +op_const_sv||| +op_dump||5.006000| +op_free||| +op_getmad_weak||| +op_getmad||| +op_null||5.007002| +op_refcnt_dec||| +op_refcnt_inc||| +op_refcnt_lock||5.009002| +op_refcnt_unlock||5.009002| +op_xmldump||| +open_script||| +pMY_CXT_|5.007003||p +pMY_CXT|5.007003||p +pTHX_|5.006000||p +pTHX|5.006000||p +packWARN|5.007003||p +pack_cat||5.007003| +pack_rec||| +package||| +packlist||5.008001| +pad_add_anon||| +pad_add_name||| +pad_alloc||| +pad_block_start||| +pad_check_dup||| +pad_compname_type||| +pad_findlex||| +pad_findmy||| +pad_fixup_inner_anons||| +pad_free||| +pad_leavemy||| +pad_new||| +pad_peg|||n +pad_push||| +pad_reset||| +pad_setsv||| +pad_sv||5.011000| +pad_swipe||| +pad_tidy||| +pad_undef||| +parse_body||| +parse_unicode_opts||| +parser_dup||| +parser_free||| +path_is_absolute|||n +peep||| +pending_Slabs_to_ro||| +perl_alloc_using|||n +perl_alloc|||n +perl_clone_using|||n +perl_clone|||n +perl_construct|||n +perl_destruct||5.007003|n +perl_free|||n +perl_parse||5.006000|n +perl_run|||n +pidgone||| +pm_description||| +pmflag||| +pmop_dump||5.006000| +pmop_xmldump||| +pmruntime||| +pmtrans||| +pop_scope||| +pregcomp||5.009005| +pregexec||| +pregfree2||5.011000| +pregfree||| +prepend_elem||| +prepend_madprops||| +printbuf||| +printf_nocontext|||vn +process_special_blocks||| +ptr_table_clear||5.009005| +ptr_table_fetch||5.009005| +ptr_table_find|||n +ptr_table_free||5.009005| +ptr_table_new||5.009005| +ptr_table_split||5.009005| +ptr_table_store||5.009005| +push_scope||| +put_byte||| +pv_display|5.006000||p +pv_escape|5.009004||p +pv_pretty|5.009004||p +pv_uni_display||5.007003| +qerror||| +qsortsvu||| +re_compile||5.009005| +re_croak2||| +re_dup_guts||| +re_intuit_start||5.009005| +re_intuit_string||5.006000| +readpipe_override||| +realloc||5.007002|n +reentrant_free||| +reentrant_init||| +reentrant_retry|||vn +reentrant_size||| +ref_array_or_hash||| +refcounted_he_chain_2hv||| +refcounted_he_fetch||| +refcounted_he_free||| +refcounted_he_new_common||| +refcounted_he_new||| +refcounted_he_value||| +refkids||| +refto||| +ref||5.011000| +reg_check_named_buff_matched||| +reg_named_buff_all||5.009005| +reg_named_buff_exists||5.009005| +reg_named_buff_fetch||5.009005| +reg_named_buff_firstkey||5.009005| +reg_named_buff_iter||| +reg_named_buff_nextkey||5.009005| +reg_named_buff_scalar||5.009005| +reg_named_buff||| +reg_namedseq||| +reg_node||| +reg_numbered_buff_fetch||| +reg_numbered_buff_length||| +reg_numbered_buff_store||| +reg_qr_package||| +reg_recode||| +reg_scan_name||| +reg_skipcomment||| +reg_temp_copy||| +reganode||| +regatom||| +regbranch||| +regclass_swash||5.009004| +regclass||| +regcppop||| +regcppush||| +regcurly|||n +regdump_extflags||| +regdump||5.005000| +regdupe_internal||| +regexec_flags||5.005000| +regfree_internal||5.009005| +reghop3|||n +reghop4|||n +reghopmaybe3|||n +reginclass||| +reginitcolors||5.006000| +reginsert||| +regmatch||| +regnext||5.005000| +regpiece||| +regpposixcc||| +regprop||| +regrepeat||| +regtail_study||| +regtail||| +regtry||| +reguni||| +regwhite|||n +reg||| +repeatcpy||| +report_evil_fh||| +report_uninit||| +require_pv||5.006000| +require_tie_mod||| +restore_magic||| +rninstr||| +rsignal_restore||| +rsignal_save||| +rsignal_state||5.004000| +rsignal||5.004000| +run_body||| +run_user_filter||| +runops_debug||5.005000| +runops_standard||5.005000| +rvpv_dup||| +rxres_free||| +rxres_restore||| +rxres_save||| +safesyscalloc||5.006000|n +safesysfree||5.006000|n +safesysmalloc||5.006000|n +safesysrealloc||5.006000|n +same_dirent||| +save_I16||5.004000| +save_I32||| +save_I8||5.006000| +save_adelete||5.011000| +save_aelem||5.004050| +save_alloc||5.006000| +save_aptr||| +save_ary||| +save_bool||5.008001| +save_clearsv||| +save_delete||| +save_destructor_x||5.006000| +save_destructor||5.006000| +save_freeop||| +save_freepv||| +save_freesv||| +save_generic_pvref||5.006001| +save_generic_svref||5.005030| +save_gp||5.004000| +save_hash||| +save_hek_flags|||n +save_helem_flags||5.011000| +save_helem||5.004050| +save_hints||| +save_hptr||| +save_int||| +save_item||| +save_iv||5.005000| +save_lines||| +save_list||| +save_long||| +save_magic||| +save_mortalizesv||5.007001| +save_nogv||| +save_op||| +save_padsv_and_mortalize||5.011000| +save_pptr||| +save_pushi32ptr||| +save_pushptri32ptr||| +save_pushptrptr||| +save_pushptr||5.011000| +save_re_context||5.006000| +save_scalar_at||| +save_scalar||| +save_set_svflags||5.009000| +save_shared_pvref||5.007003| +save_sptr||| +save_svref||| +save_vptr||5.006000| +savepvn||| +savepvs||5.009003| +savepv||| +savesharedpvn||5.009005| +savesharedpv||5.007003| +savestack_grow_cnt||5.008001| +savestack_grow||| +savesvpv||5.009002| +sawparens||| +scalar_mod_type|||n +scalarboolean||| +scalarkids||| +scalarseq||| +scalarvoid||| +scalar||| +scan_bin||5.006000| +scan_commit||| +scan_const||| +scan_formline||| +scan_heredoc||| +scan_hex||| +scan_ident||| +scan_inputsymbol||| +scan_num||5.007001| +scan_oct||| +scan_pat||| +scan_str||| +scan_subst||| +scan_trans||| +scan_version||5.009001| +scan_vstring||5.009005| +scan_word||| +scope||| +screaminstr||5.005000| +search_const||| +seed||5.008001| +sequence_num||| +sequence_tail||| +sequence||| +set_context||5.006000|n +set_numeric_local||5.006000| +set_numeric_radix||5.006000| +set_numeric_standard||5.006000| +setdefout||| +share_hek_flags||| +share_hek||5.004000| +si_dup||| +sighandler|||n +simplify_sort||| +skipspace0||| +skipspace1||| +skipspace2||| +skipspace||| +softref2xv||| +sortcv_stacked||| +sortcv_xsub||| +sortcv||| +sortsv_flags||5.009003| +sortsv||5.007003| +space_join_names_mortal||| +ss_dup||| +stack_grow||| +start_force||| +start_glob||| +start_subparse||5.004000| +stashpv_hvname_match||5.011000| +stdize_locale||| +store_cop_label||| +strEQ||| +strGE||| +strGT||| +strLE||| +strLT||| +strNE||| +str_to_version||5.006000| +strip_return||| +strnEQ||| +strnNE||| +study_chunk||| +sub_crush_depth||| +sublex_done||| +sublex_push||| +sublex_start||| +sv_2bool||| +sv_2cv||| +sv_2io||| +sv_2iuv_common||| +sv_2iuv_non_preserve||| +sv_2iv_flags||5.009001| +sv_2iv||| +sv_2mortal||| +sv_2num||| +sv_2nv||| +sv_2pv_flags|5.007002||p +sv_2pv_nolen|5.006000||p +sv_2pvbyte_nolen|5.006000||p +sv_2pvbyte|5.006000||p +sv_2pvutf8_nolen||5.006000| +sv_2pvutf8||5.006000| +sv_2pv||| +sv_2uv_flags||5.009001| +sv_2uv|5.004000||p +sv_add_arena||| +sv_add_backref||| +sv_backoff||| +sv_bless||| +sv_cat_decode||5.008001| +sv_catpv_mg|5.004050||p +sv_catpvf_mg_nocontext|||pvn +sv_catpvf_mg|5.006000|5.004000|pv +sv_catpvf_nocontext|||vn +sv_catpvf||5.004000|v +sv_catpvn_flags||5.007002| +sv_catpvn_mg|5.004050||p +sv_catpvn_nomg|5.007002||p +sv_catpvn||| +sv_catpvs|5.009003||p +sv_catpv||| +sv_catsv_flags||5.007002| +sv_catsv_mg|5.004050||p +sv_catsv_nomg|5.007002||p +sv_catsv||| +sv_catxmlpvn||| +sv_catxmlsv||| +sv_chop||| +sv_clean_all||| +sv_clean_objs||| +sv_clear||| +sv_cmp_locale||5.004000| +sv_cmp||| +sv_collxfrm||| +sv_compile_2op||5.008001| +sv_copypv||5.007003| +sv_dec||| +sv_del_backref||| +sv_derived_from||5.004000| +sv_destroyable||5.010000| +sv_does||5.009004| +sv_dump||| +sv_dup||| +sv_eq||| +sv_exp_grow||| +sv_force_normal_flags||5.007001| +sv_force_normal||5.006000| +sv_free2||| +sv_free_arenas||| +sv_free||| +sv_gets||5.004000| +sv_grow||| +sv_i_ncmp||| +sv_inc||| +sv_insert_flags||5.011000| +sv_insert||| +sv_isa||| +sv_isobject||| +sv_iv||5.005000| +sv_kill_backrefs||| +sv_len_utf8||5.006000| +sv_len||| +sv_magic_portable|5.011000|5.004000|p +sv_magicext||5.007003| +sv_magic||| +sv_mortalcopy||| +sv_ncmp||| +sv_newmortal||| +sv_newref||| +sv_nolocking||5.007003| +sv_nosharing||5.007003| +sv_nounlocking||| +sv_nv||5.005000| +sv_peek||5.005000| +sv_pos_b2u_midway||| +sv_pos_b2u||5.006000| +sv_pos_u2b_cached||| +sv_pos_u2b_forwards|||n +sv_pos_u2b_midway|||n +sv_pos_u2b||5.006000| +sv_pvbyten_force||5.006000| +sv_pvbyten||5.006000| +sv_pvbyte||5.006000| +sv_pvn_force_flags|5.007002||p +sv_pvn_force||| +sv_pvn_nomg|5.007003|5.005000|p +sv_pvn||5.005000| +sv_pvutf8n_force||5.006000| +sv_pvutf8n||5.006000| +sv_pvutf8||5.006000| +sv_pv||5.006000| +sv_recode_to_utf8||5.007003| +sv_reftype||| +sv_release_COW||| +sv_replace||| +sv_report_used||| +sv_reset||| +sv_rvweaken||5.006000| +sv_setiv_mg|5.004050||p +sv_setiv||| +sv_setnv_mg|5.006000||p +sv_setnv||| +sv_setpv_mg|5.004050||p +sv_setpvf_mg_nocontext|||pvn +sv_setpvf_mg|5.006000|5.004000|pv +sv_setpvf_nocontext|||vn +sv_setpvf||5.004000|v +sv_setpviv_mg||5.008001| +sv_setpviv||5.008001| +sv_setpvn_mg|5.004050||p +sv_setpvn||| +sv_setpvs|5.009004||p +sv_setpv||| +sv_setref_iv||| +sv_setref_nv||| +sv_setref_pvn||| +sv_setref_pv||| +sv_setref_uv||5.007001| +sv_setsv_cow||| +sv_setsv_flags||5.007002| +sv_setsv_mg|5.004050||p +sv_setsv_nomg|5.007002||p +sv_setsv||| +sv_setuv_mg|5.004050||p +sv_setuv|5.004000||p +sv_tainted||5.004000| +sv_taint||5.004000| +sv_true||5.005000| +sv_unglob||| +sv_uni_display||5.007003| +sv_unmagic||| +sv_unref_flags||5.007001| +sv_unref||| +sv_untaint||5.004000| +sv_upgrade||| +sv_usepvn_flags||5.009004| +sv_usepvn_mg|5.004050||p +sv_usepvn||| +sv_utf8_decode||5.006000| +sv_utf8_downgrade||5.006000| +sv_utf8_encode||5.006000| +sv_utf8_upgrade_flags_grow||5.011000| +sv_utf8_upgrade_flags||5.007002| +sv_utf8_upgrade_nomg||5.007002| +sv_utf8_upgrade||5.007001| +sv_uv|5.005000||p +sv_vcatpvf_mg|5.006000|5.004000|p +sv_vcatpvfn||5.004000| +sv_vcatpvf|5.006000|5.004000|p +sv_vsetpvf_mg|5.006000|5.004000|p +sv_vsetpvfn||5.004000| +sv_vsetpvf|5.006000|5.004000|p +sv_xmlpeek||| +svtype||| +swallow_bom||| +swap_match_buff||| +swash_fetch||5.007002| +swash_get||| +swash_init||5.006000| +sys_init3||5.010000|n +sys_init||5.010000|n +sys_intern_clear||| +sys_intern_dup||| +sys_intern_init||| +sys_term||5.010000|n +taint_env||| +taint_proper||| +tmps_grow||5.006000| +toLOWER||| +toUPPER||| +to_byte_substr||| +to_uni_fold||5.007003| +to_uni_lower_lc||5.006000| +to_uni_lower||5.007003| +to_uni_title_lc||5.006000| +to_uni_title||5.007003| +to_uni_upper_lc||5.006000| +to_uni_upper||5.007003| +to_utf8_case||5.007003| +to_utf8_fold||5.007003| +to_utf8_lower||5.007003| +to_utf8_substr||| +to_utf8_title||5.007003| +to_utf8_upper||5.007003| +token_free||| +token_getmad||| +tokenize_use||| +tokeq||| +tokereport||| +too_few_arguments||| +too_many_arguments||| +uiv_2buf|||n +unlnk||| +unpack_rec||| +unpack_str||5.007003| +unpackstring||5.008001| +unshare_hek_or_pvn||| +unshare_hek||| +unsharepvn||5.004000| +unwind_handler_stack||| +update_debugger_info||| +upg_version||5.009005| +usage||| +utf16_to_utf8_reversed||5.006001| +utf16_to_utf8||5.006001| +utf8_distance||5.006000| +utf8_hop||5.006000| +utf8_length||5.007001| +utf8_mg_pos_cache_update||| +utf8_to_bytes||5.006001| +utf8_to_uvchr||5.007001| +utf8_to_uvuni||5.007001| +utf8n_to_uvchr||| +utf8n_to_uvuni||5.007001| +utilize||| +uvchr_to_utf8_flags||5.007003| +uvchr_to_utf8||| +uvuni_to_utf8_flags||5.007003| +uvuni_to_utf8||5.007001| +validate_suid||| +varname||| +vcmp||5.009000| +vcroak||5.006000| +vdeb||5.007003| +vdie_common||| +vdie_croak_common||| +vdie||| +vform||5.006000| +visit||| +vivify_defelem||| +vivify_ref||| +vload_module|5.006000||p +vmess||5.006000| +vnewSVpvf|5.006000|5.004000|p +vnormal||5.009002| +vnumify||5.009000| +vstringify||5.009000| +vverify||5.009003| +vwarner||5.006000| +vwarn||5.006000| +wait4pid||| +warn_nocontext|||vn +warner_nocontext|||vn +warner|5.006000|5.004000|pv +warn|||v +watch||| +whichsig||| +write_no_mem||| +write_to_stderr||| +xmldump_all||| +xmldump_attr||| +xmldump_eval||| +xmldump_form||| +xmldump_indent|||v +xmldump_packsubs||| +xmldump_sub||| +xmldump_vindent||| +yyerror||| +yylex||| +yyparse||| +yywarn||| +); + +if (exists $opt{'list-unsupported'}) { + my $f; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $API{$f}{todo}; + print "$f ", '.'x(40-length($f)), " ", format_version($API{$f}{todo}), "\n"; + } + exit 0; +} + +# Scan for possible replacement candidates + +my(%replace, %need, %hints, %warnings, %depends); +my $replace = 0; +my($hint, $define, $function); + +sub find_api +{ + my $code = shift; + $code =~ s{ + / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]*) + | "[^"\\]*(?:\\.[^"\\]*)*" + | '[^'\\]*(?:\\.[^'\\]*)*' }{}egsx; + grep { exists $API{$_} } $code =~ /(\w+)/mg; +} + +while () { + if ($hint) { + my $h = $hint->[0] eq 'Hint' ? \%hints : \%warnings; + if (m{^\s*\*\s(.*?)\s*$}) { + for (@{$hint->[1]}) { + $h->{$_} ||= ''; # suppress warning with older perls + $h->{$_} .= "$1\n"; + } + } + else { undef $hint } + } + + $hint = [$1, [split /,?\s+/, $2]] + if m{^\s*$rccs\s+(Hint|Warning):\s+(\w+(?:,?\s+\w+)*)\s*$}; + + if ($define) { + if ($define->[1] =~ /\\$/) { + $define->[1] .= $_; + } + else { + if (exists $API{$define->[0]} && $define->[1] !~ /^DPPP_\(/) { + my @n = find_api($define->[1]); + push @{$depends{$define->[0]}}, @n if @n + } + undef $define; + } + } + + $define = [$1, $2] if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(.*)}; + + if ($function) { + if (/^}/) { + if (exists $API{$function->[0]}) { + my @n = find_api($function->[1]); + push @{$depends{$function->[0]}}, @n if @n + } + undef $function; + } + else { + $function->[1] .= $_; + } + } + + $function = [$1, ''] if m{^DPPP_\(my_(\w+)\)}; + + $replace = $1 if m{^\s*$rccs\s+Replace:\s+(\d+)\s+$rcce\s*$}; + $replace{$2} = $1 if $replace and m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+)}; + $replace{$2} = $1 if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+).*$rccs\s+Replace\s+$rcce}; + $replace{$1} = $2 if m{^\s*$rccs\s+Replace (\w+) with (\w+)\s+$rcce\s*$}; + + if (m{^\s*$rccs\s+(\w+(\s*,\s*\w+)*)\s+depends\s+on\s+(\w+(\s*,\s*\w+)*)\s+$rcce\s*$}) { + my @deps = map { s/\s+//g; $_ } split /,/, $3; + my $d; + for $d (map { s/\s+//g; $_ } split /,/, $1) { + push @{$depends{$d}}, @deps; + } + } + + $need{$1} = 1 if m{^#if\s+defined\(NEED_(\w+)(?:_GLOBAL)?\)}; +} + +for (values %depends) { + my %s; + $_ = [sort grep !$s{$_}++, @$_]; +} + +if (exists $opt{'api-info'}) { + my $f; + my $count = 0; + my $match = $opt{'api-info'} =~ m!^/(.*)/$! ? $1 : "^\Q$opt{'api-info'}\E\$"; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $f =~ /$match/; + print "\n=== $f ===\n\n"; + my $info = 0; + if ($API{$f}{base} || $API{$f}{todo}) { + my $base = format_version($API{$f}{base} || $API{$f}{todo}); + print "Supported at least starting from perl-$base.\n"; + $info++; + } + if ($API{$f}{provided}) { + my $todo = $API{$f}{todo} ? format_version($API{$f}{todo}) : "5.003"; + print "Support by $ppport provided back to perl-$todo.\n"; + print "Support needs to be explicitly requested by NEED_$f.\n" if exists $need{$f}; + print "Depends on: ", join(', ', @{$depends{$f}}), ".\n" if exists $depends{$f}; + print "\n$hints{$f}" if exists $hints{$f}; + print "\nWARNING:\n$warnings{$f}" if exists $warnings{$f}; + $info++; + } + print "No portability information available.\n" unless $info; + $count++; + } + $count or print "Found no API matching '$opt{'api-info'}'."; + print "\n"; + exit 0; +} + +if (exists $opt{'list-provided'}) { + my $f; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $API{$f}{provided}; + my @flags; + push @flags, 'explicit' if exists $need{$f}; + push @flags, 'depend' if exists $depends{$f}; + push @flags, 'hint' if exists $hints{$f}; + push @flags, 'warning' if exists $warnings{$f}; + my $flags = @flags ? ' ['.join(', ', @flags).']' : ''; + print "$f$flags\n"; + } + exit 0; +} + +my @files; +my @srcext = qw( .xs .c .h .cc .cpp -c.inc -xs.inc ); +my $srcext = join '|', map { quotemeta $_ } @srcext; + +if (@ARGV) { + my %seen; + for (@ARGV) { + if (-e) { + if (-f) { + push @files, $_ unless $seen{$_}++; + } + else { warn "'$_' is not a file.\n" } + } + else { + my @new = grep { -f } glob $_ + or warn "'$_' does not exist.\n"; + push @files, grep { !$seen{$_}++ } @new; + } + } +} +else { + eval { + require File::Find; + File::Find::find(sub { + $File::Find::name =~ /($srcext)$/i + and push @files, $File::Find::name; + }, '.'); + }; + if ($@) { + @files = map { glob "*$_" } @srcext; + } +} + +if (!@ARGV || $opt{filter}) { + my(@in, @out); + my %xsc = map { /(.*)\.xs$/ ? ("$1.c" => 1, "$1.cc" => 1) : () } @files; + for (@files) { + my $out = exists $xsc{$_} || /\b\Q$ppport\E$/i || !/($srcext)$/i; + push @{ $out ? \@out : \@in }, $_; + } + if (@ARGV && @out) { + warning("Skipping the following files (use --nofilter to avoid this):\n| ", join "\n| ", @out); + } + @files = @in; +} + +die "No input files given!\n" unless @files; + +my(%files, %global, %revreplace); +%revreplace = reverse %replace; +my $filename; +my $patch_opened = 0; + +for $filename (@files) { + unless (open IN, "<$filename") { + warn "Unable to read from $filename: $!\n"; + next; + } + + info("Scanning $filename ..."); + + my $c = do { local $/; }; + close IN; + + my %file = (orig => $c, changes => 0); + + # Temporarily remove C/XS comments and strings from the code + my @ccom; + + $c =~ s{ + ( ^$HS*\#$HS*include\b[^\r\n]+\b(?:\Q$ppport\E|XSUB\.h)\b[^\r\n]* + | ^$HS*\#$HS*(?:define|elif|if(?:def)?)\b[^\r\n]* ) + | ( ^$HS*\#[^\r\n]* + | "[^"\\]*(?:\\.[^"\\]*)*" + | '[^'\\]*(?:\\.[^'\\]*)*' + | / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]* ) ) + }{ defined $2 and push @ccom, $2; + defined $1 ? $1 : "$ccs$#ccom$cce" }mgsex; + + $file{ccom} = \@ccom; + $file{code} = $c; + $file{has_inc_ppport} = $c =~ /^$HS*#$HS*include[^\r\n]+\b\Q$ppport\E\b/m; + + my $func; + + for $func (keys %API) { + my $match = $func; + $match .= "|$revreplace{$func}" if exists $revreplace{$func}; + if ($c =~ /\b(?:Perl_)?($match)\b/) { + $file{uses_replace}{$1}++ if exists $revreplace{$func} && $1 eq $revreplace{$func}; + $file{uses_Perl}{$func}++ if $c =~ /\bPerl_$func\b/; + if (exists $API{$func}{provided}) { + $file{uses_provided}{$func}++; + if (!exists $API{$func}{base} || $API{$func}{base} > $opt{'compat-version'}) { + $file{uses}{$func}++; + my @deps = rec_depend($func); + if (@deps) { + $file{uses_deps}{$func} = \@deps; + for (@deps) { + $file{uses}{$_} = 0 unless exists $file{uses}{$_}; + } + } + for ($func, @deps) { + $file{needs}{$_} = 'static' if exists $need{$_}; + } + } + } + if (exists $API{$func}{todo} && $API{$func}{todo} > $opt{'compat-version'}) { + if ($c =~ /\b$func\b/) { + $file{uses_todo}{$func}++; + } + } + } + } + + while ($c =~ /^$HS*#$HS*define$HS+(NEED_(\w+?)(_GLOBAL)?)\b/mg) { + if (exists $need{$2}) { + $file{defined $3 ? 'needed_global' : 'needed_static'}{$2}++; + } + else { warning("Possibly wrong #define $1 in $filename") } + } + + for (qw(uses needs uses_todo needed_global needed_static)) { + for $func (keys %{$file{$_}}) { + push @{$global{$_}{$func}}, $filename; + } + } + + $files{$filename} = \%file; +} + +# Globally resolve NEED_'s +my $need; +for $need (keys %{$global{needs}}) { + if (@{$global{needs}{$need}} > 1) { + my @targets = @{$global{needs}{$need}}; + my @t = grep $files{$_}{needed_global}{$need}, @targets; + @targets = @t if @t; + @t = grep /\.xs$/i, @targets; + @targets = @t if @t; + my $target = shift @targets; + $files{$target}{needs}{$need} = 'global'; + for (@{$global{needs}{$need}}) { + $files{$_}{needs}{$need} = 'extern' if $_ ne $target; + } + } +} + +for $filename (@files) { + exists $files{$filename} or next; + + info("=== Analyzing $filename ==="); + + my %file = %{$files{$filename}}; + my $func; + my $c = $file{code}; + my $warnings = 0; + + for $func (sort keys %{$file{uses_Perl}}) { + if ($API{$func}{varargs}) { + unless ($API{$func}{nothxarg}) { + my $changes = ($c =~ s{\b(Perl_$func\s*\(\s*)(?!aTHX_?)(\)|[^\s)]*\))} + { $1 . ($2 eq ')' ? 'aTHX' : 'aTHX_ ') . $2 }ge); + if ($changes) { + warning("Doesn't pass interpreter argument aTHX to Perl_$func"); + $file{changes} += $changes; + } + } + } + else { + warning("Uses Perl_$func instead of $func"); + $file{changes} += ($c =~ s{\bPerl_$func(\s*)\((\s*aTHX_?)?\s*} + {$func$1(}g); + } + } + + for $func (sort keys %{$file{uses_replace}}) { + warning("Uses $func instead of $replace{$func}"); + $file{changes} += ($c =~ s/\b$func\b/$replace{$func}/g); + } + + for $func (sort keys %{$file{uses_provided}}) { + if ($file{uses}{$func}) { + if (exists $file{uses_deps}{$func}) { + diag("Uses $func, which depends on ", join(', ', @{$file{uses_deps}{$func}})); + } + else { + diag("Uses $func"); + } + } + $warnings += hint($func); + } + + unless ($opt{quiet}) { + for $func (sort keys %{$file{uses_todo}}) { + print "*** WARNING: Uses $func, which may not be portable below perl ", + format_version($API{$func}{todo}), ", even with '$ppport'\n"; + $warnings++; + } + } + + for $func (sort keys %{$file{needed_static}}) { + my $message = ''; + if (not exists $file{uses}{$func}) { + $message = "No need to define NEED_$func if $func is never used"; + } + elsif (exists $file{needs}{$func} && $file{needs}{$func} ne 'static') { + $message = "No need to define NEED_$func when already needed globally"; + } + if ($message) { + diag($message); + $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_$func\b.*$LF//mg); + } + } + + for $func (sort keys %{$file{needed_global}}) { + my $message = ''; + if (not exists $global{uses}{$func}) { + $message = "No need to define NEED_${func}_GLOBAL if $func is never used"; + } + elsif (exists $file{needs}{$func}) { + if ($file{needs}{$func} eq 'extern') { + $message = "No need to define NEED_${func}_GLOBAL when already needed globally"; + } + elsif ($file{needs}{$func} eq 'static') { + $message = "No need to define NEED_${func}_GLOBAL when only used in this file"; + } + } + if ($message) { + diag($message); + $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_${func}_GLOBAL\b.*$LF//mg); + } + } + + $file{needs_inc_ppport} = keys %{$file{uses}}; + + if ($file{needs_inc_ppport}) { + my $pp = ''; + + for $func (sort keys %{$file{needs}}) { + my $type = $file{needs}{$func}; + next if $type eq 'extern'; + my $suffix = $type eq 'global' ? '_GLOBAL' : ''; + unless (exists $file{"needed_$type"}{$func}) { + if ($type eq 'global') { + diag("Files [@{$global{needs}{$func}}] need $func, adding global request"); + } + else { + diag("File needs $func, adding static request"); + } + $pp .= "#define NEED_$func$suffix\n"; + } + } + + if ($pp && ($c =~ s/^(?=$HS*#$HS*define$HS+NEED_\w+)/$pp/m)) { + $pp = ''; + $file{changes}++; + } + + unless ($file{has_inc_ppport}) { + diag("Needs to include '$ppport'"); + $pp .= qq(#include "$ppport"\n) + } + + if ($pp) { + $file{changes} += ($c =~ s/^($HS*#$HS*define$HS+NEED_\w+.*?)^/$1$pp/ms) + || ($c =~ s/^(?=$HS*#$HS*include.*\Q$ppport\E)/$pp/m) + || ($c =~ s/^($HS*#$HS*include.*XSUB.*\s*?)^/$1$pp/m) + || ($c =~ s/^/$pp/); + } + } + else { + if ($file{has_inc_ppport}) { + diag("No need to include '$ppport'"); + $file{changes} += ($c =~ s/^$HS*?#$HS*include.*\Q$ppport\E.*?$LF//m); + } + } + + # put back in our C comments + my $ix; + my $cppc = 0; + my @ccom = @{$file{ccom}}; + for $ix (0 .. $#ccom) { + if (!$opt{cplusplus} && $ccom[$ix] =~ s!^//!!) { + $cppc++; + $file{changes} += $c =~ s/$rccs$ix$rcce/$ccs$ccom[$ix] $cce/; + } + else { + $c =~ s/$rccs$ix$rcce/$ccom[$ix]/; + } + } + + if ($cppc) { + my $s = $cppc != 1 ? 's' : ''; + warning("Uses $cppc C++ style comment$s, which is not portable"); + } + + my $s = $warnings != 1 ? 's' : ''; + my $warn = $warnings ? " ($warnings warning$s)" : ''; + info("Analysis completed$warn"); + + if ($file{changes}) { + if (exists $opt{copy}) { + my $newfile = "$filename$opt{copy}"; + if (-e $newfile) { + error("'$newfile' already exists, refusing to write copy of '$filename'"); + } + else { + local *F; + if (open F, ">$newfile") { + info("Writing copy of '$filename' with changes to '$newfile'"); + print F $c; + close F; + } + else { + error("Cannot open '$newfile' for writing: $!"); + } + } + } + elsif (exists $opt{patch} || $opt{changes}) { + if (exists $opt{patch}) { + unless ($patch_opened) { + if (open PATCH, ">$opt{patch}") { + $patch_opened = 1; + } + else { + error("Cannot open '$opt{patch}' for writing: $!"); + delete $opt{patch}; + $opt{changes} = 1; + goto fallback; + } + } + mydiff(\*PATCH, $filename, $c); + } + else { +fallback: + info("Suggested changes:"); + mydiff(\*STDOUT, $filename, $c); + } + } + else { + my $s = $file{changes} == 1 ? '' : 's'; + info("$file{changes} potentially required change$s detected"); + } + } + else { + info("Looks good"); + } +} + +close PATCH if $patch_opened; + +exit 0; + + +sub try_use { eval "use @_;"; return $@ eq '' } + +sub mydiff +{ + local *F = shift; + my($file, $str) = @_; + my $diff; + + if (exists $opt{diff}) { + $diff = run_diff($opt{diff}, $file, $str); + } + + if (!defined $diff and try_use('Text::Diff')) { + $diff = Text::Diff::diff($file, \$str, { STYLE => 'Unified' }); + $diff = <
$tmp") { + print F $str; + close F; + + if (open F, "$prog $file $tmp |") { + while () { + s/\Q$tmp\E/$file.patched/; + $diff .= $_; + } + close F; + unlink $tmp; + return $diff; + } + + unlink $tmp; + } + else { + error("Cannot open '$tmp' for writing: $!"); + } + + return undef; +} + +sub rec_depend +{ + my($func, $seen) = @_; + return () unless exists $depends{$func}; + $seen = {%{$seen||{}}}; + return () if $seen->{$func}++; + my %s; + grep !$s{$_}++, map { ($_, rec_depend($_, $seen)) } @{$depends{$func}}; +} + +sub parse_version +{ + my $ver = shift; + + if ($ver =~ /^(\d+)\.(\d+)\.(\d+)$/) { + return ($1, $2, $3); + } + elsif ($ver !~ /^\d+\.[\d_]+$/) { + die "cannot parse version '$ver'\n"; + } + + $ver =~ s/_//g; + $ver =~ s/$/000000/; + + my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; + + $v = int $v; + $s = int $s; + + if ($r < 5 || ($r == 5 && $v < 6)) { + if ($s % 10) { + die "cannot parse version '$ver'\n"; + } + } + + return ($r, $v, $s); +} + +sub format_version +{ + my $ver = shift; + + $ver =~ s/$/000000/; + my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; + + $v = int $v; + $s = int $s; + + if ($r < 5 || ($r == 5 && $v < 6)) { + if ($s % 10) { + die "invalid version '$ver'\n"; + } + $s /= 10; + + $ver = sprintf "%d.%03d", $r, $v; + $s > 0 and $ver .= sprintf "_%02d", $s; + + return $ver; + } + + return sprintf "%d.%d.%d", $r, $v, $s; +} + +sub info +{ + $opt{quiet} and return; + print @_, "\n"; +} + +sub diag +{ + $opt{quiet} and return; + $opt{diag} and print @_, "\n"; +} + +sub warning +{ + $opt{quiet} and return; + print "*** ", @_, "\n"; +} + +sub error +{ + print "*** ERROR: ", @_, "\n"; +} + +my %given_hints; +my %given_warnings; +sub hint +{ + $opt{quiet} and return; + my $func = shift; + my $rv = 0; + if (exists $warnings{$func} && !$given_warnings{$func}++) { + my $warn = $warnings{$func}; + $warn =~ s!^!*** !mg; + print "*** WARNING: $func\n", $warn; + $rv++; + } + if ($opt{hints} && exists $hints{$func} && !$given_hints{$func}++) { + my $hint = $hints{$func}; + $hint =~ s/^/ /mg; + print " --- hint for $func ---\n", $hint; + } + $rv; +} + +sub usage +{ + my($usage) = do { local(@ARGV,$/)=($0); <> } =~ /^=head\d$HS+SYNOPSIS\s*^(.*?)\s*^=/ms; + my %M = ( 'I' => '*' ); + $usage =~ s/^\s*perl\s+\S+/$^X $0/; + $usage =~ s/([A-Z])<([^>]+)>/$M{$1}$2$M{$1}/g; + + print < }; + my($copy) = $self =~ /^=head\d\s+COPYRIGHT\s*^(.*?)^=\w+/ms; + $copy =~ s/^(?=\S+)/ /gms; + $self =~ s/^$HS+Do NOT edit.*?(?=^-)/$copy/ms; + $self =~ s/^SKIP.*(?=^__DATA__)/SKIP +if (\@ARGV && \$ARGV[0] eq '--unstrip') { + eval { require Devel::PPPort }; + \$@ and die "Cannot require Devel::PPPort, please install.\\n"; + if (eval \$Devel::PPPort::VERSION < $VERSION) { + die "$0 was originally generated with Devel::PPPort $VERSION.\\n" + . "Your Devel::PPPort is only version \$Devel::PPPort::VERSION.\\n" + . "Please install a newer version, or --unstrip will not work.\\n"; + } + Devel::PPPort::WriteFile(\$0); + exit 0; +} +print <$0" or die "cannot strip $0: $!\n"; + print OUT "$pl$c\n"; + + exit 0; +} + +__DATA__ +*/ + +#ifndef _P_P_PORTABILITY_H_ +#define _P_P_PORTABILITY_H_ + +#ifndef DPPP_NAMESPACE +# define DPPP_NAMESPACE DPPP_ +#endif + +#define DPPP_CAT2(x,y) CAT2(x,y) +#define DPPP_(name) DPPP_CAT2(DPPP_NAMESPACE, name) + +#ifndef PERL_REVISION +# if !defined(__PATCHLEVEL_H_INCLUDED__) && !(defined(PATCHLEVEL) && defined(SUBVERSION)) +# define PERL_PATCHLEVEL_H_IMPLICIT +# include +# endif +# if !(defined(PERL_VERSION) || (defined(SUBVERSION) && defined(PATCHLEVEL))) +# include +# endif +# ifndef PERL_REVISION +# define PERL_REVISION (5) + /* Replace: 1 */ +# define PERL_VERSION PATCHLEVEL +# define PERL_SUBVERSION SUBVERSION + /* Replace PERL_PATCHLEVEL with PERL_VERSION */ + /* Replace: 0 */ +# endif +#endif + +#define _dpppDEC2BCD(dec) ((((dec)/100)<<8)|((((dec)%100)/10)<<4)|((dec)%10)) +#define PERL_BCDVERSION ((_dpppDEC2BCD(PERL_REVISION)<<24)|(_dpppDEC2BCD(PERL_VERSION)<<12)|_dpppDEC2BCD(PERL_SUBVERSION)) + +/* It is very unlikely that anyone will try to use this with Perl 6 + (or greater), but who knows. + */ +#if PERL_REVISION != 5 +# error ppport.h only works with Perl version 5 +#endif /* PERL_REVISION != 5 */ +#ifndef dTHR +# define dTHR dNOOP +#endif +#ifndef dTHX +# define dTHX dNOOP +#endif + +#ifndef dTHXa +# define dTHXa(x) dNOOP +#endif +#ifndef pTHX +# define pTHX void +#endif + +#ifndef pTHX_ +# define pTHX_ +#endif + +#ifndef aTHX +# define aTHX +#endif + +#ifndef aTHX_ +# define aTHX_ +#endif + +#if (PERL_BCDVERSION < 0x5006000) +# ifdef USE_THREADS +# define aTHXR thr +# define aTHXR_ thr, +# else +# define aTHXR +# define aTHXR_ +# endif +# define dTHXR dTHR +#else +# define aTHXR aTHX +# define aTHXR_ aTHX_ +# define dTHXR dTHX +#endif +#ifndef dTHXoa +# define dTHXoa(x) dTHXa(x) +#endif + +#ifdef I_LIMITS +# include +#endif + +#ifndef PERL_UCHAR_MIN +# define PERL_UCHAR_MIN ((unsigned char)0) +#endif + +#ifndef PERL_UCHAR_MAX +# ifdef UCHAR_MAX +# define PERL_UCHAR_MAX ((unsigned char)UCHAR_MAX) +# else +# ifdef MAXUCHAR +# define PERL_UCHAR_MAX ((unsigned char)MAXUCHAR) +# else +# define PERL_UCHAR_MAX ((unsigned char)~(unsigned)0) +# endif +# endif +#endif + +#ifndef PERL_USHORT_MIN +# define PERL_USHORT_MIN ((unsigned short)0) +#endif + +#ifndef PERL_USHORT_MAX +# ifdef USHORT_MAX +# define PERL_USHORT_MAX ((unsigned short)USHORT_MAX) +# else +# ifdef MAXUSHORT +# define PERL_USHORT_MAX ((unsigned short)MAXUSHORT) +# else +# ifdef USHRT_MAX +# define PERL_USHORT_MAX ((unsigned short)USHRT_MAX) +# else +# define PERL_USHORT_MAX ((unsigned short)~(unsigned)0) +# endif +# endif +# endif +#endif + +#ifndef PERL_SHORT_MAX +# ifdef SHORT_MAX +# define PERL_SHORT_MAX ((short)SHORT_MAX) +# else +# ifdef MAXSHORT /* Often used in */ +# define PERL_SHORT_MAX ((short)MAXSHORT) +# else +# ifdef SHRT_MAX +# define PERL_SHORT_MAX ((short)SHRT_MAX) +# else +# define PERL_SHORT_MAX ((short) (PERL_USHORT_MAX >> 1)) +# endif +# endif +# endif +#endif + +#ifndef PERL_SHORT_MIN +# ifdef SHORT_MIN +# define PERL_SHORT_MIN ((short)SHORT_MIN) +# else +# ifdef MINSHORT +# define PERL_SHORT_MIN ((short)MINSHORT) +# else +# ifdef SHRT_MIN +# define PERL_SHORT_MIN ((short)SHRT_MIN) +# else +# define PERL_SHORT_MIN (-PERL_SHORT_MAX - ((3 & -1) == 3)) +# endif +# endif +# endif +#endif + +#ifndef PERL_UINT_MAX +# ifdef UINT_MAX +# define PERL_UINT_MAX ((unsigned int)UINT_MAX) +# else +# ifdef MAXUINT +# define PERL_UINT_MAX ((unsigned int)MAXUINT) +# else +# define PERL_UINT_MAX (~(unsigned int)0) +# endif +# endif +#endif + +#ifndef PERL_UINT_MIN +# define PERL_UINT_MIN ((unsigned int)0) +#endif + +#ifndef PERL_INT_MAX +# ifdef INT_MAX +# define PERL_INT_MAX ((int)INT_MAX) +# else +# ifdef MAXINT /* Often used in */ +# define PERL_INT_MAX ((int)MAXINT) +# else +# define PERL_INT_MAX ((int)(PERL_UINT_MAX >> 1)) +# endif +# endif +#endif + +#ifndef PERL_INT_MIN +# ifdef INT_MIN +# define PERL_INT_MIN ((int)INT_MIN) +# else +# ifdef MININT +# define PERL_INT_MIN ((int)MININT) +# else +# define PERL_INT_MIN (-PERL_INT_MAX - ((3 & -1) == 3)) +# endif +# endif +#endif + +#ifndef PERL_ULONG_MAX +# ifdef ULONG_MAX +# define PERL_ULONG_MAX ((unsigned long)ULONG_MAX) +# else +# ifdef MAXULONG +# define PERL_ULONG_MAX ((unsigned long)MAXULONG) +# else +# define PERL_ULONG_MAX (~(unsigned long)0) +# endif +# endif +#endif + +#ifndef PERL_ULONG_MIN +# define PERL_ULONG_MIN ((unsigned long)0L) +#endif + +#ifndef PERL_LONG_MAX +# ifdef LONG_MAX +# define PERL_LONG_MAX ((long)LONG_MAX) +# else +# ifdef MAXLONG +# define PERL_LONG_MAX ((long)MAXLONG) +# else +# define PERL_LONG_MAX ((long) (PERL_ULONG_MAX >> 1)) +# endif +# endif +#endif + +#ifndef PERL_LONG_MIN +# ifdef LONG_MIN +# define PERL_LONG_MIN ((long)LONG_MIN) +# else +# ifdef MINLONG +# define PERL_LONG_MIN ((long)MINLONG) +# else +# define PERL_LONG_MIN (-PERL_LONG_MAX - ((3 & -1) == 3)) +# endif +# endif +#endif + +#if defined(HAS_QUAD) && (defined(convex) || defined(uts)) +# ifndef PERL_UQUAD_MAX +# ifdef ULONGLONG_MAX +# define PERL_UQUAD_MAX ((unsigned long long)ULONGLONG_MAX) +# else +# ifdef MAXULONGLONG +# define PERL_UQUAD_MAX ((unsigned long long)MAXULONGLONG) +# else +# define PERL_UQUAD_MAX (~(unsigned long long)0) +# endif +# endif +# endif + +# ifndef PERL_UQUAD_MIN +# define PERL_UQUAD_MIN ((unsigned long long)0L) +# endif + +# ifndef PERL_QUAD_MAX +# ifdef LONGLONG_MAX +# define PERL_QUAD_MAX ((long long)LONGLONG_MAX) +# else +# ifdef MAXLONGLONG +# define PERL_QUAD_MAX ((long long)MAXLONGLONG) +# else +# define PERL_QUAD_MAX ((long long) (PERL_UQUAD_MAX >> 1)) +# endif +# endif +# endif + +# ifndef PERL_QUAD_MIN +# ifdef LONGLONG_MIN +# define PERL_QUAD_MIN ((long long)LONGLONG_MIN) +# else +# ifdef MINLONGLONG +# define PERL_QUAD_MIN ((long long)MINLONGLONG) +# else +# define PERL_QUAD_MIN (-PERL_QUAD_MAX - ((3 & -1) == 3)) +# endif +# endif +# endif +#endif + +/* This is based on code from 5.003 perl.h */ +#ifdef HAS_QUAD +# ifdef cray +#ifndef IVTYPE +# define IVTYPE int +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_INT_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_INT_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_UINT_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_UINT_MAX +#endif + +# ifdef INTSIZE +#ifndef IVSIZE +# define IVSIZE INTSIZE +#endif + +# endif +# else +# if defined(convex) || defined(uts) +#ifndef IVTYPE +# define IVTYPE long long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_QUAD_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_QUAD_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_UQUAD_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_UQUAD_MAX +#endif + +# ifdef LONGLONGSIZE +#ifndef IVSIZE +# define IVSIZE LONGLONGSIZE +#endif + +# endif +# else +#ifndef IVTYPE +# define IVTYPE long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_LONG_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_LONG_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_ULONG_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_ULONG_MAX +#endif + +# ifdef LONGSIZE +#ifndef IVSIZE +# define IVSIZE LONGSIZE +#endif + +# endif +# endif +# endif +#ifndef IVSIZE +# define IVSIZE 8 +#endif + +#ifndef PERL_QUAD_MIN +# define PERL_QUAD_MIN IV_MIN +#endif + +#ifndef PERL_QUAD_MAX +# define PERL_QUAD_MAX IV_MAX +#endif + +#ifndef PERL_UQUAD_MIN +# define PERL_UQUAD_MIN UV_MIN +#endif + +#ifndef PERL_UQUAD_MAX +# define PERL_UQUAD_MAX UV_MAX +#endif + +#else +#ifndef IVTYPE +# define IVTYPE long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_LONG_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_LONG_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_ULONG_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_ULONG_MAX +#endif + +#endif + +#ifndef IVSIZE +# ifdef LONGSIZE +# define IVSIZE LONGSIZE +# else +# define IVSIZE 4 /* A bold guess, but the best we can make. */ +# endif +#endif +#ifndef UVTYPE +# define UVTYPE unsigned IVTYPE +#endif + +#ifndef UVSIZE +# define UVSIZE IVSIZE +#endif +#ifndef sv_setuv +# define sv_setuv(sv, uv) \ + STMT_START { \ + UV TeMpUv = uv; \ + if (TeMpUv <= IV_MAX) \ + sv_setiv(sv, TeMpUv); \ + else \ + sv_setnv(sv, (double)TeMpUv); \ + } STMT_END +#endif +#ifndef newSVuv +# define newSVuv(uv) ((uv) <= IV_MAX ? newSViv((IV)uv) : newSVnv((NV)uv)) +#endif +#ifndef sv_2uv +# define sv_2uv(sv) ((PL_Sv = (sv)), (UV) (SvNOK(PL_Sv) ? SvNV(PL_Sv) : sv_2nv(PL_Sv))) +#endif + +#ifndef SvUVX +# define SvUVX(sv) ((UV)SvIVX(sv)) +#endif + +#ifndef SvUVXx +# define SvUVXx(sv) SvUVX(sv) +#endif + +#ifndef SvUV +# define SvUV(sv) (SvIOK(sv) ? SvUVX(sv) : sv_2uv(sv)) +#endif + +#ifndef SvUVx +# define SvUVx(sv) ((PL_Sv = (sv)), SvUV(PL_Sv)) +#endif + +/* Hint: sv_uv + * Always use the SvUVx() macro instead of sv_uv(). + */ +#ifndef sv_uv +# define sv_uv(sv) SvUVx(sv) +#endif + +#if !defined(SvUOK) && defined(SvIOK_UV) +# define SvUOK(sv) SvIOK_UV(sv) +#endif +#ifndef XST_mUV +# define XST_mUV(i,v) (ST(i) = sv_2mortal(newSVuv(v)) ) +#endif + +#ifndef XSRETURN_UV +# define XSRETURN_UV(v) STMT_START { XST_mUV(0,v); XSRETURN(1); } STMT_END +#endif +#ifndef PUSHu +# define PUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); PUSHTARG; } STMT_END +#endif + +#ifndef XPUSHu +# define XPUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); XPUSHTARG; } STMT_END +#endif + +#ifdef HAS_MEMCMP +#ifndef memNE +# define memNE(s1,s2,l) (memcmp(s1,s2,l)) +#endif + +#ifndef memEQ +# define memEQ(s1,s2,l) (!memcmp(s1,s2,l)) +#endif + +#else +#ifndef memNE +# define memNE(s1,s2,l) (bcmp(s1,s2,l)) +#endif + +#ifndef memEQ +# define memEQ(s1,s2,l) (!bcmp(s1,s2,l)) +#endif + +#endif +#ifndef MoveD +# define MoveD(s,d,n,t) memmove((char*)(d),(char*)(s), (n) * sizeof(t)) +#endif + +#ifndef CopyD +# define CopyD(s,d,n,t) memcpy((char*)(d),(char*)(s), (n) * sizeof(t)) +#endif + +#ifdef HAS_MEMSET +#ifndef ZeroD +# define ZeroD(d,n,t) memzero((char*)(d), (n) * sizeof(t)) +#endif + +#else +#ifndef ZeroD +# define ZeroD(d,n,t) ((void)memzero((char*)(d), (n) * sizeof(t)), d) +#endif + +#endif +#ifndef PoisonWith +# define PoisonWith(d,n,t,b) (void)memset((char*)(d), (U8)(b), (n) * sizeof(t)) +#endif + +#ifndef PoisonNew +# define PoisonNew(d,n,t) PoisonWith(d,n,t,0xAB) +#endif + +#ifndef PoisonFree +# define PoisonFree(d,n,t) PoisonWith(d,n,t,0xEF) +#endif + +#ifndef Poison +# define Poison(d,n,t) PoisonFree(d,n,t) +#endif +#ifndef Newx +# define Newx(v,n,t) New(0,v,n,t) +#endif + +#ifndef Newxc +# define Newxc(v,n,t,c) Newc(0,v,n,t,c) +#endif + +#ifndef Newxz +# define Newxz(v,n,t) Newz(0,v,n,t) +#endif + +#ifndef PERL_UNUSED_DECL +# ifdef HASATTRIBUTE +# if (defined(__GNUC__) && defined(__cplusplus)) || defined(__INTEL_COMPILER) +# define PERL_UNUSED_DECL +# else +# define PERL_UNUSED_DECL __attribute__((unused)) +# endif +# else +# define PERL_UNUSED_DECL +# endif +#endif + +#ifndef PERL_UNUSED_ARG +# if defined(lint) && defined(S_SPLINT_S) /* www.splint.org */ +# include +# define PERL_UNUSED_ARG(x) NOTE(ARGUNUSED(x)) +# else +# define PERL_UNUSED_ARG(x) ((void)x) +# endif +#endif + +#ifndef PERL_UNUSED_VAR +# define PERL_UNUSED_VAR(x) ((void)x) +#endif + +#ifndef PERL_UNUSED_CONTEXT +# ifdef USE_ITHREADS +# define PERL_UNUSED_CONTEXT PERL_UNUSED_ARG(my_perl) +# else +# define PERL_UNUSED_CONTEXT +# endif +#endif +#ifndef NOOP +# define NOOP /*EMPTY*/(void)0 +#endif + +#ifndef dNOOP +# define dNOOP extern int /*@unused@*/ Perl___notused PERL_UNUSED_DECL +#endif + +#ifndef NVTYPE +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) +# define NVTYPE long double +# else +# define NVTYPE double +# endif +typedef NVTYPE NV; +#endif + +#ifndef INT2PTR +# if (IVSIZE == PTRSIZE) && (UVSIZE == PTRSIZE) +# define PTRV UV +# define INT2PTR(any,d) (any)(d) +# else +# if PTRSIZE == LONGSIZE +# define PTRV unsigned long +# else +# define PTRV unsigned +# endif +# define INT2PTR(any,d) (any)(PTRV)(d) +# endif +#endif + +#ifndef PTR2ul +# if PTRSIZE == LONGSIZE +# define PTR2ul(p) (unsigned long)(p) +# else +# define PTR2ul(p) INT2PTR(unsigned long,p) +# endif +#endif +#ifndef PTR2nat +# define PTR2nat(p) (PTRV)(p) +#endif + +#ifndef NUM2PTR +# define NUM2PTR(any,d) (any)PTR2nat(d) +#endif + +#ifndef PTR2IV +# define PTR2IV(p) INT2PTR(IV,p) +#endif + +#ifndef PTR2UV +# define PTR2UV(p) INT2PTR(UV,p) +#endif + +#ifndef PTR2NV +# define PTR2NV(p) NUM2PTR(NV,p) +#endif + +#undef START_EXTERN_C +#undef END_EXTERN_C +#undef EXTERN_C +#ifdef __cplusplus +# define START_EXTERN_C extern "C" { +# define END_EXTERN_C } +# define EXTERN_C extern "C" +#else +# define START_EXTERN_C +# define END_EXTERN_C +# define EXTERN_C extern +#endif + +#if defined(PERL_GCC_PEDANTIC) +# ifndef PERL_GCC_BRACE_GROUPS_FORBIDDEN +# define PERL_GCC_BRACE_GROUPS_FORBIDDEN +# endif +#endif + +#if defined(__GNUC__) && !defined(PERL_GCC_BRACE_GROUPS_FORBIDDEN) && !defined(__cplusplus) +# ifndef PERL_USE_GCC_BRACE_GROUPS +# define PERL_USE_GCC_BRACE_GROUPS +# endif +#endif + +#undef STMT_START +#undef STMT_END +#ifdef PERL_USE_GCC_BRACE_GROUPS +# define STMT_START (void)( /* gcc supports ``({ STATEMENTS; })'' */ +# define STMT_END ) +#else +# if defined(VOIDFLAGS) && (VOIDFLAGS) && (defined(sun) || defined(__sun__)) && !defined(__GNUC__) +# define STMT_START if (1) +# define STMT_END else (void)0 +# else +# define STMT_START do +# define STMT_END while (0) +# endif +#endif +#ifndef boolSV +# define boolSV(b) ((b) ? &PL_sv_yes : &PL_sv_no) +#endif + +/* DEFSV appears first in 5.004_56 */ +#ifndef DEFSV +# define DEFSV GvSV(PL_defgv) +#endif + +#ifndef SAVE_DEFSV +# define SAVE_DEFSV SAVESPTR(GvSV(PL_defgv)) +#endif + +#ifndef DEFSV_set +# define DEFSV_set(sv) (DEFSV = (sv)) +#endif + +/* Older perls (<=5.003) lack AvFILLp */ +#ifndef AvFILLp +# define AvFILLp AvFILL +#endif +#ifndef ERRSV +# define ERRSV get_sv("@",FALSE) +#endif + +/* Hint: gv_stashpvn + * This function's backport doesn't support the length parameter, but + * rather ignores it. Portability can only be ensured if the length + * parameter is used for speed reasons, but the length can always be + * correctly computed from the string argument. + */ +#ifndef gv_stashpvn +# define gv_stashpvn(str,len,create) gv_stashpv(str,create) +#endif + +/* Replace: 1 */ +#ifndef get_cv +# define get_cv perl_get_cv +#endif + +#ifndef get_sv +# define get_sv perl_get_sv +#endif + +#ifndef get_av +# define get_av perl_get_av +#endif + +#ifndef get_hv +# define get_hv perl_get_hv +#endif + +/* Replace: 0 */ +#ifndef dUNDERBAR +# define dUNDERBAR dNOOP +#endif + +#ifndef UNDERBAR +# define UNDERBAR DEFSV +#endif +#ifndef dAX +# define dAX I32 ax = MARK - PL_stack_base + 1 +#endif + +#ifndef dITEMS +# define dITEMS I32 items = SP - MARK +#endif +#ifndef dXSTARG +# define dXSTARG SV * targ = sv_newmortal() +#endif +#ifndef dAXMARK +# define dAXMARK I32 ax = POPMARK; \ + register SV ** const mark = PL_stack_base + ax++ +#endif +#ifndef XSprePUSH +# define XSprePUSH (sp = PL_stack_base + ax - 1) +#endif + +#if (PERL_BCDVERSION < 0x5005000) +# undef XSRETURN +# define XSRETURN(off) \ + STMT_START { \ + PL_stack_sp = PL_stack_base + ax + ((off) - 1); \ + return; \ + } STMT_END +#endif +#ifndef PERL_ABS +# define PERL_ABS(x) ((x) < 0 ? -(x) : (x)) +#endif +#ifndef dVAR +# define dVAR dNOOP +#endif +#ifndef SVf +# define SVf "_" +#endif +#ifndef UTF8_MAXBYTES +# define UTF8_MAXBYTES UTF8_MAXLEN +#endif +#ifndef CPERLscope +# define CPERLscope(x) x +#endif +#ifndef PERL_HASH +# define PERL_HASH(hash,str,len) \ + STMT_START { \ + const char *s_PeRlHaSh = str; \ + I32 i_PeRlHaSh = len; \ + U32 hash_PeRlHaSh = 0; \ + while (i_PeRlHaSh--) \ + hash_PeRlHaSh = hash_PeRlHaSh * 33 + *s_PeRlHaSh++; \ + (hash) = hash_PeRlHaSh; \ + } STMT_END +#endif + +#ifndef PERLIO_FUNCS_DECL +# ifdef PERLIO_FUNCS_CONST +# define PERLIO_FUNCS_DECL(funcs) const PerlIO_funcs funcs +# define PERLIO_FUNCS_CAST(funcs) (PerlIO_funcs*)(funcs) +# else +# define PERLIO_FUNCS_DECL(funcs) PerlIO_funcs funcs +# define PERLIO_FUNCS_CAST(funcs) (funcs) +# endif +#endif + +/* provide these typedefs for older perls */ +#if (PERL_BCDVERSION < 0x5009003) + +# ifdef ARGSproto +typedef OP* (CPERLscope(*Perl_ppaddr_t))(ARGSproto); +# else +typedef OP* (CPERLscope(*Perl_ppaddr_t))(pTHX); +# endif + +typedef OP* (CPERLscope(*Perl_check_t)) (pTHX_ OP*); + +#endif +#ifndef isPSXSPC +# define isPSXSPC(c) (isSPACE(c) || (c) == '\v') +#endif + +#ifndef isBLANK +# define isBLANK(c) ((c) == ' ' || (c) == '\t') +#endif + +#ifdef EBCDIC +#ifndef isALNUMC +# define isALNUMC(c) isalnum(c) +#endif + +#ifndef isASCII +# define isASCII(c) isascii(c) +#endif + +#ifndef isCNTRL +# define isCNTRL(c) iscntrl(c) +#endif + +#ifndef isGRAPH +# define isGRAPH(c) isgraph(c) +#endif + +#ifndef isPRINT +# define isPRINT(c) isprint(c) +#endif + +#ifndef isPUNCT +# define isPUNCT(c) ispunct(c) +#endif + +#ifndef isXDIGIT +# define isXDIGIT(c) isxdigit(c) +#endif + +#else +# if (PERL_BCDVERSION < 0x5010000) +/* Hint: isPRINT + * The implementation in older perl versions includes all of the + * isSPACE() characters, which is wrong. The version provided by + * Devel::PPPort always overrides a present buggy version. + */ +# undef isPRINT +# endif +#ifndef isALNUMC +# define isALNUMC(c) (isALPHA(c) || isDIGIT(c)) +#endif + +#ifndef isASCII +# define isASCII(c) ((c) <= 127) +#endif + +#ifndef isCNTRL +# define isCNTRL(c) ((c) < ' ' || (c) == 127) +#endif + +#ifndef isGRAPH +# define isGRAPH(c) (isALNUM(c) || isPUNCT(c)) +#endif + +#ifndef isPRINT +# define isPRINT(c) (((c) >= 32 && (c) < 127)) +#endif + +#ifndef isPUNCT +# define isPUNCT(c) (((c) >= 33 && (c) <= 47) || ((c) >= 58 && (c) <= 64) || ((c) >= 91 && (c) <= 96) || ((c) >= 123 && (c) <= 126)) +#endif + +#ifndef isXDIGIT +# define isXDIGIT(c) (isDIGIT(c) || ((c) >= 'a' && (c) <= 'f') || ((c) >= 'A' && (c) <= 'F')) +#endif + +#endif + +#ifndef PERL_SIGNALS_UNSAFE_FLAG + +#define PERL_SIGNALS_UNSAFE_FLAG 0x0001 + +#if (PERL_BCDVERSION < 0x5008000) +# define D_PPP_PERL_SIGNALS_INIT PERL_SIGNALS_UNSAFE_FLAG +#else +# define D_PPP_PERL_SIGNALS_INIT 0 +#endif + +#if defined(NEED_PL_signals) +static U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; +#elif defined(NEED_PL_signals_GLOBAL) +U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; +#else +extern U32 DPPP_(my_PL_signals); +#endif +#define PL_signals DPPP_(my_PL_signals) + +#endif + +/* Hint: PL_ppaddr + * Calling an op via PL_ppaddr requires passing a context argument + * for threaded builds. Since the context argument is different for + * 5.005 perls, you can use aTHXR (supplied by ppport.h), which will + * automatically be defined as the correct argument. + */ + +#if (PERL_BCDVERSION <= 0x5005005) +/* Replace: 1 */ +# define PL_ppaddr ppaddr +# define PL_no_modify no_modify +/* Replace: 0 */ +#endif + +#if (PERL_BCDVERSION <= 0x5004005) +/* Replace: 1 */ +# define PL_DBsignal DBsignal +# define PL_DBsingle DBsingle +# define PL_DBsub DBsub +# define PL_DBtrace DBtrace +# define PL_Sv Sv +# define PL_bufend bufend +# define PL_bufptr bufptr +# define PL_compiling compiling +# define PL_copline copline +# define PL_curcop curcop +# define PL_curstash curstash +# define PL_debstash debstash +# define PL_defgv defgv +# define PL_diehook diehook +# define PL_dirty dirty +# define PL_dowarn dowarn +# define PL_errgv errgv +# define PL_expect expect +# define PL_hexdigit hexdigit +# define PL_hints hints +# define PL_laststatval laststatval +# define PL_lex_state lex_state +# define PL_lex_stuff lex_stuff +# define PL_linestr linestr +# define PL_na na +# define PL_perl_destruct_level perl_destruct_level +# define PL_perldb perldb +# define PL_rsfp_filters rsfp_filters +# define PL_rsfp rsfp +# define PL_stack_base stack_base +# define PL_stack_sp stack_sp +# define PL_statcache statcache +# define PL_stdingv stdingv +# define PL_sv_arenaroot sv_arenaroot +# define PL_sv_no sv_no +# define PL_sv_undef sv_undef +# define PL_sv_yes sv_yes +# define PL_tainted tainted +# define PL_tainting tainting +# define PL_tokenbuf tokenbuf +/* Replace: 0 */ +#endif + +/* Warning: PL_parser + * For perl versions earlier than 5.9.5, this is an always + * non-NULL dummy. Also, it cannot be dereferenced. Don't + * use it if you can avoid is and unless you absolutely know + * what you're doing. + * If you always check that PL_parser is non-NULL, you can + * define DPPP_PL_parser_NO_DUMMY to avoid the creation of + * a dummy parser structure. + */ + +#if (PERL_BCDVERSION >= 0x5009005) +# ifdef DPPP_PL_parser_NO_DUMMY +# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ + (croak("panic: PL_parser == NULL in %s:%d", \ + __FILE__, __LINE__), (yy_parser *) NULL))->var) +# else +# ifdef DPPP_PL_parser_NO_DUMMY_WARNING +# define D_PPP_parser_dummy_warning(var) +# else +# define D_PPP_parser_dummy_warning(var) \ + warn("warning: dummy PL_" #var " used in %s:%d", __FILE__, __LINE__), +# endif +# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ + (D_PPP_parser_dummy_warning(var) &DPPP_(dummy_PL_parser)))->var) +#if defined(NEED_PL_parser) +static yy_parser DPPP_(dummy_PL_parser); +#elif defined(NEED_PL_parser_GLOBAL) +yy_parser DPPP_(dummy_PL_parser); +#else +extern yy_parser DPPP_(dummy_PL_parser); +#endif + +# endif + +/* PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf depends on PL_parser */ +/* Warning: PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf + * Do not use this variable unless you know exactly what you're + * doint. It is internal to the perl parser and may change or even + * be removed in the future. As of perl 5.9.5, you have to check + * for (PL_parser != NULL) for this variable to have any effect. + * An always non-NULL PL_parser dummy is provided for earlier + * perl versions. + * If PL_parser is NULL when you try to access this variable, a + * dummy is being accessed instead and a warning is issued unless + * you define DPPP_PL_parser_NO_DUMMY_WARNING. + * If DPPP_PL_parser_NO_DUMMY is defined, the code trying to access + * this variable will croak with a panic message. + */ + +# define PL_expect D_PPP_my_PL_parser_var(expect) +# define PL_copline D_PPP_my_PL_parser_var(copline) +# define PL_rsfp D_PPP_my_PL_parser_var(rsfp) +# define PL_rsfp_filters D_PPP_my_PL_parser_var(rsfp_filters) +# define PL_linestr D_PPP_my_PL_parser_var(linestr) +# define PL_bufptr D_PPP_my_PL_parser_var(bufptr) +# define PL_bufend D_PPP_my_PL_parser_var(bufend) +# define PL_lex_state D_PPP_my_PL_parser_var(lex_state) +# define PL_lex_stuff D_PPP_my_PL_parser_var(lex_stuff) +# define PL_tokenbuf D_PPP_my_PL_parser_var(tokenbuf) + +#else + +/* ensure that PL_parser != NULL and cannot be dereferenced */ +# define PL_parser ((void *) 1) + +#endif +#ifndef mPUSHs +# define mPUSHs(s) PUSHs(sv_2mortal(s)) +#endif + +#ifndef PUSHmortal +# define PUSHmortal PUSHs(sv_newmortal()) +#endif + +#ifndef mPUSHp +# define mPUSHp(p,l) sv_setpvn(PUSHmortal, (p), (l)) +#endif + +#ifndef mPUSHn +# define mPUSHn(n) sv_setnv(PUSHmortal, (NV)(n)) +#endif + +#ifndef mPUSHi +# define mPUSHi(i) sv_setiv(PUSHmortal, (IV)(i)) +#endif + +#ifndef mPUSHu +# define mPUSHu(u) sv_setuv(PUSHmortal, (UV)(u)) +#endif +#ifndef mXPUSHs +# define mXPUSHs(s) XPUSHs(sv_2mortal(s)) +#endif + +#ifndef XPUSHmortal +# define XPUSHmortal XPUSHs(sv_newmortal()) +#endif + +#ifndef mXPUSHp +# define mXPUSHp(p,l) STMT_START { EXTEND(sp,1); sv_setpvn(PUSHmortal, (p), (l)); } STMT_END +#endif + +#ifndef mXPUSHn +# define mXPUSHn(n) STMT_START { EXTEND(sp,1); sv_setnv(PUSHmortal, (NV)(n)); } STMT_END +#endif + +#ifndef mXPUSHi +# define mXPUSHi(i) STMT_START { EXTEND(sp,1); sv_setiv(PUSHmortal, (IV)(i)); } STMT_END +#endif + +#ifndef mXPUSHu +# define mXPUSHu(u) STMT_START { EXTEND(sp,1); sv_setuv(PUSHmortal, (UV)(u)); } STMT_END +#endif + +/* Replace: 1 */ +#ifndef call_sv +# define call_sv perl_call_sv +#endif + +#ifndef call_pv +# define call_pv perl_call_pv +#endif + +#ifndef call_argv +# define call_argv perl_call_argv +#endif + +#ifndef call_method +# define call_method perl_call_method +#endif +#ifndef eval_sv +# define eval_sv perl_eval_sv +#endif + +/* Replace: 0 */ +#ifndef PERL_LOADMOD_DENY +# define PERL_LOADMOD_DENY 0x1 +#endif + +#ifndef PERL_LOADMOD_NOIMPORT +# define PERL_LOADMOD_NOIMPORT 0x2 +#endif + +#ifndef PERL_LOADMOD_IMPORT_OPS +# define PERL_LOADMOD_IMPORT_OPS 0x4 +#endif + +#ifndef G_METHOD +# define G_METHOD 64 +# ifdef call_sv +# undef call_sv +# endif +# if (PERL_BCDVERSION < 0x5006000) +# define call_sv(sv, flags) ((flags) & G_METHOD ? perl_call_method((char *) SvPV_nolen_const(sv), \ + (flags) & ~G_METHOD) : perl_call_sv(sv, flags)) +# else +# define call_sv(sv, flags) ((flags) & G_METHOD ? Perl_call_method(aTHX_ (char *) SvPV_nolen_const(sv), \ + (flags) & ~G_METHOD) : Perl_call_sv(aTHX_ sv, flags)) +# endif +#endif + +/* Replace perl_eval_pv with eval_pv */ + +#ifndef eval_pv +#if defined(NEED_eval_pv) +static SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); +static +#else +extern SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); +#endif + +#ifdef eval_pv +# undef eval_pv +#endif +#define eval_pv(a,b) DPPP_(my_eval_pv)(aTHX_ a,b) +#define Perl_eval_pv DPPP_(my_eval_pv) + +#if defined(NEED_eval_pv) || defined(NEED_eval_pv_GLOBAL) + +SV* +DPPP_(my_eval_pv)(char *p, I32 croak_on_error) +{ + dSP; + SV* sv = newSVpv(p, 0); + + PUSHMARK(sp); + eval_sv(sv, G_SCALAR); + SvREFCNT_dec(sv); + + SPAGAIN; + sv = POPs; + PUTBACK; + + if (croak_on_error && SvTRUE(GvSV(errgv))) + croak(SvPVx(GvSV(errgv), na)); + + return sv; +} + +#endif +#endif + +#ifndef vload_module +#if defined(NEED_vload_module) +static void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); +static +#else +extern void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); +#endif + +#ifdef vload_module +# undef vload_module +#endif +#define vload_module(a,b,c,d) DPPP_(my_vload_module)(aTHX_ a,b,c,d) +#define Perl_vload_module DPPP_(my_vload_module) + +#if defined(NEED_vload_module) || defined(NEED_vload_module_GLOBAL) + +void +DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args) +{ + dTHR; + dVAR; + OP *veop, *imop; + + OP * const modname = newSVOP(OP_CONST, 0, name); + /* 5.005 has a somewhat hacky force_normal that doesn't croak on + SvREADONLY() if PL_compling is true. Current perls take care in + ck_require() to correctly turn off SvREADONLY before calling + force_normal_flags(). This seems a better fix than fudging PL_compling + */ + SvREADONLY_off(((SVOP*)modname)->op_sv); + modname->op_private |= OPpCONST_BARE; + if (ver) { + veop = newSVOP(OP_CONST, 0, ver); + } + else + veop = NULL; + if (flags & PERL_LOADMOD_NOIMPORT) { + imop = sawparens(newNULLLIST()); + } + else if (flags & PERL_LOADMOD_IMPORT_OPS) { + imop = va_arg(*args, OP*); + } + else { + SV *sv; + imop = NULL; + sv = va_arg(*args, SV*); + while (sv) { + imop = append_elem(OP_LIST, imop, newSVOP(OP_CONST, 0, sv)); + sv = va_arg(*args, SV*); + } + } + { + const line_t ocopline = PL_copline; + COP * const ocurcop = PL_curcop; + const int oexpect = PL_expect; + +#if (PERL_BCDVERSION >= 0x5004000) + utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(FALSE, 0), + veop, modname, imop); +#else + utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(), + modname, imop); +#endif + PL_expect = oexpect; + PL_copline = ocopline; + PL_curcop = ocurcop; + } +} + +#endif +#endif + +#ifndef load_module +#if defined(NEED_load_module) +static void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); +static +#else +extern void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); +#endif + +#ifdef load_module +# undef load_module +#endif +#define load_module DPPP_(my_load_module) +#define Perl_load_module DPPP_(my_load_module) + +#if defined(NEED_load_module) || defined(NEED_load_module_GLOBAL) + +void +DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...) +{ + va_list args; + va_start(args, ver); + vload_module(flags, name, ver, &args); + va_end(args); +} + +#endif +#endif +#ifndef newRV_inc +# define newRV_inc(sv) newRV(sv) /* Replace */ +#endif + +#ifndef newRV_noinc +#if defined(NEED_newRV_noinc) +static SV * DPPP_(my_newRV_noinc)(SV *sv); +static +#else +extern SV * DPPP_(my_newRV_noinc)(SV *sv); +#endif + +#ifdef newRV_noinc +# undef newRV_noinc +#endif +#define newRV_noinc(a) DPPP_(my_newRV_noinc)(aTHX_ a) +#define Perl_newRV_noinc DPPP_(my_newRV_noinc) + +#if defined(NEED_newRV_noinc) || defined(NEED_newRV_noinc_GLOBAL) +SV * +DPPP_(my_newRV_noinc)(SV *sv) +{ + SV *rv = (SV *)newRV(sv); + SvREFCNT_dec(sv); + return rv; +} +#endif +#endif + +/* Hint: newCONSTSUB + * Returns a CV* as of perl-5.7.1. This return value is not supported + * by Devel::PPPort. + */ + +/* newCONSTSUB from IO.xs is in the core starting with 5.004_63 */ +#if (PERL_BCDVERSION < 0x5004063) && (PERL_BCDVERSION != 0x5004005) +#if defined(NEED_newCONSTSUB) +static void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); +static +#else +extern void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); +#endif + +#ifdef newCONSTSUB +# undef newCONSTSUB +#endif +#define newCONSTSUB(a,b,c) DPPP_(my_newCONSTSUB)(aTHX_ a,b,c) +#define Perl_newCONSTSUB DPPP_(my_newCONSTSUB) + +#if defined(NEED_newCONSTSUB) || defined(NEED_newCONSTSUB_GLOBAL) + +/* This is just a trick to avoid a dependency of newCONSTSUB on PL_parser */ +/* (There's no PL_parser in perl < 5.005, so this is completely safe) */ +#define D_PPP_PL_copline PL_copline + +void +DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv) +{ + U32 oldhints = PL_hints; + HV *old_cop_stash = PL_curcop->cop_stash; + HV *old_curstash = PL_curstash; + line_t oldline = PL_curcop->cop_line; + PL_curcop->cop_line = D_PPP_PL_copline; + + PL_hints &= ~HINT_BLOCK_SCOPE; + if (stash) + PL_curstash = PL_curcop->cop_stash = stash; + + newSUB( + +#if (PERL_BCDVERSION < 0x5003022) + start_subparse(), +#elif (PERL_BCDVERSION == 0x5003022) + start_subparse(0), +#else /* 5.003_23 onwards */ + start_subparse(FALSE, 0), +#endif + + newSVOP(OP_CONST, 0, newSVpv((char *) name, 0)), + newSVOP(OP_CONST, 0, &PL_sv_no), /* SvPV(&PL_sv_no) == "" -- GMB */ + newSTATEOP(0, Nullch, newSVOP(OP_CONST, 0, sv)) + ); + + PL_hints = oldhints; + PL_curcop->cop_stash = old_cop_stash; + PL_curstash = old_curstash; + PL_curcop->cop_line = oldline; +} +#endif +#endif + +/* + * Boilerplate macros for initializing and accessing interpreter-local + * data from C. All statics in extensions should be reworked to use + * this, if you want to make the extension thread-safe. See ext/re/re.xs + * for an example of the use of these macros. + * + * Code that uses these macros is responsible for the following: + * 1. #define MY_CXT_KEY to a unique string, e.g. "DynaLoader_guts" + * 2. Declare a typedef named my_cxt_t that is a structure that contains + * all the data that needs to be interpreter-local. + * 3. Use the START_MY_CXT macro after the declaration of my_cxt_t. + * 4. Use the MY_CXT_INIT macro such that it is called exactly once + * (typically put in the BOOT: section). + * 5. Use the members of the my_cxt_t structure everywhere as + * MY_CXT.member. + * 6. Use the dMY_CXT macro (a declaration) in all the functions that + * access MY_CXT. + */ + +#if defined(MULTIPLICITY) || defined(PERL_OBJECT) || \ + defined(PERL_CAPI) || defined(PERL_IMPLICIT_CONTEXT) + +#ifndef START_MY_CXT + +/* This must appear in all extensions that define a my_cxt_t structure, + * right after the definition (i.e. at file scope). The non-threads + * case below uses it to declare the data as static. */ +#define START_MY_CXT + +#if (PERL_BCDVERSION < 0x5004068) +/* Fetches the SV that keeps the per-interpreter data. */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = get_sv(MY_CXT_KEY, FALSE) +#else /* >= perl5.004_68 */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = *hv_fetch(PL_modglobal, MY_CXT_KEY, \ + sizeof(MY_CXT_KEY)-1, TRUE) +#endif /* < perl5.004_68 */ + +/* This declaration should be used within all functions that use the + * interpreter-local data. */ +#define dMY_CXT \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = INT2PTR(my_cxt_t*,SvUV(my_cxt_sv)) + +/* Creates and zeroes the per-interpreter data. + * (We allocate my_cxtp in a Perl SV so that it will be released when + * the interpreter goes away.) */ +#define MY_CXT_INIT \ + dMY_CXT_SV; \ + /* newSV() allocates one more than needed */ \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Zero(my_cxtp, 1, my_cxt_t); \ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) + +/* This macro must be used to access members of the my_cxt_t structure. + * e.g. MYCXT.some_data */ +#define MY_CXT (*my_cxtp) + +/* Judicious use of these macros can reduce the number of times dMY_CXT + * is used. Use is similar to pTHX, aTHX etc. */ +#define pMY_CXT my_cxt_t *my_cxtp +#define pMY_CXT_ pMY_CXT, +#define _pMY_CXT ,pMY_CXT +#define aMY_CXT my_cxtp +#define aMY_CXT_ aMY_CXT, +#define _aMY_CXT ,aMY_CXT + +#endif /* START_MY_CXT */ + +#ifndef MY_CXT_CLONE +/* Clones the per-interpreter data. */ +#define MY_CXT_CLONE \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Copy(INT2PTR(my_cxt_t*, SvUV(my_cxt_sv)), my_cxtp, 1, my_cxt_t);\ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) +#endif + +#else /* single interpreter */ + +#ifndef START_MY_CXT + +#define START_MY_CXT static my_cxt_t my_cxt; +#define dMY_CXT_SV dNOOP +#define dMY_CXT dNOOP +#define MY_CXT_INIT NOOP +#define MY_CXT my_cxt + +#define pMY_CXT void +#define pMY_CXT_ +#define _pMY_CXT +#define aMY_CXT +#define aMY_CXT_ +#define _aMY_CXT + +#endif /* START_MY_CXT */ + +#ifndef MY_CXT_CLONE +#define MY_CXT_CLONE NOOP +#endif + +#endif + +#ifndef IVdf +# if IVSIZE == LONGSIZE +# define IVdf "ld" +# define UVuf "lu" +# define UVof "lo" +# define UVxf "lx" +# define UVXf "lX" +# else +# if IVSIZE == INTSIZE +# define IVdf "d" +# define UVuf "u" +# define UVof "o" +# define UVxf "x" +# define UVXf "X" +# endif +# endif +#endif + +#ifndef NVef +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) && \ + defined(PERL_PRIfldbl) && (PERL_BCDVERSION != 0x5006000) + /* Not very likely, but let's try anyway. */ +# define NVef PERL_PRIeldbl +# define NVff PERL_PRIfldbl +# define NVgf PERL_PRIgldbl +# else +# define NVef "e" +# define NVff "f" +# define NVgf "g" +# endif +#endif + +#ifndef SvREFCNT_inc +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + if (_sv) \ + (SvREFCNT(_sv))++; \ + _sv; \ + }) +# else +# define SvREFCNT_inc(sv) \ + ((PL_Sv=(SV*)(sv)) ? (++(SvREFCNT(PL_Sv)),PL_Sv) : NULL) +# endif +#endif + +#ifndef SvREFCNT_inc_simple +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_simple(sv) \ + ({ \ + if (sv) \ + (SvREFCNT(sv))++; \ + (SV *)(sv); \ + }) +# else +# define SvREFCNT_inc_simple(sv) \ + ((sv) ? (SvREFCNT(sv)++,(SV*)(sv)) : NULL) +# endif +#endif + +#ifndef SvREFCNT_inc_NN +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_NN(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + SvREFCNT(_sv)++; \ + _sv; \ + }) +# else +# define SvREFCNT_inc_NN(sv) \ + (PL_Sv=(SV*)(sv),++(SvREFCNT(PL_Sv)),PL_Sv) +# endif +#endif + +#ifndef SvREFCNT_inc_void +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_void(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + if (_sv) \ + (void)(SvREFCNT(_sv)++); \ + }) +# else +# define SvREFCNT_inc_void(sv) \ + (void)((PL_Sv=(SV*)(sv)) ? ++(SvREFCNT(PL_Sv)) : 0) +# endif +#endif +#ifndef SvREFCNT_inc_simple_void +# define SvREFCNT_inc_simple_void(sv) STMT_START { if (sv) SvREFCNT(sv)++; } STMT_END +#endif + +#ifndef SvREFCNT_inc_simple_NN +# define SvREFCNT_inc_simple_NN(sv) (++SvREFCNT(sv), (SV*)(sv)) +#endif + +#ifndef SvREFCNT_inc_void_NN +# define SvREFCNT_inc_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) +#endif + +#ifndef SvREFCNT_inc_simple_void_NN +# define SvREFCNT_inc_simple_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) +#endif + +#if (PERL_BCDVERSION < 0x5006000) +# define D_PPP_CONSTPV_ARG(x) ((char *) (x)) +#else +# define D_PPP_CONSTPV_ARG(x) (x) +#endif +#ifndef newSVpvn +# define newSVpvn(data,len) ((data) \ + ? ((len) ? newSVpv((data), (len)) : newSVpv("", 0)) \ + : newSV(0)) +#endif +#ifndef newSVpvn_utf8 +# define newSVpvn_utf8(s, len, u) newSVpvn_flags((s), (len), (u) ? SVf_UTF8 : 0) +#endif +#ifndef SVf_UTF8 +# define SVf_UTF8 0 +#endif + +#ifndef newSVpvn_flags + +#if defined(NEED_newSVpvn_flags) +static SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); +static +#else +extern SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); +#endif + +#ifdef newSVpvn_flags +# undef newSVpvn_flags +#endif +#define newSVpvn_flags(a,b,c) DPPP_(my_newSVpvn_flags)(aTHX_ a,b,c) +#define Perl_newSVpvn_flags DPPP_(my_newSVpvn_flags) + +#if defined(NEED_newSVpvn_flags) || defined(NEED_newSVpvn_flags_GLOBAL) + +SV * +DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags) +{ + SV *sv = newSVpvn(D_PPP_CONSTPV_ARG(s), len); + SvFLAGS(sv) |= (flags & SVf_UTF8); + return (flags & SVs_TEMP) ? sv_2mortal(sv) : sv; +} + +#endif + +#endif + +/* Backwards compatibility stuff... :-( */ +#if !defined(NEED_sv_2pv_flags) && defined(NEED_sv_2pv_nolen) +# define NEED_sv_2pv_flags +#endif +#if !defined(NEED_sv_2pv_flags_GLOBAL) && defined(NEED_sv_2pv_nolen_GLOBAL) +# define NEED_sv_2pv_flags_GLOBAL +#endif + +/* Hint: sv_2pv_nolen + * Use the SvPV_nolen() or SvPV_nolen_const() macros instead of sv_2pv_nolen(). + */ +#ifndef sv_2pv_nolen +# define sv_2pv_nolen(sv) SvPV_nolen(sv) +#endif + +#ifdef SvPVbyte + +/* Hint: SvPVbyte + * Does not work in perl-5.6.1, ppport.h implements a version + * borrowed from perl-5.7.3. + */ + +#if (PERL_BCDVERSION < 0x5007000) + +#if defined(NEED_sv_2pvbyte) +static char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); +static +#else +extern char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); +#endif + +#ifdef sv_2pvbyte +# undef sv_2pvbyte +#endif +#define sv_2pvbyte(a,b) DPPP_(my_sv_2pvbyte)(aTHX_ a,b) +#define Perl_sv_2pvbyte DPPP_(my_sv_2pvbyte) + +#if defined(NEED_sv_2pvbyte) || defined(NEED_sv_2pvbyte_GLOBAL) + +char * +DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp) +{ + sv_utf8_downgrade(sv,0); + return SvPV(sv,*lp); +} + +#endif + +/* Hint: sv_2pvbyte + * Use the SvPVbyte() macro instead of sv_2pvbyte(). + */ + +#undef SvPVbyte + +#define SvPVbyte(sv, lp) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_UTF8)) == (SVf_POK) \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pvbyte(sv, &lp)) + +#endif + +#else + +# define SvPVbyte SvPV +# define sv_2pvbyte sv_2pv + +#endif +#ifndef sv_2pvbyte_nolen +# define sv_2pvbyte_nolen(sv) sv_2pv_nolen(sv) +#endif + +/* Hint: sv_pvn + * Always use the SvPV() macro instead of sv_pvn(). + */ + +/* Hint: sv_pvn_force + * Always use the SvPV_force() macro instead of sv_pvn_force(). + */ + +/* If these are undefined, they're not handled by the core anyway */ +#ifndef SV_IMMEDIATE_UNREF +# define SV_IMMEDIATE_UNREF 0 +#endif + +#ifndef SV_GMAGIC +# define SV_GMAGIC 0 +#endif + +#ifndef SV_COW_DROP_PV +# define SV_COW_DROP_PV 0 +#endif + +#ifndef SV_UTF8_NO_ENCODING +# define SV_UTF8_NO_ENCODING 0 +#endif + +#ifndef SV_NOSTEAL +# define SV_NOSTEAL 0 +#endif + +#ifndef SV_CONST_RETURN +# define SV_CONST_RETURN 0 +#endif + +#ifndef SV_MUTABLE_RETURN +# define SV_MUTABLE_RETURN 0 +#endif + +#ifndef SV_SMAGIC +# define SV_SMAGIC 0 +#endif + +#ifndef SV_HAS_TRAILING_NUL +# define SV_HAS_TRAILING_NUL 0 +#endif + +#ifndef SV_COW_SHARED_HASH_KEYS +# define SV_COW_SHARED_HASH_KEYS 0 +#endif + +#if (PERL_BCDVERSION < 0x5007002) + +#if defined(NEED_sv_2pv_flags) +static char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +static +#else +extern char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +#endif + +#ifdef sv_2pv_flags +# undef sv_2pv_flags +#endif +#define sv_2pv_flags(a,b,c) DPPP_(my_sv_2pv_flags)(aTHX_ a,b,c) +#define Perl_sv_2pv_flags DPPP_(my_sv_2pv_flags) + +#if defined(NEED_sv_2pv_flags) || defined(NEED_sv_2pv_flags_GLOBAL) + +char * +DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) +{ + STRLEN n_a = (STRLEN) flags; + return sv_2pv(sv, lp ? lp : &n_a); +} + +#endif + +#if defined(NEED_sv_pvn_force_flags) +static char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +static +#else +extern char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +#endif + +#ifdef sv_pvn_force_flags +# undef sv_pvn_force_flags +#endif +#define sv_pvn_force_flags(a,b,c) DPPP_(my_sv_pvn_force_flags)(aTHX_ a,b,c) +#define Perl_sv_pvn_force_flags DPPP_(my_sv_pvn_force_flags) + +#if defined(NEED_sv_pvn_force_flags) || defined(NEED_sv_pvn_force_flags_GLOBAL) + +char * +DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) +{ + STRLEN n_a = (STRLEN) flags; + return sv_pvn_force(sv, lp ? lp : &n_a); +} + +#endif + +#endif + +#if (PERL_BCDVERSION < 0x5008008) || ( (PERL_BCDVERSION >= 0x5009000) && (PERL_BCDVERSION < 0x5009003) ) +# define DPPP_SVPV_NOLEN_LP_ARG &PL_na +#else +# define DPPP_SVPV_NOLEN_LP_ARG 0 +#endif +#ifndef SvPV_const +# define SvPV_const(sv, lp) SvPV_flags_const(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_mutable +# define SvPV_mutable(sv, lp) SvPV_flags_mutable(sv, lp, SV_GMAGIC) +#endif +#ifndef SvPV_flags +# define SvPV_flags(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pv_flags(sv, &lp, flags)) +#endif +#ifndef SvPV_flags_const +# define SvPV_flags_const(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_const(sv)) : \ + (const char*) sv_2pv_flags(sv, &lp, flags|SV_CONST_RETURN)) +#endif +#ifndef SvPV_flags_const_nolen +# define SvPV_flags_const_nolen(sv, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX_const(sv) : \ + (const char*) sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags|SV_CONST_RETURN)) +#endif +#ifndef SvPV_flags_mutable +# define SvPV_flags_mutable(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) : \ + sv_2pv_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) +#endif +#ifndef SvPV_force +# define SvPV_force(sv, lp) SvPV_force_flags(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_force_nolen +# define SvPV_force_nolen(sv) SvPV_force_flags_nolen(sv, SV_GMAGIC) +#endif + +#ifndef SvPV_force_mutable +# define SvPV_force_mutable(sv, lp) SvPV_force_flags_mutable(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_force_nomg +# define SvPV_force_nomg(sv, lp) SvPV_force_flags(sv, lp, 0) +#endif + +#ifndef SvPV_force_nomg_nolen +# define SvPV_force_nomg_nolen(sv) SvPV_force_flags_nolen(sv, 0) +#endif +#ifndef SvPV_force_flags +# define SvPV_force_flags(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_pvn_force_flags(sv, &lp, flags)) +#endif +#ifndef SvPV_force_flags_nolen +# define SvPV_force_flags_nolen(sv, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? SvPVX(sv) : sv_pvn_force_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags)) +#endif +#ifndef SvPV_force_flags_mutable +# define SvPV_force_flags_mutable(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) \ + : sv_pvn_force_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) +#endif +#ifndef SvPV_nolen +# define SvPV_nolen(sv) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC)) +#endif +#ifndef SvPV_nolen_const +# define SvPV_nolen_const(sv) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX_const(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC|SV_CONST_RETURN)) +#endif +#ifndef SvPV_nomg +# define SvPV_nomg(sv, lp) SvPV_flags(sv, lp, 0) +#endif + +#ifndef SvPV_nomg_const +# define SvPV_nomg_const(sv, lp) SvPV_flags_const(sv, lp, 0) +#endif + +#ifndef SvPV_nomg_const_nolen +# define SvPV_nomg_const_nolen(sv) SvPV_flags_const_nolen(sv, 0) +#endif +#ifndef SvPV_renew +# define SvPV_renew(sv,n) STMT_START { SvLEN_set(sv, n); \ + SvPV_set((sv), (char *) saferealloc( \ + (Malloc_t)SvPVX(sv), (MEM_SIZE)((n)))); \ + } STMT_END +#endif +#ifndef SvMAGIC_set +# define SvMAGIC_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ + (((XPVMG*) SvANY(sv))->xmg_magic = (val)); } STMT_END +#endif + +#if (PERL_BCDVERSION < 0x5009003) +#ifndef SvPVX_const +# define SvPVX_const(sv) ((const char*) (0 + SvPVX(sv))) +#endif + +#ifndef SvPVX_mutable +# define SvPVX_mutable(sv) (0 + SvPVX(sv)) +#endif +#ifndef SvRV_set +# define SvRV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ + (((XRV*) SvANY(sv))->xrv_rv = (val)); } STMT_END +#endif + +#else +#ifndef SvPVX_const +# define SvPVX_const(sv) ((const char*)((sv)->sv_u.svu_pv)) +#endif + +#ifndef SvPVX_mutable +# define SvPVX_mutable(sv) ((sv)->sv_u.svu_pv) +#endif +#ifndef SvRV_set +# define SvRV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ + ((sv)->sv_u.svu_rv = (val)); } STMT_END +#endif + +#endif +#ifndef SvSTASH_set +# define SvSTASH_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ + (((XPVMG*) SvANY(sv))->xmg_stash = (val)); } STMT_END +#endif + +#if (PERL_BCDVERSION < 0x5004000) +#ifndef SvUV_set +# define SvUV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ + (((XPVIV*) SvANY(sv))->xiv_iv = (IV) (val)); } STMT_END +#endif + +#else +#ifndef SvUV_set +# define SvUV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ + (((XPVUV*) SvANY(sv))->xuv_uv = (val)); } STMT_END +#endif + +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(vnewSVpvf) +#if defined(NEED_vnewSVpvf) +static SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); +static +#else +extern SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); +#endif + +#ifdef vnewSVpvf +# undef vnewSVpvf +#endif +#define vnewSVpvf(a,b) DPPP_(my_vnewSVpvf)(aTHX_ a,b) +#define Perl_vnewSVpvf DPPP_(my_vnewSVpvf) + +#if defined(NEED_vnewSVpvf) || defined(NEED_vnewSVpvf_GLOBAL) + +SV * +DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args) +{ + register SV *sv = newSV(0); + sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); + return sv; +} + +#endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf) +# define sv_vcatpvf(sv, pat, args) sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf) +# define sv_vsetpvf(sv, pat, args) sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg) +#if defined(NEED_sv_catpvf_mg) +static void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +#endif + +#define Perl_sv_catpvf_mg DPPP_(my_sv_catpvf_mg) + +#if defined(NEED_sv_catpvf_mg) || defined(NEED_sv_catpvf_mg_GLOBAL) + +void +DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...) +{ + va_list args; + va_start(args, pat); + sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif + +#ifdef PERL_IMPLICIT_CONTEXT +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg_nocontext) +#if defined(NEED_sv_catpvf_mg_nocontext) +static void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); +#endif + +#define sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) +#define Perl_sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) + +#if defined(NEED_sv_catpvf_mg_nocontext) || defined(NEED_sv_catpvf_mg_nocontext_GLOBAL) + +void +DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...) +{ + dTHX; + va_list args; + va_start(args, pat); + sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif +#endif + +/* sv_catpvf_mg depends on sv_catpvf_mg_nocontext */ +#ifndef sv_catpvf_mg +# ifdef PERL_IMPLICIT_CONTEXT +# define sv_catpvf_mg Perl_sv_catpvf_mg_nocontext +# else +# define sv_catpvf_mg Perl_sv_catpvf_mg +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf_mg) +# define sv_vcatpvf_mg(sv, pat, args) \ + STMT_START { \ + sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ + SvSETMAGIC(sv); \ + } STMT_END +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg) +#if defined(NEED_sv_setpvf_mg) +static void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +#endif + +#define Perl_sv_setpvf_mg DPPP_(my_sv_setpvf_mg) + +#if defined(NEED_sv_setpvf_mg) || defined(NEED_sv_setpvf_mg_GLOBAL) + +void +DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...) +{ + va_list args; + va_start(args, pat); + sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif + +#ifdef PERL_IMPLICIT_CONTEXT +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg_nocontext) +#if defined(NEED_sv_setpvf_mg_nocontext) +static void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); +#endif + +#define sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) +#define Perl_sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) + +#if defined(NEED_sv_setpvf_mg_nocontext) || defined(NEED_sv_setpvf_mg_nocontext_GLOBAL) + +void +DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...) +{ + dTHX; + va_list args; + va_start(args, pat); + sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif +#endif + +/* sv_setpvf_mg depends on sv_setpvf_mg_nocontext */ +#ifndef sv_setpvf_mg +# ifdef PERL_IMPLICIT_CONTEXT +# define sv_setpvf_mg Perl_sv_setpvf_mg_nocontext +# else +# define sv_setpvf_mg Perl_sv_setpvf_mg +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf_mg) +# define sv_vsetpvf_mg(sv, pat, args) \ + STMT_START { \ + sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ + SvSETMAGIC(sv); \ + } STMT_END +#endif + +#ifndef newSVpvn_share + +#if defined(NEED_newSVpvn_share) +static SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); +static +#else +extern SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); +#endif + +#ifdef newSVpvn_share +# undef newSVpvn_share +#endif +#define newSVpvn_share(a,b,c) DPPP_(my_newSVpvn_share)(aTHX_ a,b,c) +#define Perl_newSVpvn_share DPPP_(my_newSVpvn_share) + +#if defined(NEED_newSVpvn_share) || defined(NEED_newSVpvn_share_GLOBAL) + +SV * +DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash) +{ + SV *sv; + if (len < 0) + len = -len; + if (!hash) + PERL_HASH(hash, (char*) src, len); + sv = newSVpvn((char *) src, len); + sv_upgrade(sv, SVt_PVIV); + SvIVX(sv) = hash; + SvREADONLY_on(sv); + SvPOK_on(sv); + return sv; +} + +#endif + +#endif +#ifndef SvSHARED_HASH +# define SvSHARED_HASH(sv) (0 + SvUVX(sv)) +#endif +#ifndef WARN_ALL +# define WARN_ALL 0 +#endif + +#ifndef WARN_CLOSURE +# define WARN_CLOSURE 1 +#endif + +#ifndef WARN_DEPRECATED +# define WARN_DEPRECATED 2 +#endif + +#ifndef WARN_EXITING +# define WARN_EXITING 3 +#endif + +#ifndef WARN_GLOB +# define WARN_GLOB 4 +#endif + +#ifndef WARN_IO +# define WARN_IO 5 +#endif + +#ifndef WARN_CLOSED +# define WARN_CLOSED 6 +#endif + +#ifndef WARN_EXEC +# define WARN_EXEC 7 +#endif + +#ifndef WARN_LAYER +# define WARN_LAYER 8 +#endif + +#ifndef WARN_NEWLINE +# define WARN_NEWLINE 9 +#endif + +#ifndef WARN_PIPE +# define WARN_PIPE 10 +#endif + +#ifndef WARN_UNOPENED +# define WARN_UNOPENED 11 +#endif + +#ifndef WARN_MISC +# define WARN_MISC 12 +#endif + +#ifndef WARN_NUMERIC +# define WARN_NUMERIC 13 +#endif + +#ifndef WARN_ONCE +# define WARN_ONCE 14 +#endif + +#ifndef WARN_OVERFLOW +# define WARN_OVERFLOW 15 +#endif + +#ifndef WARN_PACK +# define WARN_PACK 16 +#endif + +#ifndef WARN_PORTABLE +# define WARN_PORTABLE 17 +#endif + +#ifndef WARN_RECURSION +# define WARN_RECURSION 18 +#endif + +#ifndef WARN_REDEFINE +# define WARN_REDEFINE 19 +#endif + +#ifndef WARN_REGEXP +# define WARN_REGEXP 20 +#endif + +#ifndef WARN_SEVERE +# define WARN_SEVERE 21 +#endif + +#ifndef WARN_DEBUGGING +# define WARN_DEBUGGING 22 +#endif + +#ifndef WARN_INPLACE +# define WARN_INPLACE 23 +#endif + +#ifndef WARN_INTERNAL +# define WARN_INTERNAL 24 +#endif + +#ifndef WARN_MALLOC +# define WARN_MALLOC 25 +#endif + +#ifndef WARN_SIGNAL +# define WARN_SIGNAL 26 +#endif + +#ifndef WARN_SUBSTR +# define WARN_SUBSTR 27 +#endif + +#ifndef WARN_SYNTAX +# define WARN_SYNTAX 28 +#endif + +#ifndef WARN_AMBIGUOUS +# define WARN_AMBIGUOUS 29 +#endif + +#ifndef WARN_BAREWORD +# define WARN_BAREWORD 30 +#endif + +#ifndef WARN_DIGIT +# define WARN_DIGIT 31 +#endif + +#ifndef WARN_PARENTHESIS +# define WARN_PARENTHESIS 32 +#endif + +#ifndef WARN_PRECEDENCE +# define WARN_PRECEDENCE 33 +#endif + +#ifndef WARN_PRINTF +# define WARN_PRINTF 34 +#endif + +#ifndef WARN_PROTOTYPE +# define WARN_PROTOTYPE 35 +#endif + +#ifndef WARN_QW +# define WARN_QW 36 +#endif + +#ifndef WARN_RESERVED +# define WARN_RESERVED 37 +#endif + +#ifndef WARN_SEMICOLON +# define WARN_SEMICOLON 38 +#endif + +#ifndef WARN_TAINT +# define WARN_TAINT 39 +#endif + +#ifndef WARN_THREADS +# define WARN_THREADS 40 +#endif + +#ifndef WARN_UNINITIALIZED +# define WARN_UNINITIALIZED 41 +#endif + +#ifndef WARN_UNPACK +# define WARN_UNPACK 42 +#endif + +#ifndef WARN_UNTIE +# define WARN_UNTIE 43 +#endif + +#ifndef WARN_UTF8 +# define WARN_UTF8 44 +#endif + +#ifndef WARN_VOID +# define WARN_VOID 45 +#endif + +#ifndef WARN_ASSERTIONS +# define WARN_ASSERTIONS 46 +#endif +#ifndef packWARN +# define packWARN(a) (a) +#endif + +#ifndef ckWARN +# ifdef G_WARN_ON +# define ckWARN(a) (PL_dowarn & G_WARN_ON) +# else +# define ckWARN(a) PL_dowarn +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(warner) +#if defined(NEED_warner) +static void DPPP_(my_warner)(U32 err, const char *pat, ...); +static +#else +extern void DPPP_(my_warner)(U32 err, const char *pat, ...); +#endif + +#define Perl_warner DPPP_(my_warner) + +#if defined(NEED_warner) || defined(NEED_warner_GLOBAL) + +void +DPPP_(my_warner)(U32 err, const char *pat, ...) +{ + SV *sv; + va_list args; + + PERL_UNUSED_ARG(err); + + va_start(args, pat); + sv = vnewSVpvf(pat, &args); + va_end(args); + sv_2mortal(sv); + warn("%s", SvPV_nolen(sv)); +} + +#define warner Perl_warner + +#define Perl_warner_nocontext Perl_warner + +#endif +#endif + +/* concatenating with "" ensures that only literal strings are accepted as argument + * note that STR_WITH_LEN() can't be used as argument to macros or functions that + * under some configurations might be macros + */ +#ifndef STR_WITH_LEN +# define STR_WITH_LEN(s) (s ""), (sizeof(s)-1) +#endif +#ifndef newSVpvs +# define newSVpvs(str) newSVpvn(str "", sizeof(str) - 1) +#endif + +#ifndef newSVpvs_flags +# define newSVpvs_flags(str, flags) newSVpvn_flags(str "", sizeof(str) - 1, flags) +#endif + +#ifndef sv_catpvs +# define sv_catpvs(sv, str) sv_catpvn(sv, str "", sizeof(str) - 1) +#endif + +#ifndef sv_setpvs +# define sv_setpvs(sv, str) sv_setpvn(sv, str "", sizeof(str) - 1) +#endif + +#ifndef hv_fetchs +# define hv_fetchs(hv, key, lval) hv_fetch(hv, key "", sizeof(key) - 1, lval) +#endif + +#ifndef hv_stores +# define hv_stores(hv, key, val) hv_store(hv, key "", sizeof(key) - 1, val, 0) +#endif +#ifndef SvGETMAGIC +# define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END +#endif +#ifndef PERL_MAGIC_sv +# define PERL_MAGIC_sv '\0' +#endif + +#ifndef PERL_MAGIC_overload +# define PERL_MAGIC_overload 'A' +#endif + +#ifndef PERL_MAGIC_overload_elem +# define PERL_MAGIC_overload_elem 'a' +#endif + +#ifndef PERL_MAGIC_overload_table +# define PERL_MAGIC_overload_table 'c' +#endif + +#ifndef PERL_MAGIC_bm +# define PERL_MAGIC_bm 'B' +#endif + +#ifndef PERL_MAGIC_regdata +# define PERL_MAGIC_regdata 'D' +#endif + +#ifndef PERL_MAGIC_regdatum +# define PERL_MAGIC_regdatum 'd' +#endif + +#ifndef PERL_MAGIC_env +# define PERL_MAGIC_env 'E' +#endif + +#ifndef PERL_MAGIC_envelem +# define PERL_MAGIC_envelem 'e' +#endif + +#ifndef PERL_MAGIC_fm +# define PERL_MAGIC_fm 'f' +#endif + +#ifndef PERL_MAGIC_regex_global +# define PERL_MAGIC_regex_global 'g' +#endif + +#ifndef PERL_MAGIC_isa +# define PERL_MAGIC_isa 'I' +#endif + +#ifndef PERL_MAGIC_isaelem +# define PERL_MAGIC_isaelem 'i' +#endif + +#ifndef PERL_MAGIC_nkeys +# define PERL_MAGIC_nkeys 'k' +#endif + +#ifndef PERL_MAGIC_dbfile +# define PERL_MAGIC_dbfile 'L' +#endif + +#ifndef PERL_MAGIC_dbline +# define PERL_MAGIC_dbline 'l' +#endif + +#ifndef PERL_MAGIC_mutex +# define PERL_MAGIC_mutex 'm' +#endif + +#ifndef PERL_MAGIC_shared +# define PERL_MAGIC_shared 'N' +#endif + +#ifndef PERL_MAGIC_shared_scalar +# define PERL_MAGIC_shared_scalar 'n' +#endif + +#ifndef PERL_MAGIC_collxfrm +# define PERL_MAGIC_collxfrm 'o' +#endif + +#ifndef PERL_MAGIC_tied +# define PERL_MAGIC_tied 'P' +#endif + +#ifndef PERL_MAGIC_tiedelem +# define PERL_MAGIC_tiedelem 'p' +#endif + +#ifndef PERL_MAGIC_tiedscalar +# define PERL_MAGIC_tiedscalar 'q' +#endif + +#ifndef PERL_MAGIC_qr +# define PERL_MAGIC_qr 'r' +#endif + +#ifndef PERL_MAGIC_sig +# define PERL_MAGIC_sig 'S' +#endif + +#ifndef PERL_MAGIC_sigelem +# define PERL_MAGIC_sigelem 's' +#endif + +#ifndef PERL_MAGIC_taint +# define PERL_MAGIC_taint 't' +#endif + +#ifndef PERL_MAGIC_uvar +# define PERL_MAGIC_uvar 'U' +#endif + +#ifndef PERL_MAGIC_uvar_elem +# define PERL_MAGIC_uvar_elem 'u' +#endif + +#ifndef PERL_MAGIC_vstring +# define PERL_MAGIC_vstring 'V' +#endif + +#ifndef PERL_MAGIC_vec +# define PERL_MAGIC_vec 'v' +#endif + +#ifndef PERL_MAGIC_utf8 +# define PERL_MAGIC_utf8 'w' +#endif + +#ifndef PERL_MAGIC_substr +# define PERL_MAGIC_substr 'x' +#endif + +#ifndef PERL_MAGIC_defelem +# define PERL_MAGIC_defelem 'y' +#endif + +#ifndef PERL_MAGIC_glob +# define PERL_MAGIC_glob '*' +#endif + +#ifndef PERL_MAGIC_arylen +# define PERL_MAGIC_arylen '#' +#endif + +#ifndef PERL_MAGIC_pos +# define PERL_MAGIC_pos '.' +#endif + +#ifndef PERL_MAGIC_backref +# define PERL_MAGIC_backref '<' +#endif + +#ifndef PERL_MAGIC_ext +# define PERL_MAGIC_ext '~' +#endif + +/* That's the best we can do... */ +#ifndef sv_catpvn_nomg +# define sv_catpvn_nomg sv_catpvn +#endif + +#ifndef sv_catsv_nomg +# define sv_catsv_nomg sv_catsv +#endif + +#ifndef sv_setsv_nomg +# define sv_setsv_nomg sv_setsv +#endif + +#ifndef sv_pvn_nomg +# define sv_pvn_nomg sv_pvn +#endif + +#ifndef SvIV_nomg +# define SvIV_nomg SvIV +#endif + +#ifndef SvUV_nomg +# define SvUV_nomg SvUV +#endif + +#ifndef sv_catpv_mg +# define sv_catpv_mg(sv, ptr) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_catpv(TeMpSv,ptr); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_catpvn_mg +# define sv_catpvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_catpvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_catsv_mg +# define sv_catsv_mg(dsv, ssv) \ + STMT_START { \ + SV *TeMpSv = dsv; \ + sv_catsv(TeMpSv,ssv); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setiv_mg +# define sv_setiv_mg(sv, i) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setiv(TeMpSv,i); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setnv_mg +# define sv_setnv_mg(sv, num) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setnv(TeMpSv,num); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setpv_mg +# define sv_setpv_mg(sv, ptr) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setpv(TeMpSv,ptr); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setpvn_mg +# define sv_setpvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setpvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setsv_mg +# define sv_setsv_mg(dsv, ssv) \ + STMT_START { \ + SV *TeMpSv = dsv; \ + sv_setsv(TeMpSv,ssv); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setuv_mg +# define sv_setuv_mg(sv, i) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setuv(TeMpSv,i); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_usepvn_mg +# define sv_usepvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_usepvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif +#ifndef SvVSTRING_mg +# define SvVSTRING_mg(sv) (SvMAGICAL(sv) ? mg_find(sv, PERL_MAGIC_vstring) : NULL) +#endif + +/* Hint: sv_magic_portable + * This is a compatibility function that is only available with + * Devel::PPPort. It is NOT in the perl core. + * Its purpose is to mimic the 5.8.0 behaviour of sv_magic() when + * it is being passed a name pointer with namlen == 0. In that + * case, perl 5.8.0 and later store the pointer, not a copy of it. + * The compatibility can be provided back to perl 5.004. With + * earlier versions, the code will not compile. + */ + +#if (PERL_BCDVERSION < 0x5004000) + + /* code that uses sv_magic_portable will not compile */ + +#elif (PERL_BCDVERSION < 0x5008000) + +# define sv_magic_portable(sv, obj, how, name, namlen) \ + STMT_START { \ + SV *SvMp_sv = (sv); \ + char *SvMp_name = (char *) (name); \ + I32 SvMp_namlen = (namlen); \ + if (SvMp_name && SvMp_namlen == 0) \ + { \ + MAGIC *mg; \ + sv_magic(SvMp_sv, obj, how, 0, 0); \ + mg = SvMAGIC(SvMp_sv); \ + mg->mg_len = -42; /* XXX: this is the tricky part */ \ + mg->mg_ptr = SvMp_name; \ + } \ + else \ + { \ + sv_magic(SvMp_sv, obj, how, SvMp_name, SvMp_namlen); \ + } \ + } STMT_END + +#else + +# define sv_magic_portable(a, b, c, d, e) sv_magic(a, b, c, d, e) + +#endif + +#ifdef USE_ITHREADS +#ifndef CopFILE +# define CopFILE(c) ((c)->cop_file) +#endif + +#ifndef CopFILEGV +# define CopFILEGV(c) (CopFILE(c) ? gv_fetchfile(CopFILE(c)) : Nullgv) +#endif + +#ifndef CopFILE_set +# define CopFILE_set(c,pv) ((c)->cop_file = savepv(pv)) +#endif + +#ifndef CopFILESV +# define CopFILESV(c) (CopFILE(c) ? GvSV(gv_fetchfile(CopFILE(c))) : Nullsv) +#endif + +#ifndef CopFILEAV +# define CopFILEAV(c) (CopFILE(c) ? GvAV(gv_fetchfile(CopFILE(c))) : Nullav) +#endif + +#ifndef CopSTASHPV +# define CopSTASHPV(c) ((c)->cop_stashpv) +#endif + +#ifndef CopSTASHPV_set +# define CopSTASHPV_set(c,pv) ((c)->cop_stashpv = ((pv) ? savepv(pv) : Nullch)) +#endif + +#ifndef CopSTASH +# define CopSTASH(c) (CopSTASHPV(c) ? gv_stashpv(CopSTASHPV(c),GV_ADD) : Nullhv) +#endif + +#ifndef CopSTASH_set +# define CopSTASH_set(c,hv) CopSTASHPV_set(c, (hv) ? HvNAME(hv) : Nullch) +#endif + +#ifndef CopSTASH_eq +# define CopSTASH_eq(c,hv) ((hv) && (CopSTASHPV(c) == HvNAME(hv) \ + || (CopSTASHPV(c) && HvNAME(hv) \ + && strEQ(CopSTASHPV(c), HvNAME(hv))))) +#endif + +#else +#ifndef CopFILEGV +# define CopFILEGV(c) ((c)->cop_filegv) +#endif + +#ifndef CopFILEGV_set +# define CopFILEGV_set(c,gv) ((c)->cop_filegv = (GV*)SvREFCNT_inc(gv)) +#endif + +#ifndef CopFILE_set +# define CopFILE_set(c,pv) CopFILEGV_set((c), gv_fetchfile(pv)) +#endif + +#ifndef CopFILESV +# define CopFILESV(c) (CopFILEGV(c) ? GvSV(CopFILEGV(c)) : Nullsv) +#endif + +#ifndef CopFILEAV +# define CopFILEAV(c) (CopFILEGV(c) ? GvAV(CopFILEGV(c)) : Nullav) +#endif + +#ifndef CopFILE +# define CopFILE(c) (CopFILESV(c) ? SvPVX(CopFILESV(c)) : Nullch) +#endif + +#ifndef CopSTASH +# define CopSTASH(c) ((c)->cop_stash) +#endif + +#ifndef CopSTASH_set +# define CopSTASH_set(c,hv) ((c)->cop_stash = (hv)) +#endif + +#ifndef CopSTASHPV +# define CopSTASHPV(c) (CopSTASH(c) ? HvNAME(CopSTASH(c)) : Nullch) +#endif + +#ifndef CopSTASHPV_set +# define CopSTASHPV_set(c,pv) CopSTASH_set((c), gv_stashpv(pv,GV_ADD)) +#endif + +#ifndef CopSTASH_eq +# define CopSTASH_eq(c,hv) (CopSTASH(c) == (hv)) +#endif + +#endif /* USE_ITHREADS */ +#ifndef IN_PERL_COMPILETIME +# define IN_PERL_COMPILETIME (PL_curcop == &PL_compiling) +#endif + +#ifndef IN_LOCALE_RUNTIME +# define IN_LOCALE_RUNTIME (PL_curcop->op_private & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE_COMPILETIME +# define IN_LOCALE_COMPILETIME (PL_hints & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE +# define IN_LOCALE (IN_PERL_COMPILETIME ? IN_LOCALE_COMPILETIME : IN_LOCALE_RUNTIME) +#endif +#ifndef IS_NUMBER_IN_UV +# define IS_NUMBER_IN_UV 0x01 +#endif + +#ifndef IS_NUMBER_GREATER_THAN_UV_MAX +# define IS_NUMBER_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef IS_NUMBER_NOT_INT +# define IS_NUMBER_NOT_INT 0x04 +#endif + +#ifndef IS_NUMBER_NEG +# define IS_NUMBER_NEG 0x08 +#endif + +#ifndef IS_NUMBER_INFINITY +# define IS_NUMBER_INFINITY 0x10 +#endif + +#ifndef IS_NUMBER_NAN +# define IS_NUMBER_NAN 0x20 +#endif +#ifndef GROK_NUMERIC_RADIX +# define GROK_NUMERIC_RADIX(sp, send) grok_numeric_radix(sp, send) +#endif +#ifndef PERL_SCAN_GREATER_THAN_UV_MAX +# define PERL_SCAN_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef PERL_SCAN_SILENT_ILLDIGIT +# define PERL_SCAN_SILENT_ILLDIGIT 0x04 +#endif + +#ifndef PERL_SCAN_ALLOW_UNDERSCORES +# define PERL_SCAN_ALLOW_UNDERSCORES 0x01 +#endif + +#ifndef PERL_SCAN_DISALLOW_PREFIX +# define PERL_SCAN_DISALLOW_PREFIX 0x02 +#endif + +#ifndef grok_numeric_radix +#if defined(NEED_grok_numeric_radix) +static bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); +static +#else +extern bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); +#endif + +#ifdef grok_numeric_radix +# undef grok_numeric_radix +#endif +#define grok_numeric_radix(a,b) DPPP_(my_grok_numeric_radix)(aTHX_ a,b) +#define Perl_grok_numeric_radix DPPP_(my_grok_numeric_radix) + +#if defined(NEED_grok_numeric_radix) || defined(NEED_grok_numeric_radix_GLOBAL) +bool +DPPP_(my_grok_numeric_radix)(pTHX_ const char **sp, const char *send) +{ +#ifdef USE_LOCALE_NUMERIC +#ifdef PL_numeric_radix_sv + if (PL_numeric_radix_sv && IN_LOCALE) { + STRLEN len; + char* radix = SvPV(PL_numeric_radix_sv, len); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#else + /* older perls don't have PL_numeric_radix_sv so the radix + * must manually be requested from locale.h + */ +#include + dTHR; /* needed for older threaded perls */ + struct lconv *lc = localeconv(); + char *radix = lc->decimal_point; + if (radix && IN_LOCALE) { + STRLEN len = strlen(radix); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#endif +#endif /* USE_LOCALE_NUMERIC */ + /* always try "." if numeric radix didn't match because + * we may have data from different locales mixed */ + if (*sp < send && **sp == '.') { + ++*sp; + return TRUE; + } + return FALSE; +} +#endif +#endif + +#ifndef grok_number +#if defined(NEED_grok_number) +static int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); +static +#else +extern int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); +#endif + +#ifdef grok_number +# undef grok_number +#endif +#define grok_number(a,b,c) DPPP_(my_grok_number)(aTHX_ a,b,c) +#define Perl_grok_number DPPP_(my_grok_number) + +#if defined(NEED_grok_number) || defined(NEED_grok_number_GLOBAL) +int +DPPP_(my_grok_number)(pTHX_ const char *pv, STRLEN len, UV *valuep) +{ + const char *s = pv; + const char *send = pv + len; + const UV max_div_10 = UV_MAX / 10; + const char max_mod_10 = UV_MAX % 10; + int numtype = 0; + int sawinf = 0; + int sawnan = 0; + + while (s < send && isSPACE(*s)) + s++; + if (s == send) { + return 0; + } else if (*s == '-') { + s++; + numtype = IS_NUMBER_NEG; + } + else if (*s == '+') + s++; + + if (s == send) + return 0; + + /* next must be digit or the radix separator or beginning of infinity */ + if (isDIGIT(*s)) { + /* UVs are at least 32 bits, so the first 9 decimal digits cannot + overflow. */ + UV value = *s - '0'; + /* This construction seems to be more optimiser friendly. + (without it gcc does the isDIGIT test and the *s - '0' separately) + With it gcc on arm is managing 6 instructions (6 cycles) per digit. + In theory the optimiser could deduce how far to unroll the loop + before checking for overflow. */ + if (++s < send) { + int digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + /* Now got 9 digits, so need to check + each time for overflow. */ + digit = *s - '0'; + while (digit >= 0 && digit <= 9 + && (value < max_div_10 + || (value == max_div_10 + && digit <= max_mod_10))) { + value = value * 10 + digit; + if (++s < send) + digit = *s - '0'; + else + break; + } + if (digit >= 0 && digit <= 9 + && (s < send)) { + /* value overflowed. + skip the remaining digits, don't + worry about setting *valuep. */ + do { + s++; + } while (s < send && isDIGIT(*s)); + numtype |= + IS_NUMBER_GREATER_THAN_UV_MAX; + goto skip_value; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + numtype |= IS_NUMBER_IN_UV; + if (valuep) + *valuep = value; + + skip_value: + if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT; + while (s < send && isDIGIT(*s)) /* optional digits after the radix */ + s++; + } + } + else if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT | IS_NUMBER_IN_UV; /* valuep assigned below */ + /* no digits before the radix means we need digits after it */ + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + if (valuep) { + /* integer approximation is valid - it's 0. */ + *valuep = 0; + } + } + else + return 0; + } else if (*s == 'I' || *s == 'i') { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'F' && *s != 'f')) return 0; + s++; if (s < send && (*s == 'I' || *s == 'i')) { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'I' && *s != 'i')) return 0; + s++; if (s == send || (*s != 'T' && *s != 't')) return 0; + s++; if (s == send || (*s != 'Y' && *s != 'y')) return 0; + s++; + } + sawinf = 1; + } else if (*s == 'N' || *s == 'n') { + /* XXX TODO: There are signaling NaNs and quiet NaNs. */ + s++; if (s == send || (*s != 'A' && *s != 'a')) return 0; + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; + sawnan = 1; + } else + return 0; + + if (sawinf) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_INFINITY | IS_NUMBER_NOT_INT; + } else if (sawnan) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_NAN | IS_NUMBER_NOT_INT; + } else if (s < send) { + /* we can have an optional exponent part */ + if (*s == 'e' || *s == 'E') { + /* The only flag we keep is sign. Blow away any "it's UV" */ + numtype &= IS_NUMBER_NEG; + numtype |= IS_NUMBER_NOT_INT; + s++; + if (s < send && (*s == '-' || *s == '+')) + s++; + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + } + else + return 0; + } + } + while (s < send && isSPACE(*s)) + s++; + if (s >= send) + return numtype; + if (len == 10 && memEQ(pv, "0 but true", 10)) { + if (valuep) + *valuep = 0; + return IS_NUMBER_IN_UV; + } + return 0; +} +#endif +#endif + +/* + * The grok_* routines have been modified to use warn() instead of + * Perl_warner(). Also, 'hexdigit' was the former name of PL_hexdigit, + * which is why the stack variable has been renamed to 'xdigit'. + */ + +#ifndef grok_bin +#if defined(NEED_grok_bin) +static UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_bin +# undef grok_bin +#endif +#define grok_bin(a,b,c,d) DPPP_(my_grok_bin)(aTHX_ a,b,c,d) +#define Perl_grok_bin DPPP_(my_grok_bin) + +#if defined(NEED_grok_bin) || defined(NEED_grok_bin_GLOBAL) +UV +DPPP_(my_grok_bin)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_2 = UV_MAX / 2; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + + if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { + /* strip off leading b or 0b. + for compatibility silently suffer "b" and "0b" as valid binary + numbers. */ + if (len >= 1) { + if (s[0] == 'b') { + s++; + len--; + } + else if (len >= 2 && s[0] == '0' && s[1] == 'b') { + s+=2; + len-=2; + } + } + } + + for (; len-- && *s; s++) { + char bit = *s; + if (bit == '0' || bit == '1') { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + With gcc seems to be much straighter code than old scan_bin. */ + redo: + if (!overflowed) { + if (value <= max_div_2) { + value = (value << 1) | (bit - '0'); + continue; + } + /* Bah. We're just overflowed. */ + warn("Integer overflow in binary number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 2.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount. */ + value_nv += (NV)(bit - '0'); + continue; + } + if (bit == '_' && len && allow_underscores && (bit = s[1]) + && (bit == '0' || bit == '1')) + { + --len; + ++s; + goto redo; + } + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal binary digit '%c' ignored", *s); + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Binary number > 0b11111111111111111111111111111111 non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#ifndef grok_hex +#if defined(NEED_grok_hex) +static UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_hex +# undef grok_hex +#endif +#define grok_hex(a,b,c,d) DPPP_(my_grok_hex)(aTHX_ a,b,c,d) +#define Perl_grok_hex DPPP_(my_grok_hex) + +#if defined(NEED_grok_hex) || defined(NEED_grok_hex_GLOBAL) +UV +DPPP_(my_grok_hex)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_16 = UV_MAX / 16; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + const char *xdigit; + + if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { + /* strip off leading x or 0x. + for compatibility silently suffer "x" and "0x" as valid hex numbers. + */ + if (len >= 1) { + if (s[0] == 'x') { + s++; + len--; + } + else if (len >= 2 && s[0] == '0' && s[1] == 'x') { + s+=2; + len-=2; + } + } + } + + for (; len-- && *s; s++) { + xdigit = strchr((char *) PL_hexdigit, *s); + if (xdigit) { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + With gcc seems to be much straighter code than old scan_hex. */ + redo: + if (!overflowed) { + if (value <= max_div_16) { + value = (value << 4) | ((xdigit - PL_hexdigit) & 15); + continue; + } + warn("Integer overflow in hexadecimal number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 16.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount of 16-tuples. */ + value_nv += (NV)((xdigit - PL_hexdigit) & 15); + continue; + } + if (*s == '_' && len && allow_underscores && s[1] + && (xdigit = strchr((char *) PL_hexdigit, s[1]))) + { + --len; + ++s; + goto redo; + } + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal hexadecimal digit '%c' ignored", *s); + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Hexadecimal number > 0xffffffff non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#ifndef grok_oct +#if defined(NEED_grok_oct) +static UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_oct +# undef grok_oct +#endif +#define grok_oct(a,b,c,d) DPPP_(my_grok_oct)(aTHX_ a,b,c,d) +#define Perl_grok_oct DPPP_(my_grok_oct) + +#if defined(NEED_grok_oct) || defined(NEED_grok_oct_GLOBAL) +UV +DPPP_(my_grok_oct)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_8 = UV_MAX / 8; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + + for (; len-- && *s; s++) { + /* gcc 2.95 optimiser not smart enough to figure that this subtraction + out front allows slicker code. */ + int digit = *s - '0'; + if (digit >= 0 && digit <= 7) { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + */ + redo: + if (!overflowed) { + if (value <= max_div_8) { + value = (value << 3) | digit; + continue; + } + /* Bah. We're just overflowed. */ + warn("Integer overflow in octal number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 8.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount of 8-tuples. */ + value_nv += (NV)digit; + continue; + } + if (digit == ('_' - '0') && len && allow_underscores + && (digit = s[1] - '0') && (digit >= 0 && digit <= 7)) + { + --len; + ++s; + goto redo; + } + /* Allow \octal to work the DWIM way (that is, stop scanning + * as soon as non-octal characters are seen, complain only iff + * someone seems to want to use the digits eight and nine). */ + if (digit == 8 || digit == 9) { + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal octal digit '%c' ignored", *s); + } + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Octal number > 037777777777 non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#if !defined(my_snprintf) +#if defined(NEED_my_snprintf) +static int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); +static +#else +extern int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); +#endif + +#define my_snprintf DPPP_(my_my_snprintf) +#define Perl_my_snprintf DPPP_(my_my_snprintf) + +#if defined(NEED_my_snprintf) || defined(NEED_my_snprintf_GLOBAL) + +int +DPPP_(my_my_snprintf)(char *buffer, const Size_t len, const char *format, ...) +{ + dTHX; + int retval; + va_list ap; + va_start(ap, format); +#ifdef HAS_VSNPRINTF + retval = vsnprintf(buffer, len, format, ap); +#else + retval = vsprintf(buffer, format, ap); +#endif + va_end(ap); + if (retval < 0 || (len > 0 && (Size_t)retval >= len)) + Perl_croak(aTHX_ "panic: my_snprintf buffer overflow"); + return retval; +} + +#endif +#endif + +#if !defined(my_sprintf) +#if defined(NEED_my_sprintf) +static int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); +static +#else +extern int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); +#endif + +#define my_sprintf DPPP_(my_my_sprintf) +#define Perl_my_sprintf DPPP_(my_my_sprintf) + +#if defined(NEED_my_sprintf) || defined(NEED_my_sprintf_GLOBAL) + +int +DPPP_(my_my_sprintf)(char *buffer, const char* pat, ...) +{ + va_list args; + va_start(args, pat); + vsprintf(buffer, pat, args); + va_end(args); + return strlen(buffer); +} + +#endif +#endif + +#ifdef NO_XSLOCKS +# ifdef dJMPENV +# define dXCPT dJMPENV; int rEtV = 0 +# define XCPT_TRY_START JMPENV_PUSH(rEtV); if (rEtV == 0) +# define XCPT_TRY_END JMPENV_POP; +# define XCPT_CATCH if (rEtV != 0) +# define XCPT_RETHROW JMPENV_JUMP(rEtV) +# else +# define dXCPT Sigjmp_buf oldTOP; int rEtV = 0 +# define XCPT_TRY_START Copy(top_env, oldTOP, 1, Sigjmp_buf); rEtV = Sigsetjmp(top_env, 1); if (rEtV == 0) +# define XCPT_TRY_END Copy(oldTOP, top_env, 1, Sigjmp_buf); +# define XCPT_CATCH if (rEtV != 0) +# define XCPT_RETHROW Siglongjmp(top_env, rEtV) +# endif +#endif + +#if !defined(my_strlcat) +#if defined(NEED_my_strlcat) +static Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); +static +#else +extern Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); +#endif + +#define my_strlcat DPPP_(my_my_strlcat) +#define Perl_my_strlcat DPPP_(my_my_strlcat) + +#if defined(NEED_my_strlcat) || defined(NEED_my_strlcat_GLOBAL) + +Size_t +DPPP_(my_my_strlcat)(char *dst, const char *src, Size_t size) +{ + Size_t used, length, copy; + + used = strlen(dst); + length = strlen(src); + if (size > 0 && used < size - 1) { + copy = (length >= size - used) ? size - used - 1 : length; + memcpy(dst + used, src, copy); + dst[used + copy] = '\0'; + } + return used + length; +} +#endif +#endif + +#if !defined(my_strlcpy) +#if defined(NEED_my_strlcpy) +static Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); +static +#else +extern Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); +#endif + +#define my_strlcpy DPPP_(my_my_strlcpy) +#define Perl_my_strlcpy DPPP_(my_my_strlcpy) + +#if defined(NEED_my_strlcpy) || defined(NEED_my_strlcpy_GLOBAL) + +Size_t +DPPP_(my_my_strlcpy)(char *dst, const char *src, Size_t size) +{ + Size_t length, copy; + + length = strlen(src); + if (size > 0) { + copy = (length >= size) ? size - 1 : length; + memcpy(dst, src, copy); + dst[copy] = '\0'; + } + return length; +} + +#endif +#endif +#ifndef PERL_PV_ESCAPE_QUOTE +# define PERL_PV_ESCAPE_QUOTE 0x0001 +#endif + +#ifndef PERL_PV_PRETTY_QUOTE +# define PERL_PV_PRETTY_QUOTE PERL_PV_ESCAPE_QUOTE +#endif + +#ifndef PERL_PV_PRETTY_ELLIPSES +# define PERL_PV_PRETTY_ELLIPSES 0x0002 +#endif + +#ifndef PERL_PV_PRETTY_LTGT +# define PERL_PV_PRETTY_LTGT 0x0004 +#endif + +#ifndef PERL_PV_ESCAPE_FIRSTCHAR +# define PERL_PV_ESCAPE_FIRSTCHAR 0x0008 +#endif + +#ifndef PERL_PV_ESCAPE_UNI +# define PERL_PV_ESCAPE_UNI 0x0100 +#endif + +#ifndef PERL_PV_ESCAPE_UNI_DETECT +# define PERL_PV_ESCAPE_UNI_DETECT 0x0200 +#endif + +#ifndef PERL_PV_ESCAPE_ALL +# define PERL_PV_ESCAPE_ALL 0x1000 +#endif + +#ifndef PERL_PV_ESCAPE_NOBACKSLASH +# define PERL_PV_ESCAPE_NOBACKSLASH 0x2000 +#endif + +#ifndef PERL_PV_ESCAPE_NOCLEAR +# define PERL_PV_ESCAPE_NOCLEAR 0x4000 +#endif + +#ifndef PERL_PV_ESCAPE_RE +# define PERL_PV_ESCAPE_RE 0x8000 +#endif + +#ifndef PERL_PV_PRETTY_NOCLEAR +# define PERL_PV_PRETTY_NOCLEAR PERL_PV_ESCAPE_NOCLEAR +#endif +#ifndef PERL_PV_PRETTY_DUMP +# define PERL_PV_PRETTY_DUMP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_QUOTE +#endif + +#ifndef PERL_PV_PRETTY_REGPROP +# define PERL_PV_PRETTY_REGPROP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_LTGT|PERL_PV_ESCAPE_RE +#endif + +/* Hint: pv_escape + * Note that unicode functionality is only backported to + * those perl versions that support it. For older perl + * versions, the implementation will fall back to bytes. + */ + +#ifndef pv_escape +#if defined(NEED_pv_escape) +static char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); +static +#else +extern char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); +#endif + +#ifdef pv_escape +# undef pv_escape +#endif +#define pv_escape(a,b,c,d,e,f) DPPP_(my_pv_escape)(aTHX_ a,b,c,d,e,f) +#define Perl_pv_escape DPPP_(my_pv_escape) + +#if defined(NEED_pv_escape) || defined(NEED_pv_escape_GLOBAL) + +char * +DPPP_(my_pv_escape)(pTHX_ SV *dsv, char const * const str, + const STRLEN count, const STRLEN max, + STRLEN * const escaped, const U32 flags) +{ + const char esc = flags & PERL_PV_ESCAPE_RE ? '%' : '\\'; + const char dq = flags & PERL_PV_ESCAPE_QUOTE ? '"' : esc; + char octbuf[32] = "%123456789ABCDF"; + STRLEN wrote = 0; + STRLEN chsize = 0; + STRLEN readsize = 1; +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + bool isuni = flags & PERL_PV_ESCAPE_UNI ? 1 : 0; +#endif + const char *pv = str; + const char * const end = pv + count; + octbuf[0] = esc; + + if (!(flags & PERL_PV_ESCAPE_NOCLEAR)) + sv_setpvs(dsv, ""); + +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + if ((flags & PERL_PV_ESCAPE_UNI_DETECT) && is_utf8_string((U8*)pv, count)) + isuni = 1; +#endif + + for (; pv < end && (!max || wrote < max) ; pv += readsize) { + const UV u = +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + isuni ? utf8_to_uvchr((U8*)pv, &readsize) : +#endif + (U8)*pv; + const U8 c = (U8)u & 0xFF; + + if (u > 255 || (flags & PERL_PV_ESCAPE_ALL)) { + if (flags & PERL_PV_ESCAPE_FIRSTCHAR) + chsize = my_snprintf(octbuf, sizeof octbuf, + "%"UVxf, u); + else + chsize = my_snprintf(octbuf, sizeof octbuf, + "%cx{%"UVxf"}", esc, u); + } else if (flags & PERL_PV_ESCAPE_NOBACKSLASH) { + chsize = 1; + } else { + if (c == dq || c == esc || !isPRINT(c)) { + chsize = 2; + switch (c) { + case '\\' : /* fallthrough */ + case '%' : if (c == esc) + octbuf[1] = esc; + else + chsize = 1; + break; + case '\v' : octbuf[1] = 'v'; break; + case '\t' : octbuf[1] = 't'; break; + case '\r' : octbuf[1] = 'r'; break; + case '\n' : octbuf[1] = 'n'; break; + case '\f' : octbuf[1] = 'f'; break; + case '"' : if (dq == '"') + octbuf[1] = '"'; + else + chsize = 1; + break; + default: chsize = my_snprintf(octbuf, sizeof octbuf, + pv < end && isDIGIT((U8)*(pv+readsize)) + ? "%c%03o" : "%c%o", esc, c); + } + } else { + chsize = 1; + } + } + if (max && wrote + chsize > max) { + break; + } else if (chsize > 1) { + sv_catpvn(dsv, octbuf, chsize); + wrote += chsize; + } else { + char tmp[2]; + my_snprintf(tmp, sizeof tmp, "%c", c); + sv_catpvn(dsv, tmp, 1); + wrote++; + } + if (flags & PERL_PV_ESCAPE_FIRSTCHAR) + break; + } + if (escaped != NULL) + *escaped= pv - str; + return SvPVX(dsv); +} + +#endif +#endif + +#ifndef pv_pretty +#if defined(NEED_pv_pretty) +static char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); +static +#else +extern char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); +#endif + +#ifdef pv_pretty +# undef pv_pretty +#endif +#define pv_pretty(a,b,c,d,e,f,g) DPPP_(my_pv_pretty)(aTHX_ a,b,c,d,e,f,g) +#define Perl_pv_pretty DPPP_(my_pv_pretty) + +#if defined(NEED_pv_pretty) || defined(NEED_pv_pretty_GLOBAL) + +char * +DPPP_(my_pv_pretty)(pTHX_ SV *dsv, char const * const str, const STRLEN count, + const STRLEN max, char const * const start_color, char const * const end_color, + const U32 flags) +{ + const U8 dq = (flags & PERL_PV_PRETTY_QUOTE) ? '"' : '%'; + STRLEN escaped; + + if (!(flags & PERL_PV_PRETTY_NOCLEAR)) + sv_setpvs(dsv, ""); + + if (dq == '"') + sv_catpvs(dsv, "\""); + else if (flags & PERL_PV_PRETTY_LTGT) + sv_catpvs(dsv, "<"); + + if (start_color != NULL) + sv_catpv(dsv, D_PPP_CONSTPV_ARG(start_color)); + + pv_escape(dsv, str, count, max, &escaped, flags | PERL_PV_ESCAPE_NOCLEAR); + + if (end_color != NULL) + sv_catpv(dsv, D_PPP_CONSTPV_ARG(end_color)); + + if (dq == '"') + sv_catpvs(dsv, "\""); + else if (flags & PERL_PV_PRETTY_LTGT) + sv_catpvs(dsv, ">"); + + if ((flags & PERL_PV_PRETTY_ELLIPSES) && escaped < count) + sv_catpvs(dsv, "..."); + + return SvPVX(dsv); +} + +#endif +#endif + +#ifndef pv_display +#if defined(NEED_pv_display) +static char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); +static +#else +extern char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); +#endif + +#ifdef pv_display +# undef pv_display +#endif +#define pv_display(a,b,c,d,e) DPPP_(my_pv_display)(aTHX_ a,b,c,d,e) +#define Perl_pv_display DPPP_(my_pv_display) + +#if defined(NEED_pv_display) || defined(NEED_pv_display_GLOBAL) + +char * +DPPP_(my_pv_display)(pTHX_ SV *dsv, const char *pv, STRLEN cur, STRLEN len, STRLEN pvlim) +{ + pv_pretty(dsv, pv, cur, pvlim, NULL, NULL, PERL_PV_PRETTY_DUMP); + if (len > cur && pv[cur] == '\0') + sv_catpvs(dsv, "\\0"); + return SvPVX(dsv); +} + +#endif +#endif + +#endif /* _P_P_PORTABILITY_H_ */ + +/* End of File ppport.h */ diff --git a/perl/t/00_compile.t b/perl/t/00_compile.t new file mode 100644 index 0000000..66fe8f0 --- /dev/null +++ b/perl/t/00_compile.t @@ -0,0 +1,6 @@ +use strict; +use warnings; +use Test::More tests => 1; + +use_ok 'Data::MessagePack'; + diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t new file mode 100644 index 0000000..0917f04 --- /dev/null +++ b/perl/t/01_pack.t @@ -0,0 +1,61 @@ +use t::Util; +use Test::More; +use Data::MessagePack; + +sub packit { + local $_ = unpack("H*", Data::MessagePack->pack($_[0])); + s/(..)/$1 /g; + s/ $//; + $_; +} + +sub pis ($$) { + is packit($_[0]), $_[1], 'dump ' . $_[1]; +} + +my @dat = ( + 0, '00', + 1, '01', + 127, '7f', + 128, 'cc 80', + 255, 'cc ff', + 256, 'cd 01 00', + 65535, 'cd ff ff', + 65536, 'ce 00 01 00 00', + -1, 'ff', + -32, 'e0', + -33, 'd0 df', + -128, 'd0 80', + -129, 'd1 ff 7f', + -32768, 'd1 80 00', + -32769, 'd2 ff ff 7f ff', + 1.0, 'cb 3f f0 00 00 00 00 00 00', + "", 'a0', + "a", 'a1 61', + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'bf 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61', + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'da 00 20 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61', + undef, 'c0', + Data::MessagePack::true(), 'c3', + Data::MessagePack::false(), 'c2', + [], '90', + [+[]], '91 90', + [[], undef], '92 90 c0', + {'a', 0}, '81 a1 61 00', + 8388608, 'ce 00 80 00 00', + + [undef, false, true], '93 c0 c2 c3', + ["", "a", "bc", "def"], '94 a0 a1 61 a2 62 63 a3 64 65 66', + [[], [[undef]]], '92 90 91 91 c0', + [undef, false, true], '93 c0 c2 c3', + [[0, 64, 127], [-32, -16, -1]], '92 93 00 40 7f 93 e0 f0 ff', + [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '99 00 d0 80 ff 00 d1 80 00 ff 00 d2 80 00 00 00 ff', + 2147483648, 'ce 80 00 00 00', + -2147483648, 'd2 80 00 00 00', +); +@dat = (2147483648, 'ce 80 00 00 00'); +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $iunpack($v); +} + +sub pis ($$) { + is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $inew; +sub unpackit { + my $v = $_[0]; + $v =~ s/ //g; + $v = pack 'H*', $v; + $up->reset; + my $ret = $up->execute($v, 0); + if ($ret != length($v)) { + fail "extra bytes"; + } + return $up->data; +} + +sub pis ($$) { + is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2) + 1; + +isa_ok $up, 'Data::MessagePack::Unpacker'; +for (my $i=0; $iunpack( + Data::MessagePack->pack($_[0]), + ); +} + +sub pis ($) { + is_deeply invert($_[0]), $_[0], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $iimport; + warnings->import; + + no strict 'refs'; + *{"$pkg\::true"} = sub () { + Data::MessagePack::true() + }; + *{"$pkg\::false"} = sub () { + Data::MessagePack::false() + }; +} + +1; diff --git a/perl/t/data.pl b/perl/t/data.pl new file mode 100644 index 0000000..1a2b2b4 --- /dev/null +++ b/perl/t/data.pl @@ -0,0 +1,15 @@ +no warnings 'uninitialized'; # i need this. i need this. +( + '93 c0 c2 c3' => [undef, false, true], + '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"], + '92 90 91 91 c0', [[], [[undef]]], + '93 c0 c2 c3', [undef, false, true], + 'ce 80 00 00 00', 2147483648, + '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + '92 93 00 40 7f 93 e0 f0 ff', [[0, 64, 127], [-32, -16, -1]], + '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', [[], [undef], [false, true], [], [undef], [false, true]], + '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', ["", "a", "ab", "", "a", "ab"], + '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], + '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}}, + '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], +) diff --git a/perl/unpack.c b/perl/unpack.c new file mode 100644 index 0000000..e5e069e --- /dev/null +++ b/perl/unpack.c @@ -0,0 +1,268 @@ +#ifdef __cplusplus +extern "C" { +#endif + +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" + +#ifdef __cplusplus +}; +#endif + +typedef struct { + int finished; + SV* source; +} unpack_user; + +#include "msgpack/unpack_define.h" + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object SV* + +#define msgpack_unpack_user unpack_user + +struct template_context; +typedef struct template_context msgpack_unpack_t; + +static void template_init(msgpack_unpack_t* u); + +static SV* template_data(msgpack_unpack_t* u); + +static int template_execute(msgpack_unpack_t* u, + const char* data, size_t len, size_t* off); + +static inline SV* template_callback_root(unpack_user* u) +{ return &PL_sv_undef; } + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, SV** o) +{ *o = newSViv(d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, SV** o) +{ *o = newSVnv(d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, SV** o) +{ *o = newSVnv(d); return 0; } + +static inline int template_callback_nil(unpack_user* u, SV** o) +{ *o = &PL_sv_undef; return 0; } + +static inline int template_callback_true(unpack_user* u, SV** o) +{ *o = &PL_sv_yes; return 0; } + +static inline int template_callback_false(unpack_user* u, SV** o) +{ *o = &PL_sv_no; return 0;} + +static inline int template_callback_array(unpack_user* u, unsigned int n, SV** o) +{ AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; } + +static inline int template_callback_array_item(unpack_user* u, SV** c, SV* o) +{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] + +static inline int template_callback_map(unpack_user* u, unsigned int n, SV** o) +{ HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; } + +static inline int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) +{ hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) +{ *o = (l == 0) ? newSVpv("", 0) : newSVpv(p, l); return 0; } + +#define UNPACKER(from, name) \ + msgpack_unpack_t *name; \ + name = INT2PTR(msgpack_unpack_t*, SvROK((from)) ? SvIV(SvRV((from))) : SvIV((from))); \ + if(name == NULL) { \ + Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \ + } + +#include "msgpack/unpack_template.h" + +SV* _msgpack_unpack(SV* data, int limit) { + msgpack_unpack_t mp; + unpack_user u = {0, &PL_sv_undef}; + int ret; + size_t from = 0; + STRLEN dlen; + const char * dptr = SvPV_const(data, dlen); + + template_init(&mp); + mp.user = u; + + mp.user.source = data; + ret = template_execute(&mp, dptr, (size_t)dlen, &from); + mp.user.source = &PL_sv_undef; + + if(ret < 0) { + Perl_croak(aTHX_ "parse error."); + } else if(ret == 0) { + Perl_croak(aTHX_ "insufficient bytes."); + } else { + if(from < dlen) { + Perl_croak(aTHX_ "extra bytes."); + } + return template_data(&mp); + } +} + +XS(xs_unpack_limit) { + dXSARGS; + + if (items != 3) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata', $limit)"); + } + + { + int limit = SvIV(ST(2)); + ST(0) = _msgpack_unpack(ST(1), limit); + } + XSRETURN(1); +} + + +XS(xs_unpack) { + dXSARGS; + msgpack_unpack_t mp; + + if (items != 2) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata')"); + } + + { + ST(0) = _msgpack_unpack(ST(1), sv_len(ST(1))); + } + + XSRETURN(1); +} + +/* ------------------------------ stream -- */ + +static void _reset(SV* self) { + UNPACKER(self, mp); + template_init(mp); + unpack_user u = {0, &PL_sv_undef}; + mp->user = u; +} + +XS(xs_unpacker_new) { + dXSARGS; + SV* self = sv_newmortal(); + msgpack_unpack_t *mp; + + Newx(mp, 1, msgpack_unpack_t); + + sv_setref_pv(self, "Data::MessagePack::Unpacker", mp); + _reset(self); + + ST(0) = self; + XSRETURN(1); +} + +static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { + UNPACKER(self, mp); + + size_t from = off; + const char* dptr = SvPV_nolen_const(data); + long dlen = limit; + int ret; + + if(from >= dlen) { + Perl_croak(aTHX_ "offset is bigger than data buffer size."); + } + + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = &PL_sv_undef; + + if(ret < 0) { + Perl_croak(aTHX_ "parse error."); + } else if(ret > 0) { + mp->user.finished = 1; + return newSVuv(from); + } else { + mp->user.finished = 0; + return newSVuv(from); + } +} + +XS(xs_unpacker_execute) { + dXSARGS; + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + + ST(0) = _execute_impl(self, data, off, sv_len(data)); + + XSRETURN(1); +} + +XS(xs_unpacker_execute_limit) { + dXSARGS; + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + IV limit = SvIV(ST(3)); + + ST(0) = _execute_impl(self, data, off, limit); + + XSRETURN(1); +} + +XS(xs_unpacker_is_finished) { + dXSARGS; + + UNPACKER(ST(0), mp); + ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no; + + XSRETURN(1); +} + +XS(xs_unpacker_data) { + dXSARGS; + + UNPACKER(ST(0), mp); + ST(0) = template_data(mp); + + XSRETURN(1); +} + +XS(xs_unpacker_reset) { + dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->reset()"); + } + + _reset(ST(0)); + + XSRETURN(0); +} + diff --git a/perl/xt/99_pod.t b/perl/xt/99_pod.t new file mode 100644 index 0000000..437887a --- /dev/null +++ b/perl/xt/99_pod.t @@ -0,0 +1,4 @@ +use Test::More; +eval "use Test::Pod 1.00"; +plan skip_all => "Test::Pod 1.00 required for testing POD" if $@; +all_pod_files_ok(); From d449b1d20d680242553cd692a8d3e00112f6ae7e Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 12:55:41 +0900 Subject: [PATCH 0112/1648] added perl support --- perl/Makefile.PL | 34 + perl/MessagePack.c | 39 + perl/benchmark/p1.pl | 16 + perl/lib/Data/MessagePack.pm | 33 + perl/lib/Data/MessagePack/Unpacker.pod | 52 + perl/pack.c | 122 + perl/ppport.h | 6984 ++++++++++++++++++++++++ perl/t/00_compile.t | 6 + perl/t/01_pack.t | 61 + perl/t/02_unpack.t | 24 + perl/t/03_stream_unpack.t | 32 + perl/t/04_invert.t | 24 + perl/t/Util.pm | 20 + perl/t/data.pl | 15 + perl/unpack.c | 268 + perl/xt/99_pod.t | 4 + 16 files changed, 7734 insertions(+) create mode 100644 perl/Makefile.PL create mode 100644 perl/MessagePack.c create mode 100644 perl/benchmark/p1.pl create mode 100644 perl/lib/Data/MessagePack.pm create mode 100644 perl/lib/Data/MessagePack/Unpacker.pod create mode 100644 perl/pack.c create mode 100644 perl/ppport.h create mode 100644 perl/t/00_compile.t create mode 100644 perl/t/01_pack.t create mode 100644 perl/t/02_unpack.t create mode 100644 perl/t/03_stream_unpack.t create mode 100644 perl/t/04_invert.t create mode 100644 perl/t/Util.pm create mode 100644 perl/t/data.pl create mode 100644 perl/unpack.c create mode 100644 perl/xt/99_pod.t diff --git a/perl/Makefile.PL b/perl/Makefile.PL new file mode 100644 index 0000000..f60e125 --- /dev/null +++ b/perl/Makefile.PL @@ -0,0 +1,34 @@ +use inc::Module::Install; +name 'Data-MessagePack'; +all_from 'lib/Data/MessagePack.pm'; + +perl_version '5.008005'; +license 'perl'; +can_cc or die "This module requires a C compiler"; + +my $ccflags = '-I../ '; + +makemaker_args( + OBJECT => '$(O_FILES)', + LIBS => [''], + CCFLAGS => $ccflags, + clean => { + FILES => q{ + *.stackdump + *.gcov *.gcda *.gcno + *.out + nytprof + cover_db + }, + }, +); + +tests 't/*.t'; +author_tests('xt'); + +auto_set_repository; +build_requires 'Test::More'; +use_test_base; +auto_include; +WriteAll; + diff --git a/perl/MessagePack.c b/perl/MessagePack.c new file mode 100644 index 0000000..c40e46a --- /dev/null +++ b/perl/MessagePack.c @@ -0,0 +1,39 @@ +#ifdef __cplusplus +extern "C" { +#endif +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" +#ifdef __cplusplus +}; +#endif + +XS(xs_pack); +XS(xs_unpack); +XS(xs_unpacker_new); +XS(xs_unpacker_execute); +XS(xs_unpacker_execute_limit); +XS(xs_unpacker_is_finished); +XS(xs_unpacker_data); +XS(xs_unpacker_reset); + +XS(boot_Data__MessagePack) { + dXSARGS; + HV * stash; + + newXS("Data::MessagePack::pack", xs_pack, __FILE__); + newXS("Data::MessagePack::unpack", xs_unpack, __FILE__); + stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE); + newCONSTSUB(stash, "true", &PL_sv_yes); + newCONSTSUB(stash, "false", &PL_sv_no); + + newXS("Data::MessagePack::Unpacker::new", xs_unpacker_new, __FILE__); + newXS("Data::MessagePack::Unpacker::execute", xs_unpacker_execute, __FILE__); + newXS("Data::MessagePack::Unpacker::execute_limit", xs_unpacker_execute_limit, __FILE__); + newXS("Data::MessagePack::Unpacker::is_finished", xs_unpacker_is_finished, __FILE__); + newXS("Data::MessagePack::Unpacker::data", xs_unpacker_data, __FILE__); + newXS("Data::MessagePack::Unpacker::reset", xs_unpacker_reset, __FILE__); + +} + diff --git a/perl/benchmark/p1.pl b/perl/benchmark/p1.pl new file mode 100644 index 0000000..257932e --- /dev/null +++ b/perl/benchmark/p1.pl @@ -0,0 +1,16 @@ +use strict; +use warnings; +use Data::MessagePack; +use JSON::XS; +use Benchmark ':all'; + +my $a = [0..2**24]; + +print "-- serialize\n"; +cmpthese( + -1 => { + json => sub { JSON::XS::encode_json($a) }, + mp => sub { Data::MessagePack->pack($a) }, + } +); + diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm new file mode 100644 index 0000000..d26bf9e --- /dev/null +++ b/perl/lib/Data/MessagePack.pm @@ -0,0 +1,33 @@ +package Data::MessagePack; +use strict; +use warnings; +use XSLoader; + +our $VERSION = 0.01; + +XSLoader::load(__PACKAGE__, $VERSION); + +1; +__END__ + +=head1 NAME + +Data::MessagePack - messagepack + +=head1 SYNOPSIS + + my $packed = Data::MessagePack->pack($dat); + my $unpacked = Data::MessagePack->unpack($dat); + +=head1 DESCRIPTION + +Data::MessagePack is a binary packer for perl. + +=head1 AUTHORS + +Tokuhiro Matsuno + +=head1 SEE ALSO + +L + diff --git a/perl/lib/Data/MessagePack/Unpacker.pod b/perl/lib/Data/MessagePack/Unpacker.pod new file mode 100644 index 0000000..61cbd21 --- /dev/null +++ b/perl/lib/Data/MessagePack/Unpacker.pod @@ -0,0 +1,52 @@ +=head1 NAME + +Data::MessagePack::Unpacker - messagepack streaming deserializer + +=head1 SYNOPSIS + + use Data::Dumper; + my $up = Data::MessagePack::Unpacker->new; + my $ret = $up->execute($v, 0); + if ($ret != length($v)) { + fail "extra bytes"; + } + return Dumper($up->data); + +=head1 DESCRIPTION + +This is an streaming deserializer for messagepack. + +=head1 METHODS + +=over 4 + +=item my $up = Data::MessagePack::Unpacker->new() + +create new stream deserializer + +=item $up->execute() + +=item $up->execute_limit() + +=item $up->is_finished() + +is this deserializer finished? + +=item $up->data() + +returns deserialized object. + +=item $up->reset() + +reset the stream deserializer, without memory zone. + +=back + +=head1 AUTHORS + +Tokuhiro Matsuno + +=head1 SEE ALSO + +L + diff --git a/perl/pack.c b/perl/pack.c new file mode 100644 index 0000000..5bb667b --- /dev/null +++ b/perl/pack.c @@ -0,0 +1,122 @@ +#ifdef __cplusplus +extern "C" { +#endif +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" +#ifdef __cplusplus +}; +#endif + +#include "msgpack/pack_define.h" + +#define msgpack_pack_inline_func(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_user SV* + +#define msgpack_pack_append_buffer(user, buf, len) \ + sv_catpvn(user, (const char*)(buf), len); + +#include "msgpack/pack_template.h" + +#define _PACK_WRAPPER(t) msgpack_pack_##t +#define PACK_WRAPPER(t) _PACK_WRAPPER(t) + +// move to pack.c +static void _msgpack_pack_sv(SV* buf, SV* val) { + if (val==NULL) { + msgpack_pack_nil(buf); + return; + } + + switch (SvTYPE(val)) { + case SVt_NULL: + msgpack_pack_nil(buf); + break; + case SVt_IV: + if (SvIOK_UV(val)) { + msgpack_pack_uint32(buf, SvUV(val)); + } else { + PACK_WRAPPER(IVTYPE)(buf, SvIV(val)); + } + break; + case SVt_PVNV: + { + STRLEN len = 0; + char *pv = SvPV(val, len); + if (len == 1 && *pv == '1') { + msgpack_pack_true(buf); + } else if (len == 0 && *pv==0) { + msgpack_pack_false(buf); + } else { + msgpack_pack_nil(buf); + } + } + break; + case SVt_PV: + { + STRLEN len; + char * cval = SvPV(val, len); + msgpack_pack_raw(buf, len); + msgpack_pack_raw_body(buf, cval, len); + } + break; + case SVt_NV: + PACK_WRAPPER(NVTYPE)(buf, SvNV(val)); + break; + case SVt_PVAV: + { + AV* ary = (AV*)val; + int len = av_len(ary) + 1; + int i; + msgpack_pack_array(buf, len); + for (i=0; i is designed to support operation with Perl +installations back to 5.003, and has been tested up to 5.10.0. + +=head1 OPTIONS + +=head2 --help + +Display a brief usage summary. + +=head2 --version + +Display the version of F. + +=head2 --patch=I + +If this option is given, a single patch file will be created if +any changes are suggested. This requires a working diff program +to be installed on your system. + +=head2 --copy=I + +If this option is given, a copy of each file will be saved with +the given suffix that contains the suggested changes. This does +not require any external programs. Note that this does not +automagially add a dot between the original filename and the +suffix. If you want the dot, you have to include it in the option +argument. + +If neither C<--patch> or C<--copy> are given, the default is to +simply print the diffs for each file. This requires either +C or a C program to be installed. + +=head2 --diff=I + +Manually set the diff program and options to use. The default +is to use C, when installed, and output unified +context diffs. + +=head2 --compat-version=I + +Tell F to check for compatibility with the given +Perl version. The default is to check for compatibility with Perl +version 5.003. You can use this option to reduce the output +of F if you intend to be backward compatible only +down to a certain Perl version. + +=head2 --cplusplus + +Usually, F will detect C++ style comments and +replace them with C style comments for portability reasons. +Using this option instructs F to leave C++ +comments untouched. + +=head2 --quiet + +Be quiet. Don't print anything except fatal errors. + +=head2 --nodiag + +Don't output any diagnostic messages. Only portability +alerts will be printed. + +=head2 --nohints + +Don't output any hints. Hints often contain useful portability +notes. Warnings will still be displayed. + +=head2 --nochanges + +Don't suggest any changes. Only give diagnostic output and hints +unless these are also deactivated. + +=head2 --nofilter + +Don't filter the list of input files. By default, files not looking +like source code (i.e. not *.xs, *.c, *.cc, *.cpp or *.h) are skipped. + +=head2 --strip + +Strip all script and documentation functionality from F. +This reduces the size of F dramatically and may be useful +if you want to include F in smaller modules without +increasing their distribution size too much. + +The stripped F will have a C<--unstrip> option that allows +you to undo the stripping, but only if an appropriate C +module is installed. + +=head2 --list-provided + +Lists the API elements for which compatibility is provided by +F. Also lists if it must be explicitly requested, +if it has dependencies, and if there are hints or warnings for it. + +=head2 --list-unsupported + +Lists the API elements that are known not to be supported by +F and below which version of Perl they probably +won't be available or work. + +=head2 --api-info=I + +Show portability information for API elements matching I. +If I is surrounded by slashes, it is interpreted as a regular +expression. + +=head1 DESCRIPTION + +In order for a Perl extension (XS) module to be as portable as possible +across differing versions of Perl itself, certain steps need to be taken. + +=over 4 + +=item * + +Including this header is the first major one. This alone will give you +access to a large part of the Perl API that hasn't been available in +earlier Perl releases. Use + + perl ppport.h --list-provided + +to see which API elements are provided by ppport.h. + +=item * + +You should avoid using deprecated parts of the API. For example, using +global Perl variables without the C prefix is deprecated. Also, +some API functions used to have a C prefix. Using this form is +also deprecated. You can safely use the supported API, as F +will provide wrappers for older Perl versions. + +=item * + +If you use one of a few functions or variables that were not present in +earlier versions of Perl, and that can't be provided using a macro, you +have to explicitly request support for these functions by adding one or +more C<#define>s in your source code before the inclusion of F. + +These functions or variables will be marked C in the list shown +by C<--list-provided>. + +Depending on whether you module has a single or multiple files that +use such functions or variables, you want either C or global +variants. + +For a C function or variable (used only in a single source +file), use: + + #define NEED_function + #define NEED_variable + +For a global function or variable (used in multiple source files), +use: + + #define NEED_function_GLOBAL + #define NEED_variable_GLOBAL + +Note that you mustn't have more than one global request for the +same function or variable in your project. + + Function / Variable Static Request Global Request + ----------------------------------------------------------------------------------------- + PL_parser NEED_PL_parser NEED_PL_parser_GLOBAL + PL_signals NEED_PL_signals NEED_PL_signals_GLOBAL + eval_pv() NEED_eval_pv NEED_eval_pv_GLOBAL + grok_bin() NEED_grok_bin NEED_grok_bin_GLOBAL + grok_hex() NEED_grok_hex NEED_grok_hex_GLOBAL + grok_number() NEED_grok_number NEED_grok_number_GLOBAL + grok_numeric_radix() NEED_grok_numeric_radix NEED_grok_numeric_radix_GLOBAL + grok_oct() NEED_grok_oct NEED_grok_oct_GLOBAL + load_module() NEED_load_module NEED_load_module_GLOBAL + my_snprintf() NEED_my_snprintf NEED_my_snprintf_GLOBAL + my_sprintf() NEED_my_sprintf NEED_my_sprintf_GLOBAL + my_strlcat() NEED_my_strlcat NEED_my_strlcat_GLOBAL + my_strlcpy() NEED_my_strlcpy NEED_my_strlcpy_GLOBAL + newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL + newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL + newSVpvn_flags() NEED_newSVpvn_flags NEED_newSVpvn_flags_GLOBAL + newSVpvn_share() NEED_newSVpvn_share NEED_newSVpvn_share_GLOBAL + pv_display() NEED_pv_display NEED_pv_display_GLOBAL + pv_escape() NEED_pv_escape NEED_pv_escape_GLOBAL + pv_pretty() NEED_pv_pretty NEED_pv_pretty_GLOBAL + sv_2pv_flags() NEED_sv_2pv_flags NEED_sv_2pv_flags_GLOBAL + sv_2pvbyte() NEED_sv_2pvbyte NEED_sv_2pvbyte_GLOBAL + sv_catpvf_mg() NEED_sv_catpvf_mg NEED_sv_catpvf_mg_GLOBAL + sv_catpvf_mg_nocontext() NEED_sv_catpvf_mg_nocontext NEED_sv_catpvf_mg_nocontext_GLOBAL + sv_pvn_force_flags() NEED_sv_pvn_force_flags NEED_sv_pvn_force_flags_GLOBAL + sv_setpvf_mg() NEED_sv_setpvf_mg NEED_sv_setpvf_mg_GLOBAL + sv_setpvf_mg_nocontext() NEED_sv_setpvf_mg_nocontext NEED_sv_setpvf_mg_nocontext_GLOBAL + vload_module() NEED_vload_module NEED_vload_module_GLOBAL + vnewSVpvf() NEED_vnewSVpvf NEED_vnewSVpvf_GLOBAL + warner() NEED_warner NEED_warner_GLOBAL + +To avoid namespace conflicts, you can change the namespace of the +explicitly exported functions / variables using the C +macro. Just C<#define> the macro before including C: + + #define DPPP_NAMESPACE MyOwnNamespace_ + #include "ppport.h" + +The default namespace is C. + +=back + +The good thing is that most of the above can be checked by running +F on your source code. See the next section for +details. + +=head1 EXAMPLES + +To verify whether F is needed for your module, whether you +should make any changes to your code, and whether any special defines +should be used, F can be run as a Perl script to check your +source code. Simply say: + + perl ppport.h + +The result will usually be a list of patches suggesting changes +that should at least be acceptable, if not necessarily the most +efficient solution, or a fix for all possible problems. + +If you know that your XS module uses features only available in +newer Perl releases, if you're aware that it uses C++ comments, +and if you want all suggestions as a single patch file, you could +use something like this: + + perl ppport.h --compat-version=5.6.0 --cplusplus --patch=test.diff + +If you only want your code to be scanned without any suggestions +for changes, use: + + perl ppport.h --nochanges + +You can specify a different C program or options, using +the C<--diff> option: + + perl ppport.h --diff='diff -C 10' + +This would output context diffs with 10 lines of context. + +If you want to create patched copies of your files instead, use: + + perl ppport.h --copy=.new + +To display portability information for the C function, +use: + + perl ppport.h --api-info=newSVpvn + +Since the argument to C<--api-info> can be a regular expression, +you can use + + perl ppport.h --api-info=/_nomg$/ + +to display portability information for all C<_nomg> functions or + + perl ppport.h --api-info=/./ + +to display information for all known API elements. + +=head1 BUGS + +If this version of F is causing failure during +the compilation of this module, please check if newer versions +of either this module or C are available on CPAN +before sending a bug report. + +If F was generated using the latest version of +C and is causing failure of this module, please +file a bug report using the CPAN Request Tracker at L. + +Please include the following information: + +=over 4 + +=item 1. + +The complete output from running "perl -V" + +=item 2. + +This file. + +=item 3. + +The name and version of the module you were trying to build. + +=item 4. + +A full log of the build that failed. + +=item 5. + +Any other information that you think could be relevant. + +=back + +For the latest version of this code, please get the C +module from CPAN. + +=head1 COPYRIGHT + +Version 3.x, Copyright (c) 2004-2009, Marcus Holland-Moritz. + +Version 2.x, Copyright (C) 2001, Paul Marquess. + +Version 1.x, Copyright (C) 1999, Kenneth Albanowski. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +=head1 SEE ALSO + +See L. + +=cut + +use strict; + +# Disable broken TRIE-optimization +BEGIN { eval '${^RE_TRIE_MAXBUF} = -1' if $] >= 5.009004 && $] <= 5.009005 } + +my $VERSION = 3.17; + +my %opt = ( + quiet => 0, + diag => 1, + hints => 1, + changes => 1, + cplusplus => 0, + filter => 1, + strip => 0, + version => 0, +); + +my($ppport) = $0 =~ /([\w.]+)$/; +my $LF = '(?:\r\n|[\r\n])'; # line feed +my $HS = "[ \t]"; # horizontal whitespace + +# Never use C comments in this file! +my $ccs = '/'.'*'; +my $cce = '*'.'/'; +my $rccs = quotemeta $ccs; +my $rcce = quotemeta $cce; + +eval { + require Getopt::Long; + Getopt::Long::GetOptions(\%opt, qw( + help quiet diag! filter! hints! changes! cplusplus strip version + patch=s copy=s diff=s compat-version=s + list-provided list-unsupported api-info=s + )) or usage(); +}; + +if ($@ and grep /^-/, @ARGV) { + usage() if "@ARGV" =~ /^--?h(?:elp)?$/; + die "Getopt::Long not found. Please don't use any options.\n"; +} + +if ($opt{version}) { + print "This is $0 $VERSION.\n"; + exit 0; +} + +usage() if $opt{help}; +strip() if $opt{strip}; + +if (exists $opt{'compat-version'}) { + my($r,$v,$s) = eval { parse_version($opt{'compat-version'}) }; + if ($@) { + die "Invalid version number format: '$opt{'compat-version'}'\n"; + } + die "Only Perl 5 is supported\n" if $r != 5; + die "Invalid version number: $opt{'compat-version'}\n" if $v >= 1000 || $s >= 1000; + $opt{'compat-version'} = sprintf "%d.%03d%03d", $r, $v, $s; +} +else { + $opt{'compat-version'} = 5; +} + +my %API = map { /^(\w+)\|([^|]*)\|([^|]*)\|(\w*)$/ + ? ( $1 => { + ($2 ? ( base => $2 ) : ()), + ($3 ? ( todo => $3 ) : ()), + (index($4, 'v') >= 0 ? ( varargs => 1 ) : ()), + (index($4, 'p') >= 0 ? ( provided => 1 ) : ()), + (index($4, 'n') >= 0 ? ( nothxarg => 1 ) : ()), + } ) + : die "invalid spec: $_" } qw( +AvFILLp|5.004050||p +AvFILL||| +CLASS|||n +CPERLscope|5.005000||p +CX_CURPAD_SAVE||| +CX_CURPAD_SV||| +CopFILEAV|5.006000||p +CopFILEGV_set|5.006000||p +CopFILEGV|5.006000||p +CopFILESV|5.006000||p +CopFILE_set|5.006000||p +CopFILE|5.006000||p +CopSTASHPV_set|5.006000||p +CopSTASHPV|5.006000||p +CopSTASH_eq|5.006000||p +CopSTASH_set|5.006000||p +CopSTASH|5.006000||p +CopyD|5.009002||p +Copy||| +CvPADLIST||| +CvSTASH||| +CvWEAKOUTSIDE||| +DEFSV_set|5.011000||p +DEFSV|5.004050||p +END_EXTERN_C|5.005000||p +ENTER||| +ERRSV|5.004050||p +EXTEND||| +EXTERN_C|5.005000||p +F0convert|||n +FREETMPS||| +GIMME_V||5.004000|n +GIMME|||n +GROK_NUMERIC_RADIX|5.007002||p +G_ARRAY||| +G_DISCARD||| +G_EVAL||| +G_METHOD|5.006001||p +G_NOARGS||| +G_SCALAR||| +G_VOID||5.004000| +GetVars||| +GvSV||| +Gv_AMupdate||| +HEf_SVKEY||5.004000| +HeHASH||5.004000| +HeKEY||5.004000| +HeKLEN||5.004000| +HePV||5.004000| +HeSVKEY_force||5.004000| +HeSVKEY_set||5.004000| +HeSVKEY||5.004000| +HeUTF8||5.011000| +HeVAL||5.004000| +HvNAME||| +INT2PTR|5.006000||p +IN_LOCALE_COMPILETIME|5.007002||p +IN_LOCALE_RUNTIME|5.007002||p +IN_LOCALE|5.007002||p +IN_PERL_COMPILETIME|5.008001||p +IS_NUMBER_GREATER_THAN_UV_MAX|5.007002||p +IS_NUMBER_INFINITY|5.007002||p +IS_NUMBER_IN_UV|5.007002||p +IS_NUMBER_NAN|5.007003||p +IS_NUMBER_NEG|5.007002||p +IS_NUMBER_NOT_INT|5.007002||p +IVSIZE|5.006000||p +IVTYPE|5.006000||p +IVdf|5.006000||p +LEAVE||| +LVRET||| +MARK||| +MULTICALL||5.011000| +MY_CXT_CLONE|5.009002||p +MY_CXT_INIT|5.007003||p +MY_CXT|5.007003||p +MoveD|5.009002||p +Move||| +NOOP|5.005000||p +NUM2PTR|5.006000||p +NVTYPE|5.006000||p +NVef|5.006001||p +NVff|5.006001||p +NVgf|5.006001||p +Newxc|5.009003||p +Newxz|5.009003||p +Newx|5.009003||p +Nullav||| +Nullch||| +Nullcv||| +Nullhv||| +Nullsv||| +ORIGMARK||| +PAD_BASE_SV||| +PAD_CLONE_VARS||| +PAD_COMPNAME_FLAGS||| +PAD_COMPNAME_GEN_set||| +PAD_COMPNAME_GEN||| +PAD_COMPNAME_OURSTASH||| +PAD_COMPNAME_PV||| +PAD_COMPNAME_TYPE||| +PAD_DUP||| +PAD_RESTORE_LOCAL||| +PAD_SAVE_LOCAL||| +PAD_SAVE_SETNULLPAD||| +PAD_SETSV||| +PAD_SET_CUR_NOSAVE||| +PAD_SET_CUR||| +PAD_SVl||| +PAD_SV||| +PERLIO_FUNCS_CAST|5.009003||p +PERLIO_FUNCS_DECL|5.009003||p +PERL_ABS|5.008001||p +PERL_BCDVERSION|5.011000||p +PERL_GCC_BRACE_GROUPS_FORBIDDEN|5.008001||p +PERL_HASH|5.004000||p +PERL_INT_MAX|5.004000||p +PERL_INT_MIN|5.004000||p +PERL_LONG_MAX|5.004000||p +PERL_LONG_MIN|5.004000||p +PERL_MAGIC_arylen|5.007002||p +PERL_MAGIC_backref|5.007002||p +PERL_MAGIC_bm|5.007002||p +PERL_MAGIC_collxfrm|5.007002||p +PERL_MAGIC_dbfile|5.007002||p +PERL_MAGIC_dbline|5.007002||p +PERL_MAGIC_defelem|5.007002||p +PERL_MAGIC_envelem|5.007002||p +PERL_MAGIC_env|5.007002||p +PERL_MAGIC_ext|5.007002||p +PERL_MAGIC_fm|5.007002||p +PERL_MAGIC_glob|5.011000||p +PERL_MAGIC_isaelem|5.007002||p +PERL_MAGIC_isa|5.007002||p +PERL_MAGIC_mutex|5.011000||p +PERL_MAGIC_nkeys|5.007002||p +PERL_MAGIC_overload_elem|5.007002||p +PERL_MAGIC_overload_table|5.007002||p +PERL_MAGIC_overload|5.007002||p +PERL_MAGIC_pos|5.007002||p +PERL_MAGIC_qr|5.007002||p +PERL_MAGIC_regdata|5.007002||p +PERL_MAGIC_regdatum|5.007002||p +PERL_MAGIC_regex_global|5.007002||p +PERL_MAGIC_shared_scalar|5.007003||p +PERL_MAGIC_shared|5.007003||p +PERL_MAGIC_sigelem|5.007002||p +PERL_MAGIC_sig|5.007002||p +PERL_MAGIC_substr|5.007002||p +PERL_MAGIC_sv|5.007002||p +PERL_MAGIC_taint|5.007002||p +PERL_MAGIC_tiedelem|5.007002||p +PERL_MAGIC_tiedscalar|5.007002||p +PERL_MAGIC_tied|5.007002||p +PERL_MAGIC_utf8|5.008001||p +PERL_MAGIC_uvar_elem|5.007003||p +PERL_MAGIC_uvar|5.007002||p +PERL_MAGIC_vec|5.007002||p +PERL_MAGIC_vstring|5.008001||p +PERL_PV_ESCAPE_ALL|5.009004||p +PERL_PV_ESCAPE_FIRSTCHAR|5.009004||p +PERL_PV_ESCAPE_NOBACKSLASH|5.009004||p +PERL_PV_ESCAPE_NOCLEAR|5.009004||p +PERL_PV_ESCAPE_QUOTE|5.009004||p +PERL_PV_ESCAPE_RE|5.009005||p +PERL_PV_ESCAPE_UNI_DETECT|5.009004||p +PERL_PV_ESCAPE_UNI|5.009004||p +PERL_PV_PRETTY_DUMP|5.009004||p +PERL_PV_PRETTY_ELLIPSES|5.010000||p +PERL_PV_PRETTY_LTGT|5.009004||p +PERL_PV_PRETTY_NOCLEAR|5.010000||p +PERL_PV_PRETTY_QUOTE|5.009004||p +PERL_PV_PRETTY_REGPROP|5.009004||p +PERL_QUAD_MAX|5.004000||p +PERL_QUAD_MIN|5.004000||p +PERL_REVISION|5.006000||p +PERL_SCAN_ALLOW_UNDERSCORES|5.007003||p +PERL_SCAN_DISALLOW_PREFIX|5.007003||p +PERL_SCAN_GREATER_THAN_UV_MAX|5.007003||p +PERL_SCAN_SILENT_ILLDIGIT|5.008001||p +PERL_SHORT_MAX|5.004000||p +PERL_SHORT_MIN|5.004000||p +PERL_SIGNALS_UNSAFE_FLAG|5.008001||p +PERL_SUBVERSION|5.006000||p +PERL_UCHAR_MAX|5.004000||p +PERL_UCHAR_MIN|5.004000||p +PERL_UINT_MAX|5.004000||p +PERL_UINT_MIN|5.004000||p +PERL_ULONG_MAX|5.004000||p +PERL_ULONG_MIN|5.004000||p +PERL_UNUSED_ARG|5.009003||p +PERL_UNUSED_CONTEXT|5.009004||p +PERL_UNUSED_DECL|5.007002||p +PERL_UNUSED_VAR|5.007002||p +PERL_UQUAD_MAX|5.004000||p +PERL_UQUAD_MIN|5.004000||p +PERL_USE_GCC_BRACE_GROUPS|5.009004||p +PERL_USHORT_MAX|5.004000||p +PERL_USHORT_MIN|5.004000||p +PERL_VERSION|5.006000||p +PL_DBsignal|5.005000||p +PL_DBsingle|||pn +PL_DBsub|||pn +PL_DBtrace|||pn +PL_Sv|5.005000||p +PL_bufend|5.011000||p +PL_bufptr|5.011000||p +PL_compiling|5.004050||p +PL_copline|5.011000||p +PL_curcop|5.004050||p +PL_curstash|5.004050||p +PL_debstash|5.004050||p +PL_defgv|5.004050||p +PL_diehook|5.004050||p +PL_dirty|5.004050||p +PL_dowarn|||pn +PL_errgv|5.004050||p +PL_expect|5.011000||p +PL_hexdigit|5.005000||p +PL_hints|5.005000||p +PL_last_in_gv|||n +PL_laststatval|5.005000||p +PL_lex_state|5.011000||p +PL_lex_stuff|5.011000||p +PL_linestr|5.011000||p +PL_modglobal||5.005000|n +PL_na|5.004050||pn +PL_no_modify|5.006000||p +PL_ofsgv|||n +PL_parser|5.009005||p +PL_perl_destruct_level|5.004050||p +PL_perldb|5.004050||p +PL_ppaddr|5.006000||p +PL_rsfp_filters|5.004050||p +PL_rsfp|5.004050||p +PL_rs|||n +PL_signals|5.008001||p +PL_stack_base|5.004050||p +PL_stack_sp|5.004050||p +PL_statcache|5.005000||p +PL_stdingv|5.004050||p +PL_sv_arenaroot|5.004050||p +PL_sv_no|5.004050||pn +PL_sv_undef|5.004050||pn +PL_sv_yes|5.004050||pn +PL_tainted|5.004050||p +PL_tainting|5.004050||p +PL_tokenbuf|5.011000||p +POP_MULTICALL||5.011000| +POPi|||n +POPl|||n +POPn|||n +POPpbytex||5.007001|n +POPpx||5.005030|n +POPp|||n +POPs|||n +PTR2IV|5.006000||p +PTR2NV|5.006000||p +PTR2UV|5.006000||p +PTR2nat|5.009003||p +PTR2ul|5.007001||p +PTRV|5.006000||p +PUSHMARK||| +PUSH_MULTICALL||5.011000| +PUSHi||| +PUSHmortal|5.009002||p +PUSHn||| +PUSHp||| +PUSHs||| +PUSHu|5.004000||p +PUTBACK||| +PerlIO_clearerr||5.007003| +PerlIO_close||5.007003| +PerlIO_context_layers||5.009004| +PerlIO_eof||5.007003| +PerlIO_error||5.007003| +PerlIO_fileno||5.007003| +PerlIO_fill||5.007003| +PerlIO_flush||5.007003| +PerlIO_get_base||5.007003| +PerlIO_get_bufsiz||5.007003| +PerlIO_get_cnt||5.007003| +PerlIO_get_ptr||5.007003| +PerlIO_read||5.007003| +PerlIO_seek||5.007003| +PerlIO_set_cnt||5.007003| +PerlIO_set_ptrcnt||5.007003| +PerlIO_setlinebuf||5.007003| +PerlIO_stderr||5.007003| +PerlIO_stdin||5.007003| +PerlIO_stdout||5.007003| +PerlIO_tell||5.007003| +PerlIO_unread||5.007003| +PerlIO_write||5.007003| +Perl_signbit||5.009005|n +PoisonFree|5.009004||p +PoisonNew|5.009004||p +PoisonWith|5.009004||p +Poison|5.008000||p +RETVAL|||n +Renewc||| +Renew||| +SAVECLEARSV||| +SAVECOMPPAD||| +SAVEPADSV||| +SAVETMPS||| +SAVE_DEFSV|5.004050||p +SPAGAIN||| +SP||| +START_EXTERN_C|5.005000||p +START_MY_CXT|5.007003||p +STMT_END|||p +STMT_START|||p +STR_WITH_LEN|5.009003||p +ST||| +SV_CONST_RETURN|5.009003||p +SV_COW_DROP_PV|5.008001||p +SV_COW_SHARED_HASH_KEYS|5.009005||p +SV_GMAGIC|5.007002||p +SV_HAS_TRAILING_NUL|5.009004||p +SV_IMMEDIATE_UNREF|5.007001||p +SV_MUTABLE_RETURN|5.009003||p +SV_NOSTEAL|5.009002||p +SV_SMAGIC|5.009003||p +SV_UTF8_NO_ENCODING|5.008001||p +SVf_UTF8|5.006000||p +SVf|5.006000||p +SVt_IV||| +SVt_NV||| +SVt_PVAV||| +SVt_PVCV||| +SVt_PVHV||| +SVt_PVMG||| +SVt_PV||| +Safefree||| +Slab_Alloc||| +Slab_Free||| +Slab_to_rw||| +StructCopy||| +SvCUR_set||| +SvCUR||| +SvEND||| +SvGAMAGIC||5.006001| +SvGETMAGIC|5.004050||p +SvGROW||| +SvIOK_UV||5.006000| +SvIOK_notUV||5.006000| +SvIOK_off||| +SvIOK_only_UV||5.006000| +SvIOK_only||| +SvIOK_on||| +SvIOKp||| +SvIOK||| +SvIVX||| +SvIV_nomg|5.009001||p +SvIV_set||| +SvIVx||| +SvIV||| +SvIsCOW_shared_hash||5.008003| +SvIsCOW||5.008003| +SvLEN_set||| +SvLEN||| +SvLOCK||5.007003| +SvMAGIC_set|5.009003||p +SvNIOK_off||| +SvNIOKp||| +SvNIOK||| +SvNOK_off||| +SvNOK_only||| +SvNOK_on||| +SvNOKp||| +SvNOK||| +SvNVX||| +SvNV_set||| +SvNVx||| +SvNV||| +SvOK||| +SvOOK_offset||5.011000| +SvOOK||| +SvPOK_off||| +SvPOK_only_UTF8||5.006000| +SvPOK_only||| +SvPOK_on||| +SvPOKp||| +SvPOK||| +SvPVX_const|5.009003||p +SvPVX_mutable|5.009003||p +SvPVX||| +SvPV_const|5.009003||p +SvPV_flags_const_nolen|5.009003||p +SvPV_flags_const|5.009003||p +SvPV_flags_mutable|5.009003||p +SvPV_flags|5.007002||p +SvPV_force_flags_mutable|5.009003||p +SvPV_force_flags_nolen|5.009003||p +SvPV_force_flags|5.007002||p +SvPV_force_mutable|5.009003||p +SvPV_force_nolen|5.009003||p +SvPV_force_nomg_nolen|5.009003||p +SvPV_force_nomg|5.007002||p +SvPV_force|||p +SvPV_mutable|5.009003||p +SvPV_nolen_const|5.009003||p +SvPV_nolen|5.006000||p +SvPV_nomg_const_nolen|5.009003||p +SvPV_nomg_const|5.009003||p +SvPV_nomg|5.007002||p +SvPV_renew|5.009003||p +SvPV_set||| +SvPVbyte_force||5.009002| +SvPVbyte_nolen||5.006000| +SvPVbytex_force||5.006000| +SvPVbytex||5.006000| +SvPVbyte|5.006000||p +SvPVutf8_force||5.006000| +SvPVutf8_nolen||5.006000| +SvPVutf8x_force||5.006000| +SvPVutf8x||5.006000| +SvPVutf8||5.006000| +SvPVx||| +SvPV||| +SvREFCNT_dec||| +SvREFCNT_inc_NN|5.009004||p +SvREFCNT_inc_simple_NN|5.009004||p +SvREFCNT_inc_simple_void_NN|5.009004||p +SvREFCNT_inc_simple_void|5.009004||p +SvREFCNT_inc_simple|5.009004||p +SvREFCNT_inc_void_NN|5.009004||p +SvREFCNT_inc_void|5.009004||p +SvREFCNT_inc|||p +SvREFCNT||| +SvROK_off||| +SvROK_on||| +SvROK||| +SvRV_set|5.009003||p +SvRV||| +SvRXOK||5.009005| +SvRX||5.009005| +SvSETMAGIC||| +SvSHARED_HASH|5.009003||p +SvSHARE||5.007003| +SvSTASH_set|5.009003||p +SvSTASH||| +SvSetMagicSV_nosteal||5.004000| +SvSetMagicSV||5.004000| +SvSetSV_nosteal||5.004000| +SvSetSV||| +SvTAINTED_off||5.004000| +SvTAINTED_on||5.004000| +SvTAINTED||5.004000| +SvTAINT||| +SvTRUE||| +SvTYPE||| +SvUNLOCK||5.007003| +SvUOK|5.007001|5.006000|p +SvUPGRADE||| +SvUTF8_off||5.006000| +SvUTF8_on||5.006000| +SvUTF8||5.006000| +SvUVXx|5.004000||p +SvUVX|5.004000||p +SvUV_nomg|5.009001||p +SvUV_set|5.009003||p +SvUVx|5.004000||p +SvUV|5.004000||p +SvVOK||5.008001| +SvVSTRING_mg|5.009004||p +THIS|||n +UNDERBAR|5.009002||p +UTF8_MAXBYTES|5.009002||p +UVSIZE|5.006000||p +UVTYPE|5.006000||p +UVXf|5.007001||p +UVof|5.006000||p +UVuf|5.006000||p +UVxf|5.006000||p +WARN_ALL|5.006000||p +WARN_AMBIGUOUS|5.006000||p +WARN_ASSERTIONS|5.011000||p +WARN_BAREWORD|5.006000||p +WARN_CLOSED|5.006000||p +WARN_CLOSURE|5.006000||p +WARN_DEBUGGING|5.006000||p +WARN_DEPRECATED|5.006000||p +WARN_DIGIT|5.006000||p +WARN_EXEC|5.006000||p +WARN_EXITING|5.006000||p +WARN_GLOB|5.006000||p +WARN_INPLACE|5.006000||p +WARN_INTERNAL|5.006000||p +WARN_IO|5.006000||p +WARN_LAYER|5.008000||p +WARN_MALLOC|5.006000||p +WARN_MISC|5.006000||p +WARN_NEWLINE|5.006000||p +WARN_NUMERIC|5.006000||p +WARN_ONCE|5.006000||p +WARN_OVERFLOW|5.006000||p +WARN_PACK|5.006000||p +WARN_PARENTHESIS|5.006000||p +WARN_PIPE|5.006000||p +WARN_PORTABLE|5.006000||p +WARN_PRECEDENCE|5.006000||p +WARN_PRINTF|5.006000||p +WARN_PROTOTYPE|5.006000||p +WARN_QW|5.006000||p +WARN_RECURSION|5.006000||p +WARN_REDEFINE|5.006000||p +WARN_REGEXP|5.006000||p +WARN_RESERVED|5.006000||p +WARN_SEMICOLON|5.006000||p +WARN_SEVERE|5.006000||p +WARN_SIGNAL|5.006000||p +WARN_SUBSTR|5.006000||p +WARN_SYNTAX|5.006000||p +WARN_TAINT|5.006000||p +WARN_THREADS|5.008000||p +WARN_UNINITIALIZED|5.006000||p +WARN_UNOPENED|5.006000||p +WARN_UNPACK|5.006000||p +WARN_UNTIE|5.006000||p +WARN_UTF8|5.006000||p +WARN_VOID|5.006000||p +XCPT_CATCH|5.009002||p +XCPT_RETHROW|5.009002||p +XCPT_TRY_END|5.009002||p +XCPT_TRY_START|5.009002||p +XPUSHi||| +XPUSHmortal|5.009002||p +XPUSHn||| +XPUSHp||| +XPUSHs||| +XPUSHu|5.004000||p +XSRETURN_EMPTY||| +XSRETURN_IV||| +XSRETURN_NO||| +XSRETURN_NV||| +XSRETURN_PV||| +XSRETURN_UNDEF||| +XSRETURN_UV|5.008001||p +XSRETURN_YES||| +XSRETURN|||p +XST_mIV||| +XST_mNO||| +XST_mNV||| +XST_mPV||| +XST_mUNDEF||| +XST_mUV|5.008001||p +XST_mYES||| +XS_VERSION_BOOTCHECK||| +XS_VERSION||| +XSprePUSH|5.006000||p +XS||| +ZeroD|5.009002||p +Zero||| +_aMY_CXT|5.007003||p +_pMY_CXT|5.007003||p +aMY_CXT_|5.007003||p +aMY_CXT|5.007003||p +aTHXR_|5.011000||p +aTHXR|5.011000||p +aTHX_|5.006000||p +aTHX|5.006000||p +add_data|||n +addmad||| +allocmy||| +amagic_call||| +amagic_cmp_locale||| +amagic_cmp||| +amagic_i_ncmp||| +amagic_ncmp||| +any_dup||| +ao||| +append_elem||| +append_list||| +append_madprops||| +apply_attrs_my||| +apply_attrs_string||5.006001| +apply_attrs||| +apply||| +atfork_lock||5.007003|n +atfork_unlock||5.007003|n +av_arylen_p||5.009003| +av_clear||| +av_create_and_push||5.009005| +av_create_and_unshift_one||5.009005| +av_delete||5.006000| +av_exists||5.006000| +av_extend||| +av_fetch||| +av_fill||| +av_iter_p||5.011000| +av_len||| +av_make||| +av_pop||| +av_push||| +av_reify||| +av_shift||| +av_store||| +av_undef||| +av_unshift||| +ax|||n +bad_type||| +bind_match||| +block_end||| +block_gimme||5.004000| +block_start||| +boolSV|5.004000||p +boot_core_PerlIO||| +boot_core_UNIVERSAL||| +boot_core_mro||| +boot_core_xsutils||| +bytes_from_utf8||5.007001| +bytes_to_uni|||n +bytes_to_utf8||5.006001| +call_argv|5.006000||p +call_atexit||5.006000| +call_list||5.004000| +call_method|5.006000||p +call_pv|5.006000||p +call_sv|5.006000||p +calloc||5.007002|n +cando||| +cast_i32||5.006000| +cast_iv||5.006000| +cast_ulong||5.006000| +cast_uv||5.006000| +check_type_and_open||| +check_uni||| +checkcomma||| +checkposixcc||| +ckWARN|5.006000||p +ck_anoncode||| +ck_bitop||| +ck_concat||| +ck_defined||| +ck_delete||| +ck_die||| +ck_each||| +ck_eof||| +ck_eval||| +ck_exec||| +ck_exists||| +ck_exit||| +ck_ftst||| +ck_fun||| +ck_glob||| +ck_grep||| +ck_index||| +ck_join||| +ck_lfun||| +ck_listiob||| +ck_match||| +ck_method||| +ck_null||| +ck_open||| +ck_readline||| +ck_repeat||| +ck_require||| +ck_return||| +ck_rfun||| +ck_rvconst||| +ck_sassign||| +ck_select||| +ck_shift||| +ck_sort||| +ck_spair||| +ck_split||| +ck_subr||| +ck_substr||| +ck_svconst||| +ck_trunc||| +ck_unpack||| +ckwarn_d||5.009003| +ckwarn||5.009003| +cl_and|||n +cl_anything|||n +cl_init_zero|||n +cl_init|||n +cl_is_anything|||n +cl_or|||n +clear_placeholders||| +closest_cop||| +convert||| +cop_free||| +cr_textfilter||| +create_eval_scope||| +croak_nocontext|||vn +croak_xs_usage||5.011000| +croak|||v +csighandler||5.009003|n +curmad||| +custom_op_desc||5.007003| +custom_op_name||5.007003| +cv_ckproto_len||| +cv_clone||| +cv_const_sv||5.004000| +cv_dump||| +cv_undef||| +cx_dump||5.005000| +cx_dup||| +cxinc||| +dAXMARK|5.009003||p +dAX|5.007002||p +dITEMS|5.007002||p +dMARK||| +dMULTICALL||5.009003| +dMY_CXT_SV|5.007003||p +dMY_CXT|5.007003||p +dNOOP|5.006000||p +dORIGMARK||| +dSP||| +dTHR|5.004050||p +dTHXR|5.011000||p +dTHXa|5.006000||p +dTHXoa|5.006000||p +dTHX|5.006000||p +dUNDERBAR|5.009002||p +dVAR|5.009003||p +dXCPT|5.009002||p +dXSARGS||| +dXSI32||| +dXSTARG|5.006000||p +deb_curcv||| +deb_nocontext|||vn +deb_stack_all||| +deb_stack_n||| +debop||5.005000| +debprofdump||5.005000| +debprof||| +debstackptrs||5.007003| +debstack||5.007003| +debug_start_match||| +deb||5.007003|v +del_sv||| +delete_eval_scope||| +delimcpy||5.004000| +deprecate_old||| +deprecate||| +despatch_signals||5.007001| +destroy_matcher||| +die_nocontext|||vn +die_where||| +die|||v +dirp_dup||| +div128||| +djSP||| +do_aexec5||| +do_aexec||| +do_aspawn||| +do_binmode||5.004050| +do_chomp||| +do_chop||| +do_close||| +do_dump_pad||| +do_eof||| +do_exec3||| +do_execfree||| +do_exec||| +do_gv_dump||5.006000| +do_gvgv_dump||5.006000| +do_hv_dump||5.006000| +do_ipcctl||| +do_ipcget||| +do_join||| +do_kv||| +do_magic_dump||5.006000| +do_msgrcv||| +do_msgsnd||| +do_oddball||| +do_op_dump||5.006000| +do_op_xmldump||| +do_open9||5.006000| +do_openn||5.007001| +do_open||5.004000| +do_pmop_dump||5.006000| +do_pmop_xmldump||| +do_print||| +do_readline||| +do_seek||| +do_semop||| +do_shmio||| +do_smartmatch||| +do_spawn_nowait||| +do_spawn||| +do_sprintf||| +do_sv_dump||5.006000| +do_sysseek||| +do_tell||| +do_trans_complex_utf8||| +do_trans_complex||| +do_trans_count_utf8||| +do_trans_count||| +do_trans_simple_utf8||| +do_trans_simple||| +do_trans||| +do_vecget||| +do_vecset||| +do_vop||| +docatch||| +doeval||| +dofile||| +dofindlabel||| +doform||| +doing_taint||5.008001|n +dooneliner||| +doopen_pm||| +doparseform||| +dopoptoeval||| +dopoptogiven||| +dopoptolabel||| +dopoptoloop||| +dopoptosub_at||| +dopoptowhen||| +doref||5.009003| +dounwind||| +dowantarray||| +dump_all||5.006000| +dump_eval||5.006000| +dump_exec_pos||| +dump_fds||| +dump_form||5.006000| +dump_indent||5.006000|v +dump_mstats||| +dump_packsubs||5.006000| +dump_sub||5.006000| +dump_sv_child||| +dump_trie_interim_list||| +dump_trie_interim_table||| +dump_trie||| +dump_vindent||5.006000| +dumpuntil||| +dup_attrlist||| +emulate_cop_io||| +eval_pv|5.006000||p +eval_sv|5.006000||p +exec_failed||| +expect_number||| +fbm_compile||5.005000| +fbm_instr||5.005000| +feature_is_enabled||| +fetch_cop_label||5.011000| +filter_add||| +filter_del||| +filter_gets||| +filter_read||| +find_and_forget_pmops||| +find_array_subscript||| +find_beginning||| +find_byclass||| +find_hash_subscript||| +find_in_my_stash||| +find_runcv||5.008001| +find_rundefsvoffset||5.009002| +find_script||| +find_uninit_var||| +first_symbol|||n +fold_constants||| +forbid_setid||| +force_ident||| +force_list||| +force_next||| +force_version||| +force_word||| +forget_pmop||| +form_nocontext|||vn +form||5.004000|v +fp_dup||| +fprintf_nocontext|||vn +free_global_struct||| +free_tied_hv_pool||| +free_tmps||| +gen_constant_list||| +get_arena||| +get_aux_mg||| +get_av|5.006000||p +get_context||5.006000|n +get_cvn_flags||5.009005| +get_cv|5.006000||p +get_db_sub||| +get_debug_opts||| +get_hash_seed||| +get_hv|5.006000||p +get_isa_hash||| +get_mstats||| +get_no_modify||| +get_num||| +get_op_descs||5.005000| +get_op_names||5.005000| +get_opargs||| +get_ppaddr||5.006000| +get_re_arg||| +get_sv|5.006000||p +get_vtbl||5.005030| +getcwd_sv||5.007002| +getenv_len||| +glob_2number||| +glob_2pv||| +glob_assign_glob||| +glob_assign_ref||| +gp_dup||| +gp_free||| +gp_ref||| +grok_bin|5.007003||p +grok_hex|5.007003||p +grok_number|5.007002||p +grok_numeric_radix|5.007002||p +grok_oct|5.007003||p +group_end||| +gv_AVadd||| +gv_HVadd||| +gv_IOadd||| +gv_SVadd||| +gv_autoload4||5.004000| +gv_check||| +gv_const_sv||5.009003| +gv_dump||5.006000| +gv_efullname3||5.004000| +gv_efullname4||5.006001| +gv_efullname||| +gv_ename||| +gv_fetchfile_flags||5.009005| +gv_fetchfile||| +gv_fetchmeth_autoload||5.007003| +gv_fetchmethod_autoload||5.004000| +gv_fetchmethod_flags||5.011000| +gv_fetchmethod||| +gv_fetchmeth||| +gv_fetchpvn_flags||5.009002| +gv_fetchpv||| +gv_fetchsv||5.009002| +gv_fullname3||5.004000| +gv_fullname4||5.006001| +gv_fullname||| +gv_get_super_pkg||| +gv_handler||5.007001| +gv_init_sv||| +gv_init||| +gv_name_set||5.009004| +gv_stashpvn|5.004000||p +gv_stashpvs||5.009003| +gv_stashpv||| +gv_stashsv||| +he_dup||| +hek_dup||| +hfreeentries||| +hsplit||| +hv_assert||5.011000| +hv_auxinit|||n +hv_backreferences_p||| +hv_clear_placeholders||5.009001| +hv_clear||| +hv_common_key_len||5.010000| +hv_common||5.010000| +hv_copy_hints_hv||| +hv_delayfree_ent||5.004000| +hv_delete_common||| +hv_delete_ent||5.004000| +hv_delete||| +hv_eiter_p||5.009003| +hv_eiter_set||5.009003| +hv_exists_ent||5.004000| +hv_exists||| +hv_fetch_ent||5.004000| +hv_fetchs|5.009003||p +hv_fetch||| +hv_free_ent||5.004000| +hv_iterinit||| +hv_iterkeysv||5.004000| +hv_iterkey||| +hv_iternext_flags||5.008000| +hv_iternextsv||| +hv_iternext||| +hv_iterval||| +hv_kill_backrefs||| +hv_ksplit||5.004000| +hv_magic_check|||n +hv_magic||| +hv_name_set||5.009003| +hv_notallowed||| +hv_placeholders_get||5.009003| +hv_placeholders_p||5.009003| +hv_placeholders_set||5.009003| +hv_riter_p||5.009003| +hv_riter_set||5.009003| +hv_scalar||5.009001| +hv_store_ent||5.004000| +hv_store_flags||5.008000| +hv_stores|5.009004||p +hv_store||| +hv_undef||| +ibcmp_locale||5.004000| +ibcmp_utf8||5.007003| +ibcmp||| +incline||| +incpush_if_exists||| +incpush_use_sep||| +incpush||| +ingroup||| +init_argv_symbols||| +init_debugger||| +init_global_struct||| +init_i18nl10n||5.006000| +init_i18nl14n||5.006000| +init_ids||| +init_interp||| +init_main_stash||| +init_perllib||| +init_postdump_symbols||| +init_predump_symbols||| +init_stacks||5.005000| +init_tm||5.007002| +instr||| +intro_my||| +intuit_method||| +intuit_more||| +invert||| +io_close||| +isALNUMC|5.006000||p +isALNUM||| +isALPHA||| +isASCII|5.006000||p +isBLANK|5.006001||p +isCNTRL|5.006000||p +isDIGIT||| +isGRAPH|5.006000||p +isLOWER||| +isPRINT|5.004000||p +isPSXSPC|5.006001||p +isPUNCT|5.006000||p +isSPACE||| +isUPPER||| +isXDIGIT|5.006000||p +is_an_int||| +is_gv_magical_sv||| +is_handle_constructor|||n +is_list_assignment||| +is_lvalue_sub||5.007001| +is_uni_alnum_lc||5.006000| +is_uni_alnumc_lc||5.006000| +is_uni_alnumc||5.006000| +is_uni_alnum||5.006000| +is_uni_alpha_lc||5.006000| +is_uni_alpha||5.006000| +is_uni_ascii_lc||5.006000| +is_uni_ascii||5.006000| +is_uni_cntrl_lc||5.006000| +is_uni_cntrl||5.006000| +is_uni_digit_lc||5.006000| +is_uni_digit||5.006000| +is_uni_graph_lc||5.006000| +is_uni_graph||5.006000| +is_uni_idfirst_lc||5.006000| +is_uni_idfirst||5.006000| +is_uni_lower_lc||5.006000| +is_uni_lower||5.006000| +is_uni_print_lc||5.006000| +is_uni_print||5.006000| +is_uni_punct_lc||5.006000| +is_uni_punct||5.006000| +is_uni_space_lc||5.006000| +is_uni_space||5.006000| +is_uni_upper_lc||5.006000| +is_uni_upper||5.006000| +is_uni_xdigit_lc||5.006000| +is_uni_xdigit||5.006000| +is_utf8_alnumc||5.006000| +is_utf8_alnum||5.006000| +is_utf8_alpha||5.006000| +is_utf8_ascii||5.006000| +is_utf8_char_slow|||n +is_utf8_char||5.006000| +is_utf8_cntrl||5.006000| +is_utf8_common||| +is_utf8_digit||5.006000| +is_utf8_graph||5.006000| +is_utf8_idcont||5.008000| +is_utf8_idfirst||5.006000| +is_utf8_lower||5.006000| +is_utf8_mark||5.006000| +is_utf8_print||5.006000| +is_utf8_punct||5.006000| +is_utf8_space||5.006000| +is_utf8_string_loclen||5.009003| +is_utf8_string_loc||5.008001| +is_utf8_string||5.006001| +is_utf8_upper||5.006000| +is_utf8_xdigit||5.006000| +isa_lookup||| +items|||n +ix|||n +jmaybe||| +join_exact||| +keyword||| +leave_scope||| +lex_end||| +lex_start||| +linklist||| +listkids||| +list||| +load_module_nocontext|||vn +load_module|5.006000||pv +localize||| +looks_like_bool||| +looks_like_number||| +lop||| +mPUSHi|5.009002||p +mPUSHn|5.009002||p +mPUSHp|5.009002||p +mPUSHs|5.011000||p +mPUSHu|5.009002||p +mXPUSHi|5.009002||p +mXPUSHn|5.009002||p +mXPUSHp|5.009002||p +mXPUSHs|5.011000||p +mXPUSHu|5.009002||p +mad_free||| +madlex||| +madparse||| +magic_clear_all_env||| +magic_clearenv||| +magic_clearhint||| +magic_clearisa||| +magic_clearpack||| +magic_clearsig||| +magic_dump||5.006000| +magic_existspack||| +magic_freearylen_p||| +magic_freeovrld||| +magic_getarylen||| +magic_getdefelem||| +magic_getnkeys||| +magic_getpack||| +magic_getpos||| +magic_getsig||| +magic_getsubstr||| +magic_gettaint||| +magic_getuvar||| +magic_getvec||| +magic_get||| +magic_killbackrefs||| +magic_len||| +magic_methcall||| +magic_methpack||| +magic_nextpack||| +magic_regdata_cnt||| +magic_regdatum_get||| +magic_regdatum_set||| +magic_scalarpack||| +magic_set_all_env||| +magic_setamagic||| +magic_setarylen||| +magic_setcollxfrm||| +magic_setdbline||| +magic_setdefelem||| +magic_setenv||| +magic_sethint||| +magic_setisa||| +magic_setmglob||| +magic_setnkeys||| +magic_setpack||| +magic_setpos||| +magic_setregexp||| +magic_setsig||| +magic_setsubstr||| +magic_settaint||| +magic_setutf8||| +magic_setuvar||| +magic_setvec||| +magic_set||| +magic_sizepack||| +magic_wipepack||| +make_matcher||| +make_trie_failtable||| +make_trie||| +malloc_good_size|||n +malloced_size|||n +malloc||5.007002|n +markstack_grow||| +matcher_matches_sv||| +measure_struct||| +memEQ|5.004000||p +memNE|5.004000||p +mem_collxfrm||| +mem_log_common|||n +mess_alloc||| +mess_nocontext|||vn +mess||5.006000|v +method_common||| +mfree||5.007002|n +mg_clear||| +mg_copy||| +mg_dup||| +mg_find||| +mg_free||| +mg_get||| +mg_length||5.005000| +mg_localize||| +mg_magical||| +mg_set||| +mg_size||5.005000| +mini_mktime||5.007002| +missingterm||| +mode_from_discipline||| +modkids||| +mod||| +more_bodies||| +more_sv||| +moreswitches||| +mro_get_from_name||5.011000| +mro_get_linear_isa_dfs||| +mro_get_linear_isa||5.009005| +mro_get_private_data||5.011000| +mro_isa_changed_in||| +mro_meta_dup||| +mro_meta_init||| +mro_method_changed_in||5.009005| +mro_register||5.011000| +mro_set_mro||5.011000| +mro_set_private_data||5.011000| +mul128||| +mulexp10|||n +my_atof2||5.007002| +my_atof||5.006000| +my_attrs||| +my_bcopy|||n +my_betoh16|||n +my_betoh32|||n +my_betoh64|||n +my_betohi|||n +my_betohl|||n +my_betohs|||n +my_bzero|||n +my_chsize||| +my_clearenv||| +my_cxt_index||| +my_cxt_init||| +my_dirfd||5.009005| +my_exit_jump||| +my_exit||| +my_failure_exit||5.004000| +my_fflush_all||5.006000| +my_fork||5.007003|n +my_htobe16|||n +my_htobe32|||n +my_htobe64|||n +my_htobei|||n +my_htobel|||n +my_htobes|||n +my_htole16|||n +my_htole32|||n +my_htole64|||n +my_htolei|||n +my_htolel|||n +my_htoles|||n +my_htonl||| +my_kid||| +my_letoh16|||n +my_letoh32|||n +my_letoh64|||n +my_letohi|||n +my_letohl|||n +my_letohs|||n +my_lstat||| +my_memcmp||5.004000|n +my_memset|||n +my_ntohl||| +my_pclose||5.004000| +my_popen_list||5.007001| +my_popen||5.004000| +my_setenv||| +my_snprintf|5.009004||pvn +my_socketpair||5.007003|n +my_sprintf|5.009003||pvn +my_stat||| +my_strftime||5.007002| +my_strlcat|5.009004||pn +my_strlcpy|5.009004||pn +my_swabn|||n +my_swap||| +my_unexec||| +my_vsnprintf||5.009004|n +need_utf8|||n +newANONATTRSUB||5.006000| +newANONHASH||| +newANONLIST||| +newANONSUB||| +newASSIGNOP||| +newATTRSUB||5.006000| +newAVREF||| +newAV||| +newBINOP||| +newCONDOP||| +newCONSTSUB|5.004050||p +newCVREF||| +newDEFSVOP||| +newFORM||| +newFOROP||| +newGIVENOP||5.009003| +newGIVWHENOP||| +newGP||| +newGVOP||| +newGVREF||| +newGVgen||| +newHVREF||| +newHVhv||5.005000| +newHV||| +newIO||| +newLISTOP||| +newLOGOP||| +newLOOPEX||| +newLOOPOP||| +newMADPROP||| +newMADsv||| +newMYSUB||| +newNULLLIST||| +newOP||| +newPADOP||| +newPMOP||| +newPROG||| +newPVOP||| +newRANGE||| +newRV_inc|5.004000||p +newRV_noinc|5.004000||p +newRV||| +newSLICEOP||| +newSTATEOP||| +newSUB||| +newSVOP||| +newSVREF||| +newSV_type||5.009005| +newSVhek||5.009003| +newSViv||| +newSVnv||| +newSVpvf_nocontext|||vn +newSVpvf||5.004000|v +newSVpvn_flags|5.011000||p +newSVpvn_share|5.007001||p +newSVpvn_utf8|5.011000||p +newSVpvn|5.004050||p +newSVpvs_flags|5.011000||p +newSVpvs_share||5.009003| +newSVpvs|5.009003||p +newSVpv||| +newSVrv||| +newSVsv||| +newSVuv|5.006000||p +newSV||| +newTOKEN||| +newUNOP||| +newWHENOP||5.009003| +newWHILEOP||5.009003| +newXS_flags||5.009004| +newXSproto||5.006000| +newXS||5.006000| +new_collate||5.006000| +new_constant||| +new_ctype||5.006000| +new_he||| +new_logop||| +new_numeric||5.006000| +new_stackinfo||5.005000| +new_version||5.009000| +new_warnings_bitfield||| +next_symbol||| +nextargv||| +nextchar||| +ninstr||| +no_bareword_allowed||| +no_fh_allowed||| +no_op||| +not_a_number||| +nothreadhook||5.008000| +nuke_stacks||| +num_overflow|||n +offer_nice_chunk||| +oopsAV||| +oopsHV||| +op_clear||| +op_const_sv||| +op_dump||5.006000| +op_free||| +op_getmad_weak||| +op_getmad||| +op_null||5.007002| +op_refcnt_dec||| +op_refcnt_inc||| +op_refcnt_lock||5.009002| +op_refcnt_unlock||5.009002| +op_xmldump||| +open_script||| +pMY_CXT_|5.007003||p +pMY_CXT|5.007003||p +pTHX_|5.006000||p +pTHX|5.006000||p +packWARN|5.007003||p +pack_cat||5.007003| +pack_rec||| +package||| +packlist||5.008001| +pad_add_anon||| +pad_add_name||| +pad_alloc||| +pad_block_start||| +pad_check_dup||| +pad_compname_type||| +pad_findlex||| +pad_findmy||| +pad_fixup_inner_anons||| +pad_free||| +pad_leavemy||| +pad_new||| +pad_peg|||n +pad_push||| +pad_reset||| +pad_setsv||| +pad_sv||5.011000| +pad_swipe||| +pad_tidy||| +pad_undef||| +parse_body||| +parse_unicode_opts||| +parser_dup||| +parser_free||| +path_is_absolute|||n +peep||| +pending_Slabs_to_ro||| +perl_alloc_using|||n +perl_alloc|||n +perl_clone_using|||n +perl_clone|||n +perl_construct|||n +perl_destruct||5.007003|n +perl_free|||n +perl_parse||5.006000|n +perl_run|||n +pidgone||| +pm_description||| +pmflag||| +pmop_dump||5.006000| +pmop_xmldump||| +pmruntime||| +pmtrans||| +pop_scope||| +pregcomp||5.009005| +pregexec||| +pregfree2||5.011000| +pregfree||| +prepend_elem||| +prepend_madprops||| +printbuf||| +printf_nocontext|||vn +process_special_blocks||| +ptr_table_clear||5.009005| +ptr_table_fetch||5.009005| +ptr_table_find|||n +ptr_table_free||5.009005| +ptr_table_new||5.009005| +ptr_table_split||5.009005| +ptr_table_store||5.009005| +push_scope||| +put_byte||| +pv_display|5.006000||p +pv_escape|5.009004||p +pv_pretty|5.009004||p +pv_uni_display||5.007003| +qerror||| +qsortsvu||| +re_compile||5.009005| +re_croak2||| +re_dup_guts||| +re_intuit_start||5.009005| +re_intuit_string||5.006000| +readpipe_override||| +realloc||5.007002|n +reentrant_free||| +reentrant_init||| +reentrant_retry|||vn +reentrant_size||| +ref_array_or_hash||| +refcounted_he_chain_2hv||| +refcounted_he_fetch||| +refcounted_he_free||| +refcounted_he_new_common||| +refcounted_he_new||| +refcounted_he_value||| +refkids||| +refto||| +ref||5.011000| +reg_check_named_buff_matched||| +reg_named_buff_all||5.009005| +reg_named_buff_exists||5.009005| +reg_named_buff_fetch||5.009005| +reg_named_buff_firstkey||5.009005| +reg_named_buff_iter||| +reg_named_buff_nextkey||5.009005| +reg_named_buff_scalar||5.009005| +reg_named_buff||| +reg_namedseq||| +reg_node||| +reg_numbered_buff_fetch||| +reg_numbered_buff_length||| +reg_numbered_buff_store||| +reg_qr_package||| +reg_recode||| +reg_scan_name||| +reg_skipcomment||| +reg_temp_copy||| +reganode||| +regatom||| +regbranch||| +regclass_swash||5.009004| +regclass||| +regcppop||| +regcppush||| +regcurly|||n +regdump_extflags||| +regdump||5.005000| +regdupe_internal||| +regexec_flags||5.005000| +regfree_internal||5.009005| +reghop3|||n +reghop4|||n +reghopmaybe3|||n +reginclass||| +reginitcolors||5.006000| +reginsert||| +regmatch||| +regnext||5.005000| +regpiece||| +regpposixcc||| +regprop||| +regrepeat||| +regtail_study||| +regtail||| +regtry||| +reguni||| +regwhite|||n +reg||| +repeatcpy||| +report_evil_fh||| +report_uninit||| +require_pv||5.006000| +require_tie_mod||| +restore_magic||| +rninstr||| +rsignal_restore||| +rsignal_save||| +rsignal_state||5.004000| +rsignal||5.004000| +run_body||| +run_user_filter||| +runops_debug||5.005000| +runops_standard||5.005000| +rvpv_dup||| +rxres_free||| +rxres_restore||| +rxres_save||| +safesyscalloc||5.006000|n +safesysfree||5.006000|n +safesysmalloc||5.006000|n +safesysrealloc||5.006000|n +same_dirent||| +save_I16||5.004000| +save_I32||| +save_I8||5.006000| +save_adelete||5.011000| +save_aelem||5.004050| +save_alloc||5.006000| +save_aptr||| +save_ary||| +save_bool||5.008001| +save_clearsv||| +save_delete||| +save_destructor_x||5.006000| +save_destructor||5.006000| +save_freeop||| +save_freepv||| +save_freesv||| +save_generic_pvref||5.006001| +save_generic_svref||5.005030| +save_gp||5.004000| +save_hash||| +save_hek_flags|||n +save_helem_flags||5.011000| +save_helem||5.004050| +save_hints||| +save_hptr||| +save_int||| +save_item||| +save_iv||5.005000| +save_lines||| +save_list||| +save_long||| +save_magic||| +save_mortalizesv||5.007001| +save_nogv||| +save_op||| +save_padsv_and_mortalize||5.011000| +save_pptr||| +save_pushi32ptr||| +save_pushptri32ptr||| +save_pushptrptr||| +save_pushptr||5.011000| +save_re_context||5.006000| +save_scalar_at||| +save_scalar||| +save_set_svflags||5.009000| +save_shared_pvref||5.007003| +save_sptr||| +save_svref||| +save_vptr||5.006000| +savepvn||| +savepvs||5.009003| +savepv||| +savesharedpvn||5.009005| +savesharedpv||5.007003| +savestack_grow_cnt||5.008001| +savestack_grow||| +savesvpv||5.009002| +sawparens||| +scalar_mod_type|||n +scalarboolean||| +scalarkids||| +scalarseq||| +scalarvoid||| +scalar||| +scan_bin||5.006000| +scan_commit||| +scan_const||| +scan_formline||| +scan_heredoc||| +scan_hex||| +scan_ident||| +scan_inputsymbol||| +scan_num||5.007001| +scan_oct||| +scan_pat||| +scan_str||| +scan_subst||| +scan_trans||| +scan_version||5.009001| +scan_vstring||5.009005| +scan_word||| +scope||| +screaminstr||5.005000| +search_const||| +seed||5.008001| +sequence_num||| +sequence_tail||| +sequence||| +set_context||5.006000|n +set_numeric_local||5.006000| +set_numeric_radix||5.006000| +set_numeric_standard||5.006000| +setdefout||| +share_hek_flags||| +share_hek||5.004000| +si_dup||| +sighandler|||n +simplify_sort||| +skipspace0||| +skipspace1||| +skipspace2||| +skipspace||| +softref2xv||| +sortcv_stacked||| +sortcv_xsub||| +sortcv||| +sortsv_flags||5.009003| +sortsv||5.007003| +space_join_names_mortal||| +ss_dup||| +stack_grow||| +start_force||| +start_glob||| +start_subparse||5.004000| +stashpv_hvname_match||5.011000| +stdize_locale||| +store_cop_label||| +strEQ||| +strGE||| +strGT||| +strLE||| +strLT||| +strNE||| +str_to_version||5.006000| +strip_return||| +strnEQ||| +strnNE||| +study_chunk||| +sub_crush_depth||| +sublex_done||| +sublex_push||| +sublex_start||| +sv_2bool||| +sv_2cv||| +sv_2io||| +sv_2iuv_common||| +sv_2iuv_non_preserve||| +sv_2iv_flags||5.009001| +sv_2iv||| +sv_2mortal||| +sv_2num||| +sv_2nv||| +sv_2pv_flags|5.007002||p +sv_2pv_nolen|5.006000||p +sv_2pvbyte_nolen|5.006000||p +sv_2pvbyte|5.006000||p +sv_2pvutf8_nolen||5.006000| +sv_2pvutf8||5.006000| +sv_2pv||| +sv_2uv_flags||5.009001| +sv_2uv|5.004000||p +sv_add_arena||| +sv_add_backref||| +sv_backoff||| +sv_bless||| +sv_cat_decode||5.008001| +sv_catpv_mg|5.004050||p +sv_catpvf_mg_nocontext|||pvn +sv_catpvf_mg|5.006000|5.004000|pv +sv_catpvf_nocontext|||vn +sv_catpvf||5.004000|v +sv_catpvn_flags||5.007002| +sv_catpvn_mg|5.004050||p +sv_catpvn_nomg|5.007002||p +sv_catpvn||| +sv_catpvs|5.009003||p +sv_catpv||| +sv_catsv_flags||5.007002| +sv_catsv_mg|5.004050||p +sv_catsv_nomg|5.007002||p +sv_catsv||| +sv_catxmlpvn||| +sv_catxmlsv||| +sv_chop||| +sv_clean_all||| +sv_clean_objs||| +sv_clear||| +sv_cmp_locale||5.004000| +sv_cmp||| +sv_collxfrm||| +sv_compile_2op||5.008001| +sv_copypv||5.007003| +sv_dec||| +sv_del_backref||| +sv_derived_from||5.004000| +sv_destroyable||5.010000| +sv_does||5.009004| +sv_dump||| +sv_dup||| +sv_eq||| +sv_exp_grow||| +sv_force_normal_flags||5.007001| +sv_force_normal||5.006000| +sv_free2||| +sv_free_arenas||| +sv_free||| +sv_gets||5.004000| +sv_grow||| +sv_i_ncmp||| +sv_inc||| +sv_insert_flags||5.011000| +sv_insert||| +sv_isa||| +sv_isobject||| +sv_iv||5.005000| +sv_kill_backrefs||| +sv_len_utf8||5.006000| +sv_len||| +sv_magic_portable|5.011000|5.004000|p +sv_magicext||5.007003| +sv_magic||| +sv_mortalcopy||| +sv_ncmp||| +sv_newmortal||| +sv_newref||| +sv_nolocking||5.007003| +sv_nosharing||5.007003| +sv_nounlocking||| +sv_nv||5.005000| +sv_peek||5.005000| +sv_pos_b2u_midway||| +sv_pos_b2u||5.006000| +sv_pos_u2b_cached||| +sv_pos_u2b_forwards|||n +sv_pos_u2b_midway|||n +sv_pos_u2b||5.006000| +sv_pvbyten_force||5.006000| +sv_pvbyten||5.006000| +sv_pvbyte||5.006000| +sv_pvn_force_flags|5.007002||p +sv_pvn_force||| +sv_pvn_nomg|5.007003|5.005000|p +sv_pvn||5.005000| +sv_pvutf8n_force||5.006000| +sv_pvutf8n||5.006000| +sv_pvutf8||5.006000| +sv_pv||5.006000| +sv_recode_to_utf8||5.007003| +sv_reftype||| +sv_release_COW||| +sv_replace||| +sv_report_used||| +sv_reset||| +sv_rvweaken||5.006000| +sv_setiv_mg|5.004050||p +sv_setiv||| +sv_setnv_mg|5.006000||p +sv_setnv||| +sv_setpv_mg|5.004050||p +sv_setpvf_mg_nocontext|||pvn +sv_setpvf_mg|5.006000|5.004000|pv +sv_setpvf_nocontext|||vn +sv_setpvf||5.004000|v +sv_setpviv_mg||5.008001| +sv_setpviv||5.008001| +sv_setpvn_mg|5.004050||p +sv_setpvn||| +sv_setpvs|5.009004||p +sv_setpv||| +sv_setref_iv||| +sv_setref_nv||| +sv_setref_pvn||| +sv_setref_pv||| +sv_setref_uv||5.007001| +sv_setsv_cow||| +sv_setsv_flags||5.007002| +sv_setsv_mg|5.004050||p +sv_setsv_nomg|5.007002||p +sv_setsv||| +sv_setuv_mg|5.004050||p +sv_setuv|5.004000||p +sv_tainted||5.004000| +sv_taint||5.004000| +sv_true||5.005000| +sv_unglob||| +sv_uni_display||5.007003| +sv_unmagic||| +sv_unref_flags||5.007001| +sv_unref||| +sv_untaint||5.004000| +sv_upgrade||| +sv_usepvn_flags||5.009004| +sv_usepvn_mg|5.004050||p +sv_usepvn||| +sv_utf8_decode||5.006000| +sv_utf8_downgrade||5.006000| +sv_utf8_encode||5.006000| +sv_utf8_upgrade_flags_grow||5.011000| +sv_utf8_upgrade_flags||5.007002| +sv_utf8_upgrade_nomg||5.007002| +sv_utf8_upgrade||5.007001| +sv_uv|5.005000||p +sv_vcatpvf_mg|5.006000|5.004000|p +sv_vcatpvfn||5.004000| +sv_vcatpvf|5.006000|5.004000|p +sv_vsetpvf_mg|5.006000|5.004000|p +sv_vsetpvfn||5.004000| +sv_vsetpvf|5.006000|5.004000|p +sv_xmlpeek||| +svtype||| +swallow_bom||| +swap_match_buff||| +swash_fetch||5.007002| +swash_get||| +swash_init||5.006000| +sys_init3||5.010000|n +sys_init||5.010000|n +sys_intern_clear||| +sys_intern_dup||| +sys_intern_init||| +sys_term||5.010000|n +taint_env||| +taint_proper||| +tmps_grow||5.006000| +toLOWER||| +toUPPER||| +to_byte_substr||| +to_uni_fold||5.007003| +to_uni_lower_lc||5.006000| +to_uni_lower||5.007003| +to_uni_title_lc||5.006000| +to_uni_title||5.007003| +to_uni_upper_lc||5.006000| +to_uni_upper||5.007003| +to_utf8_case||5.007003| +to_utf8_fold||5.007003| +to_utf8_lower||5.007003| +to_utf8_substr||| +to_utf8_title||5.007003| +to_utf8_upper||5.007003| +token_free||| +token_getmad||| +tokenize_use||| +tokeq||| +tokereport||| +too_few_arguments||| +too_many_arguments||| +uiv_2buf|||n +unlnk||| +unpack_rec||| +unpack_str||5.007003| +unpackstring||5.008001| +unshare_hek_or_pvn||| +unshare_hek||| +unsharepvn||5.004000| +unwind_handler_stack||| +update_debugger_info||| +upg_version||5.009005| +usage||| +utf16_to_utf8_reversed||5.006001| +utf16_to_utf8||5.006001| +utf8_distance||5.006000| +utf8_hop||5.006000| +utf8_length||5.007001| +utf8_mg_pos_cache_update||| +utf8_to_bytes||5.006001| +utf8_to_uvchr||5.007001| +utf8_to_uvuni||5.007001| +utf8n_to_uvchr||| +utf8n_to_uvuni||5.007001| +utilize||| +uvchr_to_utf8_flags||5.007003| +uvchr_to_utf8||| +uvuni_to_utf8_flags||5.007003| +uvuni_to_utf8||5.007001| +validate_suid||| +varname||| +vcmp||5.009000| +vcroak||5.006000| +vdeb||5.007003| +vdie_common||| +vdie_croak_common||| +vdie||| +vform||5.006000| +visit||| +vivify_defelem||| +vivify_ref||| +vload_module|5.006000||p +vmess||5.006000| +vnewSVpvf|5.006000|5.004000|p +vnormal||5.009002| +vnumify||5.009000| +vstringify||5.009000| +vverify||5.009003| +vwarner||5.006000| +vwarn||5.006000| +wait4pid||| +warn_nocontext|||vn +warner_nocontext|||vn +warner|5.006000|5.004000|pv +warn|||v +watch||| +whichsig||| +write_no_mem||| +write_to_stderr||| +xmldump_all||| +xmldump_attr||| +xmldump_eval||| +xmldump_form||| +xmldump_indent|||v +xmldump_packsubs||| +xmldump_sub||| +xmldump_vindent||| +yyerror||| +yylex||| +yyparse||| +yywarn||| +); + +if (exists $opt{'list-unsupported'}) { + my $f; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $API{$f}{todo}; + print "$f ", '.'x(40-length($f)), " ", format_version($API{$f}{todo}), "\n"; + } + exit 0; +} + +# Scan for possible replacement candidates + +my(%replace, %need, %hints, %warnings, %depends); +my $replace = 0; +my($hint, $define, $function); + +sub find_api +{ + my $code = shift; + $code =~ s{ + / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]*) + | "[^"\\]*(?:\\.[^"\\]*)*" + | '[^'\\]*(?:\\.[^'\\]*)*' }{}egsx; + grep { exists $API{$_} } $code =~ /(\w+)/mg; +} + +while () { + if ($hint) { + my $h = $hint->[0] eq 'Hint' ? \%hints : \%warnings; + if (m{^\s*\*\s(.*?)\s*$}) { + for (@{$hint->[1]}) { + $h->{$_} ||= ''; # suppress warning with older perls + $h->{$_} .= "$1\n"; + } + } + else { undef $hint } + } + + $hint = [$1, [split /,?\s+/, $2]] + if m{^\s*$rccs\s+(Hint|Warning):\s+(\w+(?:,?\s+\w+)*)\s*$}; + + if ($define) { + if ($define->[1] =~ /\\$/) { + $define->[1] .= $_; + } + else { + if (exists $API{$define->[0]} && $define->[1] !~ /^DPPP_\(/) { + my @n = find_api($define->[1]); + push @{$depends{$define->[0]}}, @n if @n + } + undef $define; + } + } + + $define = [$1, $2] if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(.*)}; + + if ($function) { + if (/^}/) { + if (exists $API{$function->[0]}) { + my @n = find_api($function->[1]); + push @{$depends{$function->[0]}}, @n if @n + } + undef $function; + } + else { + $function->[1] .= $_; + } + } + + $function = [$1, ''] if m{^DPPP_\(my_(\w+)\)}; + + $replace = $1 if m{^\s*$rccs\s+Replace:\s+(\d+)\s+$rcce\s*$}; + $replace{$2} = $1 if $replace and m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+)}; + $replace{$2} = $1 if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+).*$rccs\s+Replace\s+$rcce}; + $replace{$1} = $2 if m{^\s*$rccs\s+Replace (\w+) with (\w+)\s+$rcce\s*$}; + + if (m{^\s*$rccs\s+(\w+(\s*,\s*\w+)*)\s+depends\s+on\s+(\w+(\s*,\s*\w+)*)\s+$rcce\s*$}) { + my @deps = map { s/\s+//g; $_ } split /,/, $3; + my $d; + for $d (map { s/\s+//g; $_ } split /,/, $1) { + push @{$depends{$d}}, @deps; + } + } + + $need{$1} = 1 if m{^#if\s+defined\(NEED_(\w+)(?:_GLOBAL)?\)}; +} + +for (values %depends) { + my %s; + $_ = [sort grep !$s{$_}++, @$_]; +} + +if (exists $opt{'api-info'}) { + my $f; + my $count = 0; + my $match = $opt{'api-info'} =~ m!^/(.*)/$! ? $1 : "^\Q$opt{'api-info'}\E\$"; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $f =~ /$match/; + print "\n=== $f ===\n\n"; + my $info = 0; + if ($API{$f}{base} || $API{$f}{todo}) { + my $base = format_version($API{$f}{base} || $API{$f}{todo}); + print "Supported at least starting from perl-$base.\n"; + $info++; + } + if ($API{$f}{provided}) { + my $todo = $API{$f}{todo} ? format_version($API{$f}{todo}) : "5.003"; + print "Support by $ppport provided back to perl-$todo.\n"; + print "Support needs to be explicitly requested by NEED_$f.\n" if exists $need{$f}; + print "Depends on: ", join(', ', @{$depends{$f}}), ".\n" if exists $depends{$f}; + print "\n$hints{$f}" if exists $hints{$f}; + print "\nWARNING:\n$warnings{$f}" if exists $warnings{$f}; + $info++; + } + print "No portability information available.\n" unless $info; + $count++; + } + $count or print "Found no API matching '$opt{'api-info'}'."; + print "\n"; + exit 0; +} + +if (exists $opt{'list-provided'}) { + my $f; + for $f (sort { lc $a cmp lc $b } keys %API) { + next unless $API{$f}{provided}; + my @flags; + push @flags, 'explicit' if exists $need{$f}; + push @flags, 'depend' if exists $depends{$f}; + push @flags, 'hint' if exists $hints{$f}; + push @flags, 'warning' if exists $warnings{$f}; + my $flags = @flags ? ' ['.join(', ', @flags).']' : ''; + print "$f$flags\n"; + } + exit 0; +} + +my @files; +my @srcext = qw( .xs .c .h .cc .cpp -c.inc -xs.inc ); +my $srcext = join '|', map { quotemeta $_ } @srcext; + +if (@ARGV) { + my %seen; + for (@ARGV) { + if (-e) { + if (-f) { + push @files, $_ unless $seen{$_}++; + } + else { warn "'$_' is not a file.\n" } + } + else { + my @new = grep { -f } glob $_ + or warn "'$_' does not exist.\n"; + push @files, grep { !$seen{$_}++ } @new; + } + } +} +else { + eval { + require File::Find; + File::Find::find(sub { + $File::Find::name =~ /($srcext)$/i + and push @files, $File::Find::name; + }, '.'); + }; + if ($@) { + @files = map { glob "*$_" } @srcext; + } +} + +if (!@ARGV || $opt{filter}) { + my(@in, @out); + my %xsc = map { /(.*)\.xs$/ ? ("$1.c" => 1, "$1.cc" => 1) : () } @files; + for (@files) { + my $out = exists $xsc{$_} || /\b\Q$ppport\E$/i || !/($srcext)$/i; + push @{ $out ? \@out : \@in }, $_; + } + if (@ARGV && @out) { + warning("Skipping the following files (use --nofilter to avoid this):\n| ", join "\n| ", @out); + } + @files = @in; +} + +die "No input files given!\n" unless @files; + +my(%files, %global, %revreplace); +%revreplace = reverse %replace; +my $filename; +my $patch_opened = 0; + +for $filename (@files) { + unless (open IN, "<$filename") { + warn "Unable to read from $filename: $!\n"; + next; + } + + info("Scanning $filename ..."); + + my $c = do { local $/; }; + close IN; + + my %file = (orig => $c, changes => 0); + + # Temporarily remove C/XS comments and strings from the code + my @ccom; + + $c =~ s{ + ( ^$HS*\#$HS*include\b[^\r\n]+\b(?:\Q$ppport\E|XSUB\.h)\b[^\r\n]* + | ^$HS*\#$HS*(?:define|elif|if(?:def)?)\b[^\r\n]* ) + | ( ^$HS*\#[^\r\n]* + | "[^"\\]*(?:\\.[^"\\]*)*" + | '[^'\\]*(?:\\.[^'\\]*)*' + | / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]* ) ) + }{ defined $2 and push @ccom, $2; + defined $1 ? $1 : "$ccs$#ccom$cce" }mgsex; + + $file{ccom} = \@ccom; + $file{code} = $c; + $file{has_inc_ppport} = $c =~ /^$HS*#$HS*include[^\r\n]+\b\Q$ppport\E\b/m; + + my $func; + + for $func (keys %API) { + my $match = $func; + $match .= "|$revreplace{$func}" if exists $revreplace{$func}; + if ($c =~ /\b(?:Perl_)?($match)\b/) { + $file{uses_replace}{$1}++ if exists $revreplace{$func} && $1 eq $revreplace{$func}; + $file{uses_Perl}{$func}++ if $c =~ /\bPerl_$func\b/; + if (exists $API{$func}{provided}) { + $file{uses_provided}{$func}++; + if (!exists $API{$func}{base} || $API{$func}{base} > $opt{'compat-version'}) { + $file{uses}{$func}++; + my @deps = rec_depend($func); + if (@deps) { + $file{uses_deps}{$func} = \@deps; + for (@deps) { + $file{uses}{$_} = 0 unless exists $file{uses}{$_}; + } + } + for ($func, @deps) { + $file{needs}{$_} = 'static' if exists $need{$_}; + } + } + } + if (exists $API{$func}{todo} && $API{$func}{todo} > $opt{'compat-version'}) { + if ($c =~ /\b$func\b/) { + $file{uses_todo}{$func}++; + } + } + } + } + + while ($c =~ /^$HS*#$HS*define$HS+(NEED_(\w+?)(_GLOBAL)?)\b/mg) { + if (exists $need{$2}) { + $file{defined $3 ? 'needed_global' : 'needed_static'}{$2}++; + } + else { warning("Possibly wrong #define $1 in $filename") } + } + + for (qw(uses needs uses_todo needed_global needed_static)) { + for $func (keys %{$file{$_}}) { + push @{$global{$_}{$func}}, $filename; + } + } + + $files{$filename} = \%file; +} + +# Globally resolve NEED_'s +my $need; +for $need (keys %{$global{needs}}) { + if (@{$global{needs}{$need}} > 1) { + my @targets = @{$global{needs}{$need}}; + my @t = grep $files{$_}{needed_global}{$need}, @targets; + @targets = @t if @t; + @t = grep /\.xs$/i, @targets; + @targets = @t if @t; + my $target = shift @targets; + $files{$target}{needs}{$need} = 'global'; + for (@{$global{needs}{$need}}) { + $files{$_}{needs}{$need} = 'extern' if $_ ne $target; + } + } +} + +for $filename (@files) { + exists $files{$filename} or next; + + info("=== Analyzing $filename ==="); + + my %file = %{$files{$filename}}; + my $func; + my $c = $file{code}; + my $warnings = 0; + + for $func (sort keys %{$file{uses_Perl}}) { + if ($API{$func}{varargs}) { + unless ($API{$func}{nothxarg}) { + my $changes = ($c =~ s{\b(Perl_$func\s*\(\s*)(?!aTHX_?)(\)|[^\s)]*\))} + { $1 . ($2 eq ')' ? 'aTHX' : 'aTHX_ ') . $2 }ge); + if ($changes) { + warning("Doesn't pass interpreter argument aTHX to Perl_$func"); + $file{changes} += $changes; + } + } + } + else { + warning("Uses Perl_$func instead of $func"); + $file{changes} += ($c =~ s{\bPerl_$func(\s*)\((\s*aTHX_?)?\s*} + {$func$1(}g); + } + } + + for $func (sort keys %{$file{uses_replace}}) { + warning("Uses $func instead of $replace{$func}"); + $file{changes} += ($c =~ s/\b$func\b/$replace{$func}/g); + } + + for $func (sort keys %{$file{uses_provided}}) { + if ($file{uses}{$func}) { + if (exists $file{uses_deps}{$func}) { + diag("Uses $func, which depends on ", join(', ', @{$file{uses_deps}{$func}})); + } + else { + diag("Uses $func"); + } + } + $warnings += hint($func); + } + + unless ($opt{quiet}) { + for $func (sort keys %{$file{uses_todo}}) { + print "*** WARNING: Uses $func, which may not be portable below perl ", + format_version($API{$func}{todo}), ", even with '$ppport'\n"; + $warnings++; + } + } + + for $func (sort keys %{$file{needed_static}}) { + my $message = ''; + if (not exists $file{uses}{$func}) { + $message = "No need to define NEED_$func if $func is never used"; + } + elsif (exists $file{needs}{$func} && $file{needs}{$func} ne 'static') { + $message = "No need to define NEED_$func when already needed globally"; + } + if ($message) { + diag($message); + $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_$func\b.*$LF//mg); + } + } + + for $func (sort keys %{$file{needed_global}}) { + my $message = ''; + if (not exists $global{uses}{$func}) { + $message = "No need to define NEED_${func}_GLOBAL if $func is never used"; + } + elsif (exists $file{needs}{$func}) { + if ($file{needs}{$func} eq 'extern') { + $message = "No need to define NEED_${func}_GLOBAL when already needed globally"; + } + elsif ($file{needs}{$func} eq 'static') { + $message = "No need to define NEED_${func}_GLOBAL when only used in this file"; + } + } + if ($message) { + diag($message); + $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_${func}_GLOBAL\b.*$LF//mg); + } + } + + $file{needs_inc_ppport} = keys %{$file{uses}}; + + if ($file{needs_inc_ppport}) { + my $pp = ''; + + for $func (sort keys %{$file{needs}}) { + my $type = $file{needs}{$func}; + next if $type eq 'extern'; + my $suffix = $type eq 'global' ? '_GLOBAL' : ''; + unless (exists $file{"needed_$type"}{$func}) { + if ($type eq 'global') { + diag("Files [@{$global{needs}{$func}}] need $func, adding global request"); + } + else { + diag("File needs $func, adding static request"); + } + $pp .= "#define NEED_$func$suffix\n"; + } + } + + if ($pp && ($c =~ s/^(?=$HS*#$HS*define$HS+NEED_\w+)/$pp/m)) { + $pp = ''; + $file{changes}++; + } + + unless ($file{has_inc_ppport}) { + diag("Needs to include '$ppport'"); + $pp .= qq(#include "$ppport"\n) + } + + if ($pp) { + $file{changes} += ($c =~ s/^($HS*#$HS*define$HS+NEED_\w+.*?)^/$1$pp/ms) + || ($c =~ s/^(?=$HS*#$HS*include.*\Q$ppport\E)/$pp/m) + || ($c =~ s/^($HS*#$HS*include.*XSUB.*\s*?)^/$1$pp/m) + || ($c =~ s/^/$pp/); + } + } + else { + if ($file{has_inc_ppport}) { + diag("No need to include '$ppport'"); + $file{changes} += ($c =~ s/^$HS*?#$HS*include.*\Q$ppport\E.*?$LF//m); + } + } + + # put back in our C comments + my $ix; + my $cppc = 0; + my @ccom = @{$file{ccom}}; + for $ix (0 .. $#ccom) { + if (!$opt{cplusplus} && $ccom[$ix] =~ s!^//!!) { + $cppc++; + $file{changes} += $c =~ s/$rccs$ix$rcce/$ccs$ccom[$ix] $cce/; + } + else { + $c =~ s/$rccs$ix$rcce/$ccom[$ix]/; + } + } + + if ($cppc) { + my $s = $cppc != 1 ? 's' : ''; + warning("Uses $cppc C++ style comment$s, which is not portable"); + } + + my $s = $warnings != 1 ? 's' : ''; + my $warn = $warnings ? " ($warnings warning$s)" : ''; + info("Analysis completed$warn"); + + if ($file{changes}) { + if (exists $opt{copy}) { + my $newfile = "$filename$opt{copy}"; + if (-e $newfile) { + error("'$newfile' already exists, refusing to write copy of '$filename'"); + } + else { + local *F; + if (open F, ">$newfile") { + info("Writing copy of '$filename' with changes to '$newfile'"); + print F $c; + close F; + } + else { + error("Cannot open '$newfile' for writing: $!"); + } + } + } + elsif (exists $opt{patch} || $opt{changes}) { + if (exists $opt{patch}) { + unless ($patch_opened) { + if (open PATCH, ">$opt{patch}") { + $patch_opened = 1; + } + else { + error("Cannot open '$opt{patch}' for writing: $!"); + delete $opt{patch}; + $opt{changes} = 1; + goto fallback; + } + } + mydiff(\*PATCH, $filename, $c); + } + else { +fallback: + info("Suggested changes:"); + mydiff(\*STDOUT, $filename, $c); + } + } + else { + my $s = $file{changes} == 1 ? '' : 's'; + info("$file{changes} potentially required change$s detected"); + } + } + else { + info("Looks good"); + } +} + +close PATCH if $patch_opened; + +exit 0; + + +sub try_use { eval "use @_;"; return $@ eq '' } + +sub mydiff +{ + local *F = shift; + my($file, $str) = @_; + my $diff; + + if (exists $opt{diff}) { + $diff = run_diff($opt{diff}, $file, $str); + } + + if (!defined $diff and try_use('Text::Diff')) { + $diff = Text::Diff::diff($file, \$str, { STYLE => 'Unified' }); + $diff = <
$tmp") { + print F $str; + close F; + + if (open F, "$prog $file $tmp |") { + while () { + s/\Q$tmp\E/$file.patched/; + $diff .= $_; + } + close F; + unlink $tmp; + return $diff; + } + + unlink $tmp; + } + else { + error("Cannot open '$tmp' for writing: $!"); + } + + return undef; +} + +sub rec_depend +{ + my($func, $seen) = @_; + return () unless exists $depends{$func}; + $seen = {%{$seen||{}}}; + return () if $seen->{$func}++; + my %s; + grep !$s{$_}++, map { ($_, rec_depend($_, $seen)) } @{$depends{$func}}; +} + +sub parse_version +{ + my $ver = shift; + + if ($ver =~ /^(\d+)\.(\d+)\.(\d+)$/) { + return ($1, $2, $3); + } + elsif ($ver !~ /^\d+\.[\d_]+$/) { + die "cannot parse version '$ver'\n"; + } + + $ver =~ s/_//g; + $ver =~ s/$/000000/; + + my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; + + $v = int $v; + $s = int $s; + + if ($r < 5 || ($r == 5 && $v < 6)) { + if ($s % 10) { + die "cannot parse version '$ver'\n"; + } + } + + return ($r, $v, $s); +} + +sub format_version +{ + my $ver = shift; + + $ver =~ s/$/000000/; + my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; + + $v = int $v; + $s = int $s; + + if ($r < 5 || ($r == 5 && $v < 6)) { + if ($s % 10) { + die "invalid version '$ver'\n"; + } + $s /= 10; + + $ver = sprintf "%d.%03d", $r, $v; + $s > 0 and $ver .= sprintf "_%02d", $s; + + return $ver; + } + + return sprintf "%d.%d.%d", $r, $v, $s; +} + +sub info +{ + $opt{quiet} and return; + print @_, "\n"; +} + +sub diag +{ + $opt{quiet} and return; + $opt{diag} and print @_, "\n"; +} + +sub warning +{ + $opt{quiet} and return; + print "*** ", @_, "\n"; +} + +sub error +{ + print "*** ERROR: ", @_, "\n"; +} + +my %given_hints; +my %given_warnings; +sub hint +{ + $opt{quiet} and return; + my $func = shift; + my $rv = 0; + if (exists $warnings{$func} && !$given_warnings{$func}++) { + my $warn = $warnings{$func}; + $warn =~ s!^!*** !mg; + print "*** WARNING: $func\n", $warn; + $rv++; + } + if ($opt{hints} && exists $hints{$func} && !$given_hints{$func}++) { + my $hint = $hints{$func}; + $hint =~ s/^/ /mg; + print " --- hint for $func ---\n", $hint; + } + $rv; +} + +sub usage +{ + my($usage) = do { local(@ARGV,$/)=($0); <> } =~ /^=head\d$HS+SYNOPSIS\s*^(.*?)\s*^=/ms; + my %M = ( 'I' => '*' ); + $usage =~ s/^\s*perl\s+\S+/$^X $0/; + $usage =~ s/([A-Z])<([^>]+)>/$M{$1}$2$M{$1}/g; + + print < }; + my($copy) = $self =~ /^=head\d\s+COPYRIGHT\s*^(.*?)^=\w+/ms; + $copy =~ s/^(?=\S+)/ /gms; + $self =~ s/^$HS+Do NOT edit.*?(?=^-)/$copy/ms; + $self =~ s/^SKIP.*(?=^__DATA__)/SKIP +if (\@ARGV && \$ARGV[0] eq '--unstrip') { + eval { require Devel::PPPort }; + \$@ and die "Cannot require Devel::PPPort, please install.\\n"; + if (eval \$Devel::PPPort::VERSION < $VERSION) { + die "$0 was originally generated with Devel::PPPort $VERSION.\\n" + . "Your Devel::PPPort is only version \$Devel::PPPort::VERSION.\\n" + . "Please install a newer version, or --unstrip will not work.\\n"; + } + Devel::PPPort::WriteFile(\$0); + exit 0; +} +print <$0" or die "cannot strip $0: $!\n"; + print OUT "$pl$c\n"; + + exit 0; +} + +__DATA__ +*/ + +#ifndef _P_P_PORTABILITY_H_ +#define _P_P_PORTABILITY_H_ + +#ifndef DPPP_NAMESPACE +# define DPPP_NAMESPACE DPPP_ +#endif + +#define DPPP_CAT2(x,y) CAT2(x,y) +#define DPPP_(name) DPPP_CAT2(DPPP_NAMESPACE, name) + +#ifndef PERL_REVISION +# if !defined(__PATCHLEVEL_H_INCLUDED__) && !(defined(PATCHLEVEL) && defined(SUBVERSION)) +# define PERL_PATCHLEVEL_H_IMPLICIT +# include +# endif +# if !(defined(PERL_VERSION) || (defined(SUBVERSION) && defined(PATCHLEVEL))) +# include +# endif +# ifndef PERL_REVISION +# define PERL_REVISION (5) + /* Replace: 1 */ +# define PERL_VERSION PATCHLEVEL +# define PERL_SUBVERSION SUBVERSION + /* Replace PERL_PATCHLEVEL with PERL_VERSION */ + /* Replace: 0 */ +# endif +#endif + +#define _dpppDEC2BCD(dec) ((((dec)/100)<<8)|((((dec)%100)/10)<<4)|((dec)%10)) +#define PERL_BCDVERSION ((_dpppDEC2BCD(PERL_REVISION)<<24)|(_dpppDEC2BCD(PERL_VERSION)<<12)|_dpppDEC2BCD(PERL_SUBVERSION)) + +/* It is very unlikely that anyone will try to use this with Perl 6 + (or greater), but who knows. + */ +#if PERL_REVISION != 5 +# error ppport.h only works with Perl version 5 +#endif /* PERL_REVISION != 5 */ +#ifndef dTHR +# define dTHR dNOOP +#endif +#ifndef dTHX +# define dTHX dNOOP +#endif + +#ifndef dTHXa +# define dTHXa(x) dNOOP +#endif +#ifndef pTHX +# define pTHX void +#endif + +#ifndef pTHX_ +# define pTHX_ +#endif + +#ifndef aTHX +# define aTHX +#endif + +#ifndef aTHX_ +# define aTHX_ +#endif + +#if (PERL_BCDVERSION < 0x5006000) +# ifdef USE_THREADS +# define aTHXR thr +# define aTHXR_ thr, +# else +# define aTHXR +# define aTHXR_ +# endif +# define dTHXR dTHR +#else +# define aTHXR aTHX +# define aTHXR_ aTHX_ +# define dTHXR dTHX +#endif +#ifndef dTHXoa +# define dTHXoa(x) dTHXa(x) +#endif + +#ifdef I_LIMITS +# include +#endif + +#ifndef PERL_UCHAR_MIN +# define PERL_UCHAR_MIN ((unsigned char)0) +#endif + +#ifndef PERL_UCHAR_MAX +# ifdef UCHAR_MAX +# define PERL_UCHAR_MAX ((unsigned char)UCHAR_MAX) +# else +# ifdef MAXUCHAR +# define PERL_UCHAR_MAX ((unsigned char)MAXUCHAR) +# else +# define PERL_UCHAR_MAX ((unsigned char)~(unsigned)0) +# endif +# endif +#endif + +#ifndef PERL_USHORT_MIN +# define PERL_USHORT_MIN ((unsigned short)0) +#endif + +#ifndef PERL_USHORT_MAX +# ifdef USHORT_MAX +# define PERL_USHORT_MAX ((unsigned short)USHORT_MAX) +# else +# ifdef MAXUSHORT +# define PERL_USHORT_MAX ((unsigned short)MAXUSHORT) +# else +# ifdef USHRT_MAX +# define PERL_USHORT_MAX ((unsigned short)USHRT_MAX) +# else +# define PERL_USHORT_MAX ((unsigned short)~(unsigned)0) +# endif +# endif +# endif +#endif + +#ifndef PERL_SHORT_MAX +# ifdef SHORT_MAX +# define PERL_SHORT_MAX ((short)SHORT_MAX) +# else +# ifdef MAXSHORT /* Often used in */ +# define PERL_SHORT_MAX ((short)MAXSHORT) +# else +# ifdef SHRT_MAX +# define PERL_SHORT_MAX ((short)SHRT_MAX) +# else +# define PERL_SHORT_MAX ((short) (PERL_USHORT_MAX >> 1)) +# endif +# endif +# endif +#endif + +#ifndef PERL_SHORT_MIN +# ifdef SHORT_MIN +# define PERL_SHORT_MIN ((short)SHORT_MIN) +# else +# ifdef MINSHORT +# define PERL_SHORT_MIN ((short)MINSHORT) +# else +# ifdef SHRT_MIN +# define PERL_SHORT_MIN ((short)SHRT_MIN) +# else +# define PERL_SHORT_MIN (-PERL_SHORT_MAX - ((3 & -1) == 3)) +# endif +# endif +# endif +#endif + +#ifndef PERL_UINT_MAX +# ifdef UINT_MAX +# define PERL_UINT_MAX ((unsigned int)UINT_MAX) +# else +# ifdef MAXUINT +# define PERL_UINT_MAX ((unsigned int)MAXUINT) +# else +# define PERL_UINT_MAX (~(unsigned int)0) +# endif +# endif +#endif + +#ifndef PERL_UINT_MIN +# define PERL_UINT_MIN ((unsigned int)0) +#endif + +#ifndef PERL_INT_MAX +# ifdef INT_MAX +# define PERL_INT_MAX ((int)INT_MAX) +# else +# ifdef MAXINT /* Often used in */ +# define PERL_INT_MAX ((int)MAXINT) +# else +# define PERL_INT_MAX ((int)(PERL_UINT_MAX >> 1)) +# endif +# endif +#endif + +#ifndef PERL_INT_MIN +# ifdef INT_MIN +# define PERL_INT_MIN ((int)INT_MIN) +# else +# ifdef MININT +# define PERL_INT_MIN ((int)MININT) +# else +# define PERL_INT_MIN (-PERL_INT_MAX - ((3 & -1) == 3)) +# endif +# endif +#endif + +#ifndef PERL_ULONG_MAX +# ifdef ULONG_MAX +# define PERL_ULONG_MAX ((unsigned long)ULONG_MAX) +# else +# ifdef MAXULONG +# define PERL_ULONG_MAX ((unsigned long)MAXULONG) +# else +# define PERL_ULONG_MAX (~(unsigned long)0) +# endif +# endif +#endif + +#ifndef PERL_ULONG_MIN +# define PERL_ULONG_MIN ((unsigned long)0L) +#endif + +#ifndef PERL_LONG_MAX +# ifdef LONG_MAX +# define PERL_LONG_MAX ((long)LONG_MAX) +# else +# ifdef MAXLONG +# define PERL_LONG_MAX ((long)MAXLONG) +# else +# define PERL_LONG_MAX ((long) (PERL_ULONG_MAX >> 1)) +# endif +# endif +#endif + +#ifndef PERL_LONG_MIN +# ifdef LONG_MIN +# define PERL_LONG_MIN ((long)LONG_MIN) +# else +# ifdef MINLONG +# define PERL_LONG_MIN ((long)MINLONG) +# else +# define PERL_LONG_MIN (-PERL_LONG_MAX - ((3 & -1) == 3)) +# endif +# endif +#endif + +#if defined(HAS_QUAD) && (defined(convex) || defined(uts)) +# ifndef PERL_UQUAD_MAX +# ifdef ULONGLONG_MAX +# define PERL_UQUAD_MAX ((unsigned long long)ULONGLONG_MAX) +# else +# ifdef MAXULONGLONG +# define PERL_UQUAD_MAX ((unsigned long long)MAXULONGLONG) +# else +# define PERL_UQUAD_MAX (~(unsigned long long)0) +# endif +# endif +# endif + +# ifndef PERL_UQUAD_MIN +# define PERL_UQUAD_MIN ((unsigned long long)0L) +# endif + +# ifndef PERL_QUAD_MAX +# ifdef LONGLONG_MAX +# define PERL_QUAD_MAX ((long long)LONGLONG_MAX) +# else +# ifdef MAXLONGLONG +# define PERL_QUAD_MAX ((long long)MAXLONGLONG) +# else +# define PERL_QUAD_MAX ((long long) (PERL_UQUAD_MAX >> 1)) +# endif +# endif +# endif + +# ifndef PERL_QUAD_MIN +# ifdef LONGLONG_MIN +# define PERL_QUAD_MIN ((long long)LONGLONG_MIN) +# else +# ifdef MINLONGLONG +# define PERL_QUAD_MIN ((long long)MINLONGLONG) +# else +# define PERL_QUAD_MIN (-PERL_QUAD_MAX - ((3 & -1) == 3)) +# endif +# endif +# endif +#endif + +/* This is based on code from 5.003 perl.h */ +#ifdef HAS_QUAD +# ifdef cray +#ifndef IVTYPE +# define IVTYPE int +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_INT_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_INT_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_UINT_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_UINT_MAX +#endif + +# ifdef INTSIZE +#ifndef IVSIZE +# define IVSIZE INTSIZE +#endif + +# endif +# else +# if defined(convex) || defined(uts) +#ifndef IVTYPE +# define IVTYPE long long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_QUAD_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_QUAD_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_UQUAD_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_UQUAD_MAX +#endif + +# ifdef LONGLONGSIZE +#ifndef IVSIZE +# define IVSIZE LONGLONGSIZE +#endif + +# endif +# else +#ifndef IVTYPE +# define IVTYPE long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_LONG_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_LONG_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_ULONG_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_ULONG_MAX +#endif + +# ifdef LONGSIZE +#ifndef IVSIZE +# define IVSIZE LONGSIZE +#endif + +# endif +# endif +# endif +#ifndef IVSIZE +# define IVSIZE 8 +#endif + +#ifndef PERL_QUAD_MIN +# define PERL_QUAD_MIN IV_MIN +#endif + +#ifndef PERL_QUAD_MAX +# define PERL_QUAD_MAX IV_MAX +#endif + +#ifndef PERL_UQUAD_MIN +# define PERL_UQUAD_MIN UV_MIN +#endif + +#ifndef PERL_UQUAD_MAX +# define PERL_UQUAD_MAX UV_MAX +#endif + +#else +#ifndef IVTYPE +# define IVTYPE long +#endif + +#ifndef IV_MIN +# define IV_MIN PERL_LONG_MIN +#endif + +#ifndef IV_MAX +# define IV_MAX PERL_LONG_MAX +#endif + +#ifndef UV_MIN +# define UV_MIN PERL_ULONG_MIN +#endif + +#ifndef UV_MAX +# define UV_MAX PERL_ULONG_MAX +#endif + +#endif + +#ifndef IVSIZE +# ifdef LONGSIZE +# define IVSIZE LONGSIZE +# else +# define IVSIZE 4 /* A bold guess, but the best we can make. */ +# endif +#endif +#ifndef UVTYPE +# define UVTYPE unsigned IVTYPE +#endif + +#ifndef UVSIZE +# define UVSIZE IVSIZE +#endif +#ifndef sv_setuv +# define sv_setuv(sv, uv) \ + STMT_START { \ + UV TeMpUv = uv; \ + if (TeMpUv <= IV_MAX) \ + sv_setiv(sv, TeMpUv); \ + else \ + sv_setnv(sv, (double)TeMpUv); \ + } STMT_END +#endif +#ifndef newSVuv +# define newSVuv(uv) ((uv) <= IV_MAX ? newSViv((IV)uv) : newSVnv((NV)uv)) +#endif +#ifndef sv_2uv +# define sv_2uv(sv) ((PL_Sv = (sv)), (UV) (SvNOK(PL_Sv) ? SvNV(PL_Sv) : sv_2nv(PL_Sv))) +#endif + +#ifndef SvUVX +# define SvUVX(sv) ((UV)SvIVX(sv)) +#endif + +#ifndef SvUVXx +# define SvUVXx(sv) SvUVX(sv) +#endif + +#ifndef SvUV +# define SvUV(sv) (SvIOK(sv) ? SvUVX(sv) : sv_2uv(sv)) +#endif + +#ifndef SvUVx +# define SvUVx(sv) ((PL_Sv = (sv)), SvUV(PL_Sv)) +#endif + +/* Hint: sv_uv + * Always use the SvUVx() macro instead of sv_uv(). + */ +#ifndef sv_uv +# define sv_uv(sv) SvUVx(sv) +#endif + +#if !defined(SvUOK) && defined(SvIOK_UV) +# define SvUOK(sv) SvIOK_UV(sv) +#endif +#ifndef XST_mUV +# define XST_mUV(i,v) (ST(i) = sv_2mortal(newSVuv(v)) ) +#endif + +#ifndef XSRETURN_UV +# define XSRETURN_UV(v) STMT_START { XST_mUV(0,v); XSRETURN(1); } STMT_END +#endif +#ifndef PUSHu +# define PUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); PUSHTARG; } STMT_END +#endif + +#ifndef XPUSHu +# define XPUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); XPUSHTARG; } STMT_END +#endif + +#ifdef HAS_MEMCMP +#ifndef memNE +# define memNE(s1,s2,l) (memcmp(s1,s2,l)) +#endif + +#ifndef memEQ +# define memEQ(s1,s2,l) (!memcmp(s1,s2,l)) +#endif + +#else +#ifndef memNE +# define memNE(s1,s2,l) (bcmp(s1,s2,l)) +#endif + +#ifndef memEQ +# define memEQ(s1,s2,l) (!bcmp(s1,s2,l)) +#endif + +#endif +#ifndef MoveD +# define MoveD(s,d,n,t) memmove((char*)(d),(char*)(s), (n) * sizeof(t)) +#endif + +#ifndef CopyD +# define CopyD(s,d,n,t) memcpy((char*)(d),(char*)(s), (n) * sizeof(t)) +#endif + +#ifdef HAS_MEMSET +#ifndef ZeroD +# define ZeroD(d,n,t) memzero((char*)(d), (n) * sizeof(t)) +#endif + +#else +#ifndef ZeroD +# define ZeroD(d,n,t) ((void)memzero((char*)(d), (n) * sizeof(t)), d) +#endif + +#endif +#ifndef PoisonWith +# define PoisonWith(d,n,t,b) (void)memset((char*)(d), (U8)(b), (n) * sizeof(t)) +#endif + +#ifndef PoisonNew +# define PoisonNew(d,n,t) PoisonWith(d,n,t,0xAB) +#endif + +#ifndef PoisonFree +# define PoisonFree(d,n,t) PoisonWith(d,n,t,0xEF) +#endif + +#ifndef Poison +# define Poison(d,n,t) PoisonFree(d,n,t) +#endif +#ifndef Newx +# define Newx(v,n,t) New(0,v,n,t) +#endif + +#ifndef Newxc +# define Newxc(v,n,t,c) Newc(0,v,n,t,c) +#endif + +#ifndef Newxz +# define Newxz(v,n,t) Newz(0,v,n,t) +#endif + +#ifndef PERL_UNUSED_DECL +# ifdef HASATTRIBUTE +# if (defined(__GNUC__) && defined(__cplusplus)) || defined(__INTEL_COMPILER) +# define PERL_UNUSED_DECL +# else +# define PERL_UNUSED_DECL __attribute__((unused)) +# endif +# else +# define PERL_UNUSED_DECL +# endif +#endif + +#ifndef PERL_UNUSED_ARG +# if defined(lint) && defined(S_SPLINT_S) /* www.splint.org */ +# include +# define PERL_UNUSED_ARG(x) NOTE(ARGUNUSED(x)) +# else +# define PERL_UNUSED_ARG(x) ((void)x) +# endif +#endif + +#ifndef PERL_UNUSED_VAR +# define PERL_UNUSED_VAR(x) ((void)x) +#endif + +#ifndef PERL_UNUSED_CONTEXT +# ifdef USE_ITHREADS +# define PERL_UNUSED_CONTEXT PERL_UNUSED_ARG(my_perl) +# else +# define PERL_UNUSED_CONTEXT +# endif +#endif +#ifndef NOOP +# define NOOP /*EMPTY*/(void)0 +#endif + +#ifndef dNOOP +# define dNOOP extern int /*@unused@*/ Perl___notused PERL_UNUSED_DECL +#endif + +#ifndef NVTYPE +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) +# define NVTYPE long double +# else +# define NVTYPE double +# endif +typedef NVTYPE NV; +#endif + +#ifndef INT2PTR +# if (IVSIZE == PTRSIZE) && (UVSIZE == PTRSIZE) +# define PTRV UV +# define INT2PTR(any,d) (any)(d) +# else +# if PTRSIZE == LONGSIZE +# define PTRV unsigned long +# else +# define PTRV unsigned +# endif +# define INT2PTR(any,d) (any)(PTRV)(d) +# endif +#endif + +#ifndef PTR2ul +# if PTRSIZE == LONGSIZE +# define PTR2ul(p) (unsigned long)(p) +# else +# define PTR2ul(p) INT2PTR(unsigned long,p) +# endif +#endif +#ifndef PTR2nat +# define PTR2nat(p) (PTRV)(p) +#endif + +#ifndef NUM2PTR +# define NUM2PTR(any,d) (any)PTR2nat(d) +#endif + +#ifndef PTR2IV +# define PTR2IV(p) INT2PTR(IV,p) +#endif + +#ifndef PTR2UV +# define PTR2UV(p) INT2PTR(UV,p) +#endif + +#ifndef PTR2NV +# define PTR2NV(p) NUM2PTR(NV,p) +#endif + +#undef START_EXTERN_C +#undef END_EXTERN_C +#undef EXTERN_C +#ifdef __cplusplus +# define START_EXTERN_C extern "C" { +# define END_EXTERN_C } +# define EXTERN_C extern "C" +#else +# define START_EXTERN_C +# define END_EXTERN_C +# define EXTERN_C extern +#endif + +#if defined(PERL_GCC_PEDANTIC) +# ifndef PERL_GCC_BRACE_GROUPS_FORBIDDEN +# define PERL_GCC_BRACE_GROUPS_FORBIDDEN +# endif +#endif + +#if defined(__GNUC__) && !defined(PERL_GCC_BRACE_GROUPS_FORBIDDEN) && !defined(__cplusplus) +# ifndef PERL_USE_GCC_BRACE_GROUPS +# define PERL_USE_GCC_BRACE_GROUPS +# endif +#endif + +#undef STMT_START +#undef STMT_END +#ifdef PERL_USE_GCC_BRACE_GROUPS +# define STMT_START (void)( /* gcc supports ``({ STATEMENTS; })'' */ +# define STMT_END ) +#else +# if defined(VOIDFLAGS) && (VOIDFLAGS) && (defined(sun) || defined(__sun__)) && !defined(__GNUC__) +# define STMT_START if (1) +# define STMT_END else (void)0 +# else +# define STMT_START do +# define STMT_END while (0) +# endif +#endif +#ifndef boolSV +# define boolSV(b) ((b) ? &PL_sv_yes : &PL_sv_no) +#endif + +/* DEFSV appears first in 5.004_56 */ +#ifndef DEFSV +# define DEFSV GvSV(PL_defgv) +#endif + +#ifndef SAVE_DEFSV +# define SAVE_DEFSV SAVESPTR(GvSV(PL_defgv)) +#endif + +#ifndef DEFSV_set +# define DEFSV_set(sv) (DEFSV = (sv)) +#endif + +/* Older perls (<=5.003) lack AvFILLp */ +#ifndef AvFILLp +# define AvFILLp AvFILL +#endif +#ifndef ERRSV +# define ERRSV get_sv("@",FALSE) +#endif + +/* Hint: gv_stashpvn + * This function's backport doesn't support the length parameter, but + * rather ignores it. Portability can only be ensured if the length + * parameter is used for speed reasons, but the length can always be + * correctly computed from the string argument. + */ +#ifndef gv_stashpvn +# define gv_stashpvn(str,len,create) gv_stashpv(str,create) +#endif + +/* Replace: 1 */ +#ifndef get_cv +# define get_cv perl_get_cv +#endif + +#ifndef get_sv +# define get_sv perl_get_sv +#endif + +#ifndef get_av +# define get_av perl_get_av +#endif + +#ifndef get_hv +# define get_hv perl_get_hv +#endif + +/* Replace: 0 */ +#ifndef dUNDERBAR +# define dUNDERBAR dNOOP +#endif + +#ifndef UNDERBAR +# define UNDERBAR DEFSV +#endif +#ifndef dAX +# define dAX I32 ax = MARK - PL_stack_base + 1 +#endif + +#ifndef dITEMS +# define dITEMS I32 items = SP - MARK +#endif +#ifndef dXSTARG +# define dXSTARG SV * targ = sv_newmortal() +#endif +#ifndef dAXMARK +# define dAXMARK I32 ax = POPMARK; \ + register SV ** const mark = PL_stack_base + ax++ +#endif +#ifndef XSprePUSH +# define XSprePUSH (sp = PL_stack_base + ax - 1) +#endif + +#if (PERL_BCDVERSION < 0x5005000) +# undef XSRETURN +# define XSRETURN(off) \ + STMT_START { \ + PL_stack_sp = PL_stack_base + ax + ((off) - 1); \ + return; \ + } STMT_END +#endif +#ifndef PERL_ABS +# define PERL_ABS(x) ((x) < 0 ? -(x) : (x)) +#endif +#ifndef dVAR +# define dVAR dNOOP +#endif +#ifndef SVf +# define SVf "_" +#endif +#ifndef UTF8_MAXBYTES +# define UTF8_MAXBYTES UTF8_MAXLEN +#endif +#ifndef CPERLscope +# define CPERLscope(x) x +#endif +#ifndef PERL_HASH +# define PERL_HASH(hash,str,len) \ + STMT_START { \ + const char *s_PeRlHaSh = str; \ + I32 i_PeRlHaSh = len; \ + U32 hash_PeRlHaSh = 0; \ + while (i_PeRlHaSh--) \ + hash_PeRlHaSh = hash_PeRlHaSh * 33 + *s_PeRlHaSh++; \ + (hash) = hash_PeRlHaSh; \ + } STMT_END +#endif + +#ifndef PERLIO_FUNCS_DECL +# ifdef PERLIO_FUNCS_CONST +# define PERLIO_FUNCS_DECL(funcs) const PerlIO_funcs funcs +# define PERLIO_FUNCS_CAST(funcs) (PerlIO_funcs*)(funcs) +# else +# define PERLIO_FUNCS_DECL(funcs) PerlIO_funcs funcs +# define PERLIO_FUNCS_CAST(funcs) (funcs) +# endif +#endif + +/* provide these typedefs for older perls */ +#if (PERL_BCDVERSION < 0x5009003) + +# ifdef ARGSproto +typedef OP* (CPERLscope(*Perl_ppaddr_t))(ARGSproto); +# else +typedef OP* (CPERLscope(*Perl_ppaddr_t))(pTHX); +# endif + +typedef OP* (CPERLscope(*Perl_check_t)) (pTHX_ OP*); + +#endif +#ifndef isPSXSPC +# define isPSXSPC(c) (isSPACE(c) || (c) == '\v') +#endif + +#ifndef isBLANK +# define isBLANK(c) ((c) == ' ' || (c) == '\t') +#endif + +#ifdef EBCDIC +#ifndef isALNUMC +# define isALNUMC(c) isalnum(c) +#endif + +#ifndef isASCII +# define isASCII(c) isascii(c) +#endif + +#ifndef isCNTRL +# define isCNTRL(c) iscntrl(c) +#endif + +#ifndef isGRAPH +# define isGRAPH(c) isgraph(c) +#endif + +#ifndef isPRINT +# define isPRINT(c) isprint(c) +#endif + +#ifndef isPUNCT +# define isPUNCT(c) ispunct(c) +#endif + +#ifndef isXDIGIT +# define isXDIGIT(c) isxdigit(c) +#endif + +#else +# if (PERL_BCDVERSION < 0x5010000) +/* Hint: isPRINT + * The implementation in older perl versions includes all of the + * isSPACE() characters, which is wrong. The version provided by + * Devel::PPPort always overrides a present buggy version. + */ +# undef isPRINT +# endif +#ifndef isALNUMC +# define isALNUMC(c) (isALPHA(c) || isDIGIT(c)) +#endif + +#ifndef isASCII +# define isASCII(c) ((c) <= 127) +#endif + +#ifndef isCNTRL +# define isCNTRL(c) ((c) < ' ' || (c) == 127) +#endif + +#ifndef isGRAPH +# define isGRAPH(c) (isALNUM(c) || isPUNCT(c)) +#endif + +#ifndef isPRINT +# define isPRINT(c) (((c) >= 32 && (c) < 127)) +#endif + +#ifndef isPUNCT +# define isPUNCT(c) (((c) >= 33 && (c) <= 47) || ((c) >= 58 && (c) <= 64) || ((c) >= 91 && (c) <= 96) || ((c) >= 123 && (c) <= 126)) +#endif + +#ifndef isXDIGIT +# define isXDIGIT(c) (isDIGIT(c) || ((c) >= 'a' && (c) <= 'f') || ((c) >= 'A' && (c) <= 'F')) +#endif + +#endif + +#ifndef PERL_SIGNALS_UNSAFE_FLAG + +#define PERL_SIGNALS_UNSAFE_FLAG 0x0001 + +#if (PERL_BCDVERSION < 0x5008000) +# define D_PPP_PERL_SIGNALS_INIT PERL_SIGNALS_UNSAFE_FLAG +#else +# define D_PPP_PERL_SIGNALS_INIT 0 +#endif + +#if defined(NEED_PL_signals) +static U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; +#elif defined(NEED_PL_signals_GLOBAL) +U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; +#else +extern U32 DPPP_(my_PL_signals); +#endif +#define PL_signals DPPP_(my_PL_signals) + +#endif + +/* Hint: PL_ppaddr + * Calling an op via PL_ppaddr requires passing a context argument + * for threaded builds. Since the context argument is different for + * 5.005 perls, you can use aTHXR (supplied by ppport.h), which will + * automatically be defined as the correct argument. + */ + +#if (PERL_BCDVERSION <= 0x5005005) +/* Replace: 1 */ +# define PL_ppaddr ppaddr +# define PL_no_modify no_modify +/* Replace: 0 */ +#endif + +#if (PERL_BCDVERSION <= 0x5004005) +/* Replace: 1 */ +# define PL_DBsignal DBsignal +# define PL_DBsingle DBsingle +# define PL_DBsub DBsub +# define PL_DBtrace DBtrace +# define PL_Sv Sv +# define PL_bufend bufend +# define PL_bufptr bufptr +# define PL_compiling compiling +# define PL_copline copline +# define PL_curcop curcop +# define PL_curstash curstash +# define PL_debstash debstash +# define PL_defgv defgv +# define PL_diehook diehook +# define PL_dirty dirty +# define PL_dowarn dowarn +# define PL_errgv errgv +# define PL_expect expect +# define PL_hexdigit hexdigit +# define PL_hints hints +# define PL_laststatval laststatval +# define PL_lex_state lex_state +# define PL_lex_stuff lex_stuff +# define PL_linestr linestr +# define PL_na na +# define PL_perl_destruct_level perl_destruct_level +# define PL_perldb perldb +# define PL_rsfp_filters rsfp_filters +# define PL_rsfp rsfp +# define PL_stack_base stack_base +# define PL_stack_sp stack_sp +# define PL_statcache statcache +# define PL_stdingv stdingv +# define PL_sv_arenaroot sv_arenaroot +# define PL_sv_no sv_no +# define PL_sv_undef sv_undef +# define PL_sv_yes sv_yes +# define PL_tainted tainted +# define PL_tainting tainting +# define PL_tokenbuf tokenbuf +/* Replace: 0 */ +#endif + +/* Warning: PL_parser + * For perl versions earlier than 5.9.5, this is an always + * non-NULL dummy. Also, it cannot be dereferenced. Don't + * use it if you can avoid is and unless you absolutely know + * what you're doing. + * If you always check that PL_parser is non-NULL, you can + * define DPPP_PL_parser_NO_DUMMY to avoid the creation of + * a dummy parser structure. + */ + +#if (PERL_BCDVERSION >= 0x5009005) +# ifdef DPPP_PL_parser_NO_DUMMY +# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ + (croak("panic: PL_parser == NULL in %s:%d", \ + __FILE__, __LINE__), (yy_parser *) NULL))->var) +# else +# ifdef DPPP_PL_parser_NO_DUMMY_WARNING +# define D_PPP_parser_dummy_warning(var) +# else +# define D_PPP_parser_dummy_warning(var) \ + warn("warning: dummy PL_" #var " used in %s:%d", __FILE__, __LINE__), +# endif +# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ + (D_PPP_parser_dummy_warning(var) &DPPP_(dummy_PL_parser)))->var) +#if defined(NEED_PL_parser) +static yy_parser DPPP_(dummy_PL_parser); +#elif defined(NEED_PL_parser_GLOBAL) +yy_parser DPPP_(dummy_PL_parser); +#else +extern yy_parser DPPP_(dummy_PL_parser); +#endif + +# endif + +/* PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf depends on PL_parser */ +/* Warning: PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf + * Do not use this variable unless you know exactly what you're + * doint. It is internal to the perl parser and may change or even + * be removed in the future. As of perl 5.9.5, you have to check + * for (PL_parser != NULL) for this variable to have any effect. + * An always non-NULL PL_parser dummy is provided for earlier + * perl versions. + * If PL_parser is NULL when you try to access this variable, a + * dummy is being accessed instead and a warning is issued unless + * you define DPPP_PL_parser_NO_DUMMY_WARNING. + * If DPPP_PL_parser_NO_DUMMY is defined, the code trying to access + * this variable will croak with a panic message. + */ + +# define PL_expect D_PPP_my_PL_parser_var(expect) +# define PL_copline D_PPP_my_PL_parser_var(copline) +# define PL_rsfp D_PPP_my_PL_parser_var(rsfp) +# define PL_rsfp_filters D_PPP_my_PL_parser_var(rsfp_filters) +# define PL_linestr D_PPP_my_PL_parser_var(linestr) +# define PL_bufptr D_PPP_my_PL_parser_var(bufptr) +# define PL_bufend D_PPP_my_PL_parser_var(bufend) +# define PL_lex_state D_PPP_my_PL_parser_var(lex_state) +# define PL_lex_stuff D_PPP_my_PL_parser_var(lex_stuff) +# define PL_tokenbuf D_PPP_my_PL_parser_var(tokenbuf) + +#else + +/* ensure that PL_parser != NULL and cannot be dereferenced */ +# define PL_parser ((void *) 1) + +#endif +#ifndef mPUSHs +# define mPUSHs(s) PUSHs(sv_2mortal(s)) +#endif + +#ifndef PUSHmortal +# define PUSHmortal PUSHs(sv_newmortal()) +#endif + +#ifndef mPUSHp +# define mPUSHp(p,l) sv_setpvn(PUSHmortal, (p), (l)) +#endif + +#ifndef mPUSHn +# define mPUSHn(n) sv_setnv(PUSHmortal, (NV)(n)) +#endif + +#ifndef mPUSHi +# define mPUSHi(i) sv_setiv(PUSHmortal, (IV)(i)) +#endif + +#ifndef mPUSHu +# define mPUSHu(u) sv_setuv(PUSHmortal, (UV)(u)) +#endif +#ifndef mXPUSHs +# define mXPUSHs(s) XPUSHs(sv_2mortal(s)) +#endif + +#ifndef XPUSHmortal +# define XPUSHmortal XPUSHs(sv_newmortal()) +#endif + +#ifndef mXPUSHp +# define mXPUSHp(p,l) STMT_START { EXTEND(sp,1); sv_setpvn(PUSHmortal, (p), (l)); } STMT_END +#endif + +#ifndef mXPUSHn +# define mXPUSHn(n) STMT_START { EXTEND(sp,1); sv_setnv(PUSHmortal, (NV)(n)); } STMT_END +#endif + +#ifndef mXPUSHi +# define mXPUSHi(i) STMT_START { EXTEND(sp,1); sv_setiv(PUSHmortal, (IV)(i)); } STMT_END +#endif + +#ifndef mXPUSHu +# define mXPUSHu(u) STMT_START { EXTEND(sp,1); sv_setuv(PUSHmortal, (UV)(u)); } STMT_END +#endif + +/* Replace: 1 */ +#ifndef call_sv +# define call_sv perl_call_sv +#endif + +#ifndef call_pv +# define call_pv perl_call_pv +#endif + +#ifndef call_argv +# define call_argv perl_call_argv +#endif + +#ifndef call_method +# define call_method perl_call_method +#endif +#ifndef eval_sv +# define eval_sv perl_eval_sv +#endif + +/* Replace: 0 */ +#ifndef PERL_LOADMOD_DENY +# define PERL_LOADMOD_DENY 0x1 +#endif + +#ifndef PERL_LOADMOD_NOIMPORT +# define PERL_LOADMOD_NOIMPORT 0x2 +#endif + +#ifndef PERL_LOADMOD_IMPORT_OPS +# define PERL_LOADMOD_IMPORT_OPS 0x4 +#endif + +#ifndef G_METHOD +# define G_METHOD 64 +# ifdef call_sv +# undef call_sv +# endif +# if (PERL_BCDVERSION < 0x5006000) +# define call_sv(sv, flags) ((flags) & G_METHOD ? perl_call_method((char *) SvPV_nolen_const(sv), \ + (flags) & ~G_METHOD) : perl_call_sv(sv, flags)) +# else +# define call_sv(sv, flags) ((flags) & G_METHOD ? Perl_call_method(aTHX_ (char *) SvPV_nolen_const(sv), \ + (flags) & ~G_METHOD) : Perl_call_sv(aTHX_ sv, flags)) +# endif +#endif + +/* Replace perl_eval_pv with eval_pv */ + +#ifndef eval_pv +#if defined(NEED_eval_pv) +static SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); +static +#else +extern SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); +#endif + +#ifdef eval_pv +# undef eval_pv +#endif +#define eval_pv(a,b) DPPP_(my_eval_pv)(aTHX_ a,b) +#define Perl_eval_pv DPPP_(my_eval_pv) + +#if defined(NEED_eval_pv) || defined(NEED_eval_pv_GLOBAL) + +SV* +DPPP_(my_eval_pv)(char *p, I32 croak_on_error) +{ + dSP; + SV* sv = newSVpv(p, 0); + + PUSHMARK(sp); + eval_sv(sv, G_SCALAR); + SvREFCNT_dec(sv); + + SPAGAIN; + sv = POPs; + PUTBACK; + + if (croak_on_error && SvTRUE(GvSV(errgv))) + croak(SvPVx(GvSV(errgv), na)); + + return sv; +} + +#endif +#endif + +#ifndef vload_module +#if defined(NEED_vload_module) +static void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); +static +#else +extern void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); +#endif + +#ifdef vload_module +# undef vload_module +#endif +#define vload_module(a,b,c,d) DPPP_(my_vload_module)(aTHX_ a,b,c,d) +#define Perl_vload_module DPPP_(my_vload_module) + +#if defined(NEED_vload_module) || defined(NEED_vload_module_GLOBAL) + +void +DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args) +{ + dTHR; + dVAR; + OP *veop, *imop; + + OP * const modname = newSVOP(OP_CONST, 0, name); + /* 5.005 has a somewhat hacky force_normal that doesn't croak on + SvREADONLY() if PL_compling is true. Current perls take care in + ck_require() to correctly turn off SvREADONLY before calling + force_normal_flags(). This seems a better fix than fudging PL_compling + */ + SvREADONLY_off(((SVOP*)modname)->op_sv); + modname->op_private |= OPpCONST_BARE; + if (ver) { + veop = newSVOP(OP_CONST, 0, ver); + } + else + veop = NULL; + if (flags & PERL_LOADMOD_NOIMPORT) { + imop = sawparens(newNULLLIST()); + } + else if (flags & PERL_LOADMOD_IMPORT_OPS) { + imop = va_arg(*args, OP*); + } + else { + SV *sv; + imop = NULL; + sv = va_arg(*args, SV*); + while (sv) { + imop = append_elem(OP_LIST, imop, newSVOP(OP_CONST, 0, sv)); + sv = va_arg(*args, SV*); + } + } + { + const line_t ocopline = PL_copline; + COP * const ocurcop = PL_curcop; + const int oexpect = PL_expect; + +#if (PERL_BCDVERSION >= 0x5004000) + utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(FALSE, 0), + veop, modname, imop); +#else + utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(), + modname, imop); +#endif + PL_expect = oexpect; + PL_copline = ocopline; + PL_curcop = ocurcop; + } +} + +#endif +#endif + +#ifndef load_module +#if defined(NEED_load_module) +static void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); +static +#else +extern void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); +#endif + +#ifdef load_module +# undef load_module +#endif +#define load_module DPPP_(my_load_module) +#define Perl_load_module DPPP_(my_load_module) + +#if defined(NEED_load_module) || defined(NEED_load_module_GLOBAL) + +void +DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...) +{ + va_list args; + va_start(args, ver); + vload_module(flags, name, ver, &args); + va_end(args); +} + +#endif +#endif +#ifndef newRV_inc +# define newRV_inc(sv) newRV(sv) /* Replace */ +#endif + +#ifndef newRV_noinc +#if defined(NEED_newRV_noinc) +static SV * DPPP_(my_newRV_noinc)(SV *sv); +static +#else +extern SV * DPPP_(my_newRV_noinc)(SV *sv); +#endif + +#ifdef newRV_noinc +# undef newRV_noinc +#endif +#define newRV_noinc(a) DPPP_(my_newRV_noinc)(aTHX_ a) +#define Perl_newRV_noinc DPPP_(my_newRV_noinc) + +#if defined(NEED_newRV_noinc) || defined(NEED_newRV_noinc_GLOBAL) +SV * +DPPP_(my_newRV_noinc)(SV *sv) +{ + SV *rv = (SV *)newRV(sv); + SvREFCNT_dec(sv); + return rv; +} +#endif +#endif + +/* Hint: newCONSTSUB + * Returns a CV* as of perl-5.7.1. This return value is not supported + * by Devel::PPPort. + */ + +/* newCONSTSUB from IO.xs is in the core starting with 5.004_63 */ +#if (PERL_BCDVERSION < 0x5004063) && (PERL_BCDVERSION != 0x5004005) +#if defined(NEED_newCONSTSUB) +static void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); +static +#else +extern void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); +#endif + +#ifdef newCONSTSUB +# undef newCONSTSUB +#endif +#define newCONSTSUB(a,b,c) DPPP_(my_newCONSTSUB)(aTHX_ a,b,c) +#define Perl_newCONSTSUB DPPP_(my_newCONSTSUB) + +#if defined(NEED_newCONSTSUB) || defined(NEED_newCONSTSUB_GLOBAL) + +/* This is just a trick to avoid a dependency of newCONSTSUB on PL_parser */ +/* (There's no PL_parser in perl < 5.005, so this is completely safe) */ +#define D_PPP_PL_copline PL_copline + +void +DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv) +{ + U32 oldhints = PL_hints; + HV *old_cop_stash = PL_curcop->cop_stash; + HV *old_curstash = PL_curstash; + line_t oldline = PL_curcop->cop_line; + PL_curcop->cop_line = D_PPP_PL_copline; + + PL_hints &= ~HINT_BLOCK_SCOPE; + if (stash) + PL_curstash = PL_curcop->cop_stash = stash; + + newSUB( + +#if (PERL_BCDVERSION < 0x5003022) + start_subparse(), +#elif (PERL_BCDVERSION == 0x5003022) + start_subparse(0), +#else /* 5.003_23 onwards */ + start_subparse(FALSE, 0), +#endif + + newSVOP(OP_CONST, 0, newSVpv((char *) name, 0)), + newSVOP(OP_CONST, 0, &PL_sv_no), /* SvPV(&PL_sv_no) == "" -- GMB */ + newSTATEOP(0, Nullch, newSVOP(OP_CONST, 0, sv)) + ); + + PL_hints = oldhints; + PL_curcop->cop_stash = old_cop_stash; + PL_curstash = old_curstash; + PL_curcop->cop_line = oldline; +} +#endif +#endif + +/* + * Boilerplate macros for initializing and accessing interpreter-local + * data from C. All statics in extensions should be reworked to use + * this, if you want to make the extension thread-safe. See ext/re/re.xs + * for an example of the use of these macros. + * + * Code that uses these macros is responsible for the following: + * 1. #define MY_CXT_KEY to a unique string, e.g. "DynaLoader_guts" + * 2. Declare a typedef named my_cxt_t that is a structure that contains + * all the data that needs to be interpreter-local. + * 3. Use the START_MY_CXT macro after the declaration of my_cxt_t. + * 4. Use the MY_CXT_INIT macro such that it is called exactly once + * (typically put in the BOOT: section). + * 5. Use the members of the my_cxt_t structure everywhere as + * MY_CXT.member. + * 6. Use the dMY_CXT macro (a declaration) in all the functions that + * access MY_CXT. + */ + +#if defined(MULTIPLICITY) || defined(PERL_OBJECT) || \ + defined(PERL_CAPI) || defined(PERL_IMPLICIT_CONTEXT) + +#ifndef START_MY_CXT + +/* This must appear in all extensions that define a my_cxt_t structure, + * right after the definition (i.e. at file scope). The non-threads + * case below uses it to declare the data as static. */ +#define START_MY_CXT + +#if (PERL_BCDVERSION < 0x5004068) +/* Fetches the SV that keeps the per-interpreter data. */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = get_sv(MY_CXT_KEY, FALSE) +#else /* >= perl5.004_68 */ +#define dMY_CXT_SV \ + SV *my_cxt_sv = *hv_fetch(PL_modglobal, MY_CXT_KEY, \ + sizeof(MY_CXT_KEY)-1, TRUE) +#endif /* < perl5.004_68 */ + +/* This declaration should be used within all functions that use the + * interpreter-local data. */ +#define dMY_CXT \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = INT2PTR(my_cxt_t*,SvUV(my_cxt_sv)) + +/* Creates and zeroes the per-interpreter data. + * (We allocate my_cxtp in a Perl SV so that it will be released when + * the interpreter goes away.) */ +#define MY_CXT_INIT \ + dMY_CXT_SV; \ + /* newSV() allocates one more than needed */ \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Zero(my_cxtp, 1, my_cxt_t); \ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) + +/* This macro must be used to access members of the my_cxt_t structure. + * e.g. MYCXT.some_data */ +#define MY_CXT (*my_cxtp) + +/* Judicious use of these macros can reduce the number of times dMY_CXT + * is used. Use is similar to pTHX, aTHX etc. */ +#define pMY_CXT my_cxt_t *my_cxtp +#define pMY_CXT_ pMY_CXT, +#define _pMY_CXT ,pMY_CXT +#define aMY_CXT my_cxtp +#define aMY_CXT_ aMY_CXT, +#define _aMY_CXT ,aMY_CXT + +#endif /* START_MY_CXT */ + +#ifndef MY_CXT_CLONE +/* Clones the per-interpreter data. */ +#define MY_CXT_CLONE \ + dMY_CXT_SV; \ + my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ + Copy(INT2PTR(my_cxt_t*, SvUV(my_cxt_sv)), my_cxtp, 1, my_cxt_t);\ + sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) +#endif + +#else /* single interpreter */ + +#ifndef START_MY_CXT + +#define START_MY_CXT static my_cxt_t my_cxt; +#define dMY_CXT_SV dNOOP +#define dMY_CXT dNOOP +#define MY_CXT_INIT NOOP +#define MY_CXT my_cxt + +#define pMY_CXT void +#define pMY_CXT_ +#define _pMY_CXT +#define aMY_CXT +#define aMY_CXT_ +#define _aMY_CXT + +#endif /* START_MY_CXT */ + +#ifndef MY_CXT_CLONE +#define MY_CXT_CLONE NOOP +#endif + +#endif + +#ifndef IVdf +# if IVSIZE == LONGSIZE +# define IVdf "ld" +# define UVuf "lu" +# define UVof "lo" +# define UVxf "lx" +# define UVXf "lX" +# else +# if IVSIZE == INTSIZE +# define IVdf "d" +# define UVuf "u" +# define UVof "o" +# define UVxf "x" +# define UVXf "X" +# endif +# endif +#endif + +#ifndef NVef +# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) && \ + defined(PERL_PRIfldbl) && (PERL_BCDVERSION != 0x5006000) + /* Not very likely, but let's try anyway. */ +# define NVef PERL_PRIeldbl +# define NVff PERL_PRIfldbl +# define NVgf PERL_PRIgldbl +# else +# define NVef "e" +# define NVff "f" +# define NVgf "g" +# endif +#endif + +#ifndef SvREFCNT_inc +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + if (_sv) \ + (SvREFCNT(_sv))++; \ + _sv; \ + }) +# else +# define SvREFCNT_inc(sv) \ + ((PL_Sv=(SV*)(sv)) ? (++(SvREFCNT(PL_Sv)),PL_Sv) : NULL) +# endif +#endif + +#ifndef SvREFCNT_inc_simple +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_simple(sv) \ + ({ \ + if (sv) \ + (SvREFCNT(sv))++; \ + (SV *)(sv); \ + }) +# else +# define SvREFCNT_inc_simple(sv) \ + ((sv) ? (SvREFCNT(sv)++,(SV*)(sv)) : NULL) +# endif +#endif + +#ifndef SvREFCNT_inc_NN +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_NN(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + SvREFCNT(_sv)++; \ + _sv; \ + }) +# else +# define SvREFCNT_inc_NN(sv) \ + (PL_Sv=(SV*)(sv),++(SvREFCNT(PL_Sv)),PL_Sv) +# endif +#endif + +#ifndef SvREFCNT_inc_void +# ifdef PERL_USE_GCC_BRACE_GROUPS +# define SvREFCNT_inc_void(sv) \ + ({ \ + SV * const _sv = (SV*)(sv); \ + if (_sv) \ + (void)(SvREFCNT(_sv)++); \ + }) +# else +# define SvREFCNT_inc_void(sv) \ + (void)((PL_Sv=(SV*)(sv)) ? ++(SvREFCNT(PL_Sv)) : 0) +# endif +#endif +#ifndef SvREFCNT_inc_simple_void +# define SvREFCNT_inc_simple_void(sv) STMT_START { if (sv) SvREFCNT(sv)++; } STMT_END +#endif + +#ifndef SvREFCNT_inc_simple_NN +# define SvREFCNT_inc_simple_NN(sv) (++SvREFCNT(sv), (SV*)(sv)) +#endif + +#ifndef SvREFCNT_inc_void_NN +# define SvREFCNT_inc_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) +#endif + +#ifndef SvREFCNT_inc_simple_void_NN +# define SvREFCNT_inc_simple_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) +#endif + +#if (PERL_BCDVERSION < 0x5006000) +# define D_PPP_CONSTPV_ARG(x) ((char *) (x)) +#else +# define D_PPP_CONSTPV_ARG(x) (x) +#endif +#ifndef newSVpvn +# define newSVpvn(data,len) ((data) \ + ? ((len) ? newSVpv((data), (len)) : newSVpv("", 0)) \ + : newSV(0)) +#endif +#ifndef newSVpvn_utf8 +# define newSVpvn_utf8(s, len, u) newSVpvn_flags((s), (len), (u) ? SVf_UTF8 : 0) +#endif +#ifndef SVf_UTF8 +# define SVf_UTF8 0 +#endif + +#ifndef newSVpvn_flags + +#if defined(NEED_newSVpvn_flags) +static SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); +static +#else +extern SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); +#endif + +#ifdef newSVpvn_flags +# undef newSVpvn_flags +#endif +#define newSVpvn_flags(a,b,c) DPPP_(my_newSVpvn_flags)(aTHX_ a,b,c) +#define Perl_newSVpvn_flags DPPP_(my_newSVpvn_flags) + +#if defined(NEED_newSVpvn_flags) || defined(NEED_newSVpvn_flags_GLOBAL) + +SV * +DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags) +{ + SV *sv = newSVpvn(D_PPP_CONSTPV_ARG(s), len); + SvFLAGS(sv) |= (flags & SVf_UTF8); + return (flags & SVs_TEMP) ? sv_2mortal(sv) : sv; +} + +#endif + +#endif + +/* Backwards compatibility stuff... :-( */ +#if !defined(NEED_sv_2pv_flags) && defined(NEED_sv_2pv_nolen) +# define NEED_sv_2pv_flags +#endif +#if !defined(NEED_sv_2pv_flags_GLOBAL) && defined(NEED_sv_2pv_nolen_GLOBAL) +# define NEED_sv_2pv_flags_GLOBAL +#endif + +/* Hint: sv_2pv_nolen + * Use the SvPV_nolen() or SvPV_nolen_const() macros instead of sv_2pv_nolen(). + */ +#ifndef sv_2pv_nolen +# define sv_2pv_nolen(sv) SvPV_nolen(sv) +#endif + +#ifdef SvPVbyte + +/* Hint: SvPVbyte + * Does not work in perl-5.6.1, ppport.h implements a version + * borrowed from perl-5.7.3. + */ + +#if (PERL_BCDVERSION < 0x5007000) + +#if defined(NEED_sv_2pvbyte) +static char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); +static +#else +extern char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); +#endif + +#ifdef sv_2pvbyte +# undef sv_2pvbyte +#endif +#define sv_2pvbyte(a,b) DPPP_(my_sv_2pvbyte)(aTHX_ a,b) +#define Perl_sv_2pvbyte DPPP_(my_sv_2pvbyte) + +#if defined(NEED_sv_2pvbyte) || defined(NEED_sv_2pvbyte_GLOBAL) + +char * +DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp) +{ + sv_utf8_downgrade(sv,0); + return SvPV(sv,*lp); +} + +#endif + +/* Hint: sv_2pvbyte + * Use the SvPVbyte() macro instead of sv_2pvbyte(). + */ + +#undef SvPVbyte + +#define SvPVbyte(sv, lp) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_UTF8)) == (SVf_POK) \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pvbyte(sv, &lp)) + +#endif + +#else + +# define SvPVbyte SvPV +# define sv_2pvbyte sv_2pv + +#endif +#ifndef sv_2pvbyte_nolen +# define sv_2pvbyte_nolen(sv) sv_2pv_nolen(sv) +#endif + +/* Hint: sv_pvn + * Always use the SvPV() macro instead of sv_pvn(). + */ + +/* Hint: sv_pvn_force + * Always use the SvPV_force() macro instead of sv_pvn_force(). + */ + +/* If these are undefined, they're not handled by the core anyway */ +#ifndef SV_IMMEDIATE_UNREF +# define SV_IMMEDIATE_UNREF 0 +#endif + +#ifndef SV_GMAGIC +# define SV_GMAGIC 0 +#endif + +#ifndef SV_COW_DROP_PV +# define SV_COW_DROP_PV 0 +#endif + +#ifndef SV_UTF8_NO_ENCODING +# define SV_UTF8_NO_ENCODING 0 +#endif + +#ifndef SV_NOSTEAL +# define SV_NOSTEAL 0 +#endif + +#ifndef SV_CONST_RETURN +# define SV_CONST_RETURN 0 +#endif + +#ifndef SV_MUTABLE_RETURN +# define SV_MUTABLE_RETURN 0 +#endif + +#ifndef SV_SMAGIC +# define SV_SMAGIC 0 +#endif + +#ifndef SV_HAS_TRAILING_NUL +# define SV_HAS_TRAILING_NUL 0 +#endif + +#ifndef SV_COW_SHARED_HASH_KEYS +# define SV_COW_SHARED_HASH_KEYS 0 +#endif + +#if (PERL_BCDVERSION < 0x5007002) + +#if defined(NEED_sv_2pv_flags) +static char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +static +#else +extern char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +#endif + +#ifdef sv_2pv_flags +# undef sv_2pv_flags +#endif +#define sv_2pv_flags(a,b,c) DPPP_(my_sv_2pv_flags)(aTHX_ a,b,c) +#define Perl_sv_2pv_flags DPPP_(my_sv_2pv_flags) + +#if defined(NEED_sv_2pv_flags) || defined(NEED_sv_2pv_flags_GLOBAL) + +char * +DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) +{ + STRLEN n_a = (STRLEN) flags; + return sv_2pv(sv, lp ? lp : &n_a); +} + +#endif + +#if defined(NEED_sv_pvn_force_flags) +static char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +static +#else +extern char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); +#endif + +#ifdef sv_pvn_force_flags +# undef sv_pvn_force_flags +#endif +#define sv_pvn_force_flags(a,b,c) DPPP_(my_sv_pvn_force_flags)(aTHX_ a,b,c) +#define Perl_sv_pvn_force_flags DPPP_(my_sv_pvn_force_flags) + +#if defined(NEED_sv_pvn_force_flags) || defined(NEED_sv_pvn_force_flags_GLOBAL) + +char * +DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) +{ + STRLEN n_a = (STRLEN) flags; + return sv_pvn_force(sv, lp ? lp : &n_a); +} + +#endif + +#endif + +#if (PERL_BCDVERSION < 0x5008008) || ( (PERL_BCDVERSION >= 0x5009000) && (PERL_BCDVERSION < 0x5009003) ) +# define DPPP_SVPV_NOLEN_LP_ARG &PL_na +#else +# define DPPP_SVPV_NOLEN_LP_ARG 0 +#endif +#ifndef SvPV_const +# define SvPV_const(sv, lp) SvPV_flags_const(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_mutable +# define SvPV_mutable(sv, lp) SvPV_flags_mutable(sv, lp, SV_GMAGIC) +#endif +#ifndef SvPV_flags +# define SvPV_flags(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pv_flags(sv, &lp, flags)) +#endif +#ifndef SvPV_flags_const +# define SvPV_flags_const(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_const(sv)) : \ + (const char*) sv_2pv_flags(sv, &lp, flags|SV_CONST_RETURN)) +#endif +#ifndef SvPV_flags_const_nolen +# define SvPV_flags_const_nolen(sv, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX_const(sv) : \ + (const char*) sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags|SV_CONST_RETURN)) +#endif +#ifndef SvPV_flags_mutable +# define SvPV_flags_mutable(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) : \ + sv_2pv_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) +#endif +#ifndef SvPV_force +# define SvPV_force(sv, lp) SvPV_force_flags(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_force_nolen +# define SvPV_force_nolen(sv) SvPV_force_flags_nolen(sv, SV_GMAGIC) +#endif + +#ifndef SvPV_force_mutable +# define SvPV_force_mutable(sv, lp) SvPV_force_flags_mutable(sv, lp, SV_GMAGIC) +#endif + +#ifndef SvPV_force_nomg +# define SvPV_force_nomg(sv, lp) SvPV_force_flags(sv, lp, 0) +#endif + +#ifndef SvPV_force_nomg_nolen +# define SvPV_force_nomg_nolen(sv) SvPV_force_flags_nolen(sv, 0) +#endif +#ifndef SvPV_force_flags +# define SvPV_force_flags(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_pvn_force_flags(sv, &lp, flags)) +#endif +#ifndef SvPV_force_flags_nolen +# define SvPV_force_flags_nolen(sv, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? SvPVX(sv) : sv_pvn_force_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags)) +#endif +#ifndef SvPV_force_flags_mutable +# define SvPV_force_flags_mutable(sv, lp, flags) \ + ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ + ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) \ + : sv_pvn_force_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) +#endif +#ifndef SvPV_nolen +# define SvPV_nolen(sv) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC)) +#endif +#ifndef SvPV_nolen_const +# define SvPV_nolen_const(sv) \ + ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ + ? SvPVX_const(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC|SV_CONST_RETURN)) +#endif +#ifndef SvPV_nomg +# define SvPV_nomg(sv, lp) SvPV_flags(sv, lp, 0) +#endif + +#ifndef SvPV_nomg_const +# define SvPV_nomg_const(sv, lp) SvPV_flags_const(sv, lp, 0) +#endif + +#ifndef SvPV_nomg_const_nolen +# define SvPV_nomg_const_nolen(sv) SvPV_flags_const_nolen(sv, 0) +#endif +#ifndef SvPV_renew +# define SvPV_renew(sv,n) STMT_START { SvLEN_set(sv, n); \ + SvPV_set((sv), (char *) saferealloc( \ + (Malloc_t)SvPVX(sv), (MEM_SIZE)((n)))); \ + } STMT_END +#endif +#ifndef SvMAGIC_set +# define SvMAGIC_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ + (((XPVMG*) SvANY(sv))->xmg_magic = (val)); } STMT_END +#endif + +#if (PERL_BCDVERSION < 0x5009003) +#ifndef SvPVX_const +# define SvPVX_const(sv) ((const char*) (0 + SvPVX(sv))) +#endif + +#ifndef SvPVX_mutable +# define SvPVX_mutable(sv) (0 + SvPVX(sv)) +#endif +#ifndef SvRV_set +# define SvRV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ + (((XRV*) SvANY(sv))->xrv_rv = (val)); } STMT_END +#endif + +#else +#ifndef SvPVX_const +# define SvPVX_const(sv) ((const char*)((sv)->sv_u.svu_pv)) +#endif + +#ifndef SvPVX_mutable +# define SvPVX_mutable(sv) ((sv)->sv_u.svu_pv) +#endif +#ifndef SvRV_set +# define SvRV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ + ((sv)->sv_u.svu_rv = (val)); } STMT_END +#endif + +#endif +#ifndef SvSTASH_set +# define SvSTASH_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ + (((XPVMG*) SvANY(sv))->xmg_stash = (val)); } STMT_END +#endif + +#if (PERL_BCDVERSION < 0x5004000) +#ifndef SvUV_set +# define SvUV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ + (((XPVIV*) SvANY(sv))->xiv_iv = (IV) (val)); } STMT_END +#endif + +#else +#ifndef SvUV_set +# define SvUV_set(sv, val) \ + STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ + (((XPVUV*) SvANY(sv))->xuv_uv = (val)); } STMT_END +#endif + +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(vnewSVpvf) +#if defined(NEED_vnewSVpvf) +static SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); +static +#else +extern SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); +#endif + +#ifdef vnewSVpvf +# undef vnewSVpvf +#endif +#define vnewSVpvf(a,b) DPPP_(my_vnewSVpvf)(aTHX_ a,b) +#define Perl_vnewSVpvf DPPP_(my_vnewSVpvf) + +#if defined(NEED_vnewSVpvf) || defined(NEED_vnewSVpvf_GLOBAL) + +SV * +DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args) +{ + register SV *sv = newSV(0); + sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); + return sv; +} + +#endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf) +# define sv_vcatpvf(sv, pat, args) sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf) +# define sv_vsetpvf(sv, pat, args) sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg) +#if defined(NEED_sv_catpvf_mg) +static void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +#endif + +#define Perl_sv_catpvf_mg DPPP_(my_sv_catpvf_mg) + +#if defined(NEED_sv_catpvf_mg) || defined(NEED_sv_catpvf_mg_GLOBAL) + +void +DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...) +{ + va_list args; + va_start(args, pat); + sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif + +#ifdef PERL_IMPLICIT_CONTEXT +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg_nocontext) +#if defined(NEED_sv_catpvf_mg_nocontext) +static void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); +#endif + +#define sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) +#define Perl_sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) + +#if defined(NEED_sv_catpvf_mg_nocontext) || defined(NEED_sv_catpvf_mg_nocontext_GLOBAL) + +void +DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...) +{ + dTHX; + va_list args; + va_start(args, pat); + sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif +#endif + +/* sv_catpvf_mg depends on sv_catpvf_mg_nocontext */ +#ifndef sv_catpvf_mg +# ifdef PERL_IMPLICIT_CONTEXT +# define sv_catpvf_mg Perl_sv_catpvf_mg_nocontext +# else +# define sv_catpvf_mg Perl_sv_catpvf_mg +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf_mg) +# define sv_vcatpvf_mg(sv, pat, args) \ + STMT_START { \ + sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ + SvSETMAGIC(sv); \ + } STMT_END +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg) +#if defined(NEED_sv_setpvf_mg) +static void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); +#endif + +#define Perl_sv_setpvf_mg DPPP_(my_sv_setpvf_mg) + +#if defined(NEED_sv_setpvf_mg) || defined(NEED_sv_setpvf_mg_GLOBAL) + +void +DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...) +{ + va_list args; + va_start(args, pat); + sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif + +#ifdef PERL_IMPLICIT_CONTEXT +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg_nocontext) +#if defined(NEED_sv_setpvf_mg_nocontext) +static void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); +static +#else +extern void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); +#endif + +#define sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) +#define Perl_sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) + +#if defined(NEED_sv_setpvf_mg_nocontext) || defined(NEED_sv_setpvf_mg_nocontext_GLOBAL) + +void +DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...) +{ + dTHX; + va_list args; + va_start(args, pat); + sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); + SvSETMAGIC(sv); + va_end(args); +} + +#endif +#endif +#endif + +/* sv_setpvf_mg depends on sv_setpvf_mg_nocontext */ +#ifndef sv_setpvf_mg +# ifdef PERL_IMPLICIT_CONTEXT +# define sv_setpvf_mg Perl_sv_setpvf_mg_nocontext +# else +# define sv_setpvf_mg Perl_sv_setpvf_mg +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf_mg) +# define sv_vsetpvf_mg(sv, pat, args) \ + STMT_START { \ + sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ + SvSETMAGIC(sv); \ + } STMT_END +#endif + +#ifndef newSVpvn_share + +#if defined(NEED_newSVpvn_share) +static SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); +static +#else +extern SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); +#endif + +#ifdef newSVpvn_share +# undef newSVpvn_share +#endif +#define newSVpvn_share(a,b,c) DPPP_(my_newSVpvn_share)(aTHX_ a,b,c) +#define Perl_newSVpvn_share DPPP_(my_newSVpvn_share) + +#if defined(NEED_newSVpvn_share) || defined(NEED_newSVpvn_share_GLOBAL) + +SV * +DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash) +{ + SV *sv; + if (len < 0) + len = -len; + if (!hash) + PERL_HASH(hash, (char*) src, len); + sv = newSVpvn((char *) src, len); + sv_upgrade(sv, SVt_PVIV); + SvIVX(sv) = hash; + SvREADONLY_on(sv); + SvPOK_on(sv); + return sv; +} + +#endif + +#endif +#ifndef SvSHARED_HASH +# define SvSHARED_HASH(sv) (0 + SvUVX(sv)) +#endif +#ifndef WARN_ALL +# define WARN_ALL 0 +#endif + +#ifndef WARN_CLOSURE +# define WARN_CLOSURE 1 +#endif + +#ifndef WARN_DEPRECATED +# define WARN_DEPRECATED 2 +#endif + +#ifndef WARN_EXITING +# define WARN_EXITING 3 +#endif + +#ifndef WARN_GLOB +# define WARN_GLOB 4 +#endif + +#ifndef WARN_IO +# define WARN_IO 5 +#endif + +#ifndef WARN_CLOSED +# define WARN_CLOSED 6 +#endif + +#ifndef WARN_EXEC +# define WARN_EXEC 7 +#endif + +#ifndef WARN_LAYER +# define WARN_LAYER 8 +#endif + +#ifndef WARN_NEWLINE +# define WARN_NEWLINE 9 +#endif + +#ifndef WARN_PIPE +# define WARN_PIPE 10 +#endif + +#ifndef WARN_UNOPENED +# define WARN_UNOPENED 11 +#endif + +#ifndef WARN_MISC +# define WARN_MISC 12 +#endif + +#ifndef WARN_NUMERIC +# define WARN_NUMERIC 13 +#endif + +#ifndef WARN_ONCE +# define WARN_ONCE 14 +#endif + +#ifndef WARN_OVERFLOW +# define WARN_OVERFLOW 15 +#endif + +#ifndef WARN_PACK +# define WARN_PACK 16 +#endif + +#ifndef WARN_PORTABLE +# define WARN_PORTABLE 17 +#endif + +#ifndef WARN_RECURSION +# define WARN_RECURSION 18 +#endif + +#ifndef WARN_REDEFINE +# define WARN_REDEFINE 19 +#endif + +#ifndef WARN_REGEXP +# define WARN_REGEXP 20 +#endif + +#ifndef WARN_SEVERE +# define WARN_SEVERE 21 +#endif + +#ifndef WARN_DEBUGGING +# define WARN_DEBUGGING 22 +#endif + +#ifndef WARN_INPLACE +# define WARN_INPLACE 23 +#endif + +#ifndef WARN_INTERNAL +# define WARN_INTERNAL 24 +#endif + +#ifndef WARN_MALLOC +# define WARN_MALLOC 25 +#endif + +#ifndef WARN_SIGNAL +# define WARN_SIGNAL 26 +#endif + +#ifndef WARN_SUBSTR +# define WARN_SUBSTR 27 +#endif + +#ifndef WARN_SYNTAX +# define WARN_SYNTAX 28 +#endif + +#ifndef WARN_AMBIGUOUS +# define WARN_AMBIGUOUS 29 +#endif + +#ifndef WARN_BAREWORD +# define WARN_BAREWORD 30 +#endif + +#ifndef WARN_DIGIT +# define WARN_DIGIT 31 +#endif + +#ifndef WARN_PARENTHESIS +# define WARN_PARENTHESIS 32 +#endif + +#ifndef WARN_PRECEDENCE +# define WARN_PRECEDENCE 33 +#endif + +#ifndef WARN_PRINTF +# define WARN_PRINTF 34 +#endif + +#ifndef WARN_PROTOTYPE +# define WARN_PROTOTYPE 35 +#endif + +#ifndef WARN_QW +# define WARN_QW 36 +#endif + +#ifndef WARN_RESERVED +# define WARN_RESERVED 37 +#endif + +#ifndef WARN_SEMICOLON +# define WARN_SEMICOLON 38 +#endif + +#ifndef WARN_TAINT +# define WARN_TAINT 39 +#endif + +#ifndef WARN_THREADS +# define WARN_THREADS 40 +#endif + +#ifndef WARN_UNINITIALIZED +# define WARN_UNINITIALIZED 41 +#endif + +#ifndef WARN_UNPACK +# define WARN_UNPACK 42 +#endif + +#ifndef WARN_UNTIE +# define WARN_UNTIE 43 +#endif + +#ifndef WARN_UTF8 +# define WARN_UTF8 44 +#endif + +#ifndef WARN_VOID +# define WARN_VOID 45 +#endif + +#ifndef WARN_ASSERTIONS +# define WARN_ASSERTIONS 46 +#endif +#ifndef packWARN +# define packWARN(a) (a) +#endif + +#ifndef ckWARN +# ifdef G_WARN_ON +# define ckWARN(a) (PL_dowarn & G_WARN_ON) +# else +# define ckWARN(a) PL_dowarn +# endif +#endif + +#if (PERL_BCDVERSION >= 0x5004000) && !defined(warner) +#if defined(NEED_warner) +static void DPPP_(my_warner)(U32 err, const char *pat, ...); +static +#else +extern void DPPP_(my_warner)(U32 err, const char *pat, ...); +#endif + +#define Perl_warner DPPP_(my_warner) + +#if defined(NEED_warner) || defined(NEED_warner_GLOBAL) + +void +DPPP_(my_warner)(U32 err, const char *pat, ...) +{ + SV *sv; + va_list args; + + PERL_UNUSED_ARG(err); + + va_start(args, pat); + sv = vnewSVpvf(pat, &args); + va_end(args); + sv_2mortal(sv); + warn("%s", SvPV_nolen(sv)); +} + +#define warner Perl_warner + +#define Perl_warner_nocontext Perl_warner + +#endif +#endif + +/* concatenating with "" ensures that only literal strings are accepted as argument + * note that STR_WITH_LEN() can't be used as argument to macros or functions that + * under some configurations might be macros + */ +#ifndef STR_WITH_LEN +# define STR_WITH_LEN(s) (s ""), (sizeof(s)-1) +#endif +#ifndef newSVpvs +# define newSVpvs(str) newSVpvn(str "", sizeof(str) - 1) +#endif + +#ifndef newSVpvs_flags +# define newSVpvs_flags(str, flags) newSVpvn_flags(str "", sizeof(str) - 1, flags) +#endif + +#ifndef sv_catpvs +# define sv_catpvs(sv, str) sv_catpvn(sv, str "", sizeof(str) - 1) +#endif + +#ifndef sv_setpvs +# define sv_setpvs(sv, str) sv_setpvn(sv, str "", sizeof(str) - 1) +#endif + +#ifndef hv_fetchs +# define hv_fetchs(hv, key, lval) hv_fetch(hv, key "", sizeof(key) - 1, lval) +#endif + +#ifndef hv_stores +# define hv_stores(hv, key, val) hv_store(hv, key "", sizeof(key) - 1, val, 0) +#endif +#ifndef SvGETMAGIC +# define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END +#endif +#ifndef PERL_MAGIC_sv +# define PERL_MAGIC_sv '\0' +#endif + +#ifndef PERL_MAGIC_overload +# define PERL_MAGIC_overload 'A' +#endif + +#ifndef PERL_MAGIC_overload_elem +# define PERL_MAGIC_overload_elem 'a' +#endif + +#ifndef PERL_MAGIC_overload_table +# define PERL_MAGIC_overload_table 'c' +#endif + +#ifndef PERL_MAGIC_bm +# define PERL_MAGIC_bm 'B' +#endif + +#ifndef PERL_MAGIC_regdata +# define PERL_MAGIC_regdata 'D' +#endif + +#ifndef PERL_MAGIC_regdatum +# define PERL_MAGIC_regdatum 'd' +#endif + +#ifndef PERL_MAGIC_env +# define PERL_MAGIC_env 'E' +#endif + +#ifndef PERL_MAGIC_envelem +# define PERL_MAGIC_envelem 'e' +#endif + +#ifndef PERL_MAGIC_fm +# define PERL_MAGIC_fm 'f' +#endif + +#ifndef PERL_MAGIC_regex_global +# define PERL_MAGIC_regex_global 'g' +#endif + +#ifndef PERL_MAGIC_isa +# define PERL_MAGIC_isa 'I' +#endif + +#ifndef PERL_MAGIC_isaelem +# define PERL_MAGIC_isaelem 'i' +#endif + +#ifndef PERL_MAGIC_nkeys +# define PERL_MAGIC_nkeys 'k' +#endif + +#ifndef PERL_MAGIC_dbfile +# define PERL_MAGIC_dbfile 'L' +#endif + +#ifndef PERL_MAGIC_dbline +# define PERL_MAGIC_dbline 'l' +#endif + +#ifndef PERL_MAGIC_mutex +# define PERL_MAGIC_mutex 'm' +#endif + +#ifndef PERL_MAGIC_shared +# define PERL_MAGIC_shared 'N' +#endif + +#ifndef PERL_MAGIC_shared_scalar +# define PERL_MAGIC_shared_scalar 'n' +#endif + +#ifndef PERL_MAGIC_collxfrm +# define PERL_MAGIC_collxfrm 'o' +#endif + +#ifndef PERL_MAGIC_tied +# define PERL_MAGIC_tied 'P' +#endif + +#ifndef PERL_MAGIC_tiedelem +# define PERL_MAGIC_tiedelem 'p' +#endif + +#ifndef PERL_MAGIC_tiedscalar +# define PERL_MAGIC_tiedscalar 'q' +#endif + +#ifndef PERL_MAGIC_qr +# define PERL_MAGIC_qr 'r' +#endif + +#ifndef PERL_MAGIC_sig +# define PERL_MAGIC_sig 'S' +#endif + +#ifndef PERL_MAGIC_sigelem +# define PERL_MAGIC_sigelem 's' +#endif + +#ifndef PERL_MAGIC_taint +# define PERL_MAGIC_taint 't' +#endif + +#ifndef PERL_MAGIC_uvar +# define PERL_MAGIC_uvar 'U' +#endif + +#ifndef PERL_MAGIC_uvar_elem +# define PERL_MAGIC_uvar_elem 'u' +#endif + +#ifndef PERL_MAGIC_vstring +# define PERL_MAGIC_vstring 'V' +#endif + +#ifndef PERL_MAGIC_vec +# define PERL_MAGIC_vec 'v' +#endif + +#ifndef PERL_MAGIC_utf8 +# define PERL_MAGIC_utf8 'w' +#endif + +#ifndef PERL_MAGIC_substr +# define PERL_MAGIC_substr 'x' +#endif + +#ifndef PERL_MAGIC_defelem +# define PERL_MAGIC_defelem 'y' +#endif + +#ifndef PERL_MAGIC_glob +# define PERL_MAGIC_glob '*' +#endif + +#ifndef PERL_MAGIC_arylen +# define PERL_MAGIC_arylen '#' +#endif + +#ifndef PERL_MAGIC_pos +# define PERL_MAGIC_pos '.' +#endif + +#ifndef PERL_MAGIC_backref +# define PERL_MAGIC_backref '<' +#endif + +#ifndef PERL_MAGIC_ext +# define PERL_MAGIC_ext '~' +#endif + +/* That's the best we can do... */ +#ifndef sv_catpvn_nomg +# define sv_catpvn_nomg sv_catpvn +#endif + +#ifndef sv_catsv_nomg +# define sv_catsv_nomg sv_catsv +#endif + +#ifndef sv_setsv_nomg +# define sv_setsv_nomg sv_setsv +#endif + +#ifndef sv_pvn_nomg +# define sv_pvn_nomg sv_pvn +#endif + +#ifndef SvIV_nomg +# define SvIV_nomg SvIV +#endif + +#ifndef SvUV_nomg +# define SvUV_nomg SvUV +#endif + +#ifndef sv_catpv_mg +# define sv_catpv_mg(sv, ptr) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_catpv(TeMpSv,ptr); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_catpvn_mg +# define sv_catpvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_catpvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_catsv_mg +# define sv_catsv_mg(dsv, ssv) \ + STMT_START { \ + SV *TeMpSv = dsv; \ + sv_catsv(TeMpSv,ssv); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setiv_mg +# define sv_setiv_mg(sv, i) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setiv(TeMpSv,i); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setnv_mg +# define sv_setnv_mg(sv, num) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setnv(TeMpSv,num); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setpv_mg +# define sv_setpv_mg(sv, ptr) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setpv(TeMpSv,ptr); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setpvn_mg +# define sv_setpvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setpvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setsv_mg +# define sv_setsv_mg(dsv, ssv) \ + STMT_START { \ + SV *TeMpSv = dsv; \ + sv_setsv(TeMpSv,ssv); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_setuv_mg +# define sv_setuv_mg(sv, i) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_setuv(TeMpSv,i); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif + +#ifndef sv_usepvn_mg +# define sv_usepvn_mg(sv, ptr, len) \ + STMT_START { \ + SV *TeMpSv = sv; \ + sv_usepvn(TeMpSv,ptr,len); \ + SvSETMAGIC(TeMpSv); \ + } STMT_END +#endif +#ifndef SvVSTRING_mg +# define SvVSTRING_mg(sv) (SvMAGICAL(sv) ? mg_find(sv, PERL_MAGIC_vstring) : NULL) +#endif + +/* Hint: sv_magic_portable + * This is a compatibility function that is only available with + * Devel::PPPort. It is NOT in the perl core. + * Its purpose is to mimic the 5.8.0 behaviour of sv_magic() when + * it is being passed a name pointer with namlen == 0. In that + * case, perl 5.8.0 and later store the pointer, not a copy of it. + * The compatibility can be provided back to perl 5.004. With + * earlier versions, the code will not compile. + */ + +#if (PERL_BCDVERSION < 0x5004000) + + /* code that uses sv_magic_portable will not compile */ + +#elif (PERL_BCDVERSION < 0x5008000) + +# define sv_magic_portable(sv, obj, how, name, namlen) \ + STMT_START { \ + SV *SvMp_sv = (sv); \ + char *SvMp_name = (char *) (name); \ + I32 SvMp_namlen = (namlen); \ + if (SvMp_name && SvMp_namlen == 0) \ + { \ + MAGIC *mg; \ + sv_magic(SvMp_sv, obj, how, 0, 0); \ + mg = SvMAGIC(SvMp_sv); \ + mg->mg_len = -42; /* XXX: this is the tricky part */ \ + mg->mg_ptr = SvMp_name; \ + } \ + else \ + { \ + sv_magic(SvMp_sv, obj, how, SvMp_name, SvMp_namlen); \ + } \ + } STMT_END + +#else + +# define sv_magic_portable(a, b, c, d, e) sv_magic(a, b, c, d, e) + +#endif + +#ifdef USE_ITHREADS +#ifndef CopFILE +# define CopFILE(c) ((c)->cop_file) +#endif + +#ifndef CopFILEGV +# define CopFILEGV(c) (CopFILE(c) ? gv_fetchfile(CopFILE(c)) : Nullgv) +#endif + +#ifndef CopFILE_set +# define CopFILE_set(c,pv) ((c)->cop_file = savepv(pv)) +#endif + +#ifndef CopFILESV +# define CopFILESV(c) (CopFILE(c) ? GvSV(gv_fetchfile(CopFILE(c))) : Nullsv) +#endif + +#ifndef CopFILEAV +# define CopFILEAV(c) (CopFILE(c) ? GvAV(gv_fetchfile(CopFILE(c))) : Nullav) +#endif + +#ifndef CopSTASHPV +# define CopSTASHPV(c) ((c)->cop_stashpv) +#endif + +#ifndef CopSTASHPV_set +# define CopSTASHPV_set(c,pv) ((c)->cop_stashpv = ((pv) ? savepv(pv) : Nullch)) +#endif + +#ifndef CopSTASH +# define CopSTASH(c) (CopSTASHPV(c) ? gv_stashpv(CopSTASHPV(c),GV_ADD) : Nullhv) +#endif + +#ifndef CopSTASH_set +# define CopSTASH_set(c,hv) CopSTASHPV_set(c, (hv) ? HvNAME(hv) : Nullch) +#endif + +#ifndef CopSTASH_eq +# define CopSTASH_eq(c,hv) ((hv) && (CopSTASHPV(c) == HvNAME(hv) \ + || (CopSTASHPV(c) && HvNAME(hv) \ + && strEQ(CopSTASHPV(c), HvNAME(hv))))) +#endif + +#else +#ifndef CopFILEGV +# define CopFILEGV(c) ((c)->cop_filegv) +#endif + +#ifndef CopFILEGV_set +# define CopFILEGV_set(c,gv) ((c)->cop_filegv = (GV*)SvREFCNT_inc(gv)) +#endif + +#ifndef CopFILE_set +# define CopFILE_set(c,pv) CopFILEGV_set((c), gv_fetchfile(pv)) +#endif + +#ifndef CopFILESV +# define CopFILESV(c) (CopFILEGV(c) ? GvSV(CopFILEGV(c)) : Nullsv) +#endif + +#ifndef CopFILEAV +# define CopFILEAV(c) (CopFILEGV(c) ? GvAV(CopFILEGV(c)) : Nullav) +#endif + +#ifndef CopFILE +# define CopFILE(c) (CopFILESV(c) ? SvPVX(CopFILESV(c)) : Nullch) +#endif + +#ifndef CopSTASH +# define CopSTASH(c) ((c)->cop_stash) +#endif + +#ifndef CopSTASH_set +# define CopSTASH_set(c,hv) ((c)->cop_stash = (hv)) +#endif + +#ifndef CopSTASHPV +# define CopSTASHPV(c) (CopSTASH(c) ? HvNAME(CopSTASH(c)) : Nullch) +#endif + +#ifndef CopSTASHPV_set +# define CopSTASHPV_set(c,pv) CopSTASH_set((c), gv_stashpv(pv,GV_ADD)) +#endif + +#ifndef CopSTASH_eq +# define CopSTASH_eq(c,hv) (CopSTASH(c) == (hv)) +#endif + +#endif /* USE_ITHREADS */ +#ifndef IN_PERL_COMPILETIME +# define IN_PERL_COMPILETIME (PL_curcop == &PL_compiling) +#endif + +#ifndef IN_LOCALE_RUNTIME +# define IN_LOCALE_RUNTIME (PL_curcop->op_private & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE_COMPILETIME +# define IN_LOCALE_COMPILETIME (PL_hints & HINT_LOCALE) +#endif + +#ifndef IN_LOCALE +# define IN_LOCALE (IN_PERL_COMPILETIME ? IN_LOCALE_COMPILETIME : IN_LOCALE_RUNTIME) +#endif +#ifndef IS_NUMBER_IN_UV +# define IS_NUMBER_IN_UV 0x01 +#endif + +#ifndef IS_NUMBER_GREATER_THAN_UV_MAX +# define IS_NUMBER_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef IS_NUMBER_NOT_INT +# define IS_NUMBER_NOT_INT 0x04 +#endif + +#ifndef IS_NUMBER_NEG +# define IS_NUMBER_NEG 0x08 +#endif + +#ifndef IS_NUMBER_INFINITY +# define IS_NUMBER_INFINITY 0x10 +#endif + +#ifndef IS_NUMBER_NAN +# define IS_NUMBER_NAN 0x20 +#endif +#ifndef GROK_NUMERIC_RADIX +# define GROK_NUMERIC_RADIX(sp, send) grok_numeric_radix(sp, send) +#endif +#ifndef PERL_SCAN_GREATER_THAN_UV_MAX +# define PERL_SCAN_GREATER_THAN_UV_MAX 0x02 +#endif + +#ifndef PERL_SCAN_SILENT_ILLDIGIT +# define PERL_SCAN_SILENT_ILLDIGIT 0x04 +#endif + +#ifndef PERL_SCAN_ALLOW_UNDERSCORES +# define PERL_SCAN_ALLOW_UNDERSCORES 0x01 +#endif + +#ifndef PERL_SCAN_DISALLOW_PREFIX +# define PERL_SCAN_DISALLOW_PREFIX 0x02 +#endif + +#ifndef grok_numeric_radix +#if defined(NEED_grok_numeric_radix) +static bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); +static +#else +extern bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); +#endif + +#ifdef grok_numeric_radix +# undef grok_numeric_radix +#endif +#define grok_numeric_radix(a,b) DPPP_(my_grok_numeric_radix)(aTHX_ a,b) +#define Perl_grok_numeric_radix DPPP_(my_grok_numeric_radix) + +#if defined(NEED_grok_numeric_radix) || defined(NEED_grok_numeric_radix_GLOBAL) +bool +DPPP_(my_grok_numeric_radix)(pTHX_ const char **sp, const char *send) +{ +#ifdef USE_LOCALE_NUMERIC +#ifdef PL_numeric_radix_sv + if (PL_numeric_radix_sv && IN_LOCALE) { + STRLEN len; + char* radix = SvPV(PL_numeric_radix_sv, len); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#else + /* older perls don't have PL_numeric_radix_sv so the radix + * must manually be requested from locale.h + */ +#include + dTHR; /* needed for older threaded perls */ + struct lconv *lc = localeconv(); + char *radix = lc->decimal_point; + if (radix && IN_LOCALE) { + STRLEN len = strlen(radix); + if (*sp + len <= send && memEQ(*sp, radix, len)) { + *sp += len; + return TRUE; + } + } +#endif +#endif /* USE_LOCALE_NUMERIC */ + /* always try "." if numeric radix didn't match because + * we may have data from different locales mixed */ + if (*sp < send && **sp == '.') { + ++*sp; + return TRUE; + } + return FALSE; +} +#endif +#endif + +#ifndef grok_number +#if defined(NEED_grok_number) +static int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); +static +#else +extern int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); +#endif + +#ifdef grok_number +# undef grok_number +#endif +#define grok_number(a,b,c) DPPP_(my_grok_number)(aTHX_ a,b,c) +#define Perl_grok_number DPPP_(my_grok_number) + +#if defined(NEED_grok_number) || defined(NEED_grok_number_GLOBAL) +int +DPPP_(my_grok_number)(pTHX_ const char *pv, STRLEN len, UV *valuep) +{ + const char *s = pv; + const char *send = pv + len; + const UV max_div_10 = UV_MAX / 10; + const char max_mod_10 = UV_MAX % 10; + int numtype = 0; + int sawinf = 0; + int sawnan = 0; + + while (s < send && isSPACE(*s)) + s++; + if (s == send) { + return 0; + } else if (*s == '-') { + s++; + numtype = IS_NUMBER_NEG; + } + else if (*s == '+') + s++; + + if (s == send) + return 0; + + /* next must be digit or the radix separator or beginning of infinity */ + if (isDIGIT(*s)) { + /* UVs are at least 32 bits, so the first 9 decimal digits cannot + overflow. */ + UV value = *s - '0'; + /* This construction seems to be more optimiser friendly. + (without it gcc does the isDIGIT test and the *s - '0' separately) + With it gcc on arm is managing 6 instructions (6 cycles) per digit. + In theory the optimiser could deduce how far to unroll the loop + before checking for overflow. */ + if (++s < send) { + int digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + digit = *s - '0'; + if (digit >= 0 && digit <= 9) { + value = value * 10 + digit; + if (++s < send) { + /* Now got 9 digits, so need to check + each time for overflow. */ + digit = *s - '0'; + while (digit >= 0 && digit <= 9 + && (value < max_div_10 + || (value == max_div_10 + && digit <= max_mod_10))) { + value = value * 10 + digit; + if (++s < send) + digit = *s - '0'; + else + break; + } + if (digit >= 0 && digit <= 9 + && (s < send)) { + /* value overflowed. + skip the remaining digits, don't + worry about setting *valuep. */ + do { + s++; + } while (s < send && isDIGIT(*s)); + numtype |= + IS_NUMBER_GREATER_THAN_UV_MAX; + goto skip_value; + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + } + numtype |= IS_NUMBER_IN_UV; + if (valuep) + *valuep = value; + + skip_value: + if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT; + while (s < send && isDIGIT(*s)) /* optional digits after the radix */ + s++; + } + } + else if (GROK_NUMERIC_RADIX(&s, send)) { + numtype |= IS_NUMBER_NOT_INT | IS_NUMBER_IN_UV; /* valuep assigned below */ + /* no digits before the radix means we need digits after it */ + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + if (valuep) { + /* integer approximation is valid - it's 0. */ + *valuep = 0; + } + } + else + return 0; + } else if (*s == 'I' || *s == 'i') { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'F' && *s != 'f')) return 0; + s++; if (s < send && (*s == 'I' || *s == 'i')) { + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; if (s == send || (*s != 'I' && *s != 'i')) return 0; + s++; if (s == send || (*s != 'T' && *s != 't')) return 0; + s++; if (s == send || (*s != 'Y' && *s != 'y')) return 0; + s++; + } + sawinf = 1; + } else if (*s == 'N' || *s == 'n') { + /* XXX TODO: There are signaling NaNs and quiet NaNs. */ + s++; if (s == send || (*s != 'A' && *s != 'a')) return 0; + s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; + s++; + sawnan = 1; + } else + return 0; + + if (sawinf) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_INFINITY | IS_NUMBER_NOT_INT; + } else if (sawnan) { + numtype &= IS_NUMBER_NEG; /* Keep track of sign */ + numtype |= IS_NUMBER_NAN | IS_NUMBER_NOT_INT; + } else if (s < send) { + /* we can have an optional exponent part */ + if (*s == 'e' || *s == 'E') { + /* The only flag we keep is sign. Blow away any "it's UV" */ + numtype &= IS_NUMBER_NEG; + numtype |= IS_NUMBER_NOT_INT; + s++; + if (s < send && (*s == '-' || *s == '+')) + s++; + if (s < send && isDIGIT(*s)) { + do { + s++; + } while (s < send && isDIGIT(*s)); + } + else + return 0; + } + } + while (s < send && isSPACE(*s)) + s++; + if (s >= send) + return numtype; + if (len == 10 && memEQ(pv, "0 but true", 10)) { + if (valuep) + *valuep = 0; + return IS_NUMBER_IN_UV; + } + return 0; +} +#endif +#endif + +/* + * The grok_* routines have been modified to use warn() instead of + * Perl_warner(). Also, 'hexdigit' was the former name of PL_hexdigit, + * which is why the stack variable has been renamed to 'xdigit'. + */ + +#ifndef grok_bin +#if defined(NEED_grok_bin) +static UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_bin +# undef grok_bin +#endif +#define grok_bin(a,b,c,d) DPPP_(my_grok_bin)(aTHX_ a,b,c,d) +#define Perl_grok_bin DPPP_(my_grok_bin) + +#if defined(NEED_grok_bin) || defined(NEED_grok_bin_GLOBAL) +UV +DPPP_(my_grok_bin)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_2 = UV_MAX / 2; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + + if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { + /* strip off leading b or 0b. + for compatibility silently suffer "b" and "0b" as valid binary + numbers. */ + if (len >= 1) { + if (s[0] == 'b') { + s++; + len--; + } + else if (len >= 2 && s[0] == '0' && s[1] == 'b') { + s+=2; + len-=2; + } + } + } + + for (; len-- && *s; s++) { + char bit = *s; + if (bit == '0' || bit == '1') { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + With gcc seems to be much straighter code than old scan_bin. */ + redo: + if (!overflowed) { + if (value <= max_div_2) { + value = (value << 1) | (bit - '0'); + continue; + } + /* Bah. We're just overflowed. */ + warn("Integer overflow in binary number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 2.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount. */ + value_nv += (NV)(bit - '0'); + continue; + } + if (bit == '_' && len && allow_underscores && (bit = s[1]) + && (bit == '0' || bit == '1')) + { + --len; + ++s; + goto redo; + } + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal binary digit '%c' ignored", *s); + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Binary number > 0b11111111111111111111111111111111 non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#ifndef grok_hex +#if defined(NEED_grok_hex) +static UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_hex +# undef grok_hex +#endif +#define grok_hex(a,b,c,d) DPPP_(my_grok_hex)(aTHX_ a,b,c,d) +#define Perl_grok_hex DPPP_(my_grok_hex) + +#if defined(NEED_grok_hex) || defined(NEED_grok_hex_GLOBAL) +UV +DPPP_(my_grok_hex)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_16 = UV_MAX / 16; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + const char *xdigit; + + if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { + /* strip off leading x or 0x. + for compatibility silently suffer "x" and "0x" as valid hex numbers. + */ + if (len >= 1) { + if (s[0] == 'x') { + s++; + len--; + } + else if (len >= 2 && s[0] == '0' && s[1] == 'x') { + s+=2; + len-=2; + } + } + } + + for (; len-- && *s; s++) { + xdigit = strchr((char *) PL_hexdigit, *s); + if (xdigit) { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + With gcc seems to be much straighter code than old scan_hex. */ + redo: + if (!overflowed) { + if (value <= max_div_16) { + value = (value << 4) | ((xdigit - PL_hexdigit) & 15); + continue; + } + warn("Integer overflow in hexadecimal number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 16.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount of 16-tuples. */ + value_nv += (NV)((xdigit - PL_hexdigit) & 15); + continue; + } + if (*s == '_' && len && allow_underscores && s[1] + && (xdigit = strchr((char *) PL_hexdigit, s[1]))) + { + --len; + ++s; + goto redo; + } + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal hexadecimal digit '%c' ignored", *s); + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Hexadecimal number > 0xffffffff non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#ifndef grok_oct +#if defined(NEED_grok_oct) +static UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +static +#else +extern UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); +#endif + +#ifdef grok_oct +# undef grok_oct +#endif +#define grok_oct(a,b,c,d) DPPP_(my_grok_oct)(aTHX_ a,b,c,d) +#define Perl_grok_oct DPPP_(my_grok_oct) + +#if defined(NEED_grok_oct) || defined(NEED_grok_oct_GLOBAL) +UV +DPPP_(my_grok_oct)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) +{ + const char *s = start; + STRLEN len = *len_p; + UV value = 0; + NV value_nv = 0; + + const UV max_div_8 = UV_MAX / 8; + bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; + bool overflowed = FALSE; + + for (; len-- && *s; s++) { + /* gcc 2.95 optimiser not smart enough to figure that this subtraction + out front allows slicker code. */ + int digit = *s - '0'; + if (digit >= 0 && digit <= 7) { + /* Write it in this wonky order with a goto to attempt to get the + compiler to make the common case integer-only loop pretty tight. + */ + redo: + if (!overflowed) { + if (value <= max_div_8) { + value = (value << 3) | digit; + continue; + } + /* Bah. We're just overflowed. */ + warn("Integer overflow in octal number"); + overflowed = TRUE; + value_nv = (NV) value; + } + value_nv *= 8.0; + /* If an NV has not enough bits in its mantissa to + * represent a UV this summing of small low-order numbers + * is a waste of time (because the NV cannot preserve + * the low-order bits anyway): we could just remember when + * did we overflow and in the end just multiply value_nv by the + * right amount of 8-tuples. */ + value_nv += (NV)digit; + continue; + } + if (digit == ('_' - '0') && len && allow_underscores + && (digit = s[1] - '0') && (digit >= 0 && digit <= 7)) + { + --len; + ++s; + goto redo; + } + /* Allow \octal to work the DWIM way (that is, stop scanning + * as soon as non-octal characters are seen, complain only iff + * someone seems to want to use the digits eight and nine). */ + if (digit == 8 || digit == 9) { + if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) + warn("Illegal octal digit '%c' ignored", *s); + } + break; + } + + if ( ( overflowed && value_nv > 4294967295.0) +#if UVSIZE > 4 + || (!overflowed && value > 0xffffffff ) +#endif + ) { + warn("Octal number > 037777777777 non-portable"); + } + *len_p = s - start; + if (!overflowed) { + *flags = 0; + return value; + } + *flags = PERL_SCAN_GREATER_THAN_UV_MAX; + if (result) + *result = value_nv; + return UV_MAX; +} +#endif +#endif + +#if !defined(my_snprintf) +#if defined(NEED_my_snprintf) +static int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); +static +#else +extern int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); +#endif + +#define my_snprintf DPPP_(my_my_snprintf) +#define Perl_my_snprintf DPPP_(my_my_snprintf) + +#if defined(NEED_my_snprintf) || defined(NEED_my_snprintf_GLOBAL) + +int +DPPP_(my_my_snprintf)(char *buffer, const Size_t len, const char *format, ...) +{ + dTHX; + int retval; + va_list ap; + va_start(ap, format); +#ifdef HAS_VSNPRINTF + retval = vsnprintf(buffer, len, format, ap); +#else + retval = vsprintf(buffer, format, ap); +#endif + va_end(ap); + if (retval < 0 || (len > 0 && (Size_t)retval >= len)) + Perl_croak(aTHX_ "panic: my_snprintf buffer overflow"); + return retval; +} + +#endif +#endif + +#if !defined(my_sprintf) +#if defined(NEED_my_sprintf) +static int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); +static +#else +extern int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); +#endif + +#define my_sprintf DPPP_(my_my_sprintf) +#define Perl_my_sprintf DPPP_(my_my_sprintf) + +#if defined(NEED_my_sprintf) || defined(NEED_my_sprintf_GLOBAL) + +int +DPPP_(my_my_sprintf)(char *buffer, const char* pat, ...) +{ + va_list args; + va_start(args, pat); + vsprintf(buffer, pat, args); + va_end(args); + return strlen(buffer); +} + +#endif +#endif + +#ifdef NO_XSLOCKS +# ifdef dJMPENV +# define dXCPT dJMPENV; int rEtV = 0 +# define XCPT_TRY_START JMPENV_PUSH(rEtV); if (rEtV == 0) +# define XCPT_TRY_END JMPENV_POP; +# define XCPT_CATCH if (rEtV != 0) +# define XCPT_RETHROW JMPENV_JUMP(rEtV) +# else +# define dXCPT Sigjmp_buf oldTOP; int rEtV = 0 +# define XCPT_TRY_START Copy(top_env, oldTOP, 1, Sigjmp_buf); rEtV = Sigsetjmp(top_env, 1); if (rEtV == 0) +# define XCPT_TRY_END Copy(oldTOP, top_env, 1, Sigjmp_buf); +# define XCPT_CATCH if (rEtV != 0) +# define XCPT_RETHROW Siglongjmp(top_env, rEtV) +# endif +#endif + +#if !defined(my_strlcat) +#if defined(NEED_my_strlcat) +static Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); +static +#else +extern Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); +#endif + +#define my_strlcat DPPP_(my_my_strlcat) +#define Perl_my_strlcat DPPP_(my_my_strlcat) + +#if defined(NEED_my_strlcat) || defined(NEED_my_strlcat_GLOBAL) + +Size_t +DPPP_(my_my_strlcat)(char *dst, const char *src, Size_t size) +{ + Size_t used, length, copy; + + used = strlen(dst); + length = strlen(src); + if (size > 0 && used < size - 1) { + copy = (length >= size - used) ? size - used - 1 : length; + memcpy(dst + used, src, copy); + dst[used + copy] = '\0'; + } + return used + length; +} +#endif +#endif + +#if !defined(my_strlcpy) +#if defined(NEED_my_strlcpy) +static Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); +static +#else +extern Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); +#endif + +#define my_strlcpy DPPP_(my_my_strlcpy) +#define Perl_my_strlcpy DPPP_(my_my_strlcpy) + +#if defined(NEED_my_strlcpy) || defined(NEED_my_strlcpy_GLOBAL) + +Size_t +DPPP_(my_my_strlcpy)(char *dst, const char *src, Size_t size) +{ + Size_t length, copy; + + length = strlen(src); + if (size > 0) { + copy = (length >= size) ? size - 1 : length; + memcpy(dst, src, copy); + dst[copy] = '\0'; + } + return length; +} + +#endif +#endif +#ifndef PERL_PV_ESCAPE_QUOTE +# define PERL_PV_ESCAPE_QUOTE 0x0001 +#endif + +#ifndef PERL_PV_PRETTY_QUOTE +# define PERL_PV_PRETTY_QUOTE PERL_PV_ESCAPE_QUOTE +#endif + +#ifndef PERL_PV_PRETTY_ELLIPSES +# define PERL_PV_PRETTY_ELLIPSES 0x0002 +#endif + +#ifndef PERL_PV_PRETTY_LTGT +# define PERL_PV_PRETTY_LTGT 0x0004 +#endif + +#ifndef PERL_PV_ESCAPE_FIRSTCHAR +# define PERL_PV_ESCAPE_FIRSTCHAR 0x0008 +#endif + +#ifndef PERL_PV_ESCAPE_UNI +# define PERL_PV_ESCAPE_UNI 0x0100 +#endif + +#ifndef PERL_PV_ESCAPE_UNI_DETECT +# define PERL_PV_ESCAPE_UNI_DETECT 0x0200 +#endif + +#ifndef PERL_PV_ESCAPE_ALL +# define PERL_PV_ESCAPE_ALL 0x1000 +#endif + +#ifndef PERL_PV_ESCAPE_NOBACKSLASH +# define PERL_PV_ESCAPE_NOBACKSLASH 0x2000 +#endif + +#ifndef PERL_PV_ESCAPE_NOCLEAR +# define PERL_PV_ESCAPE_NOCLEAR 0x4000 +#endif + +#ifndef PERL_PV_ESCAPE_RE +# define PERL_PV_ESCAPE_RE 0x8000 +#endif + +#ifndef PERL_PV_PRETTY_NOCLEAR +# define PERL_PV_PRETTY_NOCLEAR PERL_PV_ESCAPE_NOCLEAR +#endif +#ifndef PERL_PV_PRETTY_DUMP +# define PERL_PV_PRETTY_DUMP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_QUOTE +#endif + +#ifndef PERL_PV_PRETTY_REGPROP +# define PERL_PV_PRETTY_REGPROP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_LTGT|PERL_PV_ESCAPE_RE +#endif + +/* Hint: pv_escape + * Note that unicode functionality is only backported to + * those perl versions that support it. For older perl + * versions, the implementation will fall back to bytes. + */ + +#ifndef pv_escape +#if defined(NEED_pv_escape) +static char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); +static +#else +extern char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); +#endif + +#ifdef pv_escape +# undef pv_escape +#endif +#define pv_escape(a,b,c,d,e,f) DPPP_(my_pv_escape)(aTHX_ a,b,c,d,e,f) +#define Perl_pv_escape DPPP_(my_pv_escape) + +#if defined(NEED_pv_escape) || defined(NEED_pv_escape_GLOBAL) + +char * +DPPP_(my_pv_escape)(pTHX_ SV *dsv, char const * const str, + const STRLEN count, const STRLEN max, + STRLEN * const escaped, const U32 flags) +{ + const char esc = flags & PERL_PV_ESCAPE_RE ? '%' : '\\'; + const char dq = flags & PERL_PV_ESCAPE_QUOTE ? '"' : esc; + char octbuf[32] = "%123456789ABCDF"; + STRLEN wrote = 0; + STRLEN chsize = 0; + STRLEN readsize = 1; +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + bool isuni = flags & PERL_PV_ESCAPE_UNI ? 1 : 0; +#endif + const char *pv = str; + const char * const end = pv + count; + octbuf[0] = esc; + + if (!(flags & PERL_PV_ESCAPE_NOCLEAR)) + sv_setpvs(dsv, ""); + +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + if ((flags & PERL_PV_ESCAPE_UNI_DETECT) && is_utf8_string((U8*)pv, count)) + isuni = 1; +#endif + + for (; pv < end && (!max || wrote < max) ; pv += readsize) { + const UV u = +#if defined(is_utf8_string) && defined(utf8_to_uvchr) + isuni ? utf8_to_uvchr((U8*)pv, &readsize) : +#endif + (U8)*pv; + const U8 c = (U8)u & 0xFF; + + if (u > 255 || (flags & PERL_PV_ESCAPE_ALL)) { + if (flags & PERL_PV_ESCAPE_FIRSTCHAR) + chsize = my_snprintf(octbuf, sizeof octbuf, + "%"UVxf, u); + else + chsize = my_snprintf(octbuf, sizeof octbuf, + "%cx{%"UVxf"}", esc, u); + } else if (flags & PERL_PV_ESCAPE_NOBACKSLASH) { + chsize = 1; + } else { + if (c == dq || c == esc || !isPRINT(c)) { + chsize = 2; + switch (c) { + case '\\' : /* fallthrough */ + case '%' : if (c == esc) + octbuf[1] = esc; + else + chsize = 1; + break; + case '\v' : octbuf[1] = 'v'; break; + case '\t' : octbuf[1] = 't'; break; + case '\r' : octbuf[1] = 'r'; break; + case '\n' : octbuf[1] = 'n'; break; + case '\f' : octbuf[1] = 'f'; break; + case '"' : if (dq == '"') + octbuf[1] = '"'; + else + chsize = 1; + break; + default: chsize = my_snprintf(octbuf, sizeof octbuf, + pv < end && isDIGIT((U8)*(pv+readsize)) + ? "%c%03o" : "%c%o", esc, c); + } + } else { + chsize = 1; + } + } + if (max && wrote + chsize > max) { + break; + } else if (chsize > 1) { + sv_catpvn(dsv, octbuf, chsize); + wrote += chsize; + } else { + char tmp[2]; + my_snprintf(tmp, sizeof tmp, "%c", c); + sv_catpvn(dsv, tmp, 1); + wrote++; + } + if (flags & PERL_PV_ESCAPE_FIRSTCHAR) + break; + } + if (escaped != NULL) + *escaped= pv - str; + return SvPVX(dsv); +} + +#endif +#endif + +#ifndef pv_pretty +#if defined(NEED_pv_pretty) +static char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); +static +#else +extern char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); +#endif + +#ifdef pv_pretty +# undef pv_pretty +#endif +#define pv_pretty(a,b,c,d,e,f,g) DPPP_(my_pv_pretty)(aTHX_ a,b,c,d,e,f,g) +#define Perl_pv_pretty DPPP_(my_pv_pretty) + +#if defined(NEED_pv_pretty) || defined(NEED_pv_pretty_GLOBAL) + +char * +DPPP_(my_pv_pretty)(pTHX_ SV *dsv, char const * const str, const STRLEN count, + const STRLEN max, char const * const start_color, char const * const end_color, + const U32 flags) +{ + const U8 dq = (flags & PERL_PV_PRETTY_QUOTE) ? '"' : '%'; + STRLEN escaped; + + if (!(flags & PERL_PV_PRETTY_NOCLEAR)) + sv_setpvs(dsv, ""); + + if (dq == '"') + sv_catpvs(dsv, "\""); + else if (flags & PERL_PV_PRETTY_LTGT) + sv_catpvs(dsv, "<"); + + if (start_color != NULL) + sv_catpv(dsv, D_PPP_CONSTPV_ARG(start_color)); + + pv_escape(dsv, str, count, max, &escaped, flags | PERL_PV_ESCAPE_NOCLEAR); + + if (end_color != NULL) + sv_catpv(dsv, D_PPP_CONSTPV_ARG(end_color)); + + if (dq == '"') + sv_catpvs(dsv, "\""); + else if (flags & PERL_PV_PRETTY_LTGT) + sv_catpvs(dsv, ">"); + + if ((flags & PERL_PV_PRETTY_ELLIPSES) && escaped < count) + sv_catpvs(dsv, "..."); + + return SvPVX(dsv); +} + +#endif +#endif + +#ifndef pv_display +#if defined(NEED_pv_display) +static char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); +static +#else +extern char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); +#endif + +#ifdef pv_display +# undef pv_display +#endif +#define pv_display(a,b,c,d,e) DPPP_(my_pv_display)(aTHX_ a,b,c,d,e) +#define Perl_pv_display DPPP_(my_pv_display) + +#if defined(NEED_pv_display) || defined(NEED_pv_display_GLOBAL) + +char * +DPPP_(my_pv_display)(pTHX_ SV *dsv, const char *pv, STRLEN cur, STRLEN len, STRLEN pvlim) +{ + pv_pretty(dsv, pv, cur, pvlim, NULL, NULL, PERL_PV_PRETTY_DUMP); + if (len > cur && pv[cur] == '\0') + sv_catpvs(dsv, "\\0"); + return SvPVX(dsv); +} + +#endif +#endif + +#endif /* _P_P_PORTABILITY_H_ */ + +/* End of File ppport.h */ diff --git a/perl/t/00_compile.t b/perl/t/00_compile.t new file mode 100644 index 0000000..66fe8f0 --- /dev/null +++ b/perl/t/00_compile.t @@ -0,0 +1,6 @@ +use strict; +use warnings; +use Test::More tests => 1; + +use_ok 'Data::MessagePack'; + diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t new file mode 100644 index 0000000..0917f04 --- /dev/null +++ b/perl/t/01_pack.t @@ -0,0 +1,61 @@ +use t::Util; +use Test::More; +use Data::MessagePack; + +sub packit { + local $_ = unpack("H*", Data::MessagePack->pack($_[0])); + s/(..)/$1 /g; + s/ $//; + $_; +} + +sub pis ($$) { + is packit($_[0]), $_[1], 'dump ' . $_[1]; +} + +my @dat = ( + 0, '00', + 1, '01', + 127, '7f', + 128, 'cc 80', + 255, 'cc ff', + 256, 'cd 01 00', + 65535, 'cd ff ff', + 65536, 'ce 00 01 00 00', + -1, 'ff', + -32, 'e0', + -33, 'd0 df', + -128, 'd0 80', + -129, 'd1 ff 7f', + -32768, 'd1 80 00', + -32769, 'd2 ff ff 7f ff', + 1.0, 'cb 3f f0 00 00 00 00 00 00', + "", 'a0', + "a", 'a1 61', + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'bf 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61', + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'da 00 20 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61', + undef, 'c0', + Data::MessagePack::true(), 'c3', + Data::MessagePack::false(), 'c2', + [], '90', + [+[]], '91 90', + [[], undef], '92 90 c0', + {'a', 0}, '81 a1 61 00', + 8388608, 'ce 00 80 00 00', + + [undef, false, true], '93 c0 c2 c3', + ["", "a", "bc", "def"], '94 a0 a1 61 a2 62 63 a3 64 65 66', + [[], [[undef]]], '92 90 91 91 c0', + [undef, false, true], '93 c0 c2 c3', + [[0, 64, 127], [-32, -16, -1]], '92 93 00 40 7f 93 e0 f0 ff', + [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '99 00 d0 80 ff 00 d1 80 00 ff 00 d2 80 00 00 00 ff', + 2147483648, 'ce 80 00 00 00', + -2147483648, 'd2 80 00 00 00', +); +@dat = (2147483648, 'ce 80 00 00 00'); +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $iunpack($v); +} + +sub pis ($$) { + is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $inew; +sub unpackit { + my $v = $_[0]; + $v =~ s/ //g; + $v = pack 'H*', $v; + $up->reset; + my $ret = $up->execute($v, 0); + if ($ret != length($v)) { + fail "extra bytes"; + } + return $up->data; +} + +sub pis ($$) { + is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2) + 1; + +isa_ok $up, 'Data::MessagePack::Unpacker'; +for (my $i=0; $iunpack( + Data::MessagePack->pack($_[0]), + ); +} + +sub pis ($) { + is_deeply invert($_[0]), $_[0], 'dump ' . $_[0]; +} + +my @dat = do 't/data.pl'; + +plan tests => 1*(scalar(@dat)/2); + +for (my $i=0; $iimport; + warnings->import; + + no strict 'refs'; + *{"$pkg\::true"} = sub () { + Data::MessagePack::true() + }; + *{"$pkg\::false"} = sub () { + Data::MessagePack::false() + }; +} + +1; diff --git a/perl/t/data.pl b/perl/t/data.pl new file mode 100644 index 0000000..1a2b2b4 --- /dev/null +++ b/perl/t/data.pl @@ -0,0 +1,15 @@ +no warnings 'uninitialized'; # i need this. i need this. +( + '93 c0 c2 c3' => [undef, false, true], + '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"], + '92 90 91 91 c0', [[], [[undef]]], + '93 c0 c2 c3', [undef, false, true], + 'ce 80 00 00 00', 2147483648, + '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + '92 93 00 40 7f 93 e0 f0 ff', [[0, 64, 127], [-32, -16, -1]], + '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', [[], [undef], [false, true], [], [undef], [false, true]], + '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', ["", "a", "ab", "", "a", "ab"], + '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], + '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}}, + '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], +) diff --git a/perl/unpack.c b/perl/unpack.c new file mode 100644 index 0000000..e5e069e --- /dev/null +++ b/perl/unpack.c @@ -0,0 +1,268 @@ +#ifdef __cplusplus +extern "C" { +#endif + +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" +#include "ppport.h" + +#ifdef __cplusplus +}; +#endif + +typedef struct { + int finished; + SV* source; +} unpack_user; + +#include "msgpack/unpack_define.h" + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object SV* + +#define msgpack_unpack_user unpack_user + +struct template_context; +typedef struct template_context msgpack_unpack_t; + +static void template_init(msgpack_unpack_t* u); + +static SV* template_data(msgpack_unpack_t* u); + +static int template_execute(msgpack_unpack_t* u, + const char* data, size_t len, size_t* off); + +static inline SV* template_callback_root(unpack_user* u) +{ return &PL_sv_undef; } + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) +{ *o = newSVuv(d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, SV** o) +{ *o = newSViv((long)d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, SV** o) +{ *o = newSViv(d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, SV** o) +{ *o = newSVnv(d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, SV** o) +{ *o = newSVnv(d); return 0; } + +static inline int template_callback_nil(unpack_user* u, SV** o) +{ *o = &PL_sv_undef; return 0; } + +static inline int template_callback_true(unpack_user* u, SV** o) +{ *o = &PL_sv_yes; return 0; } + +static inline int template_callback_false(unpack_user* u, SV** o) +{ *o = &PL_sv_no; return 0;} + +static inline int template_callback_array(unpack_user* u, unsigned int n, SV** o) +{ AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; } + +static inline int template_callback_array_item(unpack_user* u, SV** c, SV* o) +{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] + +static inline int template_callback_map(unpack_user* u, unsigned int n, SV** o) +{ HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; } + +static inline int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) +{ hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) +{ *o = (l == 0) ? newSVpv("", 0) : newSVpv(p, l); return 0; } + +#define UNPACKER(from, name) \ + msgpack_unpack_t *name; \ + name = INT2PTR(msgpack_unpack_t*, SvROK((from)) ? SvIV(SvRV((from))) : SvIV((from))); \ + if(name == NULL) { \ + Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \ + } + +#include "msgpack/unpack_template.h" + +SV* _msgpack_unpack(SV* data, int limit) { + msgpack_unpack_t mp; + unpack_user u = {0, &PL_sv_undef}; + int ret; + size_t from = 0; + STRLEN dlen; + const char * dptr = SvPV_const(data, dlen); + + template_init(&mp); + mp.user = u; + + mp.user.source = data; + ret = template_execute(&mp, dptr, (size_t)dlen, &from); + mp.user.source = &PL_sv_undef; + + if(ret < 0) { + Perl_croak(aTHX_ "parse error."); + } else if(ret == 0) { + Perl_croak(aTHX_ "insufficient bytes."); + } else { + if(from < dlen) { + Perl_croak(aTHX_ "extra bytes."); + } + return template_data(&mp); + } +} + +XS(xs_unpack_limit) { + dXSARGS; + + if (items != 3) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata', $limit)"); + } + + { + int limit = SvIV(ST(2)); + ST(0) = _msgpack_unpack(ST(1), limit); + } + XSRETURN(1); +} + + +XS(xs_unpack) { + dXSARGS; + msgpack_unpack_t mp; + + if (items != 2) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata')"); + } + + { + ST(0) = _msgpack_unpack(ST(1), sv_len(ST(1))); + } + + XSRETURN(1); +} + +/* ------------------------------ stream -- */ + +static void _reset(SV* self) { + UNPACKER(self, mp); + template_init(mp); + unpack_user u = {0, &PL_sv_undef}; + mp->user = u; +} + +XS(xs_unpacker_new) { + dXSARGS; + SV* self = sv_newmortal(); + msgpack_unpack_t *mp; + + Newx(mp, 1, msgpack_unpack_t); + + sv_setref_pv(self, "Data::MessagePack::Unpacker", mp); + _reset(self); + + ST(0) = self; + XSRETURN(1); +} + +static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { + UNPACKER(self, mp); + + size_t from = off; + const char* dptr = SvPV_nolen_const(data); + long dlen = limit; + int ret; + + if(from >= dlen) { + Perl_croak(aTHX_ "offset is bigger than data buffer size."); + } + + mp->user.source = data; + ret = template_execute(mp, dptr, (size_t)dlen, &from); + mp->user.source = &PL_sv_undef; + + if(ret < 0) { + Perl_croak(aTHX_ "parse error."); + } else if(ret > 0) { + mp->user.finished = 1; + return newSVuv(from); + } else { + mp->user.finished = 0; + return newSVuv(from); + } +} + +XS(xs_unpacker_execute) { + dXSARGS; + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + + ST(0) = _execute_impl(self, data, off, sv_len(data)); + + XSRETURN(1); +} + +XS(xs_unpacker_execute_limit) { + dXSARGS; + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + IV limit = SvIV(ST(3)); + + ST(0) = _execute_impl(self, data, off, limit); + + XSRETURN(1); +} + +XS(xs_unpacker_is_finished) { + dXSARGS; + + UNPACKER(ST(0), mp); + ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no; + + XSRETURN(1); +} + +XS(xs_unpacker_data) { + dXSARGS; + + UNPACKER(ST(0), mp); + ST(0) = template_data(mp); + + XSRETURN(1); +} + +XS(xs_unpacker_reset) { + dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->reset()"); + } + + _reset(ST(0)); + + XSRETURN(0); +} + diff --git a/perl/xt/99_pod.t b/perl/xt/99_pod.t new file mode 100644 index 0000000..437887a --- /dev/null +++ b/perl/xt/99_pod.t @@ -0,0 +1,4 @@ +use Test::More; +eval "use Test::Pod 1.00"; +plan skip_all => "Test::Pod 1.00 required for testing POD" if $@; +all_pod_files_ok(); From 5710b87b0619b4754000a86da922708de40406af Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 13:09:05 +0900 Subject: [PATCH 0113/1648] perl: enhancement portability --- perl/MessagePack.c | 1 + perl/pack.c | 1 - perl/unpack.c | 4 +++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/perl/MessagePack.c b/perl/MessagePack.c index c40e46a..3b33fee 100644 --- a/perl/MessagePack.c +++ b/perl/MessagePack.c @@ -4,6 +4,7 @@ extern "C" { #include "EXTERN.h" #include "perl.h" #include "XSUB.h" +#define NEED_newCONSTSUB #include "ppport.h" #ifdef __cplusplus }; diff --git a/perl/pack.c b/perl/pack.c index 5bb667b..e9a5023 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -27,7 +27,6 @@ extern "C" { #define _PACK_WRAPPER(t) msgpack_pack_##t #define PACK_WRAPPER(t) _PACK_WRAPPER(t) -// move to pack.c static void _msgpack_pack_sv(SV* buf, SV* val) { if (val==NULL) { msgpack_pack_nil(buf); diff --git a/perl/unpack.c b/perl/unpack.c index e5e069e..fcbd2e2 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -5,6 +5,8 @@ extern "C" { #include "EXTERN.h" #include "perl.h" #include "XSUB.h" +#define NEED_newRV_noinc +#define NEED_sv_2pv_flags #include "ppport.h" #ifdef __cplusplus @@ -87,7 +89,7 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, SV** o { AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; } static inline int template_callback_array_item(unpack_user* u, SV** c, SV* o) -{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] +{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */ static inline int template_callback_map(unpack_user* u, unsigned int n, SV** o) { HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; } From c7b6bb730362f674ac573f4efae8169544a113a2 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 13:09:05 +0900 Subject: [PATCH 0114/1648] perl: enhancement portability --- perl/MessagePack.c | 1 + perl/pack.c | 1 - perl/unpack.c | 4 +++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/perl/MessagePack.c b/perl/MessagePack.c index c40e46a..3b33fee 100644 --- a/perl/MessagePack.c +++ b/perl/MessagePack.c @@ -4,6 +4,7 @@ extern "C" { #include "EXTERN.h" #include "perl.h" #include "XSUB.h" +#define NEED_newCONSTSUB #include "ppport.h" #ifdef __cplusplus }; diff --git a/perl/pack.c b/perl/pack.c index 5bb667b..e9a5023 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -27,7 +27,6 @@ extern "C" { #define _PACK_WRAPPER(t) msgpack_pack_##t #define PACK_WRAPPER(t) _PACK_WRAPPER(t) -// move to pack.c static void _msgpack_pack_sv(SV* buf, SV* val) { if (val==NULL) { msgpack_pack_nil(buf); diff --git a/perl/unpack.c b/perl/unpack.c index e5e069e..fcbd2e2 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -5,6 +5,8 @@ extern "C" { #include "EXTERN.h" #include "perl.h" #include "XSUB.h" +#define NEED_newRV_noinc +#define NEED_sv_2pv_flags #include "ppport.h" #ifdef __cplusplus @@ -87,7 +89,7 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, SV** o { AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; } static inline int template_callback_array_item(unpack_user* u, SV** c, SV* o) -{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } // FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] +{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */ static inline int template_callback_map(unpack_user* u, unsigned int n, SV** o) { HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; } From 28e113fd006a213c56a8b01057ebf71570bee2b1 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 22:43:16 +0900 Subject: [PATCH 0115/1648] oops. remove debugging code. --- perl/t/01_pack.t | 1 - 1 file changed, 1 deletion(-) diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t index 0917f04..60c67f8 100644 --- a/perl/t/01_pack.t +++ b/perl/t/01_pack.t @@ -52,7 +52,6 @@ my @dat = ( 2147483648, 'ce 80 00 00 00', -2147483648, 'd2 80 00 00 00', ); -@dat = (2147483648, 'ce 80 00 00 00'); plan tests => 1*(scalar(@dat)/2); for (my $i=0; $i Date: Wed, 15 Apr 2009 22:43:16 +0900 Subject: [PATCH 0116/1648] oops. remove debugging code. --- perl/t/01_pack.t | 1 - 1 file changed, 1 deletion(-) diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t index 0917f04..60c67f8 100644 --- a/perl/t/01_pack.t +++ b/perl/t/01_pack.t @@ -52,7 +52,6 @@ my @dat = ( 2147483648, 'ce 80 00 00 00', -2147483648, 'd2 80 00 00 00', ); -@dat = (2147483648, 'ce 80 00 00 00'); plan tests => 1*(scalar(@dat)/2); for (my $i=0; $i Date: Wed, 15 Apr 2009 22:43:59 +0900 Subject: [PATCH 0117/1648] perl: use more efficent strategy for memory allocation.this code taken from JSON::XS. thanks to mlehmann++ --- perl/pack.c | 85 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index e9a5023..bd0c715 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -1,3 +1,7 @@ +/* + * code is written by tokuhirom. + * buffer alocation technique is taken from JSON::XS. thanks to mlehmann. + */ #ifdef __cplusplus extern "C" { #endif @@ -17,43 +21,64 @@ extern "C" { #define msgpack_pack_inline_func_cint(name) \ static inline void msgpack_pack ## name -#define msgpack_pack_user SV* +typedef struct { + char *cur; /* SvPVX (sv) + current output position */ + char *end; /* SvEND (sv) */ + SV *sv; /* result scalar */ +} enc_t; +void need(enc_t *enc, STRLEN len); -#define msgpack_pack_append_buffer(user, buf, len) \ - sv_catpvn(user, (const char*)(buf), len); +#define msgpack_pack_user enc_t* + +#define msgpack_pack_append_buffer(enc, buf, len) \ + need(enc, len); \ + memcpy(enc->cur, buf, len); \ + enc->cur += len; #include "msgpack/pack_template.h" #define _PACK_WRAPPER(t) msgpack_pack_##t #define PACK_WRAPPER(t) _PACK_WRAPPER(t) +#define INIT_SIZE 32 /* initial scalar size to be allocated */ -static void _msgpack_pack_sv(SV* buf, SV* val) { +void need(enc_t *enc, STRLEN len) +{ + if (enc->cur + len >= enc->end) { + STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); + SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); + enc->cur = SvPVX (enc->sv) + cur; + enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1; + } +} + +static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (val==NULL) { - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); return; } switch (SvTYPE(val)) { case SVt_NULL: - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); break; case SVt_IV: if (SvIOK_UV(val)) { - msgpack_pack_uint32(buf, SvUV(val)); + msgpack_pack_uint32(enc, SvUV(val)); } else { - PACK_WRAPPER(IVTYPE)(buf, SvIV(val)); + PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); } break; case SVt_PVNV: { STRLEN len = 0; + need(enc, 1); char *pv = SvPV(val, len); if (len == 1 && *pv == '1') { - msgpack_pack_true(buf); + msgpack_pack_true(enc); } else if (len == 0 && *pv==0) { - msgpack_pack_false(buf); + msgpack_pack_false(enc); } else { - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); } } break; @@ -61,25 +86,26 @@ static void _msgpack_pack_sv(SV* buf, SV* val) { { STRLEN len; char * cval = SvPV(val, len); - msgpack_pack_raw(buf, len); - msgpack_pack_raw_body(buf, cval, len); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, cval, len); } break; case SVt_NV: - PACK_WRAPPER(NVTYPE)(buf, SvNV(val)); + PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); break; case SVt_PVAV: { AV* ary = (AV*)val; int len = av_len(ary) + 1; int i; - msgpack_pack_array(buf, len); + need(enc, 1); + msgpack_pack_array(enc, len); for (i=0; i Date: Wed, 15 Apr 2009 22:43:59 +0900 Subject: [PATCH 0118/1648] perl: use more efficent strategy for memory allocation.this code taken from JSON::XS. thanks to mlehmann++ --- perl/pack.c | 85 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index e9a5023..bd0c715 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -1,3 +1,7 @@ +/* + * code is written by tokuhirom. + * buffer alocation technique is taken from JSON::XS. thanks to mlehmann. + */ #ifdef __cplusplus extern "C" { #endif @@ -17,43 +21,64 @@ extern "C" { #define msgpack_pack_inline_func_cint(name) \ static inline void msgpack_pack ## name -#define msgpack_pack_user SV* +typedef struct { + char *cur; /* SvPVX (sv) + current output position */ + char *end; /* SvEND (sv) */ + SV *sv; /* result scalar */ +} enc_t; +void need(enc_t *enc, STRLEN len); -#define msgpack_pack_append_buffer(user, buf, len) \ - sv_catpvn(user, (const char*)(buf), len); +#define msgpack_pack_user enc_t* + +#define msgpack_pack_append_buffer(enc, buf, len) \ + need(enc, len); \ + memcpy(enc->cur, buf, len); \ + enc->cur += len; #include "msgpack/pack_template.h" #define _PACK_WRAPPER(t) msgpack_pack_##t #define PACK_WRAPPER(t) _PACK_WRAPPER(t) +#define INIT_SIZE 32 /* initial scalar size to be allocated */ -static void _msgpack_pack_sv(SV* buf, SV* val) { +void need(enc_t *enc, STRLEN len) +{ + if (enc->cur + len >= enc->end) { + STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); + SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1); + enc->cur = SvPVX (enc->sv) + cur; + enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1; + } +} + +static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (val==NULL) { - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); return; } switch (SvTYPE(val)) { case SVt_NULL: - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); break; case SVt_IV: if (SvIOK_UV(val)) { - msgpack_pack_uint32(buf, SvUV(val)); + msgpack_pack_uint32(enc, SvUV(val)); } else { - PACK_WRAPPER(IVTYPE)(buf, SvIV(val)); + PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); } break; case SVt_PVNV: { STRLEN len = 0; + need(enc, 1); char *pv = SvPV(val, len); if (len == 1 && *pv == '1') { - msgpack_pack_true(buf); + msgpack_pack_true(enc); } else if (len == 0 && *pv==0) { - msgpack_pack_false(buf); + msgpack_pack_false(enc); } else { - msgpack_pack_nil(buf); + msgpack_pack_nil(enc); } } break; @@ -61,25 +86,26 @@ static void _msgpack_pack_sv(SV* buf, SV* val) { { STRLEN len; char * cval = SvPV(val, len); - msgpack_pack_raw(buf, len); - msgpack_pack_raw_body(buf, cval, len); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, cval, len); } break; case SVt_NV: - PACK_WRAPPER(NVTYPE)(buf, SvNV(val)); + PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); break; case SVt_PVAV: { AV* ary = (AV*)val; int len = av_len(ary) + 1; int i; - msgpack_pack_array(buf, len); + need(enc, 1); + msgpack_pack_array(enc, len); for (i=0; i Date: Wed, 15 Apr 2009 23:02:27 +0900 Subject: [PATCH 0119/1648] perl: renamed benchmark script --- perl/benchmark/{p1.pl => serialize.pl} | 2 ++ 1 file changed, 2 insertions(+) rename perl/benchmark/{p1.pl => serialize.pl} (73%) diff --git a/perl/benchmark/p1.pl b/perl/benchmark/serialize.pl similarity index 73% rename from perl/benchmark/p1.pl rename to perl/benchmark/serialize.pl index 257932e..626ae03 100644 --- a/perl/benchmark/p1.pl +++ b/perl/benchmark/serialize.pl @@ -7,6 +7,8 @@ use Benchmark ':all'; my $a = [0..2**24]; print "-- serialize\n"; +print "JSON::XS: $JSON::XS::VERSION\n"; +print "Data::MessagePack: $Data::MessagePack::VERSION\n"; cmpthese( -1 => { json => sub { JSON::XS::encode_json($a) }, From 0b083030b106113c4cc92d7b1d0cae4db1b8c00a Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:02:27 +0900 Subject: [PATCH 0120/1648] perl: renamed benchmark script --- perl/benchmark/{p1.pl => serialize.pl} | 2 ++ 1 file changed, 2 insertions(+) rename perl/benchmark/{p1.pl => serialize.pl} (73%) diff --git a/perl/benchmark/p1.pl b/perl/benchmark/serialize.pl similarity index 73% rename from perl/benchmark/p1.pl rename to perl/benchmark/serialize.pl index 257932e..626ae03 100644 --- a/perl/benchmark/p1.pl +++ b/perl/benchmark/serialize.pl @@ -7,6 +7,8 @@ use Benchmark ':all'; my $a = [0..2**24]; print "-- serialize\n"; +print "JSON::XS: $JSON::XS::VERSION\n"; +print "Data::MessagePack: $Data::MessagePack::VERSION\n"; cmpthese( -1 => { json => sub { JSON::XS::encode_json($a) }, From e0bd2a291182bb6cfe1971de59bcf18f7bd52b04 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:06:47 +0900 Subject: [PATCH 0121/1648] perl: added benchmark script for deserialization --- perl/benchmark/deserialize.pl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 perl/benchmark/deserialize.pl diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl new file mode 100644 index 0000000..fd21f08 --- /dev/null +++ b/perl/benchmark/deserialize.pl @@ -0,0 +1,20 @@ +use strict; +use warnings; +use Data::MessagePack; +use JSON::XS; +use Benchmark ':all'; + +my $a = [0..2**24]; +my $j = JSON::XS::encode_json($a); +my $m = Data::MessagePack->pack($a); + +print "-- deserialize\n"; +print "JSON::XS: $JSON::XS::VERSION\n"; +print "Data::MessagePack: $Data::MessagePack::VERSION\n"; +cmpthese( + -1 => { + json => sub { JSON::XS::decode_json($j) }, + mp => sub { Data::MessagePack->unpack($m) }, + } +); + From 45321baa669435e048f396b22c46efbbaf753b85 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:06:47 +0900 Subject: [PATCH 0122/1648] perl: added benchmark script for deserialization --- perl/benchmark/deserialize.pl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 perl/benchmark/deserialize.pl diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl new file mode 100644 index 0000000..fd21f08 --- /dev/null +++ b/perl/benchmark/deserialize.pl @@ -0,0 +1,20 @@ +use strict; +use warnings; +use Data::MessagePack; +use JSON::XS; +use Benchmark ':all'; + +my $a = [0..2**24]; +my $j = JSON::XS::encode_json($a); +my $m = Data::MessagePack->pack($a); + +print "-- deserialize\n"; +print "JSON::XS: $JSON::XS::VERSION\n"; +print "Data::MessagePack: $Data::MessagePack::VERSION\n"; +cmpthese( + -1 => { + json => sub { JSON::XS::decode_json($j) }, + mp => sub { Data::MessagePack->unpack($m) }, + } +); + From b140b27b9af50fe3216a7c5571cb9c957a3259ee Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:11:26 +0900 Subject: [PATCH 0123/1648] perl: added argument check --- perl/pack.c | 4 +++- perl/unpack.c | 28 ++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index bd0c715..d30735e 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -136,7 +136,9 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { XS(xs_pack) { dXSARGS; - PERL_UNUSED_VAR(items); /* TODO: check argument count */ + if (items != 2) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->pack($dat)"); + } SV* val = ST(1); diff --git a/perl/unpack.c b/perl/unpack.c index fcbd2e2..c2e32dd 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -177,6 +177,10 @@ static void _reset(SV* self) { XS(xs_unpacker_new) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: Data::MessagePack::Unpacker->new()"); + } + SV* self = sv_newmortal(); msgpack_unpack_t *mp; @@ -218,17 +222,27 @@ static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { XS(xs_unpacker_execute) { dXSARGS; - SV* self = ST(0); - SV* data = ST(1); - IV off = SvIV(ST(2)); + if (items != 3) { + Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)"); + } - ST(0) = _execute_impl(self, data, off, sv_len(data)); + { + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + + ST(0) = _execute_impl(self, data, off, sv_len(data)); + } XSRETURN(1); } XS(xs_unpacker_execute_limit) { dXSARGS; + if (items != 4) { + Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off, limit)"); + } + SV* self = ST(0); SV* data = ST(1); IV off = SvIV(ST(2)); @@ -241,6 +255,9 @@ XS(xs_unpacker_execute_limit) { XS(xs_unpacker_is_finished) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->is_finished()"); + } UNPACKER(ST(0), mp); ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no; @@ -250,6 +267,9 @@ XS(xs_unpacker_is_finished) { XS(xs_unpacker_data) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->data()"); + } UNPACKER(ST(0), mp); ST(0) = template_data(mp); From 0b3db489768ed91e42eaaccce2657abacf8972ab Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:11:26 +0900 Subject: [PATCH 0124/1648] perl: added argument check --- perl/pack.c | 4 +++- perl/unpack.c | 28 ++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index bd0c715..d30735e 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -136,7 +136,9 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { XS(xs_pack) { dXSARGS; - PERL_UNUSED_VAR(items); /* TODO: check argument count */ + if (items != 2) { + Perl_croak(aTHX_ "Usage: Data::MessagePack->pack($dat)"); + } SV* val = ST(1); diff --git a/perl/unpack.c b/perl/unpack.c index fcbd2e2..c2e32dd 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -177,6 +177,10 @@ static void _reset(SV* self) { XS(xs_unpacker_new) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: Data::MessagePack::Unpacker->new()"); + } + SV* self = sv_newmortal(); msgpack_unpack_t *mp; @@ -218,17 +222,27 @@ static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { XS(xs_unpacker_execute) { dXSARGS; - SV* self = ST(0); - SV* data = ST(1); - IV off = SvIV(ST(2)); + if (items != 3) { + Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)"); + } - ST(0) = _execute_impl(self, data, off, sv_len(data)); + { + SV* self = ST(0); + SV* data = ST(1); + IV off = SvIV(ST(2)); + + ST(0) = _execute_impl(self, data, off, sv_len(data)); + } XSRETURN(1); } XS(xs_unpacker_execute_limit) { dXSARGS; + if (items != 4) { + Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off, limit)"); + } + SV* self = ST(0); SV* data = ST(1); IV off = SvIV(ST(2)); @@ -241,6 +255,9 @@ XS(xs_unpacker_execute_limit) { XS(xs_unpacker_is_finished) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->is_finished()"); + } UNPACKER(ST(0), mp); ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no; @@ -250,6 +267,9 @@ XS(xs_unpacker_is_finished) { XS(xs_unpacker_data) { dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->data()"); + } UNPACKER(ST(0), mp); ST(0) = template_data(mp); From b0062a7f6f47b9a4340d14c6fb0a3ae0f60c8259 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 15 Apr 2009 23:14:56 +0900 Subject: [PATCH 0125/1648] perl: oops. this doens't needed. --- perl/pack.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index d30735e..9c32dc1 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -71,7 +71,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { case SVt_PVNV: { STRLEN len = 0; - need(enc, 1); char *pv = SvPV(val, len); if (len == 1 && *pv == '1') { msgpack_pack_true(enc); @@ -98,7 +97,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { AV* ary = (AV*)val; int len = av_len(ary) + 1; int i; - need(enc, 1); msgpack_pack_array(enc, len); for (i=0; i Date: Wed, 15 Apr 2009 23:14:56 +0900 Subject: [PATCH 0126/1648] perl: oops. this doens't needed. --- perl/pack.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index d30735e..9c32dc1 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -71,7 +71,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { case SVt_PVNV: { STRLEN len = 0; - need(enc, 1); char *pv = SvPV(val, len); if (len == 1 && *pv == '1') { msgpack_pack_true(enc); @@ -98,7 +97,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { AV* ary = (AV*)val; int len = av_len(ary) + 1; int i; - need(enc, 1); msgpack_pack_array(enc, len); for (i=0; i Date: Wed, 13 May 2009 17:53:27 +0900 Subject: [PATCH 0127/1648] cpp: const --- cpp/object.hpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 09ddb89..07917c2 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -80,13 +80,13 @@ struct object { type::object_type type; union_type via; - bool is_nil() { return type == type::NIL; } + bool is_nil() const { return type == type::NIL; } template - T as(); + T as() const; template - void convert(T* v); + void convert(T* v) const; object(); object(msgpack_object obj); @@ -96,7 +96,7 @@ private: struct implicit_type; public: - implicit_type convert(); + implicit_type convert() const; }; struct object_kv { @@ -201,25 +201,25 @@ inline object::operator msgpack_object() } -inline object::implicit_type object::convert() +inline object::implicit_type object::convert() const { return implicit_type(*this); } template -inline T object::as() +inline void object::convert(T* v) const +{ + *this >> *v; +} + +template +inline T object::as() const { T v; convert(&v); return v; } -template -inline void object::convert(T* v) -{ - *this >> *v; -} - // obsolete template From 8903e2dea9f7f719897ca2420fae9376982c2c3b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 13 May 2009 17:53:27 +0900 Subject: [PATCH 0128/1648] cpp: const --- cpp/object.hpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 09ddb89..07917c2 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -80,13 +80,13 @@ struct object { type::object_type type; union_type via; - bool is_nil() { return type == type::NIL; } + bool is_nil() const { return type == type::NIL; } template - T as(); + T as() const; template - void convert(T* v); + void convert(T* v) const; object(); object(msgpack_object obj); @@ -96,7 +96,7 @@ private: struct implicit_type; public: - implicit_type convert(); + implicit_type convert() const; }; struct object_kv { @@ -201,25 +201,25 @@ inline object::operator msgpack_object() } -inline object::implicit_type object::convert() +inline object::implicit_type object::convert() const { return implicit_type(*this); } template -inline T object::as() +inline void object::convert(T* v) const +{ + *this >> *v; +} + +template +inline T object::as() const { T v; convert(&v); return v; } -template -inline void object::convert(T* v) -{ - *this >> *v; -} - // obsolete template From c8bb68d4334977ddd725804e23b2def0435a8d99 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Fri, 22 May 2009 14:31:20 +0900 Subject: [PATCH 0129/1648] add pyx --- msgpack.pyx | 162 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 msgpack.pyx diff --git a/msgpack.pyx b/msgpack.pyx new file mode 100644 index 0000000..6885317 --- /dev/null +++ b/msgpack.pyx @@ -0,0 +1,162 @@ +# coding: utf-8 + + +cdef extern from "Python.h": + ctypedef char* const_char_ptr "const char*" + cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + +cdef extern from "stdlib.h": + void* malloc(int) + void free(void*) +cdef extern from "string.h": + int memcpy(char*dst, char*src, unsigned int size) + +cdef extern from "msgpack/pack.h": + ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) + + struct msgpack_packer: + void *data + msgpack_packer_write callback + + void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) + void msgpack_pack_int(msgpack_packer* pk, int d) + void msgpack_pack_nil(msgpack_packer* pk) + void msgpack_pack_true(msgpack_packer* pk) + void msgpack_pack_false(msgpack_packer* pk) + void msgpack_pack_long_long(msgpack_packer* pk, long long d) + void msgpack_pack_double(msgpack_packer* pk, double d) + void msgpack_pack_array(msgpack_packer* pk, size_t l) + void msgpack_pack_map(msgpack_packer* pk, size_t l) + void msgpack_pack_raw(msgpack_packer* pk, size_t l) + void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) + +cdef extern from "msgpack/unpack.h": + ctypedef struct msgpack_unpacker + + +cdef int BUFF_SIZE=2*1024 + +cdef class Packer: + cdef char* buff + cdef unsigned int length + cdef unsigned int allocated + cdef msgpack_packer pk + cdef object strm + + def __init__(self, strm, int size=0): + """Make packer that pack data into strm. + + strm must have `write(bytes)` method. + size specifies local buffer size. + """ + if size <= 0: + size = BUFF_SIZE + + self.strm = strm + self.buff = malloc(size) + self.allocated = size + self.length = 0 + + msgpack_packer_init(&self.pk, self, _packer_write) + + + def flush(self): + """Flash local buffer and output stream if it has 'flush()' method.""" + if self.length > 0: + self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + self.length = 0 + if hasattr(self.strm, 'flush'): + self.strm.flush() + + def pack_list(self, len): + """Start packing sequential objects. + + Example: + + packer.pack_list(2) + packer.pack('foo') + packer.pack('bar') + + This code is same as below code: + + packer.pack(['foo', 'bar']) + """ + msgpack_pack_array(&self.pk, len) + + def pack_dict(self, len): + """Start packing key-value objects. + + Example: + + packer.pack_dict(1) + packer.pack('foo') + packer.pack('bar') + + This code is same as below code: + + packer.pack({'foo', 'bar'}) + """ + msgpack_pack_map(&self.pk, len) + + def __call__(self, object o): + cdef long long intval + cdef double fval + cdef char* rawval + + if o is None: + msgpack_pack_nil(&self.pk) + elif o is True: + msgpack_pack_true(&self.pk) + elif o is False: + msgpack_pack_false(&self.pk) + elif isinstance(o, int): + intval = o + msgpack_pack_long_long(&self.pk, intval) + elif isinstance(o, float): + fval = 9 + msgpack_pack_double(&self.pk, fval) + elif isinstance(o, str): + rawval = o + msgpack_pack_raw(&self.pk, len(o)) + msgpack_pack_raw_body(&self.pk, rawval, len(o)) + elif isinstance(o, unicode): + # todo + pass + elif isinstance(o, dict): + msgpack_pack_map(&self.pk, len(o)) + for k,v in o.iteritems(): + self(k) + self(v) + elif isinstance(o, tuple): + msgpack_pack_array(&self.pk, len(o)) + for v in o: + self(v) + elif isinstance(o, list): + msgpack_pack_array(&self.pk, len(o)) + for v in o: + self(v) + elif hasattr(o, "__msgpack__"): + o.__msgpack__(self) + else: + raise TypeError, "can't serialize %r" % (o,) + +cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + if packer.length + l > packer.allocated: + if packer.length > 0: + packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + if l > 64: + packer.strm.write(PyString_FromStringAndSize(b, l)) + packer.length = 0 + else: + memcpy(packer.buff, b, l) + packer.length = l + else: + memcpy(packer.buff + packer.length, b, l) + packer.length += l + return 0 + +cdef class Unpacker: + def __init__(self): + pass + def unpack(strm): + pass From 3628ea22d4079fa0a235a938eadf7b24943e326c Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Fri, 22 May 2009 14:31:20 +0900 Subject: [PATCH 0130/1648] add pyx --- python/msgpack.pyx | 162 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 python/msgpack.pyx diff --git a/python/msgpack.pyx b/python/msgpack.pyx new file mode 100644 index 0000000..6885317 --- /dev/null +++ b/python/msgpack.pyx @@ -0,0 +1,162 @@ +# coding: utf-8 + + +cdef extern from "Python.h": + ctypedef char* const_char_ptr "const char*" + cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + +cdef extern from "stdlib.h": + void* malloc(int) + void free(void*) +cdef extern from "string.h": + int memcpy(char*dst, char*src, unsigned int size) + +cdef extern from "msgpack/pack.h": + ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) + + struct msgpack_packer: + void *data + msgpack_packer_write callback + + void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) + void msgpack_pack_int(msgpack_packer* pk, int d) + void msgpack_pack_nil(msgpack_packer* pk) + void msgpack_pack_true(msgpack_packer* pk) + void msgpack_pack_false(msgpack_packer* pk) + void msgpack_pack_long_long(msgpack_packer* pk, long long d) + void msgpack_pack_double(msgpack_packer* pk, double d) + void msgpack_pack_array(msgpack_packer* pk, size_t l) + void msgpack_pack_map(msgpack_packer* pk, size_t l) + void msgpack_pack_raw(msgpack_packer* pk, size_t l) + void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) + +cdef extern from "msgpack/unpack.h": + ctypedef struct msgpack_unpacker + + +cdef int BUFF_SIZE=2*1024 + +cdef class Packer: + cdef char* buff + cdef unsigned int length + cdef unsigned int allocated + cdef msgpack_packer pk + cdef object strm + + def __init__(self, strm, int size=0): + """Make packer that pack data into strm. + + strm must have `write(bytes)` method. + size specifies local buffer size. + """ + if size <= 0: + size = BUFF_SIZE + + self.strm = strm + self.buff = malloc(size) + self.allocated = size + self.length = 0 + + msgpack_packer_init(&self.pk, self, _packer_write) + + + def flush(self): + """Flash local buffer and output stream if it has 'flush()' method.""" + if self.length > 0: + self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + self.length = 0 + if hasattr(self.strm, 'flush'): + self.strm.flush() + + def pack_list(self, len): + """Start packing sequential objects. + + Example: + + packer.pack_list(2) + packer.pack('foo') + packer.pack('bar') + + This code is same as below code: + + packer.pack(['foo', 'bar']) + """ + msgpack_pack_array(&self.pk, len) + + def pack_dict(self, len): + """Start packing key-value objects. + + Example: + + packer.pack_dict(1) + packer.pack('foo') + packer.pack('bar') + + This code is same as below code: + + packer.pack({'foo', 'bar'}) + """ + msgpack_pack_map(&self.pk, len) + + def __call__(self, object o): + cdef long long intval + cdef double fval + cdef char* rawval + + if o is None: + msgpack_pack_nil(&self.pk) + elif o is True: + msgpack_pack_true(&self.pk) + elif o is False: + msgpack_pack_false(&self.pk) + elif isinstance(o, int): + intval = o + msgpack_pack_long_long(&self.pk, intval) + elif isinstance(o, float): + fval = 9 + msgpack_pack_double(&self.pk, fval) + elif isinstance(o, str): + rawval = o + msgpack_pack_raw(&self.pk, len(o)) + msgpack_pack_raw_body(&self.pk, rawval, len(o)) + elif isinstance(o, unicode): + # todo + pass + elif isinstance(o, dict): + msgpack_pack_map(&self.pk, len(o)) + for k,v in o.iteritems(): + self(k) + self(v) + elif isinstance(o, tuple): + msgpack_pack_array(&self.pk, len(o)) + for v in o: + self(v) + elif isinstance(o, list): + msgpack_pack_array(&self.pk, len(o)) + for v in o: + self(v) + elif hasattr(o, "__msgpack__"): + o.__msgpack__(self) + else: + raise TypeError, "can't serialize %r" % (o,) + +cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + if packer.length + l > packer.allocated: + if packer.length > 0: + packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + if l > 64: + packer.strm.write(PyString_FromStringAndSize(b, l)) + packer.length = 0 + else: + memcpy(packer.buff, b, l) + packer.length = l + else: + memcpy(packer.buff + packer.length, b, l) + packer.length += l + return 0 + +cdef class Unpacker: + def __init__(self): + pass + def unpack(strm): + pass From 861721314965cf89beaa17f3dfcdce90ccd8d089 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 3 Jun 2009 22:01:27 +0900 Subject: [PATCH 0131/1648] cpp: fix map converter --- cpp/type/map.hpp | 10 +++++----- ruby/test_case.rb | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index c136d53..552de57 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -80,13 +80,13 @@ inline std::map operator>> (object o, std::map& v) for(; p != pend; ++p) { K key; p->key.convert(&key); - typename std::map::iterator it(v.find(key)); - if(it != v.end()) { + typename std::map::iterator it(v.lower_bound(key)); + if(it != v.end() && !(key < it->first)) { + p->val.convert(&it->second); + } else { V val; p->val.convert(&val); - it->insert( std::pair(key, val) ); - } else { - p->val.convert(&it->second); + v.insert(it, std::pair(key, val)); } } return v; diff --git a/ruby/test_case.rb b/ruby/test_case.rb index 2d897df..4fbcea3 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -219,6 +219,7 @@ class MessagePackTestFormat < Test::Unit::TestCase def match(obj, buf) assert_equal(obj.to_msgpack, buf) + assert_equal(MessagePack::unpack(buf), obj) end end From b3846a411fb148c30cf99c7297b70fe8ad3bfaa9 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 3 Jun 2009 22:01:27 +0900 Subject: [PATCH 0132/1648] cpp: fix map converter --- cpp/type/map.hpp | 10 +++++----- ruby/test_case.rb | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cpp/type/map.hpp b/cpp/type/map.hpp index c136d53..552de57 100644 --- a/cpp/type/map.hpp +++ b/cpp/type/map.hpp @@ -80,13 +80,13 @@ inline std::map operator>> (object o, std::map& v) for(; p != pend; ++p) { K key; p->key.convert(&key); - typename std::map::iterator it(v.find(key)); - if(it != v.end()) { + typename std::map::iterator it(v.lower_bound(key)); + if(it != v.end() && !(key < it->first)) { + p->val.convert(&it->second); + } else { V val; p->val.convert(&val); - it->insert( std::pair(key, val) ); - } else { - p->val.convert(&it->second); + v.insert(it, std::pair(key, val)); } } return v; diff --git a/ruby/test_case.rb b/ruby/test_case.rb index 2d897df..4fbcea3 100644 --- a/ruby/test_case.rb +++ b/ruby/test_case.rb @@ -219,6 +219,7 @@ class MessagePackTestFormat < Test::Unit::TestCase def match(obj, buf) assert_equal(obj.to_msgpack, buf) + assert_equal(MessagePack::unpack(buf), obj) end end From 7cd359c1fdab21adb0fd5760741a6b1b0e4a806a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 7 Jun 2009 14:59:50 +0900 Subject: [PATCH 0133/1648] add msgpack_vrefbuffer, msgpack::vrefbuffer --- README | 2 +- c/Makefile.am | 2 + c/vrefbuffer.c | 135 +++++++++++++++++++++++++++++++++++++++++++++ c/vrefbuffer.h | 97 ++++++++++++++++++++++++++++++++ c/zone.c | 2 +- c/zone.h | 11 +++- configure.in | 2 +- cpp/Makefile.am | 1 + cpp/vrefbuffer.hpp | 85 ++++++++++++++++++++++++++++ cpp/zone.hpp.erb | 10 ++++ 10 files changed, 341 insertions(+), 6 deletions(-) create mode 100644 c/vrefbuffer.c create mode 100644 c/vrefbuffer.h create mode 100644 cpp/vrefbuffer.hpp diff --git a/README b/README index 0c1d440..2a1a3e0 100644 --- a/README +++ b/README @@ -8,7 +8,7 @@ Binary-based efficient data interchange format. MessagePack is only tested on Linux and Mac OS X, but it may run on other UNIX-like platforms. - Following programs is required to build: + Following programs are required to build: - gcc >= 4.1 with C++ support - ruby >= 1.8 (ruby is used as a preprocessor) diff --git a/c/Makefile.am b/c/Makefile.am index e7cdc10..daa8a76 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -3,11 +3,13 @@ lib_LTLIBRARIES = libmsgpackc.la libmsgpackc_la_SOURCES = \ unpack.c \ object.c \ + vrefbuffer.c \ zone.c nobase_include_HEADERS = \ msgpack.h \ msgpack/sbuffer.h \ + msgpack/vrefbuffer.h \ msgpack/pack.h \ msgpack/unpack.h \ msgpack/object.h \ diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c new file mode 100644 index 0000000..bbaf61d --- /dev/null +++ b/c/vrefbuffer.c @@ -0,0 +1,135 @@ +/* + * MessagePack for C zero-copy buffer implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/vrefbuffer.h" +#include +#include + +bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, + size_t ref_size, size_t chunk_size) +{ + if(chunk_size < sizeof(msgpack_vrefbuffer_chunk)+72) { + chunk_size = 72; + } else { + chunk_size -= sizeof(msgpack_vrefbuffer_chunk); + } + + vbuf->chunk_size = chunk_size; + vbuf->ref_size = ref_size; + + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + size_t nfirst = (sizeof(struct iovec) < 72/2) ? + 72 / sizeof(struct iovec) : 8; + + struct iovec* array = (struct iovec*)malloc( + sizeof(struct iovec) * nfirst); + if(array == NULL) { + return false; + } + + vbuf->tail = array; + vbuf->end = array + nfirst; + vbuf->array = array; + + vbuf->chunk = (msgpack_vrefbuffer_chunk*)malloc( + chunk_size + sizeof(msgpack_vrefbuffer_chunk)); + if(vbuf->chunk == NULL) { + free(array); + return false; + } + + vbuf->chunk->next = NULL; + vbuf->chunk->free = chunk_size; + + return true; +} + +void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf) +{ + msgpack_vrefbuffer_chunk* c = vbuf->chunk; + while(true) { + msgpack_vrefbuffer_chunk* n = c->next; + free(c); + if(n) { + c = n; + } else { + break; + } + } + free(vbuf->array); +} + +int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len) +{ + if(vbuf->tail == vbuf->end) { + const size_t nused = vbuf->end - vbuf->array; + const size_t nnext = nused * 2; + + struct iovec* nvec = (struct iovec*)realloc( + vbuf->array, sizeof(struct iovec)*nnext); + if(nvec == NULL) { + return -1; + } + + vbuf->array = nvec; + vbuf->end = nvec + nnext; + vbuf->tail = nvec + nused; + } + + vbuf->tail->iov_base = (char*)buf; + vbuf->tail->iov_len = len; + ++vbuf->tail; + + return 0; +} + +int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len) +{ + msgpack_vrefbuffer_chunk* chunk = vbuf->chunk; + size_t cur_size = vbuf->chunk_size; + + if(chunk->free < len) { + cur_size = (cur_size > len) ? cur_size : len; + + chunk = (msgpack_vrefbuffer_chunk*)malloc( + cur_size + sizeof(msgpack_vrefbuffer_chunk)); + if(chunk == NULL) { + return -1; + } + + chunk->free = cur_size; + chunk->next = vbuf->chunk; + vbuf->chunk = chunk; + } + + char* m = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk) + + (cur_size - chunk->free); + + memcpy(m, buf, len); + chunk->free -= len; + + if(vbuf->tail != vbuf->array && m == + (const char*)((vbuf->tail-1)->iov_base) + (vbuf->tail-1)->iov_len) { + (vbuf->tail-1)->iov_len += len; + return 0; + } else { + return msgpack_vrefbuffer_append_ref(vbuf, m, len); + } +} + diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h new file mode 100644 index 0000000..baa7c03 --- /dev/null +++ b/c/vrefbuffer.h @@ -0,0 +1,97 @@ +/* + * MessagePack for C zero-copy buffer implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_VREFBUFFER_H__ +#define MSGPACK_VREFBUFFER_H__ + +#include "msgpack/zone.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_VREFBUFFER_REF_SIZE +#define MSGPACK_VREFBUFFER_REF_SIZE 32 +#endif + +#ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE +#define MSGPACK_VREFBUFFER_CHUNK_SIZE 2048 +#endif + +typedef struct msgpack_vrefbuffer_chunk { + size_t free; + struct msgpack_vrefbuffer_chunk* next; + /* data ... */ +} msgpack_vrefbuffer_chunk; + +typedef struct msgpack_vrefbuffer { + size_t chunk_size; + size_t ref_size; + + struct iovec* tail; + struct iovec* end; + struct iovec* array; + + msgpack_vrefbuffer_chunk* chunk; +} msgpack_vrefbuffer; + + +bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, + size_t ref_size, size_t chunk_size); +void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf); + +static inline int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len); + +static inline const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref); +static inline size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref); + +int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len); + +int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len); + + +int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len) +{ + msgpack_vrefbuffer* vbuf = (msgpack_vrefbuffer*)data; + + if(len < vbuf->ref_size) { + return msgpack_vrefbuffer_append_copy(vbuf, buf, len); + } else { + return msgpack_vrefbuffer_append_ref(vbuf, buf, len); + } +} + +const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref) +{ + return vref->array; +} + +size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref) +{ + return vref->tail - vref->array; +} + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/vrefbuffer.h */ + diff --git a/c/zone.c b/c/zone.c index 1aaad9f..79e90dc 100644 --- a/c/zone.c +++ b/c/zone.c @@ -27,7 +27,7 @@ static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_s msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc( sizeof(msgpack_zone_chunk) * nfirst); - if(!array) { + if(array == NULL) { return false; } diff --git a/c/zone.h b/c/zone.h index a3dfe41..79b51f8 100644 --- a/c/zone.h +++ b/c/zone.h @@ -66,6 +66,7 @@ msgpack_zone* msgpack_zone_new(size_t chunk_size); void msgpack_zone_free(msgpack_zone* zone); static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); +static inline void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size); static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); @@ -82,10 +83,8 @@ void msgpack_zone_clear(msgpack_zone* zone); void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size); -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size) { - size = ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1); - msgpack_zone_chunk* chunk = zone->chunk_array.tail; if(chunk->free < size) { @@ -99,6 +98,12 @@ void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) return ptr; } +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + return msgpack_zone_malloc_no_align(zone, + ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1)); +} + bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, void (*func)(void* data), void* data); diff --git a/configure.in b/configure.in index 3569244..47b28cf 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.1) +AM_INIT_AUTOMAKE(msgpack, 0.3.2) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 76770f4..b9126f8 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -6,6 +6,7 @@ libmsgpack_la_SOURCES = \ nobase_include_HEADERS = \ msgpack.hpp \ msgpack/sbuffer.hpp \ + msgpack/vrefbuffer.hpp \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ diff --git a/cpp/vrefbuffer.hpp b/cpp/vrefbuffer.hpp new file mode 100644 index 0000000..549d77f --- /dev/null +++ b/cpp/vrefbuffer.hpp @@ -0,0 +1,85 @@ +// +// MessagePack for C++ zero-copy buffer implementation +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_VREFBUFFER_HPP__ +#define MSGPACK_VREFBUFFER_HPP__ + +#include "msgpack/vrefbuffer.h" +#include + +namespace msgpack { + + +class vrefbuffer : public msgpack_vrefbuffer { +public: + vrefbuffer(size_t ref_size = MSGPACK_VREFBUFFER_REF_SIZE, + size_t chunk_size = MSGPACK_VREFBUFFER_CHUNK_SIZE) + { + msgpack_vrefbuffer_init(this, ref_size, chunk_size); + } + + ~vrefbuffer() + { + msgpack_vrefbuffer_destroy(this); + } + +public: + void write(const char* buf, unsigned int len) + { + if(len < base::ref_size) { + append_copy(buf, len); + } else { + append_ref(buf, len); + } + } + + void append_ref(const char* buf, size_t len) + { + if(msgpack_vrefbuffer_append_ref(this, buf, len) < 0) { + throw std::bad_alloc(); + } + } + + void append_copy(const char* buf, size_t len) + { + if(msgpack_vrefbuffer_append_copy(this, buf, len) < 0) { + throw std::bad_alloc(); + } + } + + const struct iovec* vector() const + { + return msgpack_vrefbuffer_vec(this); + } + + size_t vector_size() const + { + return msgpack_vrefbuffer_veclen(this); + } + +private: + typedef msgpack_vrefbuffer base; + +private: + vrefbuffer(const vrefbuffer&); +}; + + +} // namespace msgpack + +#endif /* msgpack/vrefbuffer.hpp */ + diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 9e85080..8fd14a6 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -34,6 +34,7 @@ public: public: void* malloc(size_t size); + void* malloc_no_align(size_t size); void push_finalizer(void (*func)(void*), void* data); @@ -77,6 +78,15 @@ inline void* zone::malloc(size_t size) return ptr; } +inline void* zone::malloc_no_align(size_t size) +{ + void* ptr = msgpack_zone_malloc_no_align(this, size); + if(!ptr) { + throw std::bad_alloc(); + } + return ptr; +} + inline void zone::push_finalizer(void (*func)(void*), void* data) { if(!msgpack_zone_push_finalizer(this, func, data)) { From 8ed1d61529cb7d7e9577e648773fd8c91d87b9cf Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 7 Jun 2009 14:59:50 +0900 Subject: [PATCH 0134/1648] add msgpack_vrefbuffer, msgpack::vrefbuffer --- README | 2 +- c/Makefile.am | 2 + c/vrefbuffer.c | 135 +++++++++++++++++++++++++++++++++++++++++++++ c/vrefbuffer.h | 97 ++++++++++++++++++++++++++++++++ c/zone.c | 2 +- c/zone.h | 11 +++- configure.in | 2 +- cpp/Makefile.am | 1 + cpp/vrefbuffer.hpp | 85 ++++++++++++++++++++++++++++ cpp/zone.hpp.erb | 10 ++++ 10 files changed, 341 insertions(+), 6 deletions(-) create mode 100644 c/vrefbuffer.c create mode 100644 c/vrefbuffer.h create mode 100644 cpp/vrefbuffer.hpp diff --git a/README b/README index 0c1d440..2a1a3e0 100644 --- a/README +++ b/README @@ -8,7 +8,7 @@ Binary-based efficient data interchange format. MessagePack is only tested on Linux and Mac OS X, but it may run on other UNIX-like platforms. - Following programs is required to build: + Following programs are required to build: - gcc >= 4.1 with C++ support - ruby >= 1.8 (ruby is used as a preprocessor) diff --git a/c/Makefile.am b/c/Makefile.am index e7cdc10..daa8a76 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -3,11 +3,13 @@ lib_LTLIBRARIES = libmsgpackc.la libmsgpackc_la_SOURCES = \ unpack.c \ object.c \ + vrefbuffer.c \ zone.c nobase_include_HEADERS = \ msgpack.h \ msgpack/sbuffer.h \ + msgpack/vrefbuffer.h \ msgpack/pack.h \ msgpack/unpack.h \ msgpack/object.h \ diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c new file mode 100644 index 0000000..bbaf61d --- /dev/null +++ b/c/vrefbuffer.c @@ -0,0 +1,135 @@ +/* + * MessagePack for C zero-copy buffer implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/vrefbuffer.h" +#include +#include + +bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, + size_t ref_size, size_t chunk_size) +{ + if(chunk_size < sizeof(msgpack_vrefbuffer_chunk)+72) { + chunk_size = 72; + } else { + chunk_size -= sizeof(msgpack_vrefbuffer_chunk); + } + + vbuf->chunk_size = chunk_size; + vbuf->ref_size = ref_size; + + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + size_t nfirst = (sizeof(struct iovec) < 72/2) ? + 72 / sizeof(struct iovec) : 8; + + struct iovec* array = (struct iovec*)malloc( + sizeof(struct iovec) * nfirst); + if(array == NULL) { + return false; + } + + vbuf->tail = array; + vbuf->end = array + nfirst; + vbuf->array = array; + + vbuf->chunk = (msgpack_vrefbuffer_chunk*)malloc( + chunk_size + sizeof(msgpack_vrefbuffer_chunk)); + if(vbuf->chunk == NULL) { + free(array); + return false; + } + + vbuf->chunk->next = NULL; + vbuf->chunk->free = chunk_size; + + return true; +} + +void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf) +{ + msgpack_vrefbuffer_chunk* c = vbuf->chunk; + while(true) { + msgpack_vrefbuffer_chunk* n = c->next; + free(c); + if(n) { + c = n; + } else { + break; + } + } + free(vbuf->array); +} + +int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len) +{ + if(vbuf->tail == vbuf->end) { + const size_t nused = vbuf->end - vbuf->array; + const size_t nnext = nused * 2; + + struct iovec* nvec = (struct iovec*)realloc( + vbuf->array, sizeof(struct iovec)*nnext); + if(nvec == NULL) { + return -1; + } + + vbuf->array = nvec; + vbuf->end = nvec + nnext; + vbuf->tail = nvec + nused; + } + + vbuf->tail->iov_base = (char*)buf; + vbuf->tail->iov_len = len; + ++vbuf->tail; + + return 0; +} + +int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len) +{ + msgpack_vrefbuffer_chunk* chunk = vbuf->chunk; + size_t cur_size = vbuf->chunk_size; + + if(chunk->free < len) { + cur_size = (cur_size > len) ? cur_size : len; + + chunk = (msgpack_vrefbuffer_chunk*)malloc( + cur_size + sizeof(msgpack_vrefbuffer_chunk)); + if(chunk == NULL) { + return -1; + } + + chunk->free = cur_size; + chunk->next = vbuf->chunk; + vbuf->chunk = chunk; + } + + char* m = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk) + + (cur_size - chunk->free); + + memcpy(m, buf, len); + chunk->free -= len; + + if(vbuf->tail != vbuf->array && m == + (const char*)((vbuf->tail-1)->iov_base) + (vbuf->tail-1)->iov_len) { + (vbuf->tail-1)->iov_len += len; + return 0; + } else { + return msgpack_vrefbuffer_append_ref(vbuf, m, len); + } +} + diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h new file mode 100644 index 0000000..baa7c03 --- /dev/null +++ b/c/vrefbuffer.h @@ -0,0 +1,97 @@ +/* + * MessagePack for C zero-copy buffer implementation + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_VREFBUFFER_H__ +#define MSGPACK_VREFBUFFER_H__ + +#include "msgpack/zone.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_VREFBUFFER_REF_SIZE +#define MSGPACK_VREFBUFFER_REF_SIZE 32 +#endif + +#ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE +#define MSGPACK_VREFBUFFER_CHUNK_SIZE 2048 +#endif + +typedef struct msgpack_vrefbuffer_chunk { + size_t free; + struct msgpack_vrefbuffer_chunk* next; + /* data ... */ +} msgpack_vrefbuffer_chunk; + +typedef struct msgpack_vrefbuffer { + size_t chunk_size; + size_t ref_size; + + struct iovec* tail; + struct iovec* end; + struct iovec* array; + + msgpack_vrefbuffer_chunk* chunk; +} msgpack_vrefbuffer; + + +bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, + size_t ref_size, size_t chunk_size); +void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf); + +static inline int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len); + +static inline const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref); +static inline size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref); + +int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len); + +int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, + const char* buf, unsigned int len); + + +int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len) +{ + msgpack_vrefbuffer* vbuf = (msgpack_vrefbuffer*)data; + + if(len < vbuf->ref_size) { + return msgpack_vrefbuffer_append_copy(vbuf, buf, len); + } else { + return msgpack_vrefbuffer_append_ref(vbuf, buf, len); + } +} + +const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref) +{ + return vref->array; +} + +size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref) +{ + return vref->tail - vref->array; +} + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/vrefbuffer.h */ + diff --git a/c/zone.c b/c/zone.c index 1aaad9f..79e90dc 100644 --- a/c/zone.c +++ b/c/zone.c @@ -27,7 +27,7 @@ static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_s msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc( sizeof(msgpack_zone_chunk) * nfirst); - if(!array) { + if(array == NULL) { return false; } diff --git a/c/zone.h b/c/zone.h index a3dfe41..79b51f8 100644 --- a/c/zone.h +++ b/c/zone.h @@ -66,6 +66,7 @@ msgpack_zone* msgpack_zone_new(size_t chunk_size); void msgpack_zone_free(msgpack_zone* zone); static inline void* msgpack_zone_malloc(msgpack_zone* zone, size_t size); +static inline void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size); static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone, void (*func)(void* data), void* data); @@ -82,10 +83,8 @@ void msgpack_zone_clear(msgpack_zone* zone); void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size); -void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size) { - size = ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1); - msgpack_zone_chunk* chunk = zone->chunk_array.tail; if(chunk->free < size) { @@ -99,6 +98,12 @@ void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) return ptr; } +void* msgpack_zone_malloc(msgpack_zone* zone, size_t size) +{ + return msgpack_zone_malloc_no_align(zone, + ((size)+((MSGPACK_ZONE_ALIGN)-1)) & ~((MSGPACK_ZONE_ALIGN)-1)); +} + bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, void (*func)(void* data), void* data); diff --git a/configure.in b/configure.in index 3569244..47b28cf 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.1) +AM_INIT_AUTOMAKE(msgpack, 0.3.2) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 76770f4..b9126f8 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -6,6 +6,7 @@ libmsgpack_la_SOURCES = \ nobase_include_HEADERS = \ msgpack.hpp \ msgpack/sbuffer.hpp \ + msgpack/vrefbuffer.hpp \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ diff --git a/cpp/vrefbuffer.hpp b/cpp/vrefbuffer.hpp new file mode 100644 index 0000000..549d77f --- /dev/null +++ b/cpp/vrefbuffer.hpp @@ -0,0 +1,85 @@ +// +// MessagePack for C++ zero-copy buffer implementation +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_VREFBUFFER_HPP__ +#define MSGPACK_VREFBUFFER_HPP__ + +#include "msgpack/vrefbuffer.h" +#include + +namespace msgpack { + + +class vrefbuffer : public msgpack_vrefbuffer { +public: + vrefbuffer(size_t ref_size = MSGPACK_VREFBUFFER_REF_SIZE, + size_t chunk_size = MSGPACK_VREFBUFFER_CHUNK_SIZE) + { + msgpack_vrefbuffer_init(this, ref_size, chunk_size); + } + + ~vrefbuffer() + { + msgpack_vrefbuffer_destroy(this); + } + +public: + void write(const char* buf, unsigned int len) + { + if(len < base::ref_size) { + append_copy(buf, len); + } else { + append_ref(buf, len); + } + } + + void append_ref(const char* buf, size_t len) + { + if(msgpack_vrefbuffer_append_ref(this, buf, len) < 0) { + throw std::bad_alloc(); + } + } + + void append_copy(const char* buf, size_t len) + { + if(msgpack_vrefbuffer_append_copy(this, buf, len) < 0) { + throw std::bad_alloc(); + } + } + + const struct iovec* vector() const + { + return msgpack_vrefbuffer_vec(this); + } + + size_t vector_size() const + { + return msgpack_vrefbuffer_veclen(this); + } + +private: + typedef msgpack_vrefbuffer base; + +private: + vrefbuffer(const vrefbuffer&); +}; + + +} // namespace msgpack + +#endif /* msgpack/vrefbuffer.hpp */ + diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 9e85080..8fd14a6 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -34,6 +34,7 @@ public: public: void* malloc(size_t size); + void* malloc_no_align(size_t size); void push_finalizer(void (*func)(void*), void* data); @@ -77,6 +78,15 @@ inline void* zone::malloc(size_t size) return ptr; } +inline void* zone::malloc_no_align(size_t size) +{ + void* ptr = msgpack_zone_malloc_no_align(this, size); + if(!ptr) { + throw std::bad_alloc(); + } + return ptr; +} + inline void zone::push_finalizer(void (*func)(void*), void* data) { if(!msgpack_zone_push_finalizer(this, func, data)) { From 711e4817a5e35a32c4b577664d8096de00fa802b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 00:23:38 +0900 Subject: [PATCH 0135/1648] support packing long and tuple. add missing files. --- python/msgpack.pyx | 53 ++++-- python/pack.h | 130 +++++++++++++++ python/unpack.h | 397 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 563 insertions(+), 17 deletions(-) create mode 100644 python/pack.h create mode 100644 python/unpack.h diff --git a/python/msgpack.pyx b/python/msgpack.pyx index 6885317..ceae9b6 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -3,15 +3,17 @@ cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" + ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) cdef extern from "stdlib.h": void* malloc(int) void free(void*) + cdef extern from "string.h": int memcpy(char*dst, char*src, unsigned int size) -cdef extern from "msgpack/pack.h": +cdef extern from "pack.h": ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) struct msgpack_packer: @@ -30,7 +32,7 @@ cdef extern from "msgpack/pack.h": void msgpack_pack_raw(msgpack_packer* pk, size_t l) void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef extern from "msgpack/unpack.h": +cdef extern from "unpack.h": ctypedef struct msgpack_unpacker @@ -98,7 +100,7 @@ cdef class Packer: """ msgpack_pack_map(&self.pk, len) - def __call__(self, object o): + def pack(self, object o): cdef long long intval cdef double fval cdef char* rawval @@ -109,6 +111,9 @@ cdef class Packer: msgpack_pack_true(&self.pk) elif o is False: msgpack_pack_false(&self.pk) + elif isinstance(o, long): + intval = o + msgpack_pack_long_long(&self.pk, intval) elif isinstance(o, int): intval = o msgpack_pack_long_long(&self.pk, intval) @@ -120,24 +125,21 @@ cdef class Packer: msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif isinstance(o, unicode): - # todo - pass + o = o.encode('utf-8') + rawval = o + msgpack_pack_raw(&self.pk, len(o)) + msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif isinstance(o, dict): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): - self(k) - self(v) - elif isinstance(o, tuple): + self.pack(k) + self.pack(v) + elif isinstance(o, tuple) or isinstance(o, list): msgpack_pack_array(&self.pk, len(o)) for v in o: - self(v) - elif isinstance(o, list): - msgpack_pack_array(&self.pk, len(o)) - for v in o: - self(v) - elif hasattr(o, "__msgpack__"): - o.__msgpack__(self) + self.pack(v) else: + # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): @@ -155,8 +157,25 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): packer.length += l return 0 +cdef extern from "msgpack/zone.h": + ctypedef struct msgpack_zone + +cdef extern from "unpack.c": + ctypedef struct template_context: + pass + int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) + void template_init(template_context* ctx) + PyObject* template_data(template_context* ctx) + + cdef class Unpacker: def __init__(self): pass - def unpack(strm): - pass + + def unpack(self, bytes_): + cdef const_char_ptr p = bytes_ + cdef template_context ctx + cdef size_t off = 0 + template_init(&ctx) + template_execute(&ctx, p, len(bytes_), &off) + return template_data(&ctx) diff --git a/python/pack.h b/python/pack.h new file mode 100644 index 0000000..4a336d5 --- /dev/null +++ b/python/pack.h @@ -0,0 +1,130 @@ +/* + * MessagePack for Python packing routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_H__ +#define MSGPACK_PACK_H__ + +#if _MSC_VER +typedef signed char uint8_t; +typedef unsigned char uint8_t; +typedef short int16_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef long long int64_t; +typedef unsigned long long uint64_t; +#elif +#include +#endif + +#include +#include +#include "msgpack/pack_define.h" +#include "msgpack/object.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); + +typedef struct msgpack_packer { + void* data; + msgpack_packer_write callback; +} msgpack_packer; + +static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); + +static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static void msgpack_packer_free(msgpack_packer* pk); + +static int msgpack_pack_short(msgpack_packer* pk, short d); +static int msgpack_pack_int(msgpack_packer* pk, int d); +static int msgpack_pack_long(msgpack_packer* pk, long d); +static int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); + +static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); + +static int msgpack_pack_float(msgpack_packer* pk, float d); +static int msgpack_pack_double(msgpack_packer* pk, double d); + +static int msgpack_pack_nil(msgpack_packer* pk); +static int msgpack_pack_true(msgpack_packer* pk); +static int msgpack_pack_false(msgpack_packer* pk); + +static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); + +static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); + +static int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); + +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); + + + +#define msgpack_pack_inline_func(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_user msgpack_packer* + +#define msgpack_pack_append_buffer(user, buf, len) \ + return (*(user)->callback)((user)->data, (const char*)buf, len) + +#include "msgpack/pack_template.h" + +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) +{ + pk->data = data; + pk->callback = callback; +} + +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) +{ + msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); + if(!pk) { return NULL; } + msgpack_packer_init(pk, data, callback); + return pk; +} + +static inline void msgpack_packer_free(msgpack_packer* pk) +{ + free(pk); +} + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/pack.h */ + diff --git a/python/unpack.h b/python/unpack.h new file mode 100644 index 0000000..d578360 --- /dev/null +++ b/python/unpack.h @@ -0,0 +1,397 @@ +/* + * MessagePack for Python unpacking routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/unpack.h" +#include "msgpack/unpack_define.h" +#include + +#include "Python.h" + + +typedef struct { + int reserved; +} unpack_user; + + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object PyObject* + +#define msgpack_unpack_user unpack_user + + +struct template_context; +typedef struct template_context template_context; + +static void template_init(template_context* ctx); + +static msgpack_unpack_object template_data(template_context* ctx); + +static int template_execute(template_context* ctx, + const char* data, size_t len, size_t* off); + + +static inline msgpack_unpack_object template_callback_root(unpack_user* u) +{ PyObject *o = Py_None; Py_INCREF(o); return o; } + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) +{ + if (d >= 0x80000000UL) { + *o = PyLong_FromUnsignedLongLong((unsigned long long)d); + } else { + *o = PyInt_FromLong((long)d); + } + return 0; +} + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromUnsignedLongLong(d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromLongLong(d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble((double)d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble(d); return 0; } + +static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_None; Py_INCREF(o); return 0; } + +static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_True; Py_INCREF(o); return 0; } + +static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_False; Py_INCREF(o); return 0; } + +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + /* TODO: use PyList_New(n). */ + *o = PyList_New(0); + return 0; +} + +static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) +{ + PyList_Append(*c, o); + return 0; +} + +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + *o = PyDict_New(); + return 0; +} + +static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +{ + PyDict_SetItem(*c, k, v); + return 0; +} + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) +{ + *o = PyString_FromStringAndSize(p, l); + return 0; +} + +#include "msgpack/unpack_template.h" + + +#if 0 +#define CTX_CAST(m) ((template_context*)(m)) +#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced + + +static const size_t COUNTER_SIZE = sizeof(unsigned int); + +static inline void init_count(void* buffer) +{ + *(volatile unsigned int*)buffer = 1; +} + +static inline void decl_count(void* buffer) +{ + //if(--*(unsigned int*)buffer == 0) { + if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + free(buffer); + } +} + +static inline void incr_count(void* buffer) +{ + //++*(unsigned int*)buffer; + __sync_add_and_fetch((unsigned int*)buffer, 1); +} + +static inline unsigned int get_count(void* buffer) +{ + return *(volatile unsigned int*)buffer; +} + + + +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) +{ + if(initial_buffer_size < COUNTER_SIZE) { + initial_buffer_size = COUNTER_SIZE; + } + + char* buffer = (char*)malloc(initial_buffer_size); + if(buffer == NULL) { + return false; + } + + void* ctx = malloc(sizeof(template_context)); + if(ctx == NULL) { + free(buffer); + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + free(ctx); + free(buffer); + return false; + } + + mpac->buffer = buffer; + mpac->used = COUNTER_SIZE; + mpac->free = initial_buffer_size - mpac->used; + mpac->off = COUNTER_SIZE; + mpac->parsed = 0; + mpac->initial_buffer_size = initial_buffer_size; + mpac->z = z; + mpac->ctx = ctx; + + init_count(mpac->buffer); + + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = mpac->z; + CTX_CAST(mpac->ctx)->user.referenced = false; + + return true; +} + +void msgpack_unpacker_destroy(msgpack_unpacker* mpac) +{ + msgpack_zone_free(mpac->z); + free(mpac->ctx); + decl_count(mpac->buffer); +} + + +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) +{ + msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); + if(mpac == NULL) { + return NULL; + } + + if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { + free(mpac); + return NULL; + } + + return mpac; +} + +void msgpack_unpacker_free(msgpack_unpacker* mpac) +{ + msgpack_unpacker_destroy(mpac); + free(mpac); +} + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 + && !CTX_REFERENCED(mpac)) { + // rewind buffer + mpac->free += mpac->used - COUNTER_SIZE; + mpac->used = COUNTER_SIZE; + mpac->off = COUNTER_SIZE; + + if(mpac->free >= size) { + return true; + } + } + + if(mpac->off == COUNTER_SIZE) { + size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE + while(next_size < size + mpac->used) { + next_size *= 2; + } + + char* tmp = (char*)realloc(mpac->buffer, next_size); + if(tmp == NULL) { + return false; + } + + mpac->buffer = tmp; + mpac->free = next_size - mpac->used; + + } else { + size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE + size_t not_parsed = mpac->used - mpac->off; + while(next_size < size + not_parsed + COUNTER_SIZE) { + next_size *= 2; + } + + char* tmp = (char*)malloc(next_size); + if(tmp == NULL) { + return false; + } + + init_count(tmp); + + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); + + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { + free(tmp); + return false; + } + CTX_REFERENCED(mpac) = false; + } else { + decl_count(mpac->buffer); + } + + mpac->buffer = tmp; + mpac->used = not_parsed + COUNTER_SIZE; + mpac->free = next_size - mpac->used; + mpac->off = COUNTER_SIZE; + } + + return true; +} + +int msgpack_unpacker_execute(msgpack_unpacker* mpac) +{ + size_t off = mpac->off; + int ret = template_execute(CTX_CAST(mpac->ctx), + mpac->buffer, mpac->used, &mpac->off); + if(mpac->off > off) { + mpac->parsed += mpac->off - off; + } + return ret; +} + +msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) +{ + return template_data(CTX_CAST(mpac->ctx)); +} + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) +{ + if(!msgpack_unpacker_flush_zone(mpac)) { + return false; + } + + msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(r == NULL) { + return NULL; + } + + msgpack_zone* old = mpac->z; + mpac->z = r; + + return old; +} + +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) +{ + msgpack_zone_clear(mpac->z); +} + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) +{ + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { + return false; + } + CTX_REFERENCED(mpac) = false; + + incr_count(mpac->buffer); + } + + return true; +} + +void msgpack_unpacker_reset(msgpack_unpacker* mpac) +{ + template_init(CTX_CAST(mpac->ctx)); + // don't reset referenced flag + mpac->parsed = 0; +} + + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_unpack_object* result) +{ + template_context ctx; + template_init(&ctx); + + ctx.user.z = z; + ctx.user.referenced = false; + + size_t noff = 0; + if(off != NULL) { noff = *off; } + + int ret = template_execute(&ctx, data, len, &noff); + if(ret < 0) { + return MSGPACK_UNPACK_PARSE_ERROR; + } + + if(off != NULL) { *off = noff; } + + if(ret == 0) { + return MSGPACK_UNPACK_CONTINUE; + } + + *result = template_data(&ctx); + + if(noff < len) { + return MSGPACK_UNPACK_EXTRA_BYTES; + } + + return MSGPACK_UNPACK_SUCCESS; +} +#endif From 342d3ca71027b9a1f949fe03b5243557ee53dc94 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 00:23:38 +0900 Subject: [PATCH 0136/1648] support packing long and tuple. add missing files. --- msgpack.pyx | 53 ++++--- pack.h | 130 +++++++++++++++++ unpack.h | 397 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 563 insertions(+), 17 deletions(-) create mode 100644 pack.h create mode 100644 unpack.h diff --git a/msgpack.pyx b/msgpack.pyx index 6885317..ceae9b6 100644 --- a/msgpack.pyx +++ b/msgpack.pyx @@ -3,15 +3,17 @@ cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" + ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) cdef extern from "stdlib.h": void* malloc(int) void free(void*) + cdef extern from "string.h": int memcpy(char*dst, char*src, unsigned int size) -cdef extern from "msgpack/pack.h": +cdef extern from "pack.h": ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) struct msgpack_packer: @@ -30,7 +32,7 @@ cdef extern from "msgpack/pack.h": void msgpack_pack_raw(msgpack_packer* pk, size_t l) void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef extern from "msgpack/unpack.h": +cdef extern from "unpack.h": ctypedef struct msgpack_unpacker @@ -98,7 +100,7 @@ cdef class Packer: """ msgpack_pack_map(&self.pk, len) - def __call__(self, object o): + def pack(self, object o): cdef long long intval cdef double fval cdef char* rawval @@ -109,6 +111,9 @@ cdef class Packer: msgpack_pack_true(&self.pk) elif o is False: msgpack_pack_false(&self.pk) + elif isinstance(o, long): + intval = o + msgpack_pack_long_long(&self.pk, intval) elif isinstance(o, int): intval = o msgpack_pack_long_long(&self.pk, intval) @@ -120,24 +125,21 @@ cdef class Packer: msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif isinstance(o, unicode): - # todo - pass + o = o.encode('utf-8') + rawval = o + msgpack_pack_raw(&self.pk, len(o)) + msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif isinstance(o, dict): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): - self(k) - self(v) - elif isinstance(o, tuple): + self.pack(k) + self.pack(v) + elif isinstance(o, tuple) or isinstance(o, list): msgpack_pack_array(&self.pk, len(o)) for v in o: - self(v) - elif isinstance(o, list): - msgpack_pack_array(&self.pk, len(o)) - for v in o: - self(v) - elif hasattr(o, "__msgpack__"): - o.__msgpack__(self) + self.pack(v) else: + # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): @@ -155,8 +157,25 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): packer.length += l return 0 +cdef extern from "msgpack/zone.h": + ctypedef struct msgpack_zone + +cdef extern from "unpack.c": + ctypedef struct template_context: + pass + int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) + void template_init(template_context* ctx) + PyObject* template_data(template_context* ctx) + + cdef class Unpacker: def __init__(self): pass - def unpack(strm): - pass + + def unpack(self, bytes_): + cdef const_char_ptr p = bytes_ + cdef template_context ctx + cdef size_t off = 0 + template_init(&ctx) + template_execute(&ctx, p, len(bytes_), &off) + return template_data(&ctx) diff --git a/pack.h b/pack.h new file mode 100644 index 0000000..4a336d5 --- /dev/null +++ b/pack.h @@ -0,0 +1,130 @@ +/* + * MessagePack for Python packing routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_H__ +#define MSGPACK_PACK_H__ + +#if _MSC_VER +typedef signed char uint8_t; +typedef unsigned char uint8_t; +typedef short int16_t; +typedef unsigned short uint16_t; +typedef int int32_t; +typedef unsigned int uint32_t; +typedef long long int64_t; +typedef unsigned long long uint64_t; +#elif +#include +#endif + +#include +#include +#include "msgpack/pack_define.h" +#include "msgpack/object.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); + +typedef struct msgpack_packer { + void* data; + msgpack_packer_write callback; +} msgpack_packer; + +static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); + +static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static void msgpack_packer_free(msgpack_packer* pk); + +static int msgpack_pack_short(msgpack_packer* pk, short d); +static int msgpack_pack_int(msgpack_packer* pk, int d); +static int msgpack_pack_long(msgpack_packer* pk, long d); +static int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); + +static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); + +static int msgpack_pack_float(msgpack_packer* pk, float d); +static int msgpack_pack_double(msgpack_packer* pk, double d); + +static int msgpack_pack_nil(msgpack_packer* pk); +static int msgpack_pack_true(msgpack_packer* pk); +static int msgpack_pack_false(msgpack_packer* pk); + +static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); + +static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); + +static int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); + +int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); + + + +#define msgpack_pack_inline_func(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline int msgpack_pack ## name + +#define msgpack_pack_user msgpack_packer* + +#define msgpack_pack_append_buffer(user, buf, len) \ + return (*(user)->callback)((user)->data, (const char*)buf, len) + +#include "msgpack/pack_template.h" + +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) +{ + pk->data = data; + pk->callback = callback; +} + +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) +{ + msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); + if(!pk) { return NULL; } + msgpack_packer_init(pk, data, callback); + return pk; +} + +static inline void msgpack_packer_free(msgpack_packer* pk) +{ + free(pk); +} + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/pack.h */ + diff --git a/unpack.h b/unpack.h new file mode 100644 index 0000000..d578360 --- /dev/null +++ b/unpack.h @@ -0,0 +1,397 @@ +/* + * MessagePack for Python unpacking routine + * + * Copyright (C) 2009 Naoki INADA + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "msgpack/unpack.h" +#include "msgpack/unpack_define.h" +#include + +#include "Python.h" + + +typedef struct { + int reserved; +} unpack_user; + + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object PyObject* + +#define msgpack_unpack_user unpack_user + + +struct template_context; +typedef struct template_context template_context; + +static void template_init(template_context* ctx); + +static msgpack_unpack_object template_data(template_context* ctx); + +static int template_execute(template_context* ctx, + const char* data, size_t len, size_t* off); + + +static inline msgpack_unpack_object template_callback_root(unpack_user* u) +{ PyObject *o = Py_None; Py_INCREF(o); return o; } + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong((long)d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) +{ + if (d >= 0x80000000UL) { + *o = PyLong_FromUnsignedLongLong((unsigned long long)d); + } else { + *o = PyInt_FromLong((long)d); + } + return 0; +} + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromUnsignedLongLong(d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +{ *o = PyInt_FromLong(d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) +{ *o = PyLong_FromLongLong(d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble((double)d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) +{ *o = PyFloat_FromDouble(d); return 0; } + +static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_None; Py_INCREF(o); return 0; } + +static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_True; Py_INCREF(o); return 0; } + +static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) +{ *o = Py_False; Py_INCREF(o); return 0; } + +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + /* TODO: use PyList_New(n). */ + *o = PyList_New(0); + return 0; +} + +static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) +{ + PyList_Append(*c, o); + return 0; +} + +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ + *o = PyDict_New(); + return 0; +} + +static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +{ + PyDict_SetItem(*c, k, v); + return 0; +} + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) +{ + *o = PyString_FromStringAndSize(p, l); + return 0; +} + +#include "msgpack/unpack_template.h" + + +#if 0 +#define CTX_CAST(m) ((template_context*)(m)) +#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced + + +static const size_t COUNTER_SIZE = sizeof(unsigned int); + +static inline void init_count(void* buffer) +{ + *(volatile unsigned int*)buffer = 1; +} + +static inline void decl_count(void* buffer) +{ + //if(--*(unsigned int*)buffer == 0) { + if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + free(buffer); + } +} + +static inline void incr_count(void* buffer) +{ + //++*(unsigned int*)buffer; + __sync_add_and_fetch((unsigned int*)buffer, 1); +} + +static inline unsigned int get_count(void* buffer) +{ + return *(volatile unsigned int*)buffer; +} + + + +bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) +{ + if(initial_buffer_size < COUNTER_SIZE) { + initial_buffer_size = COUNTER_SIZE; + } + + char* buffer = (char*)malloc(initial_buffer_size); + if(buffer == NULL) { + return false; + } + + void* ctx = malloc(sizeof(template_context)); + if(ctx == NULL) { + free(buffer); + return false; + } + + msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(z == NULL) { + free(ctx); + free(buffer); + return false; + } + + mpac->buffer = buffer; + mpac->used = COUNTER_SIZE; + mpac->free = initial_buffer_size - mpac->used; + mpac->off = COUNTER_SIZE; + mpac->parsed = 0; + mpac->initial_buffer_size = initial_buffer_size; + mpac->z = z; + mpac->ctx = ctx; + + init_count(mpac->buffer); + + template_init(CTX_CAST(mpac->ctx)); + CTX_CAST(mpac->ctx)->user.z = mpac->z; + CTX_CAST(mpac->ctx)->user.referenced = false; + + return true; +} + +void msgpack_unpacker_destroy(msgpack_unpacker* mpac) +{ + msgpack_zone_free(mpac->z); + free(mpac->ctx); + decl_count(mpac->buffer); +} + + +msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) +{ + msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); + if(mpac == NULL) { + return NULL; + } + + if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { + free(mpac); + return NULL; + } + + return mpac; +} + +void msgpack_unpacker_free(msgpack_unpacker* mpac) +{ + msgpack_unpacker_destroy(mpac); + free(mpac); +} + +bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) +{ + if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 + && !CTX_REFERENCED(mpac)) { + // rewind buffer + mpac->free += mpac->used - COUNTER_SIZE; + mpac->used = COUNTER_SIZE; + mpac->off = COUNTER_SIZE; + + if(mpac->free >= size) { + return true; + } + } + + if(mpac->off == COUNTER_SIZE) { + size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE + while(next_size < size + mpac->used) { + next_size *= 2; + } + + char* tmp = (char*)realloc(mpac->buffer, next_size); + if(tmp == NULL) { + return false; + } + + mpac->buffer = tmp; + mpac->free = next_size - mpac->used; + + } else { + size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE + size_t not_parsed = mpac->used - mpac->off; + while(next_size < size + not_parsed + COUNTER_SIZE) { + next_size *= 2; + } + + char* tmp = (char*)malloc(next_size); + if(tmp == NULL) { + return false; + } + + init_count(tmp); + + memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); + + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { + free(tmp); + return false; + } + CTX_REFERENCED(mpac) = false; + } else { + decl_count(mpac->buffer); + } + + mpac->buffer = tmp; + mpac->used = not_parsed + COUNTER_SIZE; + mpac->free = next_size - mpac->used; + mpac->off = COUNTER_SIZE; + } + + return true; +} + +int msgpack_unpacker_execute(msgpack_unpacker* mpac) +{ + size_t off = mpac->off; + int ret = template_execute(CTX_CAST(mpac->ctx), + mpac->buffer, mpac->used, &mpac->off); + if(mpac->off > off) { + mpac->parsed += mpac->off - off; + } + return ret; +} + +msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) +{ + return template_data(CTX_CAST(mpac->ctx)); +} + +msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) +{ + if(!msgpack_unpacker_flush_zone(mpac)) { + return false; + } + + msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if(r == NULL) { + return NULL; + } + + msgpack_zone* old = mpac->z; + mpac->z = r; + + return old; +} + +void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) +{ + msgpack_zone_clear(mpac->z); +} + +bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) +{ + if(CTX_REFERENCED(mpac)) { + if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { + return false; + } + CTX_REFERENCED(mpac) = false; + + incr_count(mpac->buffer); + } + + return true; +} + +void msgpack_unpacker_reset(msgpack_unpacker* mpac) +{ + template_init(CTX_CAST(mpac->ctx)); + // don't reset referenced flag + mpac->parsed = 0; +} + + +msgpack_unpack_return +msgpack_unpack(const char* data, size_t len, size_t* off, + msgpack_zone* z, msgpack_unpack_object* result) +{ + template_context ctx; + template_init(&ctx); + + ctx.user.z = z; + ctx.user.referenced = false; + + size_t noff = 0; + if(off != NULL) { noff = *off; } + + int ret = template_execute(&ctx, data, len, &noff); + if(ret < 0) { + return MSGPACK_UNPACK_PARSE_ERROR; + } + + if(off != NULL) { *off = noff; } + + if(ret == 0) { + return MSGPACK_UNPACK_CONTINUE; + } + + *result = template_data(&ctx); + + if(noff < len) { + return MSGPACK_UNPACK_EXTRA_BYTES; + } + + return MSGPACK_UNPACK_SUCCESS; +} +#endif From d8a3bc920caa06f21adcfc288c4c597bb9d33157 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 01:30:43 +0900 Subject: [PATCH 0137/1648] refactoring --- python/msgpack.pyx | 58 ++++++---- python/pack.h | 67 +++++------ python/unpack.h | 273 +-------------------------------------------- 3 files changed, 68 insertions(+), 330 deletions(-) diff --git a/python/msgpack.pyx b/python/msgpack.pyx index ceae9b6..c454c5d 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -1,5 +1,6 @@ # coding: utf-8 +from cStringIO import StringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" @@ -32,13 +33,15 @@ cdef extern from "pack.h": void msgpack_pack_raw(msgpack_packer* pk, size_t l) void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef extern from "unpack.h": - ctypedef struct msgpack_unpacker - cdef int BUFF_SIZE=2*1024 cdef class Packer: + """Packer that pack data into strm. + + strm must have `write(bytes)` method. + size specifies local buffer size. + """ cdef char* buff cdef unsigned int length cdef unsigned int allocated @@ -46,11 +49,6 @@ cdef class Packer: cdef object strm def __init__(self, strm, int size=0): - """Make packer that pack data into strm. - - strm must have `write(bytes)` method. - size specifies local buffer size. - """ if size <= 0: size = BUFF_SIZE @@ -157,25 +155,41 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): packer.length += l return 0 -cdef extern from "msgpack/zone.h": - ctypedef struct msgpack_zone +def pack(object o, object stream): + packer = Packer(stream) + packer.pack(o) + packer.flush() -cdef extern from "unpack.c": +def packs(object o): + buf = StringIO() + packer = Packer(buf) + packer.pack(o) + packer.flush() + return buf.getvalue() + +cdef extern from "unpack.h": ctypedef struct template_context: pass - int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) + int template_execute(template_context* ctx, const_char_ptr data, + size_t len, size_t* off) void template_init(template_context* ctx) PyObject* template_data(template_context* ctx) -cdef class Unpacker: - def __init__(self): - pass +def unpacks(object packed_bytes): + """Unpack packed_bytes to object. Returns unpacked object.""" + cdef const_char_ptr p = packed_bytes + cdef template_context ctx + cdef size_t off = 0 + template_init(&ctx) + template_execute(&ctx, p, len(packed_bytes), &off) + return template_data(&ctx) - def unpack(self, bytes_): - cdef const_char_ptr p = bytes_ - cdef template_context ctx - cdef size_t off = 0 - template_init(&ctx) - template_execute(&ctx, p, len(bytes_), &off) - return template_data(&ctx) +def unpack(object stream): + """unpack from stream.""" + packed = stream.read() + return unpacks(packed) + +cdef class Unpacker: + """Do nothing. This function is for symmetric to Packer""" + unpack = staticmethod(unpacks) diff --git a/python/pack.h b/python/pack.h index 4a336d5..f3935fb 100644 --- a/python/pack.h +++ b/python/pack.h @@ -15,9 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MSGPACK_PACK_H__ -#define MSGPACK_PACK_H__ - #if _MSC_VER typedef signed char uint8_t; typedef unsigned char uint8_t; @@ -27,14 +24,13 @@ typedef int int32_t; typedef unsigned int uint32_t; typedef long long int64_t; typedef unsigned long long uint64_t; -#elif +#else #include #endif #include #include #include "msgpack/pack_define.h" -#include "msgpack/object.h" #ifdef __cplusplus extern "C" { @@ -48,44 +44,42 @@ typedef struct msgpack_packer { msgpack_packer_write callback; } msgpack_packer; -static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); -static void msgpack_packer_free(msgpack_packer* pk); +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static inline void msgpack_packer_free(msgpack_packer* pk); -static int msgpack_pack_short(msgpack_packer* pk, short d); -static int msgpack_pack_int(msgpack_packer* pk, int d); -static int msgpack_pack_long(msgpack_packer* pk, long d); -static int msgpack_pack_long_long(msgpack_packer* pk, long long d); -static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); -static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); -static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); -static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); +static inline int msgpack_pack_short(msgpack_packer* pk, short d); +static inline int msgpack_pack_int(msgpack_packer* pk, int d); +static inline int msgpack_pack_long(msgpack_packer* pk, long d); +static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); -static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); -static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); -static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); -static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); -static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); -static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); -static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); -static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); +static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d); -static int msgpack_pack_float(msgpack_packer* pk, float d); -static int msgpack_pack_double(msgpack_packer* pk, double d); +static inline int msgpack_pack_float(msgpack_packer* pk, float d); +static inline int msgpack_pack_double(msgpack_packer* pk, double d); -static int msgpack_pack_nil(msgpack_packer* pk); -static int msgpack_pack_true(msgpack_packer* pk); -static int msgpack_pack_false(msgpack_packer* pk); +static inline int msgpack_pack_nil(msgpack_packer* pk); +static inline int msgpack_pack_true(msgpack_packer* pk); +static inline int msgpack_pack_false(msgpack_packer* pk); -static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); +static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); +static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_raw(msgpack_packer* pk, size_t l); -static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); - -int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); +static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); @@ -125,6 +119,3 @@ static inline void msgpack_packer_free(msgpack_packer* pk) #ifdef __cplusplus } #endif - -#endif /* msgpack/pack.h */ - diff --git a/python/unpack.h b/python/unpack.h index d578360..eb72001 100644 --- a/python/unpack.h +++ b/python/unpack.h @@ -15,12 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "msgpack/unpack.h" #include "msgpack/unpack_define.h" -#include - -#include "Python.h" - typedef struct { int reserved; @@ -44,11 +39,11 @@ typedef struct { struct template_context; typedef struct template_context template_context; -static void template_init(template_context* ctx); +static inline void template_init(template_context* ctx); -static msgpack_unpack_object template_data(template_context* ctx); +static inline msgpack_unpack_object template_data(template_context* ctx); -static int template_execute(template_context* ctx, +static inline int template_execute(template_context* ctx, const char* data, size_t len, size_t* off); @@ -133,265 +128,3 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha } #include "msgpack/unpack_template.h" - - -#if 0 -#define CTX_CAST(m) ((template_context*)(m)) -#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced - - -static const size_t COUNTER_SIZE = sizeof(unsigned int); - -static inline void init_count(void* buffer) -{ - *(volatile unsigned int*)buffer = 1; -} - -static inline void decl_count(void* buffer) -{ - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { - free(buffer); - } -} - -static inline void incr_count(void* buffer) -{ - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); -} - -static inline unsigned int get_count(void* buffer) -{ - return *(volatile unsigned int*)buffer; -} - - - -bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) -{ - if(initial_buffer_size < COUNTER_SIZE) { - initial_buffer_size = COUNTER_SIZE; - } - - char* buffer = (char*)malloc(initial_buffer_size); - if(buffer == NULL) { - return false; - } - - void* ctx = malloc(sizeof(template_context)); - if(ctx == NULL) { - free(buffer); - return false; - } - - msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(z == NULL) { - free(ctx); - free(buffer); - return false; - } - - mpac->buffer = buffer; - mpac->used = COUNTER_SIZE; - mpac->free = initial_buffer_size - mpac->used; - mpac->off = COUNTER_SIZE; - mpac->parsed = 0; - mpac->initial_buffer_size = initial_buffer_size; - mpac->z = z; - mpac->ctx = ctx; - - init_count(mpac->buffer); - - template_init(CTX_CAST(mpac->ctx)); - CTX_CAST(mpac->ctx)->user.z = mpac->z; - CTX_CAST(mpac->ctx)->user.referenced = false; - - return true; -} - -void msgpack_unpacker_destroy(msgpack_unpacker* mpac) -{ - msgpack_zone_free(mpac->z); - free(mpac->ctx); - decl_count(mpac->buffer); -} - - -msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) -{ - msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); - if(mpac == NULL) { - return NULL; - } - - if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { - free(mpac); - return NULL; - } - - return mpac; -} - -void msgpack_unpacker_free(msgpack_unpacker* mpac) -{ - msgpack_unpacker_destroy(mpac); - free(mpac); -} - -bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) -{ - if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 - && !CTX_REFERENCED(mpac)) { - // rewind buffer - mpac->free += mpac->used - COUNTER_SIZE; - mpac->used = COUNTER_SIZE; - mpac->off = COUNTER_SIZE; - - if(mpac->free >= size) { - return true; - } - } - - if(mpac->off == COUNTER_SIZE) { - size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE - while(next_size < size + mpac->used) { - next_size *= 2; - } - - char* tmp = (char*)realloc(mpac->buffer, next_size); - if(tmp == NULL) { - return false; - } - - mpac->buffer = tmp; - mpac->free = next_size - mpac->used; - - } else { - size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE - size_t not_parsed = mpac->used - mpac->off; - while(next_size < size + not_parsed + COUNTER_SIZE) { - next_size *= 2; - } - - char* tmp = (char*)malloc(next_size); - if(tmp == NULL) { - return false; - } - - init_count(tmp); - - memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - - if(CTX_REFERENCED(mpac)) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { - free(tmp); - return false; - } - CTX_REFERENCED(mpac) = false; - } else { - decl_count(mpac->buffer); - } - - mpac->buffer = tmp; - mpac->used = not_parsed + COUNTER_SIZE; - mpac->free = next_size - mpac->used; - mpac->off = COUNTER_SIZE; - } - - return true; -} - -int msgpack_unpacker_execute(msgpack_unpacker* mpac) -{ - size_t off = mpac->off; - int ret = template_execute(CTX_CAST(mpac->ctx), - mpac->buffer, mpac->used, &mpac->off); - if(mpac->off > off) { - mpac->parsed += mpac->off - off; - } - return ret; -} - -msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) -{ - return template_data(CTX_CAST(mpac->ctx)); -} - -msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) -{ - if(!msgpack_unpacker_flush_zone(mpac)) { - return false; - } - - msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(r == NULL) { - return NULL; - } - - msgpack_zone* old = mpac->z; - mpac->z = r; - - return old; -} - -void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) -{ - msgpack_zone_clear(mpac->z); -} - -bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) -{ - if(CTX_REFERENCED(mpac)) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { - return false; - } - CTX_REFERENCED(mpac) = false; - - incr_count(mpac->buffer); - } - - return true; -} - -void msgpack_unpacker_reset(msgpack_unpacker* mpac) -{ - template_init(CTX_CAST(mpac->ctx)); - // don't reset referenced flag - mpac->parsed = 0; -} - - -msgpack_unpack_return -msgpack_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_unpack_object* result) -{ - template_context ctx; - template_init(&ctx); - - ctx.user.z = z; - ctx.user.referenced = false; - - size_t noff = 0; - if(off != NULL) { noff = *off; } - - int ret = template_execute(&ctx, data, len, &noff); - if(ret < 0) { - return MSGPACK_UNPACK_PARSE_ERROR; - } - - if(off != NULL) { *off = noff; } - - if(ret == 0) { - return MSGPACK_UNPACK_CONTINUE; - } - - *result = template_data(&ctx); - - if(noff < len) { - return MSGPACK_UNPACK_EXTRA_BYTES; - } - - return MSGPACK_UNPACK_SUCCESS; -} -#endif From 5425b72247ca4f0de9e6780fb4405acaef5ae986 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 01:30:43 +0900 Subject: [PATCH 0138/1648] refactoring --- msgpack.pyx | 58 ++++++----- pack.h | 67 ++++++------- unpack.h | 273 +--------------------------------------------------- 3 files changed, 68 insertions(+), 330 deletions(-) diff --git a/msgpack.pyx b/msgpack.pyx index ceae9b6..c454c5d 100644 --- a/msgpack.pyx +++ b/msgpack.pyx @@ -1,5 +1,6 @@ # coding: utf-8 +from cStringIO import StringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" @@ -32,13 +33,15 @@ cdef extern from "pack.h": void msgpack_pack_raw(msgpack_packer* pk, size_t l) void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef extern from "unpack.h": - ctypedef struct msgpack_unpacker - cdef int BUFF_SIZE=2*1024 cdef class Packer: + """Packer that pack data into strm. + + strm must have `write(bytes)` method. + size specifies local buffer size. + """ cdef char* buff cdef unsigned int length cdef unsigned int allocated @@ -46,11 +49,6 @@ cdef class Packer: cdef object strm def __init__(self, strm, int size=0): - """Make packer that pack data into strm. - - strm must have `write(bytes)` method. - size specifies local buffer size. - """ if size <= 0: size = BUFF_SIZE @@ -157,25 +155,41 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): packer.length += l return 0 -cdef extern from "msgpack/zone.h": - ctypedef struct msgpack_zone +def pack(object o, object stream): + packer = Packer(stream) + packer.pack(o) + packer.flush() -cdef extern from "unpack.c": +def packs(object o): + buf = StringIO() + packer = Packer(buf) + packer.pack(o) + packer.flush() + return buf.getvalue() + +cdef extern from "unpack.h": ctypedef struct template_context: pass - int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) + int template_execute(template_context* ctx, const_char_ptr data, + size_t len, size_t* off) void template_init(template_context* ctx) PyObject* template_data(template_context* ctx) -cdef class Unpacker: - def __init__(self): - pass +def unpacks(object packed_bytes): + """Unpack packed_bytes to object. Returns unpacked object.""" + cdef const_char_ptr p = packed_bytes + cdef template_context ctx + cdef size_t off = 0 + template_init(&ctx) + template_execute(&ctx, p, len(packed_bytes), &off) + return template_data(&ctx) - def unpack(self, bytes_): - cdef const_char_ptr p = bytes_ - cdef template_context ctx - cdef size_t off = 0 - template_init(&ctx) - template_execute(&ctx, p, len(bytes_), &off) - return template_data(&ctx) +def unpack(object stream): + """unpack from stream.""" + packed = stream.read() + return unpacks(packed) + +cdef class Unpacker: + """Do nothing. This function is for symmetric to Packer""" + unpack = staticmethod(unpacks) diff --git a/pack.h b/pack.h index 4a336d5..f3935fb 100644 --- a/pack.h +++ b/pack.h @@ -15,9 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef MSGPACK_PACK_H__ -#define MSGPACK_PACK_H__ - #if _MSC_VER typedef signed char uint8_t; typedef unsigned char uint8_t; @@ -27,14 +24,13 @@ typedef int int32_t; typedef unsigned int uint32_t; typedef long long int64_t; typedef unsigned long long uint64_t; -#elif +#else #include #endif #include #include #include "msgpack/pack_define.h" -#include "msgpack/object.h" #ifdef __cplusplus extern "C" { @@ -48,44 +44,42 @@ typedef struct msgpack_packer { msgpack_packer_write callback; } msgpack_packer; -static void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); +static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); -static void msgpack_packer_free(msgpack_packer* pk); +static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); +static inline void msgpack_packer_free(msgpack_packer* pk); -static int msgpack_pack_short(msgpack_packer* pk, short d); -static int msgpack_pack_int(msgpack_packer* pk, int d); -static int msgpack_pack_long(msgpack_packer* pk, long d); -static int msgpack_pack_long_long(msgpack_packer* pk, long long d); -static int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); -static int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); -static int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); -static int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); +static inline int msgpack_pack_short(msgpack_packer* pk, short d); +static inline int msgpack_pack_int(msgpack_packer* pk, int d); +static inline int msgpack_pack_long(msgpack_packer* pk, long d); +static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d); +static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d); +static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d); +static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d); +static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d); -static int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); -static int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); -static int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); -static int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); -static int msgpack_pack_int8(msgpack_packer* pk, int8_t d); -static int msgpack_pack_int16(msgpack_packer* pk, int16_t d); -static int msgpack_pack_int32(msgpack_packer* pk, int32_t d); -static int msgpack_pack_int64(msgpack_packer* pk, int64_t d); +static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d); +static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d); +static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d); +static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d); +static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d); +static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d); +static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d); +static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d); -static int msgpack_pack_float(msgpack_packer* pk, float d); -static int msgpack_pack_double(msgpack_packer* pk, double d); +static inline int msgpack_pack_float(msgpack_packer* pk, float d); +static inline int msgpack_pack_double(msgpack_packer* pk, double d); -static int msgpack_pack_nil(msgpack_packer* pk); -static int msgpack_pack_true(msgpack_packer* pk); -static int msgpack_pack_false(msgpack_packer* pk); +static inline int msgpack_pack_nil(msgpack_packer* pk); +static inline int msgpack_pack_true(msgpack_packer* pk); +static inline int msgpack_pack_false(msgpack_packer* pk); -static int msgpack_pack_array(msgpack_packer* pk, unsigned int n); +static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_map(msgpack_packer* pk, unsigned int n); +static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); -static int msgpack_pack_raw(msgpack_packer* pk, size_t l); -static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); - -int msgpack_pack_object(msgpack_packer* pk, msgpack_object d); +static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); +static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); @@ -125,6 +119,3 @@ static inline void msgpack_packer_free(msgpack_packer* pk) #ifdef __cplusplus } #endif - -#endif /* msgpack/pack.h */ - diff --git a/unpack.h b/unpack.h index d578360..eb72001 100644 --- a/unpack.h +++ b/unpack.h @@ -15,12 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "msgpack/unpack.h" #include "msgpack/unpack_define.h" -#include - -#include "Python.h" - typedef struct { int reserved; @@ -44,11 +39,11 @@ typedef struct { struct template_context; typedef struct template_context template_context; -static void template_init(template_context* ctx); +static inline void template_init(template_context* ctx); -static msgpack_unpack_object template_data(template_context* ctx); +static inline msgpack_unpack_object template_data(template_context* ctx); -static int template_execute(template_context* ctx, +static inline int template_execute(template_context* ctx, const char* data, size_t len, size_t* off); @@ -133,265 +128,3 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha } #include "msgpack/unpack_template.h" - - -#if 0 -#define CTX_CAST(m) ((template_context*)(m)) -#define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced - - -static const size_t COUNTER_SIZE = sizeof(unsigned int); - -static inline void init_count(void* buffer) -{ - *(volatile unsigned int*)buffer = 1; -} - -static inline void decl_count(void* buffer) -{ - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { - free(buffer); - } -} - -static inline void incr_count(void* buffer) -{ - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); -} - -static inline unsigned int get_count(void* buffer) -{ - return *(volatile unsigned int*)buffer; -} - - - -bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) -{ - if(initial_buffer_size < COUNTER_SIZE) { - initial_buffer_size = COUNTER_SIZE; - } - - char* buffer = (char*)malloc(initial_buffer_size); - if(buffer == NULL) { - return false; - } - - void* ctx = malloc(sizeof(template_context)); - if(ctx == NULL) { - free(buffer); - return false; - } - - msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(z == NULL) { - free(ctx); - free(buffer); - return false; - } - - mpac->buffer = buffer; - mpac->used = COUNTER_SIZE; - mpac->free = initial_buffer_size - mpac->used; - mpac->off = COUNTER_SIZE; - mpac->parsed = 0; - mpac->initial_buffer_size = initial_buffer_size; - mpac->z = z; - mpac->ctx = ctx; - - init_count(mpac->buffer); - - template_init(CTX_CAST(mpac->ctx)); - CTX_CAST(mpac->ctx)->user.z = mpac->z; - CTX_CAST(mpac->ctx)->user.referenced = false; - - return true; -} - -void msgpack_unpacker_destroy(msgpack_unpacker* mpac) -{ - msgpack_zone_free(mpac->z); - free(mpac->ctx); - decl_count(mpac->buffer); -} - - -msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size) -{ - msgpack_unpacker* mpac = (msgpack_unpacker*)malloc(sizeof(msgpack_unpacker)); - if(mpac == NULL) { - return NULL; - } - - if(!msgpack_unpacker_init(mpac, initial_buffer_size)) { - free(mpac); - return NULL; - } - - return mpac; -} - -void msgpack_unpacker_free(msgpack_unpacker* mpac) -{ - msgpack_unpacker_destroy(mpac); - free(mpac); -} - -bool msgpack_unpacker_expand_buffer(msgpack_unpacker* mpac, size_t size) -{ - if(mpac->used == mpac->off && get_count(mpac->buffer) == 1 - && !CTX_REFERENCED(mpac)) { - // rewind buffer - mpac->free += mpac->used - COUNTER_SIZE; - mpac->used = COUNTER_SIZE; - mpac->off = COUNTER_SIZE; - - if(mpac->free >= size) { - return true; - } - } - - if(mpac->off == COUNTER_SIZE) { - size_t next_size = (mpac->used + mpac->free) * 2; // include COUNTER_SIZE - while(next_size < size + mpac->used) { - next_size *= 2; - } - - char* tmp = (char*)realloc(mpac->buffer, next_size); - if(tmp == NULL) { - return false; - } - - mpac->buffer = tmp; - mpac->free = next_size - mpac->used; - - } else { - size_t next_size = mpac->initial_buffer_size; // include COUNTER_SIZE - size_t not_parsed = mpac->used - mpac->off; - while(next_size < size + not_parsed + COUNTER_SIZE) { - next_size *= 2; - } - - char* tmp = (char*)malloc(next_size); - if(tmp == NULL) { - return false; - } - - init_count(tmp); - - memcpy(tmp+COUNTER_SIZE, mpac->buffer+mpac->off, not_parsed); - - if(CTX_REFERENCED(mpac)) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { - free(tmp); - return false; - } - CTX_REFERENCED(mpac) = false; - } else { - decl_count(mpac->buffer); - } - - mpac->buffer = tmp; - mpac->used = not_parsed + COUNTER_SIZE; - mpac->free = next_size - mpac->used; - mpac->off = COUNTER_SIZE; - } - - return true; -} - -int msgpack_unpacker_execute(msgpack_unpacker* mpac) -{ - size_t off = mpac->off; - int ret = template_execute(CTX_CAST(mpac->ctx), - mpac->buffer, mpac->used, &mpac->off); - if(mpac->off > off) { - mpac->parsed += mpac->off - off; - } - return ret; -} - -msgpack_unpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) -{ - return template_data(CTX_CAST(mpac->ctx)); -} - -msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) -{ - if(!msgpack_unpacker_flush_zone(mpac)) { - return false; - } - - msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); - if(r == NULL) { - return NULL; - } - - msgpack_zone* old = mpac->z; - mpac->z = r; - - return old; -} - -void msgpack_unpacker_reset_zone(msgpack_unpacker* mpac) -{ - msgpack_zone_clear(mpac->z); -} - -bool msgpack_unpacker_flush_zone(msgpack_unpacker* mpac) -{ - if(CTX_REFERENCED(mpac)) { - if(!msgpack_zone_push_finalizer(mpac->z, decl_count, mpac->buffer)) { - return false; - } - CTX_REFERENCED(mpac) = false; - - incr_count(mpac->buffer); - } - - return true; -} - -void msgpack_unpacker_reset(msgpack_unpacker* mpac) -{ - template_init(CTX_CAST(mpac->ctx)); - // don't reset referenced flag - mpac->parsed = 0; -} - - -msgpack_unpack_return -msgpack_unpack(const char* data, size_t len, size_t* off, - msgpack_zone* z, msgpack_unpack_object* result) -{ - template_context ctx; - template_init(&ctx); - - ctx.user.z = z; - ctx.user.referenced = false; - - size_t noff = 0; - if(off != NULL) { noff = *off; } - - int ret = template_execute(&ctx, data, len, &noff); - if(ret < 0) { - return MSGPACK_UNPACK_PARSE_ERROR; - } - - if(off != NULL) { *off = noff; } - - if(ret == 0) { - return MSGPACK_UNPACK_CONTINUE; - } - - *result = template_data(&ctx); - - if(noff < len) { - return MSGPACK_UNPACK_EXTRA_BYTES; - } - - return MSGPACK_UNPACK_SUCCESS; -} -#endif From c930f5367b41ef141e80a76506fd91b49fe7d426 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 01:43:50 +0900 Subject: [PATCH 0139/1648] add cythoned source and setup script. --- python/msgpack | 1 + python/msgpack.c | 3231 ++++++++++++++++++++++++++++++++++++++++++++++ python/setup.py | 29 + 3 files changed, 3261 insertions(+) create mode 120000 python/msgpack create mode 100644 python/msgpack.c create mode 100644 python/setup.py diff --git a/python/msgpack b/python/msgpack new file mode 120000 index 0000000..430db49 --- /dev/null +++ b/python/msgpack @@ -0,0 +1 @@ +../msgpack \ No newline at end of file diff --git a/python/msgpack.c b/python/msgpack.c new file mode 100644 index 0000000..50ec6fc --- /dev/null +++ b/python/msgpack.c @@ -0,0 +1,3231 @@ +/* Generated by Cython 0.11.2 on Mon Jun 8 01:28:30 2009 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#if PY_VERSION_HEX < 0x02040000 + #define METH_COEXIST 0 + #define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type) +#endif +#if PY_VERSION_HEX < 0x02050000 + typedef int Py_ssize_t; + #define PY_SSIZE_T_MAX INT_MAX + #define PY_SSIZE_T_MIN INT_MIN + #define PY_FORMAT_SIZE_T "" + #define PyInt_FromSsize_t(z) PyInt_FromLong(z) + #define PyInt_AsSsize_t(o) PyInt_AsLong(o) + #define PyNumber_Index(o) PyNumber_Int(o) + #define PyIndex_Check(o) PyNumber_Check(o) +#endif +#if PY_VERSION_HEX < 0x02060000 + #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) + #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) + #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) + #define PyVarObject_HEAD_INIT(type, size) \ + PyObject_HEAD_INIT(type) size, + #define PyType_Modified(t) + + typedef struct { + void *buf; + PyObject *obj; + Py_ssize_t len; + Py_ssize_t itemsize; + int readonly; + int ndim; + char *format; + Py_ssize_t *shape; + Py_ssize_t *strides; + Py_ssize_t *suboffsets; + void *internal; + } Py_buffer; + + #define PyBUF_SIMPLE 0 + #define PyBUF_WRITABLE 0x0001 + #define PyBUF_FORMAT 0x0004 + #define PyBUF_ND 0x0008 + #define PyBUF_STRIDES (0x0010 | PyBUF_ND) + #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) + #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) + #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) + #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) + +#endif +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" +#endif +#if PY_MAJOR_VERSION >= 3 + #define Py_TPFLAGS_CHECKTYPES 0 + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyString_Type PyBytes_Type + #define PyString_CheckExact PyBytes_CheckExact + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define PyBytes_Type PyString_Type +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyMethod_New(func, self, klass) PyInstanceMethod_New(func) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#else + #define _USE_MATH_DEFINES +#endif +#if PY_VERSION_HEX < 0x02050000 + #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),((char *)(n))) + #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a)) + #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),((char *)(n))) +#else + #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),(n)) + #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a)) + #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),(n)) +#endif +#if PY_VERSION_HEX < 0x02050000 + #define __Pyx_NAMESTR(n) ((char *)(n)) + #define __Pyx_DOCSTR(n) ((char *)(n)) +#else + #define __Pyx_NAMESTR(n) (n) + #define __Pyx_DOCSTR(n) (n) +#endif +#ifdef __cplusplus +#define __PYX_EXTERN_C extern "C" +#else +#define __PYX_EXTERN_C extern +#endif +#include +#define __PYX_HAVE_API__msgpack +#include "stdlib.h" +#include "string.h" +#include "pack.h" +#include "unpack.h" +#define __PYX_USE_C99_COMPLEX defined(_Complex_I) + + +#ifdef __GNUC__ +#define INLINE __inline__ +#elif _WIN32 +#define INLINE __inline +#else +#define INLINE +#endif + +typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/ + + + +static int __pyx_skip_dispatch = 0; + + +/* Type Conversion Predeclarations */ + +#if PY_MAJOR_VERSION < 3 +#define __Pyx_PyBytes_FromString PyString_FromString +#define __Pyx_PyBytes_FromStringAndSize PyString_FromStringAndSize +#define __Pyx_PyBytes_AsString PyString_AsString +#else +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +#define __Pyx_PyBytes_AsString PyBytes_AsString +#endif + +#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) +static INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); + +#if !defined(T_PYSSIZET) +#if PY_VERSION_HEX < 0x02050000 +#define T_PYSSIZET T_INT +#elif !defined(T_LONGLONG) +#define T_PYSSIZET \ + ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ + ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : -1)) +#else +#define T_PYSSIZET \ + ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ + ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : \ + ((sizeof(Py_ssize_t) == sizeof(PY_LONG_LONG)) ? T_LONGLONG : -1))) +#endif +#endif + +#if !defined(T_SIZET) +#if !defined(T_ULONGLONG) +#define T_SIZET \ + ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ + ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : -1)) +#else +#define T_SIZET \ + ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ + ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : \ + ((sizeof(size_t) == sizeof(unsigned PY_LONG_LONG)) ? T_ULONGLONG : -1))) +#endif +#endif + +static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); + +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) + + +#ifdef __GNUC__ +/* Test for GCC > 2.95 */ +#if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)) +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else /* __GNUC__ > 2 ... */ +#define likely(x) (x) +#define unlikely(x) (x) +#endif /* __GNUC__ > 2 ... */ +#else /* __GNUC__ */ +#define likely(x) (x) +#define unlikely(x) (x) +#endif /* __GNUC__ */ + +static PyObject *__pyx_m; +static PyObject *__pyx_b; +static PyObject *__pyx_empty_tuple; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; +static const char **__pyx_f; + + +#ifdef CYTHON_REFNANNY +typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*NewContext)(const char*, int, const char*); + void (*FinishContext)(void**); +} __Pyx_RefnannyAPIStruct; +static __Pyx_RefnannyAPIStruct *__Pyx_Refnanny = NULL; +#define __Pyx_ImportRefcountAPI(name) (__Pyx_RefnannyAPIStruct *) PyCObject_Import((char *)name, (char *)"RefnannyAPI") +#define __Pyx_INCREF(r) __Pyx_Refnanny->INCREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_DECREF(r) __Pyx_Refnanny->DECREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_GOTREF(r) __Pyx_Refnanny->GOTREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_GIVEREF(r) __Pyx_Refnanny->GIVEREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_XDECREF(r) if((r) == NULL) ; else __Pyx_DECREF(r) +#define __Pyx_SetupRefcountContext(name) void* __pyx_refchk = __Pyx_Refnanny->NewContext((name), __LINE__, __FILE__) +#define __Pyx_FinishRefcountContext() __Pyx_Refnanny->FinishContext(&__pyx_refchk) +#else +#define __Pyx_INCREF(r) Py_INCREF(r) +#define __Pyx_DECREF(r) Py_DECREF(r) +#define __Pyx_GOTREF(r) +#define __Pyx_GIVEREF(r) +#define __Pyx_XDECREF(r) Py_XDECREF(r) +#define __Pyx_SetupRefcountContext(name) +#define __Pyx_FinishRefcountContext() +#endif /* CYTHON_REFNANNY */ +#define __Pyx_XGIVEREF(r) if((r) == NULL) ; else __Pyx_GIVEREF(r) +#define __Pyx_XGOTREF(r) if((r) == NULL) ; else __Pyx_GOTREF(r) + +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, PyObject* kw_name); /*proto*/ + +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ + +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list); /*proto*/ + +static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/ + +static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +static INLINE void __Pyx_RaiseTooManyValuesError(void); + +static PyObject *__Pyx_UnpackItem(PyObject *, Py_ssize_t index); /*proto*/ +static int __Pyx_EndUnpack(PyObject *); /*proto*/ + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ + +static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ +static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ + +static INLINE int __Pyx_StrEq(const char *, const char *); /*proto*/ + +static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *); + +static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *); + +static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *); + +static INLINE char __Pyx_PyInt_AsChar(PyObject *); + +static INLINE short __Pyx_PyInt_AsShort(PyObject *); + +static INLINE int __Pyx_PyInt_AsInt(PyObject *); + +static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *); + +static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *); + +static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *); + +static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *); + +static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *); + +static INLINE long __Pyx_PyInt_AsLong(PyObject *); + +static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *); + +static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); + +static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); + +static void __Pyx_WriteUnraisable(const char *name); /*proto*/ + +static void __Pyx_AddTraceback(const char *funcname); /*proto*/ + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ + +/* Type declarations */ + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":39 + * cdef int BUFF_SIZE=2*1024 + * + * cdef class Packer: # <<<<<<<<<<<<<< + * """Packer that pack data into strm. + * + */ + +struct __pyx_obj_7msgpack_Packer { + PyObject_HEAD + char *buff; + unsigned int length; + unsigned int allocated; + struct msgpack_packer pk; + PyObject *strm; +}; + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":193 + * return unpacks(packed) + * + * cdef class Unpacker: # <<<<<<<<<<<<<< + * """Do nothing. This function is for symmetric to Packer""" + * unpack = staticmethod(unpacks) + */ + +struct __pyx_obj_7msgpack_Unpacker { + PyObject_HEAD +}; +/* Module declarations from msgpack */ + +static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; +static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; +static int __pyx_v_7msgpack_BUFF_SIZE; +static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ +#define __Pyx_MODULE_NAME "msgpack" +int __pyx_module_is_main_msgpack = 0; + +/* Implementation of msgpack */ +static char __pyx_k___main__[] = "__main__"; +static PyObject *__pyx_kp___main__; +static char __pyx_k___init__[] = "__init__"; +static PyObject *__pyx_kp___init__; +static char __pyx_k_flush[] = "flush"; +static PyObject *__pyx_kp_flush; +static char __pyx_k_pack_list[] = "pack_list"; +static PyObject *__pyx_kp_pack_list; +static char __pyx_k_pack_dict[] = "pack_dict"; +static PyObject *__pyx_kp_pack_dict; +static char __pyx_k_pack[] = "pack"; +static PyObject *__pyx_kp_pack; +static char __pyx_k_unpack[] = "unpack"; +static PyObject *__pyx_kp_unpack; +static char __pyx_k_strm[] = "strm"; +static PyObject *__pyx_kp_strm; +static char __pyx_k_size[] = "size"; +static PyObject *__pyx_kp_size; +static char __pyx_k_len[] = "len"; +static PyObject *__pyx_kp_len; +static char __pyx_k_o[] = "o"; +static PyObject *__pyx_kp_o; +static char __pyx_k_stream[] = "stream"; +static PyObject *__pyx_kp_stream; +static char __pyx_k_packed_bytes[] = "packed_bytes"; +static PyObject *__pyx_kp_packed_bytes; +static char __pyx_k_cStringIO[] = "cStringIO"; +static PyObject *__pyx_kp_cStringIO; +static char __pyx_k_StringIO[] = "StringIO"; +static PyObject *__pyx_kp_StringIO; +static char __pyx_k_staticmethod[] = "staticmethod"; +static PyObject *__pyx_kp_staticmethod; +static char __pyx_k_unpacks[] = "unpacks"; +static PyObject *__pyx_kp_unpacks; +static char __pyx_k_write[] = "write"; +static PyObject *__pyx_kp_write; +static char __pyx_k_1[] = "flush"; +static PyObject *__pyx_kp_1; +static char __pyx_k_encode[] = "encode"; +static PyObject *__pyx_kp_encode; +static char __pyx_k_iteritems[] = "iteritems"; +static PyObject *__pyx_kp_iteritems; +static char __pyx_k_TypeError[] = "TypeError"; +static PyObject *__pyx_kp_TypeError; +static char __pyx_k_getvalue[] = "getvalue"; +static PyObject *__pyx_kp_getvalue; +static char __pyx_k_read[] = "read"; +static PyObject *__pyx_kp_read; +static PyObject *__pyx_builtin_staticmethod; +static PyObject *__pyx_builtin_TypeError; +static PyObject *__pyx_kp_2; +static PyObject *__pyx_kp_3; +static char __pyx_k_2[] = "utf-8"; +static char __pyx_k_3[] = "can't serialize %r"; + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":51 + * cdef object strm + * + * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< + * if size <= 0: + * size = BUFF_SIZE + */ + +static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_strm = 0; + int __pyx_v_size; + int __pyx_r; + int __pyx_t_1; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_strm,&__pyx_kp_size,0}; + __Pyx_SetupRefcountContext("__init__"); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_strm); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 1) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_size); + if (unlikely(value)) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "__init__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_strm = values[0]; + if (values[1]) { + __pyx_v_size = __Pyx_PyInt_AsInt(values[1]); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } else { + __pyx_v_size = 0; + } + } else { + __pyx_v_size = 0; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: __pyx_v_size = __Pyx_PyInt_AsInt(PyTuple_GET_ITEM(__pyx_args, 1)); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + case 1: __pyx_v_strm = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.Packer.__init__"); + return -1; + __pyx_L4_argument_unpacking_done:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":52 + * + * def __init__(self, strm, int size=0): + * if size <= 0: # <<<<<<<<<<<<<< + * size = BUFF_SIZE + * + */ + __pyx_t_1 = (__pyx_v_size <= 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":53 + * def __init__(self, strm, int size=0): + * if size <= 0: + * size = BUFF_SIZE # <<<<<<<<<<<<<< + * + * self.strm = strm + */ + __pyx_v_size = __pyx_v_7msgpack_BUFF_SIZE; + goto __pyx_L6; + } + __pyx_L6:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":55 + * size = BUFF_SIZE + * + * self.strm = strm # <<<<<<<<<<<<<< + * self.buff = malloc(size) + * self.allocated = size + */ + __Pyx_INCREF(__pyx_v_strm); + __Pyx_GIVEREF(__pyx_v_strm); + __Pyx_GOTREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); + __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":56 + * + * self.strm = strm + * self.buff = malloc(size) # <<<<<<<<<<<<<< + * self.allocated = size + * self.length = 0 + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":57 + * self.strm = strm + * self.buff = malloc(size) + * self.allocated = size # <<<<<<<<<<<<<< + * self.length = 0 + * + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":58 + * self.buff = malloc(size) + * self.allocated = size + * self.length = 0 # <<<<<<<<<<<<<< + * + * msgpack_packer_init(&self.pk, self, _packer_write) + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":60 + * self.length = 0 + * + * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< + * + * + */ + msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); + + __pyx_r = 0; + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":63 + * + * + * def flush(self): # <<<<<<<<<<<<<< + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_flush[] = "Flash local buffer and output stream if it has 'flush()' method."; +static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused) { + PyObject *__pyx_r = NULL; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_SetupRefcountContext("flush"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":65 + * def flush(self): + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: # <<<<<<<<<<<<<< + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 + */ + __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":66 + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< + * self.length = 0 + * if hasattr(self.strm, 'flush'): + */ + __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":67 + * if self.length > 0: + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 # <<<<<<<<<<<<<< + * if hasattr(self.strm, 'flush'): + * self.strm.flush() + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; + goto __pyx_L5; + } + __pyx_L5:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":68 + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 + * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< + * self.strm.flush() + * + */ + __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_1); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":69 + * self.length = 0 + * if hasattr(self.strm, 'flush'): + * self.strm.flush() # <<<<<<<<<<<<<< + * + * def pack_list(self, len): + */ + __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_L6:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("msgpack.Packer.flush"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":71 + * self.strm.flush() + * + * def pack_list(self, len): # <<<<<<<<<<<<<< + * """Start packing sequential objects. + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_pack_list[] = "Start packing sequential objects.\n\n Example:\n\n packer.pack_list(2)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack(['foo', 'bar'])\n "; +static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { + PyObject *__pyx_r = NULL; + size_t __pyx_t_1; + __Pyx_SetupRefcountContext("pack_list"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":84 + * packer.pack(['foo', 'bar']) + * """ + * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< + * + * def pack_dict(self, len): + */ + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.Packer.pack_list"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":86 + * msgpack_pack_array(&self.pk, len) + * + * def pack_dict(self, len): # <<<<<<<<<<<<<< + * """Start packing key-value objects. + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_pack_dict[] = "Start packing key-value objects.\n\n Example:\n\n packer.pack_dict(1)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack({'foo', 'bar'})\n "; +static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { + PyObject *__pyx_r = NULL; + size_t __pyx_t_1; + __Pyx_SetupRefcountContext("pack_dict"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":99 + * packer.pack({'foo', 'bar'}) + * """ + * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< + * + * def pack(self, object o): + */ + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.Packer.pack_dict"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":101 + * msgpack_pack_map(&self.pk, len) + * + * def pack(self, object o): # <<<<<<<<<<<<<< + * cdef long long intval + * cdef double fval + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o) { + PY_LONG_LONG __pyx_v_intval; + double __pyx_v_fval; + char *__pyx_v_rawval; + PyObject *__pyx_v_k; + PyObject *__pyx_v_v; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_2 = 0; + PyObject *__pyx_3 = 0; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PY_LONG_LONG __pyx_t_3; + char *__pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + int __pyx_t_10; + __Pyx_SetupRefcountContext("pack"); + __Pyx_INCREF(__pyx_v_o); + __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); + __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":106 + * cdef char* rawval + * + * if o is None: # <<<<<<<<<<<<<< + * msgpack_pack_nil(&self.pk) + * elif o is True: + */ + __pyx_t_1 = (__pyx_v_o == Py_None); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":107 + * + * if o is None: + * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< + * elif o is True: + * msgpack_pack_true(&self.pk) + */ + msgpack_pack_nil((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":108 + * if o is None: + * msgpack_pack_nil(&self.pk) + * elif o is True: # <<<<<<<<<<<<<< + * msgpack_pack_true(&self.pk) + * elif o is False: + */ + __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = (__pyx_v_o == __pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":109 + * msgpack_pack_nil(&self.pk) + * elif o is True: + * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< + * elif o is False: + * msgpack_pack_false(&self.pk) + */ + msgpack_pack_true((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":110 + * elif o is True: + * msgpack_pack_true(&self.pk) + * elif o is False: # <<<<<<<<<<<<<< + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): + */ + __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = (__pyx_v_o == __pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":111 + * msgpack_pack_true(&self.pk) + * elif o is False: + * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< + * elif isinstance(o, long): + * intval = o + */ + msgpack_pack_false((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":112 + * elif o is False: + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): # <<<<<<<<<<<<<< + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":113 + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): + * intval = o # <<<<<<<<<<<<<< + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): + */ + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_intval = __pyx_t_3; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":114 + * elif isinstance(o, long): + * intval = o + * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< + * elif isinstance(o, int): + * intval = o + */ + msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":115 + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): # <<<<<<<<<<<<<< + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":116 + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): + * intval = o # <<<<<<<<<<<<<< + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): + */ + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_intval = __pyx_t_3; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":117 + * elif isinstance(o, int): + * intval = o + * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< + * elif isinstance(o, float): + * fval = 9 + */ + msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":118 + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): # <<<<<<<<<<<<<< + * fval = 9 + * msgpack_pack_double(&self.pk, fval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":119 + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): + * fval = 9 # <<<<<<<<<<<<<< + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): + */ + __pyx_v_fval = 9; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":120 + * elif isinstance(o, float): + * fval = 9 + * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< + * elif isinstance(o, str): + * rawval = o + */ + msgpack_pack_double((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_fval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":121 + * fval = 9 + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): # <<<<<<<<<<<<<< + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":122 + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): + * rawval = o # <<<<<<<<<<<<<< + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + */ + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_4; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":123 + * elif isinstance(o, str): + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":124 + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< + * elif isinstance(o, unicode): + * o = o.encode('utf-8') + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":125 + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): # <<<<<<<<<<<<<< + * o = o.encode('utf-8') + * rawval = o + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":126 + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): + * o = o.encode('utf-8') # <<<<<<<<<<<<<< + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_6)); + __Pyx_INCREF(__pyx_kp_2); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_2); + __Pyx_GIVEREF(__pyx_kp_2); + __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_v_o); + __pyx_v_o = __pyx_t_7; + __pyx_t_7 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":127 + * elif isinstance(o, unicode): + * o = o.encode('utf-8') + * rawval = o # <<<<<<<<<<<<<< + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + */ + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_4; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":128 + * o = o.encode('utf-8') + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":129 + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":130 + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): # <<<<<<<<<<<<<< + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":131 + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< + * for k,v in o.iteritems(): + * self.pack(k) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":132 + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): # <<<<<<<<<<<<<< + * self.pack(k) + * self.pack(v) + */ + __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { + __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); + } else { + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + for (;;) { + if (likely(PyList_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else { + __pyx_t_6 = PyIter_Next(__pyx_t_7); + if (!__pyx_t_6) { + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + break; + } + __Pyx_GOTREF(__pyx_t_6); + } + if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { + PyObject* tuple = __pyx_t_6; + __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); + __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_v_k); + __pyx_v_k = __pyx_2; + __pyx_2 = 0; + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_3; + __pyx_3 = 0; + } else { + __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_2); + __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_3); + if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(__pyx_v_k); + __pyx_v_k = __pyx_2; + __pyx_2 = 0; + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_3; + __pyx_3 = 0; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":133 + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): + * self.pack(k) # <<<<<<<<<<<<<< + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): + */ + __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_k); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); + __Pyx_GIVEREF(__pyx_v_k); + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":134 + * for k,v in o.iteritems(): + * self.pack(k) + * self.pack(v) # <<<<<<<<<<<<<< + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) + */ + __pyx_t_8 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_v); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); + __Pyx_GIVEREF(__pyx_v_v); + __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":135 + * self.pack(k) + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); + if (!__pyx_t_1) { + __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); + __pyx_t_10 = __pyx_t_9; + } else { + __pyx_t_10 = __pyx_t_1; + } + if (__pyx_t_10) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":136 + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< + * for v in o: + * self.pack(v) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":137 + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: # <<<<<<<<<<<<<< + * self.pack(v) + * else: + */ + if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { + __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); + } else { + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + } + for (;;) { + if (likely(PyList_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else { + __pyx_t_6 = PyIter_Next(__pyx_t_7); + if (!__pyx_t_6) { + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + break; + } + __Pyx_GOTREF(__pyx_t_6); + } + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_t_6; + __pyx_t_6 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":138 + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: + * self.pack(v) # <<<<<<<<<<<<<< + * else: + * # TODO: Serialize with defalt() like simplejson. + */ + __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_v); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); + __Pyx_GIVEREF(__pyx_v_v); + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L5; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":141 + * else: + * # TODO: Serialize with defalt() like simplejson. + * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + */ + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_7)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_8 = PyNumber_Remainder(__pyx_kp_3, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; + __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } + __pyx_L5:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_2); + __Pyx_XDECREF(__pyx_3); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("msgpack.Packer.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_k); + __Pyx_DECREF(__pyx_v_v); + __Pyx_DECREF(__pyx_v_o); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":143 + * raise TypeError, "can't serialize %r" % (o,) + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< + * if packer.length + l > packer.allocated: + * if packer.length > 0: + */ + +static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__pyx_v_packer, const char* __pyx_v_b, unsigned int __pyx_v_l) { + int __pyx_r; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_SetupRefcountContext("_packer_write"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":144 + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + */ + __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":145 + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * if packer.length + l > packer.allocated: + * if packer.length > 0: # <<<<<<<<<<<<<< + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: + */ + __pyx_t_1 = (__pyx_v_packer->length > 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":146 + * if packer.length + l > packer.allocated: + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L4; + } + __pyx_L4:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":147 + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: # <<<<<<<<<<<<<< + * packer.strm.write(PyString_FromStringAndSize(b, l)) + * packer.length = 0 + */ + __pyx_t_1 = (__pyx_v_l > 64); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":148 + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< + * packer.length = 0 + * else: + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":149 + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) + * packer.length = 0 # <<<<<<<<<<<<<< + * else: + * memcpy(packer.buff, b, l) + */ + __pyx_v_packer->length = 0; + goto __pyx_L5; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":151 + * packer.length = 0 + * else: + * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< + * packer.length = l + * else: + */ + memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":152 + * else: + * memcpy(packer.buff, b, l) + * packer.length = l # <<<<<<<<<<<<<< + * else: + * memcpy(packer.buff + packer.length, b, l) + */ + __pyx_v_packer->length = __pyx_v_l; + } + __pyx_L5:; + goto __pyx_L3; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":154 + * packer.length = l + * else: + * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< + * packer.length += l + * return 0 + */ + memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":155 + * else: + * memcpy(packer.buff + packer.length, b, l) + * packer.length += l # <<<<<<<<<<<<<< + * return 0 + * + */ + __pyx_v_packer->length += __pyx_v_l; + } + __pyx_L3:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":156 + * memcpy(packer.buff + packer.length, b, l) + * packer.length += l + * return 0 # <<<<<<<<<<<<<< + * + * def pack(object o, object stream): + */ + __pyx_r = 0; + goto __pyx_L0; + + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_WriteUnraisable("msgpack._packer_write"); + __pyx_r = 0; + __pyx_L0:; + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":158 + * return 0 + * + * def pack(object o, object stream): # <<<<<<<<<<<<<< + * packer = Packer(stream) + * packer.pack(o) + */ + +static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_o = 0; + PyObject *__pyx_v_stream = 0; + PyObject *__pyx_v_packer; + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_o,&__pyx_kp_stream,0}; + __Pyx_SetupRefcountContext("pack"); + __pyx_self = __pyx_self; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_o); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); + if (likely(values[1])) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_o = values[0]; + __pyx_v_stream = values[1]; + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + __pyx_v_o = PyTuple_GET_ITEM(__pyx_args, 0); + __pyx_v_stream = PyTuple_GET_ITEM(__pyx_args, 1); + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.pack"); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":159 + * + * def pack(object o, object stream): + * packer = Packer(stream) # <<<<<<<<<<<<<< + * packer.pack(o) + * packer.flush() + */ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_stream); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); + __Pyx_GIVEREF(__pyx_v_stream); + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packer); + __pyx_v_packer = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":160 + * def pack(object o, object stream): + * packer = Packer(stream) + * packer.pack(o) # <<<<<<<<<<<<<< + * packer.flush() + * + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":161 + * packer = Packer(stream) + * packer.pack(o) + * packer.flush() # <<<<<<<<<<<<<< + * + * def packs(object o): + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_packer); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":163 + * packer.flush() + * + * def packs(object o): # <<<<<<<<<<<<<< + * buf = StringIO() + * packer = Packer(buf) + */ + +static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o); /*proto*/ +static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o) { + PyObject *__pyx_v_buf; + PyObject *__pyx_v_packer; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_SetupRefcountContext("packs"); + __pyx_self = __pyx_self; + __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); + __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":164 + * + * def packs(object o): + * buf = StringIO() # <<<<<<<<<<<<<< + * packer = Packer(buf) + * packer.pack(o) + */ + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(__pyx_v_buf); + __pyx_v_buf = __pyx_t_1; + __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":165 + * def packs(object o): + * buf = StringIO() + * packer = Packer(buf) # <<<<<<<<<<<<<< + * packer.pack(o) + * packer.flush() + */ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_buf); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); + __Pyx_GIVEREF(__pyx_v_buf); + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packer); + __pyx_v_packer = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":166 + * buf = StringIO() + * packer = Packer(buf) + * packer.pack(o) # <<<<<<<<<<<<<< + * packer.flush() + * return buf.getvalue() + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":167 + * packer = Packer(buf) + * packer.pack(o) + * packer.flush() # <<<<<<<<<<<<<< + * return buf.getvalue() + * + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":168 + * packer.pack(o) + * packer.flush() + * return buf.getvalue() # <<<<<<<<<<<<<< + * + * cdef extern from "unpack.h": + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.packs"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_buf); + __Pyx_DECREF(__pyx_v_packer); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":179 + * + * + * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< + * """Unpack packed_bytes to object. Returns unpacked object.""" + * cdef const_char_ptr p = packed_bytes + */ + +static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes); /*proto*/ +static char __pyx_doc_7msgpack_unpacks[] = "Unpack packed_bytes to object. Returns unpacked object."; +static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes) { + const char* __pyx_v_p; + template_context __pyx_v_ctx; + size_t __pyx_v_off; + PyObject *__pyx_r = NULL; + const char* __pyx_t_1; + Py_ssize_t __pyx_t_2; + PyObject *__pyx_t_3; + __Pyx_SetupRefcountContext("unpacks"); + __pyx_self = __pyx_self; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":181 + * def unpacks(object packed_bytes): + * """Unpack packed_bytes to object. Returns unpacked object.""" + * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< + * cdef template_context ctx + * cdef size_t off = 0 + */ + __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 181; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_p = __pyx_t_1; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":183 + * cdef const_char_ptr p = packed_bytes + * cdef template_context ctx + * cdef size_t off = 0 # <<<<<<<<<<<<<< + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) + */ + __pyx_v_off = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":184 + * cdef template_context ctx + * cdef size_t off = 0 + * template_init(&ctx) # <<<<<<<<<<<<<< + * template_execute(&ctx, p, len(packed_bytes), &off) + * return template_data(&ctx) + */ + template_init((&__pyx_v_ctx)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":185 + * cdef size_t off = 0 + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< + * return template_data(&ctx) + * + */ + __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) + * return template_data(&ctx) # <<<<<<<<<<<<<< + * + * def unpack(object stream): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = template_data((&__pyx_v_ctx)); + __Pyx_INCREF(((PyObject *)__pyx_t_3)); + __pyx_r = ((PyObject *)__pyx_t_3); + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.unpacks"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 + * return template_data(&ctx) + * + * def unpack(object stream): # <<<<<<<<<<<<<< + * """unpack from stream.""" + * packed = stream.read() + */ + +static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream); /*proto*/ +static char __pyx_doc_7msgpack_unpack[] = "unpack from stream."; +static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream) { + PyObject *__pyx_v_packed; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + __Pyx_SetupRefcountContext("unpack"); + __pyx_self = __pyx_self; + __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":190 + * def unpack(object stream): + * """unpack from stream.""" + * packed = stream.read() # <<<<<<<<<<<<<< + * return unpacks(packed) + * + */ + __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packed); + __pyx_v_packed = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":191 + * """unpack from stream.""" + * packed = stream.read() + * return unpacks(packed) # <<<<<<<<<<<<<< + * + * cdef class Unpacker: + */ + __Pyx_XDECREF(__pyx_r); + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_packed); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); + __Pyx_GIVEREF(__pyx_v_packed); + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("msgpack.unpack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_packed); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_7msgpack_Packer *p; + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + p = ((struct __pyx_obj_7msgpack_Packer *)o); + p->strm = Py_None; Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_7msgpack_Packer(PyObject *o) { + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + Py_XDECREF(p->strm); + (*Py_TYPE(o)->tp_free)(o); +} + +static int __pyx_tp_traverse_7msgpack_Packer(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + if (p->strm) { + e = (*v)(p->strm, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + PyObject* tmp; + tmp = ((PyObject*)p->strm); + p->strm = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { + {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, + {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, + {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_O, __Pyx_DOCSTR(0)}, + {0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number_Packer = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else + 0, /*nb_long*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence_Packer = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Packer = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_Packer = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +PyTypeObject __pyx_type_7msgpack_Packer = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("msgpack.Packer"), /*tp_name*/ + sizeof(struct __pyx_obj_7msgpack_Packer), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_7msgpack_Packer, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + &__pyx_tp_as_number_Packer, /*tp_as_number*/ + &__pyx_tp_as_sequence_Packer, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Packer, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_Packer, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __Pyx_DOCSTR("Packer that pack data into strm.\n\n strm must have `write(bytes)` method.\n size specifies local buffer size.\n "), /*tp_doc*/ + __pyx_tp_traverse_7msgpack_Packer, /*tp_traverse*/ + __pyx_tp_clear_7msgpack_Packer, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_7msgpack_Packer, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + __pyx_pf_7msgpack_6Packer___init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_7msgpack_Packer, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ +}; + +static PyObject *__pyx_tp_new_7msgpack_Unpacker(PyTypeObject *t, PyObject *a, PyObject *k) { + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + return o; +} + +static void __pyx_tp_dealloc_7msgpack_Unpacker(PyObject *o) { + (*Py_TYPE(o)->tp_free)(o); +} + +static struct PyMethodDef __pyx_methods_7msgpack_Unpacker[] = { + {0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number_Unpacker = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else + 0, /*nb_long*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence_Unpacker = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Unpacker = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_Unpacker = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +PyTypeObject __pyx_type_7msgpack_Unpacker = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("msgpack.Unpacker"), /*tp_name*/ + sizeof(struct __pyx_obj_7msgpack_Unpacker), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_7msgpack_Unpacker, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + &__pyx_tp_as_number_Unpacker, /*tp_as_number*/ + &__pyx_tp_as_sequence_Unpacker, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Unpacker, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_Unpacker, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ + __Pyx_DOCSTR("Do nothing. This function is for symmetric to Packer"), /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_7msgpack_Unpacker, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_7msgpack_Unpacker, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ +}; + +static struct PyMethodDef __pyx_methods[] = { + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("packs"), (PyCFunction)__pyx_pf_7msgpack_packs, METH_O, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("unpacks"), (PyCFunction)__pyx_pf_7msgpack_unpacks, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpacks)}, + {__Pyx_NAMESTR("unpack"), (PyCFunction)__pyx_pf_7msgpack_unpack, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpack)}, + {0, 0, 0, 0} +}; + +static void __pyx_init_filenames(void); /*proto*/ + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef __pyx_moduledef = { + PyModuleDef_HEAD_INIT, + __Pyx_NAMESTR("msgpack"), + 0, /* m_doc */ + -1, /* m_size */ + __pyx_methods /* m_methods */, + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, + {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, + {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, + {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, + {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, + {&__pyx_kp_pack, __pyx_k_pack, sizeof(__pyx_k_pack), 1, 1, 1}, + {&__pyx_kp_unpack, __pyx_k_unpack, sizeof(__pyx_k_unpack), 1, 1, 1}, + {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, + {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, + {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, + {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, + {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, + {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, + {&__pyx_kp_cStringIO, __pyx_k_cStringIO, sizeof(__pyx_k_cStringIO), 1, 1, 1}, + {&__pyx_kp_StringIO, __pyx_k_StringIO, sizeof(__pyx_k_StringIO), 1, 1, 1}, + {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, + {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, + {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, + {&__pyx_kp_1, __pyx_k_1, sizeof(__pyx_k_1), 0, 1, 0}, + {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, + {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, + {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, + {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, + {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, + {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 0}, + {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, + {0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_InitGlobals(void) { + if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + return 0; + __pyx_L1_error:; + return -1; +} + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC initmsgpack(void); /*proto*/ +PyMODINIT_FUNC initmsgpack(void) +#else +PyMODINIT_FUNC PyInit_msgpack(void); /*proto*/ +PyMODINIT_FUNC PyInit_msgpack(void) +#endif +{ + PyObject *__pyx_1 = 0; + PyObject *__pyx_2 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + #ifdef CYTHON_REFNANNY + void* __pyx_refchk = NULL; + __Pyx_Refnanny = __Pyx_ImportRefcountAPI("refnanny"); + if (!__Pyx_Refnanny) { + PyErr_Clear(); + __Pyx_Refnanny = __Pyx_ImportRefcountAPI("Cython.Runtime.refnanny"); + if (!__Pyx_Refnanny) + Py_FatalError("failed to import refnanny module"); + } + __pyx_refchk = __Pyx_Refnanny->NewContext("PyMODINIT_FUNC PyInit_msgpack(void)", __LINE__, __FILE__); + #endif + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + /*--- Library function declarations ---*/ + __pyx_init_filenames(); + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Initialize various global constants etc. ---*/ + if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + /*--- Module creation code ---*/ + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4(__Pyx_NAMESTR("msgpack"), __pyx_methods, 0, 0, PYTHON_API_VERSION); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (!__pyx_m) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + #if PY_MAJOR_VERSION < 3 + Py_INCREF(__pyx_m); + #endif + __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); + if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__pyx_module_is_main_msgpack) { + if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + } + /*--- Builtin init code ---*/ + if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_skip_dispatch = 0; + /*--- Global init code ---*/ + /*--- Function export code ---*/ + /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; + if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; + /*--- Type import code ---*/ + /*--- Function import code ---*/ + /*--- Execution code ---*/ + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":3 + * # coding: utf-8 + * + * from cStringIO import StringIO # <<<<<<<<<<<<<< + * + * cdef extern from "Python.h": + */ + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_kp_StringIO); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_kp_StringIO); + __Pyx_GIVEREF(__pyx_kp_StringIO); + __pyx_1 = __Pyx_Import(__pyx_kp_cStringIO, ((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __pyx_2 = PyObject_GetAttr(__pyx_1, __pyx_kp_StringIO); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_2); + if (PyObject_SetAttr(__pyx_m, __pyx_kp_StringIO, __pyx_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_2); __pyx_2 = 0; + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":37 + * + * + * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< + * + * cdef class Packer: + */ + __pyx_v_7msgpack_BUFF_SIZE = 2048; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":195 + * cdef class Unpacker: + * """Do nothing. This function is for symmetric to Packer""" + * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< + */ + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); + __Pyx_GIVEREF(__pyx_1); + __pyx_1 = 0; + __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + PyType_Modified(__pyx_ptype_7msgpack_Unpacker); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_2); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("msgpack"); + Py_DECREF(__pyx_m); __pyx_m = 0; + __pyx_L0:; + __Pyx_FinishRefcountContext(); + #if PY_MAJOR_VERSION < 3 + return; + #else + return __pyx_m; + #endif +} + +static const char *__pyx_filenames[] = { + "msgpack.pyx", +}; + +/* Runtime support code */ + +static void __pyx_init_filenames(void) { + __pyx_f = __pyx_filenames; +} + +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AS_STRING(kw_name)); + #endif +} + +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *number, *more_or_less; + + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + number = (num_expected == 1) ? "" : "s"; + PyErr_Format(PyExc_TypeError, + #if PY_VERSION_HEX < 0x02050000 + "%s() takes %s %d positional argument%s (%d given)", + #else + "%s() takes %s %zd positional argument%s (%zd given)", + #endif + func_name, more_or_less, num_expected, number, num_found); +} + +static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + } else { + #if PY_MAJOR_VERSION < 3 + if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) { + #else + if (unlikely(!PyUnicode_CheckExact(key)) && unlikely(!PyUnicode_Check(key))) { + #endif + goto invalid_keyword_type; + } else { + for (name = first_kw_arg; *name; name++) { + #if PY_MAJOR_VERSION >= 3 + if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && + PyUnicode_Compare(**name, key) == 0) break; + #else + if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && + _PyString_Eq(**name, key)) break; + #endif + } + if (*name) { + values[name-argnames] = value; + } else { + /* unexpected keyword found */ + for (name=argnames; name != first_kw_arg; name++) { + if (**name == key) goto arg_passed_twice; + #if PY_MAJOR_VERSION >= 3 + if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && + PyUnicode_Compare(**name, key) == 0) goto arg_passed_twice; + #else + if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && + _PyString_Eq(**name, key)) goto arg_passed_twice; + #endif + } + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + } + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, **name); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%s() got an unexpected keyword argument '%s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list) { + PyObject *__import__ = 0; + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + __import__ = __Pyx_GetAttrString(__pyx_b, "__import__"); + if (!__import__) + goto bad; + if (from_list) + list = from_list; + else { + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; + } + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + module = PyObject_CallFunctionObjArgs(__import__, + name, global_dict, empty_dict, list, NULL); +bad: + Py_XDECREF(empty_list); + Py_XDECREF(__import__); + Py_XDECREF(empty_dict); + return module; +} + +static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) { + PyObject *result; + result = PyObject_GetAttr(dict, name); + if (!result) + PyErr_SetObject(PyExc_NameError, name); + return result; +} + +static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + #if PY_VERSION_HEX < 0x02050000 + "need more than %d value%s to unpack", (int)index, + #else + "need more than %zd value%s to unpack", index, + #endif + (index == 1) ? "" : "s"); +} + +static INLINE void __Pyx_RaiseTooManyValuesError(void) { + PyErr_SetString(PyExc_ValueError, "too many values to unpack"); +} + +static PyObject *__Pyx_UnpackItem(PyObject *iter, Py_ssize_t index) { + PyObject *item; + if (!(item = PyIter_Next(iter))) { + if (!PyErr_Occurred()) { + __Pyx_RaiseNeedMoreValuesError(index); + } + } + return item; +} + +static int __Pyx_EndUnpack(PyObject *iter) { + PyObject *item; + if ((item = PyIter_Next(iter))) { + Py_DECREF(item); + __Pyx_RaiseTooManyValuesError(); + return -1; + } + else if (!PyErr_Occurred()) + return 0; + else + return -1; +} + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) { + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(tb); + /* First, check the traceback argument, replacing None with NULL. */ + if (tb == Py_None) { + Py_DECREF(tb); + tb = 0; + } + else if (tb != NULL && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + /* Next, replace a missing value with None */ + if (value == NULL) { + value = Py_None; + Py_INCREF(value); + } + #if PY_VERSION_HEX < 0x02050000 + if (!PyClass_Check(type)) + #else + if (!PyType_Check(type)) + #endif + { + /* Raising an instance. The value should be a dummy. */ + if (value != Py_None) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + /* Normalize to raise , */ + Py_DECREF(value); + value = type; + #if PY_VERSION_HEX < 0x02050000 + if (PyInstance_Check(type)) { + type = (PyObject*) ((PyInstanceObject*)type)->in_class; + Py_INCREF(type); + } + else { + type = 0; + PyErr_SetString(PyExc_TypeError, + "raise: exception must be an old-style class or instance"); + goto raise_error; + } + #else + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + #endif + } + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} + +static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyThreadState *tstate = PyThreadState_GET(); + +#if PY_MAJOR_VERSION >= 3 + /* Note: this is a temporary work-around to prevent crashes in Python 3.0 */ + if ((tstate->exc_type != NULL) & (tstate->exc_type != Py_None)) { + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + PyErr_NormalizeException(&type, &value, &tb); + PyErr_NormalizeException(&tmp_type, &tmp_value, &tmp_tb); + tstate->exc_type = 0; + tstate->exc_value = 0; + tstate->exc_traceback = 0; + PyException_SetContext(value, tmp_value); + Py_DECREF(tmp_type); + Py_XDECREF(tmp_tb); + } +#endif + + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} + +static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) { + PyThreadState *tstate = PyThreadState_GET(); + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} + + +static INLINE int __Pyx_StrEq(const char *s1, const char *s2) { + while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } + return *s1 == *s2; +} + +static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { + if (sizeof(unsigned char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned char)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned char"); + return (unsigned char)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned char"); + return (unsigned char)-1; + } + return (unsigned char)val; + } + return (unsigned char)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject* x) { + if (sizeof(unsigned short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned short)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned short"); + return (unsigned short)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned short"); + return (unsigned short)-1; + } + return (unsigned short)val; + } + return (unsigned short)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject* x) { + if (sizeof(unsigned int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned int)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned int"); + return (unsigned int)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned int"); + return (unsigned int)-1; + } + return (unsigned int)val; + } + return (unsigned int)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE char __Pyx_PyInt_AsChar(PyObject* x) { + if (sizeof(char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (char)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char)-1; + } + return (char)val; + } + return (char)__Pyx_PyInt_AsLong(x); +} + +static INLINE short __Pyx_PyInt_AsShort(PyObject* x) { + if (sizeof(short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (short)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to short"); + return (short)-1; + } + return (short)val; + } + return (short)__Pyx_PyInt_AsLong(x); +} + +static INLINE int __Pyx_PyInt_AsInt(PyObject* x) { + if (sizeof(int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (int)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int)-1; + } + return (int)val; + } + return (int)__Pyx_PyInt_AsLong(x); +} + +static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject* x) { + if (sizeof(signed char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed char)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed char"); + return (signed char)-1; + } + return (signed char)val; + } + return (signed char)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject* x) { + if (sizeof(signed short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed short)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed short"); + return (signed short)-1; + } + return (signed short)val; + } + return (signed short)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject* x) { + if (sizeof(signed int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed int)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed int"); + return (signed int)-1; + } + return (signed int)val; + } + return (signed int)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned long"); + return (unsigned long)-1; + } + return (unsigned long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned long"); + return (unsigned long)-1; + } + return PyLong_AsUnsignedLong(x); + } else { + unsigned long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (unsigned long)-1; + val = __Pyx_PyInt_AsUnsignedLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned PY_LONG_LONG"); + return (unsigned PY_LONG_LONG)-1; + } + return (unsigned PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned PY_LONG_LONG"); + return (unsigned PY_LONG_LONG)-1; + } + return PyLong_AsUnsignedLongLong(x); + } else { + unsigned PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (unsigned PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsUnsignedLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE long __Pyx_PyInt_AsLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLong(x); + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (long)-1; + val = __Pyx_PyInt_AsLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLongLong(x); + } else { + PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (signed long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLong(x); + } else { + signed long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (signed long)-1; + val = __Pyx_PyInt_AsSignedLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (signed PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLongLong(x); + } else { + signed PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (signed PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsSignedLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static void __Pyx_WriteUnraisable(const char *name) { + PyObject *old_exc, *old_val, *old_tb; + PyObject *ctx; + __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); + #if PY_MAJOR_VERSION < 3 + ctx = PyString_FromString(name); + #else + ctx = PyUnicode_FromString(name); + #endif + __Pyx_ErrRestore(old_exc, old_val, old_tb); + if (!ctx) { + PyErr_WriteUnraisable(Py_None); + } else { + PyErr_WriteUnraisable(ctx); + Py_DECREF(ctx); + } +} + +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" + +static void __Pyx_AddTraceback(const char *funcname) { + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + PyObject *py_globals = 0; + PyObject *empty_string = 0; + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(__pyx_filename); + #else + py_srcfile = PyUnicode_FromString(__pyx_filename); + #endif + if (!py_srcfile) goto bad; + if (__pyx_clineno) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_globals = PyModule_GetDict(__pyx_m); + if (!py_globals) goto bad; + #if PY_MAJOR_VERSION < 3 + empty_string = PyString_FromStringAndSize("", 0); + #else + empty_string = PyBytes_FromStringAndSize("", 0); + #endif + if (!empty_string) goto bad; + py_code = PyCode_New( + 0, /*int argcount,*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*int kwonlyargcount,*/ + #endif + 0, /*int nlocals,*/ + 0, /*int stacksize,*/ + 0, /*int flags,*/ + empty_string, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + __pyx_lineno, /*int firstlineno,*/ + empty_string /*PyObject *lnotab*/ + ); + if (!py_code) goto bad; + py_frame = PyFrame_New( + PyThreadState_GET(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + py_globals, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + py_frame->f_lineno = __pyx_lineno; + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + Py_XDECREF(empty_string); + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode && (!t->is_identifier)) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else /* Python 3+ has unicode identifiers */ + if (t->is_identifier || (t->is_unicode && t->intern)) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->is_unicode) { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + ++t; + } + return 0; +} + +/* Type Conversion Functions */ + +static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + if (x == Py_True) return 1; + else if ((x == Py_False) | (x == Py_None)) return 0; + else return PyObject_IsTrue(x); +} + +static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { + PyNumberMethods *m; + const char *name = NULL; + PyObject *res = NULL; +#if PY_VERSION_HEX < 0x03000000 + if (PyInt_Check(x) || PyLong_Check(x)) +#else + if (PyLong_Check(x)) +#endif + return Py_INCREF(x), x; + m = Py_TYPE(x)->tp_as_number; +#if PY_VERSION_HEX < 0x03000000 + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } +#else + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Long(x); + } +#endif + if (res) { +#if PY_VERSION_HEX < 0x03000000 + if (!PyInt_Check(res) && !PyLong_Check(res)) { +#else + if (!PyLong_Check(res)) { +#endif + PyErr_Format(PyExc_TypeError, + "__%s__ returned non-%s (type %.200s)", + name, name, Py_TYPE(res)->tp_name); + Py_DECREF(res); + return NULL; + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} + +static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject* x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} + +static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { +#if PY_VERSION_HEX < 0x02050000 + if (ival <= LONG_MAX) + return PyInt_FromLong((long)ival); + else { + unsigned char *bytes = (unsigned char *) &ival; + int one = 1; int little = (int)*(unsigned char*)&one; + return _PyLong_FromByteArray(bytes, sizeof(size_t), little, 0); + } +#else + return PyInt_FromSize_t(ival); +#endif +} + +static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { + unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); + if (unlikely(val == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())) { + return (size_t)-1; + } else if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t)-1; + } + return (size_t)val; +} + + diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 0000000..8bfdf82 --- /dev/null +++ b/python/setup.py @@ -0,0 +1,29 @@ +from distutils.core import setup, Extension + +version = '0.0.1' + +msgpack_mod = Extension('msgpack', sources=['msgpack.c']) + +desc = 'MessagePack serializer/desirializer.' +long_desc = desc + """ + +Python binding of MessagePack_. + +This package is under development. + +.. _MessagePack: http://msgpack.sourceforge.jp/ + +What's MessagePack? (from http://msgpack.sourceforge.jp/) + + MessagePack is a binary-based efficient data interchange format that is + focused on high performance. It is like JSON, but very fast and small. +""" + +setup(name='msgpack', + author='Naoki INADA', + author_email='songofacandy@gmail.com', + version=version, + ext_modules=[msgpack_mod], + description='The MessagePack serializer/desirializer.', + long_description=long_desc, + ) From abe2a99ad2d57ef38c96643e93e0a1bb33dee1c9 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 01:43:50 +0900 Subject: [PATCH 0140/1648] add cythoned source and setup script. --- msgpack | 1 + msgpack.c | 3231 +++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 29 + 3 files changed, 3261 insertions(+) create mode 120000 msgpack create mode 100644 msgpack.c create mode 100644 setup.py diff --git a/msgpack b/msgpack new file mode 120000 index 0000000..430db49 --- /dev/null +++ b/msgpack @@ -0,0 +1 @@ +../msgpack \ No newline at end of file diff --git a/msgpack.c b/msgpack.c new file mode 100644 index 0000000..50ec6fc --- /dev/null +++ b/msgpack.c @@ -0,0 +1,3231 @@ +/* Generated by Cython 0.11.2 on Mon Jun 8 01:28:30 2009 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#if PY_VERSION_HEX < 0x02040000 + #define METH_COEXIST 0 + #define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type) +#endif +#if PY_VERSION_HEX < 0x02050000 + typedef int Py_ssize_t; + #define PY_SSIZE_T_MAX INT_MAX + #define PY_SSIZE_T_MIN INT_MIN + #define PY_FORMAT_SIZE_T "" + #define PyInt_FromSsize_t(z) PyInt_FromLong(z) + #define PyInt_AsSsize_t(o) PyInt_AsLong(o) + #define PyNumber_Index(o) PyNumber_Int(o) + #define PyIndex_Check(o) PyNumber_Check(o) +#endif +#if PY_VERSION_HEX < 0x02060000 + #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) + #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) + #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) + #define PyVarObject_HEAD_INIT(type, size) \ + PyObject_HEAD_INIT(type) size, + #define PyType_Modified(t) + + typedef struct { + void *buf; + PyObject *obj; + Py_ssize_t len; + Py_ssize_t itemsize; + int readonly; + int ndim; + char *format; + Py_ssize_t *shape; + Py_ssize_t *strides; + Py_ssize_t *suboffsets; + void *internal; + } Py_buffer; + + #define PyBUF_SIMPLE 0 + #define PyBUF_WRITABLE 0x0001 + #define PyBUF_FORMAT 0x0004 + #define PyBUF_ND 0x0008 + #define PyBUF_STRIDES (0x0010 | PyBUF_ND) + #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) + #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) + #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) + #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) + +#endif +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" +#endif +#if PY_MAJOR_VERSION >= 3 + #define Py_TPFLAGS_CHECKTYPES 0 + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyString_Type PyBytes_Type + #define PyString_CheckExact PyBytes_CheckExact + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define PyBytes_Type PyString_Type +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyMethod_New(func, self, klass) PyInstanceMethod_New(func) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#else + #define _USE_MATH_DEFINES +#endif +#if PY_VERSION_HEX < 0x02050000 + #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),((char *)(n))) + #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a)) + #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),((char *)(n))) +#else + #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),(n)) + #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a)) + #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),(n)) +#endif +#if PY_VERSION_HEX < 0x02050000 + #define __Pyx_NAMESTR(n) ((char *)(n)) + #define __Pyx_DOCSTR(n) ((char *)(n)) +#else + #define __Pyx_NAMESTR(n) (n) + #define __Pyx_DOCSTR(n) (n) +#endif +#ifdef __cplusplus +#define __PYX_EXTERN_C extern "C" +#else +#define __PYX_EXTERN_C extern +#endif +#include +#define __PYX_HAVE_API__msgpack +#include "stdlib.h" +#include "string.h" +#include "pack.h" +#include "unpack.h" +#define __PYX_USE_C99_COMPLEX defined(_Complex_I) + + +#ifdef __GNUC__ +#define INLINE __inline__ +#elif _WIN32 +#define INLINE __inline +#else +#define INLINE +#endif + +typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/ + + + +static int __pyx_skip_dispatch = 0; + + +/* Type Conversion Predeclarations */ + +#if PY_MAJOR_VERSION < 3 +#define __Pyx_PyBytes_FromString PyString_FromString +#define __Pyx_PyBytes_FromStringAndSize PyString_FromStringAndSize +#define __Pyx_PyBytes_AsString PyString_AsString +#else +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +#define __Pyx_PyBytes_AsString PyBytes_AsString +#endif + +#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) +static INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); + +#if !defined(T_PYSSIZET) +#if PY_VERSION_HEX < 0x02050000 +#define T_PYSSIZET T_INT +#elif !defined(T_LONGLONG) +#define T_PYSSIZET \ + ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ + ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : -1)) +#else +#define T_PYSSIZET \ + ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ + ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : \ + ((sizeof(Py_ssize_t) == sizeof(PY_LONG_LONG)) ? T_LONGLONG : -1))) +#endif +#endif + +#if !defined(T_SIZET) +#if !defined(T_ULONGLONG) +#define T_SIZET \ + ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ + ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : -1)) +#else +#define T_SIZET \ + ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ + ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : \ + ((sizeof(size_t) == sizeof(unsigned PY_LONG_LONG)) ? T_ULONGLONG : -1))) +#endif +#endif + +static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); + +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) + + +#ifdef __GNUC__ +/* Test for GCC > 2.95 */ +#if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)) +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else /* __GNUC__ > 2 ... */ +#define likely(x) (x) +#define unlikely(x) (x) +#endif /* __GNUC__ > 2 ... */ +#else /* __GNUC__ */ +#define likely(x) (x) +#define unlikely(x) (x) +#endif /* __GNUC__ */ + +static PyObject *__pyx_m; +static PyObject *__pyx_b; +static PyObject *__pyx_empty_tuple; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; +static const char **__pyx_f; + + +#ifdef CYTHON_REFNANNY +typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*NewContext)(const char*, int, const char*); + void (*FinishContext)(void**); +} __Pyx_RefnannyAPIStruct; +static __Pyx_RefnannyAPIStruct *__Pyx_Refnanny = NULL; +#define __Pyx_ImportRefcountAPI(name) (__Pyx_RefnannyAPIStruct *) PyCObject_Import((char *)name, (char *)"RefnannyAPI") +#define __Pyx_INCREF(r) __Pyx_Refnanny->INCREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_DECREF(r) __Pyx_Refnanny->DECREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_GOTREF(r) __Pyx_Refnanny->GOTREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_GIVEREF(r) __Pyx_Refnanny->GIVEREF(__pyx_refchk, (PyObject *)(r), __LINE__) +#define __Pyx_XDECREF(r) if((r) == NULL) ; else __Pyx_DECREF(r) +#define __Pyx_SetupRefcountContext(name) void* __pyx_refchk = __Pyx_Refnanny->NewContext((name), __LINE__, __FILE__) +#define __Pyx_FinishRefcountContext() __Pyx_Refnanny->FinishContext(&__pyx_refchk) +#else +#define __Pyx_INCREF(r) Py_INCREF(r) +#define __Pyx_DECREF(r) Py_DECREF(r) +#define __Pyx_GOTREF(r) +#define __Pyx_GIVEREF(r) +#define __Pyx_XDECREF(r) Py_XDECREF(r) +#define __Pyx_SetupRefcountContext(name) +#define __Pyx_FinishRefcountContext() +#endif /* CYTHON_REFNANNY */ +#define __Pyx_XGIVEREF(r) if((r) == NULL) ; else __Pyx_GIVEREF(r) +#define __Pyx_XGOTREF(r) if((r) == NULL) ; else __Pyx_GOTREF(r) + +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, PyObject* kw_name); /*proto*/ + +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ + +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list); /*proto*/ + +static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/ + +static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); + +static INLINE void __Pyx_RaiseTooManyValuesError(void); + +static PyObject *__Pyx_UnpackItem(PyObject *, Py_ssize_t index); /*proto*/ +static int __Pyx_EndUnpack(PyObject *); /*proto*/ + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ + +static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ +static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ + +static INLINE int __Pyx_StrEq(const char *, const char *); /*proto*/ + +static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *); + +static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *); + +static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *); + +static INLINE char __Pyx_PyInt_AsChar(PyObject *); + +static INLINE short __Pyx_PyInt_AsShort(PyObject *); + +static INLINE int __Pyx_PyInt_AsInt(PyObject *); + +static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *); + +static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *); + +static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *); + +static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *); + +static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *); + +static INLINE long __Pyx_PyInt_AsLong(PyObject *); + +static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *); + +static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); + +static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); + +static void __Pyx_WriteUnraisable(const char *name); /*proto*/ + +static void __Pyx_AddTraceback(const char *funcname); /*proto*/ + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ + +/* Type declarations */ + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":39 + * cdef int BUFF_SIZE=2*1024 + * + * cdef class Packer: # <<<<<<<<<<<<<< + * """Packer that pack data into strm. + * + */ + +struct __pyx_obj_7msgpack_Packer { + PyObject_HEAD + char *buff; + unsigned int length; + unsigned int allocated; + struct msgpack_packer pk; + PyObject *strm; +}; + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":193 + * return unpacks(packed) + * + * cdef class Unpacker: # <<<<<<<<<<<<<< + * """Do nothing. This function is for symmetric to Packer""" + * unpack = staticmethod(unpacks) + */ + +struct __pyx_obj_7msgpack_Unpacker { + PyObject_HEAD +}; +/* Module declarations from msgpack */ + +static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; +static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; +static int __pyx_v_7msgpack_BUFF_SIZE; +static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ +#define __Pyx_MODULE_NAME "msgpack" +int __pyx_module_is_main_msgpack = 0; + +/* Implementation of msgpack */ +static char __pyx_k___main__[] = "__main__"; +static PyObject *__pyx_kp___main__; +static char __pyx_k___init__[] = "__init__"; +static PyObject *__pyx_kp___init__; +static char __pyx_k_flush[] = "flush"; +static PyObject *__pyx_kp_flush; +static char __pyx_k_pack_list[] = "pack_list"; +static PyObject *__pyx_kp_pack_list; +static char __pyx_k_pack_dict[] = "pack_dict"; +static PyObject *__pyx_kp_pack_dict; +static char __pyx_k_pack[] = "pack"; +static PyObject *__pyx_kp_pack; +static char __pyx_k_unpack[] = "unpack"; +static PyObject *__pyx_kp_unpack; +static char __pyx_k_strm[] = "strm"; +static PyObject *__pyx_kp_strm; +static char __pyx_k_size[] = "size"; +static PyObject *__pyx_kp_size; +static char __pyx_k_len[] = "len"; +static PyObject *__pyx_kp_len; +static char __pyx_k_o[] = "o"; +static PyObject *__pyx_kp_o; +static char __pyx_k_stream[] = "stream"; +static PyObject *__pyx_kp_stream; +static char __pyx_k_packed_bytes[] = "packed_bytes"; +static PyObject *__pyx_kp_packed_bytes; +static char __pyx_k_cStringIO[] = "cStringIO"; +static PyObject *__pyx_kp_cStringIO; +static char __pyx_k_StringIO[] = "StringIO"; +static PyObject *__pyx_kp_StringIO; +static char __pyx_k_staticmethod[] = "staticmethod"; +static PyObject *__pyx_kp_staticmethod; +static char __pyx_k_unpacks[] = "unpacks"; +static PyObject *__pyx_kp_unpacks; +static char __pyx_k_write[] = "write"; +static PyObject *__pyx_kp_write; +static char __pyx_k_1[] = "flush"; +static PyObject *__pyx_kp_1; +static char __pyx_k_encode[] = "encode"; +static PyObject *__pyx_kp_encode; +static char __pyx_k_iteritems[] = "iteritems"; +static PyObject *__pyx_kp_iteritems; +static char __pyx_k_TypeError[] = "TypeError"; +static PyObject *__pyx_kp_TypeError; +static char __pyx_k_getvalue[] = "getvalue"; +static PyObject *__pyx_kp_getvalue; +static char __pyx_k_read[] = "read"; +static PyObject *__pyx_kp_read; +static PyObject *__pyx_builtin_staticmethod; +static PyObject *__pyx_builtin_TypeError; +static PyObject *__pyx_kp_2; +static PyObject *__pyx_kp_3; +static char __pyx_k_2[] = "utf-8"; +static char __pyx_k_3[] = "can't serialize %r"; + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":51 + * cdef object strm + * + * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< + * if size <= 0: + * size = BUFF_SIZE + */ + +static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_strm = 0; + int __pyx_v_size; + int __pyx_r; + int __pyx_t_1; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_strm,&__pyx_kp_size,0}; + __Pyx_SetupRefcountContext("__init__"); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_strm); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 1) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_size); + if (unlikely(value)) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "__init__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_strm = values[0]; + if (values[1]) { + __pyx_v_size = __Pyx_PyInt_AsInt(values[1]); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } else { + __pyx_v_size = 0; + } + } else { + __pyx_v_size = 0; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: __pyx_v_size = __Pyx_PyInt_AsInt(PyTuple_GET_ITEM(__pyx_args, 1)); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + case 1: __pyx_v_strm = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.Packer.__init__"); + return -1; + __pyx_L4_argument_unpacking_done:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":52 + * + * def __init__(self, strm, int size=0): + * if size <= 0: # <<<<<<<<<<<<<< + * size = BUFF_SIZE + * + */ + __pyx_t_1 = (__pyx_v_size <= 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":53 + * def __init__(self, strm, int size=0): + * if size <= 0: + * size = BUFF_SIZE # <<<<<<<<<<<<<< + * + * self.strm = strm + */ + __pyx_v_size = __pyx_v_7msgpack_BUFF_SIZE; + goto __pyx_L6; + } + __pyx_L6:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":55 + * size = BUFF_SIZE + * + * self.strm = strm # <<<<<<<<<<<<<< + * self.buff = malloc(size) + * self.allocated = size + */ + __Pyx_INCREF(__pyx_v_strm); + __Pyx_GIVEREF(__pyx_v_strm); + __Pyx_GOTREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); + __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":56 + * + * self.strm = strm + * self.buff = malloc(size) # <<<<<<<<<<<<<< + * self.allocated = size + * self.length = 0 + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":57 + * self.strm = strm + * self.buff = malloc(size) + * self.allocated = size # <<<<<<<<<<<<<< + * self.length = 0 + * + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":58 + * self.buff = malloc(size) + * self.allocated = size + * self.length = 0 # <<<<<<<<<<<<<< + * + * msgpack_packer_init(&self.pk, self, _packer_write) + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":60 + * self.length = 0 + * + * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< + * + * + */ + msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); + + __pyx_r = 0; + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":63 + * + * + * def flush(self): # <<<<<<<<<<<<<< + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_flush[] = "Flash local buffer and output stream if it has 'flush()' method."; +static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused) { + PyObject *__pyx_r = NULL; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_SetupRefcountContext("flush"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":65 + * def flush(self): + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: # <<<<<<<<<<<<<< + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 + */ + __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":66 + * """Flash local buffer and output stream if it has 'flush()' method.""" + * if self.length > 0: + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< + * self.length = 0 + * if hasattr(self.strm, 'flush'): + */ + __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":67 + * if self.length > 0: + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 # <<<<<<<<<<<<<< + * if hasattr(self.strm, 'flush'): + * self.strm.flush() + */ + ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; + goto __pyx_L5; + } + __pyx_L5:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":68 + * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) + * self.length = 0 + * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< + * self.strm.flush() + * + */ + __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_1); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":69 + * self.length = 0 + * if hasattr(self.strm, 'flush'): + * self.strm.flush() # <<<<<<<<<<<<<< + * + * def pack_list(self, len): + */ + __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + goto __pyx_L6; + } + __pyx_L6:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("msgpack.Packer.flush"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":71 + * self.strm.flush() + * + * def pack_list(self, len): # <<<<<<<<<<<<<< + * """Start packing sequential objects. + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_pack_list[] = "Start packing sequential objects.\n\n Example:\n\n packer.pack_list(2)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack(['foo', 'bar'])\n "; +static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { + PyObject *__pyx_r = NULL; + size_t __pyx_t_1; + __Pyx_SetupRefcountContext("pack_list"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":84 + * packer.pack(['foo', 'bar']) + * """ + * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< + * + * def pack_dict(self, len): + */ + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.Packer.pack_list"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":86 + * msgpack_pack_array(&self.pk, len) + * + * def pack_dict(self, len): # <<<<<<<<<<<<<< + * """Start packing key-value objects. + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ +static char __pyx_doc_7msgpack_6Packer_pack_dict[] = "Start packing key-value objects.\n\n Example:\n\n packer.pack_dict(1)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack({'foo', 'bar'})\n "; +static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { + PyObject *__pyx_r = NULL; + size_t __pyx_t_1; + __Pyx_SetupRefcountContext("pack_dict"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":99 + * packer.pack({'foo', 'bar'}) + * """ + * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< + * + * def pack(self, object o): + */ + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.Packer.pack_dict"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":101 + * msgpack_pack_map(&self.pk, len) + * + * def pack(self, object o): # <<<<<<<<<<<<<< + * cdef long long intval + * cdef double fval + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o) { + PY_LONG_LONG __pyx_v_intval; + double __pyx_v_fval; + char *__pyx_v_rawval; + PyObject *__pyx_v_k; + PyObject *__pyx_v_v; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_2 = 0; + PyObject *__pyx_3 = 0; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PY_LONG_LONG __pyx_t_3; + char *__pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + int __pyx_t_10; + __Pyx_SetupRefcountContext("pack"); + __Pyx_INCREF(__pyx_v_o); + __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); + __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":106 + * cdef char* rawval + * + * if o is None: # <<<<<<<<<<<<<< + * msgpack_pack_nil(&self.pk) + * elif o is True: + */ + __pyx_t_1 = (__pyx_v_o == Py_None); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":107 + * + * if o is None: + * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< + * elif o is True: + * msgpack_pack_true(&self.pk) + */ + msgpack_pack_nil((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":108 + * if o is None: + * msgpack_pack_nil(&self.pk) + * elif o is True: # <<<<<<<<<<<<<< + * msgpack_pack_true(&self.pk) + * elif o is False: + */ + __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = (__pyx_v_o == __pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":109 + * msgpack_pack_nil(&self.pk) + * elif o is True: + * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< + * elif o is False: + * msgpack_pack_false(&self.pk) + */ + msgpack_pack_true((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":110 + * elif o is True: + * msgpack_pack_true(&self.pk) + * elif o is False: # <<<<<<<<<<<<<< + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): + */ + __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = (__pyx_v_o == __pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":111 + * msgpack_pack_true(&self.pk) + * elif o is False: + * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< + * elif isinstance(o, long): + * intval = o + */ + msgpack_pack_false((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":112 + * elif o is False: + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): # <<<<<<<<<<<<<< + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":113 + * msgpack_pack_false(&self.pk) + * elif isinstance(o, long): + * intval = o # <<<<<<<<<<<<<< + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): + */ + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_intval = __pyx_t_3; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":114 + * elif isinstance(o, long): + * intval = o + * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< + * elif isinstance(o, int): + * intval = o + */ + msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":115 + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): # <<<<<<<<<<<<<< + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":116 + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, int): + * intval = o # <<<<<<<<<<<<<< + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): + */ + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_intval = __pyx_t_3; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":117 + * elif isinstance(o, int): + * intval = o + * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< + * elif isinstance(o, float): + * fval = 9 + */ + msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":118 + * intval = o + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): # <<<<<<<<<<<<<< + * fval = 9 + * msgpack_pack_double(&self.pk, fval) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":119 + * msgpack_pack_long_long(&self.pk, intval) + * elif isinstance(o, float): + * fval = 9 # <<<<<<<<<<<<<< + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): + */ + __pyx_v_fval = 9; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":120 + * elif isinstance(o, float): + * fval = 9 + * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< + * elif isinstance(o, str): + * rawval = o + */ + msgpack_pack_double((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_fval); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":121 + * fval = 9 + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): # <<<<<<<<<<<<<< + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":122 + * msgpack_pack_double(&self.pk, fval) + * elif isinstance(o, str): + * rawval = o # <<<<<<<<<<<<<< + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + */ + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_4; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":123 + * elif isinstance(o, str): + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":124 + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< + * elif isinstance(o, unicode): + * o = o.encode('utf-8') + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":125 + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): # <<<<<<<<<<<<<< + * o = o.encode('utf-8') + * rawval = o + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":126 + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, unicode): + * o = o.encode('utf-8') # <<<<<<<<<<<<<< + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_6)); + __Pyx_INCREF(__pyx_kp_2); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_2); + __Pyx_GIVEREF(__pyx_kp_2); + __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_v_o); + __pyx_v_o = __pyx_t_7; + __pyx_t_7 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":127 + * elif isinstance(o, unicode): + * o = o.encode('utf-8') + * rawval = o # <<<<<<<<<<<<<< + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + */ + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_4; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":128 + * o = o.encode('utf-8') + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":129 + * rawval = o + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":130 + * msgpack_pack_raw(&self.pk, len(o)) + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): # <<<<<<<<<<<<<< + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":131 + * msgpack_pack_raw_body(&self.pk, rawval, len(o)) + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< + * for k,v in o.iteritems(): + * self.pack(k) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":132 + * elif isinstance(o, dict): + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): # <<<<<<<<<<<<<< + * self.pack(k) + * self.pack(v) + */ + __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { + __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); + } else { + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + for (;;) { + if (likely(PyList_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else { + __pyx_t_6 = PyIter_Next(__pyx_t_7); + if (!__pyx_t_6) { + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + break; + } + __Pyx_GOTREF(__pyx_t_6); + } + if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { + PyObject* tuple = __pyx_t_6; + __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); + __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_v_k); + __pyx_v_k = __pyx_2; + __pyx_2 = 0; + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_3; + __pyx_3 = 0; + } else { + __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_2); + __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_3); + if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(__pyx_v_k); + __pyx_v_k = __pyx_2; + __pyx_2 = 0; + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_3; + __pyx_3 = 0; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":133 + * msgpack_pack_map(&self.pk, len(o)) + * for k,v in o.iteritems(): + * self.pack(k) # <<<<<<<<<<<<<< + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): + */ + __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_k); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); + __Pyx_GIVEREF(__pyx_v_k); + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":134 + * for k,v in o.iteritems(): + * self.pack(k) + * self.pack(v) # <<<<<<<<<<<<<< + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) + */ + __pyx_t_8 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_v); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); + __Pyx_GIVEREF(__pyx_v_v); + __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L5; + } + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":135 + * self.pack(k) + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: + */ + __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); + if (!__pyx_t_1) { + __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); + __pyx_t_10 = __pyx_t_9; + } else { + __pyx_t_10 = __pyx_t_1; + } + if (__pyx_t_10) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":136 + * self.pack(v) + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< + * for v in o: + * self.pack(v) + */ + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":137 + * elif isinstance(o, tuple) or isinstance(o, list): + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: # <<<<<<<<<<<<<< + * self.pack(v) + * else: + */ + if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { + __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); + } else { + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + } + for (;;) { + if (likely(PyList_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { + if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; + __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + } else { + __pyx_t_6 = PyIter_Next(__pyx_t_7); + if (!__pyx_t_6) { + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + break; + } + __Pyx_GOTREF(__pyx_t_6); + } + __Pyx_DECREF(__pyx_v_v); + __pyx_v_v = __pyx_t_6; + __pyx_t_6 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":138 + * msgpack_pack_array(&self.pk, len(o)) + * for v in o: + * self.pack(v) # <<<<<<<<<<<<<< + * else: + * # TODO: Serialize with defalt() like simplejson. + */ + __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_v); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); + __Pyx_GIVEREF(__pyx_v_v); + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L5; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":141 + * else: + * # TODO: Serialize with defalt() like simplejson. + * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + */ + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_7)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_8 = PyNumber_Remainder(__pyx_kp_3, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; + __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } + __pyx_L5:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_2); + __Pyx_XDECREF(__pyx_3); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("msgpack.Packer.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_k); + __Pyx_DECREF(__pyx_v_v); + __Pyx_DECREF(__pyx_v_o); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":143 + * raise TypeError, "can't serialize %r" % (o,) + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< + * if packer.length + l > packer.allocated: + * if packer.length > 0: + */ + +static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__pyx_v_packer, const char* __pyx_v_b, unsigned int __pyx_v_l) { + int __pyx_r; + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_SetupRefcountContext("_packer_write"); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":144 + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + */ + __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":145 + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * if packer.length + l > packer.allocated: + * if packer.length > 0: # <<<<<<<<<<<<<< + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: + */ + __pyx_t_1 = (__pyx_v_packer->length > 0); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":146 + * if packer.length + l > packer.allocated: + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L4; + } + __pyx_L4:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":147 + * if packer.length > 0: + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: # <<<<<<<<<<<<<< + * packer.strm.write(PyString_FromStringAndSize(b, l)) + * packer.length = 0 + */ + __pyx_t_1 = (__pyx_v_l > 64); + if (__pyx_t_1) { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":148 + * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< + * packer.length = 0 + * else: + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":149 + * if l > 64: + * packer.strm.write(PyString_FromStringAndSize(b, l)) + * packer.length = 0 # <<<<<<<<<<<<<< + * else: + * memcpy(packer.buff, b, l) + */ + __pyx_v_packer->length = 0; + goto __pyx_L5; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":151 + * packer.length = 0 + * else: + * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< + * packer.length = l + * else: + */ + memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":152 + * else: + * memcpy(packer.buff, b, l) + * packer.length = l # <<<<<<<<<<<<<< + * else: + * memcpy(packer.buff + packer.length, b, l) + */ + __pyx_v_packer->length = __pyx_v_l; + } + __pyx_L5:; + goto __pyx_L3; + } + /*else*/ { + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":154 + * packer.length = l + * else: + * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< + * packer.length += l + * return 0 + */ + memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":155 + * else: + * memcpy(packer.buff + packer.length, b, l) + * packer.length += l # <<<<<<<<<<<<<< + * return 0 + * + */ + __pyx_v_packer->length += __pyx_v_l; + } + __pyx_L3:; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":156 + * memcpy(packer.buff + packer.length, b, l) + * packer.length += l + * return 0 # <<<<<<<<<<<<<< + * + * def pack(object o, object stream): + */ + __pyx_r = 0; + goto __pyx_L0; + + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_WriteUnraisable("msgpack._packer_write"); + __pyx_r = 0; + __pyx_L0:; + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":158 + * return 0 + * + * def pack(object o, object stream): # <<<<<<<<<<<<<< + * packer = Packer(stream) + * packer.pack(o) + */ + +static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_o = 0; + PyObject *__pyx_v_stream = 0; + PyObject *__pyx_v_packer; + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_o,&__pyx_kp_stream,0}; + __Pyx_SetupRefcountContext("pack"); + __pyx_self = __pyx_self; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_o); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); + if (likely(values[1])) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_o = values[0]; + __pyx_v_stream = values[1]; + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + __pyx_v_o = PyTuple_GET_ITEM(__pyx_args, 0); + __pyx_v_stream = PyTuple_GET_ITEM(__pyx_args, 1); + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.pack"); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":159 + * + * def pack(object o, object stream): + * packer = Packer(stream) # <<<<<<<<<<<<<< + * packer.pack(o) + * packer.flush() + */ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_stream); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); + __Pyx_GIVEREF(__pyx_v_stream); + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packer); + __pyx_v_packer = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":160 + * def pack(object o, object stream): + * packer = Packer(stream) + * packer.pack(o) # <<<<<<<<<<<<<< + * packer.flush() + * + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":161 + * packer = Packer(stream) + * packer.pack(o) + * packer.flush() # <<<<<<<<<<<<<< + * + * def packs(object o): + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_packer); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":163 + * packer.flush() + * + * def packs(object o): # <<<<<<<<<<<<<< + * buf = StringIO() + * packer = Packer(buf) + */ + +static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o); /*proto*/ +static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o) { + PyObject *__pyx_v_buf; + PyObject *__pyx_v_packer; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_SetupRefcountContext("packs"); + __pyx_self = __pyx_self; + __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); + __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":164 + * + * def packs(object o): + * buf = StringIO() # <<<<<<<<<<<<<< + * packer = Packer(buf) + * packer.pack(o) + */ + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(__pyx_v_buf); + __pyx_v_buf = __pyx_t_1; + __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":165 + * def packs(object o): + * buf = StringIO() + * packer = Packer(buf) # <<<<<<<<<<<<<< + * packer.pack(o) + * packer.flush() + */ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_buf); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); + __Pyx_GIVEREF(__pyx_v_buf); + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packer); + __pyx_v_packer = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":166 + * buf = StringIO() + * packer = Packer(buf) + * packer.pack(o) # <<<<<<<<<<<<<< + * packer.flush() + * return buf.getvalue() + */ + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); + __Pyx_GIVEREF(__pyx_v_o); + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":167 + * packer = Packer(buf) + * packer.pack(o) + * packer.flush() # <<<<<<<<<<<<<< + * return buf.getvalue() + * + */ + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":168 + * packer.pack(o) + * packer.flush() + * return buf.getvalue() # <<<<<<<<<<<<<< + * + * cdef extern from "unpack.h": + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.packs"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_buf); + __Pyx_DECREF(__pyx_v_packer); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":179 + * + * + * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< + * """Unpack packed_bytes to object. Returns unpacked object.""" + * cdef const_char_ptr p = packed_bytes + */ + +static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes); /*proto*/ +static char __pyx_doc_7msgpack_unpacks[] = "Unpack packed_bytes to object. Returns unpacked object."; +static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes) { + const char* __pyx_v_p; + template_context __pyx_v_ctx; + size_t __pyx_v_off; + PyObject *__pyx_r = NULL; + const char* __pyx_t_1; + Py_ssize_t __pyx_t_2; + PyObject *__pyx_t_3; + __Pyx_SetupRefcountContext("unpacks"); + __pyx_self = __pyx_self; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":181 + * def unpacks(object packed_bytes): + * """Unpack packed_bytes to object. Returns unpacked object.""" + * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< + * cdef template_context ctx + * cdef size_t off = 0 + */ + __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 181; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_p = __pyx_t_1; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":183 + * cdef const_char_ptr p = packed_bytes + * cdef template_context ctx + * cdef size_t off = 0 # <<<<<<<<<<<<<< + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) + */ + __pyx_v_off = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":184 + * cdef template_context ctx + * cdef size_t off = 0 + * template_init(&ctx) # <<<<<<<<<<<<<< + * template_execute(&ctx, p, len(packed_bytes), &off) + * return template_data(&ctx) + */ + template_init((&__pyx_v_ctx)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":185 + * cdef size_t off = 0 + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< + * return template_data(&ctx) + * + */ + __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 + * template_init(&ctx) + * template_execute(&ctx, p, len(packed_bytes), &off) + * return template_data(&ctx) # <<<<<<<<<<<<<< + * + * def unpack(object stream): + */ + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = template_data((&__pyx_v_ctx)); + __Pyx_INCREF(((PyObject *)__pyx_t_3)); + __pyx_r = ((PyObject *)__pyx_t_3); + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("msgpack.unpacks"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 + * return template_data(&ctx) + * + * def unpack(object stream): # <<<<<<<<<<<<<< + * """unpack from stream.""" + * packed = stream.read() + */ + +static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream); /*proto*/ +static char __pyx_doc_7msgpack_unpack[] = "unpack from stream."; +static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream) { + PyObject *__pyx_v_packed; + PyObject *__pyx_r = NULL; + PyObject *__pyx_1 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + __Pyx_SetupRefcountContext("unpack"); + __pyx_self = __pyx_self; + __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":190 + * def unpack(object stream): + * """unpack from stream.""" + * packed = stream.read() # <<<<<<<<<<<<<< + * return unpacks(packed) + * + */ + __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_v_packed); + __pyx_v_packed = __pyx_t_2; + __pyx_t_2 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":191 + * """unpack from stream.""" + * packed = stream.read() + * return unpacks(packed) # <<<<<<<<<<<<<< + * + * cdef class Unpacker: + */ + __Pyx_XDECREF(__pyx_r); + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_2)); + __Pyx_INCREF(__pyx_v_packed); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); + __Pyx_GIVEREF(__pyx_v_packed); + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("msgpack.unpack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(__pyx_v_packed); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_7msgpack_Packer *p; + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + p = ((struct __pyx_obj_7msgpack_Packer *)o); + p->strm = Py_None; Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_7msgpack_Packer(PyObject *o) { + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + Py_XDECREF(p->strm); + (*Py_TYPE(o)->tp_free)(o); +} + +static int __pyx_tp_traverse_7msgpack_Packer(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + if (p->strm) { + e = (*v)(p->strm, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { + struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; + PyObject* tmp; + tmp = ((PyObject*)p->strm); + p->strm = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { + {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, + {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, + {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_O, __Pyx_DOCSTR(0)}, + {0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number_Packer = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else + 0, /*nb_long*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence_Packer = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Packer = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_Packer = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +PyTypeObject __pyx_type_7msgpack_Packer = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("msgpack.Packer"), /*tp_name*/ + sizeof(struct __pyx_obj_7msgpack_Packer), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_7msgpack_Packer, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + &__pyx_tp_as_number_Packer, /*tp_as_number*/ + &__pyx_tp_as_sequence_Packer, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Packer, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_Packer, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __Pyx_DOCSTR("Packer that pack data into strm.\n\n strm must have `write(bytes)` method.\n size specifies local buffer size.\n "), /*tp_doc*/ + __pyx_tp_traverse_7msgpack_Packer, /*tp_traverse*/ + __pyx_tp_clear_7msgpack_Packer, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_7msgpack_Packer, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + __pyx_pf_7msgpack_6Packer___init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_7msgpack_Packer, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ +}; + +static PyObject *__pyx_tp_new_7msgpack_Unpacker(PyTypeObject *t, PyObject *a, PyObject *k) { + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + return o; +} + +static void __pyx_tp_dealloc_7msgpack_Unpacker(PyObject *o) { + (*Py_TYPE(o)->tp_free)(o); +} + +static struct PyMethodDef __pyx_methods_7msgpack_Unpacker[] = { + {0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number_Unpacker = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else + 0, /*nb_long*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence_Unpacker = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Unpacker = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer_Unpacker = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +PyTypeObject __pyx_type_7msgpack_Unpacker = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("msgpack.Unpacker"), /*tp_name*/ + sizeof(struct __pyx_obj_7msgpack_Unpacker), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_7msgpack_Unpacker, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + &__pyx_tp_as_number_Unpacker, /*tp_as_number*/ + &__pyx_tp_as_sequence_Unpacker, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Unpacker, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer_Unpacker, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ + __Pyx_DOCSTR("Do nothing. This function is for symmetric to Packer"), /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_7msgpack_Unpacker, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_7msgpack_Unpacker, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ +}; + +static struct PyMethodDef __pyx_methods[] = { + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("packs"), (PyCFunction)__pyx_pf_7msgpack_packs, METH_O, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("unpacks"), (PyCFunction)__pyx_pf_7msgpack_unpacks, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpacks)}, + {__Pyx_NAMESTR("unpack"), (PyCFunction)__pyx_pf_7msgpack_unpack, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpack)}, + {0, 0, 0, 0} +}; + +static void __pyx_init_filenames(void); /*proto*/ + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef __pyx_moduledef = { + PyModuleDef_HEAD_INIT, + __Pyx_NAMESTR("msgpack"), + 0, /* m_doc */ + -1, /* m_size */ + __pyx_methods /* m_methods */, + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, + {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, + {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, + {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, + {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, + {&__pyx_kp_pack, __pyx_k_pack, sizeof(__pyx_k_pack), 1, 1, 1}, + {&__pyx_kp_unpack, __pyx_k_unpack, sizeof(__pyx_k_unpack), 1, 1, 1}, + {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, + {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, + {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, + {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, + {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, + {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, + {&__pyx_kp_cStringIO, __pyx_k_cStringIO, sizeof(__pyx_k_cStringIO), 1, 1, 1}, + {&__pyx_kp_StringIO, __pyx_k_StringIO, sizeof(__pyx_k_StringIO), 1, 1, 1}, + {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, + {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, + {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, + {&__pyx_kp_1, __pyx_k_1, sizeof(__pyx_k_1), 0, 1, 0}, + {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, + {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, + {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, + {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, + {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, + {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 0}, + {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, + {0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_InitGlobals(void) { + if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + return 0; + __pyx_L1_error:; + return -1; +} + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC initmsgpack(void); /*proto*/ +PyMODINIT_FUNC initmsgpack(void) +#else +PyMODINIT_FUNC PyInit_msgpack(void); /*proto*/ +PyMODINIT_FUNC PyInit_msgpack(void) +#endif +{ + PyObject *__pyx_1 = 0; + PyObject *__pyx_2 = 0; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + #ifdef CYTHON_REFNANNY + void* __pyx_refchk = NULL; + __Pyx_Refnanny = __Pyx_ImportRefcountAPI("refnanny"); + if (!__Pyx_Refnanny) { + PyErr_Clear(); + __Pyx_Refnanny = __Pyx_ImportRefcountAPI("Cython.Runtime.refnanny"); + if (!__Pyx_Refnanny) + Py_FatalError("failed to import refnanny module"); + } + __pyx_refchk = __Pyx_Refnanny->NewContext("PyMODINIT_FUNC PyInit_msgpack(void)", __LINE__, __FILE__); + #endif + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + /*--- Library function declarations ---*/ + __pyx_init_filenames(); + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Initialize various global constants etc. ---*/ + if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + /*--- Module creation code ---*/ + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4(__Pyx_NAMESTR("msgpack"), __pyx_methods, 0, 0, PYTHON_API_VERSION); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (!__pyx_m) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + #if PY_MAJOR_VERSION < 3 + Py_INCREF(__pyx_m); + #endif + __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); + if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__pyx_module_is_main_msgpack) { + if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + } + /*--- Builtin init code ---*/ + if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_skip_dispatch = 0; + /*--- Global init code ---*/ + /*--- Function export code ---*/ + /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; + if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; + /*--- Type import code ---*/ + /*--- Function import code ---*/ + /*--- Execution code ---*/ + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":3 + * # coding: utf-8 + * + * from cStringIO import StringIO # <<<<<<<<<<<<<< + * + * cdef extern from "Python.h": + */ + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + __Pyx_INCREF(__pyx_kp_StringIO); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_kp_StringIO); + __Pyx_GIVEREF(__pyx_kp_StringIO); + __pyx_1 = __Pyx_Import(__pyx_kp_cStringIO, ((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + __pyx_2 = PyObject_GetAttr(__pyx_1, __pyx_kp_StringIO); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_2); + if (PyObject_SetAttr(__pyx_m, __pyx_kp_StringIO, __pyx_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_2); __pyx_2 = 0; + __Pyx_DECREF(__pyx_1); __pyx_1 = 0; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":37 + * + * + * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< + * + * cdef class Packer: + */ + __pyx_v_7msgpack_BUFF_SIZE = 2048; + + /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":195 + * cdef class Unpacker: + * """Do nothing. This function is for symmetric to Packer""" + * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< + */ + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_1); + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_1)); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); + __Pyx_GIVEREF(__pyx_1); + __pyx_1 = 0; + __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + PyType_Modified(__pyx_ptype_7msgpack_Unpacker); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_1); + __Pyx_XDECREF(__pyx_2); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("msgpack"); + Py_DECREF(__pyx_m); __pyx_m = 0; + __pyx_L0:; + __Pyx_FinishRefcountContext(); + #if PY_MAJOR_VERSION < 3 + return; + #else + return __pyx_m; + #endif +} + +static const char *__pyx_filenames[] = { + "msgpack.pyx", +}; + +/* Runtime support code */ + +static void __pyx_init_filenames(void) { + __pyx_f = __pyx_filenames; +} + +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AS_STRING(kw_name)); + #endif +} + +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *number, *more_or_less; + + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + number = (num_expected == 1) ? "" : "s"; + PyErr_Format(PyExc_TypeError, + #if PY_VERSION_HEX < 0x02050000 + "%s() takes %s %d positional argument%s (%d given)", + #else + "%s() takes %s %zd positional argument%s (%zd given)", + #endif + func_name, more_or_less, num_expected, number, num_found); +} + +static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + } else { + #if PY_MAJOR_VERSION < 3 + if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) { + #else + if (unlikely(!PyUnicode_CheckExact(key)) && unlikely(!PyUnicode_Check(key))) { + #endif + goto invalid_keyword_type; + } else { + for (name = first_kw_arg; *name; name++) { + #if PY_MAJOR_VERSION >= 3 + if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && + PyUnicode_Compare(**name, key) == 0) break; + #else + if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && + _PyString_Eq(**name, key)) break; + #endif + } + if (*name) { + values[name-argnames] = value; + } else { + /* unexpected keyword found */ + for (name=argnames; name != first_kw_arg; name++) { + if (**name == key) goto arg_passed_twice; + #if PY_MAJOR_VERSION >= 3 + if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && + PyUnicode_Compare(**name, key) == 0) goto arg_passed_twice; + #else + if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && + _PyString_Eq(**name, key)) goto arg_passed_twice; + #endif + } + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + } + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, **name); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%s() got an unexpected keyword argument '%s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list) { + PyObject *__import__ = 0; + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + __import__ = __Pyx_GetAttrString(__pyx_b, "__import__"); + if (!__import__) + goto bad; + if (from_list) + list = from_list; + else { + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; + } + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + module = PyObject_CallFunctionObjArgs(__import__, + name, global_dict, empty_dict, list, NULL); +bad: + Py_XDECREF(empty_list); + Py_XDECREF(__import__); + Py_XDECREF(empty_dict); + return module; +} + +static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) { + PyObject *result; + result = PyObject_GetAttr(dict, name); + if (!result) + PyErr_SetObject(PyExc_NameError, name); + return result; +} + +static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { + PyErr_Format(PyExc_ValueError, + #if PY_VERSION_HEX < 0x02050000 + "need more than %d value%s to unpack", (int)index, + #else + "need more than %zd value%s to unpack", index, + #endif + (index == 1) ? "" : "s"); +} + +static INLINE void __Pyx_RaiseTooManyValuesError(void) { + PyErr_SetString(PyExc_ValueError, "too many values to unpack"); +} + +static PyObject *__Pyx_UnpackItem(PyObject *iter, Py_ssize_t index) { + PyObject *item; + if (!(item = PyIter_Next(iter))) { + if (!PyErr_Occurred()) { + __Pyx_RaiseNeedMoreValuesError(index); + } + } + return item; +} + +static int __Pyx_EndUnpack(PyObject *iter) { + PyObject *item; + if ((item = PyIter_Next(iter))) { + Py_DECREF(item); + __Pyx_RaiseTooManyValuesError(); + return -1; + } + else if (!PyErr_Occurred()) + return 0; + else + return -1; +} + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) { + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(tb); + /* First, check the traceback argument, replacing None with NULL. */ + if (tb == Py_None) { + Py_DECREF(tb); + tb = 0; + } + else if (tb != NULL && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + /* Next, replace a missing value with None */ + if (value == NULL) { + value = Py_None; + Py_INCREF(value); + } + #if PY_VERSION_HEX < 0x02050000 + if (!PyClass_Check(type)) + #else + if (!PyType_Check(type)) + #endif + { + /* Raising an instance. The value should be a dummy. */ + if (value != Py_None) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + /* Normalize to raise , */ + Py_DECREF(value); + value = type; + #if PY_VERSION_HEX < 0x02050000 + if (PyInstance_Check(type)) { + type = (PyObject*) ((PyInstanceObject*)type)->in_class; + Py_INCREF(type); + } + else { + type = 0; + PyErr_SetString(PyExc_TypeError, + "raise: exception must be an old-style class or instance"); + goto raise_error; + } + #else + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + #endif + } + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} + +static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyThreadState *tstate = PyThreadState_GET(); + +#if PY_MAJOR_VERSION >= 3 + /* Note: this is a temporary work-around to prevent crashes in Python 3.0 */ + if ((tstate->exc_type != NULL) & (tstate->exc_type != Py_None)) { + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + PyErr_NormalizeException(&type, &value, &tb); + PyErr_NormalizeException(&tmp_type, &tmp_value, &tmp_tb); + tstate->exc_type = 0; + tstate->exc_value = 0; + tstate->exc_traceback = 0; + PyException_SetContext(value, tmp_value); + Py_DECREF(tmp_type); + Py_XDECREF(tmp_tb); + } +#endif + + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} + +static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) { + PyThreadState *tstate = PyThreadState_GET(); + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} + + +static INLINE int __Pyx_StrEq(const char *s1, const char *s2) { + while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } + return *s1 == *s2; +} + +static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { + if (sizeof(unsigned char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned char)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned char"); + return (unsigned char)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned char"); + return (unsigned char)-1; + } + return (unsigned char)val; + } + return (unsigned char)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject* x) { + if (sizeof(unsigned short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned short)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned short"); + return (unsigned short)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned short"); + return (unsigned short)-1; + } + return (unsigned short)val; + } + return (unsigned short)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject* x) { + if (sizeof(unsigned int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(unsigned int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (unsigned int)-1; + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned int"); + return (unsigned int)-1; + } + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned int"); + return (unsigned int)-1; + } + return (unsigned int)val; + } + return (unsigned int)__Pyx_PyInt_AsUnsignedLong(x); +} + +static INLINE char __Pyx_PyInt_AsChar(PyObject* x) { + if (sizeof(char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (char)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char)-1; + } + return (char)val; + } + return (char)__Pyx_PyInt_AsLong(x); +} + +static INLINE short __Pyx_PyInt_AsShort(PyObject* x) { + if (sizeof(short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (short)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to short"); + return (short)-1; + } + return (short)val; + } + return (short)__Pyx_PyInt_AsLong(x); +} + +static INLINE int __Pyx_PyInt_AsInt(PyObject* x) { + if (sizeof(int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (int)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int)-1; + } + return (int)val; + } + return (int)__Pyx_PyInt_AsLong(x); +} + +static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject* x) { + if (sizeof(signed char) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed char)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed char)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed char"); + return (signed char)-1; + } + return (signed char)val; + } + return (signed char)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject* x) { + if (sizeof(signed short) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed short)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed short)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed short"); + return (signed short)-1; + } + return (signed short)val; + } + return (signed short)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject* x) { + if (sizeof(signed int) < sizeof(long)) { + long val = __Pyx_PyInt_AsLong(x); + if (unlikely(val != (long)(signed int)val)) { + if (unlikely(val == -1 && PyErr_Occurred())) + return (signed int)-1; + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to signed int"); + return (signed int)-1; + } + return (signed int)val; + } + return (signed int)__Pyx_PyInt_AsSignedLong(x); +} + +static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned long"); + return (unsigned long)-1; + } + return (unsigned long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned long"); + return (unsigned long)-1; + } + return PyLong_AsUnsignedLong(x); + } else { + unsigned long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (unsigned long)-1; + val = __Pyx_PyInt_AsUnsignedLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + if (unlikely(val < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned PY_LONG_LONG"); + return (unsigned PY_LONG_LONG)-1; + } + return (unsigned PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + if (unlikely(Py_SIZE(x) < 0)) { + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned PY_LONG_LONG"); + return (unsigned PY_LONG_LONG)-1; + } + return PyLong_AsUnsignedLongLong(x); + } else { + unsigned PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (unsigned PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsUnsignedLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE long __Pyx_PyInt_AsLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLong(x); + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (long)-1; + val = __Pyx_PyInt_AsLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLongLong(x); + } else { + PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (signed long)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLong(x); + } else { + signed long val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (signed long)-1; + val = __Pyx_PyInt_AsSignedLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { +#if PY_VERSION_HEX < 0x03000000 + if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { + long val = PyInt_AS_LONG(x); + return (signed PY_LONG_LONG)val; + } else +#endif + if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { + return PyLong_AsLongLong(x); + } else { + signed PY_LONG_LONG val; + PyObject *tmp = __Pyx_PyNumber_Int(x); + if (!tmp) return (signed PY_LONG_LONG)-1; + val = __Pyx_PyInt_AsSignedLongLong(tmp); + Py_DECREF(tmp); + return val; + } +} + +static void __Pyx_WriteUnraisable(const char *name) { + PyObject *old_exc, *old_val, *old_tb; + PyObject *ctx; + __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); + #if PY_MAJOR_VERSION < 3 + ctx = PyString_FromString(name); + #else + ctx = PyUnicode_FromString(name); + #endif + __Pyx_ErrRestore(old_exc, old_val, old_tb); + if (!ctx) { + PyErr_WriteUnraisable(Py_None); + } else { + PyErr_WriteUnraisable(ctx); + Py_DECREF(ctx); + } +} + +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" + +static void __Pyx_AddTraceback(const char *funcname) { + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + PyObject *py_globals = 0; + PyObject *empty_string = 0; + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(__pyx_filename); + #else + py_srcfile = PyUnicode_FromString(__pyx_filename); + #endif + if (!py_srcfile) goto bad; + if (__pyx_clineno) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_globals = PyModule_GetDict(__pyx_m); + if (!py_globals) goto bad; + #if PY_MAJOR_VERSION < 3 + empty_string = PyString_FromStringAndSize("", 0); + #else + empty_string = PyBytes_FromStringAndSize("", 0); + #endif + if (!empty_string) goto bad; + py_code = PyCode_New( + 0, /*int argcount,*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*int kwonlyargcount,*/ + #endif + 0, /*int nlocals,*/ + 0, /*int stacksize,*/ + 0, /*int flags,*/ + empty_string, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + __pyx_lineno, /*int firstlineno,*/ + empty_string /*PyObject *lnotab*/ + ); + if (!py_code) goto bad; + py_frame = PyFrame_New( + PyThreadState_GET(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + py_globals, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + py_frame->f_lineno = __pyx_lineno; + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + Py_XDECREF(empty_string); + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode && (!t->is_identifier)) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else /* Python 3+ has unicode identifiers */ + if (t->is_identifier || (t->is_unicode && t->intern)) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->is_unicode) { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + ++t; + } + return 0; +} + +/* Type Conversion Functions */ + +static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + if (x == Py_True) return 1; + else if ((x == Py_False) | (x == Py_None)) return 0; + else return PyObject_IsTrue(x); +} + +static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { + PyNumberMethods *m; + const char *name = NULL; + PyObject *res = NULL; +#if PY_VERSION_HEX < 0x03000000 + if (PyInt_Check(x) || PyLong_Check(x)) +#else + if (PyLong_Check(x)) +#endif + return Py_INCREF(x), x; + m = Py_TYPE(x)->tp_as_number; +#if PY_VERSION_HEX < 0x03000000 + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } +#else + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Long(x); + } +#endif + if (res) { +#if PY_VERSION_HEX < 0x03000000 + if (!PyInt_Check(res) && !PyLong_Check(res)) { +#else + if (!PyLong_Check(res)) { +#endif + PyErr_Format(PyExc_TypeError, + "__%s__ returned non-%s (type %.200s)", + name, name, Py_TYPE(res)->tp_name); + Py_DECREF(res); + return NULL; + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} + +static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject* x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} + +static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { +#if PY_VERSION_HEX < 0x02050000 + if (ival <= LONG_MAX) + return PyInt_FromLong((long)ival); + else { + unsigned char *bytes = (unsigned char *) &ival; + int one = 1; int little = (int)*(unsigned char*)&one; + return _PyLong_FromByteArray(bytes, sizeof(size_t), little, 0); + } +#else + return PyInt_FromSize_t(ival); +#endif +} + +static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { + unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); + if (unlikely(val == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())) { + return (size_t)-1; + } else if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t)-1; + } + return (size_t)val; +} + + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..8bfdf82 --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +from distutils.core import setup, Extension + +version = '0.0.1' + +msgpack_mod = Extension('msgpack', sources=['msgpack.c']) + +desc = 'MessagePack serializer/desirializer.' +long_desc = desc + """ + +Python binding of MessagePack_. + +This package is under development. + +.. _MessagePack: http://msgpack.sourceforge.jp/ + +What's MessagePack? (from http://msgpack.sourceforge.jp/) + + MessagePack is a binary-based efficient data interchange format that is + focused on high performance. It is like JSON, but very fast and small. +""" + +setup(name='msgpack', + author='Naoki INADA', + author_email='songofacandy@gmail.com', + version=version, + ext_modules=[msgpack_mod], + description='The MessagePack serializer/desirializer.', + long_description=long_desc, + ) From 17d2ca2d63b3eb8b9d6028c180dc00f1f10be51e Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 04:33:47 +0900 Subject: [PATCH 0141/1648] Fix unpacking True, False and True. --- python/unpack.h | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/python/unpack.h b/python/unpack.h index eb72001..e51557f 100644 --- a/python/unpack.h +++ b/python/unpack.h @@ -26,7 +26,7 @@ typedef struct { struct template ## name #define msgpack_unpack_func(ret, name) \ - ret template ## name + static inline ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name @@ -39,16 +39,8 @@ typedef struct { struct template_context; typedef struct template_context template_context; -static inline void template_init(template_context* ctx); - -static inline msgpack_unpack_object template_data(template_context* ctx); - -static inline int template_execute(template_context* ctx, - const char* data, size_t len, size_t* off); - - static inline msgpack_unpack_object template_callback_root(unpack_user* u) -{ PyObject *o = Py_None; Py_INCREF(o); return o; } +{ return NULL; } static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) { *o = PyInt_FromLong((long)d); return 0; } @@ -88,13 +80,13 @@ static inline int template_callback_double(unpack_user* u, double d, msgpack_unp { *o = PyFloat_FromDouble(d); return 0; } static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_None; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_None); *o = Py_None; return 0; } static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_True; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_True); *o = Py_True; return 0; } static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_False; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { From 9b5a25ec32f716715311f2d2386762a1fd5714b7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 04:33:47 +0900 Subject: [PATCH 0142/1648] Fix unpacking True, False and True. --- unpack.h | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/unpack.h b/unpack.h index eb72001..e51557f 100644 --- a/unpack.h +++ b/unpack.h @@ -26,7 +26,7 @@ typedef struct { struct template ## name #define msgpack_unpack_func(ret, name) \ - ret template ## name + static inline ret template ## name #define msgpack_unpack_callback(name) \ template_callback ## name @@ -39,16 +39,8 @@ typedef struct { struct template_context; typedef struct template_context template_context; -static inline void template_init(template_context* ctx); - -static inline msgpack_unpack_object template_data(template_context* ctx); - -static inline int template_execute(template_context* ctx, - const char* data, size_t len, size_t* off); - - static inline msgpack_unpack_object template_callback_root(unpack_user* u) -{ PyObject *o = Py_None; Py_INCREF(o); return o; } +{ return NULL; } static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) { *o = PyInt_FromLong((long)d); return 0; } @@ -88,13 +80,13 @@ static inline int template_callback_double(unpack_user* u, double d, msgpack_unp { *o = PyFloat_FromDouble(d); return 0; } static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_None; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_None); *o = Py_None; return 0; } static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_True; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_True); *o = Py_True; return 0; } static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) -{ *o = Py_False; Py_INCREF(o); return 0; } +{ Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { From 935db853f0d70576c84f908c69d809877796503b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 04:33:58 +0900 Subject: [PATCH 0143/1648] add test. --- python/testformat.py | 64 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 python/testformat.py diff --git a/python/testformat.py b/python/testformat.py new file mode 100644 index 0000000..a00123c --- /dev/null +++ b/python/testformat.py @@ -0,0 +1,64 @@ +from unittest import TestCase, main +from msgpack import packs, unpacks + +class TestFormat(TestCase): + def __check(self, obj, expected_packed): + packed = packs(obj) + self.assertEqual(packed, expected_packed) + unpacked = unpacks(packed) + self.assertEqual(unpacked, obj) + + def testSimpleValues(self): + self.__check(None, '\xc0') + self.__check(True, '\xc3') + self.__check(False, '\xc2') + self.__check( + [None, False, True], + '\x93\xc0\xc2\xc3' + ) + + def testFixnum(self): + self.__check( + [[0,64,127], [-32,-16,-1]], + "\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff" + ) + + def testFixArray(self): + self.__check( + [[],[[None]]], + "\x92\x90\x91\x91\xc0" + ) + + def testFixRaw(self): + self.__check( + ["", "a", "bc", "def"], + "\x94\xa0\xa1a\xa2bc\xa3def" + ) + pass + + def testFixMap(self): + self.__check( + {False: {None: None}, True:{None:{}}}, + "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80" + ) + pass + + +class TestUnpack(TestCase): + def __check(self, packed, obj): + self.assertEqual(unpacks(packed), obj) + + def testuint(self): + self.__check( + "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + "\xce\xff\xff\xff\xff", + [0, 128, 255, 0, 32768, 65535, 0, + 2147483648, 4294967295], + ) + + + + +if __name__ == '__main__': + main() From b0f42ca49eb4ad4770efe6587b7fe3b6f0e06da1 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 04:33:58 +0900 Subject: [PATCH 0144/1648] add test. --- testformat.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 testformat.py diff --git a/testformat.py b/testformat.py new file mode 100644 index 0000000..a00123c --- /dev/null +++ b/testformat.py @@ -0,0 +1,64 @@ +from unittest import TestCase, main +from msgpack import packs, unpacks + +class TestFormat(TestCase): + def __check(self, obj, expected_packed): + packed = packs(obj) + self.assertEqual(packed, expected_packed) + unpacked = unpacks(packed) + self.assertEqual(unpacked, obj) + + def testSimpleValues(self): + self.__check(None, '\xc0') + self.__check(True, '\xc3') + self.__check(False, '\xc2') + self.__check( + [None, False, True], + '\x93\xc0\xc2\xc3' + ) + + def testFixnum(self): + self.__check( + [[0,64,127], [-32,-16,-1]], + "\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff" + ) + + def testFixArray(self): + self.__check( + [[],[[None]]], + "\x92\x90\x91\x91\xc0" + ) + + def testFixRaw(self): + self.__check( + ["", "a", "bc", "def"], + "\x94\xa0\xa1a\xa2bc\xa3def" + ) + pass + + def testFixMap(self): + self.__check( + {False: {None: None}, True:{None:{}}}, + "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80" + ) + pass + + +class TestUnpack(TestCase): + def __check(self, packed, obj): + self.assertEqual(unpacks(packed), obj) + + def testuint(self): + self.__check( + "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + "\xce\xff\xff\xff\xff", + [0, 128, 255, 0, 32768, 65535, 0, + 2147483648, 4294967295], + ) + + + + +if __name__ == '__main__': + main() From 560bd901f83c9024ed88b738b80e14de337847c5 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 12:46:02 +0900 Subject: [PATCH 0145/1648] Fix double INCREF-ing when unpacking. --- python/msgpack.c | 20 +++++++++++--------- python/msgpack.pyx | 4 ++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/python/msgpack.c b/python/msgpack.c index 50ec6fc..e044f18 100644 --- a/python/msgpack.c +++ b/python/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Mon Jun 8 01:28:30 2009 */ +/* Generated by Cython 0.11.2 on Mon Jun 8 12:41:02 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -1724,7 +1724,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx PyObject *__pyx_r = NULL; const char* __pyx_t_1; Py_ssize_t __pyx_t_2; - PyObject *__pyx_t_3; + PyObject *__pyx_t_3 = NULL; __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; @@ -1752,7 +1752,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) + * return template_data(&ctx) */ template_init((&__pyx_v_ctx)); @@ -1760,7 +1760,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< - * return template_data(&ctx) + * return template_data(&ctx) * */ __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -1769,19 +1769,21 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) # <<<<<<<<<<<<<< + * return template_data(&ctx) # <<<<<<<<<<<<<< * * def unpack(object stream): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); - __Pyx_INCREF(((PyObject *)__pyx_t_3)); - __pyx_r = ((PyObject *)__pyx_t_3); + __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); __Pyx_AddTraceback("msgpack.unpacks"); __pyx_r = NULL; __pyx_L0:; @@ -1791,7 +1793,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx } /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 - * return template_data(&ctx) + * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< * """unpack from stream.""" diff --git a/python/msgpack.pyx b/python/msgpack.pyx index c454c5d..8b2006a 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -173,7 +173,7 @@ cdef extern from "unpack.h": int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) void template_init(template_context* ctx) - PyObject* template_data(template_context* ctx) + object template_data(template_context* ctx) def unpacks(object packed_bytes): @@ -183,7 +183,7 @@ def unpacks(object packed_bytes): cdef size_t off = 0 template_init(&ctx) template_execute(&ctx, p, len(packed_bytes), &off) - return template_data(&ctx) + return template_data(&ctx) def unpack(object stream): """unpack from stream.""" From 46fae3d893be1fd75195c71b1503352f02d7ef38 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 12:46:02 +0900 Subject: [PATCH 0146/1648] Fix double INCREF-ing when unpacking. --- msgpack.c | 20 +++++++++++--------- msgpack.pyx | 4 ++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/msgpack.c b/msgpack.c index 50ec6fc..e044f18 100644 --- a/msgpack.c +++ b/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Mon Jun 8 01:28:30 2009 */ +/* Generated by Cython 0.11.2 on Mon Jun 8 12:41:02 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -1724,7 +1724,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx PyObject *__pyx_r = NULL; const char* __pyx_t_1; Py_ssize_t __pyx_t_2; - PyObject *__pyx_t_3; + PyObject *__pyx_t_3 = NULL; __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; @@ -1752,7 +1752,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) + * return template_data(&ctx) */ template_init((&__pyx_v_ctx)); @@ -1760,7 +1760,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< - * return template_data(&ctx) + * return template_data(&ctx) * */ __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -1769,19 +1769,21 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) # <<<<<<<<<<<<<< + * return template_data(&ctx) # <<<<<<<<<<<<<< * * def unpack(object stream): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); - __Pyx_INCREF(((PyObject *)__pyx_t_3)); - __pyx_r = ((PyObject *)__pyx_t_3); + __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); __Pyx_AddTraceback("msgpack.unpacks"); __pyx_r = NULL; __pyx_L0:; @@ -1791,7 +1793,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx } /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 - * return template_data(&ctx) + * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< * """unpack from stream.""" diff --git a/msgpack.pyx b/msgpack.pyx index c454c5d..8b2006a 100644 --- a/msgpack.pyx +++ b/msgpack.pyx @@ -173,7 +173,7 @@ cdef extern from "unpack.h": int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) void template_init(template_context* ctx) - PyObject* template_data(template_context* ctx) + object template_data(template_context* ctx) def unpacks(object packed_bytes): @@ -183,7 +183,7 @@ def unpacks(object packed_bytes): cdef size_t off = 0 template_init(&ctx) template_execute(&ctx, p, len(packed_bytes), &off) - return template_data(&ctx) + return template_data(&ctx) def unpack(object stream): """unpack from stream.""" From 9c9393bff919fa4ad9e8ff64feaf28859a0c22e6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 13:21:38 +0900 Subject: [PATCH 0147/1648] Fix setup script doesn't work. --- python/MANIFEST | 8 ++++++++ python/setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 python/MANIFEST diff --git a/python/MANIFEST b/python/MANIFEST new file mode 100644 index 0000000..c7dd3a3 --- /dev/null +++ b/python/MANIFEST @@ -0,0 +1,8 @@ +msgpack.c +setup.py +pack.h +unpack.h +msgpack/pack_define.h +msgpack/pack_template.h +msgpack/unpack_define.h +msgpack/unpack_template.h diff --git a/python/setup.py b/python/setup.py index 8bfdf82..e5651a0 100644 --- a/python/setup.py +++ b/python/setup.py @@ -24,6 +24,6 @@ setup(name='msgpack', author_email='songofacandy@gmail.com', version=version, ext_modules=[msgpack_mod], - description='The MessagePack serializer/desirializer.', + description=desc, long_description=long_desc, ) From a06a5ce46f1b87a8684c347657a3a748da84bf58 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 8 Jun 2009 13:21:38 +0900 Subject: [PATCH 0148/1648] Fix setup script doesn't work. --- MANIFEST | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 MANIFEST diff --git a/MANIFEST b/MANIFEST new file mode 100644 index 0000000..c7dd3a3 --- /dev/null +++ b/MANIFEST @@ -0,0 +1,8 @@ +msgpack.c +setup.py +pack.h +unpack.h +msgpack/pack_define.h +msgpack/pack_template.h +msgpack/unpack_define.h +msgpack/unpack_template.h diff --git a/setup.py b/setup.py index 8bfdf82..e5651a0 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,6 @@ setup(name='msgpack', author_email='songofacandy@gmail.com', version=version, ext_modules=[msgpack_mod], - description='The MessagePack serializer/desirializer.', + description=desc, long_description=long_desc, ) From 85ca59411868ab84d874ad28e56213f5374e5ca8 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 9 Jun 2009 13:12:29 +0900 Subject: [PATCH 0149/1648] free buffer when packer deleted. --- python/msgpack.c | 653 +++++++++++++++++++++++++++++---------------- python/msgpack.pyx | 9 +- 2 files changed, 437 insertions(+), 225 deletions(-) diff --git a/python/msgpack.c b/python/msgpack.c index e044f18..821f65b 100644 --- a/python/msgpack.c +++ b/python/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Mon Jun 8 12:41:02 2009 */ +/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -317,13 +317,15 @@ static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); static void __Pyx_WriteUnraisable(const char *name); /*proto*/ +static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ + static void __Pyx_AddTraceback(const char *funcname); /*proto*/ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -333,6 +335,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ struct __pyx_obj_7msgpack_Packer { PyObject_HEAD + struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtab; char *buff; unsigned int length; unsigned int allocated; @@ -340,7 +343,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":193 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -351,11 +354,26 @@ struct __pyx_obj_7msgpack_Packer { struct __pyx_obj_7msgpack_Unpacker { PyObject_HEAD }; + + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 + * cdef int BUFF_SIZE=2*1024 + * + * cdef class Packer: # <<<<<<<<<<<<<< + * """Packer that pack data into strm. + * + */ + +struct __pyx_vtabstruct_7msgpack_Packer { + PyObject *(*__pack)(struct __pyx_obj_7msgpack_Packer *, PyObject *); +}; +static struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtabptr_7msgpack_Packer; /* Module declarations from msgpack */ static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; +static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ #define __Pyx_MODULE_NAME "msgpack" int __pyx_module_is_main_msgpack = 0; @@ -365,6 +383,8 @@ static char __pyx_k___main__[] = "__main__"; static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; +static char __pyx_k___del__[] = "__del__"; +static PyObject *__pyx_kp___del__; static char __pyx_k_flush[] = "flush"; static PyObject *__pyx_kp_flush; static char __pyx_k_pack_list[] = "pack_list"; @@ -381,6 +401,8 @@ static char __pyx_k_size[] = "size"; static PyObject *__pyx_kp_size; static char __pyx_k_len[] = "len"; static PyObject *__pyx_kp_len; +static char __pyx_k_obj[] = "obj"; +static PyObject *__pyx_kp_obj; static char __pyx_k_o[] = "o"; static PyObject *__pyx_kp_o; static char __pyx_k_stream[] = "stream"; @@ -397,8 +419,8 @@ static char __pyx_k_unpacks[] = "unpacks"; static PyObject *__pyx_kp_unpacks; static char __pyx_k_write[] = "write"; static PyObject *__pyx_kp_write; -static char __pyx_k_1[] = "flush"; -static PyObject *__pyx_kp_1; +static char __pyx_k_2[] = "flush"; +static PyObject *__pyx_kp_2; static char __pyx_k_encode[] = "encode"; static PyObject *__pyx_kp_encode; static char __pyx_k_iteritems[] = "iteritems"; @@ -411,12 +433,12 @@ static char __pyx_k_read[] = "read"; static PyObject *__pyx_kp_read; static PyObject *__pyx_builtin_staticmethod; static PyObject *__pyx_builtin_TypeError; -static PyObject *__pyx_kp_2; static PyObject *__pyx_kp_3; -static char __pyx_k_2[] = "utf-8"; -static char __pyx_k_3[] = "can't serialize %r"; +static PyObject *__pyx_kp_4; +static char __pyx_k_3[] = "utf-8"; +static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -478,7 +500,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -488,7 +510,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -500,7 +522,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -513,7 +535,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -522,7 +544,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -531,7 +553,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -540,12 +562,12 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< * - * + * def __del__(self): */ msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); @@ -554,8 +576,36 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":63 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 + * msgpack_packer_init(&self.pk, self, _packer_write) * + * def __del__(self): # <<<<<<<<<<<<<< + * free(self.buff); + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused) { + PyObject *__pyx_r = NULL; + __Pyx_SetupRefcountContext("__del__"); + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 + * + * def __del__(self): + * free(self.buff); # <<<<<<<<<<<<<< + * + * def flush(self): + */ + free(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 + * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< * """Flash local buffer and output stream if it has 'flush()' method.""" @@ -572,7 +622,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":65 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -582,29 +632,29 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":66 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< * self.length = 0 * if hasattr(self.strm, 'flush'): */ - __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_4)); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -616,26 +666,26 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< * self.strm.flush() * */ - __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_1); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< * * def pack_list(self, len): */ - __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -657,7 +707,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":71 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -672,14 +722,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":84 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< * * def pack_dict(self, len): */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); __pyx_r = Py_None; __Pyx_INCREF(Py_None); @@ -693,7 +743,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":86 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -708,14 +758,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":99 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< * - * def pack(self, object o): + * cdef __pack(self, object o): */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); __pyx_r = Py_None; __Pyx_INCREF(Py_None); @@ -729,16 +779,15 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":101 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * - * def pack(self, object o): # <<<<<<<<<<<<<< + * cdef __pack(self, object o): # <<<<<<<<<<<<<< * cdef long long intval * cdef double fval */ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o) { +static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Packer *__pyx_v_self, PyObject *__pyx_v_o) { PY_LONG_LONG __pyx_v_intval; double __pyx_v_fval; char *__pyx_v_rawval; @@ -758,12 +807,12 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject PyObject *__pyx_t_8 = NULL; int __pyx_t_9; int __pyx_t_10; - __Pyx_SetupRefcountContext("pack"); + __Pyx_SetupRefcountContext("__pack"); __Pyx_INCREF(__pyx_v_o); __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":106 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -773,66 +822,66 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":107 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< * elif o is True: * msgpack_pack_true(&self.pk) */ - msgpack_pack_nil((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_nil((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< * msgpack_pack_true(&self.pk) * elif o is False: */ - __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_1 = (__pyx_v_o == __pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< * elif o is False: * msgpack_pack_false(&self.pk) */ - msgpack_pack_true((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_true((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< * msgpack_pack_false(&self.pk) * elif isinstance(o, long): */ - __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_1 = (__pyx_v_o == __pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< * elif isinstance(o, long): * intval = o */ - msgpack_pack_false((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_false((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -842,28 +891,28 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, int): * intval = o */ - msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); - goto __pyx_L5; + msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -873,28 +922,28 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, float): * fval = 9 */ - msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); - goto __pyx_L5; + msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -904,7 +953,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = 9 # <<<<<<<<<<<<<< @@ -913,18 +962,18 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject */ __pyx_v_fval = 9; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = 9 * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< * elif isinstance(o, str): * rawval = o */ - msgpack_pack_double((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_fval); - goto __pyx_L5; + msgpack_pack_double((&__pyx_v_self->pk), __pyx_v_fval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 * fval = 9 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -934,39 +983,39 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_4; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< * elif isinstance(o, unicode): * o = o.encode('utf-8') */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L5; + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -976,21 +1025,21 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< * rawval = o * msgpack_pack_raw(&self.pk, len(o)) */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_6)); - __Pyx_INCREF(__pyx_kp_2); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_2); - __Pyx_GIVEREF(__pyx_kp_2); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_INCREF(__pyx_kp_3); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); + __Pyx_GIVEREF(__pyx_kp_3); + __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; @@ -998,39 +1047,39 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_o = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_4; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L5; + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1040,32 +1089,32 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< * for k,v in o.iteritems(): * self.pack(k) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< * self.pack(k) * self.pack(v) */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -1079,7 +1128,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } else { __pyx_t_6 = PyIter_Next(__pyx_t_7); if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } __Pyx_GOTREF(__pyx_t_6); @@ -1096,14 +1145,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_v = __pyx_3; __pyx_3 = 0; } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_2); - __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_3); - if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(__pyx_v_k); __pyx_v_k = __pyx_2; @@ -1113,51 +1162,51 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): */ - __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_k); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) */ - __pyx_t_8 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L5; + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1173,17 +1222,17 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } if (__pyx_t_10) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< * for v in o: * self.pack(v) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1193,7 +1242,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); } for (;;) { @@ -1206,7 +1255,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } else { __pyx_t_6 = PyIter_Next(__pyx_t_7); if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } __Pyx_GOTREF(__pyx_t_6); @@ -1215,51 +1264,51 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_v = __pyx_t_6; __pyx_t_6 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< * else: * # TODO: Serialize with defalt() like simplejson. */ - __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L5; + goto __pyx_L3; } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":141 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * def pack(self, obj, flush=True): */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_7)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_3, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } - __pyx_L5:; + __pyx_L3:; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; @@ -1271,8 +1320,8 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("msgpack.Packer.pack"); - __pyx_r = NULL; + __Pyx_AddTraceback("msgpack.Packer.__pack"); + __pyx_r = 0; __pyx_L0:; __Pyx_DECREF(__pyx_v_k); __Pyx_DECREF(__pyx_v_v); @@ -1282,9 +1331,121 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":143 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * + * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< + * self.__pack(obj) + * if flush: + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_obj = 0; + PyObject *__pyx_v_flush = 0; + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_obj,&__pyx_kp_flush,0}; + __Pyx_SetupRefcountContext("pack"); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + values[1] = __pyx_k_1; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_obj); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 1) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_flush); + if (unlikely(value)) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_obj = values[0]; + __pyx_v_flush = values[1]; + } else { + __pyx_v_flush = __pyx_k_1; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: __pyx_v_flush = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: __pyx_v_obj = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("pack", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.Packer.pack"); + return NULL; + __pyx_L4_argument_unpacking_done:; + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 + * + * def pack(self, obj, flush=True): + * self.__pack(obj) # <<<<<<<<<<<<<< + * if flush: + * self.flush() + */ + __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_Packer *)((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->__pyx_vtab)->__pack(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self), __pyx_v_obj); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 + * def pack(self, obj, flush=True): + * self.__pack(obj) + * if flush: # <<<<<<<<<<<<<< + * self.flush() + * + */ + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_2) { + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 + * self.__pack(obj) + * if flush: + * self.flush() # <<<<<<<<<<<<<< + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + */ + __pyx_t_1 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_flush); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L6; + } + __pyx_L6:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.Packer.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 + * self.flush() + * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< * if packer.length + l > packer.allocated: * if packer.length > 0: @@ -1298,7 +1459,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":144 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1308,7 +1469,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1318,23 +1479,23 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_4)); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; @@ -1343,7 +1504,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1353,29 +1514,29 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< * packer.length = 0 * else: */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":149 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1387,7 +1548,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1396,7 +1557,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1410,7 +1571,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1419,7 +1580,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1430,7 +1591,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1453,7 +1614,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":158 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1491,11 +1652,11 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); if (likely(values[1])) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } __pyx_v_o = values[0]; __pyx_v_stream = values[1]; @@ -1507,62 +1668,62 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("msgpack.pack"); return NULL; __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< * packer.pack(o) * packer.flush() */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_stream); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); __Pyx_GIVEREF(__pyx_v_stream); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packer); __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":160 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< * packer.flush() * */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< * * def packs(object o): */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -1582,7 +1743,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":163 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1604,76 +1765,76 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":164 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< * packer = Packer(buf) * packer.pack(o) */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(__pyx_v_buf); __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":165 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< * packer.pack(o) * packer.flush() */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_buf); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); __Pyx_GIVEREF(__pyx_v_buf); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packer); __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< * packer.flush() * return buf.getvalue() */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< * return buf.getvalue() * */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1681,9 +1842,9 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v * cdef extern from "unpack.h": */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; @@ -1707,7 +1868,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":179 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1728,17 +1889,17 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":181 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< * cdef template_context ctx * cdef size_t off = 0 */ - __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 181; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":183 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1747,7 +1908,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":184 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1756,17 +1917,17 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":185 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< * return template_data(&ctx) * */ - __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1774,7 +1935,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * def unpack(object stream): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __pyx_r = __pyx_t_3; __pyx_t_3 = 0; @@ -1792,7 +1953,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1812,23 +1973,23 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< * return unpacks(packed) * */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packed); __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -1836,14 +1997,14 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ * cdef class Unpacker: */ __Pyx_XDECREF(__pyx_r); - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_packed); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); __Pyx_GIVEREF(__pyx_v_packed); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; @@ -1865,12 +2026,14 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __Pyx_FinishRefcountContext(); return __pyx_r; } +static struct __pyx_vtabstruct_7msgpack_Packer __pyx_vtable_7msgpack_Packer; static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_7msgpack_Packer *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; p = ((struct __pyx_obj_7msgpack_Packer *)o); + p->__pyx_vtab = __pyx_vtabptr_7msgpack_Packer; p->strm = Py_None; Py_INCREF(Py_None); return o; } @@ -1900,10 +2063,11 @@ static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { } static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { + {__Pyx_NAMESTR("__del__"), (PyCFunction)__pyx_pf_7msgpack_6Packer___del__, METH_NOARGS, __Pyx_DOCSTR(0)}, {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_O, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, {0, 0, 0, 0} }; @@ -2240,6 +2404,7 @@ static struct PyModuleDef __pyx_moduledef = { static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, + {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, @@ -2248,6 +2413,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, + {&__pyx_kp_obj, __pyx_k_obj, sizeof(__pyx_k_obj), 1, 1, 1}, {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, @@ -2256,19 +2422,19 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, - {&__pyx_kp_1, __pyx_k_1, sizeof(__pyx_k_1), 0, 1, 0}, + {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 1, 0}, {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, - {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 0}, {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, + {&__pyx_kp_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 0}, {0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} return 0; __pyx_L1_error:; return -1; @@ -2337,17 +2503,24 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Global init code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ + __pyx_vtabptr_7msgpack_Packer = &__pyx_vtable_7msgpack_Packer; + #if PY_MAJOR_VERSION >= 3 + __pyx_vtable_7msgpack_Packer.__pack = (PyObject *(*)(struct __pyx_obj_7msgpack_Packer *, PyObject *))__pyx_f_7msgpack_6Packer___pack; + #else + *(void(**)(void))&__pyx_vtable_7msgpack_Packer.__pack = (void(*)(void))__pyx_f_7msgpack_6Packer___pack; + #endif if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetVtable(__pyx_type_7msgpack_Packer.tp_dict, __pyx_vtabptr_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; - if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; /*--- Type import code ---*/ /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2368,7 +2541,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2377,22 +2550,35 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":195 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 + * raise TypeError, "can't serialize %r" % (o,) + * + * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< + * self.__pack(obj) + * if flush: + */ + __pyx_t_1 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_k_1 = __pyx_t_1; + __pyx_t_1 = 0; + __Pyx_GIVEREF(__pyx_k_1); + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); __Pyx_GIVEREF(__pyx_1); __pyx_1 = 0; - __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; PyType_Modified(__pyx_ptype_7msgpack_Unpacker); goto __pyx_L0; @@ -3037,6 +3223,25 @@ static void __Pyx_WriteUnraisable(const char *name) { } } +static int __Pyx_SetVtable(PyObject *dict, void *vtable) { + PyObject *pycobj = 0; + int result; + + pycobj = PyCObject_FromVoidPtr(vtable, 0); + if (!pycobj) + goto bad; + if (PyDict_SetItemString(dict, "__pyx_vtable__", pycobj) < 0) + goto bad; + result = 0; + goto done; + +bad: + result = -1; +done: + Py_XDECREF(pycobj); + return result; +} + #include "compile.h" #include "frameobject.h" #include "traceback.h" diff --git a/python/msgpack.pyx b/python/msgpack.pyx index 8b2006a..f24f403 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -59,6 +59,8 @@ cdef class Packer: msgpack_packer_init(&self.pk, self, _packer_write) + def __del__(self): + free(self.buff); def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" @@ -98,7 +100,7 @@ cdef class Packer: """ msgpack_pack_map(&self.pk, len) - def pack(self, object o): + cdef __pack(self, object o): cdef long long intval cdef double fval cdef char* rawval @@ -140,6 +142,11 @@ cdef class Packer: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) + def pack(self, obj, flush=True): + self.__pack(obj) + if flush: + self.flush() + cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: From 85b68fbf8c33fb7daaaa1caafe50be8361c4b1d1 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 9 Jun 2009 13:12:29 +0900 Subject: [PATCH 0150/1648] free buffer when packer deleted. --- msgpack.c | 653 ++++++++++++++++++++++++++++++++++------------------ msgpack.pyx | 9 +- 2 files changed, 437 insertions(+), 225 deletions(-) diff --git a/msgpack.c b/msgpack.c index e044f18..821f65b 100644 --- a/msgpack.c +++ b/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Mon Jun 8 12:41:02 2009 */ +/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -317,13 +317,15 @@ static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); static void __Pyx_WriteUnraisable(const char *name); /*proto*/ +static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ + static void __Pyx_AddTraceback(const char *funcname); /*proto*/ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -333,6 +335,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ struct __pyx_obj_7msgpack_Packer { PyObject_HEAD + struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtab; char *buff; unsigned int length; unsigned int allocated; @@ -340,7 +343,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":193 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -351,11 +354,26 @@ struct __pyx_obj_7msgpack_Packer { struct __pyx_obj_7msgpack_Unpacker { PyObject_HEAD }; + + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 + * cdef int BUFF_SIZE=2*1024 + * + * cdef class Packer: # <<<<<<<<<<<<<< + * """Packer that pack data into strm. + * + */ + +struct __pyx_vtabstruct_7msgpack_Packer { + PyObject *(*__pack)(struct __pyx_obj_7msgpack_Packer *, PyObject *); +}; +static struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtabptr_7msgpack_Packer; /* Module declarations from msgpack */ static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; +static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ #define __Pyx_MODULE_NAME "msgpack" int __pyx_module_is_main_msgpack = 0; @@ -365,6 +383,8 @@ static char __pyx_k___main__[] = "__main__"; static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; +static char __pyx_k___del__[] = "__del__"; +static PyObject *__pyx_kp___del__; static char __pyx_k_flush[] = "flush"; static PyObject *__pyx_kp_flush; static char __pyx_k_pack_list[] = "pack_list"; @@ -381,6 +401,8 @@ static char __pyx_k_size[] = "size"; static PyObject *__pyx_kp_size; static char __pyx_k_len[] = "len"; static PyObject *__pyx_kp_len; +static char __pyx_k_obj[] = "obj"; +static PyObject *__pyx_kp_obj; static char __pyx_k_o[] = "o"; static PyObject *__pyx_kp_o; static char __pyx_k_stream[] = "stream"; @@ -397,8 +419,8 @@ static char __pyx_k_unpacks[] = "unpacks"; static PyObject *__pyx_kp_unpacks; static char __pyx_k_write[] = "write"; static PyObject *__pyx_kp_write; -static char __pyx_k_1[] = "flush"; -static PyObject *__pyx_kp_1; +static char __pyx_k_2[] = "flush"; +static PyObject *__pyx_kp_2; static char __pyx_k_encode[] = "encode"; static PyObject *__pyx_kp_encode; static char __pyx_k_iteritems[] = "iteritems"; @@ -411,12 +433,12 @@ static char __pyx_k_read[] = "read"; static PyObject *__pyx_kp_read; static PyObject *__pyx_builtin_staticmethod; static PyObject *__pyx_builtin_TypeError; -static PyObject *__pyx_kp_2; static PyObject *__pyx_kp_3; -static char __pyx_k_2[] = "utf-8"; -static char __pyx_k_3[] = "can't serialize %r"; +static PyObject *__pyx_kp_4; +static char __pyx_k_3[] = "utf-8"; +static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -478,7 +500,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -488,7 +510,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -500,7 +522,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -513,7 +535,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -522,7 +544,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -531,7 +553,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -540,12 +562,12 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< * - * + * def __del__(self): */ msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); @@ -554,8 +576,36 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":63 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 + * msgpack_packer_init(&self.pk, self, _packer_write) * + * def __del__(self): # <<<<<<<<<<<<<< + * free(self.buff); + * + */ + +static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused) { + PyObject *__pyx_r = NULL; + __Pyx_SetupRefcountContext("__del__"); + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 + * + * def __del__(self): + * free(self.buff); # <<<<<<<<<<<<<< + * + * def flush(self): + */ + free(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff); + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 + * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< * """Flash local buffer and output stream if it has 'flush()' method.""" @@ -572,7 +622,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":65 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -582,29 +632,29 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":66 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< * self.length = 0 * if hasattr(self.strm, 'flush'): */ - __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_4)); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -616,26 +666,26 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< * self.strm.flush() * */ - __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_1); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< * * def pack_list(self, len): */ - __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -657,7 +707,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":71 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -672,14 +722,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":84 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< * * def pack_dict(self, len): */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); __pyx_r = Py_None; __Pyx_INCREF(Py_None); @@ -693,7 +743,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":86 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -708,14 +758,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":99 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< * - * def pack(self, object o): + * cdef __pack(self, object o): */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); __pyx_r = Py_None; __Pyx_INCREF(Py_None); @@ -729,16 +779,15 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":101 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * - * def pack(self, object o): # <<<<<<<<<<<<<< + * cdef __pack(self, object o): # <<<<<<<<<<<<<< * cdef long long intval * cdef double fval */ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_v_o) { +static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Packer *__pyx_v_self, PyObject *__pyx_v_o) { PY_LONG_LONG __pyx_v_intval; double __pyx_v_fval; char *__pyx_v_rawval; @@ -758,12 +807,12 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject PyObject *__pyx_t_8 = NULL; int __pyx_t_9; int __pyx_t_10; - __Pyx_SetupRefcountContext("pack"); + __Pyx_SetupRefcountContext("__pack"); __Pyx_INCREF(__pyx_v_o); __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":106 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -773,66 +822,66 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":107 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< * elif o is True: * msgpack_pack_true(&self.pk) */ - msgpack_pack_nil((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_nil((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< * msgpack_pack_true(&self.pk) * elif o is False: */ - __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_1 = (__pyx_v_o == __pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< * elif o is False: * msgpack_pack_false(&self.pk) */ - msgpack_pack_true((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_true((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< * msgpack_pack_false(&self.pk) * elif isinstance(o, long): */ - __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __pyx_t_1 = (__pyx_v_o == __pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< * elif isinstance(o, long): * intval = o */ - msgpack_pack_false((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk)); - goto __pyx_L5; + msgpack_pack_false((&__pyx_v_self->pk)); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -842,28 +891,28 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, int): * intval = o */ - msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); - goto __pyx_L5; + msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -873,28 +922,28 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, float): * fval = 9 */ - msgpack_pack_long_long((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_intval); - goto __pyx_L5; + msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -904,7 +953,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = 9 # <<<<<<<<<<<<<< @@ -913,18 +962,18 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject */ __pyx_v_fval = 9; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = 9 * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< * elif isinstance(o, str): * rawval = o */ - msgpack_pack_double((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_fval); - goto __pyx_L5; + msgpack_pack_double((&__pyx_v_self->pk), __pyx_v_fval); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 * fval = 9 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -934,39 +983,39 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_4; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< * elif isinstance(o, unicode): * o = o.encode('utf-8') */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L5; + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -976,21 +1025,21 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< * rawval = o * msgpack_pack_raw(&self.pk, len(o)) */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_6)); - __Pyx_INCREF(__pyx_kp_2); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_2); - __Pyx_GIVEREF(__pyx_kp_2); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_INCREF(__pyx_kp_3); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); + __Pyx_GIVEREF(__pyx_kp_3); + __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; @@ -998,39 +1047,39 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_o = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_4; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L5; + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1040,32 +1089,32 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< * for k,v in o.iteritems(): * self.pack(k) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< * self.pack(k) * self.pack(v) */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -1079,7 +1128,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } else { __pyx_t_6 = PyIter_Next(__pyx_t_7); if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } __Pyx_GOTREF(__pyx_t_6); @@ -1096,14 +1145,14 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_v = __pyx_3; __pyx_3 = 0; } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_2); - __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_3); - if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(__pyx_v_k); __pyx_v_k = __pyx_2; @@ -1113,51 +1162,51 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): */ - __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_k); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) */ - __pyx_t_8 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L5; + goto __pyx_L3; } - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1173,17 +1222,17 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } if (__pyx_t_10) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< * for v in o: * self.pack(v) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_5); + __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1193,7 +1242,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); } for (;;) { @@ -1206,7 +1255,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject } else { __pyx_t_6 = PyIter_Next(__pyx_t_7); if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } __Pyx_GOTREF(__pyx_t_6); @@ -1215,51 +1264,51 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_v_v = __pyx_t_6; __pyx_t_6 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< * else: * # TODO: Serialize with defalt() like simplejson. */ - __pyx_t_6 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L5; + goto __pyx_L3; } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":141 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + * def pack(self, obj, flush=True): */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_7)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_3, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } - __pyx_L5:; + __pyx_L3:; __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; @@ -1271,8 +1320,8 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("msgpack.Packer.pack"); - __pyx_r = NULL; + __Pyx_AddTraceback("msgpack.Packer.__pack"); + __pyx_r = 0; __pyx_L0:; __Pyx_DECREF(__pyx_v_k); __Pyx_DECREF(__pyx_v_v); @@ -1282,9 +1331,121 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":143 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * + * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< + * self.__pack(obj) + * if flush: + */ + +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_obj = 0; + PyObject *__pyx_v_flush = 0; + PyObject *__pyx_r = NULL; + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + static PyObject **__pyx_pyargnames[] = {&__pyx_kp_obj,&__pyx_kp_flush,0}; + __Pyx_SetupRefcountContext("pack"); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); + PyObject* values[2] = {0,0}; + values[1] = __pyx_k_1; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 0: + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_obj); + if (likely(values[0])) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 1) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_flush); + if (unlikely(value)) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + __pyx_v_obj = values[0]; + __pyx_v_flush = values[1]; + } else { + __pyx_v_flush = __pyx_k_1; + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: __pyx_v_flush = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: __pyx_v_obj = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("pack", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_L3_error:; + __Pyx_AddTraceback("msgpack.Packer.pack"); + return NULL; + __pyx_L4_argument_unpacking_done:; + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 + * + * def pack(self, obj, flush=True): + * self.__pack(obj) # <<<<<<<<<<<<<< + * if flush: + * self.flush() + */ + __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_Packer *)((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->__pyx_vtab)->__pack(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self), __pyx_v_obj); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 + * def pack(self, obj, flush=True): + * self.__pack(obj) + * if flush: # <<<<<<<<<<<<<< + * self.flush() + * + */ + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_2) { + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 + * self.__pack(obj) + * if flush: + * self.flush() # <<<<<<<<<<<<<< + * + * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): + */ + __pyx_t_1 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_flush); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L6; + } + __pyx_L6:; + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("msgpack.Packer.pack"); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_FinishRefcountContext(); + return __pyx_r; +} + +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 + * self.flush() + * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< * if packer.length + l > packer.allocated: * if packer.length > 0: @@ -1298,7 +1459,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":144 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1308,7 +1469,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1318,23 +1479,23 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_4)); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; @@ -1343,7 +1504,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1353,29 +1514,29 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< * packer.length = 0 * else: */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":149 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1387,7 +1548,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1396,7 +1557,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1410,7 +1571,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1419,7 +1580,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1430,7 +1591,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1453,7 +1614,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":158 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1491,11 +1652,11 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); if (likely(values[1])) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } __pyx_v_o = values[0]; __pyx_v_stream = values[1]; @@ -1507,62 +1668,62 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 158; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("msgpack.pack"); return NULL; __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< * packer.pack(o) * packer.flush() */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_stream); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); __Pyx_GIVEREF(__pyx_v_stream); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packer); __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":160 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< * packer.flush() * */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 160; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< * * def packs(object o): */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 161; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -1582,7 +1743,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":163 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1604,76 +1765,76 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":164 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< * packer = Packer(buf) * packer.pack(o) */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(__pyx_v_buf); __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":165 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< * packer.pack(o) * packer.flush() */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_buf); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); __Pyx_GIVEREF(__pyx_v_buf); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packer); __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< * packer.flush() * return buf.getvalue() */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_INCREF(__pyx_v_o); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< * return buf.getvalue() * */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1681,9 +1842,9 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v * cdef extern from "unpack.h": */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; @@ -1707,7 +1868,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":179 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1728,17 +1889,17 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":181 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< * cdef template_context ctx * cdef size_t off = 0 */ - __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 181; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":183 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1747,7 +1908,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":184 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1756,17 +1917,17 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":185 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< * return template_data(&ctx) * */ - __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 185; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":186 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1774,7 +1935,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx * def unpack(object stream): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __pyx_r = __pyx_t_3; __pyx_t_3 = 0; @@ -1792,7 +1953,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":188 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1812,23 +1973,23 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< * return unpacks(packed) * */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_v_packed); __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -1836,14 +1997,14 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ * cdef class Unpacker: */ __Pyx_XDECREF(__pyx_r); - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_packed); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); __Pyx_GIVEREF(__pyx_v_packed); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_1); __pyx_1 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; @@ -1865,12 +2026,14 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __Pyx_FinishRefcountContext(); return __pyx_r; } +static struct __pyx_vtabstruct_7msgpack_Packer __pyx_vtable_7msgpack_Packer; static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_7msgpack_Packer *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; p = ((struct __pyx_obj_7msgpack_Packer *)o); + p->__pyx_vtab = __pyx_vtabptr_7msgpack_Packer; p->strm = Py_None; Py_INCREF(Py_None); return o; } @@ -1900,10 +2063,11 @@ static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { } static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { + {__Pyx_NAMESTR("__del__"), (PyCFunction)__pyx_pf_7msgpack_6Packer___del__, METH_NOARGS, __Pyx_DOCSTR(0)}, {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_O, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, {0, 0, 0, 0} }; @@ -2240,6 +2404,7 @@ static struct PyModuleDef __pyx_moduledef = { static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, + {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, @@ -2248,6 +2413,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, + {&__pyx_kp_obj, __pyx_k_obj, sizeof(__pyx_k_obj), 1, 1, 1}, {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, @@ -2256,19 +2422,19 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, - {&__pyx_kp_1, __pyx_k_1, sizeof(__pyx_k_1), 0, 1, 0}, + {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 1, 0}, {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, - {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 0}, {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, + {&__pyx_kp_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 0}, {0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} return 0; __pyx_L1_error:; return -1; @@ -2337,17 +2503,24 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Global init code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ + __pyx_vtabptr_7msgpack_Packer = &__pyx_vtable_7msgpack_Packer; + #if PY_MAJOR_VERSION >= 3 + __pyx_vtable_7msgpack_Packer.__pack = (PyObject *(*)(struct __pyx_obj_7msgpack_Packer *, PyObject *))__pyx_f_7msgpack_6Packer___pack; + #else + *(void(**)(void))&__pyx_vtable_7msgpack_Packer.__pack = (void(*)(void))__pyx_f_7msgpack_6Packer___pack; + #endif if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetVtable(__pyx_type_7msgpack_Packer.tp_dict, __pyx_vtabptr_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; - if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; /*--- Type import code ---*/ /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2368,7 +2541,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2377,22 +2550,35 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack.py/python/msgpack.pyx":195 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 + * raise TypeError, "can't serialize %r" % (o,) + * + * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< + * self.__pack(obj) + * if flush: + */ + __pyx_t_1 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_1); + __pyx_k_1 = __pyx_t_1; + __pyx_t_1 = 0; + __Pyx_GIVEREF(__pyx_k_1); + + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); __Pyx_GIVEREF(__pyx_1); __pyx_1 = 0; - __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 195; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; PyType_Modified(__pyx_ptype_7msgpack_Unpacker); goto __pyx_L0; @@ -3037,6 +3223,25 @@ static void __Pyx_WriteUnraisable(const char *name) { } } +static int __Pyx_SetVtable(PyObject *dict, void *vtable) { + PyObject *pycobj = 0; + int result; + + pycobj = PyCObject_FromVoidPtr(vtable, 0); + if (!pycobj) + goto bad; + if (PyDict_SetItemString(dict, "__pyx_vtable__", pycobj) < 0) + goto bad; + result = 0; + goto done; + +bad: + result = -1; +done: + Py_XDECREF(pycobj); + return result; +} + #include "compile.h" #include "frameobject.h" #include "traceback.h" diff --git a/msgpack.pyx b/msgpack.pyx index 8b2006a..f24f403 100644 --- a/msgpack.pyx +++ b/msgpack.pyx @@ -59,6 +59,8 @@ cdef class Packer: msgpack_packer_init(&self.pk, self, _packer_write) + def __del__(self): + free(self.buff); def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" @@ -98,7 +100,7 @@ cdef class Packer: """ msgpack_pack_map(&self.pk, len) - def pack(self, object o): + cdef __pack(self, object o): cdef long long intval cdef double fval cdef char* rawval @@ -140,6 +142,11 @@ cdef class Packer: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) + def pack(self, obj, flush=True): + self.__pack(obj) + if flush: + self.flush() + cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: From a1fb1507d45d27f0c08e8d39054987ed6dcaf9e7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 10:45:07 +0900 Subject: [PATCH 0151/1648] Refactor include path. --- python/MANIFEST | 8 ++++---- python/include | 1 + python/msgpack | 1 - python/setup.py | 7 ++++++- 4 files changed, 11 insertions(+), 6 deletions(-) create mode 120000 python/include delete mode 120000 python/msgpack diff --git a/python/MANIFEST b/python/MANIFEST index c7dd3a3..dc042ae 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -2,7 +2,7 @@ msgpack.c setup.py pack.h unpack.h -msgpack/pack_define.h -msgpack/pack_template.h -msgpack/unpack_define.h -msgpack/unpack_template.h +include/msgpack/pack_define.h +include/msgpack/pack_template.h +include/msgpack/unpack_define.h +include/msgpack/unpack_template.h diff --git a/python/include b/python/include new file mode 120000 index 0000000..a96aa0e --- /dev/null +++ b/python/include @@ -0,0 +1 @@ +.. \ No newline at end of file diff --git a/python/msgpack b/python/msgpack deleted file mode 120000 index 430db49..0000000 --- a/python/msgpack +++ /dev/null @@ -1 +0,0 @@ -../msgpack \ No newline at end of file diff --git a/python/setup.py b/python/setup.py index e5651a0..65ca412 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,8 +1,13 @@ from distutils.core import setup, Extension +import os version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c']) +PACKAGE_ROOT = os.getcwdu() +INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') +msgpack_mod = Extension('msgpack', + sources=['msgpack.c'], + include_dirs=[INCLUDE_PATH]) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ From fcc645b18bcd36b6240acdfd01ce4d193f1912f3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 10:45:07 +0900 Subject: [PATCH 0152/1648] Refactor include path. --- MANIFEST | 8 ++++---- include | 1 + msgpack | 1 - setup.py | 7 ++++++- 4 files changed, 11 insertions(+), 6 deletions(-) create mode 120000 include delete mode 120000 msgpack diff --git a/MANIFEST b/MANIFEST index c7dd3a3..dc042ae 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2,7 +2,7 @@ msgpack.c setup.py pack.h unpack.h -msgpack/pack_define.h -msgpack/pack_template.h -msgpack/unpack_define.h -msgpack/unpack_template.h +include/msgpack/pack_define.h +include/msgpack/pack_template.h +include/msgpack/unpack_define.h +include/msgpack/unpack_template.h diff --git a/include b/include new file mode 120000 index 0000000..a96aa0e --- /dev/null +++ b/include @@ -0,0 +1 @@ +.. \ No newline at end of file diff --git a/msgpack b/msgpack deleted file mode 120000 index 430db49..0000000 --- a/msgpack +++ /dev/null @@ -1 +0,0 @@ -../msgpack \ No newline at end of file diff --git a/setup.py b/setup.py index e5651a0..65ca412 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,13 @@ from distutils.core import setup, Extension +import os version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c']) +PACKAGE_ROOT = os.getcwdu() +INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') +msgpack_mod = Extension('msgpack', + sources=['msgpack.c'], + include_dirs=[INCLUDE_PATH]) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ From 3a9f74e79c3d1912d8c0c1ad4d1478c611caba0a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 10:58:09 +0900 Subject: [PATCH 0153/1648] Make msgpack package instead of module. --- python/MANIFEST | 4 +- python/msgpack.c | 3438 ------------------ python/msgpack/__init__.py | 3 + python/{msgpack.pyx => msgpack/_msgpack.pyx} | 0 python/{ => msgpack}/pack.h | 0 python/{ => msgpack}/unpack.h | 0 python/setup.py | 6 +- 7 files changed, 9 insertions(+), 3442 deletions(-) delete mode 100644 python/msgpack.c create mode 100644 python/msgpack/__init__.py rename python/{msgpack.pyx => msgpack/_msgpack.pyx} (100%) rename python/{ => msgpack}/pack.h (100%) rename python/{ => msgpack}/unpack.h (100%) diff --git a/python/MANIFEST b/python/MANIFEST index dc042ae..f2da7da 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -1,7 +1,7 @@ msgpack.c setup.py -pack.h -unpack.h +msgpack/pack.h +msgpack/unpack.h include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h diff --git a/python/msgpack.c b/python/msgpack.c deleted file mode 100644 index 821f65b..0000000 --- a/python/msgpack.c +++ /dev/null @@ -1,3438 +0,0 @@ -/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include "structmember.h" -#ifndef Py_PYTHON_H - #error Python headers needed to compile C extensions, please install development version of Python. -#endif -#ifndef PY_LONG_LONG - #define PY_LONG_LONG LONG_LONG -#endif -#ifndef DL_EXPORT - #define DL_EXPORT(t) t -#endif -#if PY_VERSION_HEX < 0x02040000 - #define METH_COEXIST 0 - #define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type) -#endif -#if PY_VERSION_HEX < 0x02050000 - typedef int Py_ssize_t; - #define PY_SSIZE_T_MAX INT_MAX - #define PY_SSIZE_T_MIN INT_MIN - #define PY_FORMAT_SIZE_T "" - #define PyInt_FromSsize_t(z) PyInt_FromLong(z) - #define PyInt_AsSsize_t(o) PyInt_AsLong(o) - #define PyNumber_Index(o) PyNumber_Int(o) - #define PyIndex_Check(o) PyNumber_Check(o) -#endif -#if PY_VERSION_HEX < 0x02060000 - #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) - #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) - #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) - #define PyVarObject_HEAD_INIT(type, size) \ - PyObject_HEAD_INIT(type) size, - #define PyType_Modified(t) - - typedef struct { - void *buf; - PyObject *obj; - Py_ssize_t len; - Py_ssize_t itemsize; - int readonly; - int ndim; - char *format; - Py_ssize_t *shape; - Py_ssize_t *strides; - Py_ssize_t *suboffsets; - void *internal; - } Py_buffer; - - #define PyBUF_SIMPLE 0 - #define PyBUF_WRITABLE 0x0001 - #define PyBUF_FORMAT 0x0004 - #define PyBUF_ND 0x0008 - #define PyBUF_STRIDES (0x0010 | PyBUF_ND) - #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) - #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) - #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) - #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) - -#endif -#if PY_MAJOR_VERSION < 3 - #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" -#else - #define __Pyx_BUILTIN_MODULE_NAME "builtins" -#endif -#if PY_MAJOR_VERSION >= 3 - #define Py_TPFLAGS_CHECKTYPES 0 - #define Py_TPFLAGS_HAVE_INDEX 0 -#endif -#if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) - #define Py_TPFLAGS_HAVE_NEWBUFFER 0 -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBaseString_Type PyUnicode_Type - #define PyString_Type PyBytes_Type - #define PyString_CheckExact PyBytes_CheckExact - #define PyInt_Type PyLong_Type - #define PyInt_Check(op) PyLong_Check(op) - #define PyInt_CheckExact(op) PyLong_CheckExact(op) - #define PyInt_FromString PyLong_FromString - #define PyInt_FromUnicode PyLong_FromUnicode - #define PyInt_FromLong PyLong_FromLong - #define PyInt_FromSize_t PyLong_FromSize_t - #define PyInt_FromSsize_t PyLong_FromSsize_t - #define PyInt_AsLong PyLong_AsLong - #define PyInt_AS_LONG PyLong_AS_LONG - #define PyInt_AsSsize_t PyLong_AsSsize_t - #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask - #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define PyBytes_Type PyString_Type -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyMethod_New(func, self, klass) PyInstanceMethod_New(func) -#endif -#if !defined(WIN32) && !defined(MS_WINDOWS) - #ifndef __stdcall - #define __stdcall - #endif - #ifndef __cdecl - #define __cdecl - #endif - #ifndef __fastcall - #define __fastcall - #endif -#else - #define _USE_MATH_DEFINES -#endif -#if PY_VERSION_HEX < 0x02050000 - #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),((char *)(n))) - #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a)) - #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),((char *)(n))) -#else - #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),(n)) - #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a)) - #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),(n)) -#endif -#if PY_VERSION_HEX < 0x02050000 - #define __Pyx_NAMESTR(n) ((char *)(n)) - #define __Pyx_DOCSTR(n) ((char *)(n)) -#else - #define __Pyx_NAMESTR(n) (n) - #define __Pyx_DOCSTR(n) (n) -#endif -#ifdef __cplusplus -#define __PYX_EXTERN_C extern "C" -#else -#define __PYX_EXTERN_C extern -#endif -#include -#define __PYX_HAVE_API__msgpack -#include "stdlib.h" -#include "string.h" -#include "pack.h" -#include "unpack.h" -#define __PYX_USE_C99_COMPLEX defined(_Complex_I) - - -#ifdef __GNUC__ -#define INLINE __inline__ -#elif _WIN32 -#define INLINE __inline -#else -#define INLINE -#endif - -typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/ - - - -static int __pyx_skip_dispatch = 0; - - -/* Type Conversion Predeclarations */ - -#if PY_MAJOR_VERSION < 3 -#define __Pyx_PyBytes_FromString PyString_FromString -#define __Pyx_PyBytes_FromStringAndSize PyString_FromStringAndSize -#define __Pyx_PyBytes_AsString PyString_AsString -#else -#define __Pyx_PyBytes_FromString PyBytes_FromString -#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize -#define __Pyx_PyBytes_AsString PyBytes_AsString -#endif - -#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) -static INLINE int __Pyx_PyObject_IsTrue(PyObject*); -static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); - -#if !defined(T_PYSSIZET) -#if PY_VERSION_HEX < 0x02050000 -#define T_PYSSIZET T_INT -#elif !defined(T_LONGLONG) -#define T_PYSSIZET \ - ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ - ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : -1)) -#else -#define T_PYSSIZET \ - ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ - ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : \ - ((sizeof(Py_ssize_t) == sizeof(PY_LONG_LONG)) ? T_LONGLONG : -1))) -#endif -#endif - -#if !defined(T_SIZET) -#if !defined(T_ULONGLONG) -#define T_SIZET \ - ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ - ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : -1)) -#else -#define T_SIZET \ - ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ - ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : \ - ((sizeof(size_t) == sizeof(unsigned PY_LONG_LONG)) ? T_ULONGLONG : -1))) -#endif -#endif - -static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); -static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); -static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); - -#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) - - -#ifdef __GNUC__ -/* Test for GCC > 2.95 */ -#if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)) -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) -#else /* __GNUC__ > 2 ... */ -#define likely(x) (x) -#define unlikely(x) (x) -#endif /* __GNUC__ > 2 ... */ -#else /* __GNUC__ */ -#define likely(x) (x) -#define unlikely(x) (x) -#endif /* __GNUC__ */ - -static PyObject *__pyx_m; -static PyObject *__pyx_b; -static PyObject *__pyx_empty_tuple; -static int __pyx_lineno; -static int __pyx_clineno = 0; -static const char * __pyx_cfilenm= __FILE__; -static const char *__pyx_filename; -static const char **__pyx_f; - - -#ifdef CYTHON_REFNANNY -typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*NewContext)(const char*, int, const char*); - void (*FinishContext)(void**); -} __Pyx_RefnannyAPIStruct; -static __Pyx_RefnannyAPIStruct *__Pyx_Refnanny = NULL; -#define __Pyx_ImportRefcountAPI(name) (__Pyx_RefnannyAPIStruct *) PyCObject_Import((char *)name, (char *)"RefnannyAPI") -#define __Pyx_INCREF(r) __Pyx_Refnanny->INCREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_DECREF(r) __Pyx_Refnanny->DECREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_GOTREF(r) __Pyx_Refnanny->GOTREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_GIVEREF(r) __Pyx_Refnanny->GIVEREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_XDECREF(r) if((r) == NULL) ; else __Pyx_DECREF(r) -#define __Pyx_SetupRefcountContext(name) void* __pyx_refchk = __Pyx_Refnanny->NewContext((name), __LINE__, __FILE__) -#define __Pyx_FinishRefcountContext() __Pyx_Refnanny->FinishContext(&__pyx_refchk) -#else -#define __Pyx_INCREF(r) Py_INCREF(r) -#define __Pyx_DECREF(r) Py_DECREF(r) -#define __Pyx_GOTREF(r) -#define __Pyx_GIVEREF(r) -#define __Pyx_XDECREF(r) Py_XDECREF(r) -#define __Pyx_SetupRefcountContext(name) -#define __Pyx_FinishRefcountContext() -#endif /* CYTHON_REFNANNY */ -#define __Pyx_XGIVEREF(r) if((r) == NULL) ; else __Pyx_GIVEREF(r) -#define __Pyx_XGOTREF(r) if((r) == NULL) ; else __Pyx_GOTREF(r) - -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, PyObject* kw_name); /*proto*/ - -static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, - Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ - -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ - -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list); /*proto*/ - -static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/ - -static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); - -static INLINE void __Pyx_RaiseTooManyValuesError(void); - -static PyObject *__Pyx_UnpackItem(PyObject *, Py_ssize_t index); /*proto*/ -static int __Pyx_EndUnpack(PyObject *); /*proto*/ - -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ - -static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ -static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ - -static INLINE int __Pyx_StrEq(const char *, const char *); /*proto*/ - -static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *); - -static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *); - -static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *); - -static INLINE char __Pyx_PyInt_AsChar(PyObject *); - -static INLINE short __Pyx_PyInt_AsShort(PyObject *); - -static INLINE int __Pyx_PyInt_AsInt(PyObject *); - -static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *); - -static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *); - -static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *); - -static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *); - -static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *); - -static INLINE long __Pyx_PyInt_AsLong(PyObject *); - -static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *); - -static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); - -static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); - -static void __Pyx_WriteUnraisable(const char *name); /*proto*/ - -static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ - -static void __Pyx_AddTraceback(const char *funcname); /*proto*/ - -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ - -/* Type declarations */ - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 - * cdef int BUFF_SIZE=2*1024 - * - * cdef class Packer: # <<<<<<<<<<<<<< - * """Packer that pack data into strm. - * - */ - -struct __pyx_obj_7msgpack_Packer { - PyObject_HEAD - struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtab; - char *buff; - unsigned int length; - unsigned int allocated; - struct msgpack_packer pk; - PyObject *strm; -}; - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 - * return unpacks(packed) - * - * cdef class Unpacker: # <<<<<<<<<<<<<< - * """Do nothing. This function is for symmetric to Packer""" - * unpack = staticmethod(unpacks) - */ - -struct __pyx_obj_7msgpack_Unpacker { - PyObject_HEAD -}; - - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 - * cdef int BUFF_SIZE=2*1024 - * - * cdef class Packer: # <<<<<<<<<<<<<< - * """Packer that pack data into strm. - * - */ - -struct __pyx_vtabstruct_7msgpack_Packer { - PyObject *(*__pack)(struct __pyx_obj_7msgpack_Packer *, PyObject *); -}; -static struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtabptr_7msgpack_Packer; -/* Module declarations from msgpack */ - -static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; -static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; -static int __pyx_v_7msgpack_BUFF_SIZE; -static PyObject *__pyx_k_1 = 0; -static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ -#define __Pyx_MODULE_NAME "msgpack" -int __pyx_module_is_main_msgpack = 0; - -/* Implementation of msgpack */ -static char __pyx_k___main__[] = "__main__"; -static PyObject *__pyx_kp___main__; -static char __pyx_k___init__[] = "__init__"; -static PyObject *__pyx_kp___init__; -static char __pyx_k___del__[] = "__del__"; -static PyObject *__pyx_kp___del__; -static char __pyx_k_flush[] = "flush"; -static PyObject *__pyx_kp_flush; -static char __pyx_k_pack_list[] = "pack_list"; -static PyObject *__pyx_kp_pack_list; -static char __pyx_k_pack_dict[] = "pack_dict"; -static PyObject *__pyx_kp_pack_dict; -static char __pyx_k_pack[] = "pack"; -static PyObject *__pyx_kp_pack; -static char __pyx_k_unpack[] = "unpack"; -static PyObject *__pyx_kp_unpack; -static char __pyx_k_strm[] = "strm"; -static PyObject *__pyx_kp_strm; -static char __pyx_k_size[] = "size"; -static PyObject *__pyx_kp_size; -static char __pyx_k_len[] = "len"; -static PyObject *__pyx_kp_len; -static char __pyx_k_obj[] = "obj"; -static PyObject *__pyx_kp_obj; -static char __pyx_k_o[] = "o"; -static PyObject *__pyx_kp_o; -static char __pyx_k_stream[] = "stream"; -static PyObject *__pyx_kp_stream; -static char __pyx_k_packed_bytes[] = "packed_bytes"; -static PyObject *__pyx_kp_packed_bytes; -static char __pyx_k_cStringIO[] = "cStringIO"; -static PyObject *__pyx_kp_cStringIO; -static char __pyx_k_StringIO[] = "StringIO"; -static PyObject *__pyx_kp_StringIO; -static char __pyx_k_staticmethod[] = "staticmethod"; -static PyObject *__pyx_kp_staticmethod; -static char __pyx_k_unpacks[] = "unpacks"; -static PyObject *__pyx_kp_unpacks; -static char __pyx_k_write[] = "write"; -static PyObject *__pyx_kp_write; -static char __pyx_k_2[] = "flush"; -static PyObject *__pyx_kp_2; -static char __pyx_k_encode[] = "encode"; -static PyObject *__pyx_kp_encode; -static char __pyx_k_iteritems[] = "iteritems"; -static PyObject *__pyx_kp_iteritems; -static char __pyx_k_TypeError[] = "TypeError"; -static PyObject *__pyx_kp_TypeError; -static char __pyx_k_getvalue[] = "getvalue"; -static PyObject *__pyx_kp_getvalue; -static char __pyx_k_read[] = "read"; -static PyObject *__pyx_kp_read; -static PyObject *__pyx_builtin_staticmethod; -static PyObject *__pyx_builtin_TypeError; -static PyObject *__pyx_kp_3; -static PyObject *__pyx_kp_4; -static char __pyx_k_3[] = "utf-8"; -static char __pyx_k_4[] = "can't serialize %r"; - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 - * cdef object strm - * - * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< - * if size <= 0: - * size = BUFF_SIZE - */ - -static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_strm = 0; - int __pyx_v_size; - int __pyx_r; - int __pyx_t_1; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_strm,&__pyx_kp_size,0}; - __Pyx_SetupRefcountContext("__init__"); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_strm); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 1) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_size); - if (unlikely(value)) { values[1] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "__init__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_strm = values[0]; - if (values[1]) { - __pyx_v_size = __Pyx_PyInt_AsInt(values[1]); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } else { - __pyx_v_size = 0; - } - } else { - __pyx_v_size = 0; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: __pyx_v_size = __Pyx_PyInt_AsInt(PyTuple_GET_ITEM(__pyx_args, 1)); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - case 1: __pyx_v_strm = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.Packer.__init__"); - return -1; - __pyx_L4_argument_unpacking_done:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 - * - * def __init__(self, strm, int size=0): - * if size <= 0: # <<<<<<<<<<<<<< - * size = BUFF_SIZE - * - */ - __pyx_t_1 = (__pyx_v_size <= 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 - * def __init__(self, strm, int size=0): - * if size <= 0: - * size = BUFF_SIZE # <<<<<<<<<<<<<< - * - * self.strm = strm - */ - __pyx_v_size = __pyx_v_7msgpack_BUFF_SIZE; - goto __pyx_L6; - } - __pyx_L6:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 - * size = BUFF_SIZE - * - * self.strm = strm # <<<<<<<<<<<<<< - * self.buff = malloc(size) - * self.allocated = size - */ - __Pyx_INCREF(__pyx_v_strm); - __Pyx_GIVEREF(__pyx_v_strm); - __Pyx_GOTREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); - __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 - * - * self.strm = strm - * self.buff = malloc(size) # <<<<<<<<<<<<<< - * self.allocated = size - * self.length = 0 - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 - * self.strm = strm - * self.buff = malloc(size) - * self.allocated = size # <<<<<<<<<<<<<< - * self.length = 0 - * - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 - * self.buff = malloc(size) - * self.allocated = size - * self.length = 0 # <<<<<<<<<<<<<< - * - * msgpack_packer_init(&self.pk, self, _packer_write) - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 - * self.length = 0 - * - * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< - * - * def __del__(self): - */ - msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); - - __pyx_r = 0; - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 - * msgpack_packer_init(&self.pk, self, _packer_write) - * - * def __del__(self): # <<<<<<<<<<<<<< - * free(self.buff); - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused) { - PyObject *__pyx_r = NULL; - __Pyx_SetupRefcountContext("__del__"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 - * - * def __del__(self): - * free(self.buff); # <<<<<<<<<<<<<< - * - * def flush(self): - */ - free(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 - * free(self.buff); - * - * def flush(self): # <<<<<<<<<<<<<< - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_flush[] = "Flash local buffer and output stream if it has 'flush()' method."; -static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused) { - PyObject *__pyx_r = NULL; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_SetupRefcountContext("flush"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 - * def flush(self): - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: # <<<<<<<<<<<<<< - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 - */ - __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< - * self.length = 0 - * if hasattr(self.strm, 'flush'): - */ - __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_4)); - PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 - * if self.length > 0: - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 # <<<<<<<<<<<<<< - * if hasattr(self.strm, 'flush'): - * self.strm.flush() - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - goto __pyx_L5; - } - __pyx_L5:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 - * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< - * self.strm.flush() - * - */ - __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 - * self.length = 0 - * if hasattr(self.strm, 'flush'): - * self.strm.flush() # <<<<<<<<<<<<<< - * - * def pack_list(self, len): - */ - __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - goto __pyx_L6; - } - __pyx_L6:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("msgpack.Packer.flush"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 - * self.strm.flush() - * - * def pack_list(self, len): # <<<<<<<<<<<<<< - * """Start packing sequential objects. - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_pack_list[] = "Start packing sequential objects.\n\n Example:\n\n packer.pack_list(2)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack(['foo', 'bar'])\n "; -static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { - PyObject *__pyx_r = NULL; - size_t __pyx_t_1; - __Pyx_SetupRefcountContext("pack_list"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 - * packer.pack(['foo', 'bar']) - * """ - * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< - * - * def pack_dict(self, len): - */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("msgpack.Packer.pack_list"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 - * msgpack_pack_array(&self.pk, len) - * - * def pack_dict(self, len): # <<<<<<<<<<<<<< - * """Start packing key-value objects. - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_pack_dict[] = "Start packing key-value objects.\n\n Example:\n\n packer.pack_dict(1)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack({'foo', 'bar'})\n "; -static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { - PyObject *__pyx_r = NULL; - size_t __pyx_t_1; - __Pyx_SetupRefcountContext("pack_dict"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 - * packer.pack({'foo', 'bar'}) - * """ - * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< - * - * cdef __pack(self, object o): - */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("msgpack.Packer.pack_dict"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 - * msgpack_pack_map(&self.pk, len) - * - * cdef __pack(self, object o): # <<<<<<<<<<<<<< - * cdef long long intval - * cdef double fval - */ - -static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Packer *__pyx_v_self, PyObject *__pyx_v_o) { - PY_LONG_LONG __pyx_v_intval; - double __pyx_v_fval; - char *__pyx_v_rawval; - PyObject *__pyx_v_k; - PyObject *__pyx_v_v; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_2 = 0; - PyObject *__pyx_3 = 0; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PY_LONG_LONG __pyx_t_3; - char *__pyx_t_4; - Py_ssize_t __pyx_t_5; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - int __pyx_t_9; - int __pyx_t_10; - __Pyx_SetupRefcountContext("__pack"); - __Pyx_INCREF(__pyx_v_o); - __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); - __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 - * cdef char* rawval - * - * if o is None: # <<<<<<<<<<<<<< - * msgpack_pack_nil(&self.pk) - * elif o is True: - */ - __pyx_t_1 = (__pyx_v_o == Py_None); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 - * - * if o is None: - * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< - * elif o is True: - * msgpack_pack_true(&self.pk) - */ - msgpack_pack_nil((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 - * if o is None: - * msgpack_pack_nil(&self.pk) - * elif o is True: # <<<<<<<<<<<<<< - * msgpack_pack_true(&self.pk) - * elif o is False: - */ - __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = (__pyx_v_o == __pyx_t_2); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 - * msgpack_pack_nil(&self.pk) - * elif o is True: - * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< - * elif o is False: - * msgpack_pack_false(&self.pk) - */ - msgpack_pack_true((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 - * elif o is True: - * msgpack_pack_true(&self.pk) - * elif o is False: # <<<<<<<<<<<<<< - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): - */ - __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = (__pyx_v_o == __pyx_t_2); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 - * msgpack_pack_true(&self.pk) - * elif o is False: - * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< - * elif isinstance(o, long): - * intval = o - */ - msgpack_pack_false((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 - * elif o is False: - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): # <<<<<<<<<<<<<< - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): - * intval = o # <<<<<<<<<<<<<< - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): - */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_intval = __pyx_t_3; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 - * elif isinstance(o, long): - * intval = o - * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< - * elif isinstance(o, int): - * intval = o - */ - msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): # <<<<<<<<<<<<<< - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): - * intval = o # <<<<<<<<<<<<<< - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): - */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_intval = __pyx_t_3; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 - * elif isinstance(o, int): - * intval = o - * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< - * elif isinstance(o, float): - * fval = 9 - */ - msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): # <<<<<<<<<<<<<< - * fval = 9 - * msgpack_pack_double(&self.pk, fval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): - * fval = 9 # <<<<<<<<<<<<<< - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): - */ - __pyx_v_fval = 9; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 - * elif isinstance(o, float): - * fval = 9 - * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< - * elif isinstance(o, str): - * rawval = o - */ - msgpack_pack_double((&__pyx_v_self->pk), __pyx_v_fval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 - * fval = 9 - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): # <<<<<<<<<<<<<< - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): - * rawval = o # <<<<<<<<<<<<<< - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 - * elif isinstance(o, str): - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< - * elif isinstance(o, unicode): - * o = o.encode('utf-8') - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): # <<<<<<<<<<<<<< - * o = o.encode('utf-8') - * rawval = o - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): - * o = o.encode('utf-8') # <<<<<<<<<<<<<< - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_6)); - __Pyx_INCREF(__pyx_kp_3); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); - __Pyx_GIVEREF(__pyx_kp_3); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_v_o); - __pyx_v_o = __pyx_t_7; - __pyx_t_7 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 - * elif isinstance(o, unicode): - * o = o.encode('utf-8') - * rawval = o # <<<<<<<<<<<<<< - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 - * o = o.encode('utf-8') - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): # <<<<<<<<<<<<<< - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< - * for k,v in o.iteritems(): - * self.pack(k) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): # <<<<<<<<<<<<<< - * self.pack(k) - * self.pack(v) - */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); - } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - } - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - break; - } - __Pyx_GOTREF(__pyx_t_6); - } - if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { - PyObject* tuple = __pyx_t_6; - __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); - __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_v_k); - __pyx_v_k = __pyx_2; - __pyx_2 = 0; - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_3; - __pyx_3 = 0; - } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_2); - __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_3); - if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(__pyx_v_k); - __pyx_v_k = __pyx_2; - __pyx_2 = 0; - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_3; - __pyx_3 = 0; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): - * self.pack(k) # <<<<<<<<<<<<<< - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): - */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_k); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); - __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 - * for k,v in o.iteritems(): - * self.pack(k) - * self.pack(v) # <<<<<<<<<<<<<< - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) - */ - __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_v); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); - __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 - * self.pack(k) - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); - if (!__pyx_t_1) { - __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); - __pyx_t_10 = __pyx_t_9; - } else { - __pyx_t_10 = __pyx_t_1; - } - if (__pyx_t_10) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< - * for v in o: - * self.pack(v) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: # <<<<<<<<<<<<<< - * self.pack(v) - * else: - */ - if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); - } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - } - for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - break; - } - __Pyx_GOTREF(__pyx_t_6); - } - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_t_6; - __pyx_t_6 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: - * self.pack(v) # <<<<<<<<<<<<<< - * else: - * # TODO: Serialize with defalt() like simplejson. - */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_v); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); - __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L3; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 - * else: - * # TODO: Serialize with defalt() like simplejson. - * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< - * - * def pack(self, obj, flush=True): - */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_7)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; - __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - __pyx_L3:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_2); - __Pyx_XDECREF(__pyx_3); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("msgpack.Packer.__pack"); - __pyx_r = 0; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_k); - __Pyx_DECREF(__pyx_v_v); - __Pyx_DECREF(__pyx_v_o); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 - * raise TypeError, "can't serialize %r" % (o,) - * - * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< - * self.__pack(obj) - * if flush: - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_obj = 0; - PyObject *__pyx_v_flush = 0; - PyObject *__pyx_r = NULL; - PyObject *__pyx_t_1 = NULL; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_obj,&__pyx_kp_flush,0}; - __Pyx_SetupRefcountContext("pack"); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - values[1] = __pyx_k_1; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_obj); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 1) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_flush); - if (unlikely(value)) { values[1] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_obj = values[0]; - __pyx_v_flush = values[1]; - } else { - __pyx_v_flush = __pyx_k_1; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: __pyx_v_flush = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: __pyx_v_obj = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.Packer.pack"); - return NULL; - __pyx_L4_argument_unpacking_done:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 - * - * def pack(self, obj, flush=True): - * self.__pack(obj) # <<<<<<<<<<<<<< - * if flush: - * self.flush() - */ - __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_Packer *)((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->__pyx_vtab)->__pack(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self), __pyx_v_obj); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 - * def pack(self, obj, flush=True): - * self.__pack(obj) - * if flush: # <<<<<<<<<<<<<< - * self.flush() - * - */ - __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_2) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 - * self.__pack(obj) - * if flush: - * self.flush() # <<<<<<<<<<<<<< - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_flush); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - goto __pyx_L6; - } - __pyx_L6:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.Packer.pack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 - * self.flush() - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< - * if packer.length + l > packer.allocated: - * if packer.length > 0: - */ - -static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__pyx_v_packer, const char* __pyx_v_b, unsigned int __pyx_v_l) { - int __pyx_r; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_SetupRefcountContext("_packer_write"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - */ - __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - * if packer.length + l > packer.allocated: - * if packer.length > 0: # <<<<<<<<<<<<<< - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: - */ - __pyx_t_1 = (__pyx_v_packer->length > 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 - * if packer.length + l > packer.allocated: - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_4)); - PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - goto __pyx_L4; - } - __pyx_L4:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: # <<<<<<<<<<<<<< - * packer.strm.write(PyString_FromStringAndSize(b, l)) - * packer.length = 0 - */ - __pyx_t_1 = (__pyx_v_l > 64); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< - * packer.length = 0 - * else: - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); - __Pyx_GIVEREF(__pyx_t_4); - __pyx_t_4 = 0; - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) - * packer.length = 0 # <<<<<<<<<<<<<< - * else: - * memcpy(packer.buff, b, l) - */ - __pyx_v_packer->length = 0; - goto __pyx_L5; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 - * packer.length = 0 - * else: - * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< - * packer.length = l - * else: - */ - memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 - * else: - * memcpy(packer.buff, b, l) - * packer.length = l # <<<<<<<<<<<<<< - * else: - * memcpy(packer.buff + packer.length, b, l) - */ - __pyx_v_packer->length = __pyx_v_l; - } - __pyx_L5:; - goto __pyx_L3; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 - * packer.length = l - * else: - * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< - * packer.length += l - * return 0 - */ - memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 - * else: - * memcpy(packer.buff + packer.length, b, l) - * packer.length += l # <<<<<<<<<<<<<< - * return 0 - * - */ - __pyx_v_packer->length += __pyx_v_l; - } - __pyx_L3:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 - * memcpy(packer.buff + packer.length, b, l) - * packer.length += l - * return 0 # <<<<<<<<<<<<<< - * - * def pack(object o, object stream): - */ - __pyx_r = 0; - goto __pyx_L0; - - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_WriteUnraisable("msgpack._packer_write"); - __pyx_r = 0; - __pyx_L0:; - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 - * return 0 - * - * def pack(object o, object stream): # <<<<<<<<<<<<<< - * packer = Packer(stream) - * packer.pack(o) - */ - -static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_o = 0; - PyObject *__pyx_v_stream = 0; - PyObject *__pyx_v_packer; - PyObject *__pyx_r = NULL; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_o,&__pyx_kp_stream,0}; - __Pyx_SetupRefcountContext("pack"); - __pyx_self = __pyx_self; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_o); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); - if (likely(values[1])) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_o = values[0]; - __pyx_v_stream = values[1]; - } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { - goto __pyx_L5_argtuple_error; - } else { - __pyx_v_o = PyTuple_GET_ITEM(__pyx_args, 0); - __pyx_v_stream = PyTuple_GET_ITEM(__pyx_args, 1); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.pack"); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 - * - * def pack(object o, object stream): - * packer = Packer(stream) # <<<<<<<<<<<<<< - * packer.pack(o) - * packer.flush() - */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_stream); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); - __Pyx_GIVEREF(__pyx_v_stream); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packer); - __pyx_v_packer = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 - * def pack(object o, object stream): - * packer = Packer(stream) - * packer.pack(o) # <<<<<<<<<<<<<< - * packer.flush() - * - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 - * packer = Packer(stream) - * packer.pack(o) - * packer.flush() # <<<<<<<<<<<<<< - * - * def packs(object o): - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.pack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_packer); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 - * packer.flush() - * - * def packs(object o): # <<<<<<<<<<<<<< - * buf = StringIO() - * packer = Packer(buf) - */ - -static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o); /*proto*/ -static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o) { - PyObject *__pyx_v_buf; - PyObject *__pyx_v_packer; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - __Pyx_SetupRefcountContext("packs"); - __pyx_self = __pyx_self; - __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); - __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 - * - * def packs(object o): - * buf = StringIO() # <<<<<<<<<<<<<< - * packer = Packer(buf) - * packer.pack(o) - */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(__pyx_v_buf); - __pyx_v_buf = __pyx_t_1; - __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 - * def packs(object o): - * buf = StringIO() - * packer = Packer(buf) # <<<<<<<<<<<<<< - * packer.pack(o) - * packer.flush() - */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_buf); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); - __Pyx_GIVEREF(__pyx_v_buf); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packer); - __pyx_v_packer = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 - * buf = StringIO() - * packer = Packer(buf) - * packer.pack(o) # <<<<<<<<<<<<<< - * packer.flush() - * return buf.getvalue() - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 - * packer = Packer(buf) - * packer.pack(o) - * packer.flush() # <<<<<<<<<<<<<< - * return buf.getvalue() - * - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 - * packer.pack(o) - * packer.flush() - * return buf.getvalue() # <<<<<<<<<<<<<< - * - * cdef extern from "unpack.h": - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.packs"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_buf); - __Pyx_DECREF(__pyx_v_packer); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 - * - * - * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< - * """Unpack packed_bytes to object. Returns unpacked object.""" - * cdef const_char_ptr p = packed_bytes - */ - -static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes); /*proto*/ -static char __pyx_doc_7msgpack_unpacks[] = "Unpack packed_bytes to object. Returns unpacked object."; -static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes) { - const char* __pyx_v_p; - template_context __pyx_v_ctx; - size_t __pyx_v_off; - PyObject *__pyx_r = NULL; - const char* __pyx_t_1; - Py_ssize_t __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - __Pyx_SetupRefcountContext("unpacks"); - __pyx_self = __pyx_self; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 - * def unpacks(object packed_bytes): - * """Unpack packed_bytes to object. Returns unpacked object.""" - * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< - * cdef template_context ctx - * cdef size_t off = 0 - */ - __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_p = __pyx_t_1; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 - * cdef const_char_ptr p = packed_bytes - * cdef template_context ctx - * cdef size_t off = 0 # <<<<<<<<<<<<<< - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) - */ - __pyx_v_off = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 - * cdef template_context ctx - * cdef size_t off = 0 - * template_init(&ctx) # <<<<<<<<<<<<<< - * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) - */ - template_init((&__pyx_v_ctx)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 - * cdef size_t off = 0 - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< - * return template_data(&ctx) - * - */ - __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) # <<<<<<<<<<<<<< - * - * def unpack(object stream): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.unpacks"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 - * return template_data(&ctx) - * - * def unpack(object stream): # <<<<<<<<<<<<<< - * """unpack from stream.""" - * packed = stream.read() - */ - -static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream); /*proto*/ -static char __pyx_doc_7msgpack_unpack[] = "unpack from stream."; -static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream) { - PyObject *__pyx_v_packed; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - __Pyx_SetupRefcountContext("unpack"); - __pyx_self = __pyx_self; - __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 - * def unpack(object stream): - * """unpack from stream.""" - * packed = stream.read() # <<<<<<<<<<<<<< - * return unpacks(packed) - * - */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packed); - __pyx_v_packed = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 - * """unpack from stream.""" - * packed = stream.read() - * return unpacks(packed) # <<<<<<<<<<<<<< - * - * cdef class Unpacker: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_packed); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); - __Pyx_GIVEREF(__pyx_v_packed); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack.unpack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_packed); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} -static struct __pyx_vtabstruct_7msgpack_Packer __pyx_vtable_7msgpack_Packer; - -static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { - struct __pyx_obj_7msgpack_Packer *p; - PyObject *o = (*t->tp_alloc)(t, 0); - if (!o) return 0; - p = ((struct __pyx_obj_7msgpack_Packer *)o); - p->__pyx_vtab = __pyx_vtabptr_7msgpack_Packer; - p->strm = Py_None; Py_INCREF(Py_None); - return o; -} - -static void __pyx_tp_dealloc_7msgpack_Packer(PyObject *o) { - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - Py_XDECREF(p->strm); - (*Py_TYPE(o)->tp_free)(o); -} - -static int __pyx_tp_traverse_7msgpack_Packer(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - if (p->strm) { - e = (*v)(p->strm, a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - PyObject* tmp; - tmp = ((PyObject*)p->strm); - p->strm = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { - {__Pyx_NAMESTR("__del__"), (PyCFunction)__pyx_pf_7msgpack_6Packer___del__, METH_NOARGS, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, - {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, - {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, - {0, 0, 0, 0} -}; - -static PyNumberMethods __pyx_tp_as_number_Packer = { - 0, /*nb_add*/ - 0, /*nb_subtract*/ - 0, /*nb_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ - #endif - 0, /*nb_remainder*/ - 0, /*nb_divmod*/ - 0, /*nb_power*/ - 0, /*nb_negative*/ - 0, /*nb_positive*/ - 0, /*nb_absolute*/ - 0, /*nb_nonzero*/ - 0, /*nb_invert*/ - 0, /*nb_lshift*/ - 0, /*nb_rshift*/ - 0, /*nb_and*/ - 0, /*nb_xor*/ - 0, /*nb_or*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ - #endif - 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else - 0, /*nb_long*/ - #endif - 0, /*nb_float*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_oct*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*nb_hex*/ - #endif - 0, /*nb_inplace_add*/ - 0, /*nb_inplace_subtract*/ - 0, /*nb_inplace_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_inplace_divide*/ - #endif - 0, /*nb_inplace_remainder*/ - 0, /*nb_inplace_power*/ - 0, /*nb_inplace_lshift*/ - 0, /*nb_inplace_rshift*/ - 0, /*nb_inplace_and*/ - 0, /*nb_inplace_xor*/ - 0, /*nb_inplace_or*/ - 0, /*nb_floor_divide*/ - 0, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) - 0, /*nb_index*/ - #endif -}; - -static PySequenceMethods __pyx_tp_as_sequence_Packer = { - 0, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping_Packer = { - 0, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyBufferProcs __pyx_tp_as_buffer_Packer = { - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getreadbuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getwritebuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getsegcount*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getcharbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_getbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_releasebuffer*/ - #endif -}; - -PyTypeObject __pyx_type_7msgpack_Packer = { - PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("msgpack.Packer"), /*tp_name*/ - sizeof(struct __pyx_obj_7msgpack_Packer), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_7msgpack_Packer, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - &__pyx_tp_as_number_Packer, /*tp_as_number*/ - &__pyx_tp_as_sequence_Packer, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_Packer, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - &__pyx_tp_as_buffer_Packer, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - __Pyx_DOCSTR("Packer that pack data into strm.\n\n strm must have `write(bytes)` method.\n size specifies local buffer size.\n "), /*tp_doc*/ - __pyx_tp_traverse_7msgpack_Packer, /*tp_traverse*/ - __pyx_tp_clear_7msgpack_Packer, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_7msgpack_Packer, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - __pyx_pf_7msgpack_6Packer___init__, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_7msgpack_Packer, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ -}; - -static PyObject *__pyx_tp_new_7msgpack_Unpacker(PyTypeObject *t, PyObject *a, PyObject *k) { - PyObject *o = (*t->tp_alloc)(t, 0); - if (!o) return 0; - return o; -} - -static void __pyx_tp_dealloc_7msgpack_Unpacker(PyObject *o) { - (*Py_TYPE(o)->tp_free)(o); -} - -static struct PyMethodDef __pyx_methods_7msgpack_Unpacker[] = { - {0, 0, 0, 0} -}; - -static PyNumberMethods __pyx_tp_as_number_Unpacker = { - 0, /*nb_add*/ - 0, /*nb_subtract*/ - 0, /*nb_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ - #endif - 0, /*nb_remainder*/ - 0, /*nb_divmod*/ - 0, /*nb_power*/ - 0, /*nb_negative*/ - 0, /*nb_positive*/ - 0, /*nb_absolute*/ - 0, /*nb_nonzero*/ - 0, /*nb_invert*/ - 0, /*nb_lshift*/ - 0, /*nb_rshift*/ - 0, /*nb_and*/ - 0, /*nb_xor*/ - 0, /*nb_or*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ - #endif - 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else - 0, /*nb_long*/ - #endif - 0, /*nb_float*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_oct*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*nb_hex*/ - #endif - 0, /*nb_inplace_add*/ - 0, /*nb_inplace_subtract*/ - 0, /*nb_inplace_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_inplace_divide*/ - #endif - 0, /*nb_inplace_remainder*/ - 0, /*nb_inplace_power*/ - 0, /*nb_inplace_lshift*/ - 0, /*nb_inplace_rshift*/ - 0, /*nb_inplace_and*/ - 0, /*nb_inplace_xor*/ - 0, /*nb_inplace_or*/ - 0, /*nb_floor_divide*/ - 0, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) - 0, /*nb_index*/ - #endif -}; - -static PySequenceMethods __pyx_tp_as_sequence_Unpacker = { - 0, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping_Unpacker = { - 0, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyBufferProcs __pyx_tp_as_buffer_Unpacker = { - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getreadbuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getwritebuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getsegcount*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getcharbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_getbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_releasebuffer*/ - #endif -}; - -PyTypeObject __pyx_type_7msgpack_Unpacker = { - PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("msgpack.Unpacker"), /*tp_name*/ - sizeof(struct __pyx_obj_7msgpack_Unpacker), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_7msgpack_Unpacker, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - &__pyx_tp_as_number_Unpacker, /*tp_as_number*/ - &__pyx_tp_as_sequence_Unpacker, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_Unpacker, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - &__pyx_tp_as_buffer_Unpacker, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ - __Pyx_DOCSTR("Do nothing. This function is for symmetric to Packer"), /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_7msgpack_Unpacker, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_7msgpack_Unpacker, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ -}; - -static struct PyMethodDef __pyx_methods[] = { - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("packs"), (PyCFunction)__pyx_pf_7msgpack_packs, METH_O, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("unpacks"), (PyCFunction)__pyx_pf_7msgpack_unpacks, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpacks)}, - {__Pyx_NAMESTR("unpack"), (PyCFunction)__pyx_pf_7msgpack_unpack, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpack)}, - {0, 0, 0, 0} -}; - -static void __pyx_init_filenames(void); /*proto*/ - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef __pyx_moduledef = { - PyModuleDef_HEAD_INIT, - __Pyx_NAMESTR("msgpack"), - 0, /* m_doc */ - -1, /* m_size */ - __pyx_methods /* m_methods */, - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; -#endif - -static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, - {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, - {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, - {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, - {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, - {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, - {&__pyx_kp_pack, __pyx_k_pack, sizeof(__pyx_k_pack), 1, 1, 1}, - {&__pyx_kp_unpack, __pyx_k_unpack, sizeof(__pyx_k_unpack), 1, 1, 1}, - {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, - {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, - {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, - {&__pyx_kp_obj, __pyx_k_obj, sizeof(__pyx_k_obj), 1, 1, 1}, - {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, - {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, - {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, - {&__pyx_kp_cStringIO, __pyx_k_cStringIO, sizeof(__pyx_k_cStringIO), 1, 1, 1}, - {&__pyx_kp_StringIO, __pyx_k_StringIO, sizeof(__pyx_k_StringIO), 1, 1, 1}, - {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, - {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, - {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, - {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 1, 0}, - {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, - {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, - {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, - {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, - {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, - {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, - {&__pyx_kp_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 0}, - {0, 0, 0, 0, 0, 0} -}; -static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - return 0; - __pyx_L1_error:; - return -1; -} - -static int __Pyx_InitGlobals(void) { - if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - return 0; - __pyx_L1_error:; - return -1; -} - -#if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC initmsgpack(void); /*proto*/ -PyMODINIT_FUNC initmsgpack(void) -#else -PyMODINIT_FUNC PyInit_msgpack(void); /*proto*/ -PyMODINIT_FUNC PyInit_msgpack(void) -#endif -{ - PyObject *__pyx_1 = 0; - PyObject *__pyx_2 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - #ifdef CYTHON_REFNANNY - void* __pyx_refchk = NULL; - __Pyx_Refnanny = __Pyx_ImportRefcountAPI("refnanny"); - if (!__Pyx_Refnanny) { - PyErr_Clear(); - __Pyx_Refnanny = __Pyx_ImportRefcountAPI("Cython.Runtime.refnanny"); - if (!__Pyx_Refnanny) - Py_FatalError("failed to import refnanny module"); - } - __pyx_refchk = __Pyx_Refnanny->NewContext("PyMODINIT_FUNC PyInit_msgpack(void)", __LINE__, __FILE__); - #endif - __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /*--- Library function declarations ---*/ - __pyx_init_filenames(); - /*--- Threads initialization code ---*/ - #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS - #ifdef WITH_THREAD /* Python build with threading support? */ - PyEval_InitThreads(); - #endif - #endif - /*--- Initialize various global constants etc. ---*/ - if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /*--- Module creation code ---*/ - #if PY_MAJOR_VERSION < 3 - __pyx_m = Py_InitModule4(__Pyx_NAMESTR("msgpack"), __pyx_methods, 0, 0, PYTHON_API_VERSION); - #else - __pyx_m = PyModule_Create(&__pyx_moduledef); - #endif - if (!__pyx_m) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #if PY_MAJOR_VERSION < 3 - Py_INCREF(__pyx_m); - #endif - __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); - if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__pyx_module_is_main_msgpack) { - if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - } - /*--- Builtin init code ---*/ - if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_skip_dispatch = 0; - /*--- Global init code ---*/ - /*--- Function export code ---*/ - /*--- Type init code ---*/ - __pyx_vtabptr_7msgpack_Packer = &__pyx_vtable_7msgpack_Packer; - #if PY_MAJOR_VERSION >= 3 - __pyx_vtable_7msgpack_Packer.__pack = (PyObject *(*)(struct __pyx_obj_7msgpack_Packer *, PyObject *))__pyx_f_7msgpack_6Packer___pack; - #else - *(void(**)(void))&__pyx_vtable_7msgpack_Packer.__pack = (void(*)(void))__pyx_f_7msgpack_6Packer___pack; - #endif - if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetVtable(__pyx_type_7msgpack_Packer.tp_dict, __pyx_vtabptr_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; - if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; - /*--- Type import code ---*/ - /*--- Function import code ---*/ - /*--- Execution code ---*/ - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 - * # coding: utf-8 - * - * from cStringIO import StringIO # <<<<<<<<<<<<<< - * - * cdef extern from "Python.h": - */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_kp_StringIO); - PyList_SET_ITEM(__pyx_t_1, 0, __pyx_kp_StringIO); - __Pyx_GIVEREF(__pyx_kp_StringIO); - __pyx_1 = __Pyx_Import(__pyx_kp_cStringIO, ((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __pyx_2 = PyObject_GetAttr(__pyx_1, __pyx_kp_StringIO); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_2); - if (PyObject_SetAttr(__pyx_m, __pyx_kp_StringIO, __pyx_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_2); __pyx_2 = 0; - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 - * - * - * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< - * - * cdef class Packer: - */ - __pyx_v_7msgpack_BUFF_SIZE = 2048; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 - * raise TypeError, "can't serialize %r" % (o,) - * - * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< - * self.__pack(obj) - * if flush: - */ - __pyx_t_1 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_k_1 = __pyx_t_1; - __pyx_t_1 = 0; - __Pyx_GIVEREF(__pyx_k_1); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 - * cdef class Unpacker: - * """Do nothing. This function is for symmetric to Packer""" - * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< - */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); - __Pyx_GIVEREF(__pyx_1); - __pyx_1 = 0; - __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - PyType_Modified(__pyx_ptype_7msgpack_Unpacker); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_2); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack"); - Py_DECREF(__pyx_m); __pyx_m = 0; - __pyx_L0:; - __Pyx_FinishRefcountContext(); - #if PY_MAJOR_VERSION < 3 - return; - #else - return __pyx_m; - #endif -} - -static const char *__pyx_filenames[] = { - "msgpack.pyx", -}; - -/* Runtime support code */ - -static void __pyx_init_filenames(void) { - __pyx_f = __pyx_filenames; -} - -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, - PyObject* kw_name) -{ - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION >= 3 - "%s() got multiple values for keyword argument '%U'", func_name, kw_name); - #else - "%s() got multiple values for keyword argument '%s'", func_name, - PyString_AS_STRING(kw_name)); - #endif -} - -static void __Pyx_RaiseArgtupleInvalid( - const char* func_name, - int exact, - Py_ssize_t num_min, - Py_ssize_t num_max, - Py_ssize_t num_found) -{ - Py_ssize_t num_expected; - const char *number, *more_or_less; - - if (num_found < num_min) { - num_expected = num_min; - more_or_less = "at least"; - } else { - num_expected = num_max; - more_or_less = "at most"; - } - if (exact) { - more_or_less = "exactly"; - } - number = (num_expected == 1) ? "" : "s"; - PyErr_Format(PyExc_TypeError, - #if PY_VERSION_HEX < 0x02050000 - "%s() takes %s %d positional argument%s (%d given)", - #else - "%s() takes %s %zd positional argument%s (%zd given)", - #endif - func_name, more_or_less, num_expected, number, num_found); -} - -static int __Pyx_ParseOptionalKeywords( - PyObject *kwds, - PyObject **argnames[], - PyObject *kwds2, - PyObject *values[], - Py_ssize_t num_pos_args, - const char* function_name) -{ - PyObject *key = 0, *value = 0; - Py_ssize_t pos = 0; - PyObject*** name; - PyObject*** first_kw_arg = argnames + num_pos_args; - - while (PyDict_Next(kwds, &pos, &key, &value)) { - name = first_kw_arg; - while (*name && (**name != key)) name++; - if (*name) { - values[name-argnames] = value; - } else { - #if PY_MAJOR_VERSION < 3 - if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) { - #else - if (unlikely(!PyUnicode_CheckExact(key)) && unlikely(!PyUnicode_Check(key))) { - #endif - goto invalid_keyword_type; - } else { - for (name = first_kw_arg; *name; name++) { - #if PY_MAJOR_VERSION >= 3 - if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && - PyUnicode_Compare(**name, key) == 0) break; - #else - if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && - _PyString_Eq(**name, key)) break; - #endif - } - if (*name) { - values[name-argnames] = value; - } else { - /* unexpected keyword found */ - for (name=argnames; name != first_kw_arg; name++) { - if (**name == key) goto arg_passed_twice; - #if PY_MAJOR_VERSION >= 3 - if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && - PyUnicode_Compare(**name, key) == 0) goto arg_passed_twice; - #else - if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && - _PyString_Eq(**name, key)) goto arg_passed_twice; - #endif - } - if (kwds2) { - if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; - } else { - goto invalid_keyword; - } - } - } - } - } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, **name); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%s() got an unexpected keyword argument '%s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; -} - -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list) { - PyObject *__import__ = 0; - PyObject *empty_list = 0; - PyObject *module = 0; - PyObject *global_dict = 0; - PyObject *empty_dict = 0; - PyObject *list; - __import__ = __Pyx_GetAttrString(__pyx_b, "__import__"); - if (!__import__) - goto bad; - if (from_list) - list = from_list; - else { - empty_list = PyList_New(0); - if (!empty_list) - goto bad; - list = empty_list; - } - global_dict = PyModule_GetDict(__pyx_m); - if (!global_dict) - goto bad; - empty_dict = PyDict_New(); - if (!empty_dict) - goto bad; - module = PyObject_CallFunctionObjArgs(__import__, - name, global_dict, empty_dict, list, NULL); -bad: - Py_XDECREF(empty_list); - Py_XDECREF(__import__); - Py_XDECREF(empty_dict); - return module; -} - -static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) { - PyObject *result; - result = PyObject_GetAttr(dict, name); - if (!result) - PyErr_SetObject(PyExc_NameError, name); - return result; -} - -static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { - PyErr_Format(PyExc_ValueError, - #if PY_VERSION_HEX < 0x02050000 - "need more than %d value%s to unpack", (int)index, - #else - "need more than %zd value%s to unpack", index, - #endif - (index == 1) ? "" : "s"); -} - -static INLINE void __Pyx_RaiseTooManyValuesError(void) { - PyErr_SetString(PyExc_ValueError, "too many values to unpack"); -} - -static PyObject *__Pyx_UnpackItem(PyObject *iter, Py_ssize_t index) { - PyObject *item; - if (!(item = PyIter_Next(iter))) { - if (!PyErr_Occurred()) { - __Pyx_RaiseNeedMoreValuesError(index); - } - } - return item; -} - -static int __Pyx_EndUnpack(PyObject *iter) { - PyObject *item; - if ((item = PyIter_Next(iter))) { - Py_DECREF(item); - __Pyx_RaiseTooManyValuesError(); - return -1; - } - else if (!PyErr_Occurred()) - return 0; - else - return -1; -} - -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) { - Py_XINCREF(type); - Py_XINCREF(value); - Py_XINCREF(tb); - /* First, check the traceback argument, replacing None with NULL. */ - if (tb == Py_None) { - Py_DECREF(tb); - tb = 0; - } - else if (tb != NULL && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } - /* Next, replace a missing value with None */ - if (value == NULL) { - value = Py_None; - Py_INCREF(value); - } - #if PY_VERSION_HEX < 0x02050000 - if (!PyClass_Check(type)) - #else - if (!PyType_Check(type)) - #endif - { - /* Raising an instance. The value should be a dummy. */ - if (value != Py_None) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - /* Normalize to raise , */ - Py_DECREF(value); - value = type; - #if PY_VERSION_HEX < 0x02050000 - if (PyInstance_Check(type)) { - type = (PyObject*) ((PyInstanceObject*)type)->in_class; - Py_INCREF(type); - } - else { - type = 0; - PyErr_SetString(PyExc_TypeError, - "raise: exception must be an old-style class or instance"); - goto raise_error; - } - #else - type = (PyObject*) Py_TYPE(type); - Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; - } - #endif - } - __Pyx_ErrRestore(type, value, tb); - return; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(tb); - return; -} - -static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyThreadState *tstate = PyThreadState_GET(); - -#if PY_MAJOR_VERSION >= 3 - /* Note: this is a temporary work-around to prevent crashes in Python 3.0 */ - if ((tstate->exc_type != NULL) & (tstate->exc_type != Py_None)) { - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - PyErr_NormalizeException(&type, &value, &tb); - PyErr_NormalizeException(&tmp_type, &tmp_value, &tmp_tb); - tstate->exc_type = 0; - tstate->exc_value = 0; - tstate->exc_traceback = 0; - PyException_SetContext(value, tmp_value); - Py_DECREF(tmp_type); - Py_XDECREF(tmp_tb); - } -#endif - - tmp_type = tstate->curexc_type; - tmp_value = tstate->curexc_value; - tmp_tb = tstate->curexc_traceback; - tstate->curexc_type = type; - tstate->curexc_value = value; - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} - -static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) { - PyThreadState *tstate = PyThreadState_GET(); - *type = tstate->curexc_type; - *value = tstate->curexc_value; - *tb = tstate->curexc_traceback; - - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -} - - -static INLINE int __Pyx_StrEq(const char *s1, const char *s2) { - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - return *s1 == *s2; -} - -static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { - if (sizeof(unsigned char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned char)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned char"); - return (unsigned char)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned char"); - return (unsigned char)-1; - } - return (unsigned char)val; - } - return (unsigned char)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject* x) { - if (sizeof(unsigned short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned short)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned short"); - return (unsigned short)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned short"); - return (unsigned short)-1; - } - return (unsigned short)val; - } - return (unsigned short)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject* x) { - if (sizeof(unsigned int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned int)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned int"); - return (unsigned int)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned int"); - return (unsigned int)-1; - } - return (unsigned int)val; - } - return (unsigned int)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE char __Pyx_PyInt_AsChar(PyObject* x) { - if (sizeof(char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (char)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to char"); - return (char)-1; - } - return (char)val; - } - return (char)__Pyx_PyInt_AsLong(x); -} - -static INLINE short __Pyx_PyInt_AsShort(PyObject* x) { - if (sizeof(short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (short)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to short"); - return (short)-1; - } - return (short)val; - } - return (short)__Pyx_PyInt_AsLong(x); -} - -static INLINE int __Pyx_PyInt_AsInt(PyObject* x) { - if (sizeof(int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (int)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int)-1; - } - return (int)val; - } - return (int)__Pyx_PyInt_AsLong(x); -} - -static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject* x) { - if (sizeof(signed char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed char)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed char"); - return (signed char)-1; - } - return (signed char)val; - } - return (signed char)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject* x) { - if (sizeof(signed short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed short)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed short"); - return (signed short)-1; - } - return (signed short)val; - } - return (signed short)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject* x) { - if (sizeof(signed int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed int)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed int"); - return (signed int)-1; - } - return (signed int)val; - } - return (signed int)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned long"); - return (unsigned long)-1; - } - return (unsigned long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - if (unlikely(Py_SIZE(x) < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned long"); - return (unsigned long)-1; - } - return PyLong_AsUnsignedLong(x); - } else { - unsigned long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (unsigned long)-1; - val = __Pyx_PyInt_AsUnsignedLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned PY_LONG_LONG"); - return (unsigned PY_LONG_LONG)-1; - } - return (unsigned PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - if (unlikely(Py_SIZE(x) < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned PY_LONG_LONG"); - return (unsigned PY_LONG_LONG)-1; - } - return PyLong_AsUnsignedLongLong(x); - } else { - unsigned PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (unsigned PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsUnsignedLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE long __Pyx_PyInt_AsLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLong(x); - } else { - long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (long)-1; - val = __Pyx_PyInt_AsLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLongLong(x); - } else { - PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (signed long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLong(x); - } else { - signed long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (signed long)-1; - val = __Pyx_PyInt_AsSignedLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (signed PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLongLong(x); - } else { - signed PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (signed PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsSignedLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static void __Pyx_WriteUnraisable(const char *name) { - PyObject *old_exc, *old_val, *old_tb; - PyObject *ctx; - __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); - #if PY_MAJOR_VERSION < 3 - ctx = PyString_FromString(name); - #else - ctx = PyUnicode_FromString(name); - #endif - __Pyx_ErrRestore(old_exc, old_val, old_tb); - if (!ctx) { - PyErr_WriteUnraisable(Py_None); - } else { - PyErr_WriteUnraisable(ctx); - Py_DECREF(ctx); - } -} - -static int __Pyx_SetVtable(PyObject *dict, void *vtable) { - PyObject *pycobj = 0; - int result; - - pycobj = PyCObject_FromVoidPtr(vtable, 0); - if (!pycobj) - goto bad; - if (PyDict_SetItemString(dict, "__pyx_vtable__", pycobj) < 0) - goto bad; - result = 0; - goto done; - -bad: - result = -1; -done: - Py_XDECREF(pycobj); - return result; -} - -#include "compile.h" -#include "frameobject.h" -#include "traceback.h" - -static void __Pyx_AddTraceback(const char *funcname) { - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - PyObject *py_globals = 0; - PyObject *empty_string = 0; - PyCodeObject *py_code = 0; - PyFrameObject *py_frame = 0; - - #if PY_MAJOR_VERSION < 3 - py_srcfile = PyString_FromString(__pyx_filename); - #else - py_srcfile = PyUnicode_FromString(__pyx_filename); - #endif - if (!py_srcfile) goto bad; - if (__pyx_clineno) { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); - #else - py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); - #endif - } - else { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); - #endif - } - if (!py_funcname) goto bad; - py_globals = PyModule_GetDict(__pyx_m); - if (!py_globals) goto bad; - #if PY_MAJOR_VERSION < 3 - empty_string = PyString_FromStringAndSize("", 0); - #else - empty_string = PyBytes_FromStringAndSize("", 0); - #endif - if (!empty_string) goto bad; - py_code = PyCode_New( - 0, /*int argcount,*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*int kwonlyargcount,*/ - #endif - 0, /*int nlocals,*/ - 0, /*int stacksize,*/ - 0, /*int flags,*/ - empty_string, /*PyObject *code,*/ - __pyx_empty_tuple, /*PyObject *consts,*/ - __pyx_empty_tuple, /*PyObject *names,*/ - __pyx_empty_tuple, /*PyObject *varnames,*/ - __pyx_empty_tuple, /*PyObject *freevars,*/ - __pyx_empty_tuple, /*PyObject *cellvars,*/ - py_srcfile, /*PyObject *filename,*/ - py_funcname, /*PyObject *name,*/ - __pyx_lineno, /*int firstlineno,*/ - empty_string /*PyObject *lnotab*/ - ); - if (!py_code) goto bad; - py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - py_globals, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ - ); - if (!py_frame) goto bad; - py_frame->f_lineno = __pyx_lineno; - PyTraceBack_Here(py_frame); -bad: - Py_XDECREF(py_srcfile); - Py_XDECREF(py_funcname); - Py_XDECREF(empty_string); - Py_XDECREF(py_code); - Py_XDECREF(py_frame); -} - -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { - while (t->p) { - #if PY_MAJOR_VERSION < 3 - if (t->is_unicode && (!t->is_identifier)) { - *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); - } else if (t->intern) { - *t->p = PyString_InternFromString(t->s); - } else { - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); - } - #else /* Python 3+ has unicode identifiers */ - if (t->is_identifier || (t->is_unicode && t->intern)) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->is_unicode) { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif - if (!*t->p) - return -1; - ++t; - } - return 0; -} - -/* Type Conversion Functions */ - -static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { - if (x == Py_True) return 1; - else if ((x == Py_False) | (x == Py_None)) return 0; - else return PyObject_IsTrue(x); -} - -static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { - PyNumberMethods *m; - const char *name = NULL; - PyObject *res = NULL; -#if PY_VERSION_HEX < 0x03000000 - if (PyInt_Check(x) || PyLong_Check(x)) -#else - if (PyLong_Check(x)) -#endif - return Py_INCREF(x), x; - m = Py_TYPE(x)->tp_as_number; -#if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } -#else - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Long(x); - } -#endif - if (res) { -#if PY_VERSION_HEX < 0x03000000 - if (!PyInt_Check(res) && !PyLong_Check(res)) { -#else - if (!PyLong_Check(res)) { -#endif - PyErr_Format(PyExc_TypeError, - "__%s__ returned non-%s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; - } - } - else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "an integer is required"); - } - return res; -} - -static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { - Py_ssize_t ival; - PyObject* x = PyNumber_Index(b); - if (!x) return -1; - ival = PyInt_AsSsize_t(x); - Py_DECREF(x); - return ival; -} - -static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { -#if PY_VERSION_HEX < 0x02050000 - if (ival <= LONG_MAX) - return PyInt_FromLong((long)ival); - else { - unsigned char *bytes = (unsigned char *) &ival; - int one = 1; int little = (int)*(unsigned char*)&one; - return _PyLong_FromByteArray(bytes, sizeof(size_t), little, 0); - } -#else - return PyInt_FromSize_t(ival); -#endif -} - -static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { - unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); - if (unlikely(val == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())) { - return (size_t)-1; - } else if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to size_t"); - return (size_t)-1; - } - return (size_t)val; -} - - diff --git a/python/msgpack/__init__.py b/python/msgpack/__init__.py new file mode 100644 index 0000000..797b29c --- /dev/null +++ b/python/msgpack/__init__.py @@ -0,0 +1,3 @@ +# coding: utf-8 +from _msgpack import * + diff --git a/python/msgpack.pyx b/python/msgpack/_msgpack.pyx similarity index 100% rename from python/msgpack.pyx rename to python/msgpack/_msgpack.pyx diff --git a/python/pack.h b/python/msgpack/pack.h similarity index 100% rename from python/pack.h rename to python/msgpack/pack.h diff --git a/python/unpack.h b/python/msgpack/unpack.h similarity index 100% rename from python/unpack.h rename to python/msgpack/unpack.h diff --git a/python/setup.py b/python/setup.py index 65ca412..56b3faa 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,12 +1,13 @@ from distutils.core import setup, Extension +from Cython.Distutils import build_ext import os version = '0.0.1' PACKAGE_ROOT = os.getcwdu() INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') -msgpack_mod = Extension('msgpack', - sources=['msgpack.c'], +msgpack_mod = Extension('msgpack._msgpack', + sources=['msgpack/_msgpack.pyx'], include_dirs=[INCLUDE_PATH]) desc = 'MessagePack serializer/desirializer.' @@ -28,6 +29,7 @@ setup(name='msgpack', author='Naoki INADA', author_email='songofacandy@gmail.com', version=version, + cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], description=desc, long_description=long_desc, From 74a5fb3592d363092ab73bd90de6ab1d550edb2b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 10:58:09 +0900 Subject: [PATCH 0154/1648] Make msgpack package instead of module. --- MANIFEST | 4 +- msgpack.c | 3438 --------------------------- msgpack/__init__.py | 3 + msgpack.pyx => msgpack/_msgpack.pyx | 0 pack.h => msgpack/pack.h | 0 unpack.h => msgpack/unpack.h | 0 setup.py | 6 +- 7 files changed, 9 insertions(+), 3442 deletions(-) delete mode 100644 msgpack.c create mode 100644 msgpack/__init__.py rename msgpack.pyx => msgpack/_msgpack.pyx (100%) rename pack.h => msgpack/pack.h (100%) rename unpack.h => msgpack/unpack.h (100%) diff --git a/MANIFEST b/MANIFEST index dc042ae..f2da7da 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1,7 +1,7 @@ msgpack.c setup.py -pack.h -unpack.h +msgpack/pack.h +msgpack/unpack.h include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h diff --git a/msgpack.c b/msgpack.c deleted file mode 100644 index 821f65b..0000000 --- a/msgpack.c +++ /dev/null @@ -1,3438 +0,0 @@ -/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include "structmember.h" -#ifndef Py_PYTHON_H - #error Python headers needed to compile C extensions, please install development version of Python. -#endif -#ifndef PY_LONG_LONG - #define PY_LONG_LONG LONG_LONG -#endif -#ifndef DL_EXPORT - #define DL_EXPORT(t) t -#endif -#if PY_VERSION_HEX < 0x02040000 - #define METH_COEXIST 0 - #define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type) -#endif -#if PY_VERSION_HEX < 0x02050000 - typedef int Py_ssize_t; - #define PY_SSIZE_T_MAX INT_MAX - #define PY_SSIZE_T_MIN INT_MIN - #define PY_FORMAT_SIZE_T "" - #define PyInt_FromSsize_t(z) PyInt_FromLong(z) - #define PyInt_AsSsize_t(o) PyInt_AsLong(o) - #define PyNumber_Index(o) PyNumber_Int(o) - #define PyIndex_Check(o) PyNumber_Check(o) -#endif -#if PY_VERSION_HEX < 0x02060000 - #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) - #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) - #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) - #define PyVarObject_HEAD_INIT(type, size) \ - PyObject_HEAD_INIT(type) size, - #define PyType_Modified(t) - - typedef struct { - void *buf; - PyObject *obj; - Py_ssize_t len; - Py_ssize_t itemsize; - int readonly; - int ndim; - char *format; - Py_ssize_t *shape; - Py_ssize_t *strides; - Py_ssize_t *suboffsets; - void *internal; - } Py_buffer; - - #define PyBUF_SIMPLE 0 - #define PyBUF_WRITABLE 0x0001 - #define PyBUF_FORMAT 0x0004 - #define PyBUF_ND 0x0008 - #define PyBUF_STRIDES (0x0010 | PyBUF_ND) - #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES) - #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES) - #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES) - #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES) - -#endif -#if PY_MAJOR_VERSION < 3 - #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" -#else - #define __Pyx_BUILTIN_MODULE_NAME "builtins" -#endif -#if PY_MAJOR_VERSION >= 3 - #define Py_TPFLAGS_CHECKTYPES 0 - #define Py_TPFLAGS_HAVE_INDEX 0 -#endif -#if (PY_VERSION_HEX < 0x02060000) || (PY_MAJOR_VERSION >= 3) - #define Py_TPFLAGS_HAVE_NEWBUFFER 0 -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyBaseString_Type PyUnicode_Type - #define PyString_Type PyBytes_Type - #define PyString_CheckExact PyBytes_CheckExact - #define PyInt_Type PyLong_Type - #define PyInt_Check(op) PyLong_Check(op) - #define PyInt_CheckExact(op) PyLong_CheckExact(op) - #define PyInt_FromString PyLong_FromString - #define PyInt_FromUnicode PyLong_FromUnicode - #define PyInt_FromLong PyLong_FromLong - #define PyInt_FromSize_t PyLong_FromSize_t - #define PyInt_FromSsize_t PyLong_FromSsize_t - #define PyInt_AsLong PyLong_AsLong - #define PyInt_AS_LONG PyLong_AS_LONG - #define PyInt_AsSsize_t PyLong_AsSsize_t - #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask - #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define PyBytes_Type PyString_Type -#endif -#if PY_MAJOR_VERSION >= 3 - #define PyMethod_New(func, self, klass) PyInstanceMethod_New(func) -#endif -#if !defined(WIN32) && !defined(MS_WINDOWS) - #ifndef __stdcall - #define __stdcall - #endif - #ifndef __cdecl - #define __cdecl - #endif - #ifndef __fastcall - #define __fastcall - #endif -#else - #define _USE_MATH_DEFINES -#endif -#if PY_VERSION_HEX < 0x02050000 - #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),((char *)(n))) - #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),((char *)(n)),(a)) - #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),((char *)(n))) -#else - #define __Pyx_GetAttrString(o,n) PyObject_GetAttrString((o),(n)) - #define __Pyx_SetAttrString(o,n,a) PyObject_SetAttrString((o),(n),(a)) - #define __Pyx_DelAttrString(o,n) PyObject_DelAttrString((o),(n)) -#endif -#if PY_VERSION_HEX < 0x02050000 - #define __Pyx_NAMESTR(n) ((char *)(n)) - #define __Pyx_DOCSTR(n) ((char *)(n)) -#else - #define __Pyx_NAMESTR(n) (n) - #define __Pyx_DOCSTR(n) (n) -#endif -#ifdef __cplusplus -#define __PYX_EXTERN_C extern "C" -#else -#define __PYX_EXTERN_C extern -#endif -#include -#define __PYX_HAVE_API__msgpack -#include "stdlib.h" -#include "string.h" -#include "pack.h" -#include "unpack.h" -#define __PYX_USE_C99_COMPLEX defined(_Complex_I) - - -#ifdef __GNUC__ -#define INLINE __inline__ -#elif _WIN32 -#define INLINE __inline -#else -#define INLINE -#endif - -typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/ - - - -static int __pyx_skip_dispatch = 0; - - -/* Type Conversion Predeclarations */ - -#if PY_MAJOR_VERSION < 3 -#define __Pyx_PyBytes_FromString PyString_FromString -#define __Pyx_PyBytes_FromStringAndSize PyString_FromStringAndSize -#define __Pyx_PyBytes_AsString PyString_AsString -#else -#define __Pyx_PyBytes_FromString PyBytes_FromString -#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize -#define __Pyx_PyBytes_AsString PyBytes_AsString -#endif - -#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False)) -static INLINE int __Pyx_PyObject_IsTrue(PyObject*); -static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x); - -#if !defined(T_PYSSIZET) -#if PY_VERSION_HEX < 0x02050000 -#define T_PYSSIZET T_INT -#elif !defined(T_LONGLONG) -#define T_PYSSIZET \ - ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ - ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : -1)) -#else -#define T_PYSSIZET \ - ((sizeof(Py_ssize_t) == sizeof(int)) ? T_INT : \ - ((sizeof(Py_ssize_t) == sizeof(long)) ? T_LONG : \ - ((sizeof(Py_ssize_t) == sizeof(PY_LONG_LONG)) ? T_LONGLONG : -1))) -#endif -#endif - -#if !defined(T_SIZET) -#if !defined(T_ULONGLONG) -#define T_SIZET \ - ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ - ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : -1)) -#else -#define T_SIZET \ - ((sizeof(size_t) == sizeof(unsigned int)) ? T_UINT : \ - ((sizeof(size_t) == sizeof(unsigned long)) ? T_ULONG : \ - ((sizeof(size_t) == sizeof(unsigned PY_LONG_LONG)) ? T_ULONGLONG : -1))) -#endif -#endif - -static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); -static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); -static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); - -#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) - - -#ifdef __GNUC__ -/* Test for GCC > 2.95 */ -#if __GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)) -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) -#else /* __GNUC__ > 2 ... */ -#define likely(x) (x) -#define unlikely(x) (x) -#endif /* __GNUC__ > 2 ... */ -#else /* __GNUC__ */ -#define likely(x) (x) -#define unlikely(x) (x) -#endif /* __GNUC__ */ - -static PyObject *__pyx_m; -static PyObject *__pyx_b; -static PyObject *__pyx_empty_tuple; -static int __pyx_lineno; -static int __pyx_clineno = 0; -static const char * __pyx_cfilenm= __FILE__; -static const char *__pyx_filename; -static const char **__pyx_f; - - -#ifdef CYTHON_REFNANNY -typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*NewContext)(const char*, int, const char*); - void (*FinishContext)(void**); -} __Pyx_RefnannyAPIStruct; -static __Pyx_RefnannyAPIStruct *__Pyx_Refnanny = NULL; -#define __Pyx_ImportRefcountAPI(name) (__Pyx_RefnannyAPIStruct *) PyCObject_Import((char *)name, (char *)"RefnannyAPI") -#define __Pyx_INCREF(r) __Pyx_Refnanny->INCREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_DECREF(r) __Pyx_Refnanny->DECREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_GOTREF(r) __Pyx_Refnanny->GOTREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_GIVEREF(r) __Pyx_Refnanny->GIVEREF(__pyx_refchk, (PyObject *)(r), __LINE__) -#define __Pyx_XDECREF(r) if((r) == NULL) ; else __Pyx_DECREF(r) -#define __Pyx_SetupRefcountContext(name) void* __pyx_refchk = __Pyx_Refnanny->NewContext((name), __LINE__, __FILE__) -#define __Pyx_FinishRefcountContext() __Pyx_Refnanny->FinishContext(&__pyx_refchk) -#else -#define __Pyx_INCREF(r) Py_INCREF(r) -#define __Pyx_DECREF(r) Py_DECREF(r) -#define __Pyx_GOTREF(r) -#define __Pyx_GIVEREF(r) -#define __Pyx_XDECREF(r) Py_XDECREF(r) -#define __Pyx_SetupRefcountContext(name) -#define __Pyx_FinishRefcountContext() -#endif /* CYTHON_REFNANNY */ -#define __Pyx_XGIVEREF(r) if((r) == NULL) ; else __Pyx_GIVEREF(r) -#define __Pyx_XGOTREF(r) if((r) == NULL) ; else __Pyx_GOTREF(r) - -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, PyObject* kw_name); /*proto*/ - -static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, - Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); /*proto*/ - -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ - -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list); /*proto*/ - -static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/ - -static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); - -static INLINE void __Pyx_RaiseTooManyValuesError(void); - -static PyObject *__Pyx_UnpackItem(PyObject *, Py_ssize_t index); /*proto*/ -static int __Pyx_EndUnpack(PyObject *); /*proto*/ - -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ - -static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ -static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ - -static INLINE int __Pyx_StrEq(const char *, const char *); /*proto*/ - -static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject *); - -static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject *); - -static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject *); - -static INLINE char __Pyx_PyInt_AsChar(PyObject *); - -static INLINE short __Pyx_PyInt_AsShort(PyObject *); - -static INLINE int __Pyx_PyInt_AsInt(PyObject *); - -static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject *); - -static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject *); - -static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject *); - -static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject *); - -static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject *); - -static INLINE long __Pyx_PyInt_AsLong(PyObject *); - -static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject *); - -static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject *); - -static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject *); - -static void __Pyx_WriteUnraisable(const char *name); /*proto*/ - -static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ - -static void __Pyx_AddTraceback(const char *funcname); /*proto*/ - -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ - -/* Type declarations */ - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 - * cdef int BUFF_SIZE=2*1024 - * - * cdef class Packer: # <<<<<<<<<<<<<< - * """Packer that pack data into strm. - * - */ - -struct __pyx_obj_7msgpack_Packer { - PyObject_HEAD - struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtab; - char *buff; - unsigned int length; - unsigned int allocated; - struct msgpack_packer pk; - PyObject *strm; -}; - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 - * return unpacks(packed) - * - * cdef class Unpacker: # <<<<<<<<<<<<<< - * """Do nothing. This function is for symmetric to Packer""" - * unpack = staticmethod(unpacks) - */ - -struct __pyx_obj_7msgpack_Unpacker { - PyObject_HEAD -}; - - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 - * cdef int BUFF_SIZE=2*1024 - * - * cdef class Packer: # <<<<<<<<<<<<<< - * """Packer that pack data into strm. - * - */ - -struct __pyx_vtabstruct_7msgpack_Packer { - PyObject *(*__pack)(struct __pyx_obj_7msgpack_Packer *, PyObject *); -}; -static struct __pyx_vtabstruct_7msgpack_Packer *__pyx_vtabptr_7msgpack_Packer; -/* Module declarations from msgpack */ - -static PyTypeObject *__pyx_ptype_7msgpack_Packer = 0; -static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; -static int __pyx_v_7msgpack_BUFF_SIZE; -static PyObject *__pyx_k_1 = 0; -static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ -#define __Pyx_MODULE_NAME "msgpack" -int __pyx_module_is_main_msgpack = 0; - -/* Implementation of msgpack */ -static char __pyx_k___main__[] = "__main__"; -static PyObject *__pyx_kp___main__; -static char __pyx_k___init__[] = "__init__"; -static PyObject *__pyx_kp___init__; -static char __pyx_k___del__[] = "__del__"; -static PyObject *__pyx_kp___del__; -static char __pyx_k_flush[] = "flush"; -static PyObject *__pyx_kp_flush; -static char __pyx_k_pack_list[] = "pack_list"; -static PyObject *__pyx_kp_pack_list; -static char __pyx_k_pack_dict[] = "pack_dict"; -static PyObject *__pyx_kp_pack_dict; -static char __pyx_k_pack[] = "pack"; -static PyObject *__pyx_kp_pack; -static char __pyx_k_unpack[] = "unpack"; -static PyObject *__pyx_kp_unpack; -static char __pyx_k_strm[] = "strm"; -static PyObject *__pyx_kp_strm; -static char __pyx_k_size[] = "size"; -static PyObject *__pyx_kp_size; -static char __pyx_k_len[] = "len"; -static PyObject *__pyx_kp_len; -static char __pyx_k_obj[] = "obj"; -static PyObject *__pyx_kp_obj; -static char __pyx_k_o[] = "o"; -static PyObject *__pyx_kp_o; -static char __pyx_k_stream[] = "stream"; -static PyObject *__pyx_kp_stream; -static char __pyx_k_packed_bytes[] = "packed_bytes"; -static PyObject *__pyx_kp_packed_bytes; -static char __pyx_k_cStringIO[] = "cStringIO"; -static PyObject *__pyx_kp_cStringIO; -static char __pyx_k_StringIO[] = "StringIO"; -static PyObject *__pyx_kp_StringIO; -static char __pyx_k_staticmethod[] = "staticmethod"; -static PyObject *__pyx_kp_staticmethod; -static char __pyx_k_unpacks[] = "unpacks"; -static PyObject *__pyx_kp_unpacks; -static char __pyx_k_write[] = "write"; -static PyObject *__pyx_kp_write; -static char __pyx_k_2[] = "flush"; -static PyObject *__pyx_kp_2; -static char __pyx_k_encode[] = "encode"; -static PyObject *__pyx_kp_encode; -static char __pyx_k_iteritems[] = "iteritems"; -static PyObject *__pyx_kp_iteritems; -static char __pyx_k_TypeError[] = "TypeError"; -static PyObject *__pyx_kp_TypeError; -static char __pyx_k_getvalue[] = "getvalue"; -static PyObject *__pyx_kp_getvalue; -static char __pyx_k_read[] = "read"; -static PyObject *__pyx_kp_read; -static PyObject *__pyx_builtin_staticmethod; -static PyObject *__pyx_builtin_TypeError; -static PyObject *__pyx_kp_3; -static PyObject *__pyx_kp_4; -static char __pyx_k_3[] = "utf-8"; -static char __pyx_k_4[] = "can't serialize %r"; - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 - * cdef object strm - * - * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< - * if size <= 0: - * size = BUFF_SIZE - */ - -static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_strm = 0; - int __pyx_v_size; - int __pyx_r; - int __pyx_t_1; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_strm,&__pyx_kp_size,0}; - __Pyx_SetupRefcountContext("__init__"); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_strm); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 1) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_size); - if (unlikely(value)) { values[1] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "__init__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_strm = values[0]; - if (values[1]) { - __pyx_v_size = __Pyx_PyInt_AsInt(values[1]); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } else { - __pyx_v_size = 0; - } - } else { - __pyx_v_size = 0; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: __pyx_v_size = __Pyx_PyInt_AsInt(PyTuple_GET_ITEM(__pyx_args, 1)); if (unlikely((__pyx_v_size == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - case 1: __pyx_v_strm = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.Packer.__init__"); - return -1; - __pyx_L4_argument_unpacking_done:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 - * - * def __init__(self, strm, int size=0): - * if size <= 0: # <<<<<<<<<<<<<< - * size = BUFF_SIZE - * - */ - __pyx_t_1 = (__pyx_v_size <= 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 - * def __init__(self, strm, int size=0): - * if size <= 0: - * size = BUFF_SIZE # <<<<<<<<<<<<<< - * - * self.strm = strm - */ - __pyx_v_size = __pyx_v_7msgpack_BUFF_SIZE; - goto __pyx_L6; - } - __pyx_L6:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 - * size = BUFF_SIZE - * - * self.strm = strm # <<<<<<<<<<<<<< - * self.buff = malloc(size) - * self.allocated = size - */ - __Pyx_INCREF(__pyx_v_strm); - __Pyx_GIVEREF(__pyx_v_strm); - __Pyx_GOTREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); - __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 - * - * self.strm = strm - * self.buff = malloc(size) # <<<<<<<<<<<<<< - * self.allocated = size - * self.length = 0 - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 - * self.strm = strm - * self.buff = malloc(size) - * self.allocated = size # <<<<<<<<<<<<<< - * self.length = 0 - * - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 - * self.buff = malloc(size) - * self.allocated = size - * self.length = 0 # <<<<<<<<<<<<<< - * - * msgpack_packer_init(&self.pk, self, _packer_write) - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 - * self.length = 0 - * - * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< - * - * def __del__(self): - */ - msgpack_packer_init((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), ((void *)__pyx_v_self), ((int (*)(void *, const char*, unsigned int))__pyx_f_7msgpack__packer_write)); - - __pyx_r = 0; - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 - * msgpack_packer_init(&self.pk, self, _packer_write) - * - * def __del__(self): # <<<<<<<<<<<<<< - * free(self.buff); - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObject *unused) { - PyObject *__pyx_r = NULL; - __Pyx_SetupRefcountContext("__del__"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 - * - * def __del__(self): - * free(self.buff); # <<<<<<<<<<<<<< - * - * def flush(self): - */ - free(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 - * free(self.buff); - * - * def flush(self): # <<<<<<<<<<<<<< - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_flush[] = "Flash local buffer and output stream if it has 'flush()' method."; -static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObject *unused) { - PyObject *__pyx_r = NULL; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_SetupRefcountContext("flush"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 - * def flush(self): - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: # <<<<<<<<<<<<<< - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 - */ - __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 - * """Flash local buffer and output stream if it has 'flush()' method.""" - * if self.length > 0: - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< - * self.length = 0 - * if hasattr(self.strm, 'flush'): - */ - __pyx_t_2 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff, ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_4)); - PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 - * if self.length > 0: - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 # <<<<<<<<<<<<<< - * if hasattr(self.strm, 'flush'): - * self.strm.flush() - */ - ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - goto __pyx_L5; - } - __pyx_L5:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 - * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - * self.length = 0 - * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< - * self.strm.flush() - * - */ - __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 - * self.length = 0 - * if hasattr(self.strm, 'flush'): - * self.strm.flush() # <<<<<<<<<<<<<< - * - * def pack_list(self, len): - */ - __pyx_t_3 = PyObject_GetAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - goto __pyx_L6; - } - __pyx_L6:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_AddTraceback("msgpack.Packer.flush"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 - * self.strm.flush() - * - * def pack_list(self, len): # <<<<<<<<<<<<<< - * """Start packing sequential objects. - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_pack_list[] = "Start packing sequential objects.\n\n Example:\n\n packer.pack_list(2)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack(['foo', 'bar'])\n "; -static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { - PyObject *__pyx_r = NULL; - size_t __pyx_t_1; - __Pyx_SetupRefcountContext("pack_list"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 - * packer.pack(['foo', 'bar']) - * """ - * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< - * - * def pack_dict(self, len): - */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("msgpack.Packer.pack_list"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 - * msgpack_pack_array(&self.pk, len) - * - * def pack_dict(self, len): # <<<<<<<<<<<<<< - * """Start packing key-value objects. - * - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len); /*proto*/ -static char __pyx_doc_7msgpack_6Packer_pack_dict[] = "Start packing key-value objects.\n\n Example:\n\n packer.pack_dict(1)\n packer.pack('foo')\n packer.pack('bar')\n\n This code is same as below code:\n\n packer.pack({'foo', 'bar'})\n "; -static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyObject *__pyx_v_len) { - PyObject *__pyx_r = NULL; - size_t __pyx_t_1; - __Pyx_SetupRefcountContext("pack_dict"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 - * packer.pack({'foo', 'bar'}) - * """ - * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< - * - * cdef __pack(self, object o): - */ - __pyx_t_1 = __Pyx_PyInt_AsSize_t(__pyx_v_len); if (unlikely((__pyx_t_1 == (size_t)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->pk), __pyx_t_1); - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_AddTraceback("msgpack.Packer.pack_dict"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 - * msgpack_pack_map(&self.pk, len) - * - * cdef __pack(self, object o): # <<<<<<<<<<<<<< - * cdef long long intval - * cdef double fval - */ - -static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Packer *__pyx_v_self, PyObject *__pyx_v_o) { - PY_LONG_LONG __pyx_v_intval; - double __pyx_v_fval; - char *__pyx_v_rawval; - PyObject *__pyx_v_k; - PyObject *__pyx_v_v; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_2 = 0; - PyObject *__pyx_3 = 0; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PY_LONG_LONG __pyx_t_3; - char *__pyx_t_4; - Py_ssize_t __pyx_t_5; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - int __pyx_t_9; - int __pyx_t_10; - __Pyx_SetupRefcountContext("__pack"); - __Pyx_INCREF(__pyx_v_o); - __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); - __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 - * cdef char* rawval - * - * if o is None: # <<<<<<<<<<<<<< - * msgpack_pack_nil(&self.pk) - * elif o is True: - */ - __pyx_t_1 = (__pyx_v_o == Py_None); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 - * - * if o is None: - * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< - * elif o is True: - * msgpack_pack_true(&self.pk) - */ - msgpack_pack_nil((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 - * if o is None: - * msgpack_pack_nil(&self.pk) - * elif o is True: # <<<<<<<<<<<<<< - * msgpack_pack_true(&self.pk) - * elif o is False: - */ - __pyx_t_2 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = (__pyx_v_o == __pyx_t_2); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 - * msgpack_pack_nil(&self.pk) - * elif o is True: - * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< - * elif o is False: - * msgpack_pack_false(&self.pk) - */ - msgpack_pack_true((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 - * elif o is True: - * msgpack_pack_true(&self.pk) - * elif o is False: # <<<<<<<<<<<<<< - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): - */ - __pyx_t_2 = __Pyx_PyBool_FromLong(0); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = (__pyx_v_o == __pyx_t_2); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 - * msgpack_pack_true(&self.pk) - * elif o is False: - * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< - * elif isinstance(o, long): - * intval = o - */ - msgpack_pack_false((&__pyx_v_self->pk)); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 - * elif o is False: - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): # <<<<<<<<<<<<<< - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 - * msgpack_pack_false(&self.pk) - * elif isinstance(o, long): - * intval = o # <<<<<<<<<<<<<< - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): - */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_intval = __pyx_t_3; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 - * elif isinstance(o, long): - * intval = o - * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< - * elif isinstance(o, int): - * intval = o - */ - msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): # <<<<<<<<<<<<<< - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, int): - * intval = o # <<<<<<<<<<<<<< - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): - */ - __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_intval = __pyx_t_3; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 - * elif isinstance(o, int): - * intval = o - * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< - * elif isinstance(o, float): - * fval = 9 - */ - msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 - * intval = o - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): # <<<<<<<<<<<<<< - * fval = 9 - * msgpack_pack_double(&self.pk, fval) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 - * msgpack_pack_long_long(&self.pk, intval) - * elif isinstance(o, float): - * fval = 9 # <<<<<<<<<<<<<< - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): - */ - __pyx_v_fval = 9; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 - * elif isinstance(o, float): - * fval = 9 - * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< - * elif isinstance(o, str): - * rawval = o - */ - msgpack_pack_double((&__pyx_v_self->pk), __pyx_v_fval); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 - * fval = 9 - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): # <<<<<<<<<<<<<< - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 - * msgpack_pack_double(&self.pk, fval) - * elif isinstance(o, str): - * rawval = o # <<<<<<<<<<<<<< - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 - * elif isinstance(o, str): - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< - * elif isinstance(o, unicode): - * o = o.encode('utf-8') - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): # <<<<<<<<<<<<<< - * o = o.encode('utf-8') - * rawval = o - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, unicode): - * o = o.encode('utf-8') # <<<<<<<<<<<<<< - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_6)); - __Pyx_INCREF(__pyx_kp_3); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); - __Pyx_GIVEREF(__pyx_kp_3); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_v_o); - __pyx_v_o = __pyx_t_7; - __pyx_t_7 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 - * elif isinstance(o, unicode): - * o = o.encode('utf-8') - * rawval = o # <<<<<<<<<<<<<< - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 - * o = o.encode('utf-8') - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 - * rawval = o - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 - * msgpack_pack_raw(&self.pk, len(o)) - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): # <<<<<<<<<<<<<< - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 - * msgpack_pack_raw_body(&self.pk, rawval, len(o)) - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< - * for k,v in o.iteritems(): - * self.pack(k) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 - * elif isinstance(o, dict): - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): # <<<<<<<<<<<<<< - * self.pack(k) - * self.pack(v) - */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); - } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - } - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - break; - } - __Pyx_GOTREF(__pyx_t_6); - } - if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { - PyObject* tuple = __pyx_t_6; - __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); - __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(__pyx_v_k); - __pyx_v_k = __pyx_2; - __pyx_2 = 0; - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_3; - __pyx_3 = 0; - } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_2); - __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_3); - if (__Pyx_EndUnpack(__pyx_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(__pyx_v_k); - __pyx_v_k = __pyx_2; - __pyx_2 = 0; - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_3; - __pyx_3 = 0; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 - * msgpack_pack_map(&self.pk, len(o)) - * for k,v in o.iteritems(): - * self.pack(k) # <<<<<<<<<<<<<< - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): - */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_k); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); - __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 - * for k,v in o.iteritems(): - * self.pack(k) - * self.pack(v) # <<<<<<<<<<<<<< - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) - */ - __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_v); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); - __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L3; - } - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 - * self.pack(k) - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: - */ - __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); - if (!__pyx_t_1) { - __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); - __pyx_t_10 = __pyx_t_9; - } else { - __pyx_t_10 = __pyx_t_1; - } - if (__pyx_t_10) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 - * self.pack(v) - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< - * for v in o: - * self.pack(v) - */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 - * elif isinstance(o, tuple) or isinstance(o, list): - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: # <<<<<<<<<<<<<< - * self.pack(v) - * else: - */ - if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); - } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); - } - for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { - if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - break; - } - __Pyx_GOTREF(__pyx_t_6); - } - __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_t_6; - __pyx_t_6 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 - * msgpack_pack_array(&self.pk, len(o)) - * for v in o: - * self.pack(v) # <<<<<<<<<<<<<< - * else: - * # TODO: Serialize with defalt() like simplejson. - */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_v); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); - __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - goto __pyx_L3; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 - * else: - * # TODO: Serialize with defalt() like simplejson. - * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< - * - * def pack(self, obj, flush=True): - */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_7)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; - __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - __pyx_L3:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_2); - __Pyx_XDECREF(__pyx_3); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_6); - __Pyx_XDECREF(__pyx_t_7); - __Pyx_XDECREF(__pyx_t_8); - __Pyx_AddTraceback("msgpack.Packer.__pack"); - __pyx_r = 0; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_k); - __Pyx_DECREF(__pyx_v_v); - __Pyx_DECREF(__pyx_v_o); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 - * raise TypeError, "can't serialize %r" % (o,) - * - * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< - * self.__pack(obj) - * if flush: - */ - -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_obj = 0; - PyObject *__pyx_v_flush = 0; - PyObject *__pyx_r = NULL; - PyObject *__pyx_t_1 = NULL; - int __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_obj,&__pyx_kp_flush,0}; - __Pyx_SetupRefcountContext("pack"); - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - values[1] = __pyx_k_1; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_obj); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - if (kw_args > 1) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_kp_flush); - if (unlikely(value)) { values[1] = value; kw_args--; } - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_obj = values[0]; - __pyx_v_flush = values[1]; - } else { - __pyx_v_flush = __pyx_k_1; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: __pyx_v_flush = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: __pyx_v_obj = PyTuple_GET_ITEM(__pyx_args, 0); - break; - default: goto __pyx_L5_argtuple_error; - } - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.Packer.pack"); - return NULL; - __pyx_L4_argument_unpacking_done:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 - * - * def pack(self, obj, flush=True): - * self.__pack(obj) # <<<<<<<<<<<<<< - * if flush: - * self.flush() - */ - __pyx_t_1 = ((struct __pyx_vtabstruct_7msgpack_Packer *)((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->__pyx_vtab)->__pack(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self), __pyx_v_obj); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 146; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 - * def pack(self, obj, flush=True): - * self.__pack(obj) - * if flush: # <<<<<<<<<<<<<< - * self.flush() - * - */ - __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_2) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 - * self.__pack(obj) - * if flush: - * self.flush() # <<<<<<<<<<<<<< - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_self, __pyx_kp_flush); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - goto __pyx_L6; - } - __pyx_L6:; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.Packer.pack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 - * self.flush() - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< - * if packer.length + l > packer.allocated: - * if packer.length > 0: - */ - -static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__pyx_v_packer, const char* __pyx_v_b, unsigned int __pyx_v_l) { - int __pyx_r; - int __pyx_t_1; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; - __Pyx_SetupRefcountContext("_packer_write"); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 - * - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - */ - __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 - * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - * if packer.length + l > packer.allocated: - * if packer.length > 0: # <<<<<<<<<<<<<< - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: - */ - __pyx_t_1 = (__pyx_v_packer->length > 0); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 - * if packer.length + l > packer.allocated: - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyString_FromStringAndSize(__pyx_v_packer->buff, __pyx_v_packer->length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_4)); - PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); - __Pyx_GIVEREF(__pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - goto __pyx_L4; - } - __pyx_L4:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 - * if packer.length > 0: - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: # <<<<<<<<<<<<<< - * packer.strm.write(PyString_FromStringAndSize(b, l)) - * packer.length = 0 - */ - __pyx_t_1 = (__pyx_v_l > 64); - if (__pyx_t_1) { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 - * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< - * packer.length = 0 - * else: - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer->strm, __pyx_kp_write); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyString_FromStringAndSize(__pyx_v_b, __pyx_v_l); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_4); - __Pyx_GIVEREF(__pyx_t_4); - __pyx_t_4 = 0; - __pyx_t_4 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 - * if l > 64: - * packer.strm.write(PyString_FromStringAndSize(b, l)) - * packer.length = 0 # <<<<<<<<<<<<<< - * else: - * memcpy(packer.buff, b, l) - */ - __pyx_v_packer->length = 0; - goto __pyx_L5; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 - * packer.length = 0 - * else: - * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< - * packer.length = l - * else: - */ - memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 - * else: - * memcpy(packer.buff, b, l) - * packer.length = l # <<<<<<<<<<<<<< - * else: - * memcpy(packer.buff + packer.length, b, l) - */ - __pyx_v_packer->length = __pyx_v_l; - } - __pyx_L5:; - goto __pyx_L3; - } - /*else*/ { - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 - * packer.length = l - * else: - * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< - * packer.length += l - * return 0 - */ - memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 - * else: - * memcpy(packer.buff + packer.length, b, l) - * packer.length += l # <<<<<<<<<<<<<< - * return 0 - * - */ - __pyx_v_packer->length += __pyx_v_l; - } - __pyx_L3:; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 - * memcpy(packer.buff + packer.length, b, l) - * packer.length += l - * return 0 # <<<<<<<<<<<<<< - * - * def pack(object o, object stream): - */ - __pyx_r = 0; - goto __pyx_L0; - - __pyx_r = 0; - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); - __Pyx_WriteUnraisable("msgpack._packer_write"); - __pyx_r = 0; - __pyx_L0:; - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 - * return 0 - * - * def pack(object o, object stream): # <<<<<<<<<<<<<< - * packer = Packer(stream) - * packer.pack(o) - */ - -static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { - PyObject *__pyx_v_o = 0; - PyObject *__pyx_v_stream = 0; - PyObject *__pyx_v_packer; - PyObject *__pyx_r = NULL; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - static PyObject **__pyx_pyargnames[] = {&__pyx_kp_o,&__pyx_kp_stream,0}; - __Pyx_SetupRefcountContext("pack"); - __pyx_self = __pyx_self; - if (unlikely(__pyx_kwds)) { - Py_ssize_t kw_args = PyDict_Size(__pyx_kwds); - PyObject* values[2] = {0,0}; - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); - case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); - case 0: break; - default: goto __pyx_L5_argtuple_error; - } - switch (PyTuple_GET_SIZE(__pyx_args)) { - case 0: - values[0] = PyDict_GetItem(__pyx_kwds, __pyx_kp_o); - if (likely(values[0])) kw_args--; - else goto __pyx_L5_argtuple_error; - case 1: - values[1] = PyDict_GetItem(__pyx_kwds, __pyx_kp_stream); - if (likely(values[1])) kw_args--; - else { - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - } - if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, PyTuple_GET_SIZE(__pyx_args), "pack") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - } - __pyx_v_o = values[0]; - __pyx_v_stream = values[1]; - } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { - goto __pyx_L5_argtuple_error; - } else { - __pyx_v_o = PyTuple_GET_ITEM(__pyx_args, 0); - __pyx_v_stream = PyTuple_GET_ITEM(__pyx_args, 1); - } - goto __pyx_L4_argument_unpacking_done; - __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("pack", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L3_error;} - __pyx_L3_error:; - __Pyx_AddTraceback("msgpack.pack"); - return NULL; - __pyx_L4_argument_unpacking_done:; - __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 - * - * def pack(object o, object stream): - * packer = Packer(stream) # <<<<<<<<<<<<<< - * packer.pack(o) - * packer.flush() - */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_stream); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_stream); - __Pyx_GIVEREF(__pyx_v_stream); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packer); - __pyx_v_packer = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 - * def pack(object o, object stream): - * packer = Packer(stream) - * packer.pack(o) # <<<<<<<<<<<<<< - * packer.flush() - * - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 - * packer = Packer(stream) - * packer.pack(o) - * packer.flush() # <<<<<<<<<<<<<< - * - * def packs(object o): - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.pack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_packer); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 - * packer.flush() - * - * def packs(object o): # <<<<<<<<<<<<<< - * buf = StringIO() - * packer = Packer(buf) - */ - -static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o); /*proto*/ -static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v_o) { - PyObject *__pyx_v_buf; - PyObject *__pyx_v_packer; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - __Pyx_SetupRefcountContext("packs"); - __pyx_self = __pyx_self; - __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); - __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 - * - * def packs(object o): - * buf = StringIO() # <<<<<<<<<<<<<< - * packer = Packer(buf) - * packer.pack(o) - */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_StringIO); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(__pyx_v_buf); - __pyx_v_buf = __pyx_t_1; - __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 - * def packs(object o): - * buf = StringIO() - * packer = Packer(buf) # <<<<<<<<<<<<<< - * packer.pack(o) - * packer.flush() - */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_buf); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_buf); - __Pyx_GIVEREF(__pyx_v_buf); - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_7msgpack_Packer)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 172; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packer); - __pyx_v_packer = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 - * buf = StringIO() - * packer = Packer(buf) - * packer.pack(o) # <<<<<<<<<<<<<< - * packer.flush() - * return buf.getvalue() - */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_pack); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_o); - __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 - * packer = Packer(buf) - * packer.pack(o) - * packer.flush() # <<<<<<<<<<<<<< - * return buf.getvalue() - * - */ - __pyx_t_3 = PyObject_GetAttr(__pyx_v_packer, __pyx_kp_flush); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 - * packer.pack(o) - * packer.flush() - * return buf.getvalue() # <<<<<<<<<<<<<< - * - * cdef extern from "unpack.h": - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(__pyx_v_buf, __pyx_kp_getvalue); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.packs"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_buf); - __Pyx_DECREF(__pyx_v_packer); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 - * - * - * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< - * """Unpack packed_bytes to object. Returns unpacked object.""" - * cdef const_char_ptr p = packed_bytes - */ - -static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes); /*proto*/ -static char __pyx_doc_7msgpack_unpacks[] = "Unpack packed_bytes to object. Returns unpacked object."; -static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx_v_packed_bytes) { - const char* __pyx_v_p; - template_context __pyx_v_ctx; - size_t __pyx_v_off; - PyObject *__pyx_r = NULL; - const char* __pyx_t_1; - Py_ssize_t __pyx_t_2; - PyObject *__pyx_t_3 = NULL; - __Pyx_SetupRefcountContext("unpacks"); - __pyx_self = __pyx_self; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 - * def unpacks(object packed_bytes): - * """Unpack packed_bytes to object. Returns unpacked object.""" - * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< - * cdef template_context ctx - * cdef size_t off = 0 - */ - __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_p = __pyx_t_1; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 - * cdef const_char_ptr p = packed_bytes - * cdef template_context ctx - * cdef size_t off = 0 # <<<<<<<<<<<<<< - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) - */ - __pyx_v_off = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 - * cdef template_context ctx - * cdef size_t off = 0 - * template_init(&ctx) # <<<<<<<<<<<<<< - * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) - */ - template_init((&__pyx_v_ctx)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 - * cdef size_t off = 0 - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< - * return template_data(&ctx) - * - */ - __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 - * template_init(&ctx) - * template_execute(&ctx, p, len(packed_bytes), &off) - * return template_data(&ctx) # <<<<<<<<<<<<<< - * - * def unpack(object stream): - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = template_data((&__pyx_v_ctx)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_3); - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_3); - __Pyx_AddTraceback("msgpack.unpacks"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} - -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 - * return template_data(&ctx) - * - * def unpack(object stream): # <<<<<<<<<<<<<< - * """unpack from stream.""" - * packed = stream.read() - */ - -static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream); /*proto*/ -static char __pyx_doc_7msgpack_unpack[] = "unpack from stream."; -static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_v_stream) { - PyObject *__pyx_v_packed; - PyObject *__pyx_r = NULL; - PyObject *__pyx_1 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - __Pyx_SetupRefcountContext("unpack"); - __pyx_self = __pyx_self; - __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 - * def unpack(object stream): - * """unpack from stream.""" - * packed = stream.read() # <<<<<<<<<<<<<< - * return unpacks(packed) - * - */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_stream, __pyx_kp_read); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_v_packed); - __pyx_v_packed = __pyx_t_2; - __pyx_t_2 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 - * """unpack from stream.""" - * packed = stream.read() - * return unpacks(packed) # <<<<<<<<<<<<<< - * - * cdef class Unpacker: - */ - __Pyx_XDECREF(__pyx_r); - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_2)); - __Pyx_INCREF(__pyx_v_packed); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_packed); - __Pyx_GIVEREF(__pyx_v_packed); - __pyx_t_1 = PyObject_Call(__pyx_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack.unpack"); - __pyx_r = NULL; - __pyx_L0:; - __Pyx_DECREF(__pyx_v_packed); - __Pyx_XGIVEREF(__pyx_r); - __Pyx_FinishRefcountContext(); - return __pyx_r; -} -static struct __pyx_vtabstruct_7msgpack_Packer __pyx_vtable_7msgpack_Packer; - -static PyObject *__pyx_tp_new_7msgpack_Packer(PyTypeObject *t, PyObject *a, PyObject *k) { - struct __pyx_obj_7msgpack_Packer *p; - PyObject *o = (*t->tp_alloc)(t, 0); - if (!o) return 0; - p = ((struct __pyx_obj_7msgpack_Packer *)o); - p->__pyx_vtab = __pyx_vtabptr_7msgpack_Packer; - p->strm = Py_None; Py_INCREF(Py_None); - return o; -} - -static void __pyx_tp_dealloc_7msgpack_Packer(PyObject *o) { - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - Py_XDECREF(p->strm); - (*Py_TYPE(o)->tp_free)(o); -} - -static int __pyx_tp_traverse_7msgpack_Packer(PyObject *o, visitproc v, void *a) { - int e; - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - if (p->strm) { - e = (*v)(p->strm, a); if (e) return e; - } - return 0; -} - -static int __pyx_tp_clear_7msgpack_Packer(PyObject *o) { - struct __pyx_obj_7msgpack_Packer *p = (struct __pyx_obj_7msgpack_Packer *)o; - PyObject* tmp; - tmp = ((PyObject*)p->strm); - p->strm = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); - return 0; -} - -static struct PyMethodDef __pyx_methods_7msgpack_Packer[] = { - {__Pyx_NAMESTR("__del__"), (PyCFunction)__pyx_pf_7msgpack_6Packer___del__, METH_NOARGS, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("flush"), (PyCFunction)__pyx_pf_7msgpack_6Packer_flush, METH_NOARGS, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_flush)}, - {__Pyx_NAMESTR("pack_list"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_list, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_list)}, - {__Pyx_NAMESTR("pack_dict"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack_dict, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_6Packer_pack_dict)}, - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_6Packer_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, - {0, 0, 0, 0} -}; - -static PyNumberMethods __pyx_tp_as_number_Packer = { - 0, /*nb_add*/ - 0, /*nb_subtract*/ - 0, /*nb_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ - #endif - 0, /*nb_remainder*/ - 0, /*nb_divmod*/ - 0, /*nb_power*/ - 0, /*nb_negative*/ - 0, /*nb_positive*/ - 0, /*nb_absolute*/ - 0, /*nb_nonzero*/ - 0, /*nb_invert*/ - 0, /*nb_lshift*/ - 0, /*nb_rshift*/ - 0, /*nb_and*/ - 0, /*nb_xor*/ - 0, /*nb_or*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ - #endif - 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else - 0, /*nb_long*/ - #endif - 0, /*nb_float*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_oct*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*nb_hex*/ - #endif - 0, /*nb_inplace_add*/ - 0, /*nb_inplace_subtract*/ - 0, /*nb_inplace_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_inplace_divide*/ - #endif - 0, /*nb_inplace_remainder*/ - 0, /*nb_inplace_power*/ - 0, /*nb_inplace_lshift*/ - 0, /*nb_inplace_rshift*/ - 0, /*nb_inplace_and*/ - 0, /*nb_inplace_xor*/ - 0, /*nb_inplace_or*/ - 0, /*nb_floor_divide*/ - 0, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) - 0, /*nb_index*/ - #endif -}; - -static PySequenceMethods __pyx_tp_as_sequence_Packer = { - 0, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping_Packer = { - 0, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyBufferProcs __pyx_tp_as_buffer_Packer = { - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getreadbuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getwritebuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getsegcount*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getcharbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_getbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_releasebuffer*/ - #endif -}; - -PyTypeObject __pyx_type_7msgpack_Packer = { - PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("msgpack.Packer"), /*tp_name*/ - sizeof(struct __pyx_obj_7msgpack_Packer), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_7msgpack_Packer, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - &__pyx_tp_as_number_Packer, /*tp_as_number*/ - &__pyx_tp_as_sequence_Packer, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_Packer, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - &__pyx_tp_as_buffer_Packer, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - __Pyx_DOCSTR("Packer that pack data into strm.\n\n strm must have `write(bytes)` method.\n size specifies local buffer size.\n "), /*tp_doc*/ - __pyx_tp_traverse_7msgpack_Packer, /*tp_traverse*/ - __pyx_tp_clear_7msgpack_Packer, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_7msgpack_Packer, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - __pyx_pf_7msgpack_6Packer___init__, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_7msgpack_Packer, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ -}; - -static PyObject *__pyx_tp_new_7msgpack_Unpacker(PyTypeObject *t, PyObject *a, PyObject *k) { - PyObject *o = (*t->tp_alloc)(t, 0); - if (!o) return 0; - return o; -} - -static void __pyx_tp_dealloc_7msgpack_Unpacker(PyObject *o) { - (*Py_TYPE(o)->tp_free)(o); -} - -static struct PyMethodDef __pyx_methods_7msgpack_Unpacker[] = { - {0, 0, 0, 0} -}; - -static PyNumberMethods __pyx_tp_as_number_Unpacker = { - 0, /*nb_add*/ - 0, /*nb_subtract*/ - 0, /*nb_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ - #endif - 0, /*nb_remainder*/ - 0, /*nb_divmod*/ - 0, /*nb_power*/ - 0, /*nb_negative*/ - 0, /*nb_positive*/ - 0, /*nb_absolute*/ - 0, /*nb_nonzero*/ - 0, /*nb_invert*/ - 0, /*nb_lshift*/ - 0, /*nb_rshift*/ - 0, /*nb_and*/ - 0, /*nb_xor*/ - 0, /*nb_or*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ - #endif - 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else - 0, /*nb_long*/ - #endif - 0, /*nb_float*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_oct*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*nb_hex*/ - #endif - 0, /*nb_inplace_add*/ - 0, /*nb_inplace_subtract*/ - 0, /*nb_inplace_multiply*/ - #if PY_MAJOR_VERSION < 3 - 0, /*nb_inplace_divide*/ - #endif - 0, /*nb_inplace_remainder*/ - 0, /*nb_inplace_power*/ - 0, /*nb_inplace_lshift*/ - 0, /*nb_inplace_rshift*/ - 0, /*nb_inplace_and*/ - 0, /*nb_inplace_xor*/ - 0, /*nb_inplace_or*/ - 0, /*nb_floor_divide*/ - 0, /*nb_true_divide*/ - 0, /*nb_inplace_floor_divide*/ - 0, /*nb_inplace_true_divide*/ - #if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & Py_TPFLAGS_HAVE_INDEX) - 0, /*nb_index*/ - #endif -}; - -static PySequenceMethods __pyx_tp_as_sequence_Unpacker = { - 0, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - 0, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - 0, /*sq_contains*/ - 0, /*sq_inplace_concat*/ - 0, /*sq_inplace_repeat*/ -}; - -static PyMappingMethods __pyx_tp_as_mapping_Unpacker = { - 0, /*mp_length*/ - 0, /*mp_subscript*/ - 0, /*mp_ass_subscript*/ -}; - -static PyBufferProcs __pyx_tp_as_buffer_Unpacker = { - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getreadbuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getwritebuffer*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getsegcount*/ - #endif - #if PY_MAJOR_VERSION < 3 - 0, /*bf_getcharbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_getbuffer*/ - #endif - #if PY_VERSION_HEX >= 0x02060000 - 0, /*bf_releasebuffer*/ - #endif -}; - -PyTypeObject __pyx_type_7msgpack_Unpacker = { - PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("msgpack.Unpacker"), /*tp_name*/ - sizeof(struct __pyx_obj_7msgpack_Unpacker), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - __pyx_tp_dealloc_7msgpack_Unpacker, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare*/ - 0, /*tp_repr*/ - &__pyx_tp_as_number_Unpacker, /*tp_as_number*/ - &__pyx_tp_as_sequence_Unpacker, /*tp_as_sequence*/ - &__pyx_tp_as_mapping_Unpacker, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - &__pyx_tp_as_buffer_Unpacker, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ - __Pyx_DOCSTR("Do nothing. This function is for symmetric to Packer"), /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - __pyx_methods_7msgpack_Unpacker, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - 0, /*tp_init*/ - 0, /*tp_alloc*/ - __pyx_tp_new_7msgpack_Unpacker, /*tp_new*/ - 0, /*tp_free*/ - 0, /*tp_is_gc*/ - 0, /*tp_bases*/ - 0, /*tp_mro*/ - 0, /*tp_cache*/ - 0, /*tp_subclasses*/ - 0, /*tp_weaklist*/ -}; - -static struct PyMethodDef __pyx_methods[] = { - {__Pyx_NAMESTR("pack"), (PyCFunction)__pyx_pf_7msgpack_pack, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("packs"), (PyCFunction)__pyx_pf_7msgpack_packs, METH_O, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("unpacks"), (PyCFunction)__pyx_pf_7msgpack_unpacks, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpacks)}, - {__Pyx_NAMESTR("unpack"), (PyCFunction)__pyx_pf_7msgpack_unpack, METH_O, __Pyx_DOCSTR(__pyx_doc_7msgpack_unpack)}, - {0, 0, 0, 0} -}; - -static void __pyx_init_filenames(void); /*proto*/ - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef __pyx_moduledef = { - PyModuleDef_HEAD_INIT, - __Pyx_NAMESTR("msgpack"), - 0, /* m_doc */ - -1, /* m_size */ - __pyx_methods /* m_methods */, - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; -#endif - -static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, - {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, - {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, - {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, - {&__pyx_kp_pack_list, __pyx_k_pack_list, sizeof(__pyx_k_pack_list), 1, 1, 1}, - {&__pyx_kp_pack_dict, __pyx_k_pack_dict, sizeof(__pyx_k_pack_dict), 1, 1, 1}, - {&__pyx_kp_pack, __pyx_k_pack, sizeof(__pyx_k_pack), 1, 1, 1}, - {&__pyx_kp_unpack, __pyx_k_unpack, sizeof(__pyx_k_unpack), 1, 1, 1}, - {&__pyx_kp_strm, __pyx_k_strm, sizeof(__pyx_k_strm), 1, 1, 1}, - {&__pyx_kp_size, __pyx_k_size, sizeof(__pyx_k_size), 1, 1, 1}, - {&__pyx_kp_len, __pyx_k_len, sizeof(__pyx_k_len), 1, 1, 1}, - {&__pyx_kp_obj, __pyx_k_obj, sizeof(__pyx_k_obj), 1, 1, 1}, - {&__pyx_kp_o, __pyx_k_o, sizeof(__pyx_k_o), 1, 1, 1}, - {&__pyx_kp_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 1, 1, 1}, - {&__pyx_kp_packed_bytes, __pyx_k_packed_bytes, sizeof(__pyx_k_packed_bytes), 1, 1, 1}, - {&__pyx_kp_cStringIO, __pyx_k_cStringIO, sizeof(__pyx_k_cStringIO), 1, 1, 1}, - {&__pyx_kp_StringIO, __pyx_k_StringIO, sizeof(__pyx_k_StringIO), 1, 1, 1}, - {&__pyx_kp_staticmethod, __pyx_k_staticmethod, sizeof(__pyx_k_staticmethod), 1, 1, 1}, - {&__pyx_kp_unpacks, __pyx_k_unpacks, sizeof(__pyx_k_unpacks), 1, 1, 1}, - {&__pyx_kp_write, __pyx_k_write, sizeof(__pyx_k_write), 1, 1, 1}, - {&__pyx_kp_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 1, 0}, - {&__pyx_kp_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 1, 1, 1}, - {&__pyx_kp_iteritems, __pyx_k_iteritems, sizeof(__pyx_k_iteritems), 1, 1, 1}, - {&__pyx_kp_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 1, 1, 1}, - {&__pyx_kp_getvalue, __pyx_k_getvalue, sizeof(__pyx_k_getvalue), 1, 1, 1}, - {&__pyx_kp_read, __pyx_k_read, sizeof(__pyx_k_read), 1, 1, 1}, - {&__pyx_kp_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 0}, - {&__pyx_kp_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 0}, - {0, 0, 0, 0, 0, 0} -}; -static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_staticmethod = __Pyx_GetName(__pyx_b, __pyx_kp_staticmethod); if (!__pyx_builtin_staticmethod) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_kp_TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - return 0; - __pyx_L1_error:; - return -1; -} - -static int __Pyx_InitGlobals(void) { - if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - return 0; - __pyx_L1_error:; - return -1; -} - -#if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC initmsgpack(void); /*proto*/ -PyMODINIT_FUNC initmsgpack(void) -#else -PyMODINIT_FUNC PyInit_msgpack(void); /*proto*/ -PyMODINIT_FUNC PyInit_msgpack(void) -#endif -{ - PyObject *__pyx_1 = 0; - PyObject *__pyx_2 = 0; - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - #ifdef CYTHON_REFNANNY - void* __pyx_refchk = NULL; - __Pyx_Refnanny = __Pyx_ImportRefcountAPI("refnanny"); - if (!__Pyx_Refnanny) { - PyErr_Clear(); - __Pyx_Refnanny = __Pyx_ImportRefcountAPI("Cython.Runtime.refnanny"); - if (!__Pyx_Refnanny) - Py_FatalError("failed to import refnanny module"); - } - __pyx_refchk = __Pyx_Refnanny->NewContext("PyMODINIT_FUNC PyInit_msgpack(void)", __LINE__, __FILE__); - #endif - __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /*--- Library function declarations ---*/ - __pyx_init_filenames(); - /*--- Threads initialization code ---*/ - #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS - #ifdef WITH_THREAD /* Python build with threading support? */ - PyEval_InitThreads(); - #endif - #endif - /*--- Initialize various global constants etc. ---*/ - if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /*--- Module creation code ---*/ - #if PY_MAJOR_VERSION < 3 - __pyx_m = Py_InitModule4(__Pyx_NAMESTR("msgpack"), __pyx_methods, 0, 0, PYTHON_API_VERSION); - #else - __pyx_m = PyModule_Create(&__pyx_moduledef); - #endif - if (!__pyx_m) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #if PY_MAJOR_VERSION < 3 - Py_INCREF(__pyx_m); - #endif - __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); - if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__pyx_module_is_main_msgpack) { - if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - } - /*--- Builtin init code ---*/ - if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_skip_dispatch = 0; - /*--- Global init code ---*/ - /*--- Function export code ---*/ - /*--- Type init code ---*/ - __pyx_vtabptr_7msgpack_Packer = &__pyx_vtable_7msgpack_Packer; - #if PY_MAJOR_VERSION >= 3 - __pyx_vtable_7msgpack_Packer.__pack = (PyObject *(*)(struct __pyx_obj_7msgpack_Packer *, PyObject *))__pyx_f_7msgpack_6Packer___pack; - #else - *(void(**)(void))&__pyx_vtable_7msgpack_Packer.__pack = (void(*)(void))__pyx_f_7msgpack_6Packer___pack; - #endif - if (PyType_Ready(&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetVtable(__pyx_type_7msgpack_Packer.tp_dict, __pyx_vtabptr_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Packer", (PyObject *)&__pyx_type_7msgpack_Packer) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_7msgpack_Packer = &__pyx_type_7msgpack_Packer; - if (PyType_Ready(&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "Unpacker", (PyObject *)&__pyx_type_7msgpack_Unpacker) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_7msgpack_Unpacker = &__pyx_type_7msgpack_Unpacker; - /*--- Type import code ---*/ - /*--- Function import code ---*/ - /*--- Execution code ---*/ - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 - * # coding: utf-8 - * - * from cStringIO import StringIO # <<<<<<<<<<<<<< - * - * cdef extern from "Python.h": - */ - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __Pyx_INCREF(__pyx_kp_StringIO); - PyList_SET_ITEM(__pyx_t_1, 0, __pyx_kp_StringIO); - __Pyx_GIVEREF(__pyx_kp_StringIO); - __pyx_1 = __Pyx_Import(__pyx_kp_cStringIO, ((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __pyx_2 = PyObject_GetAttr(__pyx_1, __pyx_kp_StringIO); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_2); - if (PyObject_SetAttr(__pyx_m, __pyx_kp_StringIO, __pyx_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_2); __pyx_2 = 0; - __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 - * - * - * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< - * - * cdef class Packer: - */ - __pyx_v_7msgpack_BUFF_SIZE = 2048; - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 - * raise TypeError, "can't serialize %r" % (o,) - * - * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< - * self.__pack(obj) - * if flush: - */ - __pyx_t_1 = __Pyx_PyBool_FromLong(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_1); - __pyx_k_1 = __pyx_t_1; - __pyx_t_1 = 0; - __Pyx_GIVEREF(__pyx_k_1); - - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 - * cdef class Unpacker: - * """Do nothing. This function is for symmetric to Packer""" - * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< - */ - __pyx_1 = __Pyx_GetName(__pyx_m, __pyx_kp_unpacks); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_1); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_1); - __Pyx_GIVEREF(__pyx_1); - __pyx_1 = 0; - __pyx_t_2 = PyObject_Call(__pyx_builtin_staticmethod, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_7msgpack_Unpacker->tp_dict, __pyx_kp_unpack, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - PyType_Modified(__pyx_ptype_7msgpack_Unpacker); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_1); - __Pyx_XDECREF(__pyx_2); - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_AddTraceback("msgpack"); - Py_DECREF(__pyx_m); __pyx_m = 0; - __pyx_L0:; - __Pyx_FinishRefcountContext(); - #if PY_MAJOR_VERSION < 3 - return; - #else - return __pyx_m; - #endif -} - -static const char *__pyx_filenames[] = { - "msgpack.pyx", -}; - -/* Runtime support code */ - -static void __pyx_init_filenames(void) { - __pyx_f = __pyx_filenames; -} - -static void __Pyx_RaiseDoubleKeywordsError( - const char* func_name, - PyObject* kw_name) -{ - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION >= 3 - "%s() got multiple values for keyword argument '%U'", func_name, kw_name); - #else - "%s() got multiple values for keyword argument '%s'", func_name, - PyString_AS_STRING(kw_name)); - #endif -} - -static void __Pyx_RaiseArgtupleInvalid( - const char* func_name, - int exact, - Py_ssize_t num_min, - Py_ssize_t num_max, - Py_ssize_t num_found) -{ - Py_ssize_t num_expected; - const char *number, *more_or_less; - - if (num_found < num_min) { - num_expected = num_min; - more_or_less = "at least"; - } else { - num_expected = num_max; - more_or_less = "at most"; - } - if (exact) { - more_or_less = "exactly"; - } - number = (num_expected == 1) ? "" : "s"; - PyErr_Format(PyExc_TypeError, - #if PY_VERSION_HEX < 0x02050000 - "%s() takes %s %d positional argument%s (%d given)", - #else - "%s() takes %s %zd positional argument%s (%zd given)", - #endif - func_name, more_or_less, num_expected, number, num_found); -} - -static int __Pyx_ParseOptionalKeywords( - PyObject *kwds, - PyObject **argnames[], - PyObject *kwds2, - PyObject *values[], - Py_ssize_t num_pos_args, - const char* function_name) -{ - PyObject *key = 0, *value = 0; - Py_ssize_t pos = 0; - PyObject*** name; - PyObject*** first_kw_arg = argnames + num_pos_args; - - while (PyDict_Next(kwds, &pos, &key, &value)) { - name = first_kw_arg; - while (*name && (**name != key)) name++; - if (*name) { - values[name-argnames] = value; - } else { - #if PY_MAJOR_VERSION < 3 - if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) { - #else - if (unlikely(!PyUnicode_CheckExact(key)) && unlikely(!PyUnicode_Check(key))) { - #endif - goto invalid_keyword_type; - } else { - for (name = first_kw_arg; *name; name++) { - #if PY_MAJOR_VERSION >= 3 - if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && - PyUnicode_Compare(**name, key) == 0) break; - #else - if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && - _PyString_Eq(**name, key)) break; - #endif - } - if (*name) { - values[name-argnames] = value; - } else { - /* unexpected keyword found */ - for (name=argnames; name != first_kw_arg; name++) { - if (**name == key) goto arg_passed_twice; - #if PY_MAJOR_VERSION >= 3 - if (PyUnicode_GET_SIZE(**name) == PyUnicode_GET_SIZE(key) && - PyUnicode_Compare(**name, key) == 0) goto arg_passed_twice; - #else - if (PyString_GET_SIZE(**name) == PyString_GET_SIZE(key) && - _PyString_Eq(**name, key)) goto arg_passed_twice; - #endif - } - if (kwds2) { - if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; - } else { - goto invalid_keyword; - } - } - } - } - } - return 0; -arg_passed_twice: - __Pyx_RaiseDoubleKeywordsError(function_name, **name); - goto bad; -invalid_keyword_type: - PyErr_Format(PyExc_TypeError, - "%s() keywords must be strings", function_name); - goto bad; -invalid_keyword: - PyErr_Format(PyExc_TypeError, - #if PY_MAJOR_VERSION < 3 - "%s() got an unexpected keyword argument '%s'", - function_name, PyString_AsString(key)); - #else - "%s() got an unexpected keyword argument '%U'", - function_name, key); - #endif -bad: - return -1; -} - -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list) { - PyObject *__import__ = 0; - PyObject *empty_list = 0; - PyObject *module = 0; - PyObject *global_dict = 0; - PyObject *empty_dict = 0; - PyObject *list; - __import__ = __Pyx_GetAttrString(__pyx_b, "__import__"); - if (!__import__) - goto bad; - if (from_list) - list = from_list; - else { - empty_list = PyList_New(0); - if (!empty_list) - goto bad; - list = empty_list; - } - global_dict = PyModule_GetDict(__pyx_m); - if (!global_dict) - goto bad; - empty_dict = PyDict_New(); - if (!empty_dict) - goto bad; - module = PyObject_CallFunctionObjArgs(__import__, - name, global_dict, empty_dict, list, NULL); -bad: - Py_XDECREF(empty_list); - Py_XDECREF(__import__); - Py_XDECREF(empty_dict); - return module; -} - -static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) { - PyObject *result; - result = PyObject_GetAttr(dict, name); - if (!result) - PyErr_SetObject(PyExc_NameError, name); - return result; -} - -static INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { - PyErr_Format(PyExc_ValueError, - #if PY_VERSION_HEX < 0x02050000 - "need more than %d value%s to unpack", (int)index, - #else - "need more than %zd value%s to unpack", index, - #endif - (index == 1) ? "" : "s"); -} - -static INLINE void __Pyx_RaiseTooManyValuesError(void) { - PyErr_SetString(PyExc_ValueError, "too many values to unpack"); -} - -static PyObject *__Pyx_UnpackItem(PyObject *iter, Py_ssize_t index) { - PyObject *item; - if (!(item = PyIter_Next(iter))) { - if (!PyErr_Occurred()) { - __Pyx_RaiseNeedMoreValuesError(index); - } - } - return item; -} - -static int __Pyx_EndUnpack(PyObject *iter) { - PyObject *item; - if ((item = PyIter_Next(iter))) { - Py_DECREF(item); - __Pyx_RaiseTooManyValuesError(); - return -1; - } - else if (!PyErr_Occurred()) - return 0; - else - return -1; -} - -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) { - Py_XINCREF(type); - Py_XINCREF(value); - Py_XINCREF(tb); - /* First, check the traceback argument, replacing None with NULL. */ - if (tb == Py_None) { - Py_DECREF(tb); - tb = 0; - } - else if (tb != NULL && !PyTraceBack_Check(tb)) { - PyErr_SetString(PyExc_TypeError, - "raise: arg 3 must be a traceback or None"); - goto raise_error; - } - /* Next, replace a missing value with None */ - if (value == NULL) { - value = Py_None; - Py_INCREF(value); - } - #if PY_VERSION_HEX < 0x02050000 - if (!PyClass_Check(type)) - #else - if (!PyType_Check(type)) - #endif - { - /* Raising an instance. The value should be a dummy. */ - if (value != Py_None) { - PyErr_SetString(PyExc_TypeError, - "instance exception may not have a separate value"); - goto raise_error; - } - /* Normalize to raise , */ - Py_DECREF(value); - value = type; - #if PY_VERSION_HEX < 0x02050000 - if (PyInstance_Check(type)) { - type = (PyObject*) ((PyInstanceObject*)type)->in_class; - Py_INCREF(type); - } - else { - type = 0; - PyErr_SetString(PyExc_TypeError, - "raise: exception must be an old-style class or instance"); - goto raise_error; - } - #else - type = (PyObject*) Py_TYPE(type); - Py_INCREF(type); - if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { - PyErr_SetString(PyExc_TypeError, - "raise: exception class must be a subclass of BaseException"); - goto raise_error; - } - #endif - } - __Pyx_ErrRestore(type, value, tb); - return; -raise_error: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(tb); - return; -} - -static INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyThreadState *tstate = PyThreadState_GET(); - -#if PY_MAJOR_VERSION >= 3 - /* Note: this is a temporary work-around to prevent crashes in Python 3.0 */ - if ((tstate->exc_type != NULL) & (tstate->exc_type != Py_None)) { - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - PyErr_NormalizeException(&type, &value, &tb); - PyErr_NormalizeException(&tmp_type, &tmp_value, &tmp_tb); - tstate->exc_type = 0; - tstate->exc_value = 0; - tstate->exc_traceback = 0; - PyException_SetContext(value, tmp_value); - Py_DECREF(tmp_type); - Py_XDECREF(tmp_tb); - } -#endif - - tmp_type = tstate->curexc_type; - tmp_value = tstate->curexc_value; - tmp_tb = tstate->curexc_traceback; - tstate->curexc_type = type; - tstate->curexc_value = value; - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} - -static INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb) { - PyThreadState *tstate = PyThreadState_GET(); - *type = tstate->curexc_type; - *value = tstate->curexc_value; - *tb = tstate->curexc_traceback; - - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -} - - -static INLINE int __Pyx_StrEq(const char *s1, const char *s2) { - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - return *s1 == *s2; -} - -static INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { - if (sizeof(unsigned char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned char)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned char"); - return (unsigned char)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned char"); - return (unsigned char)-1; - } - return (unsigned char)val; - } - return (unsigned char)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE unsigned short __Pyx_PyInt_AsUnsignedShort(PyObject* x) { - if (sizeof(unsigned short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned short)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned short"); - return (unsigned short)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned short"); - return (unsigned short)-1; - } - return (unsigned short)val; - } - return (unsigned short)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE unsigned int __Pyx_PyInt_AsUnsignedInt(PyObject* x) { - if (sizeof(unsigned int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(unsigned int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (unsigned int)-1; - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned int"); - return (unsigned int)-1; - } - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to unsigned int"); - return (unsigned int)-1; - } - return (unsigned int)val; - } - return (unsigned int)__Pyx_PyInt_AsUnsignedLong(x); -} - -static INLINE char __Pyx_PyInt_AsChar(PyObject* x) { - if (sizeof(char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (char)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to char"); - return (char)-1; - } - return (char)val; - } - return (char)__Pyx_PyInt_AsLong(x); -} - -static INLINE short __Pyx_PyInt_AsShort(PyObject* x) { - if (sizeof(short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (short)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to short"); - return (short)-1; - } - return (short)val; - } - return (short)__Pyx_PyInt_AsLong(x); -} - -static INLINE int __Pyx_PyInt_AsInt(PyObject* x) { - if (sizeof(int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (int)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to int"); - return (int)-1; - } - return (int)val; - } - return (int)__Pyx_PyInt_AsLong(x); -} - -static INLINE signed char __Pyx_PyInt_AsSignedChar(PyObject* x) { - if (sizeof(signed char) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed char)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed char)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed char"); - return (signed char)-1; - } - return (signed char)val; - } - return (signed char)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE signed short __Pyx_PyInt_AsSignedShort(PyObject* x) { - if (sizeof(signed short) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed short)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed short)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed short"); - return (signed short)-1; - } - return (signed short)val; - } - return (signed short)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE signed int __Pyx_PyInt_AsSignedInt(PyObject* x) { - if (sizeof(signed int) < sizeof(long)) { - long val = __Pyx_PyInt_AsLong(x); - if (unlikely(val != (long)(signed int)val)) { - if (unlikely(val == -1 && PyErr_Occurred())) - return (signed int)-1; - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to signed int"); - return (signed int)-1; - } - return (signed int)val; - } - return (signed int)__Pyx_PyInt_AsSignedLong(x); -} - -static INLINE unsigned long __Pyx_PyInt_AsUnsignedLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned long"); - return (unsigned long)-1; - } - return (unsigned long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - if (unlikely(Py_SIZE(x) < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned long"); - return (unsigned long)-1; - } - return PyLong_AsUnsignedLong(x); - } else { - unsigned long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (unsigned long)-1; - val = __Pyx_PyInt_AsUnsignedLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE unsigned PY_LONG_LONG __Pyx_PyInt_AsUnsignedLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - if (unlikely(val < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned PY_LONG_LONG"); - return (unsigned PY_LONG_LONG)-1; - } - return (unsigned PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - if (unlikely(Py_SIZE(x) < 0)) { - PyErr_SetString(PyExc_OverflowError, - "can't convert negative value to unsigned PY_LONG_LONG"); - return (unsigned PY_LONG_LONG)-1; - } - return PyLong_AsUnsignedLongLong(x); - } else { - unsigned PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (unsigned PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsUnsignedLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE long __Pyx_PyInt_AsLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLong(x); - } else { - long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (long)-1; - val = __Pyx_PyInt_AsLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE PY_LONG_LONG __Pyx_PyInt_AsLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLongLong(x); - } else { - PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE signed long __Pyx_PyInt_AsSignedLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (signed long)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLong(x); - } else { - signed long val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (signed long)-1; - val = __Pyx_PyInt_AsSignedLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* x) { -#if PY_VERSION_HEX < 0x03000000 - if (likely(PyInt_CheckExact(x) || PyInt_Check(x))) { - long val = PyInt_AS_LONG(x); - return (signed PY_LONG_LONG)val; - } else -#endif - if (likely(PyLong_CheckExact(x) || PyLong_Check(x))) { - return PyLong_AsLongLong(x); - } else { - signed PY_LONG_LONG val; - PyObject *tmp = __Pyx_PyNumber_Int(x); - if (!tmp) return (signed PY_LONG_LONG)-1; - val = __Pyx_PyInt_AsSignedLongLong(tmp); - Py_DECREF(tmp); - return val; - } -} - -static void __Pyx_WriteUnraisable(const char *name) { - PyObject *old_exc, *old_val, *old_tb; - PyObject *ctx; - __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); - #if PY_MAJOR_VERSION < 3 - ctx = PyString_FromString(name); - #else - ctx = PyUnicode_FromString(name); - #endif - __Pyx_ErrRestore(old_exc, old_val, old_tb); - if (!ctx) { - PyErr_WriteUnraisable(Py_None); - } else { - PyErr_WriteUnraisable(ctx); - Py_DECREF(ctx); - } -} - -static int __Pyx_SetVtable(PyObject *dict, void *vtable) { - PyObject *pycobj = 0; - int result; - - pycobj = PyCObject_FromVoidPtr(vtable, 0); - if (!pycobj) - goto bad; - if (PyDict_SetItemString(dict, "__pyx_vtable__", pycobj) < 0) - goto bad; - result = 0; - goto done; - -bad: - result = -1; -done: - Py_XDECREF(pycobj); - return result; -} - -#include "compile.h" -#include "frameobject.h" -#include "traceback.h" - -static void __Pyx_AddTraceback(const char *funcname) { - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - PyObject *py_globals = 0; - PyObject *empty_string = 0; - PyCodeObject *py_code = 0; - PyFrameObject *py_frame = 0; - - #if PY_MAJOR_VERSION < 3 - py_srcfile = PyString_FromString(__pyx_filename); - #else - py_srcfile = PyUnicode_FromString(__pyx_filename); - #endif - if (!py_srcfile) goto bad; - if (__pyx_clineno) { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); - #else - py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, __pyx_clineno); - #endif - } - else { - #if PY_MAJOR_VERSION < 3 - py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); - #endif - } - if (!py_funcname) goto bad; - py_globals = PyModule_GetDict(__pyx_m); - if (!py_globals) goto bad; - #if PY_MAJOR_VERSION < 3 - empty_string = PyString_FromStringAndSize("", 0); - #else - empty_string = PyBytes_FromStringAndSize("", 0); - #endif - if (!empty_string) goto bad; - py_code = PyCode_New( - 0, /*int argcount,*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*int kwonlyargcount,*/ - #endif - 0, /*int nlocals,*/ - 0, /*int stacksize,*/ - 0, /*int flags,*/ - empty_string, /*PyObject *code,*/ - __pyx_empty_tuple, /*PyObject *consts,*/ - __pyx_empty_tuple, /*PyObject *names,*/ - __pyx_empty_tuple, /*PyObject *varnames,*/ - __pyx_empty_tuple, /*PyObject *freevars,*/ - __pyx_empty_tuple, /*PyObject *cellvars,*/ - py_srcfile, /*PyObject *filename,*/ - py_funcname, /*PyObject *name,*/ - __pyx_lineno, /*int firstlineno,*/ - empty_string /*PyObject *lnotab*/ - ); - if (!py_code) goto bad; - py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - py_globals, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ - ); - if (!py_frame) goto bad; - py_frame->f_lineno = __pyx_lineno; - PyTraceBack_Here(py_frame); -bad: - Py_XDECREF(py_srcfile); - Py_XDECREF(py_funcname); - Py_XDECREF(empty_string); - Py_XDECREF(py_code); - Py_XDECREF(py_frame); -} - -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { - while (t->p) { - #if PY_MAJOR_VERSION < 3 - if (t->is_unicode && (!t->is_identifier)) { - *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); - } else if (t->intern) { - *t->p = PyString_InternFromString(t->s); - } else { - *t->p = PyString_FromStringAndSize(t->s, t->n - 1); - } - #else /* Python 3+ has unicode identifiers */ - if (t->is_identifier || (t->is_unicode && t->intern)) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->is_unicode) { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif - if (!*t->p) - return -1; - ++t; - } - return 0; -} - -/* Type Conversion Functions */ - -static INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { - if (x == Py_True) return 1; - else if ((x == Py_False) | (x == Py_None)) return 0; - else return PyObject_IsTrue(x); -} - -static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { - PyNumberMethods *m; - const char *name = NULL; - PyObject *res = NULL; -#if PY_VERSION_HEX < 0x03000000 - if (PyInt_Check(x) || PyLong_Check(x)) -#else - if (PyLong_Check(x)) -#endif - return Py_INCREF(x), x; - m = Py_TYPE(x)->tp_as_number; -#if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } -#else - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Long(x); - } -#endif - if (res) { -#if PY_VERSION_HEX < 0x03000000 - if (!PyInt_Check(res) && !PyLong_Check(res)) { -#else - if (!PyLong_Check(res)) { -#endif - PyErr_Format(PyExc_TypeError, - "__%s__ returned non-%s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; - } - } - else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, - "an integer is required"); - } - return res; -} - -static INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { - Py_ssize_t ival; - PyObject* x = PyNumber_Index(b); - if (!x) return -1; - ival = PyInt_AsSsize_t(x); - Py_DECREF(x); - return ival; -} - -static INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { -#if PY_VERSION_HEX < 0x02050000 - if (ival <= LONG_MAX) - return PyInt_FromLong((long)ival); - else { - unsigned char *bytes = (unsigned char *) &ival; - int one = 1; int little = (int)*(unsigned char*)&one; - return _PyLong_FromByteArray(bytes, sizeof(size_t), little, 0); - } -#else - return PyInt_FromSize_t(ival); -#endif -} - -static INLINE size_t __Pyx_PyInt_AsSize_t(PyObject* x) { - unsigned PY_LONG_LONG val = __Pyx_PyInt_AsUnsignedLongLong(x); - if (unlikely(val == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred())) { - return (size_t)-1; - } else if (unlikely(val != (unsigned PY_LONG_LONG)(size_t)val)) { - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to size_t"); - return (size_t)-1; - } - return (size_t)val; -} - - diff --git a/msgpack/__init__.py b/msgpack/__init__.py new file mode 100644 index 0000000..797b29c --- /dev/null +++ b/msgpack/__init__.py @@ -0,0 +1,3 @@ +# coding: utf-8 +from _msgpack import * + diff --git a/msgpack.pyx b/msgpack/_msgpack.pyx similarity index 100% rename from msgpack.pyx rename to msgpack/_msgpack.pyx diff --git a/pack.h b/msgpack/pack.h similarity index 100% rename from pack.h rename to msgpack/pack.h diff --git a/unpack.h b/msgpack/unpack.h similarity index 100% rename from unpack.h rename to msgpack/unpack.h diff --git a/setup.py b/setup.py index 65ca412..56b3faa 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,13 @@ from distutils.core import setup, Extension +from Cython.Distutils import build_ext import os version = '0.0.1' PACKAGE_ROOT = os.getcwdu() INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') -msgpack_mod = Extension('msgpack', - sources=['msgpack.c'], +msgpack_mod = Extension('msgpack._msgpack', + sources=['msgpack/_msgpack.pyx'], include_dirs=[INCLUDE_PATH]) desc = 'MessagePack serializer/desirializer.' @@ -28,6 +29,7 @@ setup(name='msgpack', author='Naoki INADA', author_email='songofacandy@gmail.com', version=version, + cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], description=desc, long_description=long_desc, From e814986b4ec017ab124dcf16496a6fefa65a9876 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 13:33:42 +0900 Subject: [PATCH 0155/1648] Fix document miss. --- python/msgpack/_msgpack.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index f24f403..7c07cde 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -79,7 +79,7 @@ cdef class Packer: packer.pack('foo') packer.pack('bar') - This code is same as below code: + This is same to: packer.pack(['foo', 'bar']) """ @@ -94,9 +94,9 @@ cdef class Packer: packer.pack('foo') packer.pack('bar') - This code is same as below code: + This is same to: - packer.pack({'foo', 'bar'}) + packer.pack({'foo': 'bar'}) """ msgpack_pack_map(&self.pk, len) From 9ae7b2b28c567ce0b78a90d9b16375a4568cab1f Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 10 Jun 2009 13:33:42 +0900 Subject: [PATCH 0156/1648] Fix document miss. --- msgpack/_msgpack.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index f24f403..7c07cde 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -79,7 +79,7 @@ cdef class Packer: packer.pack('foo') packer.pack('bar') - This code is same as below code: + This is same to: packer.pack(['foo', 'bar']) """ @@ -94,9 +94,9 @@ cdef class Packer: packer.pack('foo') packer.pack('bar') - This code is same as below code: + This is same to: - packer.pack({'foo', 'bar'}) + packer.pack({'foo': 'bar'}) """ msgpack_pack_map(&self.pk, len) From 6184e17a42c89993e445a668169a76cd5bcad046 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 16 Jun 2009 01:56:04 +0900 Subject: [PATCH 0157/1648] Increase stack size. --- python/unpack.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/unpack.h b/python/unpack.h index e51557f..c98c19a 100644 --- a/python/unpack.h +++ b/python/unpack.h @@ -15,6 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" typedef struct { From a90efd70c229b94b81184e94d6b36e727144ff23 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 16 Jun 2009 01:56:04 +0900 Subject: [PATCH 0158/1648] Increase stack size. --- unpack.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unpack.h b/unpack.h index e51557f..c98c19a 100644 --- a/unpack.h +++ b/unpack.h @@ -15,6 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" typedef struct { From 4d6e9ffaa2f1626e38fbc7ab5d08578213295a65 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 16 Jun 2009 01:58:07 +0900 Subject: [PATCH 0159/1648] Fix can't pack float values. --- python/msgpack.c | 212 +++++++++++++++++++++++---------------------- python/msgpack.pyx | 2 +- 2 files changed, 108 insertions(+), 106 deletions(-) diff --git a/python/msgpack.c b/python/msgpack.c index 821f65b..7e5c21f 100644 --- a/python/msgpack.c +++ b/python/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ +/* Generated by Cython 0.11.2 on Tue Jun 16 01:57:05 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -800,13 +800,14 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack int __pyx_t_1; PyObject *__pyx_t_2 = NULL; PY_LONG_LONG __pyx_t_3; - char *__pyx_t_4; - Py_ssize_t __pyx_t_5; - PyObject *__pyx_t_6 = NULL; + double __pyx_t_4; + char *__pyx_t_5; + Py_ssize_t __pyx_t_6; PyObject *__pyx_t_7 = NULL; PyObject *__pyx_t_8 = NULL; - int __pyx_t_9; + PyObject *__pyx_t_9 = NULL; int __pyx_t_10; + int __pyx_t_11; __Pyx_SetupRefcountContext("__pack"); __Pyx_INCREF(__pyx_v_o); __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); @@ -937,7 +938,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, float): - * fval = 9 + * fval = o */ msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); goto __pyx_L3; @@ -947,7 +948,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) */ __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); @@ -956,15 +957,16 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): - * fval = 9 # <<<<<<<<<<<<<< + * fval = o # <<<<<<<<<<<<<< * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): */ - __pyx_v_fval = 9; + __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_fval = __pyx_t_4; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< * elif isinstance(o, str): * rawval = o @@ -974,7 +976,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< * rawval = o @@ -990,8 +992,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; + __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_5; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): @@ -1000,8 +1002,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o @@ -1010,8 +1012,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, unicode): * o = o.encode('utf-8') */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_6); goto __pyx_L3; } @@ -1034,18 +1036,18 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack */ __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_6)); + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_7)); __Pyx_INCREF(__pyx_kp_3); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_kp_3); __Pyx_GIVEREF(__pyx_kp_3); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_7), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_v_o); - __pyx_v_o = __pyx_t_7; - __pyx_t_7 = 0; + __pyx_v_o = __pyx_t_8; + __pyx_t_8 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): @@ -1054,8 +1056,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; + __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_5; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') @@ -1064,8 +1066,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o @@ -1074,8 +1076,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_6); goto __pyx_L3; } @@ -1096,8 +1098,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * for k,v in o.iteritems(): * self.pack(k) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): @@ -1106,38 +1108,38 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * self.pack(k) * self.pack(v) */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_7 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (PyList_CheckExact(__pyx_t_7) || PyTuple_CheckExact(__pyx_t_7)) { + __pyx_t_6 = 0; __pyx_t_8 = __pyx_t_7; __Pyx_INCREF(__pyx_t_8); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_t_7); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); } - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; + } else if (likely(PyTuple_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { + __pyx_t_7 = PyIter_Next(__pyx_t_8); + if (!__pyx_t_7) { if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } - __Pyx_GOTREF(__pyx_t_6); + __Pyx_GOTREF(__pyx_t_7); } - if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { - PyObject* tuple = __pyx_t_6; + if (PyTuple_CheckExact(__pyx_t_7) && likely(PyTuple_GET_SIZE(__pyx_t_7) == 2)) { + PyObject* tuple = __pyx_t_7; __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_v_k); __pyx_v_k = __pyx_2; __pyx_2 = 0; @@ -1145,9 +1147,9 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_3; __pyx_3 = 0; } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = PyObject_GetIter(__pyx_t_7); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_2); __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -1169,18 +1171,18 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_k); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): @@ -1189,20 +1191,20 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) */ - __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_7 = PyObject_Call(__pyx_t_9, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; goto __pyx_L3; } @@ -1215,12 +1217,12 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack */ __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); if (!__pyx_t_1) { - __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); - __pyx_t_10 = __pyx_t_9; + __pyx_t_10 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); + __pyx_t_11 = __pyx_t_10; } else { - __pyx_t_10 = __pyx_t_1; + __pyx_t_11 = __pyx_t_1; } - if (__pyx_t_10) { + if (__pyx_t_11) { /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) @@ -1229,8 +1231,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * for v in o: * self.pack(v) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): @@ -1240,29 +1242,29 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * else: */ if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); + __pyx_t_6 = 0; __pyx_t_8 = __pyx_v_o; __Pyx_INCREF(__pyx_t_8); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); } for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; + } else if (likely(PyTuple_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { + __pyx_t_7 = PyIter_Next(__pyx_t_8); + if (!__pyx_t_7) { if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } - __Pyx_GOTREF(__pyx_t_6); + __Pyx_GOTREF(__pyx_t_7); } __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_t_6; - __pyx_t_6 = 0; + __pyx_v_v = __pyx_t_7; + __pyx_t_7 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) @@ -1271,20 +1273,20 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * else: * # TODO: Serialize with defalt() like simplejson. */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; goto __pyx_L3; } /*else*/ { @@ -1296,16 +1298,16 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * * def pack(self, obj, flush=True): */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_7)); + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_8)); __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; - __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_9 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_8)); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; + __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_9, 0); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __pyx_L3:; @@ -1317,9 +1319,9 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_XDECREF(__pyx_2); __Pyx_XDECREF(__pyx_3); __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); __Pyx_AddTraceback("msgpack.Packer.__pack"); __pyx_r = 0; __pyx_L0:; diff --git a/python/msgpack.pyx b/python/msgpack.pyx index f24f403..aa4006b 100644 --- a/python/msgpack.pyx +++ b/python/msgpack.pyx @@ -118,7 +118,7 @@ cdef class Packer: intval = o msgpack_pack_long_long(&self.pk, intval) elif isinstance(o, float): - fval = 9 + fval = o msgpack_pack_double(&self.pk, fval) elif isinstance(o, str): rawval = o From 4ab64bf9497eccb15081388a667b01c618878d58 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 16 Jun 2009 01:58:07 +0900 Subject: [PATCH 0160/1648] Fix can't pack float values. --- msgpack.c | 212 ++++++++++++++++++++++++++-------------------------- msgpack.pyx | 2 +- 2 files changed, 108 insertions(+), 106 deletions(-) diff --git a/msgpack.c b/msgpack.c index 821f65b..7e5c21f 100644 --- a/msgpack.c +++ b/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Tue Jun 9 13:10:11 2009 */ +/* Generated by Cython 0.11.2 on Tue Jun 16 01:57:05 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -800,13 +800,14 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack int __pyx_t_1; PyObject *__pyx_t_2 = NULL; PY_LONG_LONG __pyx_t_3; - char *__pyx_t_4; - Py_ssize_t __pyx_t_5; - PyObject *__pyx_t_6 = NULL; + double __pyx_t_4; + char *__pyx_t_5; + Py_ssize_t __pyx_t_6; PyObject *__pyx_t_7 = NULL; PyObject *__pyx_t_8 = NULL; - int __pyx_t_9; + PyObject *__pyx_t_9 = NULL; int __pyx_t_10; + int __pyx_t_11; __Pyx_SetupRefcountContext("__pack"); __Pyx_INCREF(__pyx_v_o); __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); @@ -937,7 +938,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< * elif isinstance(o, float): - * fval = 9 + * fval = o */ msgpack_pack_long_long((&__pyx_v_self->pk), __pyx_v_intval); goto __pyx_L3; @@ -947,7 +948,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) */ __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); @@ -956,15 +957,16 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): - * fval = 9 # <<<<<<<<<<<<<< + * fval = o # <<<<<<<<<<<<<< * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): */ - __pyx_v_fval = 9; + __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_fval = __pyx_t_4; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< * elif isinstance(o, str): * rawval = o @@ -974,7 +976,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 - * fval = 9 + * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< * rawval = o @@ -990,8 +992,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; + __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_5; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): @@ -1000,8 +1002,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o @@ -1010,8 +1012,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, unicode): * o = o.encode('utf-8') */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_6); goto __pyx_L3; } @@ -1034,18 +1036,18 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack */ __pyx_t_2 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_encode); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_6)); + __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_7)); __Pyx_INCREF(__pyx_kp_3); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_kp_3); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_kp_3); __Pyx_GIVEREF(__pyx_kp_3); - __pyx_t_7 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_7), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; + __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_v_o); - __pyx_v_o = __pyx_t_7; - __pyx_t_7 = 0; + __pyx_v_o = __pyx_t_8; + __pyx_t_8 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): @@ -1054,8 +1056,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) */ - __pyx_t_4 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_v_rawval = __pyx_t_4; + __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_v_rawval = __pyx_t_5; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') @@ -1064,8 +1066,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o @@ -1074,8 +1076,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_raw_body((&__pyx_v_self->pk), __pyx_v_rawval, __pyx_t_6); goto __pyx_L3; } @@ -1096,8 +1098,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * for k,v in o.iteritems(): * self.pack(k) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): @@ -1106,38 +1108,38 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * self.pack(k) * self.pack(v) */ - __pyx_t_7 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_8 = PyObject_GetAttr(__pyx_v_o, __pyx_kp_iteritems); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_7 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (PyList_CheckExact(__pyx_t_6) || PyTuple_CheckExact(__pyx_t_6)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_t_6; __Pyx_INCREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (PyList_CheckExact(__pyx_t_7) || PyTuple_CheckExact(__pyx_t_7)) { + __pyx_t_6 = 0; __pyx_t_8 = __pyx_t_7; __Pyx_INCREF(__pyx_t_8); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_t_7); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); } - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; + } else if (likely(PyTuple_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { + __pyx_t_7 = PyIter_Next(__pyx_t_8); + if (!__pyx_t_7) { if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } - __Pyx_GOTREF(__pyx_t_6); + __Pyx_GOTREF(__pyx_t_7); } - if (PyTuple_CheckExact(__pyx_t_6) && likely(PyTuple_GET_SIZE(__pyx_t_6) == 2)) { - PyObject* tuple = __pyx_t_6; + if (PyTuple_CheckExact(__pyx_t_7) && likely(PyTuple_GET_SIZE(__pyx_t_7) == 2)) { + PyObject* tuple = __pyx_t_7; __pyx_2 = PyTuple_GET_ITEM(tuple, 0); __Pyx_INCREF(__pyx_2); __pyx_3 = PyTuple_GET_ITEM(tuple, 1); __Pyx_INCREF(__pyx_3); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_v_k); __pyx_v_k = __pyx_2; __pyx_2 = 0; @@ -1145,9 +1147,9 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_3; __pyx_3 = 0; } else { - __pyx_1 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_1 = PyObject_GetIter(__pyx_t_7); if (unlikely(!__pyx_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_2 = __Pyx_UnpackItem(__pyx_1, 0); if (unlikely(!__pyx_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_2); __pyx_3 = __Pyx_UnpackItem(__pyx_1, 1); if (unlikely(!__pyx_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -1169,18 +1171,18 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_k); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_k); __Pyx_GIVEREF(__pyx_v_k); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): @@ -1189,20 +1191,20 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) */ - __pyx_t_8 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); + __pyx_t_9 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_6 = PyObject_Call(__pyx_t_8, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_7 = PyObject_Call(__pyx_t_9, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; goto __pyx_L3; } @@ -1215,12 +1217,12 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack */ __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyTuple_Type))); if (!__pyx_t_1) { - __pyx_t_9 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); - __pyx_t_10 = __pyx_t_9; + __pyx_t_10 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyList_Type))); + __pyx_t_11 = __pyx_t_10; } else { - __pyx_t_10 = __pyx_t_1; + __pyx_t_11 = __pyx_t_1; } - if (__pyx_t_10) { + if (__pyx_t_11) { /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) @@ -1229,8 +1231,8 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * for v in o: * self.pack(v) */ - __pyx_t_5 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_5 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_5); + __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): @@ -1240,29 +1242,29 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * else: */ if (PyList_CheckExact(__pyx_v_o) || PyTuple_CheckExact(__pyx_v_o)) { - __pyx_t_5 = 0; __pyx_t_7 = __pyx_v_o; __Pyx_INCREF(__pyx_t_7); + __pyx_t_6 = 0; __pyx_t_8 = __pyx_v_o; __Pyx_INCREF(__pyx_t_8); } else { - __pyx_t_5 = -1; __pyx_t_7 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = -1; __pyx_t_8 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_8); } for (;;) { - if (likely(PyList_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyList_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; - } else if (likely(PyTuple_CheckExact(__pyx_t_7))) { - if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_7)) break; - __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_7, __pyx_t_5); __Pyx_INCREF(__pyx_t_6); __pyx_t_5++; + if (likely(PyList_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyList_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; + } else if (likely(PyTuple_CheckExact(__pyx_t_8))) { + if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_8)) break; + __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_8, __pyx_t_6); __Pyx_INCREF(__pyx_t_7); __pyx_t_6++; } else { - __pyx_t_6 = PyIter_Next(__pyx_t_7); - if (!__pyx_t_6) { + __pyx_t_7 = PyIter_Next(__pyx_t_8); + if (!__pyx_t_7) { if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 139; __pyx_clineno = __LINE__; goto __pyx_L1_error;} break; } - __Pyx_GOTREF(__pyx_t_6); + __Pyx_GOTREF(__pyx_t_7); } __Pyx_DECREF(__pyx_v_v); - __pyx_v_v = __pyx_t_6; - __pyx_t_6 = 0; + __pyx_v_v = __pyx_t_7; + __pyx_t_7 = 0; /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) @@ -1271,20 +1273,20 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * else: * # TODO: Serialize with defalt() like simplejson. */ - __pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_kp_pack); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_7); __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); __Pyx_INCREF(__pyx_v_v); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_v); __Pyx_GIVEREF(__pyx_v_v); - __pyx_t_8 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 140; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; goto __pyx_L3; } /*else*/ { @@ -1296,16 +1298,16 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack * * def pack(self, obj, flush=True): */ - __pyx_t_7 = PyTuple_New(1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_7)); + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(((PyObject *)__pyx_t_8)); __Pyx_INCREF(__pyx_v_o); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_o); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); - __pyx_t_8 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_7)); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(((PyObject *)__pyx_t_7)); __pyx_t_7 = 0; - __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_8, 0); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_9 = PyNumber_Remainder(__pyx_kp_4, ((PyObject *)__pyx_t_8)); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; + __Pyx_Raise(__pyx_builtin_TypeError, __pyx_t_9, 0); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; {__pyx_filename = __pyx_f[0]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __pyx_L3:; @@ -1317,9 +1319,9 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_XDECREF(__pyx_2); __Pyx_XDECREF(__pyx_3); __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_9); __Pyx_AddTraceback("msgpack.Packer.__pack"); __pyx_r = 0; __pyx_L0:; diff --git a/msgpack.pyx b/msgpack.pyx index f24f403..aa4006b 100644 --- a/msgpack.pyx +++ b/msgpack.pyx @@ -118,7 +118,7 @@ cdef class Packer: intval = o msgpack_pack_long_long(&self.pk, intval) elif isinstance(o, float): - fval = 9 + fval = o msgpack_pack_double(&self.pk, fval) elif isinstance(o, str): rawval = o From 2475187c7d7dc32d3abf825f97f9d7acaaa58e47 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 17 Jun 2009 13:45:08 +0900 Subject: [PATCH 0161/1648] Fix refcount leak and optimize list initialization. --- python/msgpack.c | 229 ++++++++++++++++++++++------------------------- python/setup.py | 2 +- python/unpack.h | 45 ++++++++-- 3 files changed, 142 insertions(+), 134 deletions(-) diff --git a/python/msgpack.c b/python/msgpack.c index 7e5c21f..ed0ac7e 100644 --- a/python/msgpack.c +++ b/python/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Tue Jun 16 01:57:05 2009 */ +/* Generated by Cython 0.11.1 on Wed Jun 17 13:44:30 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -103,9 +103,6 @@ #ifndef __cdecl #define __cdecl #endif - #ifndef __fastcall - #define __fastcall - #endif #else #define _USE_MATH_DEFINES #endif @@ -136,7 +133,6 @@ #include "string.h" #include "pack.h" #include "unpack.h" -#define __PYX_USE_C99_COMPLEX defined(_Complex_I) #ifdef __GNUC__ @@ -228,7 +224,6 @@ static const char * __pyx_cfilenm= __FILE__; static const char *__pyx_filename; static const char **__pyx_f; - #ifdef CYTHON_REFNANNY typedef struct { void (*INCREF)(void*, PyObject*, int); @@ -325,7 +320,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -343,7 +338,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -356,7 +351,7 @@ struct __pyx_obj_7msgpack_Unpacker { }; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -375,12 +370,10 @@ static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ -#define __Pyx_MODULE_NAME "msgpack" -int __pyx_module_is_main_msgpack = 0; + +const char *__pyx_modulename = "msgpack"; /* Implementation of msgpack */ -static char __pyx_k___main__[] = "__main__"; -static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; static char __pyx_k___del__[] = "__del__"; @@ -438,7 +431,7 @@ static PyObject *__pyx_kp_4; static char __pyx_k_3[] = "utf-8"; static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -500,7 +493,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -510,7 +503,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -522,7 +515,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -535,7 +528,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -544,7 +537,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -553,7 +546,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -562,7 +555,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< @@ -576,7 +569,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":62 * msgpack_packer_init(&self.pk, self, _packer_write) * * def __del__(self): # <<<<<<<<<<<<<< @@ -589,7 +582,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj PyObject *__pyx_r = NULL; __Pyx_SetupRefcountContext("__del__"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":63 * * def __del__(self): * free(self.buff); # <<<<<<<<<<<<<< @@ -604,7 +597,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":65 * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< @@ -622,7 +615,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -632,7 +625,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< @@ -654,7 +647,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -666,7 +659,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< @@ -676,7 +669,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< @@ -707,7 +700,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -722,7 +715,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< @@ -743,7 +736,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -758,7 +751,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< @@ -779,7 +772,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * * cdef __pack(self, object o): # <<<<<<<<<<<<<< @@ -813,7 +806,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -823,7 +816,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< @@ -834,7 +827,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< @@ -847,7 +840,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< @@ -858,7 +851,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< @@ -871,7 +864,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< @@ -882,7 +875,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -892,7 +885,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< @@ -902,7 +895,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -913,7 +906,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -923,7 +916,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< @@ -933,7 +926,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -944,7 +937,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -954,7 +947,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = o # <<<<<<<<<<<<<< @@ -964,7 +957,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_fval = __pyx_t_4; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< @@ -975,7 +968,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":123 * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -985,7 +978,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< @@ -995,7 +988,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1005,7 +998,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1017,7 +1010,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -1027,7 +1020,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< @@ -1049,7 +1042,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_o = __pyx_t_8; __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< @@ -1059,7 +1052,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1069,7 +1062,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1081,7 +1074,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1091,7 +1084,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1101,7 +1094,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< @@ -1164,7 +1157,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< @@ -1184,7 +1177,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< @@ -1208,7 +1201,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1224,7 +1217,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } if (__pyx_t_11) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1234,7 +1227,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1266,7 +1259,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< @@ -1291,7 +1284,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< @@ -1333,7 +1326,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -1394,7 +1387,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return NULL; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":146 * * def pack(self, obj, flush=True): * self.__pack(obj) # <<<<<<<<<<<<<< @@ -1405,7 +1398,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":147 * def pack(self, obj, flush=True): * self.__pack(obj) * if flush: # <<<<<<<<<<<<<< @@ -1415,7 +1408,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_2) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":148 * self.__pack(obj) * if flush: * self.flush() # <<<<<<<<<<<<<< @@ -1445,7 +1438,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":150 * self.flush() * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< @@ -1461,7 +1454,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1471,7 +1464,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1481,7 +1474,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< @@ -1506,7 +1499,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1516,7 +1509,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< @@ -1538,7 +1531,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1550,7 +1543,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1559,7 +1552,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1573,7 +1566,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1582,7 +1575,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1593,7 +1586,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1616,7 +1609,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1677,7 +1670,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< @@ -1696,7 +1689,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1716,7 +1709,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1745,7 +1738,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1767,7 +1760,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< @@ -1783,7 +1776,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< @@ -1802,7 +1795,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1822,7 +1815,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1836,7 +1829,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1870,7 +1863,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1891,7 +1884,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< @@ -1901,7 +1894,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1910,7 +1903,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1919,7 +1912,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< @@ -1929,7 +1922,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1955,7 +1948,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1975,7 +1968,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< @@ -1991,7 +1984,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -2097,11 +2090,7 @@ static PyNumberMethods __pyx_tp_as_number_Packer = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else 0, /*nb_long*/ - #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2257,11 +2246,7 @@ static PyNumberMethods __pyx_tp_as_number_Unpacker = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else 0, /*nb_long*/ - #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2404,7 +2389,6 @@ static struct PyModuleDef __pyx_moduledef = { #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, @@ -2496,9 +2480,6 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__pyx_module_is_main_msgpack) { - if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - } /*--- Builtin init code ---*/ if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_skip_dispatch = 0; @@ -2522,7 +2503,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2543,7 +2524,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2552,7 +2533,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -2565,7 +2546,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_k_1); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< @@ -3368,14 +3349,14 @@ static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { return Py_INCREF(x), x; m = Py_TYPE(x)->tp_as_number; #if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { + if (m && m->nb_long) { name = "long"; res = PyNumber_Long(x); } + else if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } #else if (m && m->nb_int) { name = "int"; diff --git a/python/setup.py b/python/setup.py index e5651a0..4bb8693 100644 --- a/python/setup.py +++ b/python/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup, Extension version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c']) +msgpack_mod = Extension('msgpack', sources=['msgpack.c'], extra_compile_args=["-O3"]) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ diff --git a/python/unpack.h b/python/unpack.h index c98c19a..3e99123 100644 --- a/python/unpack.h +++ b/python/unpack.h @@ -20,7 +20,8 @@ #include "msgpack/unpack_define.h" typedef struct { - int reserved; + struct {unsigned int size, last} array_stack[MSGPACK_MAX_STACK_SIZE]; + int array_current; } unpack_user; @@ -42,7 +43,10 @@ struct template_context; typedef struct template_context template_context; static inline msgpack_unpack_object template_callback_root(unpack_user* u) -{ return NULL; } +{ + u->array_current = -1; + return NULL; +} static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) { *o = PyInt_FromLong((long)d); return 0; } @@ -52,8 +56,8 @@ static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_u static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { - if (d >= 0x80000000UL) { - *o = PyLong_FromUnsignedLongLong((unsigned long long)d); + if (d > LONG_MAX) { + *o = PyLong_FromUnsignedLong((unsigned long)d); } else { *o = PyInt_FromLong((long)d); } @@ -92,14 +96,32 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - /* TODO: use PyList_New(n). */ - *o = PyList_New(0); + if (n > 0) { + int cur = ++u->array_current; + u->array_stack[cur].size = n; + u->array_stack[cur].last = 0; + *o = PyList_New(n); + } + else { + *o = PyList_New(0); + } return 0; } static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - PyList_Append(*c, o); + int cur = u->array_current; + int n = u->array_stack[cur].size; + int last = u->array_stack[cur].last; + + PyList_SetItem(*c, last, o); + last++; + if (last >= n) { + u->array_current--; + } + else { + u->array_stack[cur].last = last; + } return 0; } @@ -112,13 +134,18 @@ static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { PyDict_SetItem(*c, k, v); - return 0; + Py_DECREF(k); + Py_DECREF(v); + return 0; } static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { *o = PyString_FromStringAndSize(p, l); - return 0; + if (l < 16) { // without foundation + PyString_InternInPlace(o); + } + return 0; } #include "msgpack/unpack_template.h" From 8ca2c441c9442a06cf55b8cc04360982dd725cb5 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 17 Jun 2009 13:45:08 +0900 Subject: [PATCH 0162/1648] Fix refcount leak and optimize list initialization. --- msgpack.c | 229 +++++++++++++++++++++++++----------------------------- setup.py | 2 +- unpack.h | 45 ++++++++--- 3 files changed, 142 insertions(+), 134 deletions(-) diff --git a/msgpack.c b/msgpack.c index 7e5c21f..ed0ac7e 100644 --- a/msgpack.c +++ b/msgpack.c @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.2 on Tue Jun 16 01:57:05 2009 */ +/* Generated by Cython 0.11.1 on Wed Jun 17 13:44:30 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -103,9 +103,6 @@ #ifndef __cdecl #define __cdecl #endif - #ifndef __fastcall - #define __fastcall - #endif #else #define _USE_MATH_DEFINES #endif @@ -136,7 +133,6 @@ #include "string.h" #include "pack.h" #include "unpack.h" -#define __PYX_USE_C99_COMPLEX defined(_Complex_I) #ifdef __GNUC__ @@ -228,7 +224,6 @@ static const char * __pyx_cfilenm= __FILE__; static const char *__pyx_filename; static const char **__pyx_f; - #ifdef CYTHON_REFNANNY typedef struct { void (*INCREF)(void*, PyObject*, int); @@ -325,7 +320,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -343,7 +338,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -356,7 +351,7 @@ struct __pyx_obj_7msgpack_Unpacker { }; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -375,12 +370,10 @@ static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ -#define __Pyx_MODULE_NAME "msgpack" -int __pyx_module_is_main_msgpack = 0; + +const char *__pyx_modulename = "msgpack"; /* Implementation of msgpack */ -static char __pyx_k___main__[] = "__main__"; -static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; static char __pyx_k___del__[] = "__del__"; @@ -438,7 +431,7 @@ static PyObject *__pyx_kp_4; static char __pyx_k_3[] = "utf-8"; static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -500,7 +493,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -510,7 +503,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -522,7 +515,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -535,7 +528,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -544,7 +537,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -553,7 +546,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -562,7 +555,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< @@ -576,7 +569,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":62 * msgpack_packer_init(&self.pk, self, _packer_write) * * def __del__(self): # <<<<<<<<<<<<<< @@ -589,7 +582,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj PyObject *__pyx_r = NULL; __Pyx_SetupRefcountContext("__del__"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":63 * * def __del__(self): * free(self.buff); # <<<<<<<<<<<<<< @@ -604,7 +597,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":65 * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< @@ -622,7 +615,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -632,7 +625,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< @@ -654,7 +647,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -666,7 +659,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< @@ -676,7 +669,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< @@ -707,7 +700,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -722,7 +715,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< @@ -743,7 +736,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -758,7 +751,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< @@ -779,7 +772,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * * cdef __pack(self, object o): # <<<<<<<<<<<<<< @@ -813,7 +806,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -823,7 +816,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< @@ -834,7 +827,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< @@ -847,7 +840,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< @@ -858,7 +851,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< @@ -871,7 +864,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< @@ -882,7 +875,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -892,7 +885,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< @@ -902,7 +895,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -913,7 +906,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -923,7 +916,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< @@ -933,7 +926,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -944,7 +937,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -954,7 +947,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = o # <<<<<<<<<<<<<< @@ -964,7 +957,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_fval = __pyx_t_4; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< @@ -975,7 +968,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":123 * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -985,7 +978,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< @@ -995,7 +988,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1005,7 +998,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1017,7 +1010,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -1027,7 +1020,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< @@ -1049,7 +1042,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_o = __pyx_t_8; __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< @@ -1059,7 +1052,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1069,7 +1062,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1081,7 +1074,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1091,7 +1084,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1101,7 +1094,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< @@ -1164,7 +1157,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< @@ -1184,7 +1177,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< @@ -1208,7 +1201,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1224,7 +1217,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } if (__pyx_t_11) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1234,7 +1227,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1266,7 +1259,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< @@ -1291,7 +1284,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< @@ -1333,7 +1326,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -1394,7 +1387,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return NULL; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":146 * * def pack(self, obj, flush=True): * self.__pack(obj) # <<<<<<<<<<<<<< @@ -1405,7 +1398,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":147 * def pack(self, obj, flush=True): * self.__pack(obj) * if flush: # <<<<<<<<<<<<<< @@ -1415,7 +1408,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_2) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":148 * self.__pack(obj) * if flush: * self.flush() # <<<<<<<<<<<<<< @@ -1445,7 +1438,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":150 * self.flush() * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< @@ -1461,7 +1454,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1471,7 +1464,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1481,7 +1474,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< @@ -1506,7 +1499,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1516,7 +1509,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< @@ -1538,7 +1531,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1550,7 +1543,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1559,7 +1552,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1573,7 +1566,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1582,7 +1575,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1593,7 +1586,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1616,7 +1609,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1677,7 +1670,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< @@ -1696,7 +1689,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1716,7 +1709,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1745,7 +1738,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1767,7 +1760,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< @@ -1783,7 +1776,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< @@ -1802,7 +1795,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1822,7 +1815,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1836,7 +1829,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1870,7 +1863,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1891,7 +1884,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< @@ -1901,7 +1894,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1910,7 +1903,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1919,7 +1912,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< @@ -1929,7 +1922,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1955,7 +1948,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 +/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1975,7 +1968,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< @@ -1991,7 +1984,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -2097,11 +2090,7 @@ static PyNumberMethods __pyx_tp_as_number_Packer = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else 0, /*nb_long*/ - #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2257,11 +2246,7 @@ static PyNumberMethods __pyx_tp_as_number_Unpacker = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*reserved*/ - #else 0, /*nb_long*/ - #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2404,7 +2389,6 @@ static struct PyModuleDef __pyx_moduledef = { #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { - {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, @@ -2496,9 +2480,6 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - if (__pyx_module_is_main_msgpack) { - if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - } /*--- Builtin init code ---*/ if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_skip_dispatch = 0; @@ -2522,7 +2503,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2543,7 +2524,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2552,7 +2533,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -2565,7 +2546,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_k_1); - /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 + /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< @@ -3368,14 +3349,14 @@ static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { return Py_INCREF(x), x; m = Py_TYPE(x)->tp_as_number; #if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_int) { - name = "int"; - res = PyNumber_Int(x); - } - else if (m && m->nb_long) { + if (m && m->nb_long) { name = "long"; res = PyNumber_Long(x); } + else if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } #else if (m && m->nb_int) { name = "int"; diff --git a/setup.py b/setup.py index e5651a0..4bb8693 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup, Extension version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c']) +msgpack_mod = Extension('msgpack', sources=['msgpack.c'], extra_compile_args=["-O3"]) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ diff --git a/unpack.h b/unpack.h index c98c19a..3e99123 100644 --- a/unpack.h +++ b/unpack.h @@ -20,7 +20,8 @@ #include "msgpack/unpack_define.h" typedef struct { - int reserved; + struct {unsigned int size, last} array_stack[MSGPACK_MAX_STACK_SIZE]; + int array_current; } unpack_user; @@ -42,7 +43,10 @@ struct template_context; typedef struct template_context template_context; static inline msgpack_unpack_object template_callback_root(unpack_user* u) -{ return NULL; } +{ + u->array_current = -1; + return NULL; +} static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) { *o = PyInt_FromLong((long)d); return 0; } @@ -52,8 +56,8 @@ static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_u static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { - if (d >= 0x80000000UL) { - *o = PyLong_FromUnsignedLongLong((unsigned long long)d); + if (d > LONG_MAX) { + *o = PyLong_FromUnsignedLong((unsigned long)d); } else { *o = PyInt_FromLong((long)d); } @@ -92,14 +96,32 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - /* TODO: use PyList_New(n). */ - *o = PyList_New(0); + if (n > 0) { + int cur = ++u->array_current; + u->array_stack[cur].size = n; + u->array_stack[cur].last = 0; + *o = PyList_New(n); + } + else { + *o = PyList_New(0); + } return 0; } static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - PyList_Append(*c, o); + int cur = u->array_current; + int n = u->array_stack[cur].size; + int last = u->array_stack[cur].last; + + PyList_SetItem(*c, last, o); + last++; + if (last >= n) { + u->array_current--; + } + else { + u->array_stack[cur].last = last; + } return 0; } @@ -112,13 +134,18 @@ static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { PyDict_SetItem(*c, k, v); - return 0; + Py_DECREF(k); + Py_DECREF(v); + return 0; } static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { *o = PyString_FromStringAndSize(p, l); - return 0; + if (l < 16) { // without foundation + PyString_InternInPlace(o); + } + return 0; } #include "msgpack/unpack_template.h" From 0d14239c2197b616196a535c9aeb818da1040e9d Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 09:51:24 +0900 Subject: [PATCH 0163/1648] Optimize to parsing data that has a number of same short raw field. --- python/{msgpack.c => msgpack.cpp} | 229 ++++++++++++++++-------------- python/setup.py | 2 +- python/unpack.h | 38 ++++- 3 files changed, 158 insertions(+), 111 deletions(-) rename python/{msgpack.c => msgpack.cpp} (94%) diff --git a/python/msgpack.c b/python/msgpack.cpp similarity index 94% rename from python/msgpack.c rename to python/msgpack.cpp index ed0ac7e..760f0c0 100644 --- a/python/msgpack.c +++ b/python/msgpack.cpp @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.1 on Wed Jun 17 13:44:30 2009 */ +/* Generated by Cython 0.11.2 on Mon Jun 22 02:56:08 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -103,6 +103,9 @@ #ifndef __cdecl #define __cdecl #endif + #ifndef __fastcall + #define __fastcall + #endif #else #define _USE_MATH_DEFINES #endif @@ -133,6 +136,7 @@ #include "string.h" #include "pack.h" #include "unpack.h" +#define __PYX_USE_C99_COMPLEX defined(_Complex_I) #ifdef __GNUC__ @@ -224,6 +228,7 @@ static const char * __pyx_cfilenm= __FILE__; static const char *__pyx_filename; static const char **__pyx_f; + #ifdef CYTHON_REFNANNY typedef struct { void (*INCREF)(void*, PyObject*, int); @@ -320,7 +325,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -338,7 +343,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":200 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -351,7 +356,7 @@ struct __pyx_obj_7msgpack_Unpacker { }; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -370,10 +375,12 @@ static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ - -const char *__pyx_modulename = "msgpack"; +#define __Pyx_MODULE_NAME "msgpack" +int __pyx_module_is_main_msgpack = 0; /* Implementation of msgpack */ +static char __pyx_k___main__[] = "__main__"; +static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; static char __pyx_k___del__[] = "__del__"; @@ -431,7 +438,7 @@ static PyObject *__pyx_kp_4; static char __pyx_k_3[] = "utf-8"; static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -493,7 +500,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -503,7 +510,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -515,7 +522,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -528,7 +535,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -537,7 +544,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -546,7 +553,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -555,7 +562,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< @@ -569,7 +576,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":62 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 * msgpack_packer_init(&self.pk, self, _packer_write) * * def __del__(self): # <<<<<<<<<<<<<< @@ -582,7 +589,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj PyObject *__pyx_r = NULL; __Pyx_SetupRefcountContext("__del__"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":63 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 * * def __del__(self): * free(self.buff); # <<<<<<<<<<<<<< @@ -597,7 +604,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":65 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< @@ -615,7 +622,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -625,7 +632,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< @@ -647,7 +654,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -659,7 +666,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":70 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< @@ -669,7 +676,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":71 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< @@ -700,7 +707,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":73 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -715,7 +722,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":86 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< @@ -736,7 +743,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":88 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -751,7 +758,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":101 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< @@ -772,7 +779,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":103 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * * cdef __pack(self, object o): # <<<<<<<<<<<<<< @@ -806,7 +813,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -816,7 +823,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< @@ -827,7 +834,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< @@ -840,7 +847,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< @@ -851,7 +858,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< @@ -864,7 +871,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< @@ -875,7 +882,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -885,7 +892,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< @@ -895,7 +902,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -906,7 +913,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -916,7 +923,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< @@ -926,7 +933,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -937,7 +944,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -947,7 +954,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = o # <<<<<<<<<<<<<< @@ -957,7 +964,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_fval = __pyx_t_4; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< @@ -968,7 +975,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -978,7 +985,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< @@ -988,7 +995,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -998,7 +1005,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1010,7 +1017,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -1020,7 +1027,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< @@ -1042,7 +1049,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_o = __pyx_t_8; __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< @@ -1052,7 +1059,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1062,7 +1069,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1074,7 +1081,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1084,7 +1091,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1094,7 +1101,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< @@ -1157,7 +1164,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< @@ -1177,7 +1184,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< @@ -1201,7 +1208,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1217,7 +1224,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } if (__pyx_t_11) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1227,7 +1234,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":139 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1259,7 +1266,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":140 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< @@ -1284,7 +1291,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":143 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< @@ -1326,7 +1333,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -1387,7 +1394,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return NULL; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 * * def pack(self, obj, flush=True): * self.__pack(obj) # <<<<<<<<<<<<<< @@ -1398,7 +1405,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 * def pack(self, obj, flush=True): * self.__pack(obj) * if flush: # <<<<<<<<<<<<<< @@ -1408,7 +1415,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_2) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 * self.__pack(obj) * if flush: * self.flush() # <<<<<<<<<<<<<< @@ -1438,7 +1445,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":150 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 * self.flush() * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< @@ -1454,7 +1461,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1464,7 +1471,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1474,7 +1481,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":153 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< @@ -1499,7 +1506,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1509,7 +1516,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< @@ -1531,7 +1538,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1543,7 +1550,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":158 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1552,7 +1559,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1566,7 +1573,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1575,7 +1582,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":162 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1586,7 +1593,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":163 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1609,7 +1616,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":165 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1670,7 +1677,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< @@ -1689,7 +1696,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1709,7 +1716,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1738,7 +1745,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":170 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1760,7 +1767,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":171 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< @@ -1776,7 +1783,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":172 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< @@ -1795,7 +1802,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":173 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1815,7 +1822,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":174 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1829,7 +1836,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":175 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1863,7 +1870,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":186 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1884,7 +1891,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":188 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< @@ -1894,7 +1901,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1903,7 +1910,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1912,7 +1919,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":192 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< @@ -1922,7 +1929,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":193 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1948,7 +1955,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":195 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1968,7 +1975,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":197 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< @@ -1984,7 +1991,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":198 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -2090,7 +2097,11 @@ static PyNumberMethods __pyx_tp_as_number_Packer = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else 0, /*nb_long*/ + #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2246,7 +2257,11 @@ static PyNumberMethods __pyx_tp_as_number_Unpacker = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else 0, /*nb_long*/ + #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2389,6 +2404,7 @@ static struct PyModuleDef __pyx_moduledef = { #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, @@ -2480,6 +2496,9 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__pyx_module_is_main_msgpack) { + if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + } /*--- Builtin init code ---*/ if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_skip_dispatch = 0; @@ -2503,7 +2522,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2524,7 +2543,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2533,7 +2552,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -2546,7 +2565,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_k_1); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":202 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< @@ -3349,14 +3368,14 @@ static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { return Py_INCREF(x), x; m = Py_TYPE(x)->tp_as_number; #if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } - else if (m && m->nb_int) { + if (m && m->nb_int) { name = "int"; res = PyNumber_Int(x); } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } #else if (m && m->nb_int) { name = "int"; diff --git a/python/setup.py b/python/setup.py index 4bb8693..f4c84dc 100644 --- a/python/setup.py +++ b/python/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup, Extension version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c'], extra_compile_args=["-O3"]) +msgpack_mod = Extension('msgpack', sources=['msgpack.cpp']) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ diff --git a/python/unpack.h b/python/unpack.h index 3e99123..694e816 100644 --- a/python/unpack.h +++ b/python/unpack.h @@ -16,12 +16,28 @@ * limitations under the License. */ +#include +#include + #define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" -typedef struct { - struct {unsigned int size, last} array_stack[MSGPACK_MAX_STACK_SIZE]; +using namespace std; + +typedef struct unpack_user { + struct array_stack_type{unsigned int size, last;}; + array_stack_type array_stack[MSGPACK_MAX_STACK_SIZE]; int array_current; + + map str_cache; + + ~unpack_user() { + map::iterator it, itend; + itend = str_cache.end(); + for (it = str_cache.begin(); it != itend; ++it) { + Py_DECREF(it->second); + } + } } unpack_user; @@ -141,9 +157,21 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { - *o = PyString_FromStringAndSize(p, l); - if (l < 16) { // without foundation - PyString_InternInPlace(o); + if (l < 16) { + string s(p, l); + map::iterator it = u->str_cache.find(s); + if (it != u->str_cache.end()) { + *o = it->second; + Py_INCREF(*o); + } + else { + *o = PyString_FromStringAndSize(p, l); + Py_INCREF(*o); + u->str_cache[s] = *o; + } + } + else { + *o = PyString_FromStringAndSize(p, l); } return 0; } From dee7b2092474d00090f22db95623f2ac134eada2 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 09:51:24 +0900 Subject: [PATCH 0164/1648] Optimize to parsing data that has a number of same short raw field. --- msgpack.c => msgpack.cpp | 229 +++++++++++++++++++++------------------ setup.py | 2 +- unpack.h | 38 ++++++- 3 files changed, 158 insertions(+), 111 deletions(-) rename msgpack.c => msgpack.cpp (94%) diff --git a/msgpack.c b/msgpack.cpp similarity index 94% rename from msgpack.c rename to msgpack.cpp index ed0ac7e..760f0c0 100644 --- a/msgpack.c +++ b/msgpack.cpp @@ -1,4 +1,4 @@ -/* Generated by Cython 0.11.1 on Wed Jun 17 13:44:30 2009 */ +/* Generated by Cython 0.11.2 on Mon Jun 22 02:56:08 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -103,6 +103,9 @@ #ifndef __cdecl #define __cdecl #endif + #ifndef __fastcall + #define __fastcall + #endif #else #define _USE_MATH_DEFINES #endif @@ -133,6 +136,7 @@ #include "string.h" #include "pack.h" #include "unpack.h" +#define __PYX_USE_C99_COMPLEX defined(_Complex_I) #ifdef __GNUC__ @@ -224,6 +228,7 @@ static const char * __pyx_cfilenm= __FILE__; static const char *__pyx_filename; static const char **__pyx_f; + #ifdef CYTHON_REFNANNY typedef struct { void (*INCREF)(void*, PyObject*, int); @@ -320,7 +325,7 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ /* Type declarations */ -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -338,7 +343,7 @@ struct __pyx_obj_7msgpack_Packer { PyObject *strm; }; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":200 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":200 * return unpacks(packed) * * cdef class Unpacker: # <<<<<<<<<<<<<< @@ -351,7 +356,7 @@ struct __pyx_obj_7msgpack_Unpacker { }; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":39 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":39 * cdef int BUFF_SIZE=2*1024 * * cdef class Packer: # <<<<<<<<<<<<<< @@ -370,10 +375,12 @@ static PyTypeObject *__pyx_ptype_7msgpack_Unpacker = 0; static int __pyx_v_7msgpack_BUFF_SIZE; static PyObject *__pyx_k_1 = 0; static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *, const char*, unsigned int); /*proto*/ - -const char *__pyx_modulename = "msgpack"; +#define __Pyx_MODULE_NAME "msgpack" +int __pyx_module_is_main_msgpack = 0; /* Implementation of msgpack */ +static char __pyx_k___main__[] = "__main__"; +static PyObject *__pyx_kp___main__; static char __pyx_k___init__[] = "__init__"; static PyObject *__pyx_kp___init__; static char __pyx_k___del__[] = "__del__"; @@ -431,7 +438,7 @@ static PyObject *__pyx_kp_4; static char __pyx_k_3[] = "utf-8"; static char __pyx_k_4[] = "can't serialize %r"; -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":51 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":51 * cdef object strm * * def __init__(self, strm, int size=0): # <<<<<<<<<<<<<< @@ -493,7 +500,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return -1; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":52 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":52 * * def __init__(self, strm, int size=0): * if size <= 0: # <<<<<<<<<<<<<< @@ -503,7 +510,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __pyx_t_1 = (__pyx_v_size <= 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":53 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":53 * def __init__(self, strm, int size=0): * if size <= 0: * size = BUFF_SIZE # <<<<<<<<<<<<<< @@ -515,7 +522,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * } __pyx_L6:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":55 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":55 * size = BUFF_SIZE * * self.strm = strm # <<<<<<<<<<<<<< @@ -528,7 +535,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * __Pyx_DECREF(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm); ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm = __pyx_v_strm; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":56 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":56 * * self.strm = strm * self.buff = malloc(size) # <<<<<<<<<<<<<< @@ -537,7 +544,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->buff = ((char *)malloc(__pyx_v_size)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":57 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":57 * self.strm = strm * self.buff = malloc(size) * self.allocated = size # <<<<<<<<<<<<<< @@ -546,7 +553,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->allocated = __pyx_v_size; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":58 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":58 * self.buff = malloc(size) * self.allocated = size * self.length = 0 # <<<<<<<<<<<<<< @@ -555,7 +562,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * */ ((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":60 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":60 * self.length = 0 * * msgpack_packer_init(&self.pk, self, _packer_write) # <<<<<<<<<<<<<< @@ -569,7 +576,7 @@ static int __pyx_pf_7msgpack_6Packer___init__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":62 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":62 * msgpack_packer_init(&self.pk, self, _packer_write) * * def __del__(self): # <<<<<<<<<<<<<< @@ -582,7 +589,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj PyObject *__pyx_r = NULL; __Pyx_SetupRefcountContext("__del__"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":63 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":63 * * def __del__(self): * free(self.buff); # <<<<<<<<<<<<<< @@ -597,7 +604,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer___del__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":65 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":65 * free(self.buff); * * def flush(self): # <<<<<<<<<<<<<< @@ -615,7 +622,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("flush"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":67 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":67 * def flush(self): * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: # <<<<<<<<<<<<<< @@ -625,7 +632,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = (((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":68 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":68 * """Flash local buffer and output stream if it has 'flush()' method.""" * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) # <<<<<<<<<<<<<< @@ -647,7 +654,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":69 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":69 * if self.length > 0: * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 # <<<<<<<<<<<<<< @@ -659,7 +666,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec } __pyx_L5:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":70 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":70 * self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) * self.length = 0 * if hasattr(self.strm, 'flush'): # <<<<<<<<<<<<<< @@ -669,7 +676,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec __pyx_t_1 = PyObject_HasAttr(((struct __pyx_obj_7msgpack_Packer *)__pyx_v_self)->strm, __pyx_kp_2); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":71 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":71 * self.length = 0 * if hasattr(self.strm, 'flush'): * self.strm.flush() # <<<<<<<<<<<<<< @@ -700,7 +707,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_flush(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":73 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":73 * self.strm.flush() * * def pack_list(self, len): # <<<<<<<<<<<<<< @@ -715,7 +722,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_list"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":86 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":86 * packer.pack(['foo', 'bar']) * """ * msgpack_pack_array(&self.pk, len) # <<<<<<<<<<<<<< @@ -736,7 +743,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_list(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":88 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":88 * msgpack_pack_array(&self.pk, len) * * def pack_dict(self, len): # <<<<<<<<<<<<<< @@ -751,7 +758,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO size_t __pyx_t_1; __Pyx_SetupRefcountContext("pack_dict"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":101 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":101 * packer.pack({'foo', 'bar'}) * """ * msgpack_pack_map(&self.pk, len) # <<<<<<<<<<<<<< @@ -772,7 +779,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack_dict(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":103 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":103 * msgpack_pack_map(&self.pk, len) * * cdef __pack(self, object o): # <<<<<<<<<<<<<< @@ -806,7 +813,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_k = Py_None; __Pyx_INCREF(Py_None); __pyx_v_v = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":108 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":108 * cdef char* rawval * * if o is None: # <<<<<<<<<<<<<< @@ -816,7 +823,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = (__pyx_v_o == Py_None); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":109 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":109 * * if o is None: * msgpack_pack_nil(&self.pk) # <<<<<<<<<<<<<< @@ -827,7 +834,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":110 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":110 * if o is None: * msgpack_pack_nil(&self.pk) * elif o is True: # <<<<<<<<<<<<<< @@ -840,7 +847,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":111 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":111 * msgpack_pack_nil(&self.pk) * elif o is True: * msgpack_pack_true(&self.pk) # <<<<<<<<<<<<<< @@ -851,7 +858,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":112 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":112 * elif o is True: * msgpack_pack_true(&self.pk) * elif o is False: # <<<<<<<<<<<<<< @@ -864,7 +871,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":113 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":113 * msgpack_pack_true(&self.pk) * elif o is False: * msgpack_pack_false(&self.pk) # <<<<<<<<<<<<<< @@ -875,7 +882,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":114 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":114 * elif o is False: * msgpack_pack_false(&self.pk) * elif isinstance(o, long): # <<<<<<<<<<<<<< @@ -885,7 +892,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyLong_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":115 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":115 * msgpack_pack_false(&self.pk) * elif isinstance(o, long): * intval = o # <<<<<<<<<<<<<< @@ -895,7 +902,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":116 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":116 * elif isinstance(o, long): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -906,7 +913,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":117 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":117 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): # <<<<<<<<<<<<<< @@ -916,7 +923,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyInt_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":118 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":118 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, int): * intval = o # <<<<<<<<<<<<<< @@ -926,7 +933,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_3 = __Pyx_PyInt_AsLongLong(__pyx_v_o); if (unlikely((__pyx_t_3 == (PY_LONG_LONG)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 118; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_intval = __pyx_t_3; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":119 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":119 * elif isinstance(o, int): * intval = o * msgpack_pack_long_long(&self.pk, intval) # <<<<<<<<<<<<<< @@ -937,7 +944,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":120 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":120 * intval = o * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): # <<<<<<<<<<<<<< @@ -947,7 +954,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyFloat_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":121 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":121 * msgpack_pack_long_long(&self.pk, intval) * elif isinstance(o, float): * fval = o # <<<<<<<<<<<<<< @@ -957,7 +964,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_4 = __pyx_PyFloat_AsDouble(__pyx_v_o); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_fval = __pyx_t_4; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":122 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":122 * elif isinstance(o, float): * fval = o * msgpack_pack_double(&self.pk, fval) # <<<<<<<<<<<<<< @@ -968,7 +975,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":123 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":123 * fval = o * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): # <<<<<<<<<<<<<< @@ -978,7 +985,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyString_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":124 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":124 * msgpack_pack_double(&self.pk, fval) * elif isinstance(o, str): * rawval = o # <<<<<<<<<<<<<< @@ -988,7 +995,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":125 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":125 * elif isinstance(o, str): * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -998,7 +1005,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":126 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":126 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1010,7 +1017,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":127 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":127 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): # <<<<<<<<<<<<<< @@ -1020,7 +1027,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyUnicode_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":128 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":128 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, unicode): * o = o.encode('utf-8') # <<<<<<<<<<<<<< @@ -1042,7 +1049,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_o = __pyx_t_8; __pyx_t_8 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":129 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":129 * elif isinstance(o, unicode): * o = o.encode('utf-8') * rawval = o # <<<<<<<<<<<<<< @@ -1052,7 +1059,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_5 = __Pyx_PyBytes_AsString(__pyx_v_o); if (unlikely((!__pyx_t_5) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_rawval = __pyx_t_5; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":130 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":130 * o = o.encode('utf-8') * rawval = o * msgpack_pack_raw(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1062,7 +1069,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_raw((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":131 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":131 * rawval = o * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) # <<<<<<<<<<<<<< @@ -1074,7 +1081,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":132 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":132 * msgpack_pack_raw(&self.pk, len(o)) * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): # <<<<<<<<<<<<<< @@ -1084,7 +1091,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_1 = PyObject_TypeCheck(__pyx_v_o, ((PyTypeObject *)((PyObject*)&PyDict_Type))); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":133 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":133 * msgpack_pack_raw_body(&self.pk, rawval, len(o)) * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1094,7 +1101,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 133; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_map((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":134 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":134 * elif isinstance(o, dict): * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): # <<<<<<<<<<<<<< @@ -1157,7 +1164,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_3 = 0; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":135 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":135 * msgpack_pack_map(&self.pk, len(o)) * for k,v in o.iteritems(): * self.pack(k) # <<<<<<<<<<<<<< @@ -1177,7 +1184,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":136 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":136 * for k,v in o.iteritems(): * self.pack(k) * self.pack(v) # <<<<<<<<<<<<<< @@ -1201,7 +1208,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack goto __pyx_L3; } - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":137 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":137 * self.pack(k) * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): # <<<<<<<<<<<<<< @@ -1217,7 +1224,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } if (__pyx_t_11) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":138 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":138 * self.pack(v) * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) # <<<<<<<<<<<<<< @@ -1227,7 +1234,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_t_6 = PyObject_Length(__pyx_v_o); if (unlikely(__pyx_t_6 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} msgpack_pack_array((&__pyx_v_self->pk), __pyx_t_6); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":139 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":139 * elif isinstance(o, tuple) or isinstance(o, list): * msgpack_pack_array(&self.pk, len(o)) * for v in o: # <<<<<<<<<<<<<< @@ -1259,7 +1266,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack __pyx_v_v = __pyx_t_7; __pyx_t_7 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":140 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":140 * msgpack_pack_array(&self.pk, len(o)) * for v in o: * self.pack(v) # <<<<<<<<<<<<<< @@ -1284,7 +1291,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":143 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":143 * else: * # TODO: Serialize with defalt() like simplejson. * raise TypeError, "can't serialize %r" % (o,) # <<<<<<<<<<<<<< @@ -1326,7 +1333,7 @@ static PyObject *__pyx_f_7msgpack_6Packer___pack(struct __pyx_obj_7msgpack_Pack return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -1387,7 +1394,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return NULL; __pyx_L4_argument_unpacking_done:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":146 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":146 * * def pack(self, obj, flush=True): * self.__pack(obj) # <<<<<<<<<<<<<< @@ -1398,7 +1405,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":147 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":147 * def pack(self, obj, flush=True): * self.__pack(obj) * if flush: # <<<<<<<<<<<<<< @@ -1408,7 +1415,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_flush); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_2) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":148 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":148 * self.__pack(obj) * if flush: * self.flush() # <<<<<<<<<<<<<< @@ -1438,7 +1445,7 @@ static PyObject *__pyx_pf_7msgpack_6Packer_pack(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":150 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":150 * self.flush() * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): # <<<<<<<<<<<<<< @@ -1454,7 +1461,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p PyObject *__pyx_t_4 = NULL; __Pyx_SetupRefcountContext("_packer_write"); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":151 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":151 * * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: # <<<<<<<<<<<<<< @@ -1464,7 +1471,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = ((__pyx_v_packer->length + __pyx_v_l) > __pyx_v_packer->allocated); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":152 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":152 * cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): * if packer.length + l > packer.allocated: * if packer.length > 0: # <<<<<<<<<<<<<< @@ -1474,7 +1481,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_packer->length > 0); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":153 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":153 * if packer.length + l > packer.allocated: * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) # <<<<<<<<<<<<<< @@ -1499,7 +1506,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L4:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":154 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":154 * if packer.length > 0: * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: # <<<<<<<<<<<<<< @@ -1509,7 +1516,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __pyx_t_1 = (__pyx_v_l > 64); if (__pyx_t_1) { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":155 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":155 * packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) # <<<<<<<<<<<<<< @@ -1531,7 +1538,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":156 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":156 * if l > 64: * packer.strm.write(PyString_FromStringAndSize(b, l)) * packer.length = 0 # <<<<<<<<<<<<<< @@ -1543,7 +1550,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":158 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":158 * packer.length = 0 * else: * memcpy(packer.buff, b, l) # <<<<<<<<<<<<<< @@ -1552,7 +1559,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy(__pyx_v_packer->buff, __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":159 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":159 * else: * memcpy(packer.buff, b, l) * packer.length = l # <<<<<<<<<<<<<< @@ -1566,7 +1573,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } /*else*/ { - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":161 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":161 * packer.length = l * else: * memcpy(packer.buff + packer.length, b, l) # <<<<<<<<<<<<<< @@ -1575,7 +1582,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p */ memcpy((__pyx_v_packer->buff + __pyx_v_packer->length), __pyx_v_b, __pyx_v_l); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":162 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":162 * else: * memcpy(packer.buff + packer.length, b, l) * packer.length += l # <<<<<<<<<<<<<< @@ -1586,7 +1593,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p } __pyx_L3:; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":163 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":163 * memcpy(packer.buff + packer.length, b, l) * packer.length += l * return 0 # <<<<<<<<<<<<<< @@ -1609,7 +1616,7 @@ static int __pyx_f_7msgpack__packer_write(struct __pyx_obj_7msgpack_Packer *__p return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":165 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":165 * return 0 * * def pack(object o, object stream): # <<<<<<<<<<<<<< @@ -1670,7 +1677,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_L4_argument_unpacking_done:; __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":166 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":166 * * def pack(object o, object stream): * packer = Packer(stream) # <<<<<<<<<<<<<< @@ -1689,7 +1696,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":167 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":167 * def pack(object o, object stream): * packer = Packer(stream) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1709,7 +1716,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":168 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":168 * packer = Packer(stream) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1738,7 +1745,7 @@ static PyObject *__pyx_pf_7msgpack_pack(PyObject *__pyx_self, PyObject *__pyx_ar return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":170 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":170 * packer.flush() * * def packs(object o): # <<<<<<<<<<<<<< @@ -1760,7 +1767,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = Py_None; __Pyx_INCREF(Py_None); __pyx_v_packer = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":171 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":171 * * def packs(object o): * buf = StringIO() # <<<<<<<<<<<<<< @@ -1776,7 +1783,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_buf = __pyx_t_1; __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":172 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":172 * def packs(object o): * buf = StringIO() * packer = Packer(buf) # <<<<<<<<<<<<<< @@ -1795,7 +1802,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __pyx_v_packer = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":173 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":173 * buf = StringIO() * packer = Packer(buf) * packer.pack(o) # <<<<<<<<<<<<<< @@ -1815,7 +1822,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":174 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":174 * packer = Packer(buf) * packer.pack(o) * packer.flush() # <<<<<<<<<<<<<< @@ -1829,7 +1836,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":175 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":175 * packer.pack(o) * packer.flush() * return buf.getvalue() # <<<<<<<<<<<<<< @@ -1863,7 +1870,7 @@ static PyObject *__pyx_pf_7msgpack_packs(PyObject *__pyx_self, PyObject *__pyx_v return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":186 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":186 * * * def unpacks(object packed_bytes): # <<<<<<<<<<<<<< @@ -1884,7 +1891,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __Pyx_SetupRefcountContext("unpacks"); __pyx_self = __pyx_self; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":188 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":188 * def unpacks(object packed_bytes): * """Unpack packed_bytes to object. Returns unpacked object.""" * cdef const_char_ptr p = packed_bytes # <<<<<<<<<<<<<< @@ -1894,7 +1901,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_1 = __Pyx_PyBytes_AsString(__pyx_v_packed_bytes); if (unlikely((!__pyx_t_1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_p = __pyx_t_1; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":190 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":190 * cdef const_char_ptr p = packed_bytes * cdef template_context ctx * cdef size_t off = 0 # <<<<<<<<<<<<<< @@ -1903,7 +1910,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ __pyx_v_off = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":191 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":191 * cdef template_context ctx * cdef size_t off = 0 * template_init(&ctx) # <<<<<<<<<<<<<< @@ -1912,7 +1919,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx */ template_init((&__pyx_v_ctx)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":192 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":192 * cdef size_t off = 0 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) # <<<<<<<<<<<<<< @@ -1922,7 +1929,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx __pyx_t_2 = PyObject_Length(__pyx_v_packed_bytes); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;} template_execute((&__pyx_v_ctx), __pyx_v_p, __pyx_t_2, (&__pyx_v_off)); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":193 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":193 * template_init(&ctx) * template_execute(&ctx, p, len(packed_bytes), &off) * return template_data(&ctx) # <<<<<<<<<<<<<< @@ -1948,7 +1955,7 @@ static PyObject *__pyx_pf_7msgpack_unpacks(PyObject *__pyx_self, PyObject *__pyx return __pyx_r; } -/* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":195 +/* "/home/inada-n/work/msgpack/python/msgpack.pyx":195 * return template_data(&ctx) * * def unpack(object stream): # <<<<<<<<<<<<<< @@ -1968,7 +1975,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_self = __pyx_self; __pyx_v_packed = Py_None; __Pyx_INCREF(Py_None); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":197 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":197 * def unpack(object stream): * """unpack from stream.""" * packed = stream.read() # <<<<<<<<<<<<<< @@ -1984,7 +1991,7 @@ static PyObject *__pyx_pf_7msgpack_unpack(PyObject *__pyx_self, PyObject *__pyx_ __pyx_v_packed = __pyx_t_2; __pyx_t_2 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":198 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":198 * """unpack from stream.""" * packed = stream.read() * return unpacks(packed) # <<<<<<<<<<<<<< @@ -2090,7 +2097,11 @@ static PyNumberMethods __pyx_tp_as_number_Packer = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else 0, /*nb_long*/ + #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2246,7 +2257,11 @@ static PyNumberMethods __pyx_tp_as_number_Unpacker = { 0, /*nb_coerce*/ #endif 0, /*nb_int*/ + #if PY_MAJOR_VERSION >= 3 + 0, /*reserved*/ + #else 0, /*nb_long*/ + #endif 0, /*nb_float*/ #if PY_MAJOR_VERSION < 3 0, /*nb_oct*/ @@ -2389,6 +2404,7 @@ static struct PyModuleDef __pyx_moduledef = { #endif static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp___main__, __pyx_k___main__, sizeof(__pyx_k___main__), 1, 1, 1}, {&__pyx_kp___init__, __pyx_k___init__, sizeof(__pyx_k___init__), 1, 1, 1}, {&__pyx_kp___del__, __pyx_k___del__, sizeof(__pyx_k___del__), 1, 1, 1}, {&__pyx_kp_flush, __pyx_k_flush, sizeof(__pyx_k_flush), 1, 1, 1}, @@ -2480,6 +2496,9 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + if (__pyx_module_is_main_msgpack) { + if (__Pyx_SetAttrString(__pyx_m, "__name__", __pyx_kp___main__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; + } /*--- Builtin init code ---*/ if (unlikely(__Pyx_InitCachedBuiltins() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_skip_dispatch = 0; @@ -2503,7 +2522,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) /*--- Function import code ---*/ /*--- Execution code ---*/ - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":3 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":3 * # coding: utf-8 * * from cStringIO import StringIO # <<<<<<<<<<<<<< @@ -2524,7 +2543,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __Pyx_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_DECREF(__pyx_1); __pyx_1 = 0; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":37 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":37 * * * cdef int BUFF_SIZE=2*1024 # <<<<<<<<<<<<<< @@ -2533,7 +2552,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) */ __pyx_v_7msgpack_BUFF_SIZE = 2048; - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":145 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":145 * raise TypeError, "can't serialize %r" % (o,) * * def pack(self, obj, flush=True): # <<<<<<<<<<<<<< @@ -2546,7 +2565,7 @@ PyMODINIT_FUNC PyInit_msgpack(void) __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_k_1); - /* "/home/inada-n/work/msgpack/msgpack-py/python/msgpack.pyx":202 + /* "/home/inada-n/work/msgpack/python/msgpack.pyx":202 * cdef class Unpacker: * """Do nothing. This function is for symmetric to Packer""" * unpack = staticmethod(unpacks) # <<<<<<<<<<<<<< @@ -3349,14 +3368,14 @@ static INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x) { return Py_INCREF(x), x; m = Py_TYPE(x)->tp_as_number; #if PY_VERSION_HEX < 0x03000000 - if (m && m->nb_long) { - name = "long"; - res = PyNumber_Long(x); - } - else if (m && m->nb_int) { + if (m && m->nb_int) { name = "int"; res = PyNumber_Int(x); } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } #else if (m && m->nb_int) { name = "int"; diff --git a/setup.py b/setup.py index 4bb8693..f4c84dc 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup, Extension version = '0.0.1' -msgpack_mod = Extension('msgpack', sources=['msgpack.c'], extra_compile_args=["-O3"]) +msgpack_mod = Extension('msgpack', sources=['msgpack.cpp']) desc = 'MessagePack serializer/desirializer.' long_desc = desc + """ diff --git a/unpack.h b/unpack.h index 3e99123..694e816 100644 --- a/unpack.h +++ b/unpack.h @@ -16,12 +16,28 @@ * limitations under the License. */ +#include +#include + #define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" -typedef struct { - struct {unsigned int size, last} array_stack[MSGPACK_MAX_STACK_SIZE]; +using namespace std; + +typedef struct unpack_user { + struct array_stack_type{unsigned int size, last;}; + array_stack_type array_stack[MSGPACK_MAX_STACK_SIZE]; int array_current; + + map str_cache; + + ~unpack_user() { + map::iterator it, itend; + itend = str_cache.end(); + for (it = str_cache.begin(); it != itend; ++it) { + Py_DECREF(it->second); + } + } } unpack_user; @@ -141,9 +157,21 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { - *o = PyString_FromStringAndSize(p, l); - if (l < 16) { // without foundation - PyString_InternInPlace(o); + if (l < 16) { + string s(p, l); + map::iterator it = u->str_cache.find(s); + if (it != u->str_cache.end()) { + *o = it->second; + Py_INCREF(*o); + } + else { + *o = PyString_FromStringAndSize(p, l); + Py_INCREF(*o); + u->str_cache[s] = *o; + } + } + else { + *o = PyString_FromStringAndSize(p, l); } return 0; } From 075081a521c8287759847f1aa846b88fef1f0046 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 14:28:04 +0900 Subject: [PATCH 0165/1648] Fix manifest. --- python/MANIFEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/MANIFEST b/python/MANIFEST index dc042ae..dd0c7d2 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -1,4 +1,4 @@ -msgpack.c +msgpack.cpp setup.py pack.h unpack.h From 07806c92630480337fe9dcf3d79a46368f40bf5c Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 14:28:04 +0900 Subject: [PATCH 0166/1648] Fix manifest. --- MANIFEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST b/MANIFEST index dc042ae..dd0c7d2 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1,4 +1,4 @@ -msgpack.c +msgpack.cpp setup.py pack.h unpack.h From 87f5df1503e70efd2bb9e9565259f1ed37a45933 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 15:59:02 +0900 Subject: [PATCH 0167/1648] Use std::stack. --- python/msgpack/unpack.h | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 694e816..daeb54c 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -18,17 +18,21 @@ #include #include +#include #define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" using namespace std; -typedef struct unpack_user { - struct array_stack_type{unsigned int size, last;}; - array_stack_type array_stack[MSGPACK_MAX_STACK_SIZE]; - int array_current; - +struct array_context { + unsigned int size; + unsigned int last; + stack_item(unsigned int size) : size(size), last(0) + {} +}; +struct unpack_user { + stack array_stack; map str_cache; ~unpack_user() { @@ -38,7 +42,7 @@ typedef struct unpack_user { Py_DECREF(it->second); } } -} unpack_user; +}; #define msgpack_unpack_struct(name) \ @@ -60,7 +64,6 @@ typedef struct template_context template_context; static inline msgpack_unpack_object template_callback_root(unpack_user* u) { - u->array_current = -1; return NULL; } @@ -113,9 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - int cur = ++u->array_current; - u->array_stack[cur].size = n; - u->array_stack[cur].last = 0; + u->array_stack.push(stack_item(n)); *o = PyList_New(n); } else { @@ -126,17 +127,13 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - int cur = u->array_current; - int n = u->array_stack[cur].size; - int last = u->array_stack[cur].last; + unsigned int n = u->array_stack.top().size; + unsigned int &last = u->array_stack.top().last; PyList_SetItem(*c, last, o); last++; if (last >= n) { - u->array_current--; - } - else { - u->array_stack[cur].last = last; + u->array_stack.pop(); } return 0; } From d44108c70977c859a96caa8847ec2514f2313fdd Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 15:59:02 +0900 Subject: [PATCH 0168/1648] Use std::stack. --- msgpack/unpack.h | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 694e816..daeb54c 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -18,17 +18,21 @@ #include #include +#include #define MSGPACK_MAX_STACK_SIZE (1024) #include "msgpack/unpack_define.h" using namespace std; -typedef struct unpack_user { - struct array_stack_type{unsigned int size, last;}; - array_stack_type array_stack[MSGPACK_MAX_STACK_SIZE]; - int array_current; - +struct array_context { + unsigned int size; + unsigned int last; + stack_item(unsigned int size) : size(size), last(0) + {} +}; +struct unpack_user { + stack array_stack; map str_cache; ~unpack_user() { @@ -38,7 +42,7 @@ typedef struct unpack_user { Py_DECREF(it->second); } } -} unpack_user; +}; #define msgpack_unpack_struct(name) \ @@ -60,7 +64,6 @@ typedef struct template_context template_context; static inline msgpack_unpack_object template_callback_root(unpack_user* u) { - u->array_current = -1; return NULL; } @@ -113,9 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - int cur = ++u->array_current; - u->array_stack[cur].size = n; - u->array_stack[cur].last = 0; + u->array_stack.push(stack_item(n)); *o = PyList_New(n); } else { @@ -126,17 +127,13 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - int cur = u->array_current; - int n = u->array_stack[cur].size; - int last = u->array_stack[cur].last; + unsigned int n = u->array_stack.top().size; + unsigned int &last = u->array_stack.top().last; PyList_SetItem(*c, last, o); last++; if (last >= n) { - u->array_current--; - } - else { - u->array_stack[cur].last = last; + u->array_stack.pop(); } return 0; } From 46d7c656214073cde4c458c7cf1eb03e2d33dbd9 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 19:49:02 +0900 Subject: [PATCH 0169/1648] Fix compile error. --- python/msgpack/pack.h | 12 ------------ python/msgpack/unpack.h | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index f3935fb..544950b 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -15,18 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#if _MSC_VER -typedef signed char uint8_t; -typedef unsigned char uint8_t; -typedef short int16_t; -typedef unsigned short uint16_t; -typedef int int32_t; -typedef unsigned int uint32_t; -typedef long long int64_t; -typedef unsigned long long uint64_t; -#else -#include -#endif #include #include diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index daeb54c..b753493 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -28,7 +28,7 @@ using namespace std; struct array_context { unsigned int size; unsigned int last; - stack_item(unsigned int size) : size(size), last(0) + array_context(unsigned int size) : size(size), last(0) {} }; struct unpack_user { @@ -116,7 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(stack_item(n)); + u->array_stack.push(array_context(n)); *o = PyList_New(n); } else { From 8e850dca4ae7481df612e12fd3bf6391f488decf Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 19:49:02 +0900 Subject: [PATCH 0170/1648] Fix compile error. --- msgpack/pack.h | 12 ------------ msgpack/unpack.h | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/msgpack/pack.h b/msgpack/pack.h index f3935fb..544950b 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -15,18 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#if _MSC_VER -typedef signed char uint8_t; -typedef unsigned char uint8_t; -typedef short int16_t; -typedef unsigned short uint16_t; -typedef int int32_t; -typedef unsigned int uint32_t; -typedef long long int64_t; -typedef unsigned long long uint64_t; -#else -#include -#endif #include #include diff --git a/msgpack/unpack.h b/msgpack/unpack.h index daeb54c..b753493 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -28,7 +28,7 @@ using namespace std; struct array_context { unsigned int size; unsigned int last; - stack_item(unsigned int size) : size(size), last(0) + array_context(unsigned int size) : size(size), last(0) {} }; struct unpack_user { @@ -116,7 +116,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(stack_item(n)); + u->array_stack.push(array_context(n)); *o = PyList_New(n); } else { From 7d5f04917ea4986db096ad647b36ce509583ae5c Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 19:55:46 +0900 Subject: [PATCH 0171/1648] Update manifest. --- python/MANIFEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/MANIFEST b/python/MANIFEST index 8b21b4c..6983a33 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -1,7 +1,7 @@ -msgpack.cpp setup.py msgpack/pack.h msgpack/unpack.h +msgpack/_msgpack.pyx include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h From d21e117a9228d09699bc08750fd66f093b8a27a3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 19:55:46 +0900 Subject: [PATCH 0172/1648] Update manifest. --- MANIFEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST b/MANIFEST index 8b21b4c..6983a33 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1,7 +1,7 @@ -msgpack.cpp setup.py msgpack/pack.h msgpack/unpack.h +msgpack/_msgpack.pyx include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h From 5c0a89c258f5f875a2c8c38f8876991372db3260 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 22:50:05 +0900 Subject: [PATCH 0173/1648] Update manifest. --- MANIFEST | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST b/MANIFEST index 6983a33..1fe22ec 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2,6 +2,7 @@ setup.py msgpack/pack.h msgpack/unpack.h msgpack/_msgpack.pyx +msgpack/__init__.py include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h From b8e5b918a3c76da04fc1e5653b94dd99bf0b83c6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 22 Jun 2009 22:50:05 +0900 Subject: [PATCH 0174/1648] Update manifest. --- python/MANIFEST | 1 + 1 file changed, 1 insertion(+) diff --git a/python/MANIFEST b/python/MANIFEST index 6983a33..1fe22ec 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -2,6 +2,7 @@ setup.py msgpack/pack.h msgpack/unpack.h msgpack/_msgpack.pyx +msgpack/__init__.py include/msgpack/pack_define.h include/msgpack/pack_template.h include/msgpack/unpack_define.h From df2acc856915cffbf932376c92fd6bdc58110a5b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:13:22 +0900 Subject: [PATCH 0175/1648] Fix setup script bug. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index eb897f2..1cbc24e 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ setup(name='msgpack', version=version, cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], + packages=['msgpack'], description=desc, long_description=long_desc, ) From 99d0a41ec6a6169456dcfe64cd6972520a768b71 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:13:22 +0900 Subject: [PATCH 0176/1648] Fix setup script bug. --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index eb897f2..1cbc24e 100644 --- a/python/setup.py +++ b/python/setup.py @@ -32,6 +32,7 @@ setup(name='msgpack', version=version, cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], + packages=['msgpack'], description=desc, long_description=long_desc, ) From f2d07e5d6906567ce06d87dd21085828b3ebc1cc Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:13:39 +0900 Subject: [PATCH 0177/1648] Remove unneccessary value. --- msgpack/unpack.h | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index b753493..12702d8 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -25,14 +25,8 @@ using namespace std; -struct array_context { - unsigned int size; - unsigned int last; - array_context(unsigned int size) : size(size), last(0) - {} -}; struct unpack_user { - stack array_stack; + stack array_stack; map str_cache; ~unpack_user() { @@ -116,7 +110,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(array_context(n)); + u->array_stack.push(0); *o = PyList_New(n); } else { @@ -127,12 +121,12 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - unsigned int n = u->array_stack.top().size; - unsigned int &last = u->array_stack.top().last; - - PyList_SetItem(*c, last, o); + unsigned int &last = u->array_stack.top(); + PyList_SET_ITEM(*c, last, o); last++; - if (last >= n) { + + Py_ssize_t len = PyList_GET_SIZE(*c); + if (last >= len) { u->array_stack.pop(); } return 0; From dd53b141ef6d2495d0cc14c8949180a74ae3caf6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:13:39 +0900 Subject: [PATCH 0178/1648] Remove unneccessary value. --- python/msgpack/unpack.h | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index b753493..12702d8 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -25,14 +25,8 @@ using namespace std; -struct array_context { - unsigned int size; - unsigned int last; - array_context(unsigned int size) : size(size), last(0) - {} -}; struct unpack_user { - stack array_stack; + stack array_stack; map str_cache; ~unpack_user() { @@ -116,7 +110,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { if (n > 0) { - u->array_stack.push(array_context(n)); + u->array_stack.push(0); *o = PyList_New(n); } else { @@ -127,12 +121,12 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) { - unsigned int n = u->array_stack.top().size; - unsigned int &last = u->array_stack.top().last; - - PyList_SetItem(*c, last, o); + unsigned int &last = u->array_stack.top(); + PyList_SET_ITEM(*c, last, o); last++; - if (last >= n) { + + Py_ssize_t len = PyList_GET_SIZE(*c); + if (last >= len) { u->array_stack.pop(); } return 0; From d32b48f98f09be8c829a8d48f8d7fcdb4bd0c61e Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:38:48 +0900 Subject: [PATCH 0179/1648] Remove duplicated values. --- include | 1 - msgpack/pack.h | 4 +- msgpack/pack_define.h | 26 ++ msgpack/pack_template.h | 741 ++++++++++++++++++++++++++++++++++++++ msgpack/unpack.h | 53 +-- msgpack/unpack_define.h | 129 +++++++ msgpack/unpack_template.h | 363 +++++++++++++++++++ 7 files changed, 1276 insertions(+), 41 deletions(-) delete mode 120000 include create mode 100644 msgpack/pack_define.h create mode 100644 msgpack/pack_template.h create mode 100644 msgpack/unpack_define.h create mode 100644 msgpack/unpack_template.h diff --git a/include b/include deleted file mode 120000 index a96aa0e..0000000 --- a/include +++ /dev/null @@ -1 +0,0 @@ -.. \ No newline at end of file diff --git a/msgpack/pack.h b/msgpack/pack.h index 544950b..9bd6b68 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -18,7 +18,7 @@ #include #include -#include "msgpack/pack_define.h" +#include "pack_define.h" #ifdef __cplusplus extern "C" { @@ -82,7 +82,7 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ #define msgpack_pack_append_buffer(user, buf, len) \ return (*(user)->callback)((user)->data, (const char*)buf, len) -#include "msgpack/pack_template.h" +#include "pack_template.h" static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) { diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h new file mode 100644 index 0000000..33408e5 --- /dev/null +++ b/msgpack/pack_define.h @@ -0,0 +1,26 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_DEFINE_H__ +#define MSGPACK_PACK_DEFINE_H__ + +#include +#include +#include + +#endif /* msgpack/pack_define.h */ + diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h new file mode 100644 index 0000000..aa620f5 --- /dev/null +++ b/msgpack/pack_template.h @@ -0,0 +1,741 @@ +/* + * MessagePack packing routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + + +#ifdef __LITTLE_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#elif __BIG_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[1] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[3] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[7] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#endif + +#ifndef msgpack_pack_inline_func +#error msgpack_pack_inline_func template is not defined +#endif + +#ifndef msgpack_pack_user +#error msgpack_pack_user type is not defined +#endif + +#ifndef msgpack_pack_append_buffer +#error msgpack_pack_append_buffer callback is not defined +#endif + + +/* + * Integer + */ + +#define msgpack_pack_real_uint8(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ +} while(0) + +#define msgpack_pack_real_uint16(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ +} while(0) + +#define msgpack_pack_real_uint32(x, d) \ +do { \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_uint64(x, d) \ +do { \ + if(d < (1ULL<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int8(x, d) \ +do { \ + if(d < -(1<<5)) { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } \ +} while(0) + +#define msgpack_pack_real_int16(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int32(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<15)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int64(x, d) \ +do { \ + if(d < -(1LL<<5)) { \ + if(d < -(1LL<<15)) { \ + if(d < -(1LL<<31)) { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } else { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } else { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + if(d < (1LL<<16)) { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } else { \ + if(d < (1LL<<32)) { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* unsigned 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ + } \ +} while(0) + + +#ifdef msgpack_pack_inline_func_fastint + +msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) +{ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) +{ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) +{ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) +{ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) +{ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) +{ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) +{ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) +{ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +#undef msgpack_pack_inline_func_fastint +#endif + + +msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d) +{ + msgpack_pack_real_uint8(x, d); +} + +msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d) +{ + msgpack_pack_real_uint16(x, d); +} + +msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d) +{ + msgpack_pack_real_uint32(x, d); +} + +msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d) +{ + msgpack_pack_real_uint64(x, d); +} + +msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d) +{ + msgpack_pack_real_int8(x, d); +} + +msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d) +{ + msgpack_pack_real_int16(x, d); +} + +msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d) +{ + msgpack_pack_real_int32(x, d); +} + +msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) +{ + msgpack_pack_real_int64(x, d); +} + + +#ifdef msgpack_pack_inline_func_cint + +msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) +{ +#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) +#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(short) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(short) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) +{ +#if defined(SIZEOF_INT) || defined(INT_MAX) +#if SIZEOF_INT == 2 || INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(int) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(int) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) +{ +#if defined(SIZEOF_LONG) || defined(LONG_MAX) +#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) +{ +#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) +#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned short) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned short) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) +{ +#if defined(SIZEOF_INT) || defined(UINT_MAX) +#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) +{ +#if defined(SIZEOF_LONG) || defined(ULONG_MAX) +#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned long long) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned long long) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +#undef msgpack_pack_inline_func_cint +#endif + + + +/* + * Float + */ + +msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) +{ + union { char buf[4]; uint32_t num; } f; + *((float*)&f.buf) = d; // FIXME + const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) +{ + union { char buf[8]; uint64_t num; } f; + *((double*)&f.buf) = d; // FIXME + const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + msgpack_pack_append_buffer(x, buf, 9); +} + + +/* + * Nil + */ + +msgpack_pack_inline_func(_nil)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc0; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Boolean + */ + +msgpack_pack_inline_func(_true)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc3; + msgpack_pack_append_buffer(x, &d, 1); +} + +msgpack_pack_inline_func(_false)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc2; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Array + */ + +msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x90 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Map + */ + +msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x80 | n; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Raw + */ + +msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) +{ + if(l < 32) { + unsigned char d = 0xa0 | l; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(l < 65536) { + uint16_t d = (uint16_t)l; + unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)l; + unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + +msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l) +{ + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); +} + +#undef msgpack_pack_inline_func +#undef msgpack_pack_user +#undef msgpack_pack_append_buffer + +#undef STORE8_BE8 + +#undef STORE16_BE8 +#undef STORE16_BE16 + +#undef STORE32_BE8 +#undef STORE32_BE16 +#undef STORE32_BE32 + +#undef STORE64_BE8 +#undef STORE64_BE16 +#undef STORE64_BE32 +#undef STORE64_BE64 + +#undef msgpack_pack_real_uint8 +#undef msgpack_pack_real_uint16 +#undef msgpack_pack_real_uint32 +#undef msgpack_pack_real_uint64 +#undef msgpack_pack_real_int8 +#undef msgpack_pack_real_int16 +#undef msgpack_pack_real_int32 +#undef msgpack_pack_real_int64 + diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 12702d8..cc48d9c 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -21,18 +21,18 @@ #include #define MSGPACK_MAX_STACK_SIZE (1024) -#include "msgpack/unpack_define.h" +#include "unpack_define.h" using namespace std; +typedef map str_cach_t; struct unpack_user { - stack array_stack; - map str_cache; + str_cach_t strcache; ~unpack_user() { - map::iterator it, itend; - itend = str_cache.end(); - for (it = str_cache.begin(); it != itend; ++it) { + str_cach_t::iterator it, itend; + itend = strcache.end(); + for (it = strcache.begin(); it != itend; ++it) { Py_DECREF(it->second); } } @@ -108,35 +108,13 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - if (n > 0) { - u->array_stack.push(0); - *o = PyList_New(n); - } - else { - *o = PyList_New(0); - } - return 0; -} +{ *o = PyList_New(n); return 0; } -static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) -{ - unsigned int &last = u->array_stack.top(); - PyList_SET_ITEM(*c, last, o); - last++; - - Py_ssize_t len = PyList_GET_SIZE(*c); - if (last >= len) { - u->array_stack.pop(); - } - return 0; -} +static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) +{ PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - *o = PyDict_New(); - return 0; -} +{ *o = PyDict_New(); return 0; } static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { @@ -150,16 +128,15 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha { if (l < 16) { string s(p, l); - map::iterator it = u->str_cache.find(s); - if (it != u->str_cache.end()) { + str_cach_t ::iterator it = u->strcache.find(s); + if (it != u->strcache.end()) { *o = it->second; - Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); - Py_INCREF(*o); - u->str_cache[s] = *o; + u->strcache[s] = *o; } + Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); @@ -167,4 +144,4 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha return 0; } -#include "msgpack/unpack_template.h" +#include "unpack_template.h" diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h new file mode 100644 index 0000000..63668c2 --- /dev/null +++ b/msgpack/unpack_define.h @@ -0,0 +1,129 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_DEFINE_H__ +#define MSGPACK_UNPACK_DEFINE_H__ + +#include +#include +#include +#include +#include +#ifndef __WIN32__ +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_MAX_STACK_SIZE +#define MSGPACK_MAX_STACK_SIZE 16 +#endif + + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#define msgpack_betoh16(x) ntohs(x) +#define msgpack_betoh32(x) ntohl(x) + +#ifdef __LITTLE_ENDIAN__ +#if defined(__bswap_64) +# define msgpack_betoh64(x) __bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) +#else +static inline uint64_t msgpack_betoh64(uint64_t x) { + return ((x << 56) & 0xff00000000000000ULL ) | + ((x << 40) & 0x00ff000000000000ULL ) | + ((x << 24) & 0x0000ff0000000000ULL ) | + ((x << 8) & 0x000000ff00000000ULL ) | + ((x >> 8) & 0x00000000ff000000ULL ) | + ((x >> 24) & 0x0000000000ff0000ULL ) | + ((x >> 40) & 0x000000000000ff00ULL ) | + ((x >> 56) & 0x00000000000000ffULL ) ; +} +#endif +#else +#define msgpack_betoh64(x) (x) +#endif + + +typedef enum { + CS_HEADER = 0x00, // nil + + //CS_ = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true + + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, + + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UINT_8 = 0x0c, + CS_UINT_16 = 0x0d, + CS_UINT_32 = 0x0e, + CS_UINT_64 = 0x0f, + CS_INT_8 = 0x10, + CS_INT_16 = 0x11, + CS_INT_32 = 0x12, + CS_INT_64 = 0x13, + + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, + + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, +} msgpack_unpack_state; + + +typedef enum { + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, +} msgpack_container_type; + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/unpack_define.h */ + diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h new file mode 100644 index 0000000..db33368 --- /dev/null +++ b/msgpack/unpack_template.h @@ -0,0 +1,363 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef msgpack_unpack_func +#error msgpack_unpack_func template is not defined +#endif + +#ifndef msgpack_unpack_callback +#error msgpack_unpack_callback template is not defined +#endif + +#ifndef msgpack_unpack_struct +#error msgpack_unpack_struct template is not defined +#endif + +#ifndef msgpack_unpack_struct_decl +#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name) +#endif + +#ifndef msgpack_unpack_object +#error msgpack_unpack_object type is not defined +#endif + +#ifndef msgpack_unpack_user +#error msgpack_unpack_user type is not defined +#endif + + +msgpack_unpack_struct_decl(_stack) { + msgpack_unpack_object obj; + size_t curr; + size_t count; + unsigned int ct; + msgpack_unpack_object map_key; +}; + +msgpack_unpack_struct_decl(_context) { + msgpack_unpack_user user; + unsigned int cs; + unsigned int trail; + unsigned int top; + msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; +}; + + +msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) +{ + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); +} + +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) +{ + return (ctx)->stack[0].obj; +} + + +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; + + unsigned int trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + msgpack_unpack_struct(_stack)* stack = ctx->stack; + msgpack_unpack_user* user = &ctx->user; + + msgpack_unpack_object obj; + msgpack_unpack_struct(_stack)* c = NULL; + + int ret; + +#define push_simple_value(func) \ + if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ + goto _push +#define push_fixed_value(func, arg) \ + if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ + goto _push +#define push_variable_value(func, base, pos, len) \ + if(msgpack_unpack_callback(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ + goto _push + +#define again_fixed_trail(_cs, trail_len) \ + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again +#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again + +#define start_container(func, count_, ct_) \ + if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ + if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ + stack[top].ct = ct_; \ + stack[top].curr = 0; \ + stack[top].count = count_; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + ++top; \ + goto _header_again + +#define NEXT_CS(p) \ + ((unsigned int)*p & 0x1f) + +#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) + + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + switch(*p) { + case 0x00 ... 0x7f: // Positive Fixnum + push_fixed_value(_uint8, *(uint8_t*)p); + case 0xe0 ... 0xff: // Negative Fixnum + push_fixed_value(_int8, *(int8_t*)p); + case 0xc0 ... 0xdf: // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(_nil); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(_false); + case 0xc3: // true + push_simple_value(_true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + case 0xa0 ... 0xbf: // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + case 0x90 ... 0x9f: // FixArray + start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + case 0x80 ... 0x8f: // FixMap + start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + + default: + goto _failed; + } + // end CS_HEADER + + + _fixed_trail_again: + ++p; + + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + union { uint32_t num; char buf[4]; } f; + f.num = PTR_CAST_32(n); // FIXME + push_fixed_value(_float, *((float*)f.buf)); } + case CS_DOUBLE: { + union { uint64_t num; char buf[8]; } f; + f.num = PTR_CAST_64(n); // FIXME + push_fixed_value(_double, *((double*)f.buf)); } + case CS_UINT_8: + push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); + case CS_UINT_16: + push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); + case CS_UINT_32: + push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); + case CS_UINT_64: + push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); + + case CS_INT_8: + push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); + case CS_INT_16: + push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); + case CS_INT_32: + push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); + case CS_INT_64: + push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(_big_int, data, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(_big_float, data, n, trail); + + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(_raw, data, n, trail); + + case CS_ARRAY_16: + start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + /* FIXME security guard */ + start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + + case CS_MAP_16: + start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + case CS_MAP_32: + /* FIXME security guard */ + start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + + default: + goto _failed; + } + } + +_push: + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; } + if(++c->curr == c->count) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } + +_header_again: + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; + + +_finish: + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; + +_failed: + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; + +_out: + ret = 0; + goto _end; + +_end: + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; + + return ret; +} + + +#undef msgpack_unpack_func +#undef msgpack_unpack_callback +#undef msgpack_unpack_struct +#undef msgpack_unpack_object +#undef msgpack_unpack_user + +#undef push_simple_value +#undef push_fixed_value +#undef push_variable_value +#undef again_fixed_trail +#undef again_fixed_trail_if_zero +#undef start_container + +#undef NEXT_CS +#undef PTR_CAST_8 +#undef PTR_CAST_16 +#undef PTR_CAST_32 +#undef PTR_CAST_64 + From 3fd28d07928c51cb563f35d9dddb0c7867ac9875 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:38:48 +0900 Subject: [PATCH 0180/1648] Remove duplicated values. --- python/include | 1 - python/msgpack/pack.h | 4 +- python/msgpack/pack_define.h | 26 ++ python/msgpack/pack_template.h | 741 +++++++++++++++++++++++++++++++ python/msgpack/unpack.h | 53 +-- python/msgpack/unpack_define.h | 129 ++++++ python/msgpack/unpack_template.h | 363 +++++++++++++++ 7 files changed, 1276 insertions(+), 41 deletions(-) delete mode 120000 python/include create mode 100644 python/msgpack/pack_define.h create mode 100644 python/msgpack/pack_template.h create mode 100644 python/msgpack/unpack_define.h create mode 100644 python/msgpack/unpack_template.h diff --git a/python/include b/python/include deleted file mode 120000 index a96aa0e..0000000 --- a/python/include +++ /dev/null @@ -1 +0,0 @@ -.. \ No newline at end of file diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index 544950b..9bd6b68 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -18,7 +18,7 @@ #include #include -#include "msgpack/pack_define.h" +#include "pack_define.h" #ifdef __cplusplus extern "C" { @@ -82,7 +82,7 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ #define msgpack_pack_append_buffer(user, buf, len) \ return (*(user)->callback)((user)->data, (const char*)buf, len) -#include "msgpack/pack_template.h" +#include "pack_template.h" static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) { diff --git a/python/msgpack/pack_define.h b/python/msgpack/pack_define.h new file mode 100644 index 0000000..33408e5 --- /dev/null +++ b/python/msgpack/pack_define.h @@ -0,0 +1,26 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_DEFINE_H__ +#define MSGPACK_PACK_DEFINE_H__ + +#include +#include +#include + +#endif /* msgpack/pack_define.h */ + diff --git a/python/msgpack/pack_template.h b/python/msgpack/pack_template.h new file mode 100644 index 0000000..aa620f5 --- /dev/null +++ b/python/msgpack/pack_template.h @@ -0,0 +1,741 @@ +/* + * MessagePack packing routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + + +#ifdef __LITTLE_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#elif __BIG_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[1] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[3] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[7] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#endif + +#ifndef msgpack_pack_inline_func +#error msgpack_pack_inline_func template is not defined +#endif + +#ifndef msgpack_pack_user +#error msgpack_pack_user type is not defined +#endif + +#ifndef msgpack_pack_append_buffer +#error msgpack_pack_append_buffer callback is not defined +#endif + + +/* + * Integer + */ + +#define msgpack_pack_real_uint8(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ +} while(0) + +#define msgpack_pack_real_uint16(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ +} while(0) + +#define msgpack_pack_real_uint32(x, d) \ +do { \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_uint64(x, d) \ +do { \ + if(d < (1ULL<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int8(x, d) \ +do { \ + if(d < -(1<<5)) { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } \ +} while(0) + +#define msgpack_pack_real_int16(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int32(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<15)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int64(x, d) \ +do { \ + if(d < -(1LL<<5)) { \ + if(d < -(1LL<<15)) { \ + if(d < -(1LL<<31)) { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } else { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } else { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + if(d < (1LL<<16)) { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } else { \ + if(d < (1LL<<32)) { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* unsigned 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ + } \ +} while(0) + + +#ifdef msgpack_pack_inline_func_fastint + +msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) +{ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) +{ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) +{ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) +{ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) +{ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) +{ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) +{ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) +{ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +#undef msgpack_pack_inline_func_fastint +#endif + + +msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d) +{ + msgpack_pack_real_uint8(x, d); +} + +msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d) +{ + msgpack_pack_real_uint16(x, d); +} + +msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d) +{ + msgpack_pack_real_uint32(x, d); +} + +msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d) +{ + msgpack_pack_real_uint64(x, d); +} + +msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d) +{ + msgpack_pack_real_int8(x, d); +} + +msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d) +{ + msgpack_pack_real_int16(x, d); +} + +msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d) +{ + msgpack_pack_real_int32(x, d); +} + +msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) +{ + msgpack_pack_real_int64(x, d); +} + + +#ifdef msgpack_pack_inline_func_cint + +msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) +{ +#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) +#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(short) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(short) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) +{ +#if defined(SIZEOF_INT) || defined(INT_MAX) +#if SIZEOF_INT == 2 || INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(int) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(int) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) +{ +#if defined(SIZEOF_LONG) || defined(LONG_MAX) +#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) +{ +#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) +#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned short) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned short) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) +{ +#if defined(SIZEOF_INT) || defined(UINT_MAX) +#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) +{ +#if defined(SIZEOF_LONG) || defined(ULONG_MAX) +#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned long long) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned long long) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +#undef msgpack_pack_inline_func_cint +#endif + + + +/* + * Float + */ + +msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) +{ + union { char buf[4]; uint32_t num; } f; + *((float*)&f.buf) = d; // FIXME + const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) +{ + union { char buf[8]; uint64_t num; } f; + *((double*)&f.buf) = d; // FIXME + const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + msgpack_pack_append_buffer(x, buf, 9); +} + + +/* + * Nil + */ + +msgpack_pack_inline_func(_nil)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc0; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Boolean + */ + +msgpack_pack_inline_func(_true)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc3; + msgpack_pack_append_buffer(x, &d, 1); +} + +msgpack_pack_inline_func(_false)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc2; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Array + */ + +msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x90 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Map + */ + +msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x80 | n; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Raw + */ + +msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) +{ + if(l < 32) { + unsigned char d = 0xa0 | l; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(l < 65536) { + uint16_t d = (uint16_t)l; + unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)l; + unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + +msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l) +{ + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); +} + +#undef msgpack_pack_inline_func +#undef msgpack_pack_user +#undef msgpack_pack_append_buffer + +#undef STORE8_BE8 + +#undef STORE16_BE8 +#undef STORE16_BE16 + +#undef STORE32_BE8 +#undef STORE32_BE16 +#undef STORE32_BE32 + +#undef STORE64_BE8 +#undef STORE64_BE16 +#undef STORE64_BE32 +#undef STORE64_BE64 + +#undef msgpack_pack_real_uint8 +#undef msgpack_pack_real_uint16 +#undef msgpack_pack_real_uint32 +#undef msgpack_pack_real_uint64 +#undef msgpack_pack_real_int8 +#undef msgpack_pack_real_int16 +#undef msgpack_pack_real_int32 +#undef msgpack_pack_real_int64 + diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 12702d8..cc48d9c 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -21,18 +21,18 @@ #include #define MSGPACK_MAX_STACK_SIZE (1024) -#include "msgpack/unpack_define.h" +#include "unpack_define.h" using namespace std; +typedef map str_cach_t; struct unpack_user { - stack array_stack; - map str_cache; + str_cach_t strcache; ~unpack_user() { - map::iterator it, itend; - itend = str_cache.end(); - for (it = str_cache.begin(); it != itend; ++it) { + str_cach_t::iterator it, itend; + itend = strcache.end(); + for (it = strcache.begin(); it != itend; ++it) { Py_DECREF(it->second); } } @@ -108,35 +108,13 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - if (n > 0) { - u->array_stack.push(0); - *o = PyList_New(n); - } - else { - *o = PyList_New(0); - } - return 0; -} +{ *o = PyList_New(n); return 0; } -static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) -{ - unsigned int &last = u->array_stack.top(); - PyList_SET_ITEM(*c, last, o); - last++; - - Py_ssize_t len = PyList_GET_SIZE(*c); - if (last >= len) { - u->array_stack.pop(); - } - return 0; -} +static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) +{ PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ - *o = PyDict_New(); - return 0; -} +{ *o = PyDict_New(); return 0; } static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { @@ -150,16 +128,15 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha { if (l < 16) { string s(p, l); - map::iterator it = u->str_cache.find(s); - if (it != u->str_cache.end()) { + str_cach_t ::iterator it = u->strcache.find(s); + if (it != u->strcache.end()) { *o = it->second; - Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); - Py_INCREF(*o); - u->str_cache[s] = *o; + u->strcache[s] = *o; } + Py_INCREF(*o); } else { *o = PyString_FromStringAndSize(p, l); @@ -167,4 +144,4 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha return 0; } -#include "msgpack/unpack_template.h" +#include "unpack_template.h" diff --git a/python/msgpack/unpack_define.h b/python/msgpack/unpack_define.h new file mode 100644 index 0000000..63668c2 --- /dev/null +++ b/python/msgpack/unpack_define.h @@ -0,0 +1,129 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_DEFINE_H__ +#define MSGPACK_UNPACK_DEFINE_H__ + +#include +#include +#include +#include +#include +#ifndef __WIN32__ +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_MAX_STACK_SIZE +#define MSGPACK_MAX_STACK_SIZE 16 +#endif + + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#define msgpack_betoh16(x) ntohs(x) +#define msgpack_betoh32(x) ntohl(x) + +#ifdef __LITTLE_ENDIAN__ +#if defined(__bswap_64) +# define msgpack_betoh64(x) __bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) +#else +static inline uint64_t msgpack_betoh64(uint64_t x) { + return ((x << 56) & 0xff00000000000000ULL ) | + ((x << 40) & 0x00ff000000000000ULL ) | + ((x << 24) & 0x0000ff0000000000ULL ) | + ((x << 8) & 0x000000ff00000000ULL ) | + ((x >> 8) & 0x00000000ff000000ULL ) | + ((x >> 24) & 0x0000000000ff0000ULL ) | + ((x >> 40) & 0x000000000000ff00ULL ) | + ((x >> 56) & 0x00000000000000ffULL ) ; +} +#endif +#else +#define msgpack_betoh64(x) (x) +#endif + + +typedef enum { + CS_HEADER = 0x00, // nil + + //CS_ = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true + + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, + + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UINT_8 = 0x0c, + CS_UINT_16 = 0x0d, + CS_UINT_32 = 0x0e, + CS_UINT_64 = 0x0f, + CS_INT_8 = 0x10, + CS_INT_16 = 0x11, + CS_INT_32 = 0x12, + CS_INT_64 = 0x13, + + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, + + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, +} msgpack_unpack_state; + + +typedef enum { + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, +} msgpack_container_type; + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/unpack_define.h */ + diff --git a/python/msgpack/unpack_template.h b/python/msgpack/unpack_template.h new file mode 100644 index 0000000..db33368 --- /dev/null +++ b/python/msgpack/unpack_template.h @@ -0,0 +1,363 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef msgpack_unpack_func +#error msgpack_unpack_func template is not defined +#endif + +#ifndef msgpack_unpack_callback +#error msgpack_unpack_callback template is not defined +#endif + +#ifndef msgpack_unpack_struct +#error msgpack_unpack_struct template is not defined +#endif + +#ifndef msgpack_unpack_struct_decl +#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name) +#endif + +#ifndef msgpack_unpack_object +#error msgpack_unpack_object type is not defined +#endif + +#ifndef msgpack_unpack_user +#error msgpack_unpack_user type is not defined +#endif + + +msgpack_unpack_struct_decl(_stack) { + msgpack_unpack_object obj; + size_t curr; + size_t count; + unsigned int ct; + msgpack_unpack_object map_key; +}; + +msgpack_unpack_struct_decl(_context) { + msgpack_unpack_user user; + unsigned int cs; + unsigned int trail; + unsigned int top; + msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; +}; + + +msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) +{ + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); +} + +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) +{ + return (ctx)->stack[0].obj; +} + + +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; + + unsigned int trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + msgpack_unpack_struct(_stack)* stack = ctx->stack; + msgpack_unpack_user* user = &ctx->user; + + msgpack_unpack_object obj; + msgpack_unpack_struct(_stack)* c = NULL; + + int ret; + +#define push_simple_value(func) \ + if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ + goto _push +#define push_fixed_value(func, arg) \ + if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ + goto _push +#define push_variable_value(func, base, pos, len) \ + if(msgpack_unpack_callback(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ + goto _push + +#define again_fixed_trail(_cs, trail_len) \ + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again +#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again + +#define start_container(func, count_, ct_) \ + if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ + if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ + stack[top].ct = ct_; \ + stack[top].curr = 0; \ + stack[top].count = count_; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + ++top; \ + goto _header_again + +#define NEXT_CS(p) \ + ((unsigned int)*p & 0x1f) + +#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) + + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + switch(*p) { + case 0x00 ... 0x7f: // Positive Fixnum + push_fixed_value(_uint8, *(uint8_t*)p); + case 0xe0 ... 0xff: // Negative Fixnum + push_fixed_value(_int8, *(int8_t*)p); + case 0xc0 ... 0xdf: // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(_nil); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(_false); + case 0xc3: // true + push_simple_value(_true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + case 0xa0 ... 0xbf: // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + case 0x90 ... 0x9f: // FixArray + start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + case 0x80 ... 0x8f: // FixMap + start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + + default: + goto _failed; + } + // end CS_HEADER + + + _fixed_trail_again: + ++p; + + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + union { uint32_t num; char buf[4]; } f; + f.num = PTR_CAST_32(n); // FIXME + push_fixed_value(_float, *((float*)f.buf)); } + case CS_DOUBLE: { + union { uint64_t num; char buf[8]; } f; + f.num = PTR_CAST_64(n); // FIXME + push_fixed_value(_double, *((double*)f.buf)); } + case CS_UINT_8: + push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); + case CS_UINT_16: + push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); + case CS_UINT_32: + push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); + case CS_UINT_64: + push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); + + case CS_INT_8: + push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); + case CS_INT_16: + push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); + case CS_INT_32: + push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); + case CS_INT_64: + push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(_big_int, data, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(_big_float, data, n, trail); + + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(_raw, data, n, trail); + + case CS_ARRAY_16: + start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + /* FIXME security guard */ + start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + + case CS_MAP_16: + start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + case CS_MAP_32: + /* FIXME security guard */ + start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + + default: + goto _failed; + } + } + +_push: + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; } + if(++c->curr == c->count) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } + +_header_again: + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; + + +_finish: + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; + +_failed: + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; + +_out: + ret = 0; + goto _end; + +_end: + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; + + return ret; +} + + +#undef msgpack_unpack_func +#undef msgpack_unpack_callback +#undef msgpack_unpack_struct +#undef msgpack_unpack_object +#undef msgpack_unpack_user + +#undef push_simple_value +#undef push_fixed_value +#undef push_variable_value +#undef again_fixed_trail +#undef again_fixed_trail_if_zero +#undef start_container + +#undef NEXT_CS +#undef PTR_CAST_8 +#undef PTR_CAST_16 +#undef PTR_CAST_32 +#undef PTR_CAST_64 + From de6afa5285c2f748d0c0c72b930fce77f68fcb85 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:54:47 +0900 Subject: [PATCH 0181/1648] Reduce memory footprint. --- msgpack/unpack.h | 6 ++---- msgpack/unpack_template.h | 7 +++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index cc48d9c..12afb99 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -23,9 +23,7 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -using namespace std; - -typedef map str_cach_t; +typedef std::map str_cach_t; struct unpack_user { str_cach_t strcache; @@ -127,7 +125,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { if (l < 16) { - string s(p, l); + std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index db33368..c960c3a 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -43,10 +43,13 @@ msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; - size_t curr; size_t count; unsigned int ct; - msgpack_unpack_object map_key; + + union { + size_t curr; + msgpack_unpack_object map_key; + }; }; msgpack_unpack_struct_decl(_context) { From f61b282886bb9d7f7932fbcbe2affee06d150a5f Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 01:54:47 +0900 Subject: [PATCH 0182/1648] Reduce memory footprint. --- python/msgpack/unpack.h | 6 ++---- python/msgpack/unpack_template.h | 7 +++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index cc48d9c..12afb99 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -23,9 +23,7 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -using namespace std; - -typedef map str_cach_t; +typedef std::map str_cach_t; struct unpack_user { str_cach_t strcache; @@ -127,7 +125,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { if (l < 16) { - string s(p, l); + std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; diff --git a/python/msgpack/unpack_template.h b/python/msgpack/unpack_template.h index db33368..c960c3a 100644 --- a/python/msgpack/unpack_template.h +++ b/python/msgpack/unpack_template.h @@ -43,10 +43,13 @@ msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; - size_t curr; size_t count; unsigned int ct; - msgpack_unpack_object map_key; + + union { + size_t curr; + msgpack_unpack_object map_key; + }; }; msgpack_unpack_struct_decl(_context) { From 318b4b63ec295484e9fc8029a17b50f5181681ce Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 04:25:05 +0900 Subject: [PATCH 0183/1648] Check return value of c-api. --- msgpack/unpack.h | 124 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 95 insertions(+), 29 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 12afb99..e30b72a 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -59,42 +59,84 @@ static inline msgpack_unpack_object template_callback_root(unpack_user* u) return NULL; } -static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } - static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } +{ + PyObject *p = PyInt_FromLong((long)d); + if (!p) + return -1; + *o = p; + return 0; +} +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ + return template_callback_uint16(u, d, o); +} + static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { + PyObject *p; if (d > LONG_MAX) { - *o = PyLong_FromUnsignedLong((unsigned long)d); + p = PyLong_FromUnsignedLong((unsigned long)d); } else { - *o = PyInt_FromLong((long)d); + p = PyInt_FromLong((long)d); } + if (!p) + return -1; + *o = p; return 0; } static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromUnsignedLongLong(d); return 0; } - -static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } - -static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +{ + PyObject *p = PyLong_FromUnsignedLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +{ + PyObject *p = PyInt_FromLong(d); + if (!p) + return -1; + *o = p; + return 0; +} + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ + return template_callback_int32(u, d, o); +} + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ + return template_callback_int32(u, d, o); +} static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromLongLong(d); return 0; } - -static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble((double)d); return 0; } +{ + PyObject *p = PyLong_FromLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble(d); return 0; } +{ + PyObject *p = PyFloat_FromDouble(d); + if (!p) + return -1; + *o = p; + return 0; +} + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ + return template_callback_double(u, d, o); +} static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) { Py_INCREF(Py_None); *o = Py_None; return 0; } @@ -106,40 +148,64 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyList_New(n); return 0; } +{ + PyObject *p = PyList_New(n); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyDict_New(); return 0; } +{ + PyObject *p = PyDict_New(); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { - PyDict_SetItem(*c, k, v); - Py_DECREF(k); - Py_DECREF(v); - return 0; + if (PyDict_SetItem(*c, k, v) == 0) { + Py_DECREF(k); + Py_DECREF(v); + return 0; + } + return -1; } static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { + PyObject *py; if (l < 16) { std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; + Py_INCREF(*o); + return 0; } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + Py_INCREF(*o); u->strcache[s] = *o; + return 0; } - Py_INCREF(*o); } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } - return 0; } #include "unpack_template.h" From 5d4189306ae0b6ea23e9410d34434b41aee384b3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 04:25:05 +0900 Subject: [PATCH 0184/1648] Check return value of c-api. --- python/msgpack/unpack.h | 124 ++++++++++++++++++++++++++++++---------- 1 file changed, 95 insertions(+), 29 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 12afb99..e30b72a 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -59,42 +59,84 @@ static inline msgpack_unpack_object template_callback_root(unpack_user* u) return NULL; } -static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } - static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong((long)d); return 0; } +{ + PyObject *p = PyInt_FromLong((long)d); + if (!p) + return -1; + *o = p; + return 0; +} +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ + return template_callback_uint16(u, d, o); +} + static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { + PyObject *p; if (d > LONG_MAX) { - *o = PyLong_FromUnsignedLong((unsigned long)d); + p = PyLong_FromUnsignedLong((unsigned long)d); } else { - *o = PyInt_FromLong((long)d); + p = PyInt_FromLong((long)d); } + if (!p) + return -1; + *o = p; return 0; } static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromUnsignedLongLong(d); return 0; } - -static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } - -static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +{ + PyObject *p = PyLong_FromUnsignedLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) -{ *o = PyInt_FromLong(d); return 0; } +{ + PyObject *p = PyInt_FromLong(d); + if (!p) + return -1; + *o = p; + return 0; +} + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ + return template_callback_int32(u, d, o); +} + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ + return template_callback_int32(u, d, o); +} static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) -{ *o = PyLong_FromLongLong(d); return 0; } - -static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble((double)d); return 0; } +{ + PyObject *p = PyLong_FromLongLong(d); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) -{ *o = PyFloat_FromDouble(d); return 0; } +{ + PyObject *p = PyFloat_FromDouble(d); + if (!p) + return -1; + *o = p; + return 0; +} + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ + return template_callback_double(u, d, o); +} static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) { Py_INCREF(Py_None); *o = Py_None; return 0; } @@ -106,40 +148,64 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* { Py_INCREF(Py_False); *o = Py_False; return 0; } static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyList_New(n); return 0; } +{ + PyObject *p = PyList_New(n); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { PyList_SET_ITEM(*c, current, o); return 0; } static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) -{ *o = PyDict_New(); return 0; } +{ + PyObject *p = PyDict_New(); + if (!p) + return -1; + *o = p; + return 0; +} static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) { - PyDict_SetItem(*c, k, v); - Py_DECREF(k); - Py_DECREF(v); - return 0; + if (PyDict_SetItem(*c, k, v) == 0) { + Py_DECREF(k); + Py_DECREF(v); + return 0; + } + return -1; } static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { + PyObject *py; if (l < 16) { std::string s(p, l); str_cach_t ::iterator it = u->strcache.find(s); if (it != u->strcache.end()) { *o = it->second; + Py_INCREF(*o); + return 0; } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + Py_INCREF(*o); u->strcache[s] = *o; + return 0; } - Py_INCREF(*o); } else { - *o = PyString_FromStringAndSize(p, l); + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } - return 0; } #include "unpack_template.h" From b6ba0d9baccf116ec29c5a0eb7605ae21d760740 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 14:33:36 +0900 Subject: [PATCH 0185/1648] Make setup.py executable. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) mode change 100644 => 100755 setup.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 1cbc24e..e0e0ac2 --- a/setup.py +++ b/setup.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python +# coding: utf-8 + from distutils.core import setup, Extension from Cython.Distutils import build_ext import os From 1581acfd142b68ad4ddf56fbce6c7760f1a2e920 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 14:33:36 +0900 Subject: [PATCH 0186/1648] Make setup.py executable. --- python/setup.py | 3 +++ 1 file changed, 3 insertions(+) mode change 100644 => 100755 python/setup.py diff --git a/python/setup.py b/python/setup.py old mode 100644 new mode 100755 index 1cbc24e..e0e0ac2 --- a/python/setup.py +++ b/python/setup.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python +# coding: utf-8 + from distutils.core import setup, Extension from Cython.Distutils import build_ext import os From 8596d36d02b13424b18d3f95c9b80dd0006661ac Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 24 Jun 2009 13:53:12 +0900 Subject: [PATCH 0187/1648] perl package --- Makefile.am | 20 +++++++++++++++++++- c/msgpack.h | 1 + cpp/msgpack.hpp | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index d067fe0..9e36092 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,5 +26,23 @@ EXTRA_DIST = \ ruby/test_format.rb \ ruby/test_pack.rb \ ruby/unpack.c \ - ruby/unpack.h + ruby/unpack.h \ + perl/Makefile.PL \ + perl/MessagePack.c \ + perl/benchmark/deserialize.pl \ + perl/benchmark/serialize.pl \ + perl/lib/Data/MessagePack.pm \ + perl/lib/Data/MessagePack \ + perl/lib/Data/MessagePack/Unpacker.pod \ + perl/pack.c \ + perl/ppport.h \ + perl/t/00_compile.t \ + perl/t/01_pack.t \ + perl/t/02_unpack.t \ + perl/t/03_stream_unpack.t \ + perl/t/04_invert.t \ + perl/t/Util.pm \ + perl/t/data.pl \ + perl/unpack.c \ + perl/xt/99_pod.t diff --git a/c/msgpack.h b/c/msgpack.h index a59ef03..21729f4 100644 --- a/c/msgpack.h +++ b/c/msgpack.h @@ -20,3 +20,4 @@ #include "msgpack/pack.h" #include "msgpack/unpack.h" #include "msgpack/sbuffer.h" +#include "msgpack/vrefbuffer.h" diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp index 58b40ac..e14680d 100644 --- a/cpp/msgpack.hpp +++ b/cpp/msgpack.hpp @@ -20,4 +20,5 @@ #include "msgpack/pack.hpp" #include "msgpack/unpack.hpp" #include "msgpack/sbuffer.hpp" +#include "msgpack/vrefbuffer.hpp" #include "msgpack.h" From a961f646e5dc1ea0c2881646ebf7373eb24ed564 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 24 Jun 2009 13:53:12 +0900 Subject: [PATCH 0188/1648] perl package --- Makefile.am | 20 +++++++++++++++++++- c/msgpack.h | 1 + cpp/msgpack.hpp | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index d067fe0..9e36092 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,5 +26,23 @@ EXTRA_DIST = \ ruby/test_format.rb \ ruby/test_pack.rb \ ruby/unpack.c \ - ruby/unpack.h + ruby/unpack.h \ + perl/Makefile.PL \ + perl/MessagePack.c \ + perl/benchmark/deserialize.pl \ + perl/benchmark/serialize.pl \ + perl/lib/Data/MessagePack.pm \ + perl/lib/Data/MessagePack \ + perl/lib/Data/MessagePack/Unpacker.pod \ + perl/pack.c \ + perl/ppport.h \ + perl/t/00_compile.t \ + perl/t/01_pack.t \ + perl/t/02_unpack.t \ + perl/t/03_stream_unpack.t \ + perl/t/04_invert.t \ + perl/t/Util.pm \ + perl/t/data.pl \ + perl/unpack.c \ + perl/xt/99_pod.t diff --git a/c/msgpack.h b/c/msgpack.h index a59ef03..21729f4 100644 --- a/c/msgpack.h +++ b/c/msgpack.h @@ -20,3 +20,4 @@ #include "msgpack/pack.h" #include "msgpack/unpack.h" #include "msgpack/sbuffer.h" +#include "msgpack/vrefbuffer.h" diff --git a/cpp/msgpack.hpp b/cpp/msgpack.hpp index 58b40ac..e14680d 100644 --- a/cpp/msgpack.hpp +++ b/cpp/msgpack.hpp @@ -20,4 +20,5 @@ #include "msgpack/pack.hpp" #include "msgpack/unpack.hpp" #include "msgpack/sbuffer.hpp" +#include "msgpack/vrefbuffer.hpp" #include "msgpack.h" From 9a342ab7f896cd3b6f66452278f80a424055b443 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 24 Jun 2009 14:11:06 +0900 Subject: [PATCH 0189/1648] update ppport.h --- perl/ppport.h | 95 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 8 deletions(-) diff --git a/perl/ppport.h b/perl/ppport.h index 3e3d52f..ec2f1cc 100644 --- a/perl/ppport.h +++ b/perl/ppport.h @@ -4,7 +4,7 @@ /* ---------------------------------------------------------------------- - ppport.h -- Perl/Pollution/Portability Version 3.17 + ppport.h -- Perl/Pollution/Portability Version 3.19 Automatically created by Devel::PPPort running under perl 5.010000. @@ -21,7 +21,7 @@ SKIP =head1 NAME -ppport.h - Perl/Pollution/Portability version 3.17 +ppport.h - Perl/Pollution/Portability version 3.19 =head1 SYNOPSIS @@ -232,6 +232,7 @@ same function or variable in your project. my_strlcpy() NEED_my_strlcpy NEED_my_strlcpy_GLOBAL newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL + newSV_type() NEED_newSV_type NEED_newSV_type_GLOBAL newSVpvn_flags() NEED_newSVpvn_flags NEED_newSVpvn_flags_GLOBAL newSVpvn_share() NEED_newSVpvn_share NEED_newSVpvn_share_GLOBAL pv_display() NEED_pv_display NEED_pv_display_GLOBAL @@ -377,7 +378,7 @@ use strict; # Disable broken TRIE-optimization BEGIN { eval '${^RE_TRIE_MAXBUF} = -1' if $] >= 5.009004 && $] <= 5.009005 } -my $VERSION = 3.17; +my $VERSION = 3.19; my %opt = ( quiet => 0, @@ -486,6 +487,7 @@ G_NOARGS||| G_SCALAR||| G_VOID||5.004000| GetVars||| +GvSVn|5.009003||p GvSV||| Gv_AMupdate||| HEf_SVKEY||5.004000| @@ -498,6 +500,8 @@ HeSVKEY_set||5.004000| HeSVKEY||5.004000| HeUTF8||5.011000| HeVAL||5.004000| +HvNAMELEN_get|5.009003||p +HvNAME_get|5.009003||p HvNAME||| INT2PTR|5.006000||p IN_LOCALE_COMPILETIME|5.007002||p @@ -628,6 +632,9 @@ PERL_SHORT_MAX|5.004000||p PERL_SHORT_MIN|5.004000||p PERL_SIGNALS_UNSAFE_FLAG|5.008001||p PERL_SUBVERSION|5.006000||p +PERL_SYS_INIT3||5.006000| +PERL_SYS_INIT||| +PERL_SYS_TERM||5.011000| PERL_UCHAR_MAX|5.004000||p PERL_UCHAR_MIN|5.004000||p PERL_UINT_MAX|5.004000||p @@ -661,9 +668,12 @@ PL_diehook|5.004050||p PL_dirty|5.004050||p PL_dowarn|||pn PL_errgv|5.004050||p +PL_error_count|5.011000||p PL_expect|5.011000||p PL_hexdigit|5.005000||p PL_hints|5.005000||p +PL_in_my_stash|5.011000||p +PL_in_my|5.011000||p PL_last_in_gv|||n PL_laststatval|5.005000||p PL_lex_state|5.011000||p @@ -769,6 +779,7 @@ SV_MUTABLE_RETURN|5.009003||p SV_NOSTEAL|5.009002||p SV_SMAGIC|5.009003||p SV_UTF8_NO_ENCODING|5.008001||p +SVfARG|5.009005||p SVf_UTF8|5.006000||p SVf|5.006000||p SVt_IV||| @@ -977,6 +988,7 @@ XPUSHn||| XPUSHp||| XPUSHs||| XPUSHu|5.004000||p +XSPROTO|5.010000||p XSRETURN_EMPTY||| XSRETURN_IV||| XSRETURN_NO||| @@ -1055,7 +1067,6 @@ boolSV|5.004000||p boot_core_PerlIO||| boot_core_UNIVERSAL||| boot_core_mro||| -boot_core_xsutils||| bytes_from_utf8||5.007001| bytes_to_uni|||n bytes_to_utf8||5.006001| @@ -1341,7 +1352,6 @@ get_vtbl||5.005030| getcwd_sv||5.007002| getenv_len||| glob_2number||| -glob_2pv||| glob_assign_glob||| glob_assign_ref||| gp_dup||| @@ -1372,7 +1382,8 @@ gv_fetchmethod_autoload||5.004000| gv_fetchmethod_flags||5.011000| gv_fetchmethod||| gv_fetchmeth||| -gv_fetchpvn_flags||5.009002| +gv_fetchpvn_flags|5.009002||p +gv_fetchpvs|5.009004||p gv_fetchpv||| gv_fetchsv||5.009002| gv_fullname3||5.004000| @@ -1384,7 +1395,7 @@ gv_init_sv||| gv_init||| gv_name_set||5.009004| gv_stashpvn|5.004000||p -gv_stashpvs||5.009003| +gv_stashpvs|5.009003||p gv_stashpv||| gv_stashsv||| he_dup||| @@ -1470,6 +1481,7 @@ isBLANK|5.006001||p isCNTRL|5.006000||p isDIGIT||| isGRAPH|5.006000||p +isGV_with_GP|5.009004||p isLOWER||| isPRINT|5.004000||p isPSXSPC|5.006001||p @@ -1774,7 +1786,7 @@ newSTATEOP||| newSUB||| newSVOP||| newSVREF||| -newSV_type||5.009005| +newSV_type|5.009005||p newSVhek||5.009003| newSViv||| newSVnv||| @@ -2195,6 +2207,7 @@ sv_derived_from||5.004000| sv_destroyable||5.010000| sv_does||5.009004| sv_dump||| +sv_dup_inc_multiple||| sv_dup||| sv_eq||| sv_exp_grow||| @@ -3907,6 +3920,13 @@ typedef NVTYPE NV; return; \ } STMT_END #endif +#ifndef XSPROTO +# define XSPROTO(name) void name(pTHX_ CV* cv) +#endif + +#ifndef SVfARG +# define SVfARG(p) ((void*)(p)) +#endif #ifndef PERL_ABS # define PERL_ABS(x) ((x) < 0 ? -(x) : (x)) #endif @@ -4086,9 +4106,11 @@ extern U32 DPPP_(my_PL_signals); # define PL_dirty dirty # define PL_dowarn dowarn # define PL_errgv errgv +# define PL_error_count error_count # define PL_expect expect # define PL_hexdigit hexdigit # define PL_hints hints +# define PL_in_my in_my # define PL_laststatval laststatval # define PL_lex_state lex_state # define PL_lex_stuff lex_stuff @@ -4171,6 +4193,10 @@ extern yy_parser DPPP_(dummy_PL_parser); # define PL_lex_state D_PPP_my_PL_parser_var(lex_state) # define PL_lex_stuff D_PPP_my_PL_parser_var(lex_stuff) # define PL_tokenbuf D_PPP_my_PL_parser_var(tokenbuf) +# define PL_in_my D_PPP_my_PL_parser_var(in_my) +# define PL_in_my_stash D_PPP_my_PL_parser_var(in_my_stash) +# define PL_error_count D_PPP_my_PL_parser_var(error_count) + #else @@ -4711,6 +4737,35 @@ DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv) # define SvREFCNT_inc_simple_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) #endif +#ifndef newSV_type + +#if defined(NEED_newSV_type) +static SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); +static +#else +extern SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); +#endif + +#ifdef newSV_type +# undef newSV_type +#endif +#define newSV_type(a) DPPP_(my_newSV_type)(aTHX_ a) +#define Perl_newSV_type DPPP_(my_newSV_type) + +#if defined(NEED_newSV_type) || defined(NEED_newSV_type_GLOBAL) + +SV* +DPPP_(my_newSV_type)(pTHX_ svtype const t) +{ + SV* const sv = newSV(0); + sv_upgrade(sv, t); + return sv; +} + +#endif + +#endif + #if (PERL_BCDVERSION < 0x5006000) # define D_PPP_CONSTPV_ARG(x) ((char *) (x)) #else @@ -5298,6 +5353,19 @@ DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash) #ifndef SvSHARED_HASH # define SvSHARED_HASH(sv) (0 + SvUVX(sv)) #endif +#ifndef HvNAME_get +# define HvNAME_get(hv) HvNAME(hv) +#endif +#ifndef HvNAMELEN_get +# define HvNAMELEN_get(hv) (HvNAME_get(hv) ? (I32)strlen(HvNAME_get(hv)) : 0) +#endif +#ifndef GvSVn +# define GvSVn(gv) GvSV(gv) +#endif + +#ifndef isGV_with_GP +# define isGV_with_GP(gv) isGV(gv) +#endif #ifndef WARN_ALL # define WARN_ALL 0 #endif @@ -5561,6 +5629,17 @@ DPPP_(my_warner)(U32 err, const char *pat, ...) #ifndef hv_stores # define hv_stores(hv, key, val) hv_store(hv, key "", sizeof(key) - 1, val, 0) #endif +#ifndef gv_fetchpvn_flags +# define gv_fetchpvn_flags(name, len, flags, svt) gv_fetchpv(name, flags, svt) +#endif + +#ifndef gv_fetchpvs +# define gv_fetchpvs(name, flags, svt) gv_fetchpvn_flags(name "", sizeof(name) - 1, flags, svt) +#endif + +#ifndef gv_stashpvs +# define gv_stashpvs(name, flags) gv_stashpvn(name "", sizeof(name) - 1, flags) +#endif #ifndef SvGETMAGIC # define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END #endif From b2fdaac99efc9a6c43bcc87f59103ad98d2c5bd3 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 24 Jun 2009 14:11:06 +0900 Subject: [PATCH 0190/1648] update ppport.h --- perl/ppport.h | 95 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 8 deletions(-) diff --git a/perl/ppport.h b/perl/ppport.h index 3e3d52f..ec2f1cc 100644 --- a/perl/ppport.h +++ b/perl/ppport.h @@ -4,7 +4,7 @@ /* ---------------------------------------------------------------------- - ppport.h -- Perl/Pollution/Portability Version 3.17 + ppport.h -- Perl/Pollution/Portability Version 3.19 Automatically created by Devel::PPPort running under perl 5.010000. @@ -21,7 +21,7 @@ SKIP =head1 NAME -ppport.h - Perl/Pollution/Portability version 3.17 +ppport.h - Perl/Pollution/Portability version 3.19 =head1 SYNOPSIS @@ -232,6 +232,7 @@ same function or variable in your project. my_strlcpy() NEED_my_strlcpy NEED_my_strlcpy_GLOBAL newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL + newSV_type() NEED_newSV_type NEED_newSV_type_GLOBAL newSVpvn_flags() NEED_newSVpvn_flags NEED_newSVpvn_flags_GLOBAL newSVpvn_share() NEED_newSVpvn_share NEED_newSVpvn_share_GLOBAL pv_display() NEED_pv_display NEED_pv_display_GLOBAL @@ -377,7 +378,7 @@ use strict; # Disable broken TRIE-optimization BEGIN { eval '${^RE_TRIE_MAXBUF} = -1' if $] >= 5.009004 && $] <= 5.009005 } -my $VERSION = 3.17; +my $VERSION = 3.19; my %opt = ( quiet => 0, @@ -486,6 +487,7 @@ G_NOARGS||| G_SCALAR||| G_VOID||5.004000| GetVars||| +GvSVn|5.009003||p GvSV||| Gv_AMupdate||| HEf_SVKEY||5.004000| @@ -498,6 +500,8 @@ HeSVKEY_set||5.004000| HeSVKEY||5.004000| HeUTF8||5.011000| HeVAL||5.004000| +HvNAMELEN_get|5.009003||p +HvNAME_get|5.009003||p HvNAME||| INT2PTR|5.006000||p IN_LOCALE_COMPILETIME|5.007002||p @@ -628,6 +632,9 @@ PERL_SHORT_MAX|5.004000||p PERL_SHORT_MIN|5.004000||p PERL_SIGNALS_UNSAFE_FLAG|5.008001||p PERL_SUBVERSION|5.006000||p +PERL_SYS_INIT3||5.006000| +PERL_SYS_INIT||| +PERL_SYS_TERM||5.011000| PERL_UCHAR_MAX|5.004000||p PERL_UCHAR_MIN|5.004000||p PERL_UINT_MAX|5.004000||p @@ -661,9 +668,12 @@ PL_diehook|5.004050||p PL_dirty|5.004050||p PL_dowarn|||pn PL_errgv|5.004050||p +PL_error_count|5.011000||p PL_expect|5.011000||p PL_hexdigit|5.005000||p PL_hints|5.005000||p +PL_in_my_stash|5.011000||p +PL_in_my|5.011000||p PL_last_in_gv|||n PL_laststatval|5.005000||p PL_lex_state|5.011000||p @@ -769,6 +779,7 @@ SV_MUTABLE_RETURN|5.009003||p SV_NOSTEAL|5.009002||p SV_SMAGIC|5.009003||p SV_UTF8_NO_ENCODING|5.008001||p +SVfARG|5.009005||p SVf_UTF8|5.006000||p SVf|5.006000||p SVt_IV||| @@ -977,6 +988,7 @@ XPUSHn||| XPUSHp||| XPUSHs||| XPUSHu|5.004000||p +XSPROTO|5.010000||p XSRETURN_EMPTY||| XSRETURN_IV||| XSRETURN_NO||| @@ -1055,7 +1067,6 @@ boolSV|5.004000||p boot_core_PerlIO||| boot_core_UNIVERSAL||| boot_core_mro||| -boot_core_xsutils||| bytes_from_utf8||5.007001| bytes_to_uni|||n bytes_to_utf8||5.006001| @@ -1341,7 +1352,6 @@ get_vtbl||5.005030| getcwd_sv||5.007002| getenv_len||| glob_2number||| -glob_2pv||| glob_assign_glob||| glob_assign_ref||| gp_dup||| @@ -1372,7 +1382,8 @@ gv_fetchmethod_autoload||5.004000| gv_fetchmethod_flags||5.011000| gv_fetchmethod||| gv_fetchmeth||| -gv_fetchpvn_flags||5.009002| +gv_fetchpvn_flags|5.009002||p +gv_fetchpvs|5.009004||p gv_fetchpv||| gv_fetchsv||5.009002| gv_fullname3||5.004000| @@ -1384,7 +1395,7 @@ gv_init_sv||| gv_init||| gv_name_set||5.009004| gv_stashpvn|5.004000||p -gv_stashpvs||5.009003| +gv_stashpvs|5.009003||p gv_stashpv||| gv_stashsv||| he_dup||| @@ -1470,6 +1481,7 @@ isBLANK|5.006001||p isCNTRL|5.006000||p isDIGIT||| isGRAPH|5.006000||p +isGV_with_GP|5.009004||p isLOWER||| isPRINT|5.004000||p isPSXSPC|5.006001||p @@ -1774,7 +1786,7 @@ newSTATEOP||| newSUB||| newSVOP||| newSVREF||| -newSV_type||5.009005| +newSV_type|5.009005||p newSVhek||5.009003| newSViv||| newSVnv||| @@ -2195,6 +2207,7 @@ sv_derived_from||5.004000| sv_destroyable||5.010000| sv_does||5.009004| sv_dump||| +sv_dup_inc_multiple||| sv_dup||| sv_eq||| sv_exp_grow||| @@ -3907,6 +3920,13 @@ typedef NVTYPE NV; return; \ } STMT_END #endif +#ifndef XSPROTO +# define XSPROTO(name) void name(pTHX_ CV* cv) +#endif + +#ifndef SVfARG +# define SVfARG(p) ((void*)(p)) +#endif #ifndef PERL_ABS # define PERL_ABS(x) ((x) < 0 ? -(x) : (x)) #endif @@ -4086,9 +4106,11 @@ extern U32 DPPP_(my_PL_signals); # define PL_dirty dirty # define PL_dowarn dowarn # define PL_errgv errgv +# define PL_error_count error_count # define PL_expect expect # define PL_hexdigit hexdigit # define PL_hints hints +# define PL_in_my in_my # define PL_laststatval laststatval # define PL_lex_state lex_state # define PL_lex_stuff lex_stuff @@ -4171,6 +4193,10 @@ extern yy_parser DPPP_(dummy_PL_parser); # define PL_lex_state D_PPP_my_PL_parser_var(lex_state) # define PL_lex_stuff D_PPP_my_PL_parser_var(lex_stuff) # define PL_tokenbuf D_PPP_my_PL_parser_var(tokenbuf) +# define PL_in_my D_PPP_my_PL_parser_var(in_my) +# define PL_in_my_stash D_PPP_my_PL_parser_var(in_my_stash) +# define PL_error_count D_PPP_my_PL_parser_var(error_count) + #else @@ -4711,6 +4737,35 @@ DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv) # define SvREFCNT_inc_simple_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) #endif +#ifndef newSV_type + +#if defined(NEED_newSV_type) +static SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); +static +#else +extern SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); +#endif + +#ifdef newSV_type +# undef newSV_type +#endif +#define newSV_type(a) DPPP_(my_newSV_type)(aTHX_ a) +#define Perl_newSV_type DPPP_(my_newSV_type) + +#if defined(NEED_newSV_type) || defined(NEED_newSV_type_GLOBAL) + +SV* +DPPP_(my_newSV_type)(pTHX_ svtype const t) +{ + SV* const sv = newSV(0); + sv_upgrade(sv, t); + return sv; +} + +#endif + +#endif + #if (PERL_BCDVERSION < 0x5006000) # define D_PPP_CONSTPV_ARG(x) ((char *) (x)) #else @@ -5298,6 +5353,19 @@ DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash) #ifndef SvSHARED_HASH # define SvSHARED_HASH(sv) (0 + SvUVX(sv)) #endif +#ifndef HvNAME_get +# define HvNAME_get(hv) HvNAME(hv) +#endif +#ifndef HvNAMELEN_get +# define HvNAMELEN_get(hv) (HvNAME_get(hv) ? (I32)strlen(HvNAME_get(hv)) : 0) +#endif +#ifndef GvSVn +# define GvSVn(gv) GvSV(gv) +#endif + +#ifndef isGV_with_GP +# define isGV_with_GP(gv) isGV(gv) +#endif #ifndef WARN_ALL # define WARN_ALL 0 #endif @@ -5561,6 +5629,17 @@ DPPP_(my_warner)(U32 err, const char *pat, ...) #ifndef hv_stores # define hv_stores(hv, key, val) hv_store(hv, key "", sizeof(key) - 1, val, 0) #endif +#ifndef gv_fetchpvn_flags +# define gv_fetchpvn_flags(name, len, flags, svt) gv_fetchpv(name, flags, svt) +#endif + +#ifndef gv_fetchpvs +# define gv_fetchpvs(name, flags, svt) gv_fetchpvn_flags(name "", sizeof(name) - 1, flags, svt) +#endif + +#ifndef gv_stashpvs +# define gv_stashpvs(name, flags) gv_stashpvn(name "", sizeof(name) - 1, flags) +#endif #ifndef SvGETMAGIC # define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END #endif From 3aef3edbad682ecd38ae0bd83cfb1060800335a4 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 24 Jun 2009 14:36:37 +0900 Subject: [PATCH 0191/1648] works correctly under Perl5.8.x. Perl5.8.x makes "a" as PVIV in {"a", 0}. --- perl/pack.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 9c32dc1..5aec963 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -81,14 +81,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { } } break; - case SVt_PV: - { - STRLEN len; - char * cval = SvPV(val, len); - msgpack_pack_raw(enc, len); - msgpack_pack_raw_body(enc, cval, len); - } - break; case SVt_NV: PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); break; @@ -126,8 +118,16 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { _msgpack_pack_sv(enc, SvRV(val)); break; default: - sv_dump(val); - Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + if (SvPOKp(val)) { + STRLEN len; + char * cval = SvPV(val, len); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, cval, len); + return; + } else { + sv_dump(val); + Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + } } } From 8e166de1d1b6279dfe94583631711fde5bb22bea Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 24 Jun 2009 14:46:28 +0900 Subject: [PATCH 0192/1648] works correctly under Perl5.8.x. Perl5.8.x makes "a" as PVIV in {"a", 0}. --- perl/pack.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 9c32dc1..5aec963 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -81,14 +81,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { } } break; - case SVt_PV: - { - STRLEN len; - char * cval = SvPV(val, len); - msgpack_pack_raw(enc, len); - msgpack_pack_raw_body(enc, cval, len); - } - break; case SVt_NV: PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); break; @@ -126,8 +118,16 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { _msgpack_pack_sv(enc, SvRV(val)); break; default: - sv_dump(val); - Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + if (SvPOKp(val)) { + STRLEN len; + char * cval = SvPV(val, len); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, cval, len); + return; + } else { + sv_dump(val); + Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + } } } From 87e1adcb40ae4946c7443a7c1483a3e521869052 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 24 Jun 2009 14:36:37 +0900 Subject: [PATCH 0193/1648] works correctly under Perl5.8.x. Perl5.8.x makes "a" as PVIV in {"a", 0}. --- perl/pack.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 9c32dc1..5aec963 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -81,14 +81,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { } } break; - case SVt_PV: - { - STRLEN len; - char * cval = SvPV(val, len); - msgpack_pack_raw(enc, len); - msgpack_pack_raw_body(enc, cval, len); - } - break; case SVt_NV: PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); break; @@ -126,8 +118,16 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { _msgpack_pack_sv(enc, SvRV(val)); break; default: - sv_dump(val); - Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + if (SvPOKp(val)) { + STRLEN len; + char * cval = SvPV(val, len); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, cval, len); + return; + } else { + sv_dump(val); + Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + } } } From 479263989b02693f0e449ee591ca58ed84fd904b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 14:58:02 +0900 Subject: [PATCH 0194/1648] Stop unnecessary caching. --- python/msgpack/unpack.h | 44 +++++------------------------------------ 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index e30b72a..dd23130 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -16,24 +16,10 @@ * limitations under the License. */ -#include -#include -#include - #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -typedef std::map str_cach_t; struct unpack_user { - str_cach_t strcache; - - ~unpack_user() { - str_cach_t::iterator it, itend; - itend = strcache.end(); - for (it = strcache.begin(); it != itend; ++it) { - Py_DECREF(it->second); - } - } }; @@ -181,31 +167,11 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { PyObject *py; - if (l < 16) { - std::string s(p, l); - str_cach_t ::iterator it = u->strcache.find(s); - if (it != u->strcache.end()) { - *o = it->second; - Py_INCREF(*o); - return 0; - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - Py_INCREF(*o); - u->strcache[s] = *o; - return 0; - } - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - return 0; - } + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } #include "unpack_template.h" From b944eefb96a0f3ad907c8fe704fd14ba87384ed5 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 24 Jun 2009 14:58:02 +0900 Subject: [PATCH 0195/1648] Stop unnecessary caching. --- msgpack/unpack.h | 44 +++++--------------------------------------- 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index e30b72a..dd23130 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -16,24 +16,10 @@ * limitations under the License. */ -#include -#include -#include - #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -typedef std::map str_cach_t; struct unpack_user { - str_cach_t strcache; - - ~unpack_user() { - str_cach_t::iterator it, itend; - itend = strcache.end(); - for (it = strcache.begin(); it != itend; ++it) { - Py_DECREF(it->second); - } - } }; @@ -181,31 +167,11 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) { PyObject *py; - if (l < 16) { - std::string s(p, l); - str_cach_t ::iterator it = u->strcache.find(s); - if (it != u->strcache.end()) { - *o = it->second; - Py_INCREF(*o); - return 0; - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - Py_INCREF(*o); - u->strcache[s] = *o; - return 0; - } - } - else { - py = PyString_FromStringAndSize(p, l); - if (!py) - return -1; - *o = py; - return 0; - } + py = PyString_FromStringAndSize(p, l); + if (!py) + return -1; + *o = py; + return 0; } #include "unpack_template.h" From 8f52ed26c7ecbc28ebcc8deb3f65bdce940fd22b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 24 Jun 2009 16:01:13 +0900 Subject: [PATCH 0196/1648] version 0.3.3 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 47b28cf..45a04a9 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.2) +AM_INIT_AUTOMAKE(msgpack, 0.3.3) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From dd407410d25b4364897f9518f0fd173304b4718b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 24 Jun 2009 16:01:13 +0900 Subject: [PATCH 0197/1648] version 0.3.3 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 47b28cf..45a04a9 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.2) +AM_INIT_AUTOMAKE(msgpack, 0.3.3) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From 3c3df3133c6f6320a03ebcaef546ab9c2c53154d Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Fri, 26 Jun 2009 14:10:20 +0900 Subject: [PATCH 0198/1648] Implement streaming deserializer. --- python/msgpack/_msgpack.pyx | 162 ++++++++++++++++++++++++++++++++---- python/test_sequnpack.py | 36 ++++++++ 2 files changed, 183 insertions(+), 15 deletions(-) create mode 100644 python/test_sequnpack.py diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index cbdcfc5..c37f8ba 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -6,13 +6,16 @@ cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + char* PyString_AsString(object o) cdef extern from "stdlib.h": - void* malloc(int) + void* malloc(size_t) + void* realloc(void*, size_t) void free(void*) cdef extern from "string.h": - int memcpy(char*dst, char*src, unsigned int size) + void* memcpy(char* dst, char* src, size_t size) + void* memmove(char* dst, char* src, size_t size) cdef extern from "pack.h": ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) @@ -34,8 +37,6 @@ cdef extern from "pack.h": void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef int BUFF_SIZE=2*1024 - cdef class Packer: """Packer that pack data into strm. @@ -48,10 +49,7 @@ cdef class Packer: cdef msgpack_packer pk cdef object strm - def __init__(self, strm, int size=0): - if size <= 0: - size = BUFF_SIZE - + def __init__(self, strm, int size=4*1024): self.strm = strm self.buff = malloc(size) self.allocated = size @@ -147,6 +145,8 @@ cdef class Packer: if flush: self.flush() + close = flush + cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: @@ -163,20 +163,28 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): return 0 def pack(object o, object stream): + u"""pack o and write to stream).""" packer = Packer(stream) packer.pack(o) packer.flush() -def packs(object o): +def packb(object o): + u"""pack o and return packed bytes.""" buf = StringIO() packer = Packer(buf) packer.pack(o) packer.flush() return buf.getvalue() +packs = packb + cdef extern from "unpack.h": ctypedef struct template_context: - pass + PyObject* obj + size_t count + unsigned int ct + PyObject* key + int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) void template_init(template_context* ctx) @@ -188,15 +196,139 @@ def unpacks(object packed_bytes): cdef const_char_ptr p = packed_bytes cdef template_context ctx cdef size_t off = 0 + cdef int ret template_init(&ctx) - template_execute(&ctx, p, len(packed_bytes), &off) - return template_data(&ctx) + ret = template_execute(&ctx, p, len(packed_bytes), &off) + if ret == 1: + return template_data(&ctx) + else: + return None def unpack(object stream): """unpack from stream.""" packed = stream.read() return unpacks(packed) -cdef class Unpacker: - """Do nothing. This function is for symmetric to Packer""" - unpack = staticmethod(unpacks) +cdef class UnpackIterator(object): + cdef object unpacker + + def __init__(self, unpacker): + self.unpacker = unpacker + + def __next__(self): + return self.unpacker.unpack() + + def __iter__(self): + return self + +cdef class Unpacker(object): + """Unpacker(file_like=None, read_size=4096) + + Streaming unpacker. + file_like must have read(n) method. + read_size is used like file_like.read(read_size) + + If file_like is None, you can feed() bytes. feed() is useful + for unpack from non-blocking stream. + + exsample 1: + unpacker = Unpacker(afile) + for o in unpacker: + do_something(o) + + example 2: + unpacker = Unpacker() + while 1: + buf = astream.read() + unpacker.feed(buf) + for o in unpacker: + do_something(o) + """ + + cdef template_context ctx + cdef char* buf + cdef size_t buf_size, buf_head, buf_tail + cdef object file_like + cdef int read_size + cdef object waiting_bytes + + def __init__(self, file_like=None, int read_size=4096): + self.file_like = file_like + self.read_size = read_size + self.waiting_bytes = [] + self.buf = malloc(read_size) + self.buf_size = read_size + self.buf_head = 0 + self.buf_tail = 0 + template_init(&self.ctx) + + def feed(self, next_bytes): + if not isinstance(next_bytes, str): + raise ValueError, "Argument must be bytes object" + self.waiting_bytes.append(next_bytes) + + cdef append_buffer(self): + cdef char* buf = self.buf + cdef Py_ssize_t tail = self.buf_tail + cdef Py_ssize_t l + + for b in self.waiting_bytes: + l = len(b) + memcpy(buf + tail, PyString_AsString(b), l) + tail += l + self.buf_tail = tail + del self.waiting_bytes[:] + + # prepare self.buf + cdef fill_buffer(self): + cdef Py_ssize_t add_size + + if self.file_like is not None: + self.waiting_bytes.append(self.file_like.read(self.read_size)) + + if not self.waiting_bytes: + return + + add_size = 0 + for b in self.waiting_bytes: + add_size += len(b) + + cdef char* buf = self.buf + cdef size_t head = self.buf_head + cdef size_t tail = self.buf_tail + cdef size_t size = self.buf_size + + if self.buf_tail + add_size <= self.buf_size: + # do nothing. + pass + if self.buf_tail - self.buf_head + add_size < self.buf_size: + # move to front. + memmove(buf, buf + head, tail - head) + tail -= head + head = 0 + else: + # expand buffer + size = tail + add_size + buf = realloc(buf, size) + + self.buf = buf + self.buf_head = head + self.buf_tail = tail + self.buf_size = size + + self.append_buffer() + + cpdef unpack(self): + """unpack one object""" + cdef int ret + self.fill_buffer() + ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + if ret == 1: + return template_data(&self.ctx) + elif ret == 0: + raise StopIteration, "No more unpack data." + else: + raise ValueError, "Unpack failed." + + def __iter__(self): + return UnpackIterator(self) diff --git a/python/test_sequnpack.py b/python/test_sequnpack.py new file mode 100644 index 0000000..789ccd2 --- /dev/null +++ b/python/test_sequnpack.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import unicode_literals, print_function + +from msgpack import Unpacker + +def test_foobar(): + unpacker = Unpacker(read_size=3) + unpacker.feed(b'foobar') + assert unpacker.unpack() == ord('f') + assert unpacker.unpack() == ord('o') + assert unpacker.unpack() == ord('o') + assert unpacker.unpack() == ord('b') + assert unpacker.unpack() == ord('a') + assert unpacker.unpack() == ord('r') + try: + o = unpacker.unpack() + print("Oops!", o) + assert 0 + except StopIteration: + assert 1 + else: + assert 0 + unpacker.feed(b'foo') + unpacker.feed(b'bar') + + k = 0 + for o, e in zip(unpacker, b'foobarbaz'): + assert o == ord(e) + k += 1 + assert k == len(b'foobar') + +if __name__ == '__main__': + test_foobar() + From 1781092bd8da6e014f78873516dd1e403205fcd7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Fri, 26 Jun 2009 14:10:20 +0900 Subject: [PATCH 0199/1648] Implement streaming deserializer. --- msgpack/_msgpack.pyx | 162 +++++++++++++++++++++++++++++++++++++++---- test_sequnpack.py | 36 ++++++++++ 2 files changed, 183 insertions(+), 15 deletions(-) create mode 100644 test_sequnpack.py diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index cbdcfc5..c37f8ba 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -6,13 +6,16 @@ cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + char* PyString_AsString(object o) cdef extern from "stdlib.h": - void* malloc(int) + void* malloc(size_t) + void* realloc(void*, size_t) void free(void*) cdef extern from "string.h": - int memcpy(char*dst, char*src, unsigned int size) + void* memcpy(char* dst, char* src, size_t size) + void* memmove(char* dst, char* src, size_t size) cdef extern from "pack.h": ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) @@ -34,8 +37,6 @@ cdef extern from "pack.h": void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef int BUFF_SIZE=2*1024 - cdef class Packer: """Packer that pack data into strm. @@ -48,10 +49,7 @@ cdef class Packer: cdef msgpack_packer pk cdef object strm - def __init__(self, strm, int size=0): - if size <= 0: - size = BUFF_SIZE - + def __init__(self, strm, int size=4*1024): self.strm = strm self.buff = malloc(size) self.allocated = size @@ -147,6 +145,8 @@ cdef class Packer: if flush: self.flush() + close = flush + cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: @@ -163,20 +163,28 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): return 0 def pack(object o, object stream): + u"""pack o and write to stream).""" packer = Packer(stream) packer.pack(o) packer.flush() -def packs(object o): +def packb(object o): + u"""pack o and return packed bytes.""" buf = StringIO() packer = Packer(buf) packer.pack(o) packer.flush() return buf.getvalue() +packs = packb + cdef extern from "unpack.h": ctypedef struct template_context: - pass + PyObject* obj + size_t count + unsigned int ct + PyObject* key + int template_execute(template_context* ctx, const_char_ptr data, size_t len, size_t* off) void template_init(template_context* ctx) @@ -188,15 +196,139 @@ def unpacks(object packed_bytes): cdef const_char_ptr p = packed_bytes cdef template_context ctx cdef size_t off = 0 + cdef int ret template_init(&ctx) - template_execute(&ctx, p, len(packed_bytes), &off) - return template_data(&ctx) + ret = template_execute(&ctx, p, len(packed_bytes), &off) + if ret == 1: + return template_data(&ctx) + else: + return None def unpack(object stream): """unpack from stream.""" packed = stream.read() return unpacks(packed) -cdef class Unpacker: - """Do nothing. This function is for symmetric to Packer""" - unpack = staticmethod(unpacks) +cdef class UnpackIterator(object): + cdef object unpacker + + def __init__(self, unpacker): + self.unpacker = unpacker + + def __next__(self): + return self.unpacker.unpack() + + def __iter__(self): + return self + +cdef class Unpacker(object): + """Unpacker(file_like=None, read_size=4096) + + Streaming unpacker. + file_like must have read(n) method. + read_size is used like file_like.read(read_size) + + If file_like is None, you can feed() bytes. feed() is useful + for unpack from non-blocking stream. + + exsample 1: + unpacker = Unpacker(afile) + for o in unpacker: + do_something(o) + + example 2: + unpacker = Unpacker() + while 1: + buf = astream.read() + unpacker.feed(buf) + for o in unpacker: + do_something(o) + """ + + cdef template_context ctx + cdef char* buf + cdef size_t buf_size, buf_head, buf_tail + cdef object file_like + cdef int read_size + cdef object waiting_bytes + + def __init__(self, file_like=None, int read_size=4096): + self.file_like = file_like + self.read_size = read_size + self.waiting_bytes = [] + self.buf = malloc(read_size) + self.buf_size = read_size + self.buf_head = 0 + self.buf_tail = 0 + template_init(&self.ctx) + + def feed(self, next_bytes): + if not isinstance(next_bytes, str): + raise ValueError, "Argument must be bytes object" + self.waiting_bytes.append(next_bytes) + + cdef append_buffer(self): + cdef char* buf = self.buf + cdef Py_ssize_t tail = self.buf_tail + cdef Py_ssize_t l + + for b in self.waiting_bytes: + l = len(b) + memcpy(buf + tail, PyString_AsString(b), l) + tail += l + self.buf_tail = tail + del self.waiting_bytes[:] + + # prepare self.buf + cdef fill_buffer(self): + cdef Py_ssize_t add_size + + if self.file_like is not None: + self.waiting_bytes.append(self.file_like.read(self.read_size)) + + if not self.waiting_bytes: + return + + add_size = 0 + for b in self.waiting_bytes: + add_size += len(b) + + cdef char* buf = self.buf + cdef size_t head = self.buf_head + cdef size_t tail = self.buf_tail + cdef size_t size = self.buf_size + + if self.buf_tail + add_size <= self.buf_size: + # do nothing. + pass + if self.buf_tail - self.buf_head + add_size < self.buf_size: + # move to front. + memmove(buf, buf + head, tail - head) + tail -= head + head = 0 + else: + # expand buffer + size = tail + add_size + buf = realloc(buf, size) + + self.buf = buf + self.buf_head = head + self.buf_tail = tail + self.buf_size = size + + self.append_buffer() + + cpdef unpack(self): + """unpack one object""" + cdef int ret + self.fill_buffer() + ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) + if ret == 1: + return template_data(&self.ctx) + elif ret == 0: + raise StopIteration, "No more unpack data." + else: + raise ValueError, "Unpack failed." + + def __iter__(self): + return UnpackIterator(self) diff --git a/test_sequnpack.py b/test_sequnpack.py new file mode 100644 index 0000000..789ccd2 --- /dev/null +++ b/test_sequnpack.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import unicode_literals, print_function + +from msgpack import Unpacker + +def test_foobar(): + unpacker = Unpacker(read_size=3) + unpacker.feed(b'foobar') + assert unpacker.unpack() == ord('f') + assert unpacker.unpack() == ord('o') + assert unpacker.unpack() == ord('o') + assert unpacker.unpack() == ord('b') + assert unpacker.unpack() == ord('a') + assert unpacker.unpack() == ord('r') + try: + o = unpacker.unpack() + print("Oops!", o) + assert 0 + except StopIteration: + assert 1 + else: + assert 0 + unpacker.feed(b'foo') + unpacker.feed(b'bar') + + k = 0 + for o, e in zip(unpacker, b'foobarbaz'): + assert o == ord(e) + k += 1 + assert k == len(b'foobar') + +if __name__ == '__main__': + test_foobar() + From a345131aaa4a87c2e917913eae053a8382fd4e67 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sat, 27 Jun 2009 12:03:00 +0900 Subject: [PATCH 0200/1648] Add: README --- python/README | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 python/README diff --git a/python/README b/python/README new file mode 100644 index 0000000..12a27b2 --- /dev/null +++ b/python/README @@ -0,0 +1,31 @@ +=========================== +MessagePack Python Binding +=========================== + +:author: Naoki INADA +:version: 0.0.1 +:date: 2009-06-27 + +HOW TO USE +----------- +You can read document in docstring after `import msgpack` + + +INSTALL +--------- +Cython_ is required to build the binding. + +.. _Cython: http://www.cython.org/ + +Posix +'''''' +You can install msgpack in common way. + + $ python setup.py install + +Windows +'''''''' +MessagePack requires gcc currently. So you need to prepare +MinGW GCC. + + $ python setup.py install -c mingw From 26a8d5d90f487728f2ab1d3cd0289500d68300e1 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sat, 27 Jun 2009 12:03:00 +0900 Subject: [PATCH 0201/1648] Add: README --- README | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 README diff --git a/README b/README new file mode 100644 index 0000000..12a27b2 --- /dev/null +++ b/README @@ -0,0 +1,31 @@ +=========================== +MessagePack Python Binding +=========================== + +:author: Naoki INADA +:version: 0.0.1 +:date: 2009-06-27 + +HOW TO USE +----------- +You can read document in docstring after `import msgpack` + + +INSTALL +--------- +Cython_ is required to build the binding. + +.. _Cython: http://www.cython.org/ + +Posix +'''''' +You can install msgpack in common way. + + $ python setup.py install + +Windows +'''''''' +MessagePack requires gcc currently. So you need to prepare +MinGW GCC. + + $ python setup.py install -c mingw From fa2efcdb5ba8a5cca141f7882271abb8ea697caa Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sat, 27 Jun 2009 12:04:11 +0900 Subject: [PATCH 0202/1648] Rename test files. --- python/{testformat.py => test/test_format.py} | 0 python/{ => test}/test_sequnpack.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename python/{testformat.py => test/test_format.py} (100%) rename python/{ => test}/test_sequnpack.py (100%) diff --git a/python/testformat.py b/python/test/test_format.py similarity index 100% rename from python/testformat.py rename to python/test/test_format.py diff --git a/python/test_sequnpack.py b/python/test/test_sequnpack.py similarity index 100% rename from python/test_sequnpack.py rename to python/test/test_sequnpack.py From 04d8fc114b256a80f1fac4e04352d487c47f6768 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sat, 27 Jun 2009 12:04:11 +0900 Subject: [PATCH 0203/1648] Rename test files. --- testformat.py => test/test_format.py | 0 test_sequnpack.py => test/test_sequnpack.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename testformat.py => test/test_format.py (100%) rename test_sequnpack.py => test/test_sequnpack.py (100%) diff --git a/testformat.py b/test/test_format.py similarity index 100% rename from testformat.py rename to test/test_format.py diff --git a/test_sequnpack.py b/test/test_sequnpack.py similarity index 100% rename from test_sequnpack.py rename to test/test_sequnpack.py From 1b07b61c048f749c7a5e505617b0e59974f81077 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 28 Jun 2009 21:24:02 +0900 Subject: [PATCH 0204/1648] Ues more suitable type when packing. --- python/msgpack/_msgpack.pyx | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index c37f8ba..cde5313 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -7,6 +7,8 @@ cdef extern from "Python.h": ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) char* PyString_AsString(object o) + int PyMapping_Check(object o) + int PySequence_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -37,7 +39,7 @@ cdef extern from "pack.h": void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef class Packer: +cdef class Packer(object): """Packer that pack data into strm. strm must have `write(bytes)` method. @@ -99,7 +101,8 @@ cdef class Packer: msgpack_pack_map(&self.pk, len) cdef __pack(self, object o): - cdef long long intval + cdef long long llval + cdef long longval cdef double fval cdef char* rawval @@ -110,11 +113,11 @@ cdef class Packer: elif o is False: msgpack_pack_false(&self.pk) elif isinstance(o, long): - intval = o - msgpack_pack_long_long(&self.pk, intval) + llval = o + msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): - intval = o - msgpack_pack_long_long(&self.pk, intval) + longval = o + msgpack_pack_long_long(&self.pk, longval) elif isinstance(o, float): fval = o msgpack_pack_double(&self.pk, fval) @@ -127,12 +130,12 @@ cdef class Packer: rawval = o msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif isinstance(o, dict): + elif PyMapping_Check(o): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): self.pack(k) self.pack(v) - elif isinstance(o, tuple) or isinstance(o, list): + elif PySequence_Check(o): msgpack_pack_array(&self.pk, len(o)) for v in o: self.pack(v) From 1374bce15f3e6643c49a728a773407994a8207b9 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 28 Jun 2009 21:24:02 +0900 Subject: [PATCH 0205/1648] Ues more suitable type when packing. --- msgpack/_msgpack.pyx | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c37f8ba..cde5313 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -7,6 +7,8 @@ cdef extern from "Python.h": ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) char* PyString_AsString(object o) + int PyMapping_Check(object o) + int PySequence_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -37,7 +39,7 @@ cdef extern from "pack.h": void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) -cdef class Packer: +cdef class Packer(object): """Packer that pack data into strm. strm must have `write(bytes)` method. @@ -99,7 +101,8 @@ cdef class Packer: msgpack_pack_map(&self.pk, len) cdef __pack(self, object o): - cdef long long intval + cdef long long llval + cdef long longval cdef double fval cdef char* rawval @@ -110,11 +113,11 @@ cdef class Packer: elif o is False: msgpack_pack_false(&self.pk) elif isinstance(o, long): - intval = o - msgpack_pack_long_long(&self.pk, intval) + llval = o + msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): - intval = o - msgpack_pack_long_long(&self.pk, intval) + longval = o + msgpack_pack_long_long(&self.pk, longval) elif isinstance(o, float): fval = o msgpack_pack_double(&self.pk, fval) @@ -127,12 +130,12 @@ cdef class Packer: rawval = o msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif isinstance(o, dict): + elif PyMapping_Check(o): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): self.pack(k) self.pack(v) - elif isinstance(o, tuple) or isinstance(o, list): + elif PySequence_Check(o): msgpack_pack_array(&self.pk, len(o)) for v in o: self.pack(v) From 3e396ef146940ada28fc8b154189ae9bb206babc Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 28 Jun 2009 21:24:16 +0900 Subject: [PATCH 0206/1648] Don't use C++. --- python/msgpack/unpack.h | 4 ++-- python/setup.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index dd23130..40058d0 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -19,8 +19,8 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -struct unpack_user { -}; +typedef struct unpack_user { +} unpack_user; #define msgpack_unpack_struct(name) \ diff --git a/python/setup.py b/python/setup.py index e0e0ac2..faadb33 100755 --- a/python/setup.py +++ b/python/setup.py @@ -10,7 +10,6 @@ version = '0.0.1dev' PACKAGE_ROOT = os.getcwdu() INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') msgpack_mod = Extension('msgpack._msgpack', - language="c++", sources=['msgpack/_msgpack.pyx'], include_dirs=[INCLUDE_PATH]) From 5a87c94333904aea236f646c99c73ce25ba3c74a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 28 Jun 2009 21:24:16 +0900 Subject: [PATCH 0207/1648] Don't use C++. --- msgpack/unpack.h | 4 ++-- setup.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index dd23130..40058d0 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -19,8 +19,8 @@ #define MSGPACK_MAX_STACK_SIZE (1024) #include "unpack_define.h" -struct unpack_user { -}; +typedef struct unpack_user { +} unpack_user; #define msgpack_unpack_struct(name) \ diff --git a/setup.py b/setup.py index e0e0ac2..faadb33 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ version = '0.0.1dev' PACKAGE_ROOT = os.getcwdu() INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') msgpack_mod = Extension('msgpack._msgpack', - language="c++", sources=['msgpack/_msgpack.pyx'], include_dirs=[INCLUDE_PATH]) From 0b33a634a668a478ee86f2c3d9d29c6be85afa6b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 08:23:27 +0900 Subject: [PATCH 0208/1648] Update test_format. --- python/README | 7 +++ python/test/test_format.py | 110 ++++++++++++++++++++----------------- 2 files changed, 67 insertions(+), 50 deletions(-) diff --git a/python/README b/python/README index 12a27b2..243def2 100644 --- a/python/README +++ b/python/README @@ -29,3 +29,10 @@ MessagePack requires gcc currently. So you need to prepare MinGW GCC. $ python setup.py install -c mingw + +TEST +---- +MessagePack uses nosetest for testing. +Run test with following command: + + $ nosetests test diff --git a/python/test/test_format.py b/python/test/test_format.py index a00123c..009a764 100644 --- a/python/test/test_format.py +++ b/python/test/test_format.py @@ -1,64 +1,74 @@ -from unittest import TestCase, main -from msgpack import packs, unpacks +#!/usr/bin/env python +# coding: utf-8 -class TestFormat(TestCase): - def __check(self, obj, expected_packed): - packed = packs(obj) - self.assertEqual(packed, expected_packed) - unpacked = unpacks(packed) - self.assertEqual(unpacked, obj) +from nose import main +from nose.tools import * +from msgpack import unpacks - def testSimpleValues(self): - self.__check(None, '\xc0') - self.__check(True, '\xc3') - self.__check(False, '\xc2') - self.__check( - [None, False, True], - '\x93\xc0\xc2\xc3' - ) +def check(src, should): + assert_equal(unpacks(src), should) - def testFixnum(self): - self.__check( - [[0,64,127], [-32,-16,-1]], - "\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff" - ) - - def testFixArray(self): - self.__check( - [[],[[None]]], - "\x92\x90\x91\x91\xc0" - ) +def testSimpleValue(): + check("\x93\xc0\xc2\xc3", + [None, False, True]) - def testFixRaw(self): - self.__check( - ["", "a", "bc", "def"], - "\x94\xa0\xa1a\xa2bc\xa3def" - ) - pass +def testFixnum(): + check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", + [[0,64,127], [-32,-16,-1]] + ) - def testFixMap(self): - self.__check( - {False: {None: None}, True:{None:{}}}, - "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80" - ) - pass +def testFixArray(): + check("\x92\x90\x91\x91\xc0", + [[],[[None]]], + ) +def testFixRaw(): + check("\x94\xa0\xa1a\xa2bc\xa3def", + ["", "a", "bc", "def"], + ) -class TestUnpack(TestCase): - def __check(self, packed, obj): - self.assertEqual(unpacks(packed), obj) +def testFixMap(): + check( + "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", + {False: {None: None}, True:{None:{}}}, + ) - def testuint(self): - self.__check( - "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" - "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" - "\xce\xff\xff\xff\xff", - [0, 128, 255, 0, 32768, 65535, 0, - 2147483648, 4294967295], - ) +def testUnsignedInt(): + check( + "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + "\xce\xff\xff\xff\xff", + [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + ) +def testSignedInt(): + check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" + "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" + "\xd2\xff\xff\xff\xff", + [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) +def testRaw(): + check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" + "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", + ["", "a", "ab", "", "a", "ab"]) +def testArray(): + check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" + "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" + "\xc2\xc3", + [[], [None], [False,True], [], [None], [False,True]]) + +def testMap(): + check( + "\x96" + "\xde\x00\x00" + "\xde\x00\x01\xc0\xc2" + "\xde\x00\x02\xc0\xc2\xc3\xc2" + "\xdf\x00\x00\x00\x00" + "\xdf\x00\x00\x00\x01\xc0\xc2" + "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", + [{}, {None: False}, {True: False, None: False}, {}, + {None: False}, {True: False, None: False}]) if __name__ == '__main__': main() From 9105c3513b2c35e225e4286bec582a5b4dc09f0b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 08:23:27 +0900 Subject: [PATCH 0209/1648] Update test_format. --- README | 7 +++ test/test_format.py | 110 ++++++++++++++++++++++++-------------------- 2 files changed, 67 insertions(+), 50 deletions(-) diff --git a/README b/README index 12a27b2..243def2 100644 --- a/README +++ b/README @@ -29,3 +29,10 @@ MessagePack requires gcc currently. So you need to prepare MinGW GCC. $ python setup.py install -c mingw + +TEST +---- +MessagePack uses nosetest for testing. +Run test with following command: + + $ nosetests test diff --git a/test/test_format.py b/test/test_format.py index a00123c..009a764 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -1,64 +1,74 @@ -from unittest import TestCase, main -from msgpack import packs, unpacks +#!/usr/bin/env python +# coding: utf-8 -class TestFormat(TestCase): - def __check(self, obj, expected_packed): - packed = packs(obj) - self.assertEqual(packed, expected_packed) - unpacked = unpacks(packed) - self.assertEqual(unpacked, obj) +from nose import main +from nose.tools import * +from msgpack import unpacks - def testSimpleValues(self): - self.__check(None, '\xc0') - self.__check(True, '\xc3') - self.__check(False, '\xc2') - self.__check( - [None, False, True], - '\x93\xc0\xc2\xc3' - ) +def check(src, should): + assert_equal(unpacks(src), should) - def testFixnum(self): - self.__check( - [[0,64,127], [-32,-16,-1]], - "\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff" - ) - - def testFixArray(self): - self.__check( - [[],[[None]]], - "\x92\x90\x91\x91\xc0" - ) +def testSimpleValue(): + check("\x93\xc0\xc2\xc3", + [None, False, True]) - def testFixRaw(self): - self.__check( - ["", "a", "bc", "def"], - "\x94\xa0\xa1a\xa2bc\xa3def" - ) - pass +def testFixnum(): + check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", + [[0,64,127], [-32,-16,-1]] + ) - def testFixMap(self): - self.__check( - {False: {None: None}, True:{None:{}}}, - "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80" - ) - pass +def testFixArray(): + check("\x92\x90\x91\x91\xc0", + [[],[[None]]], + ) +def testFixRaw(): + check("\x94\xa0\xa1a\xa2bc\xa3def", + ["", "a", "bc", "def"], + ) -class TestUnpack(TestCase): - def __check(self, packed, obj): - self.assertEqual(unpacks(packed), obj) +def testFixMap(): + check( + "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", + {False: {None: None}, True:{None:{}}}, + ) - def testuint(self): - self.__check( - "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" - "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" - "\xce\xff\xff\xff\xff", - [0, 128, 255, 0, 32768, 65535, 0, - 2147483648, 4294967295], - ) +def testUnsignedInt(): + check( + "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" + "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" + "\xce\xff\xff\xff\xff", + [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + ) +def testSignedInt(): + check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" + "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" + "\xd2\xff\xff\xff\xff", + [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) +def testRaw(): + check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" + "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", + ["", "a", "ab", "", "a", "ab"]) +def testArray(): + check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" + "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" + "\xc2\xc3", + [[], [None], [False,True], [], [None], [False,True]]) + +def testMap(): + check( + "\x96" + "\xde\x00\x00" + "\xde\x00\x01\xc0\xc2" + "\xde\x00\x02\xc0\xc2\xc3\xc2" + "\xdf\x00\x00\x00\x00" + "\xdf\x00\x00\x00\x01\xc0\xc2" + "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", + [{}, {None: False}, {True: False, None: False}, {}, + {None: False}, {True: False, None: False}]) if __name__ == '__main__': main() From 257270c1ebc5bb6ac6c60e47130e2a577ffb6714 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 08:23:49 +0900 Subject: [PATCH 0210/1648] Refactor packing code. --- python/msgpack/_msgpack.pyx | 14 +++++++------- python/msgpack/pack.h | 17 ----------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index cde5313..279fdf5 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -31,6 +31,7 @@ cdef extern from "pack.h": void msgpack_pack_nil(msgpack_packer* pk) void msgpack_pack_true(msgpack_packer* pk) void msgpack_pack_false(msgpack_packer* pk) + void msgpack_pack_long(msgpack_packer* pk, long d) void msgpack_pack_long_long(msgpack_packer* pk, long long d) void msgpack_pack_double(msgpack_packer* pk, double d) void msgpack_pack_array(msgpack_packer* pk, size_t l) @@ -60,7 +61,7 @@ cdef class Packer(object): msgpack_packer_init(&self.pk, self, _packer_write) def __del__(self): - free(self.buff); + free(self.buff) def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" @@ -117,7 +118,7 @@ cdef class Packer(object): msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): longval = o - msgpack_pack_long_long(&self.pk, longval) + msgpack_pack_long(&self.pk, longval) elif isinstance(o, float): fval = o msgpack_pack_double(&self.pk, fval) @@ -133,12 +134,12 @@ cdef class Packer(object): elif PyMapping_Check(o): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): - self.pack(k) - self.pack(v) + self.__pack(k) + self.__pack(v) elif PySequence_Check(o): msgpack_pack_array(&self.pk, len(o)) for v in o: - self.pack(v) + self.__pack(v) else: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) @@ -154,7 +155,7 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - if l > 64: + if l > packer.allocated/4: packer.strm.write(PyString_FromStringAndSize(b, l)) packer.length = 0 else: @@ -176,7 +177,6 @@ def packb(object o): buf = StringIO() packer = Packer(buf) packer.pack(o) - packer.flush() return buf.getvalue() packs = packb diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index 9bd6b68..cdac819 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -34,9 +34,6 @@ typedef struct msgpack_packer { static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); -static inline void msgpack_packer_free(msgpack_packer* pk); - static inline int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_int(msgpack_packer* pk, int d); static inline int msgpack_pack_long(msgpack_packer* pk, long d); @@ -90,20 +87,6 @@ static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_p pk->callback = callback; } -static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) -{ - msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); - if(!pk) { return NULL; } - msgpack_packer_init(pk, data, callback); - return pk; -} - -static inline void msgpack_packer_free(msgpack_packer* pk) -{ - free(pk); -} - - #ifdef __cplusplus } #endif From 3a6f6626ebf42aae84bfe9a7803bd8ab81185be4 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 08:23:49 +0900 Subject: [PATCH 0211/1648] Refactor packing code. --- msgpack/_msgpack.pyx | 14 +++++++------- msgpack/pack.h | 17 ----------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index cde5313..279fdf5 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -31,6 +31,7 @@ cdef extern from "pack.h": void msgpack_pack_nil(msgpack_packer* pk) void msgpack_pack_true(msgpack_packer* pk) void msgpack_pack_false(msgpack_packer* pk) + void msgpack_pack_long(msgpack_packer* pk, long d) void msgpack_pack_long_long(msgpack_packer* pk, long long d) void msgpack_pack_double(msgpack_packer* pk, double d) void msgpack_pack_array(msgpack_packer* pk, size_t l) @@ -60,7 +61,7 @@ cdef class Packer(object): msgpack_packer_init(&self.pk, self, _packer_write) def __del__(self): - free(self.buff); + free(self.buff) def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" @@ -117,7 +118,7 @@ cdef class Packer(object): msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): longval = o - msgpack_pack_long_long(&self.pk, longval) + msgpack_pack_long(&self.pk, longval) elif isinstance(o, float): fval = o msgpack_pack_double(&self.pk, fval) @@ -133,12 +134,12 @@ cdef class Packer(object): elif PyMapping_Check(o): msgpack_pack_map(&self.pk, len(o)) for k,v in o.iteritems(): - self.pack(k) - self.pack(v) + self.__pack(k) + self.__pack(v) elif PySequence_Check(o): msgpack_pack_array(&self.pk, len(o)) for v in o: - self.pack(v) + self.__pack(v) else: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) @@ -154,7 +155,7 @@ cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): if packer.length + l > packer.allocated: if packer.length > 0: packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - if l > 64: + if l > packer.allocated/4: packer.strm.write(PyString_FromStringAndSize(b, l)) packer.length = 0 else: @@ -176,7 +177,6 @@ def packb(object o): buf = StringIO() packer = Packer(buf) packer.pack(o) - packer.flush() return buf.getvalue() packs = packb diff --git a/msgpack/pack.h b/msgpack/pack.h index 9bd6b68..cdac819 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -34,9 +34,6 @@ typedef struct msgpack_packer { static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); -static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback); -static inline void msgpack_packer_free(msgpack_packer* pk); - static inline int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_int(msgpack_packer* pk, int d); static inline int msgpack_pack_long(msgpack_packer* pk, long d); @@ -90,20 +87,6 @@ static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_p pk->callback = callback; } -static inline msgpack_packer* msgpack_packer_new(void* data, msgpack_packer_write callback) -{ - msgpack_packer* pk = (msgpack_packer*)calloc(1, sizeof(msgpack_packer)); - if(!pk) { return NULL; } - msgpack_packer_init(pk, data, callback); - return pk; -} - -static inline void msgpack_packer_free(msgpack_packer* pk) -{ - free(pk); -} - - #ifdef __cplusplus } #endif From fe2421275d1e0809a9658dbaa5b31fc535bcf32a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 09:31:43 +0900 Subject: [PATCH 0212/1648] Add some test cases. --- python/test/test_case.py | 101 +++++++++++++++++++++++++++++++++++++++ python/test/test_pack.py | 28 +++++++++++ 2 files changed, 129 insertions(+) create mode 100644 python/test/test_case.py create mode 100644 python/test/test_pack.py diff --git a/python/test/test_case.py b/python/test/test_case.py new file mode 100644 index 0000000..3fafd8b --- /dev/null +++ b/python/test/test_case.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * +from msgpack import packs, unpacks + +def check(length, obj): + v = packs(obj) + assert_equal(len(v), length) + assert_equal(unpacks(v), obj) + +def test_1(): + for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, + -((1<<5)-1), -(1<<5)]: + check(1, o) + +def test_2(): + for o in [1 << 7, (1 << 8) - 1, + -((1<<5)+1), -(1<<7) + ]: + check(2, o) + +def test_3(): + for o in [1 << 8, (1 << 16) - 1, + -((1<<7)+1), -(1<<15)]: + check(3, o) + +def test_5(): + for o in [1 << 16, (1 << 32) - 1, + -((1<<15)+1), -(1<<31)]: + check(5, o) + +def test_9(): + for o in [1 << 32, (1 << 64) - 1, + -((1<<31)+1), -(1<<63), + 1.0, 0.1, -0.1, -1.0]: + check(9, o) + + +def check_raw(overhead, num): + check(num + overhead, " " * num) + +def test_fixraw(): + check_raw(1, 0) + check_raw(1, (1<<5) - 1) + +def test_raw16(): + check_raw(3, 1<<5) + check_raw(3, (1<<16) - 1) + +def test_raw32(): + check_raw(5, 1<<16) + + +def check_array(overhead, num): + check(num + overhead, [None] * num) + +def test_fixarray(): + check_array(1, 0) + check_array(1, (1 << 4) - 1) + +def test_array16(): + check_array(3, 1 << 4) + check_array(3, (1<<16)-1) + +def test_array32(): + check_array(5, (1<<16)) + + +def match(obj, buf): + assert_equal(packs(obj), buf) + assert_equal(unpacks(buf), obj) + +def test_match(): + cases = [ + (None, '\xc0'), + (False, '\xc2'), + (True, '\xc3'), + (0, '\x00'), + (127, '\x7f'), + (128, '\xcc\x80'), + (256, '\xcd\x01\x00'), + (-1, '\xff'), + (-33, '\xd0\xdf'), + (-129, '\xd1\xff\x7f'), + ({1:1}, '\x81\x01\x01'), + (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), + ([], '\x90'), + (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ({}, '\x80'), + (dict([(x,x) for x in range(15)]), "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"), + (dict([(x,x) for x in range(16)]), "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"), + ] + + for v, p in cases: + match(v, p) + +if __name__ == '__main__': + main() diff --git a/python/test/test_pack.py b/python/test/test_pack.py new file mode 100644 index 0000000..86badb5 --- /dev/null +++ b/python/test/test_pack.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * + +from msgpack import packs, unpacks + +def check(data): + re = unpacks(packs(data)) + assert_equal(re, data) + +def testPack(): + test_data = [ + 0, 1, 127, 128, 255, 256, 65535, 65536, + -1, -32, -33, -128, -129, -32768, -32769, + 1.0, + "", "a", "a"*31, "a"*32, + None, True, False, + [], [[]], [[], None], + {None: 0}, + (1<<23), + ] + for td in test_data: + check(td) + +if __name__ == '__main__': + main() From ca0bda01f116f04f98009f13e1bc9def8200cbb6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 09:31:43 +0900 Subject: [PATCH 0213/1648] Add some test cases. --- test/test_case.py | 101 ++++++++++++++++++++++++++++++++++++++++++++++ test/test_pack.py | 28 +++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 test/test_case.py create mode 100644 test/test_pack.py diff --git a/test/test_case.py b/test/test_case.py new file mode 100644 index 0000000..3fafd8b --- /dev/null +++ b/test/test_case.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * +from msgpack import packs, unpacks + +def check(length, obj): + v = packs(obj) + assert_equal(len(v), length) + assert_equal(unpacks(v), obj) + +def test_1(): + for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1, + -((1<<5)-1), -(1<<5)]: + check(1, o) + +def test_2(): + for o in [1 << 7, (1 << 8) - 1, + -((1<<5)+1), -(1<<7) + ]: + check(2, o) + +def test_3(): + for o in [1 << 8, (1 << 16) - 1, + -((1<<7)+1), -(1<<15)]: + check(3, o) + +def test_5(): + for o in [1 << 16, (1 << 32) - 1, + -((1<<15)+1), -(1<<31)]: + check(5, o) + +def test_9(): + for o in [1 << 32, (1 << 64) - 1, + -((1<<31)+1), -(1<<63), + 1.0, 0.1, -0.1, -1.0]: + check(9, o) + + +def check_raw(overhead, num): + check(num + overhead, " " * num) + +def test_fixraw(): + check_raw(1, 0) + check_raw(1, (1<<5) - 1) + +def test_raw16(): + check_raw(3, 1<<5) + check_raw(3, (1<<16) - 1) + +def test_raw32(): + check_raw(5, 1<<16) + + +def check_array(overhead, num): + check(num + overhead, [None] * num) + +def test_fixarray(): + check_array(1, 0) + check_array(1, (1 << 4) - 1) + +def test_array16(): + check_array(3, 1 << 4) + check_array(3, (1<<16)-1) + +def test_array32(): + check_array(5, (1<<16)) + + +def match(obj, buf): + assert_equal(packs(obj), buf) + assert_equal(unpacks(buf), obj) + +def test_match(): + cases = [ + (None, '\xc0'), + (False, '\xc2'), + (True, '\xc3'), + (0, '\x00'), + (127, '\x7f'), + (128, '\xcc\x80'), + (256, '\xcd\x01\x00'), + (-1, '\xff'), + (-33, '\xd0\xdf'), + (-129, '\xd1\xff\x7f'), + ({1:1}, '\x81\x01\x01'), + (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), + ([], '\x90'), + (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ({}, '\x80'), + (dict([(x,x) for x in range(15)]), "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"), + (dict([(x,x) for x in range(16)]), "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"), + ] + + for v, p in cases: + match(v, p) + +if __name__ == '__main__': + main() diff --git a/test/test_pack.py b/test/test_pack.py new file mode 100644 index 0000000..86badb5 --- /dev/null +++ b/test/test_pack.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose import main +from nose.tools import * + +from msgpack import packs, unpacks + +def check(data): + re = unpacks(packs(data)) + assert_equal(re, data) + +def testPack(): + test_data = [ + 0, 1, 127, 128, 255, 256, 65535, 65536, + -1, -32, -33, -128, -129, -32768, -32769, + 1.0, + "", "a", "a"*31, "a"*32, + None, True, False, + [], [[]], [[], None], + {None: 0}, + (1<<23), + ] + for td in test_data: + check(td) + +if __name__ == '__main__': + main() From 9015bd4ecfe7c30aef47d7c3c6fbc6fd4219eae0 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 10:09:04 +0900 Subject: [PATCH 0214/1648] Fix error on packing unsigned long long. --- python/msgpack/_msgpack.pyx | 34 +++++++++++++++++++++------------- python/test/test_case.py | 2 +- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 279fdf5..e1c497b 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -9,6 +9,8 @@ cdef extern from "Python.h": char* PyString_AsString(object o) int PyMapping_Check(object o) int PySequence_Check(object o) + long long PyLong_AsLongLong(object o) + unsigned long long PyLong_AsUnsignedLongLong(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -27,17 +29,18 @@ cdef extern from "pack.h": msgpack_packer_write callback void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) - void msgpack_pack_int(msgpack_packer* pk, int d) - void msgpack_pack_nil(msgpack_packer* pk) - void msgpack_pack_true(msgpack_packer* pk) - void msgpack_pack_false(msgpack_packer* pk) - void msgpack_pack_long(msgpack_packer* pk, long d) - void msgpack_pack_long_long(msgpack_packer* pk, long long d) - void msgpack_pack_double(msgpack_packer* pk, double d) - void msgpack_pack_array(msgpack_packer* pk, size_t l) - void msgpack_pack_map(msgpack_packer* pk, size_t l) - void msgpack_pack_raw(msgpack_packer* pk, size_t l) - void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) + int msgpack_pack_int(msgpack_packer* pk, int d) + int msgpack_pack_nil(msgpack_packer* pk) + int msgpack_pack_true(msgpack_packer* pk) + int msgpack_pack_false(msgpack_packer* pk) + int msgpack_pack_long(msgpack_packer* pk, long d) + int msgpack_pack_long_long(msgpack_packer* pk, long long d) + int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) + int msgpack_pack_double(msgpack_packer* pk, double d) + int msgpack_pack_array(msgpack_packer* pk, size_t l) + int msgpack_pack_map(msgpack_packer* pk, size_t l) + int msgpack_pack_raw(msgpack_packer* pk, size_t l) + int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) cdef class Packer(object): @@ -103,6 +106,7 @@ cdef class Packer(object): cdef __pack(self, object o): cdef long long llval + cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval @@ -114,8 +118,12 @@ cdef class Packer(object): elif o is False: msgpack_pack_false(&self.pk) elif isinstance(o, long): - llval = o - msgpack_pack_long_long(&self.pk, llval) + if o > 0: + ullval = PyLong_AsUnsignedLongLong(o) + msgpack_pack_unsigned_long_long(&self.pk, ullval) + else: + llval = PyLong_AsLongLong(o) + msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): longval = o msgpack_pack_long(&self.pk, longval) diff --git a/python/test/test_case.py b/python/test/test_case.py index 3fafd8b..997027a 100644 --- a/python/test/test_case.py +++ b/python/test/test_case.py @@ -7,7 +7,7 @@ from msgpack import packs, unpacks def check(length, obj): v = packs(obj) - assert_equal(len(v), length) + assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) assert_equal(unpacks(v), obj) def test_1(): From f46498bcdfc862f9073ea7f520514cd722b03616 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 10:09:04 +0900 Subject: [PATCH 0215/1648] Fix error on packing unsigned long long. --- msgpack/_msgpack.pyx | 34 +++++++++++++++++++++------------- test/test_case.py | 2 +- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 279fdf5..e1c497b 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -9,6 +9,8 @@ cdef extern from "Python.h": char* PyString_AsString(object o) int PyMapping_Check(object o) int PySequence_Check(object o) + long long PyLong_AsLongLong(object o) + unsigned long long PyLong_AsUnsignedLongLong(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -27,17 +29,18 @@ cdef extern from "pack.h": msgpack_packer_write callback void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) - void msgpack_pack_int(msgpack_packer* pk, int d) - void msgpack_pack_nil(msgpack_packer* pk) - void msgpack_pack_true(msgpack_packer* pk) - void msgpack_pack_false(msgpack_packer* pk) - void msgpack_pack_long(msgpack_packer* pk, long d) - void msgpack_pack_long_long(msgpack_packer* pk, long long d) - void msgpack_pack_double(msgpack_packer* pk, double d) - void msgpack_pack_array(msgpack_packer* pk, size_t l) - void msgpack_pack_map(msgpack_packer* pk, size_t l) - void msgpack_pack_raw(msgpack_packer* pk, size_t l) - void msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) + int msgpack_pack_int(msgpack_packer* pk, int d) + int msgpack_pack_nil(msgpack_packer* pk) + int msgpack_pack_true(msgpack_packer* pk) + int msgpack_pack_false(msgpack_packer* pk) + int msgpack_pack_long(msgpack_packer* pk, long d) + int msgpack_pack_long_long(msgpack_packer* pk, long long d) + int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) + int msgpack_pack_double(msgpack_packer* pk, double d) + int msgpack_pack_array(msgpack_packer* pk, size_t l) + int msgpack_pack_map(msgpack_packer* pk, size_t l) + int msgpack_pack_raw(msgpack_packer* pk, size_t l) + int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) cdef class Packer(object): @@ -103,6 +106,7 @@ cdef class Packer(object): cdef __pack(self, object o): cdef long long llval + cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval @@ -114,8 +118,12 @@ cdef class Packer(object): elif o is False: msgpack_pack_false(&self.pk) elif isinstance(o, long): - llval = o - msgpack_pack_long_long(&self.pk, llval) + if o > 0: + ullval = PyLong_AsUnsignedLongLong(o) + msgpack_pack_unsigned_long_long(&self.pk, ullval) + else: + llval = PyLong_AsLongLong(o) + msgpack_pack_long_long(&self.pk, llval) elif isinstance(o, int): longval = o msgpack_pack_long(&self.pk, longval) diff --git a/test/test_case.py b/test/test_case.py index 3fafd8b..997027a 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -7,7 +7,7 @@ from msgpack import packs, unpacks def check(length, obj): v = packs(obj) - assert_equal(len(v), length) + assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) assert_equal(unpacks(v), obj) def test_1(): From b5010c71a9810515db1aa82d2e7eebf2ee4f474f Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 11:21:28 +0900 Subject: [PATCH 0216/1648] Fix tests. --- python/Makefile | 7 +++++++ python/test/test_case.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 python/Makefile diff --git a/python/Makefile b/python/Makefile new file mode 100644 index 0000000..5f16fae --- /dev/null +++ b/python/Makefile @@ -0,0 +1,7 @@ +all: + python setup.py build + python setup.py sdist + +.PHONY: test +test: + nosetests test diff --git a/python/test/test_case.py b/python/test/test_case.py index 997027a..d754fb0 100644 --- a/python/test/test_case.py +++ b/python/test/test_case.py @@ -90,8 +90,8 @@ def test_match(): (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), ({}, '\x80'), - (dict([(x,x) for x in range(15)]), "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"), - (dict([(x,x) for x in range(16)]), "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"), + (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), + (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), ] for v, p in cases: From 6201e623e29b4162701a9ccdce07af893baaaa5e Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 29 Jun 2009 11:21:28 +0900 Subject: [PATCH 0217/1648] Fix tests. --- Makefile | 7 +++++++ test/test_case.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5f16fae --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +all: + python setup.py build + python setup.py sdist + +.PHONY: test +test: + nosetests test diff --git a/test/test_case.py b/test/test_case.py index 997027a..d754fb0 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -90,8 +90,8 @@ def test_match(): (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), ({}, '\x80'), - (dict([(x,x) for x in range(15)]), "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"), - (dict([(x,x) for x in range(16)]), "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"), + (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), + (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), ] for v, p in cases: From d4317fdc853c7dbe3bc57718dc009cb8b0a2b351 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 30 Jun 2009 23:03:33 +0900 Subject: [PATCH 0218/1648] Some optimization on packing. --- python/Makefile | 1 + python/msgpack/_msgpack.pyx | 69 +++++++++++++------------------------ python/msgpack/pack.h | 29 +++++++++------- 3 files changed, 40 insertions(+), 59 deletions(-) diff --git a/python/Makefile b/python/Makefile index 5f16fae..d90789a 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,4 +1,5 @@ all: + python setup.py build_ext -i -f python setup.py build python setup.py sdist diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index e1c497b..26222aa 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -1,16 +1,23 @@ # coding: utf-8 -from cStringIO import StringIO +import cStringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + PyObject* Py_True + PyObject* Py_False char* PyString_AsString(object o) int PyMapping_Check(object o) int PySequence_Check(object o) long long PyLong_AsLongLong(object o) unsigned long long PyLong_AsUnsignedLongLong(object o) + int PyLong_Check(object o) + int PyInt_Check(object o) + int PyFloat_Check(object o) + int PyString_Check(object o) + int PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -22,13 +29,9 @@ cdef extern from "string.h": void* memmove(char* dst, char* src, size_t size) cdef extern from "pack.h": - ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) - struct msgpack_packer: - void *data - msgpack_packer_write callback + PyObject* writer - void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) int msgpack_pack_true(msgpack_packer* pk) @@ -49,28 +52,17 @@ cdef class Packer(object): strm must have `write(bytes)` method. size specifies local buffer size. """ - cdef char* buff - cdef unsigned int length - cdef unsigned int allocated cdef msgpack_packer pk cdef object strm + cdef object writer - def __init__(self, strm, int size=4*1024): - self.strm = strm - self.buff = malloc(size) - self.allocated = size - self.length = 0 - - msgpack_packer_init(&self.pk, self, _packer_write) - - def __del__(self): - free(self.buff) + def __init__(self, strm_, int size=4*1024): + self.strm = strm_ + self.writer = strm_.write + self.pk.writer = self.writer def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" - if self.length > 0: - self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - self.length = 0 if hasattr(self.strm, 'flush'): self.strm.flush() @@ -113,28 +105,28 @@ cdef class Packer(object): if o is None: msgpack_pack_nil(&self.pk) - elif o is True: + elif o == Py_True: msgpack_pack_true(&self.pk) - elif o is False: + elif o == Py_False: msgpack_pack_false(&self.pk) - elif isinstance(o, long): + elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) msgpack_pack_unsigned_long_long(&self.pk, ullval) else: llval = PyLong_AsLongLong(o) msgpack_pack_long_long(&self.pk, llval) - elif isinstance(o, int): + elif PyInt_Check(o): longval = o msgpack_pack_long(&self.pk, longval) - elif isinstance(o, float): + elif PyFloat_Check(o): fval = o msgpack_pack_double(&self.pk, fval) - elif isinstance(o, str): + elif PyString_Check(o): rawval = o msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif isinstance(o, unicode): + elif PyUnicode_Check(o): o = o.encode('utf-8') rawval = o msgpack_pack_raw(&self.pk, len(o)) @@ -152,28 +144,13 @@ cdef class Packer(object): # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) - def pack(self, obj, flush=True): + def pack(self, object obj, flush=True): self.__pack(obj) if flush: self.flush() close = flush -cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - if packer.length + l > packer.allocated: - if packer.length > 0: - packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - if l > packer.allocated/4: - packer.strm.write(PyString_FromStringAndSize(b, l)) - packer.length = 0 - else: - memcpy(packer.buff, b, l) - packer.length = l - else: - memcpy(packer.buff + packer.length, b, l) - packer.length += l - return 0 - def pack(object o, object stream): u"""pack o and write to stream).""" packer = Packer(stream) @@ -182,7 +159,7 @@ def pack(object o, object stream): def packb(object o): u"""pack o and return packed bytes.""" - buf = StringIO() + buf = cStringIO.StringIO() packer = Packer(buf) packer.pack(o) return buf.getvalue() diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index cdac819..ea97601 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -24,15 +24,11 @@ extern "C" { #endif - -typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); - typedef struct msgpack_packer { - void* data; - msgpack_packer_write callback; + PyObject* writer; } msgpack_packer; -static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); +typedef struct Packer Packer; static inline int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_int(msgpack_packer* pk, int d); @@ -66,7 +62,20 @@ static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); +static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) +{ + PyObject *buf, *ret; + buf = PyBuffer_FromMemory((void*)data, l); + //buf = PyString_FromStringAndSize(data, l); + if (buf == NULL) return -1; + + ret = PyObject_CallFunctionObjArgs(pk->writer, buf, NULL); + Py_DECREF(buf); + if (ret == NULL) return -1; + Py_DECREF(ret); + return 0; +} #define msgpack_pack_inline_func(name) \ static inline int msgpack_pack ## name @@ -77,16 +86,10 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ #define msgpack_pack_user msgpack_packer* #define msgpack_pack_append_buffer(user, buf, len) \ - return (*(user)->callback)((user)->data, (const char*)buf, len) + return msgpack_pack_write(user, (const char*)buf, len) #include "pack_template.h" -static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) -{ - pk->data = data; - pk->callback = callback; -} - #ifdef __cplusplus } #endif From f7b3ef18b0ca44a14ba39087f04cad14433542a0 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Tue, 30 Jun 2009 23:03:33 +0900 Subject: [PATCH 0219/1648] Some optimization on packing. --- Makefile | 1 + msgpack/_msgpack.pyx | 69 +++++++++++++++----------------------------- msgpack/pack.h | 29 ++++++++++--------- 3 files changed, 40 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index 5f16fae..d90789a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ all: + python setup.py build_ext -i -f python setup.py build python setup.py sdist diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index e1c497b..26222aa 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -1,16 +1,23 @@ # coding: utf-8 -from cStringIO import StringIO +import cStringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) + PyObject* Py_True + PyObject* Py_False char* PyString_AsString(object o) int PyMapping_Check(object o) int PySequence_Check(object o) long long PyLong_AsLongLong(object o) unsigned long long PyLong_AsUnsignedLongLong(object o) + int PyLong_Check(object o) + int PyInt_Check(object o) + int PyFloat_Check(object o) + int PyString_Check(object o) + int PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -22,13 +29,9 @@ cdef extern from "string.h": void* memmove(char* dst, char* src, size_t size) cdef extern from "pack.h": - ctypedef int (*msgpack_packer_write)(void* data, const_char_ptr buf, unsigned int len) - struct msgpack_packer: - void *data - msgpack_packer_write callback + PyObject* writer - void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) int msgpack_pack_true(msgpack_packer* pk) @@ -49,28 +52,17 @@ cdef class Packer(object): strm must have `write(bytes)` method. size specifies local buffer size. """ - cdef char* buff - cdef unsigned int length - cdef unsigned int allocated cdef msgpack_packer pk cdef object strm + cdef object writer - def __init__(self, strm, int size=4*1024): - self.strm = strm - self.buff = malloc(size) - self.allocated = size - self.length = 0 - - msgpack_packer_init(&self.pk, self, _packer_write) - - def __del__(self): - free(self.buff) + def __init__(self, strm_, int size=4*1024): + self.strm = strm_ + self.writer = strm_.write + self.pk.writer = self.writer def flush(self): """Flash local buffer and output stream if it has 'flush()' method.""" - if self.length > 0: - self.strm.write(PyString_FromStringAndSize(self.buff, self.length)) - self.length = 0 if hasattr(self.strm, 'flush'): self.strm.flush() @@ -113,28 +105,28 @@ cdef class Packer(object): if o is None: msgpack_pack_nil(&self.pk) - elif o is True: + elif o == Py_True: msgpack_pack_true(&self.pk) - elif o is False: + elif o == Py_False: msgpack_pack_false(&self.pk) - elif isinstance(o, long): + elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) msgpack_pack_unsigned_long_long(&self.pk, ullval) else: llval = PyLong_AsLongLong(o) msgpack_pack_long_long(&self.pk, llval) - elif isinstance(o, int): + elif PyInt_Check(o): longval = o msgpack_pack_long(&self.pk, longval) - elif isinstance(o, float): + elif PyFloat_Check(o): fval = o msgpack_pack_double(&self.pk, fval) - elif isinstance(o, str): + elif PyString_Check(o): rawval = o msgpack_pack_raw(&self.pk, len(o)) msgpack_pack_raw_body(&self.pk, rawval, len(o)) - elif isinstance(o, unicode): + elif PyUnicode_Check(o): o = o.encode('utf-8') rawval = o msgpack_pack_raw(&self.pk, len(o)) @@ -152,28 +144,13 @@ cdef class Packer(object): # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) - def pack(self, obj, flush=True): + def pack(self, object obj, flush=True): self.__pack(obj) if flush: self.flush() close = flush -cdef int _packer_write(Packer packer, const_char_ptr b, unsigned int l): - if packer.length + l > packer.allocated: - if packer.length > 0: - packer.strm.write(PyString_FromStringAndSize(packer.buff, packer.length)) - if l > packer.allocated/4: - packer.strm.write(PyString_FromStringAndSize(b, l)) - packer.length = 0 - else: - memcpy(packer.buff, b, l) - packer.length = l - else: - memcpy(packer.buff + packer.length, b, l) - packer.length += l - return 0 - def pack(object o, object stream): u"""pack o and write to stream).""" packer = Packer(stream) @@ -182,7 +159,7 @@ def pack(object o, object stream): def packb(object o): u"""pack o and return packed bytes.""" - buf = StringIO() + buf = cStringIO.StringIO() packer = Packer(buf) packer.pack(o) return buf.getvalue() diff --git a/msgpack/pack.h b/msgpack/pack.h index cdac819..ea97601 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -24,15 +24,11 @@ extern "C" { #endif - -typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len); - typedef struct msgpack_packer { - void* data; - msgpack_packer_write callback; + PyObject* writer; } msgpack_packer; -static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback); +typedef struct Packer Packer; static inline int msgpack_pack_short(msgpack_packer* pk, short d); static inline int msgpack_pack_int(msgpack_packer* pk, int d); @@ -66,7 +62,20 @@ static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n); static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l); static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l); +static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) +{ + PyObject *buf, *ret; + buf = PyBuffer_FromMemory((void*)data, l); + //buf = PyString_FromStringAndSize(data, l); + if (buf == NULL) return -1; + + ret = PyObject_CallFunctionObjArgs(pk->writer, buf, NULL); + Py_DECREF(buf); + if (ret == NULL) return -1; + Py_DECREF(ret); + return 0; +} #define msgpack_pack_inline_func(name) \ static inline int msgpack_pack ## name @@ -77,16 +86,10 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ #define msgpack_pack_user msgpack_packer* #define msgpack_pack_append_buffer(user, buf, len) \ - return (*(user)->callback)((user)->data, (const char*)buf, len) + return msgpack_pack_write(user, (const char*)buf, len) #include "pack_template.h" -static inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback) -{ - pk->data = data; - pk->callback = callback; -} - #ifdef __cplusplus } #endif From 03942a1b9020b00bf47e93b7d5ec606e8160c054 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 1 Jul 2009 00:57:46 +0900 Subject: [PATCH 0220/1648] Major speedup on packing. --- python/msgpack/_msgpack.pyx | 60 +++++++++++++------------------------ python/msgpack/pack.h | 25 ++++++++++------ 2 files changed, 37 insertions(+), 48 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 26222aa..9d766c5 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -18,6 +18,7 @@ cdef extern from "Python.h": int PyFloat_Check(object o) int PyString_Check(object o) int PyUnicode_Check(object o) + object PyBuffer_FromMemory(const_char_ptr b, Py_ssize_t len) cdef extern from "stdlib.h": void* malloc(size_t) @@ -30,7 +31,9 @@ cdef extern from "string.h": cdef extern from "pack.h": struct msgpack_packer: - PyObject* writer + char* buf + size_t length + size_t buf_size int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) @@ -56,45 +59,16 @@ cdef class Packer(object): cdef object strm cdef object writer - def __init__(self, strm_, int size=4*1024): - self.strm = strm_ - self.writer = strm_.write - self.pk.writer = self.writer + def __init__(self, strm): + self.strm = strm - def flush(self): - """Flash local buffer and output stream if it has 'flush()' method.""" - if hasattr(self.strm, 'flush'): - self.strm.flush() + cdef int buf_size = 1024*1024 + self.pk.buf = malloc(buf_size); + self.pk.buf_size = buf_size + self.pk.length = 0 - def pack_list(self, len): - """Start packing sequential objects. - - Example: - - packer.pack_list(2) - packer.pack('foo') - packer.pack('bar') - - This is same to: - - packer.pack(['foo', 'bar']) - """ - msgpack_pack_array(&self.pk, len) - - def pack_dict(self, len): - """Start packing key-value objects. - - Example: - - packer.pack_dict(1) - packer.pack('foo') - packer.pack('bar') - - This is same to: - - packer.pack({'foo': 'bar'}) - """ - msgpack_pack_map(&self.pk, len) + def __del__(self): + free(self.pk.buf); cdef __pack(self, object o): cdef long long llval @@ -144,11 +118,19 @@ cdef class Packer(object): # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) - def pack(self, object obj, flush=True): + def pack(self, object obj, object flush=True): self.__pack(obj) + buf = PyBuffer_FromMemory(self.pk.buf, self.pk.length) + self.pk.length = 0 + self.strm.write(buf) if flush: self.flush() + def flush(self): + """Flash local buffer and output stream if it has 'flush()' method.""" + if hasattr(self.strm, 'flush'): + self.strm.flush() + close = flush def pack(object o, object stream): diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index ea97601..d7e0867 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -25,7 +25,9 @@ extern "C" { #endif typedef struct msgpack_packer { - PyObject* writer; + char *buf; + size_t length; + size_t buf_size; } msgpack_packer; typedef struct Packer Packer; @@ -64,16 +66,21 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) { - PyObject *buf, *ret; + char* buf = pk->buf; + size_t bs = pk->buf_size; + size_t len = pk->length; - buf = PyBuffer_FromMemory((void*)data, l); - //buf = PyString_FromStringAndSize(data, l); - if (buf == NULL) return -1; + if (len + l > bs) { + bs = (len + l) * 2; + buf = realloc(pk->buf, bs); + if (!buf) return -1; + } + memcpy(buf + len, data, l); + len += l; - ret = PyObject_CallFunctionObjArgs(pk->writer, buf, NULL); - Py_DECREF(buf); - if (ret == NULL) return -1; - Py_DECREF(ret); + pk->buf = buf; + pk->buf_size = bs; + pk->length = len; return 0; } From 17cbfa66ae0d6171b0e29699459a3dfbf30e06f2 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 1 Jul 2009 00:57:46 +0900 Subject: [PATCH 0221/1648] Major speedup on packing. --- msgpack/_msgpack.pyx | 60 ++++++++++++++++---------------------------- msgpack/pack.h | 25 +++++++++++------- 2 files changed, 37 insertions(+), 48 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 26222aa..9d766c5 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -18,6 +18,7 @@ cdef extern from "Python.h": int PyFloat_Check(object o) int PyString_Check(object o) int PyUnicode_Check(object o) + object PyBuffer_FromMemory(const_char_ptr b, Py_ssize_t len) cdef extern from "stdlib.h": void* malloc(size_t) @@ -30,7 +31,9 @@ cdef extern from "string.h": cdef extern from "pack.h": struct msgpack_packer: - PyObject* writer + char* buf + size_t length + size_t buf_size int msgpack_pack_int(msgpack_packer* pk, int d) int msgpack_pack_nil(msgpack_packer* pk) @@ -56,45 +59,16 @@ cdef class Packer(object): cdef object strm cdef object writer - def __init__(self, strm_, int size=4*1024): - self.strm = strm_ - self.writer = strm_.write - self.pk.writer = self.writer + def __init__(self, strm): + self.strm = strm - def flush(self): - """Flash local buffer and output stream if it has 'flush()' method.""" - if hasattr(self.strm, 'flush'): - self.strm.flush() + cdef int buf_size = 1024*1024 + self.pk.buf = malloc(buf_size); + self.pk.buf_size = buf_size + self.pk.length = 0 - def pack_list(self, len): - """Start packing sequential objects. - - Example: - - packer.pack_list(2) - packer.pack('foo') - packer.pack('bar') - - This is same to: - - packer.pack(['foo', 'bar']) - """ - msgpack_pack_array(&self.pk, len) - - def pack_dict(self, len): - """Start packing key-value objects. - - Example: - - packer.pack_dict(1) - packer.pack('foo') - packer.pack('bar') - - This is same to: - - packer.pack({'foo': 'bar'}) - """ - msgpack_pack_map(&self.pk, len) + def __del__(self): + free(self.pk.buf); cdef __pack(self, object o): cdef long long llval @@ -144,11 +118,19 @@ cdef class Packer(object): # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) - def pack(self, object obj, flush=True): + def pack(self, object obj, object flush=True): self.__pack(obj) + buf = PyBuffer_FromMemory(self.pk.buf, self.pk.length) + self.pk.length = 0 + self.strm.write(buf) if flush: self.flush() + def flush(self): + """Flash local buffer and output stream if it has 'flush()' method.""" + if hasattr(self.strm, 'flush'): + self.strm.flush() + close = flush def pack(object o, object stream): diff --git a/msgpack/pack.h b/msgpack/pack.h index ea97601..d7e0867 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -25,7 +25,9 @@ extern "C" { #endif typedef struct msgpack_packer { - PyObject* writer; + char *buf; + size_t length; + size_t buf_size; } msgpack_packer; typedef struct Packer Packer; @@ -64,16 +66,21 @@ static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l) { - PyObject *buf, *ret; + char* buf = pk->buf; + size_t bs = pk->buf_size; + size_t len = pk->length; - buf = PyBuffer_FromMemory((void*)data, l); - //buf = PyString_FromStringAndSize(data, l); - if (buf == NULL) return -1; + if (len + l > bs) { + bs = (len + l) * 2; + buf = realloc(pk->buf, bs); + if (!buf) return -1; + } + memcpy(buf + len, data, l); + len += l; - ret = PyObject_CallFunctionObjArgs(pk->writer, buf, NULL); - Py_DECREF(buf); - if (ret == NULL) return -1; - Py_DECREF(ret); + pk->buf = buf; + pk->buf_size = bs; + pk->length = len; return 0; } From a95e8c790c11927708f29a1640492530955369f7 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 17:45:18 +0900 Subject: [PATCH 0222/1648] this module requires perl 5.8 --- perl/lib/Data/MessagePack.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index d26bf9e..7b5604d 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -2,6 +2,7 @@ package Data::MessagePack; use strict; use warnings; use XSLoader; +use 5.008001; our $VERSION = 0.01; From c6496ddf1334d4dc037cff390c5c227c050bf20d Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:08:39 +0900 Subject: [PATCH 0223/1648] added .gitignore file --- perl/.gitignore | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 perl/.gitignore diff --git a/perl/.gitignore b/perl/.gitignore new file mode 100644 index 0000000..53c9b78 --- /dev/null +++ b/perl/.gitignore @@ -0,0 +1,11 @@ +META.yml +Makefile +Makefile.old +MessagePack.bs +MessagePack.o +blib/ +inc/ +msgpack/ +pack.o +pm_to_blib +unpack.o From 37972dfe3ea0adbd9c5f9ffd8898ab3274ec72ba Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:20:14 +0900 Subject: [PATCH 0224/1648] copy files into this dir. --- perl/Makefile.PL | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index f60e125..167bebe 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -6,7 +6,7 @@ perl_version '5.008005'; license 'perl'; can_cc or die "This module requires a C compiler"; -my $ccflags = '-I../ '; +my $ccflags = '.'; makemaker_args( OBJECT => '$(O_FILES)', @@ -26,6 +26,15 @@ makemaker_args( tests 't/*.t'; author_tests('xt'); +# copy modules +if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) { + mkdir 'msgpack' unless -d 'msgpack'; + require File::Copy; + for my $src (<../msgpack/*.h>) { + File::Copy::copy($src, 'msgpack/') or die "copy failed: $!"; + } +} + auto_set_repository; build_requires 'Test::More'; use_test_base; From d72bef9f9ac7d26e54e0778062d575e887ff3df4 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:33:56 +0900 Subject: [PATCH 0225/1648] added MANIFEST.SKIP files --- perl/MANIFEST.SKIP | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 perl/MANIFEST.SKIP diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP new file mode 100644 index 0000000..366a1e8 --- /dev/null +++ b/perl/MANIFEST.SKIP @@ -0,0 +1,21 @@ +\bRCS\b +\bCVS\b +^MANIFEST\. +^Makefile$ +~$ +^# +\.old$ +^blib/ +^pm_to_blib +^MakeMaker-\d +\.gz$ +\.cvsignore +^t/9\d_.*\.t +^t/perlcritic +^tools/ +\.svn/ +^[^/]+\.yaml$ +^[^/]+\.pl$ +^\.shipit$ +^\.git/ +\.sw[pon]$ From d86b90e66344637a31b9def7c8838dd5dd9ec992 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:34:11 +0900 Subject: [PATCH 0226/1648] added Changes file --- perl/Changes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 perl/Changes diff --git a/perl/Changes b/perl/Changes new file mode 100644 index 0000000..18041d4 --- /dev/null +++ b/perl/Changes @@ -0,0 +1,3 @@ +0.01 + + - initial release to CPAN From 7c276241610ac7a2920e399346a7690038adaed9 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:34:27 +0900 Subject: [PATCH 0227/1648] added .shipit file --- perl/.shipit | 1 + 1 file changed, 1 insertion(+) create mode 100644 perl/.shipit diff --git a/perl/.shipit b/perl/.shipit new file mode 100644 index 0000000..8731dce --- /dev/null +++ b/perl/.shipit @@ -0,0 +1 @@ +steps = FindVersion, ChangeVersion, CheckChangeLog, DistTest, Commit, Tag, MakeDist, UploadCPAN From c2a63b2c54f49c2ab92c5247aa76c02414404b7c Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 1 Jul 2009 18:40:14 +0900 Subject: [PATCH 0228/1648] Checking in changes prior to tagging of version 0.01. Changelog diff is: --- perl/.gitignore | 1 + perl/.shipit | 1 + perl/MANIFEST.SKIP | 1 + perl/lib/Data/MessagePack.pm | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/perl/.gitignore b/perl/.gitignore index 53c9b78..a9096b6 100644 --- a/perl/.gitignore +++ b/perl/.gitignore @@ -9,3 +9,4 @@ msgpack/ pack.o pm_to_blib unpack.o +MANIFEST diff --git a/perl/.shipit b/perl/.shipit index 8731dce..f9997c8 100644 --- a/perl/.shipit +++ b/perl/.shipit @@ -1 +1,2 @@ steps = FindVersion, ChangeVersion, CheckChangeLog, DistTest, Commit, Tag, MakeDist, UploadCPAN +git.tagpattern = perl-%v diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index 366a1e8..1b268ef 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -19,3 +19,4 @@ ^\.shipit$ ^\.git/ \.sw[pon]$ +^\.gitignore$ diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 7b5604d..cb3f73a 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = 0.01; +our $VERSION = '0.01'; XSLoader::load(__PACKAGE__, $VERSION); From 78db826a75d9f091f2175ace5f2cd9cb81c0f115 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 1 Jul 2009 20:55:24 +0900 Subject: [PATCH 0229/1648] Fix memory leak. Remove stream packing feature. Add errorcheck in packing. --- python/msgpack/_msgpack.pyx | 135 +++++++++++++++++++----------------- python/msgpack/pack.h | 2 +- 2 files changed, 73 insertions(+), 64 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 9d766c5..c455394 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -5,20 +5,22 @@ import cStringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject + cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) - PyObject* Py_True - PyObject* Py_False - char* PyString_AsString(object o) - int PyMapping_Check(object o) - int PySequence_Check(object o) - long long PyLong_AsLongLong(object o) - unsigned long long PyLong_AsUnsignedLongLong(object o) - int PyLong_Check(object o) - int PyInt_Check(object o) - int PyFloat_Check(object o) - int PyString_Check(object o) - int PyUnicode_Check(object o) - object PyBuffer_FromMemory(const_char_ptr b, Py_ssize_t len) + cdef PyObject* Py_True + cdef PyObject* Py_False + + cdef char* PyString_AsString(object o) + cdef long long PyLong_AsLongLong(object o) + cdef unsigned long long PyLong_AsUnsignedLongLong(object o) + + cdef int PyMapping_Check(object o) + cdef int PySequence_Check(object o) + cdef int PyLong_Check(object o) + cdef int PyInt_Check(object o) + cdef int PyFloat_Check(object o) + cdef int PyString_Check(object o) + cdef int PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -50,101 +52,101 @@ cdef extern from "pack.h": cdef class Packer(object): - """Packer that pack data into strm. + """MessagePack Packer + + usage: - strm must have `write(bytes)` method. - size specifies local buffer size. + packer = Packer() + astream.write(packer.pack(a)) + astream.write(packer.pack(b)) """ cdef msgpack_packer pk - cdef object strm - cdef object writer - - def __init__(self, strm): - self.strm = strm + def __cinit__(self): cdef int buf_size = 1024*1024 self.pk.buf = malloc(buf_size); self.pk.buf_size = buf_size self.pk.length = 0 - def __del__(self): + def __dealloc__(self): free(self.pk.buf); - cdef __pack(self, object o): + cdef int __pack(self, object o): cdef long long llval cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval + cdef int ret if o is None: - msgpack_pack_nil(&self.pk) + ret = msgpack_pack_nil(&self.pk) elif o == Py_True: - msgpack_pack_true(&self.pk) + ret = msgpack_pack_true(&self.pk) elif o == Py_False: - msgpack_pack_false(&self.pk) + ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) - msgpack_pack_unsigned_long_long(&self.pk, ullval) + ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) else: llval = PyLong_AsLongLong(o) - msgpack_pack_long_long(&self.pk, llval) + ret = msgpack_pack_long_long(&self.pk, llval) elif PyInt_Check(o): longval = o - msgpack_pack_long(&self.pk, longval) + ret = msgpack_pack_long(&self.pk, longval) elif PyFloat_Check(o): fval = o - msgpack_pack_double(&self.pk, fval) + ret = msgpack_pack_double(&self.pk, fval) elif PyString_Check(o): rawval = o - msgpack_pack_raw(&self.pk, len(o)) - msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw(&self.pk, len(o)) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyUnicode_Check(o): o = o.encode('utf-8') rawval = o - msgpack_pack_raw(&self.pk, len(o)) - msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw(&self.pk, len(o)) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyMapping_Check(o): - msgpack_pack_map(&self.pk, len(o)) - for k,v in o.iteritems(): - self.__pack(k) - self.__pack(v) + ret = msgpack_pack_map(&self.pk, len(o)) + if ret == 0: + for k,v in o.iteritems(): + ret = self.__pack(k) + if ret != 0: break + ret = self.__pack(v) + if ret != 0: break elif PySequence_Check(o): - msgpack_pack_array(&self.pk, len(o)) - for v in o: - self.__pack(v) + ret = msgpack_pack_array(&self.pk, len(o)) + if ret == 0: + for v in o: + ret = self.__pack(v) + if ret != 0: break else: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) + return ret - def pack(self, object obj, object flush=True): - self.__pack(obj) - buf = PyBuffer_FromMemory(self.pk.buf, self.pk.length) + def pack(self, object obj): + cdef int ret + ret = self.__pack(obj) + if ret: + raise TypeError + buf = PyString_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 - self.strm.write(buf) - if flush: - self.flush() + return buf - def flush(self): - """Flash local buffer and output stream if it has 'flush()' method.""" - if hasattr(self.strm, 'flush'): - self.strm.flush() - - close = flush def pack(object o, object stream): - u"""pack o and write to stream).""" - packer = Packer(stream) - packer.pack(o) - packer.flush() + """pack a object `o` and write it to stream).""" + packer = Packer() + stream.write(packer.pack(o)) def packb(object o): - u"""pack o and return packed bytes.""" - buf = cStringIO.StringIO() - packer = Packer(buf) - packer.pack(o) - return buf.getvalue() + """pack o and return packed bytes.""" + packer = Packer() + return packer.pack(o) packs = packb @@ -222,7 +224,14 @@ cdef class Unpacker(object): cdef int read_size cdef object waiting_bytes - def __init__(self, file_like=None, int read_size=4096): + def __cinit__(self): + self.buf = NULL + + def __dealloc__(self): + if self.buf: + free(self.buf); + + def __init__(self, file_like=None, int read_size=1024*1024): self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index d7e0867..58f021e 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -72,7 +72,7 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ if (len + l > bs) { bs = (len + l) * 2; - buf = realloc(pk->buf, bs); + buf = realloc(buf, bs); if (!buf) return -1; } memcpy(buf + len, data, l); From 2f095515735b136d04d7f05713505e97a3d25aa6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Wed, 1 Jul 2009 20:55:24 +0900 Subject: [PATCH 0230/1648] Fix memory leak. Remove stream packing feature. Add errorcheck in packing. --- msgpack/_msgpack.pyx | 135 +++++++++++++++++++++++-------------------- msgpack/pack.h | 2 +- 2 files changed, 73 insertions(+), 64 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 9d766c5..c455394 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -5,20 +5,22 @@ import cStringIO cdef extern from "Python.h": ctypedef char* const_char_ptr "const char*" ctypedef struct PyObject + cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len) - PyObject* Py_True - PyObject* Py_False - char* PyString_AsString(object o) - int PyMapping_Check(object o) - int PySequence_Check(object o) - long long PyLong_AsLongLong(object o) - unsigned long long PyLong_AsUnsignedLongLong(object o) - int PyLong_Check(object o) - int PyInt_Check(object o) - int PyFloat_Check(object o) - int PyString_Check(object o) - int PyUnicode_Check(object o) - object PyBuffer_FromMemory(const_char_ptr b, Py_ssize_t len) + cdef PyObject* Py_True + cdef PyObject* Py_False + + cdef char* PyString_AsString(object o) + cdef long long PyLong_AsLongLong(object o) + cdef unsigned long long PyLong_AsUnsignedLongLong(object o) + + cdef int PyMapping_Check(object o) + cdef int PySequence_Check(object o) + cdef int PyLong_Check(object o) + cdef int PyInt_Check(object o) + cdef int PyFloat_Check(object o) + cdef int PyString_Check(object o) + cdef int PyUnicode_Check(object o) cdef extern from "stdlib.h": void* malloc(size_t) @@ -50,101 +52,101 @@ cdef extern from "pack.h": cdef class Packer(object): - """Packer that pack data into strm. + """MessagePack Packer + + usage: - strm must have `write(bytes)` method. - size specifies local buffer size. + packer = Packer() + astream.write(packer.pack(a)) + astream.write(packer.pack(b)) """ cdef msgpack_packer pk - cdef object strm - cdef object writer - - def __init__(self, strm): - self.strm = strm + def __cinit__(self): cdef int buf_size = 1024*1024 self.pk.buf = malloc(buf_size); self.pk.buf_size = buf_size self.pk.length = 0 - def __del__(self): + def __dealloc__(self): free(self.pk.buf); - cdef __pack(self, object o): + cdef int __pack(self, object o): cdef long long llval cdef unsigned long long ullval cdef long longval cdef double fval cdef char* rawval + cdef int ret if o is None: - msgpack_pack_nil(&self.pk) + ret = msgpack_pack_nil(&self.pk) elif o == Py_True: - msgpack_pack_true(&self.pk) + ret = msgpack_pack_true(&self.pk) elif o == Py_False: - msgpack_pack_false(&self.pk) + ret = msgpack_pack_false(&self.pk) elif PyLong_Check(o): if o > 0: ullval = PyLong_AsUnsignedLongLong(o) - msgpack_pack_unsigned_long_long(&self.pk, ullval) + ret = msgpack_pack_unsigned_long_long(&self.pk, ullval) else: llval = PyLong_AsLongLong(o) - msgpack_pack_long_long(&self.pk, llval) + ret = msgpack_pack_long_long(&self.pk, llval) elif PyInt_Check(o): longval = o - msgpack_pack_long(&self.pk, longval) + ret = msgpack_pack_long(&self.pk, longval) elif PyFloat_Check(o): fval = o - msgpack_pack_double(&self.pk, fval) + ret = msgpack_pack_double(&self.pk, fval) elif PyString_Check(o): rawval = o - msgpack_pack_raw(&self.pk, len(o)) - msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw(&self.pk, len(o)) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyUnicode_Check(o): o = o.encode('utf-8') rawval = o - msgpack_pack_raw(&self.pk, len(o)) - msgpack_pack_raw_body(&self.pk, rawval, len(o)) + ret = msgpack_pack_raw(&self.pk, len(o)) + if ret == 0: + ret = msgpack_pack_raw_body(&self.pk, rawval, len(o)) elif PyMapping_Check(o): - msgpack_pack_map(&self.pk, len(o)) - for k,v in o.iteritems(): - self.__pack(k) - self.__pack(v) + ret = msgpack_pack_map(&self.pk, len(o)) + if ret == 0: + for k,v in o.iteritems(): + ret = self.__pack(k) + if ret != 0: break + ret = self.__pack(v) + if ret != 0: break elif PySequence_Check(o): - msgpack_pack_array(&self.pk, len(o)) - for v in o: - self.__pack(v) + ret = msgpack_pack_array(&self.pk, len(o)) + if ret == 0: + for v in o: + ret = self.__pack(v) + if ret != 0: break else: # TODO: Serialize with defalt() like simplejson. raise TypeError, "can't serialize %r" % (o,) + return ret - def pack(self, object obj, object flush=True): - self.__pack(obj) - buf = PyBuffer_FromMemory(self.pk.buf, self.pk.length) + def pack(self, object obj): + cdef int ret + ret = self.__pack(obj) + if ret: + raise TypeError + buf = PyString_FromStringAndSize(self.pk.buf, self.pk.length) self.pk.length = 0 - self.strm.write(buf) - if flush: - self.flush() + return buf - def flush(self): - """Flash local buffer and output stream if it has 'flush()' method.""" - if hasattr(self.strm, 'flush'): - self.strm.flush() - - close = flush def pack(object o, object stream): - u"""pack o and write to stream).""" - packer = Packer(stream) - packer.pack(o) - packer.flush() + """pack a object `o` and write it to stream).""" + packer = Packer() + stream.write(packer.pack(o)) def packb(object o): - u"""pack o and return packed bytes.""" - buf = cStringIO.StringIO() - packer = Packer(buf) - packer.pack(o) - return buf.getvalue() + """pack o and return packed bytes.""" + packer = Packer() + return packer.pack(o) packs = packb @@ -222,7 +224,14 @@ cdef class Unpacker(object): cdef int read_size cdef object waiting_bytes - def __init__(self, file_like=None, int read_size=4096): + def __cinit__(self): + self.buf = NULL + + def __dealloc__(self): + if self.buf: + free(self.buf); + + def __init__(self, file_like=None, int read_size=1024*1024): self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] diff --git a/msgpack/pack.h b/msgpack/pack.h index d7e0867..58f021e 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -72,7 +72,7 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_ if (len + l > bs) { bs = (len + l) * 2; - buf = realloc(pk->buf, bs); + buf = realloc(buf, bs); if (!buf) return -1; } memcpy(buf + len, data, l); From 601209c83c11d9549eab86b708c6fe2f82d59f64 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 14:25:48 +0900 Subject: [PATCH 0231/1648] support $Data::MessagePack::PreferredInteger for Data::Model --- perl/lib/Data/MessagePack.pm | 10 ++++++++++ perl/pack.c | 17 ++++++++++++++++ perl/t/05_preferred_int.t | 38 ++++++++++++++++++++++++++++++++++++ perl/t/data.pl | 5 ++++- 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 perl/t/05_preferred_int.t diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index cb3f73a..082e232 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -24,6 +24,16 @@ Data::MessagePack - messagepack Data::MessagePack is a binary packer for perl. +=head1 Configuration Variables + +=over 4 + +=item $Data::MessagePack::PreferredInteger + +Pack the string as int when the value looks like int(EXPERIMENTAL). + +=back + =head1 AUTHORS Tokuhiro Matsuno diff --git a/perl/pack.c b/perl/pack.c index 5aec963..aee3bd6 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -51,6 +51,16 @@ void need(enc_t *enc, STRLEN len) } } +static int looks_like_int(const char *str, size_t len) { + int i; + for (i=0; ipack($_[0])); + s/(..)/$1 /g; + s/ $//; + $_; +} + +sub pis ($$) { + is packit($_[0]), $_[1], 'dump ' . $_[1]; + # is(Dumper(Data::MessagePack->unpack(Data::MessagePack->pack($_[0]))), Dumper($_[0])); +} + +my @dat = ( + '0', '00', + '1', '01', + '10', '0a', + ''.0xEFFF => 'cd ef ff', + ''.0xFFFF => 'cd ff ff', + ''.0xFFFFFF => 'ce 00 ff ff ff', + ''.0xFFFFFFFF => 'aa 34 32 39 34 39 36 37 32 39 35', + ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35', + ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34', + {'0' => '1'}, '81 00 01', + {'abc' => '1'}, '81 a3 61 62 63 01', +); +plan tests => 1*(scalar(@dat)/2); + +$Data::MessagePack::PreferredInteger = 1; +for (my $i=0; $i [undef, false, true], '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"], @@ -12,4 +12,7 @@ no warnings 'uninitialized'; # i need this. i need this. '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}}, '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}], + 'ce 00 ff ff ff' => ''.0xFFFFFF, + 'aa 34 32 39 34 39 36 37 32 39 35' => ''.0xFFFFFFFF, + 'ab 36 38 37 31 39 34 37 36 37 33 35' => ''.0xFFFFFFFFF, ) From 7b198d5966ee8f486425dd625fae485bc250975d Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 14:29:49 +0900 Subject: [PATCH 0232/1648] s/Preferred/Prefer/g suggested by kazuho++ --- perl/lib/Data/MessagePack.pm | 2 +- perl/pack.c | 2 +- perl/t/05_preferred_int.t | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 082e232..e7077f5 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -28,7 +28,7 @@ Data::MessagePack is a binary packer for perl. =over 4 -=item $Data::MessagePack::PreferredInteger +=item $Data::MessagePack::PreferInteger Pack the string as int when the value looks like int(EXPERIMENTAL). diff --git a/perl/pack.c b/perl/pack.c index aee3bd6..c2d871a 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -132,7 +132,7 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { STRLEN len; char * cval = SvPV(val, len); - SV* pref_int = get_sv("Data::MessagePack::PreferredInteger", 0); + SV* pref_int = get_sv("Data::MessagePack::PreferInteger", 0); if (pref_int && SvTRUE(pref_int) && looks_like_int(cval, len) && SvUV(val) < U32_MAX) { PACK_WRAPPER(uint32)(enc, SvUV(val)); return; diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t index 39167c1..176430e 100644 --- a/perl/t/05_preferred_int.t +++ b/perl/t/05_preferred_int.t @@ -31,7 +31,7 @@ my @dat = ( ); plan tests => 1*(scalar(@dat)/2); -$Data::MessagePack::PreferredInteger = 1; +$Data::MessagePack::PreferInteger = 1; for (my $i=0; $i Date: Thu, 2 Jul 2009 14:37:54 +0900 Subject: [PATCH 0233/1648] Checking in changes prior to tagging of version 0.02. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 18041d4..fb31a69 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.02 + + - added $Data::MessagePack::PreferInteger + (requested by yappo++) + 0.01 - initial release to CPAN --- perl/Changes | 5 +++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 18041d4..fb31a69 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.02 + + - added $Data::MessagePack::PreferInteger + (requested by yappo++) + 0.01 - initial release to CPAN diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index e7077f5..712d3b5 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.01'; +our $VERSION = '0.02'; XSLoader::load(__PACKAGE__, $VERSION); From e92e99c4d11cf35652c4184feeab5fa6ac29cb97 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 14:43:22 +0900 Subject: [PATCH 0234/1648] Perl: I don't need CCFLAGS --- perl/Makefile.PL | 3 --- 1 file changed, 3 deletions(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 167bebe..773d65d 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -6,12 +6,9 @@ perl_version '5.008005'; license 'perl'; can_cc or die "This module requires a C compiler"; -my $ccflags = '.'; - makemaker_args( OBJECT => '$(O_FILES)', LIBS => [''], - CCFLAGS => $ccflags, clean => { FILES => q{ *.stackdump From 580fbe77e9301786234ae47bfa0300736d9d8a84 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 16:40:41 +0900 Subject: [PATCH 0235/1648] do not check the string longer than 10. --- perl/pack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perl/pack.c b/perl/pack.c index c2d871a..c8987fe 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -131,9 +131,10 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (SvPOKp(val)) { STRLEN len; char * cval = SvPV(val, len); + const int U32_STRLEN = 10; /* length(0xFFFFFFFF) */ SV* pref_int = get_sv("Data::MessagePack::PreferInteger", 0); - if (pref_int && SvTRUE(pref_int) && looks_like_int(cval, len) && SvUV(val) < U32_MAX) { + if (pref_int && SvTRUE(pref_int) && len <= U32_STRLEN && looks_like_int(cval, len) && SvUV(val) < U32_MAX) { PACK_WRAPPER(uint32)(enc, SvUV(val)); return; } From ffef0a0b6fd6963e0465652c38185ead937e9545 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 17:42:28 +0900 Subject: [PATCH 0236/1648] oops. fixed memory leaks in stream unpacker. --- perl/MessagePack.c | 3 ++- perl/unpack.c | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/perl/MessagePack.c b/perl/MessagePack.c index 3b33fee..e583526 100644 --- a/perl/MessagePack.c +++ b/perl/MessagePack.c @@ -18,6 +18,7 @@ XS(xs_unpacker_execute_limit); XS(xs_unpacker_is_finished); XS(xs_unpacker_data); XS(xs_unpacker_reset); +XS(xs_unpacker_destroy); XS(boot_Data__MessagePack) { dXSARGS; @@ -35,6 +36,6 @@ XS(boot_Data__MessagePack) { newXS("Data::MessagePack::Unpacker::is_finished", xs_unpacker_is_finished, __FILE__); newXS("Data::MessagePack::Unpacker::data", xs_unpacker_data, __FILE__); newXS("Data::MessagePack::Unpacker::reset", xs_unpacker_reset, __FILE__); - + newXS("Data::MessagePack::Unpacker::DESTROY", xs_unpacker_destroy, __FILE__); } diff --git a/perl/unpack.c b/perl/unpack.c index c2e32dd..5a28fef 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -288,3 +288,14 @@ XS(xs_unpacker_reset) { XSRETURN(0); } +XS(xs_unpacker_destroy) { + dXSARGS; + if (items != 1) { + Perl_croak(aTHX_ "Usage: $unpacker->DESTROY()"); + } + + UNPACKER(ST(0), mp); + Safefree(mp); + + XSRETURN(0); +} From 3446c475d753b54cb2103753610fbc6a9596cae4 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 2 Jul 2009 17:43:58 +0900 Subject: [PATCH 0237/1648] Checking in changes prior to tagging of version 0.03. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index fb31a69..0c170a3 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.03 + + - performance tuning for too long string + - fixed memory leaks in stream unpacker + 0.02 - added $Data::MessagePack::PreferInteger --- perl/Changes | 5 +++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index fb31a69..0c170a3 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.03 + + - performance tuning for too long string + - fixed memory leaks in stream unpacker + 0.02 - added $Data::MessagePack::PreferInteger diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 712d3b5..2368a5b 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.02'; +our $VERSION = '0.03'; XSLoader::load(__PACKAGE__, $VERSION); From 0f9dcb86100e34c7f10a19399cfa5dca3d4dcdbc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 3 Jul 2009 01:49:37 +0900 Subject: [PATCH 0238/1648] perl: make PreferInteger variable magical and remove get_sv from _msgpack_pack_sv --- perl/MessagePack.c | 4 ++++ perl/lib/Data/MessagePack.pm | 1 + perl/pack.c | 35 +++++++++++++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/perl/MessagePack.c b/perl/MessagePack.c index 3b33fee..f56d147 100644 --- a/perl/MessagePack.c +++ b/perl/MessagePack.c @@ -19,10 +19,14 @@ XS(xs_unpacker_is_finished); XS(xs_unpacker_data); XS(xs_unpacker_reset); +void boot_Data__MessagePack_pack(void); + XS(boot_Data__MessagePack) { dXSARGS; HV * stash; + boot_Data__MessagePack_pack(); + newXS("Data::MessagePack::pack", xs_pack, __FILE__); newXS("Data::MessagePack::unpack", xs_unpack, __FILE__); stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE); diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 712d3b5..ff179e8 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -5,6 +5,7 @@ use XSLoader; use 5.008001; our $VERSION = '0.02'; +our $PreferInteger = 0; XSLoader::load(__PACKAGE__, $VERSION); diff --git a/perl/pack.c b/perl/pack.c index c8987fe..03e6530 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -51,6 +51,38 @@ void need(enc_t *enc, STRLEN len) } } + +static int s_pref_int = 0; + +static int pref_int_set(pTHX_ SV* sv, MAGIC* mg) { + if (SvTRUE(sv)) { + s_pref_int = 1; + } else { + s_pref_int = 0; + } + return 0; +} + +MGVTBL pref_int_vtbl = { + NULL, + pref_int_set, + NULL, + NULL, + NULL, + NULL, + NULL, +#ifdef MGf_LOCAL + NULL, +#endif +}; + +void boot_Data__MessagePack_pack(void) { + SV* var = get_sv("Data::MessagePack::PreferInteger", 0); + sv_magicext(var, NULL, PERL_MAGIC_ext, &pref_int_vtbl, NULL, 0); + SvSETMAGIC(var); +} + + static int looks_like_int(const char *str, size_t len) { int i; for (i=0; i Date: Fri, 3 Jul 2009 02:08:22 +0900 Subject: [PATCH 0239/1648] perl: PreferInteger: faster string to integer conversion; support negative value --- perl/pack.c | 64 ++++++++++++++++++++++++++++++++------- perl/t/05_preferred_int.t | 13 ++++++-- 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 03e6530..b329551 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -26,7 +26,7 @@ typedef struct { char *end; /* SvEND (sv) */ SV *sv; /* result scalar */ } enc_t; -void need(enc_t *enc, STRLEN len); +static void need(enc_t *enc, STRLEN len); #define msgpack_pack_user enc_t* @@ -41,7 +41,7 @@ void need(enc_t *enc, STRLEN len); #define PACK_WRAPPER(t) _PACK_WRAPPER(t) #define INIT_SIZE 32 /* initial scalar size to be allocated */ -void need(enc_t *enc, STRLEN len) +static void need(enc_t *enc, STRLEN len) { if (enc->cur + len >= enc->end) { STRLEN cur = enc->cur - (char *)SvPVX (enc->sv); @@ -83,16 +83,60 @@ void boot_Data__MessagePack_pack(void) { } -static int looks_like_int(const char *str, size_t len) { - int i; - for (i=0; i 11) { return 0; } + negative = 1; + ++p; + } else { + /* length(0xFFFFFFFF) == 10 */ + if (len > 10) { return 0; } } + +#if '9'=='8'+1 && '8'=='7'+1 && '7'=='6'+1 && '6'=='5'+1 && '5'=='4'+1 \ + && '4'=='3'+1 && '3'=='2'+1 && '2'=='1'+1 && '1'=='0'+1 + do { + unsigned int c = ((int)*(p++)) - '0'; + if (c > 9) { return 0; } + num = num * 10 + c; + } while(p < pe); +#else + do { + switch (*(p++)) { + case '0': num = num * 10 + 0; break; + case '1': num = num * 10 + 1; break; + case '2': num = num * 10 + 2; break; + case '3': num = num * 10 + 3; break; + case '4': num = num * 10 + 4; break; + case '5': num = num * 10 + 5; break; + case '6': num = num * 10 + 6; break; + case '7': num = num * 10 + 7; break; + case '8': num = num * 10 + 8; break; + case '9': num = num * 10 + 9; break; + default: return 0; + } + } while(p < pe); +#endif + + if (negative) { + if (num > 0x80000000) { return 0; } + msgpack_pack_int32(enc, ((int32_t)num) * -1); + } else { + if (num > 0xFFFFFFFF) { return 0; } + msgpack_pack_uint32(enc, (uint32_t)num); + } + return 1; } + static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (val==NULL) { msgpack_pack_nil(enc); @@ -163,10 +207,8 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (SvPOKp(val)) { STRLEN len; char * cval = SvPV(val, len); - const int U32_STRLEN = 10; /* length(0xFFFFFFFF) */ - if (s_pref_int && len <= U32_STRLEN && looks_like_int(cval, len) && SvUV(val) < U32_MAX) { - PACK_WRAPPER(uint32)(enc, SvUV(val)); + if (s_pref_int && try_int(enc, cval, len)) { return; } diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t index 176430e..39c725a 100644 --- a/perl/t/05_preferred_int.t +++ b/perl/t/05_preferred_int.t @@ -17,15 +17,24 @@ sub pis ($$) { } my @dat = ( + '', 'a0', '0', '00', '1', '01', - '10', '0a', + '10', '0a', + '-1', 'ff', + '-10', 'f6', + '-', 'a1 2d', ''.0xEFFF => 'cd ef ff', ''.0xFFFF => 'cd ff ff', ''.0xFFFFFF => 'ce 00 ff ff ff', - ''.0xFFFFFFFF => 'aa 34 32 39 34 39 36 37 32 39 35', + ''.0xFFFFFFFF => 'ce ff ff ff ff', ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35', ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34', + '-'.0x8000000 => 'd2 f8 00 00 00', + '-'.0x80000000 => 'd2 80 00 00 00', + '-'.0x800000000 => 'ac 2d 33 34 33 35 39 37 33 38 33 36 38', + '-'.0x8000000000 => 'ad 2d 35 34 39 37 35 35 38 31 33 38 38 38', + '-'.0x800000000000000000000000000000 => 'b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35', {'0' => '1'}, '81 00 01', {'abc' => '1'}, '81 a3 61 62 63 01', ); From b45dc8db61a5f97369094592bcf80116c1019c37 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 15:14:31 +0900 Subject: [PATCH 0240/1648] check SvROK first --- perl/pack.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index b329551..b1e5677 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -141,6 +141,9 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (val==NULL) { msgpack_pack_nil(enc); return; + } else if (SvROK(val)) { + _msgpack_pack_sv(enc, SvRV(val)); + return; } switch (SvTYPE(val)) { @@ -200,9 +203,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { } } break; - case SVt_RV: - _msgpack_pack_sv(enc, SvRV(val)); - break; default: if (SvPOKp(val)) { STRLEN len; From 6fb6283463db961677a00b1ea00552e5e9679d1c Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 15:23:50 +0900 Subject: [PATCH 0241/1648] Checking in changes prior to tagging of version 0.04. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 0c170a3..33fb36c 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,11 @@ +0.04 + + - check SvROK first(reported by yappo++) + - PreferInteger: faster string to integer conversion; support negative value + (frsyuki++) + - make PreferInteger variable magical and remove get_sv from _msgpack_pack_sv + (frsyuki++) + 0.03 - performance tuning for too long string --- perl/Changes | 8 ++++++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 0c170a3..33fb36c 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,11 @@ +0.04 + + - check SvROK first(reported by yappo++) + - PreferInteger: faster string to integer conversion; support negative value + (frsyuki++) + - make PreferInteger variable magical and remove get_sv from _msgpack_pack_sv + (frsyuki++) + 0.03 - performance tuning for too long string diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index e204669..c61cac0 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.03'; +our $VERSION = '0.04'; our $PreferInteger = 0; XSLoader::load(__PACKAGE__, $VERSION); From ad7cf12128388b8432ffe33c2010d0d0580297c3 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 17:27:44 +0900 Subject: [PATCH 0242/1648] perl: added test case for "the flag is working?" --- perl/t/05_preferred_int.t | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t index 39c725a..9860711 100644 --- a/perl/t/05_preferred_int.t +++ b/perl/t/05_preferred_int.t @@ -38,10 +38,20 @@ my @dat = ( {'0' => '1'}, '81 00 01', {'abc' => '1'}, '81 a3 61 62 63 01', ); -plan tests => 1*(scalar(@dat)/2); +plan tests => 1*(scalar(@dat)/2) + 2; -$Data::MessagePack::PreferInteger = 1; for (my $i=0; $i Date: Fri, 3 Jul 2009 18:09:38 +0900 Subject: [PATCH 0243/1648] Perl: check IV by SvIOK --- perl/pack.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index b1e5677..90b18aa 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -150,13 +150,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { case SVt_NULL: msgpack_pack_nil(enc); break; - case SVt_IV: - if (SvIOK_UV(val)) { - msgpack_pack_uint32(enc, SvUV(val)); - } else { - PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); - } - break; case SVt_PVNV: { STRLEN len = 0; @@ -215,6 +208,10 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { msgpack_pack_raw(enc, len); msgpack_pack_raw_body(enc, cval, len); return; + } else if (SvIOK_UV(val)) { + msgpack_pack_uint32(enc, SvUV(val)); + } else if (SvIOK(val)) { + PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); } else { sv_dump(val); Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); From cb5c878b77c9286cc6ca47dd04978550479ae7de Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 18:15:45 +0900 Subject: [PATCH 0244/1648] Checking in changes prior to tagging of version 0.05. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 33fb36c..15603ea 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.05 + + - change type detection for old perl + 0.04 - check SvROK first(reported by yappo++) --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 33fb36c..15603ea 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.05 + + - change type detection for old perl + 0.04 - check SvROK first(reported by yappo++) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index c61cac0..46fb45d 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.04'; +our $VERSION = '0.05'; our $PreferInteger = 0; XSLoader::load(__PACKAGE__, $VERSION); From 661f27348188d15e841467ececd75e27cd77980b Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 18:18:40 +0900 Subject: [PATCH 0245/1648] Perl: use SvNOK. --- perl/pack.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 90b18aa..ff0ef2d 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -163,9 +163,6 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { } } break; - case SVt_NV: - PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); - break; case SVt_PVAV: { AV* ary = (AV*)val; @@ -212,6 +209,8 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { msgpack_pack_uint32(enc, SvUV(val)); } else if (SvIOK(val)) { PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); + } else if (SvNOK(val)) { + PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); } else { sv_dump(val); Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); From 7da1a9b3cfa5687f24cdfde4068677675bc4a076 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Fri, 3 Jul 2009 18:19:29 +0900 Subject: [PATCH 0246/1648] Checking in changes prior to tagging of version 0.06. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 15603ea..1a51ddf 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.06 + + - use SvNOK. + 0.05 - change type detection for old perl --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 15603ea..1a51ddf 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.06 + + - use SvNOK. + 0.05 - change type detection for old perl diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 46fb45d..be22203 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.05'; +our $VERSION = '0.06'; our $PreferInteger = 0; XSLoader::load(__PACKAGE__, $VERSION); From 979efbb9501e14d43e2def8a80992494690a9d39 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 9 Jul 2009 13:41:04 +0900 Subject: [PATCH 0247/1648] Support MinGW. --- python/msgpack/unpack_define.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/msgpack/unpack_define.h b/python/msgpack/unpack_define.h index 63668c2..d997569 100644 --- a/python/msgpack/unpack_define.h +++ b/python/msgpack/unpack_define.h @@ -45,8 +45,21 @@ extern "C" { #endif #endif +#ifdef __WIN32__ +static inline uint16_t msgpack_betoh16(uint16_t x) { + return ((x << 8) & 0xff00U) | + ((x >> 8) & 0x00ffU); +} +static inline uint32_t msgpack_betoh32(uint32_t x) { + return ((x << 24) & 0xff000000UL ) | + ((x << 8) & 0x00ff0000UL ) | + ((x >> 8) & 0x0000ff00UL ) | + ((x >> 24) & 0x000000ffUL ); +} +#else #define msgpack_betoh16(x) ntohs(x) #define msgpack_betoh32(x) ntohl(x) +#endif #ifdef __LITTLE_ENDIAN__ #if defined(__bswap_64) From 77ca35a865bcc8cb9d7ed2fc8508cfbfb5b91dc6 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 9 Jul 2009 13:41:04 +0900 Subject: [PATCH 0248/1648] Support MinGW. --- msgpack/unpack_define.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 63668c2..d997569 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -45,8 +45,21 @@ extern "C" { #endif #endif +#ifdef __WIN32__ +static inline uint16_t msgpack_betoh16(uint16_t x) { + return ((x << 8) & 0xff00U) | + ((x >> 8) & 0x00ffU); +} +static inline uint32_t msgpack_betoh32(uint32_t x) { + return ((x << 24) & 0xff000000UL ) | + ((x << 8) & 0x00ff0000UL ) | + ((x >> 8) & 0x0000ff00UL ) | + ((x >> 24) & 0x000000ffUL ); +} +#else #define msgpack_betoh16(x) ntohs(x) #define msgpack_betoh32(x) ntohl(x) +#endif #ifdef __LITTLE_ENDIAN__ #if defined(__bswap_64) From 900785e1aa2e031a4496f8c2a30bb95d0c950b9b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 12 Jul 2009 09:29:11 +0900 Subject: [PATCH 0249/1648] Clean up --- python/README | 6 +++--- python/msgpack/_msgpack.pyx | 6 ++++-- python/setup.py | 17 +++++++++-------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/python/README b/python/README index 243def2..6b98982 100644 --- a/python/README +++ b/python/README @@ -13,11 +13,11 @@ You can read document in docstring after `import msgpack` INSTALL --------- -Cython_ is required to build the binding. +Cython_ is required to build msgpack. .. _Cython: http://www.cython.org/ -Posix +posix '''''' You can install msgpack in common way. @@ -28,7 +28,7 @@ Windows MessagePack requires gcc currently. So you need to prepare MinGW GCC. - $ python setup.py install -c mingw + $ python setup.py install -c mingw32 TEST ---- diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index c455394..b407b92 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -163,7 +163,7 @@ cdef extern from "unpack.h": object template_data(template_context* ctx) -def unpacks(object packed_bytes): +def unpackb(object packed_bytes): """Unpack packed_bytes to object. Returns unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx @@ -176,10 +176,12 @@ def unpacks(object packed_bytes): else: return None +unpacks = unpackb + def unpack(object stream): """unpack from stream.""" packed = stream.read() - return unpacks(packed) + return unpackb(packed) cdef class UnpackIterator(object): cdef object unpacker diff --git a/python/setup.py b/python/setup.py index faadb33..fa8cb3a 100755 --- a/python/setup.py +++ b/python/setup.py @@ -7,18 +7,14 @@ import os version = '0.0.1dev' -PACKAGE_ROOT = os.getcwdu() -INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'], - include_dirs=[INCLUDE_PATH]) + sources=['msgpack/_msgpack.pyx'] + ) -desc = 'MessagePack serializer/desirializer.' +desc = 'MessagePack (de)serializer.' long_desc = desc + """ -Python binding of MessagePack_. - -This package is under development. +MessagePack_ (de)serializer for Python. .. _MessagePack: http://msgpack.sourceforge.jp/ @@ -37,4 +33,9 @@ setup(name='msgpack', packages=['msgpack'], description=desc, long_description=long_desc, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + ] ) From e963ad55d27ae0f39bec605f3a0f0362061c9655 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 12 Jul 2009 09:29:11 +0900 Subject: [PATCH 0250/1648] Clean up --- README | 6 +++--- msgpack/_msgpack.pyx | 6 ++++-- setup.py | 17 +++++++++-------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/README b/README index 243def2..6b98982 100644 --- a/README +++ b/README @@ -13,11 +13,11 @@ You can read document in docstring after `import msgpack` INSTALL --------- -Cython_ is required to build the binding. +Cython_ is required to build msgpack. .. _Cython: http://www.cython.org/ -Posix +posix '''''' You can install msgpack in common way. @@ -28,7 +28,7 @@ Windows MessagePack requires gcc currently. So you need to prepare MinGW GCC. - $ python setup.py install -c mingw + $ python setup.py install -c mingw32 TEST ---- diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index c455394..b407b92 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -163,7 +163,7 @@ cdef extern from "unpack.h": object template_data(template_context* ctx) -def unpacks(object packed_bytes): +def unpackb(object packed_bytes): """Unpack packed_bytes to object. Returns unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx @@ -176,10 +176,12 @@ def unpacks(object packed_bytes): else: return None +unpacks = unpackb + def unpack(object stream): """unpack from stream.""" packed = stream.read() - return unpacks(packed) + return unpackb(packed) cdef class UnpackIterator(object): cdef object unpacker diff --git a/setup.py b/setup.py index faadb33..fa8cb3a 100755 --- a/setup.py +++ b/setup.py @@ -7,18 +7,14 @@ import os version = '0.0.1dev' -PACKAGE_ROOT = os.getcwdu() -INCLUDE_PATH = os.path.join(PACKAGE_ROOT, 'include') msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'], - include_dirs=[INCLUDE_PATH]) + sources=['msgpack/_msgpack.pyx'] + ) -desc = 'MessagePack serializer/desirializer.' +desc = 'MessagePack (de)serializer.' long_desc = desc + """ -Python binding of MessagePack_. - -This package is under development. +MessagePack_ (de)serializer for Python. .. _MessagePack: http://msgpack.sourceforge.jp/ @@ -37,4 +33,9 @@ setup(name='msgpack', packages=['msgpack'], description=desc, long_description=long_desc, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + ] ) From e5c49dae13b26f2155dffffbc4e9915badbcecfb Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 12 Jul 2009 20:02:21 +0900 Subject: [PATCH 0251/1648] Release 0.1.0 --- python/README | 4 ++-- python/msgpack/_msgpack.pyx | 12 ++++++------ python/setup.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/README b/python/README index 6b98982..2b2ebe9 100644 --- a/python/README +++ b/python/README @@ -3,8 +3,8 @@ MessagePack Python Binding =========================== :author: Naoki INADA -:version: 0.0.1 -:date: 2009-06-27 +:version: 0.1.0 +:date: 2009-07-12 HOW TO USE ----------- diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index b407b92..7990a18 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -139,7 +139,7 @@ cdef class Packer(object): def pack(object o, object stream): - """pack a object `o` and write it to stream).""" + """pack an object `o` and write it to stream).""" packer = Packer() stream.write(packer.pack(o)) @@ -164,7 +164,7 @@ cdef extern from "unpack.h": def unpackb(object packed_bytes): - """Unpack packed_bytes to object. Returns unpacked object.""" + """Unpack packed_bytes to object. Returns an unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx cdef size_t off = 0 @@ -179,7 +179,7 @@ def unpackb(object packed_bytes): unpacks = unpackb def unpack(object stream): - """unpack from stream.""" + """unpack an object from stream.""" packed = stream.read() return unpackb(packed) @@ -196,14 +196,14 @@ cdef class UnpackIterator(object): return self cdef class Unpacker(object): - """Unpacker(file_like=None, read_size=4096) + """Unpacker(file_like=None, read_size=1024*1024) Streaming unpacker. file_like must have read(n) method. read_size is used like file_like.read(read_size) - If file_like is None, you can feed() bytes. feed() is useful - for unpack from non-blocking stream. + If file_like is None, you can ``feed()`` bytes. ``feed()`` is + useful for unpacking from non-blocking stream. exsample 1: unpacker = Unpacker(afile) diff --git a/python/setup.py b/python/setup.py index fa8cb3a..255e0db 100755 --- a/python/setup.py +++ b/python/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.0.1dev' +version = '0.1.0' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 2d885ad22afa24c5fbf62b5cf0d5970f4a4a2ff7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Sun, 12 Jul 2009 20:02:21 +0900 Subject: [PATCH 0252/1648] Release 0.1.0 --- README | 4 ++-- msgpack/_msgpack.pyx | 12 ++++++------ setup.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README b/README index 6b98982..2b2ebe9 100644 --- a/README +++ b/README @@ -3,8 +3,8 @@ MessagePack Python Binding =========================== :author: Naoki INADA -:version: 0.0.1 -:date: 2009-06-27 +:version: 0.1.0 +:date: 2009-07-12 HOW TO USE ----------- diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index b407b92..7990a18 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -139,7 +139,7 @@ cdef class Packer(object): def pack(object o, object stream): - """pack a object `o` and write it to stream).""" + """pack an object `o` and write it to stream).""" packer = Packer() stream.write(packer.pack(o)) @@ -164,7 +164,7 @@ cdef extern from "unpack.h": def unpackb(object packed_bytes): - """Unpack packed_bytes to object. Returns unpacked object.""" + """Unpack packed_bytes to object. Returns an unpacked object.""" cdef const_char_ptr p = packed_bytes cdef template_context ctx cdef size_t off = 0 @@ -179,7 +179,7 @@ def unpackb(object packed_bytes): unpacks = unpackb def unpack(object stream): - """unpack from stream.""" + """unpack an object from stream.""" packed = stream.read() return unpackb(packed) @@ -196,14 +196,14 @@ cdef class UnpackIterator(object): return self cdef class Unpacker(object): - """Unpacker(file_like=None, read_size=4096) + """Unpacker(file_like=None, read_size=1024*1024) Streaming unpacker. file_like must have read(n) method. read_size is used like file_like.read(read_size) - If file_like is None, you can feed() bytes. feed() is useful - for unpack from non-blocking stream. + If file_like is None, you can ``feed()`` bytes. ``feed()`` is + useful for unpacking from non-blocking stream. exsample 1: unpacker = Unpacker(afile) diff --git a/setup.py b/setup.py index fa8cb3a..255e0db 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.0.1dev' +version = '0.1.0' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 294e3fe7ab01ef9273b364b0d3d9df4e9b275158 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 13 Jul 2009 14:27:57 +0900 Subject: [PATCH 0253/1648] Add setup script for distribution. --- python/setup.py | 7 ++++--- python/setup_dev.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100755 python/setup_dev.py diff --git a/python/setup.py b/python/setup.py index 255e0db..c0d121b 100755 --- a/python/setup.py +++ b/python/setup.py @@ -2,13 +2,14 @@ # coding: utf-8 from distutils.core import setup, Extension -from Cython.Distutils import build_ext +#from Cython.Distutils import build_ext import os version = '0.1.0' msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'] + #sources=['msgpack/_msgpack.pyx'] + sources=['msgpack/_msgpack.c'] ) desc = 'MessagePack (de)serializer.' @@ -28,7 +29,7 @@ setup(name='msgpack', author='Naoki INADA', author_email='songofacandy@gmail.com', version=version, - cmdclass={'build_ext': build_ext}, + #cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, diff --git a/python/setup_dev.py b/python/setup_dev.py new file mode 100755 index 0000000..255e0db --- /dev/null +++ b/python/setup_dev.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# coding: utf-8 + +from distutils.core import setup, Extension +from Cython.Distutils import build_ext +import os + +version = '0.1.0' + +msgpack_mod = Extension('msgpack._msgpack', + sources=['msgpack/_msgpack.pyx'] + ) + +desc = 'MessagePack (de)serializer.' +long_desc = desc + """ + +MessagePack_ (de)serializer for Python. + +.. _MessagePack: http://msgpack.sourceforge.jp/ + +What's MessagePack? (from http://msgpack.sourceforge.jp/) + + MessagePack is a binary-based efficient data interchange format that is + focused on high performance. It is like JSON, but very fast and small. +""" + +setup(name='msgpack', + author='Naoki INADA', + author_email='songofacandy@gmail.com', + version=version, + cmdclass={'build_ext': build_ext}, + ext_modules=[msgpack_mod], + packages=['msgpack'], + description=desc, + long_description=long_desc, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + ] + ) From 7247238873d75e30b5d4d823bb39869326e89f6a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 13 Jul 2009 14:27:57 +0900 Subject: [PATCH 0254/1648] Add setup script for distribution. --- setup.py | 7 ++++--- setup_dev.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100755 setup_dev.py diff --git a/setup.py b/setup.py index 255e0db..c0d121b 100755 --- a/setup.py +++ b/setup.py @@ -2,13 +2,14 @@ # coding: utf-8 from distutils.core import setup, Extension -from Cython.Distutils import build_ext +#from Cython.Distutils import build_ext import os version = '0.1.0' msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'] + #sources=['msgpack/_msgpack.pyx'] + sources=['msgpack/_msgpack.c'] ) desc = 'MessagePack (de)serializer.' @@ -28,7 +29,7 @@ setup(name='msgpack', author='Naoki INADA', author_email='songofacandy@gmail.com', version=version, - cmdclass={'build_ext': build_ext}, + #cmdclass={'build_ext': build_ext}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, diff --git a/setup_dev.py b/setup_dev.py new file mode 100755 index 0000000..255e0db --- /dev/null +++ b/setup_dev.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# coding: utf-8 + +from distutils.core import setup, Extension +from Cython.Distutils import build_ext +import os + +version = '0.1.0' + +msgpack_mod = Extension('msgpack._msgpack', + sources=['msgpack/_msgpack.pyx'] + ) + +desc = 'MessagePack (de)serializer.' +long_desc = desc + """ + +MessagePack_ (de)serializer for Python. + +.. _MessagePack: http://msgpack.sourceforge.jp/ + +What's MessagePack? (from http://msgpack.sourceforge.jp/) + + MessagePack is a binary-based efficient data interchange format that is + focused on high performance. It is like JSON, but very fast and small. +""" + +setup(name='msgpack', + author='Naoki INADA', + author_email='songofacandy@gmail.com', + version=version, + cmdclass={'build_ext': build_ext}, + ext_modules=[msgpack_mod], + packages=['msgpack'], + description=desc, + long_description=long_desc, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + ] + ) From 6083bad5ffd646f715d756e7189041d542ee8ab0 Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 14:46:53 +0900 Subject: [PATCH 0255/1648] Fix wrong MANIFEST. --- python/MANIFEST | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/MANIFEST b/python/MANIFEST index 1fe22ec..f6c43b0 100644 --- a/python/MANIFEST +++ b/python/MANIFEST @@ -3,7 +3,7 @@ msgpack/pack.h msgpack/unpack.h msgpack/_msgpack.pyx msgpack/__init__.py -include/msgpack/pack_define.h -include/msgpack/pack_template.h -include/msgpack/unpack_define.h -include/msgpack/unpack_template.h +msgpack/pack_define.h +msgpack/pack_template.h +msgpack/unpack_define.h +msgpack/unpack_template.h From 6e0a7bdb293e53d5c1db226759ba77caec337715 Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 14:46:53 +0900 Subject: [PATCH 0256/1648] Fix wrong MANIFEST. --- MANIFEST | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/MANIFEST b/MANIFEST index 1fe22ec..f6c43b0 100644 --- a/MANIFEST +++ b/MANIFEST @@ -3,7 +3,7 @@ msgpack/pack.h msgpack/unpack.h msgpack/_msgpack.pyx msgpack/__init__.py -include/msgpack/pack_define.h -include/msgpack/pack_template.h -include/msgpack/unpack_define.h -include/msgpack/unpack_template.h +msgpack/pack_define.h +msgpack/pack_template.h +msgpack/unpack_define.h +msgpack/unpack_template.h From 0db5e0439dab0445d9f84aa5f6fbf2f535cd8d0d Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 15:49:38 +0900 Subject: [PATCH 0257/1648] Fix: Unpacker.unpack() may raise StopIteration before unpacking large object when deserializing from file. --- python/msgpack/_msgpack.pyx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 7990a18..cb95146 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -265,7 +265,11 @@ cdef class Unpacker(object): cdef Py_ssize_t add_size if self.file_like is not None: - self.waiting_bytes.append(self.file_like.read(self.read_size)) + next_bytes = self.file_like.read(self.read_size) + if next_bytes: + self.waiting_bytes.append(next_bytes) + else: + self.file_like = None if not self.waiting_bytes: return @@ -307,6 +311,8 @@ cdef class Unpacker(object): if ret == 1: return template_data(&self.ctx) elif ret == 0: + if self.file_like is not None: + return self.unpack() raise StopIteration, "No more unpack data." else: raise ValueError, "Unpack failed." From dc4e20ed35b6910211366e661382865a33fb2623 Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 15:49:38 +0900 Subject: [PATCH 0258/1648] Fix: Unpacker.unpack() may raise StopIteration before unpacking large object when deserializing from file. --- msgpack/_msgpack.pyx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 7990a18..cb95146 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -265,7 +265,11 @@ cdef class Unpacker(object): cdef Py_ssize_t add_size if self.file_like is not None: - self.waiting_bytes.append(self.file_like.read(self.read_size)) + next_bytes = self.file_like.read(self.read_size) + if next_bytes: + self.waiting_bytes.append(next_bytes) + else: + self.file_like = None if not self.waiting_bytes: return @@ -307,6 +311,8 @@ cdef class Unpacker(object): if ret == 1: return template_data(&self.ctx) elif ret == 0: + if self.file_like is not None: + return self.unpack() raise StopIteration, "No more unpack data." else: raise ValueError, "Unpack failed." From d19c4889293ef15f00ebe55c4246eaf8c48f44cc Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 15:52:25 +0900 Subject: [PATCH 0259/1648] version: python-0.1.1 --- python/setup.py | 2 +- python/setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index c0d121b..b26b9e8 100755 --- a/python/setup.py +++ b/python/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.0' +version = '0.1.1' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/python/setup_dev.py b/python/setup_dev.py index 255e0db..e28ef25 100755 --- a/python/setup_dev.py +++ b/python/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.0' +version = '0.1.1dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 3a0851b94949d730b9e69696ffc9dabc2acf6b6c Mon Sep 17 00:00:00 2001 From: inada-n Date: Mon, 13 Jul 2009 15:52:25 +0900 Subject: [PATCH 0260/1648] version: python-0.1.1 --- setup.py | 2 +- setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c0d121b..b26b9e8 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.0' +version = '0.1.1' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/setup_dev.py b/setup_dev.py index 255e0db..e28ef25 100755 --- a/setup_dev.py +++ b/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.0' +version = '0.1.1dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 390c5ad8a0424968dbcc4b69750d2c31f1872b4f Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 22 Jul 2009 13:41:31 +0900 Subject: [PATCH 0261/1648] added test case. --- perl/t/01_pack.t | 2 ++ 1 file changed, 2 insertions(+) diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t index 60c67f8..dbf7a9a 100644 --- a/perl/t/01_pack.t +++ b/perl/t/01_pack.t @@ -15,6 +15,8 @@ sub pis ($$) { my @dat = ( 0, '00', + (my $foo="0")+0, '00', + {2 => undef}, '81 a1 32 c0', 1, '01', 127, '7f', 128, 'cc 80', From 718227bc2cc7ed2d4a4152a1018f0fa046bbc57e Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 22 Jul 2009 16:40:15 +0900 Subject: [PATCH 0262/1648] Perl: do not use switch (SvTYPE) --- perl/pack.c | 114 ++++++++++++++++++++--------------------------- perl/t/01_pack.t | 1 + 2 files changed, 50 insertions(+), 65 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index ff0ef2d..50b82e5 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -140,81 +140,65 @@ static int try_int(enc_t* enc, const char *p, size_t len) { static void _msgpack_pack_sv(enc_t *enc, SV* val) { if (val==NULL) { msgpack_pack_nil(enc); - return; } else if (SvROK(val)) { _msgpack_pack_sv(enc, SvRV(val)); - return; - } - - switch (SvTYPE(val)) { - case SVt_NULL: - msgpack_pack_nil(enc); - break; - case SVt_PVNV: - { - STRLEN len = 0; - char *pv = SvPV(val, len); - if (len == 1 && *pv == '1') { - msgpack_pack_true(enc); - } else if (len == 0 && *pv==0) { - msgpack_pack_false(enc); + } else if (SVt_PVNV == SvTYPE(val)) { + STRLEN len = 0; + char *pv = SvPV(val, len); + if (len == 1 && *pv == '1') { + msgpack_pack_true(enc); + } else if (len == 0 && *pv==0) { + msgpack_pack_false(enc); + } else { + msgpack_pack_nil(enc); + } + } else if (SvTYPE(val) == SVt_PVAV) { + AV* ary = (AV*)val; + int len = av_len(ary) + 1; + int i; + msgpack_pack_array(enc, len); + for (i=0; i undef}, '81 a1 32 c0', + do {no warnings; my $foo = 10; "$foo"; $foo = undef; $foo} => 'c0', # PVIV but !POK && !IOK 1, '01', 127, '7f', 128, 'cc 80', From 794adf9469074cb4ca93c5b71d0a5ee7572d85b9 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Wed, 22 Jul 2009 16:42:12 +0900 Subject: [PATCH 0263/1648] Checking in changes prior to tagging of version 0.07. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 1a51ddf..4e7f1d7 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.07 + + - do not use switch (SvTYPE(val)). + 0.06 - use SvNOK. --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 1a51ddf..4e7f1d7 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.07 + + - do not use switch (SvTYPE(val)). + 0.06 - use SvNOK. diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index be22203..f01a92a 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.06'; +our $VERSION = '0.07'; our $PreferInteger = 0; XSLoader::load(__PACKAGE__, $VERSION); From 0fe79a7752f1a2dec796281a9a6639d126948bd3 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 13:19:49 +0900 Subject: [PATCH 0264/1648] Perl: added benchmark script for memroy-usage --- perl/benchmark/size.pl | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 perl/benchmark/size.pl diff --git a/perl/benchmark/size.pl b/perl/benchmark/size.pl new file mode 100644 index 0000000..85555ce --- /dev/null +++ b/perl/benchmark/size.pl @@ -0,0 +1,28 @@ +#!/usr/bin/perl +use strict; +use warnings; +use Data::MessagePack; +use Storable; +use Text::SimpleTable; + +my @entries = ( + 1, + 3.14, + {}, + [], + [('a')x10], + {('a')x10}, + +{1,+{1,+{}}}, + +[+[+[]]], +); + +my $table = Text::SimpleTable->new([10, 'storable'], [10, 'msgpack']); + +for my $e (@entries) { + $table->row( + length(Storable::nfreeze(ref $e ? $e : \$e)), + length(Data::MessagePack->pack($e)), + ); +} + +print $table->draw; From 3275eee28173e9c4d00641204ab966e7fa278dcd Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 13:24:38 +0900 Subject: [PATCH 0265/1648] Perl: display original data --- perl/benchmark/size.pl | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/perl/benchmark/size.pl b/perl/benchmark/size.pl index 85555ce..928daca 100644 --- a/perl/benchmark/size.pl +++ b/perl/benchmark/size.pl @@ -6,20 +6,24 @@ use Storable; use Text::SimpleTable; my @entries = ( - 1, - 3.14, - {}, - [], - [('a')x10], - {('a')x10}, - +{1,+{1,+{}}}, - +[+[+[]]], + '1', + '3.14', + '{}', + '[]', + "[('a')x10]", + "{('a')x10}", + "+{1,+{1,+{}}}", + "+[+[+[]]]", ); -my $table = Text::SimpleTable->new([10, 'storable'], [10, 'msgpack']); +my $table = Text::SimpleTable->new([15, 'src'], [9, 'storable'], [7, 'msgpack']); + +for my $src (@entries) { + my $e = eval $src; + die $@ if $@; -for my $e (@entries) { $table->row( + $src, length(Storable::nfreeze(ref $e ? $e : \$e)), length(Data::MessagePack->pack($e)), ); From bb455c1f71e859122e55673d23e8f1311852a82c Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 13:29:24 +0900 Subject: [PATCH 0266/1648] perl: display module version --- perl/benchmark/size.pl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/perl/benchmark/size.pl b/perl/benchmark/size.pl index 928daca..cf5c1ce 100644 --- a/perl/benchmark/size.pl +++ b/perl/benchmark/size.pl @@ -29,4 +29,9 @@ for my $src (@entries) { ); } +print "perl: $]\n"; +print "Storable: $Storable::VERSION\n"; +print "Data::MessagePack: $Data::MessagePack::VERSION\n"; +print "\n"; print $table->draw; + From 4db90d23ac95592ea3892513026de326926007f9 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 16:22:00 +0900 Subject: [PATCH 0267/1648] - fixed SVt_PVNV issue. --- perl/Makefile.PL | 4 ++ perl/MessagePack.c | 2 - perl/lib/Data/MessagePack.pm | 5 ++ perl/pack.c | 120 ++++++++++++++++++++++------------- perl/t/01_pack.t | 1 + perl/unpack.c | 58 +++++++++++------ perl/util.h | 11 ++++ 7 files changed, 133 insertions(+), 68 deletions(-) create mode 100644 perl/util.h diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 773d65d..d1bff06 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -23,6 +23,10 @@ makemaker_args( tests 't/*.t'; author_tests('xt'); +if ($ENV{DEBUG}) { + cc_append_to_ccflags '-g'; +} + # copy modules if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) { mkdir 'msgpack' unless -d 'msgpack'; diff --git a/perl/MessagePack.c b/perl/MessagePack.c index 6b9010c..fd1b344 100644 --- a/perl/MessagePack.c +++ b/perl/MessagePack.c @@ -31,8 +31,6 @@ XS(boot_Data__MessagePack) { newXS("Data::MessagePack::pack", xs_pack, __FILE__); newXS("Data::MessagePack::unpack", xs_unpack, __FILE__); stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE); - newCONSTSUB(stash, "true", &PL_sv_yes); - newCONSTSUB(stash, "false", &PL_sv_no); newXS("Data::MessagePack::Unpacker::new", xs_unpacker_new, __FILE__); newXS("Data::MessagePack::Unpacker::execute", xs_unpacker_execute, __FILE__); diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index f01a92a..38431eb 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -7,6 +7,11 @@ use 5.008001; our $VERSION = '0.07'; our $PreferInteger = 0; +our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; +our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" }; +sub true () { $true } +sub false () { $false } + XSLoader::load(__PACKAGE__, $VERSION); 1; diff --git a/perl/pack.c b/perl/pack.c index 50b82e5..01dd5b6 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -137,23 +137,71 @@ static int try_int(enc_t* enc, const char *p, size_t len) { } -static void _msgpack_pack_sv(enc_t *enc, SV* val) { - if (val==NULL) { +static void _msgpack_pack_rv(enc_t *enc, SV* sv); + +static void _msgpack_pack_sv(enc_t *enc, SV* sv) { + SvGETMAGIC(sv); + + if (sv==NULL) { msgpack_pack_nil(enc); - } else if (SvROK(val)) { - _msgpack_pack_sv(enc, SvRV(val)); - } else if (SVt_PVNV == SvTYPE(val)) { - STRLEN len = 0; - char *pv = SvPV(val, len); - if (len == 1 && *pv == '1') { - msgpack_pack_true(enc); - } else if (len == 0 && *pv==0) { - msgpack_pack_false(enc); + } else if (SvPOKp(sv)) { + STRLEN len; + char * csv = SvPV(sv, len); + + if (s_pref_int && try_int(enc, csv, len)) { + return; } else { - msgpack_pack_nil(enc); + msgpack_pack_raw(enc, len); + msgpack_pack_raw_body(enc, csv, len); } - } else if (SvTYPE(val) == SVt_PVAV) { - AV* ary = (AV*)val; + } else if (SvNOKp(sv)) { + PACK_WRAPPER(NVTYPE)(enc, SvNVX(sv)); + } else if (SvIOK_UV(sv)) { + msgpack_pack_uint32(enc, SvUV(sv)); + } else if (SvIOKp(sv)) { + PACK_WRAPPER(IVTYPE)(enc, SvIV(sv)); + } else if (SvROK(sv)) { + _msgpack_pack_rv(enc, SvRV(sv)); + } else if (!SvOK(sv)) { + msgpack_pack_nil(enc); + } else if (isGV(sv)) { + Perl_croak(aTHX_ "msgpack cannot pack the GV\n"); + } else { + sv_dump(sv); + Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(sv)); + } +} + +static void _msgpack_pack_rv(enc_t *enc, SV* sv) { + svtype svt; + SvGETMAGIC(sv); + svt = SvTYPE(sv); + + if (SvOBJECT (sv)) { + HV *stash = gv_stashpv ("Data::MessagePack::Boolean", 1); // TODO: cache? + if (SvSTASH (sv) == stash) { + if (SvIV(sv)) { + msgpack_pack_true(enc); + } else { + msgpack_pack_false(enc); + } + } else { + croak ("encountered object '%s', Data::MessagePack doesn't allow the object", + SvPV_nolen(sv_2mortal(newRV_inc(sv)))); + } + } else if (svt == SVt_PVHV) { + HV* hval = (HV*)sv; + int count = hv_iterinit(hval); + HE* he; + + msgpack_pack_map(enc, count); + + while (he = hv_iternext(hval)) { + _msgpack_pack_sv(enc, hv_iterkeysv(he)); + _msgpack_pack_sv(enc, HeVAL(he)); + } + } else if (svt == SVt_PVAV) { + AV* ary = (AV*)sv; int len = av_len(ary) + 1; int i; msgpack_pack_array(enc, len); @@ -165,40 +213,22 @@ static void _msgpack_pack_sv(enc_t *enc, SV* val) { msgpack_pack_nil(enc); } } - } else if (SvTYPE(val) == SVt_PVHV) { - HV* hval = (HV*)val; - int count = hv_iterinit(hval); - HE* he; + } else if (svt < SVt_PVAV) { + STRLEN len = 0; + char *pv = svt ? SvPV (sv, len) : 0; - msgpack_pack_map(enc, count); - - while (he = hv_iternext(hval)) { - _msgpack_pack_sv(enc, hv_iterkeysv(he)); - _msgpack_pack_sv(enc, HeVAL(he)); + if (len == 1 && *pv == '1') + msgpack_pack_true(enc); + else if (len == 1 && *pv == '0') + msgpack_pack_false(enc); + else { + sv_dump(sv); + croak("cannot encode reference to scalar '%s' unless the scalar is 0 or 1", + SvPV_nolen (sv_2mortal (newRV_inc (sv)))); } - } else if (SvPOKp(val)) { - STRLEN len; - char * cval = SvPV(val, len); - - if (s_pref_int && try_int(enc, cval, len)) { - return; - } - - msgpack_pack_raw(enc, len); - msgpack_pack_raw_body(enc, cval, len); - } else if (SvIOK_UV(val)) { - msgpack_pack_uint32(enc, SvUV(val)); - } else if (SvIOK(val)) { - PACK_WRAPPER(IVTYPE)(enc, SvIV(val)); - } else if (SvNOK(val)) { - PACK_WRAPPER(NVTYPE)(enc, SvNV(val)); - } else if (!SvOK(val)) { - msgpack_pack_nil(enc); - } else if (isGV(val)) { - Perl_croak(aTHX_ "msgpack cannot pack the GV\n"); } else { - sv_dump(val); - Perl_croak(aTHX_ "msgpack for perl doesn't supported this type: %d\n", SvTYPE(val)); + croak ("encountered %s, but msgpack can only represent references to arrays or hashes", + SvPV_nolen (sv_2mortal (newRV_inc (sv)))); } } diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t index 22a8c14..50fd663 100644 --- a/perl/t/01_pack.t +++ b/perl/t/01_pack.t @@ -33,6 +33,7 @@ my @dat = ( -32768, 'd1 80 00', -32769, 'd2 ff ff 7f ff', 1.0, 'cb 3f f0 00 00 00 00 00 00', + do { my $x=3.0;my $y = "$x";$x }, 'a1 33', # PVNV "", 'a0', "a", 'a1 61', "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 'bf 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61', diff --git a/perl/unpack.c b/perl/unpack.c index 5a28fef..7a4f422 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -5,6 +5,7 @@ extern "C" { #include "EXTERN.h" #include "perl.h" #include "XSUB.h" +#include "util.h" #define NEED_newRV_noinc #define NEED_sv_2pv_flags #include "ppport.h" @@ -33,6 +34,21 @@ typedef struct { #define msgpack_unpack_user unpack_user +/* ---------------------------------------------------------------------- */ +/* utility functions */ + +static INLINE SV * +get_bool (const char *name) { + SV * sv = get_sv( name, 1 ); + + SvREADONLY_on(sv); + SvREADONLY_on( SvRV(sv) ); + + return sv; +} + +/* ---------------------------------------------------------------------- */ + struct template_context; typedef struct template_context msgpack_unpack_t; @@ -43,61 +59,61 @@ static SV* template_data(msgpack_unpack_t* u); static int template_execute(msgpack_unpack_t* u, const char* data, size_t len, size_t* off); -static inline SV* template_callback_root(unpack_user* u) +static INLINE SV* template_callback_root(unpack_user* u) { return &PL_sv_undef; } -static inline int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) +static INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) { *o = newSVuv(d); return 0; } -static inline int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) +static INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) { *o = newSVuv(d); return 0; } -static inline int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) +static INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) { *o = newSVuv(d); return 0; } -static inline int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) +static INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) { *o = newSVuv(d); return 0; } -static inline int template_callback_int8(unpack_user* u, int8_t d, SV** o) +static INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o) { *o = newSViv((long)d); return 0; } -static inline int template_callback_int16(unpack_user* u, int16_t d, SV** o) +static INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o) { *o = newSViv((long)d); return 0; } -static inline int template_callback_int32(unpack_user* u, int32_t d, SV** o) +static INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o) { *o = newSViv((long)d); return 0; } -static inline int template_callback_int64(unpack_user* u, int64_t d, SV** o) +static INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o) { *o = newSViv(d); return 0; } -static inline int template_callback_float(unpack_user* u, float d, SV** o) +static INLINE int template_callback_float(unpack_user* u, float d, SV** o) { *o = newSVnv(d); return 0; } -static inline int template_callback_double(unpack_user* u, double d, SV** o) +static INLINE int template_callback_double(unpack_user* u, double d, SV** o) { *o = newSVnv(d); return 0; } -static inline int template_callback_nil(unpack_user* u, SV** o) +static INLINE int template_callback_nil(unpack_user* u, SV** o) { *o = &PL_sv_undef; return 0; } -static inline int template_callback_true(unpack_user* u, SV** o) -{ *o = &PL_sv_yes; return 0; } +static INLINE int template_callback_true(unpack_user* u, SV** o) +{ *o = get_bool("Data::MessagePack::true") ; return 0; } -static inline int template_callback_false(unpack_user* u, SV** o) -{ *o = &PL_sv_no; return 0;} +static INLINE int template_callback_false(unpack_user* u, SV** o) +{ *o = get_bool("Data::MessagePack::false") ; return 0; } -static inline int template_callback_array(unpack_user* u, unsigned int n, SV** o) +static INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o) { AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; } -static inline int template_callback_array_item(unpack_user* u, SV** c, SV* o) +static INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o) { av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */ -static inline int template_callback_map(unpack_user* u, unsigned int n, SV** o) +static INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o) { HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; } -static inline int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) +static INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) { hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } -static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) +static INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) { *o = (l == 0) ? newSVpv("", 0) : newSVpv(p, l); return 0; } #define UNPACKER(from, name) \ diff --git a/perl/util.h b/perl/util.h new file mode 100644 index 0000000..2b4ed07 --- /dev/null +++ b/perl/util.h @@ -0,0 +1,11 @@ +#ifndef __PERL_MSGPACK_UTIL_H__ +#define __PERL_MSGPACK_UTIL_H__ + +#if __GNUC__ >= 3 +# define INLINE inline +#else +# define INLINE +#endif + +#endif // __PERL_MSGPACK_UTIL_H__ + From d3906bd1aba9996b548f49c35383c7099c64f09d Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 16:38:03 +0900 Subject: [PATCH 0268/1648] Perl: use M::I::XSUtil --- perl/.gitignore | 1 + perl/Makefile.PL | 23 +- perl/ppport.h | 7063 ---------------------------------------------- 3 files changed, 10 insertions(+), 7077 deletions(-) delete mode 100644 perl/ppport.h diff --git a/perl/.gitignore b/perl/.gitignore index a9096b6..f060908 100644 --- a/perl/.gitignore +++ b/perl/.gitignore @@ -10,3 +10,4 @@ pack.o pm_to_blib unpack.o MANIFEST +ppport.h diff --git a/perl/Makefile.PL b/perl/Makefile.PL index d1bff06..dc74d23 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -6,22 +6,17 @@ perl_version '5.008005'; license 'perl'; can_cc or die "This module requires a C compiler"; -makemaker_args( - OBJECT => '$(O_FILES)', - LIBS => [''], - clean => { - FILES => q{ - *.stackdump - *.gcov *.gcda *.gcno - *.out - nytprof - cover_db - }, - }, -); - tests 't/*.t'; author_tests('xt'); +use_ppport 3.19; + +clean_files qw{ + *.stackdump + *.gcov *.gcda *.gcno + *.out + nytprof + cover_db +}; if ($ENV{DEBUG}) { cc_append_to_ccflags '-g'; diff --git a/perl/ppport.h b/perl/ppport.h deleted file mode 100644 index ec2f1cc..0000000 --- a/perl/ppport.h +++ /dev/null @@ -1,7063 +0,0 @@ -#if 0 -<<'SKIP'; -#endif -/* ----------------------------------------------------------------------- - - ppport.h -- Perl/Pollution/Portability Version 3.19 - - Automatically created by Devel::PPPort running under perl 5.010000. - - Do NOT edit this file directly! -- Edit PPPort_pm.PL and the - includes in parts/inc/ instead. - - Use 'perldoc ppport.h' to view the documentation below. - ----------------------------------------------------------------------- - -SKIP - -=pod - -=head1 NAME - -ppport.h - Perl/Pollution/Portability version 3.19 - -=head1 SYNOPSIS - - perl ppport.h [options] [source files] - - Searches current directory for files if no [source files] are given - - --help show short help - - --version show version - - --patch=file write one patch file with changes - --copy=suffix write changed copies with suffix - --diff=program use diff program and options - - --compat-version=version provide compatibility with Perl version - --cplusplus accept C++ comments - - --quiet don't output anything except fatal errors - --nodiag don't show diagnostics - --nohints don't show hints - --nochanges don't suggest changes - --nofilter don't filter input files - - --strip strip all script and doc functionality from - ppport.h - - --list-provided list provided API - --list-unsupported list unsupported API - --api-info=name show Perl API portability information - -=head1 COMPATIBILITY - -This version of F is designed to support operation with Perl -installations back to 5.003, and has been tested up to 5.10.0. - -=head1 OPTIONS - -=head2 --help - -Display a brief usage summary. - -=head2 --version - -Display the version of F. - -=head2 --patch=I - -If this option is given, a single patch file will be created if -any changes are suggested. This requires a working diff program -to be installed on your system. - -=head2 --copy=I - -If this option is given, a copy of each file will be saved with -the given suffix that contains the suggested changes. This does -not require any external programs. Note that this does not -automagially add a dot between the original filename and the -suffix. If you want the dot, you have to include it in the option -argument. - -If neither C<--patch> or C<--copy> are given, the default is to -simply print the diffs for each file. This requires either -C or a C program to be installed. - -=head2 --diff=I - -Manually set the diff program and options to use. The default -is to use C, when installed, and output unified -context diffs. - -=head2 --compat-version=I - -Tell F to check for compatibility with the given -Perl version. The default is to check for compatibility with Perl -version 5.003. You can use this option to reduce the output -of F if you intend to be backward compatible only -down to a certain Perl version. - -=head2 --cplusplus - -Usually, F will detect C++ style comments and -replace them with C style comments for portability reasons. -Using this option instructs F to leave C++ -comments untouched. - -=head2 --quiet - -Be quiet. Don't print anything except fatal errors. - -=head2 --nodiag - -Don't output any diagnostic messages. Only portability -alerts will be printed. - -=head2 --nohints - -Don't output any hints. Hints often contain useful portability -notes. Warnings will still be displayed. - -=head2 --nochanges - -Don't suggest any changes. Only give diagnostic output and hints -unless these are also deactivated. - -=head2 --nofilter - -Don't filter the list of input files. By default, files not looking -like source code (i.e. not *.xs, *.c, *.cc, *.cpp or *.h) are skipped. - -=head2 --strip - -Strip all script and documentation functionality from F. -This reduces the size of F dramatically and may be useful -if you want to include F in smaller modules without -increasing their distribution size too much. - -The stripped F will have a C<--unstrip> option that allows -you to undo the stripping, but only if an appropriate C -module is installed. - -=head2 --list-provided - -Lists the API elements for which compatibility is provided by -F. Also lists if it must be explicitly requested, -if it has dependencies, and if there are hints or warnings for it. - -=head2 --list-unsupported - -Lists the API elements that are known not to be supported by -F and below which version of Perl they probably -won't be available or work. - -=head2 --api-info=I - -Show portability information for API elements matching I. -If I is surrounded by slashes, it is interpreted as a regular -expression. - -=head1 DESCRIPTION - -In order for a Perl extension (XS) module to be as portable as possible -across differing versions of Perl itself, certain steps need to be taken. - -=over 4 - -=item * - -Including this header is the first major one. This alone will give you -access to a large part of the Perl API that hasn't been available in -earlier Perl releases. Use - - perl ppport.h --list-provided - -to see which API elements are provided by ppport.h. - -=item * - -You should avoid using deprecated parts of the API. For example, using -global Perl variables without the C prefix is deprecated. Also, -some API functions used to have a C prefix. Using this form is -also deprecated. You can safely use the supported API, as F -will provide wrappers for older Perl versions. - -=item * - -If you use one of a few functions or variables that were not present in -earlier versions of Perl, and that can't be provided using a macro, you -have to explicitly request support for these functions by adding one or -more C<#define>s in your source code before the inclusion of F. - -These functions or variables will be marked C in the list shown -by C<--list-provided>. - -Depending on whether you module has a single or multiple files that -use such functions or variables, you want either C or global -variants. - -For a C function or variable (used only in a single source -file), use: - - #define NEED_function - #define NEED_variable - -For a global function or variable (used in multiple source files), -use: - - #define NEED_function_GLOBAL - #define NEED_variable_GLOBAL - -Note that you mustn't have more than one global request for the -same function or variable in your project. - - Function / Variable Static Request Global Request - ----------------------------------------------------------------------------------------- - PL_parser NEED_PL_parser NEED_PL_parser_GLOBAL - PL_signals NEED_PL_signals NEED_PL_signals_GLOBAL - eval_pv() NEED_eval_pv NEED_eval_pv_GLOBAL - grok_bin() NEED_grok_bin NEED_grok_bin_GLOBAL - grok_hex() NEED_grok_hex NEED_grok_hex_GLOBAL - grok_number() NEED_grok_number NEED_grok_number_GLOBAL - grok_numeric_radix() NEED_grok_numeric_radix NEED_grok_numeric_radix_GLOBAL - grok_oct() NEED_grok_oct NEED_grok_oct_GLOBAL - load_module() NEED_load_module NEED_load_module_GLOBAL - my_snprintf() NEED_my_snprintf NEED_my_snprintf_GLOBAL - my_sprintf() NEED_my_sprintf NEED_my_sprintf_GLOBAL - my_strlcat() NEED_my_strlcat NEED_my_strlcat_GLOBAL - my_strlcpy() NEED_my_strlcpy NEED_my_strlcpy_GLOBAL - newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL - newRV_noinc() NEED_newRV_noinc NEED_newRV_noinc_GLOBAL - newSV_type() NEED_newSV_type NEED_newSV_type_GLOBAL - newSVpvn_flags() NEED_newSVpvn_flags NEED_newSVpvn_flags_GLOBAL - newSVpvn_share() NEED_newSVpvn_share NEED_newSVpvn_share_GLOBAL - pv_display() NEED_pv_display NEED_pv_display_GLOBAL - pv_escape() NEED_pv_escape NEED_pv_escape_GLOBAL - pv_pretty() NEED_pv_pretty NEED_pv_pretty_GLOBAL - sv_2pv_flags() NEED_sv_2pv_flags NEED_sv_2pv_flags_GLOBAL - sv_2pvbyte() NEED_sv_2pvbyte NEED_sv_2pvbyte_GLOBAL - sv_catpvf_mg() NEED_sv_catpvf_mg NEED_sv_catpvf_mg_GLOBAL - sv_catpvf_mg_nocontext() NEED_sv_catpvf_mg_nocontext NEED_sv_catpvf_mg_nocontext_GLOBAL - sv_pvn_force_flags() NEED_sv_pvn_force_flags NEED_sv_pvn_force_flags_GLOBAL - sv_setpvf_mg() NEED_sv_setpvf_mg NEED_sv_setpvf_mg_GLOBAL - sv_setpvf_mg_nocontext() NEED_sv_setpvf_mg_nocontext NEED_sv_setpvf_mg_nocontext_GLOBAL - vload_module() NEED_vload_module NEED_vload_module_GLOBAL - vnewSVpvf() NEED_vnewSVpvf NEED_vnewSVpvf_GLOBAL - warner() NEED_warner NEED_warner_GLOBAL - -To avoid namespace conflicts, you can change the namespace of the -explicitly exported functions / variables using the C -macro. Just C<#define> the macro before including C: - - #define DPPP_NAMESPACE MyOwnNamespace_ - #include "ppport.h" - -The default namespace is C. - -=back - -The good thing is that most of the above can be checked by running -F on your source code. See the next section for -details. - -=head1 EXAMPLES - -To verify whether F is needed for your module, whether you -should make any changes to your code, and whether any special defines -should be used, F can be run as a Perl script to check your -source code. Simply say: - - perl ppport.h - -The result will usually be a list of patches suggesting changes -that should at least be acceptable, if not necessarily the most -efficient solution, or a fix for all possible problems. - -If you know that your XS module uses features only available in -newer Perl releases, if you're aware that it uses C++ comments, -and if you want all suggestions as a single patch file, you could -use something like this: - - perl ppport.h --compat-version=5.6.0 --cplusplus --patch=test.diff - -If you only want your code to be scanned without any suggestions -for changes, use: - - perl ppport.h --nochanges - -You can specify a different C program or options, using -the C<--diff> option: - - perl ppport.h --diff='diff -C 10' - -This would output context diffs with 10 lines of context. - -If you want to create patched copies of your files instead, use: - - perl ppport.h --copy=.new - -To display portability information for the C function, -use: - - perl ppport.h --api-info=newSVpvn - -Since the argument to C<--api-info> can be a regular expression, -you can use - - perl ppport.h --api-info=/_nomg$/ - -to display portability information for all C<_nomg> functions or - - perl ppport.h --api-info=/./ - -to display information for all known API elements. - -=head1 BUGS - -If this version of F is causing failure during -the compilation of this module, please check if newer versions -of either this module or C are available on CPAN -before sending a bug report. - -If F was generated using the latest version of -C and is causing failure of this module, please -file a bug report using the CPAN Request Tracker at L. - -Please include the following information: - -=over 4 - -=item 1. - -The complete output from running "perl -V" - -=item 2. - -This file. - -=item 3. - -The name and version of the module you were trying to build. - -=item 4. - -A full log of the build that failed. - -=item 5. - -Any other information that you think could be relevant. - -=back - -For the latest version of this code, please get the C -module from CPAN. - -=head1 COPYRIGHT - -Version 3.x, Copyright (c) 2004-2009, Marcus Holland-Moritz. - -Version 2.x, Copyright (C) 2001, Paul Marquess. - -Version 1.x, Copyright (C) 1999, Kenneth Albanowski. - -This program is free software; you can redistribute it and/or -modify it under the same terms as Perl itself. - -=head1 SEE ALSO - -See L. - -=cut - -use strict; - -# Disable broken TRIE-optimization -BEGIN { eval '${^RE_TRIE_MAXBUF} = -1' if $] >= 5.009004 && $] <= 5.009005 } - -my $VERSION = 3.19; - -my %opt = ( - quiet => 0, - diag => 1, - hints => 1, - changes => 1, - cplusplus => 0, - filter => 1, - strip => 0, - version => 0, -); - -my($ppport) = $0 =~ /([\w.]+)$/; -my $LF = '(?:\r\n|[\r\n])'; # line feed -my $HS = "[ \t]"; # horizontal whitespace - -# Never use C comments in this file! -my $ccs = '/'.'*'; -my $cce = '*'.'/'; -my $rccs = quotemeta $ccs; -my $rcce = quotemeta $cce; - -eval { - require Getopt::Long; - Getopt::Long::GetOptions(\%opt, qw( - help quiet diag! filter! hints! changes! cplusplus strip version - patch=s copy=s diff=s compat-version=s - list-provided list-unsupported api-info=s - )) or usage(); -}; - -if ($@ and grep /^-/, @ARGV) { - usage() if "@ARGV" =~ /^--?h(?:elp)?$/; - die "Getopt::Long not found. Please don't use any options.\n"; -} - -if ($opt{version}) { - print "This is $0 $VERSION.\n"; - exit 0; -} - -usage() if $opt{help}; -strip() if $opt{strip}; - -if (exists $opt{'compat-version'}) { - my($r,$v,$s) = eval { parse_version($opt{'compat-version'}) }; - if ($@) { - die "Invalid version number format: '$opt{'compat-version'}'\n"; - } - die "Only Perl 5 is supported\n" if $r != 5; - die "Invalid version number: $opt{'compat-version'}\n" if $v >= 1000 || $s >= 1000; - $opt{'compat-version'} = sprintf "%d.%03d%03d", $r, $v, $s; -} -else { - $opt{'compat-version'} = 5; -} - -my %API = map { /^(\w+)\|([^|]*)\|([^|]*)\|(\w*)$/ - ? ( $1 => { - ($2 ? ( base => $2 ) : ()), - ($3 ? ( todo => $3 ) : ()), - (index($4, 'v') >= 0 ? ( varargs => 1 ) : ()), - (index($4, 'p') >= 0 ? ( provided => 1 ) : ()), - (index($4, 'n') >= 0 ? ( nothxarg => 1 ) : ()), - } ) - : die "invalid spec: $_" } qw( -AvFILLp|5.004050||p -AvFILL||| -CLASS|||n -CPERLscope|5.005000||p -CX_CURPAD_SAVE||| -CX_CURPAD_SV||| -CopFILEAV|5.006000||p -CopFILEGV_set|5.006000||p -CopFILEGV|5.006000||p -CopFILESV|5.006000||p -CopFILE_set|5.006000||p -CopFILE|5.006000||p -CopSTASHPV_set|5.006000||p -CopSTASHPV|5.006000||p -CopSTASH_eq|5.006000||p -CopSTASH_set|5.006000||p -CopSTASH|5.006000||p -CopyD|5.009002||p -Copy||| -CvPADLIST||| -CvSTASH||| -CvWEAKOUTSIDE||| -DEFSV_set|5.011000||p -DEFSV|5.004050||p -END_EXTERN_C|5.005000||p -ENTER||| -ERRSV|5.004050||p -EXTEND||| -EXTERN_C|5.005000||p -F0convert|||n -FREETMPS||| -GIMME_V||5.004000|n -GIMME|||n -GROK_NUMERIC_RADIX|5.007002||p -G_ARRAY||| -G_DISCARD||| -G_EVAL||| -G_METHOD|5.006001||p -G_NOARGS||| -G_SCALAR||| -G_VOID||5.004000| -GetVars||| -GvSVn|5.009003||p -GvSV||| -Gv_AMupdate||| -HEf_SVKEY||5.004000| -HeHASH||5.004000| -HeKEY||5.004000| -HeKLEN||5.004000| -HePV||5.004000| -HeSVKEY_force||5.004000| -HeSVKEY_set||5.004000| -HeSVKEY||5.004000| -HeUTF8||5.011000| -HeVAL||5.004000| -HvNAMELEN_get|5.009003||p -HvNAME_get|5.009003||p -HvNAME||| -INT2PTR|5.006000||p -IN_LOCALE_COMPILETIME|5.007002||p -IN_LOCALE_RUNTIME|5.007002||p -IN_LOCALE|5.007002||p -IN_PERL_COMPILETIME|5.008001||p -IS_NUMBER_GREATER_THAN_UV_MAX|5.007002||p -IS_NUMBER_INFINITY|5.007002||p -IS_NUMBER_IN_UV|5.007002||p -IS_NUMBER_NAN|5.007003||p -IS_NUMBER_NEG|5.007002||p -IS_NUMBER_NOT_INT|5.007002||p -IVSIZE|5.006000||p -IVTYPE|5.006000||p -IVdf|5.006000||p -LEAVE||| -LVRET||| -MARK||| -MULTICALL||5.011000| -MY_CXT_CLONE|5.009002||p -MY_CXT_INIT|5.007003||p -MY_CXT|5.007003||p -MoveD|5.009002||p -Move||| -NOOP|5.005000||p -NUM2PTR|5.006000||p -NVTYPE|5.006000||p -NVef|5.006001||p -NVff|5.006001||p -NVgf|5.006001||p -Newxc|5.009003||p -Newxz|5.009003||p -Newx|5.009003||p -Nullav||| -Nullch||| -Nullcv||| -Nullhv||| -Nullsv||| -ORIGMARK||| -PAD_BASE_SV||| -PAD_CLONE_VARS||| -PAD_COMPNAME_FLAGS||| -PAD_COMPNAME_GEN_set||| -PAD_COMPNAME_GEN||| -PAD_COMPNAME_OURSTASH||| -PAD_COMPNAME_PV||| -PAD_COMPNAME_TYPE||| -PAD_DUP||| -PAD_RESTORE_LOCAL||| -PAD_SAVE_LOCAL||| -PAD_SAVE_SETNULLPAD||| -PAD_SETSV||| -PAD_SET_CUR_NOSAVE||| -PAD_SET_CUR||| -PAD_SVl||| -PAD_SV||| -PERLIO_FUNCS_CAST|5.009003||p -PERLIO_FUNCS_DECL|5.009003||p -PERL_ABS|5.008001||p -PERL_BCDVERSION|5.011000||p -PERL_GCC_BRACE_GROUPS_FORBIDDEN|5.008001||p -PERL_HASH|5.004000||p -PERL_INT_MAX|5.004000||p -PERL_INT_MIN|5.004000||p -PERL_LONG_MAX|5.004000||p -PERL_LONG_MIN|5.004000||p -PERL_MAGIC_arylen|5.007002||p -PERL_MAGIC_backref|5.007002||p -PERL_MAGIC_bm|5.007002||p -PERL_MAGIC_collxfrm|5.007002||p -PERL_MAGIC_dbfile|5.007002||p -PERL_MAGIC_dbline|5.007002||p -PERL_MAGIC_defelem|5.007002||p -PERL_MAGIC_envelem|5.007002||p -PERL_MAGIC_env|5.007002||p -PERL_MAGIC_ext|5.007002||p -PERL_MAGIC_fm|5.007002||p -PERL_MAGIC_glob|5.011000||p -PERL_MAGIC_isaelem|5.007002||p -PERL_MAGIC_isa|5.007002||p -PERL_MAGIC_mutex|5.011000||p -PERL_MAGIC_nkeys|5.007002||p -PERL_MAGIC_overload_elem|5.007002||p -PERL_MAGIC_overload_table|5.007002||p -PERL_MAGIC_overload|5.007002||p -PERL_MAGIC_pos|5.007002||p -PERL_MAGIC_qr|5.007002||p -PERL_MAGIC_regdata|5.007002||p -PERL_MAGIC_regdatum|5.007002||p -PERL_MAGIC_regex_global|5.007002||p -PERL_MAGIC_shared_scalar|5.007003||p -PERL_MAGIC_shared|5.007003||p -PERL_MAGIC_sigelem|5.007002||p -PERL_MAGIC_sig|5.007002||p -PERL_MAGIC_substr|5.007002||p -PERL_MAGIC_sv|5.007002||p -PERL_MAGIC_taint|5.007002||p -PERL_MAGIC_tiedelem|5.007002||p -PERL_MAGIC_tiedscalar|5.007002||p -PERL_MAGIC_tied|5.007002||p -PERL_MAGIC_utf8|5.008001||p -PERL_MAGIC_uvar_elem|5.007003||p -PERL_MAGIC_uvar|5.007002||p -PERL_MAGIC_vec|5.007002||p -PERL_MAGIC_vstring|5.008001||p -PERL_PV_ESCAPE_ALL|5.009004||p -PERL_PV_ESCAPE_FIRSTCHAR|5.009004||p -PERL_PV_ESCAPE_NOBACKSLASH|5.009004||p -PERL_PV_ESCAPE_NOCLEAR|5.009004||p -PERL_PV_ESCAPE_QUOTE|5.009004||p -PERL_PV_ESCAPE_RE|5.009005||p -PERL_PV_ESCAPE_UNI_DETECT|5.009004||p -PERL_PV_ESCAPE_UNI|5.009004||p -PERL_PV_PRETTY_DUMP|5.009004||p -PERL_PV_PRETTY_ELLIPSES|5.010000||p -PERL_PV_PRETTY_LTGT|5.009004||p -PERL_PV_PRETTY_NOCLEAR|5.010000||p -PERL_PV_PRETTY_QUOTE|5.009004||p -PERL_PV_PRETTY_REGPROP|5.009004||p -PERL_QUAD_MAX|5.004000||p -PERL_QUAD_MIN|5.004000||p -PERL_REVISION|5.006000||p -PERL_SCAN_ALLOW_UNDERSCORES|5.007003||p -PERL_SCAN_DISALLOW_PREFIX|5.007003||p -PERL_SCAN_GREATER_THAN_UV_MAX|5.007003||p -PERL_SCAN_SILENT_ILLDIGIT|5.008001||p -PERL_SHORT_MAX|5.004000||p -PERL_SHORT_MIN|5.004000||p -PERL_SIGNALS_UNSAFE_FLAG|5.008001||p -PERL_SUBVERSION|5.006000||p -PERL_SYS_INIT3||5.006000| -PERL_SYS_INIT||| -PERL_SYS_TERM||5.011000| -PERL_UCHAR_MAX|5.004000||p -PERL_UCHAR_MIN|5.004000||p -PERL_UINT_MAX|5.004000||p -PERL_UINT_MIN|5.004000||p -PERL_ULONG_MAX|5.004000||p -PERL_ULONG_MIN|5.004000||p -PERL_UNUSED_ARG|5.009003||p -PERL_UNUSED_CONTEXT|5.009004||p -PERL_UNUSED_DECL|5.007002||p -PERL_UNUSED_VAR|5.007002||p -PERL_UQUAD_MAX|5.004000||p -PERL_UQUAD_MIN|5.004000||p -PERL_USE_GCC_BRACE_GROUPS|5.009004||p -PERL_USHORT_MAX|5.004000||p -PERL_USHORT_MIN|5.004000||p -PERL_VERSION|5.006000||p -PL_DBsignal|5.005000||p -PL_DBsingle|||pn -PL_DBsub|||pn -PL_DBtrace|||pn -PL_Sv|5.005000||p -PL_bufend|5.011000||p -PL_bufptr|5.011000||p -PL_compiling|5.004050||p -PL_copline|5.011000||p -PL_curcop|5.004050||p -PL_curstash|5.004050||p -PL_debstash|5.004050||p -PL_defgv|5.004050||p -PL_diehook|5.004050||p -PL_dirty|5.004050||p -PL_dowarn|||pn -PL_errgv|5.004050||p -PL_error_count|5.011000||p -PL_expect|5.011000||p -PL_hexdigit|5.005000||p -PL_hints|5.005000||p -PL_in_my_stash|5.011000||p -PL_in_my|5.011000||p -PL_last_in_gv|||n -PL_laststatval|5.005000||p -PL_lex_state|5.011000||p -PL_lex_stuff|5.011000||p -PL_linestr|5.011000||p -PL_modglobal||5.005000|n -PL_na|5.004050||pn -PL_no_modify|5.006000||p -PL_ofsgv|||n -PL_parser|5.009005||p -PL_perl_destruct_level|5.004050||p -PL_perldb|5.004050||p -PL_ppaddr|5.006000||p -PL_rsfp_filters|5.004050||p -PL_rsfp|5.004050||p -PL_rs|||n -PL_signals|5.008001||p -PL_stack_base|5.004050||p -PL_stack_sp|5.004050||p -PL_statcache|5.005000||p -PL_stdingv|5.004050||p -PL_sv_arenaroot|5.004050||p -PL_sv_no|5.004050||pn -PL_sv_undef|5.004050||pn -PL_sv_yes|5.004050||pn -PL_tainted|5.004050||p -PL_tainting|5.004050||p -PL_tokenbuf|5.011000||p -POP_MULTICALL||5.011000| -POPi|||n -POPl|||n -POPn|||n -POPpbytex||5.007001|n -POPpx||5.005030|n -POPp|||n -POPs|||n -PTR2IV|5.006000||p -PTR2NV|5.006000||p -PTR2UV|5.006000||p -PTR2nat|5.009003||p -PTR2ul|5.007001||p -PTRV|5.006000||p -PUSHMARK||| -PUSH_MULTICALL||5.011000| -PUSHi||| -PUSHmortal|5.009002||p -PUSHn||| -PUSHp||| -PUSHs||| -PUSHu|5.004000||p -PUTBACK||| -PerlIO_clearerr||5.007003| -PerlIO_close||5.007003| -PerlIO_context_layers||5.009004| -PerlIO_eof||5.007003| -PerlIO_error||5.007003| -PerlIO_fileno||5.007003| -PerlIO_fill||5.007003| -PerlIO_flush||5.007003| -PerlIO_get_base||5.007003| -PerlIO_get_bufsiz||5.007003| -PerlIO_get_cnt||5.007003| -PerlIO_get_ptr||5.007003| -PerlIO_read||5.007003| -PerlIO_seek||5.007003| -PerlIO_set_cnt||5.007003| -PerlIO_set_ptrcnt||5.007003| -PerlIO_setlinebuf||5.007003| -PerlIO_stderr||5.007003| -PerlIO_stdin||5.007003| -PerlIO_stdout||5.007003| -PerlIO_tell||5.007003| -PerlIO_unread||5.007003| -PerlIO_write||5.007003| -Perl_signbit||5.009005|n -PoisonFree|5.009004||p -PoisonNew|5.009004||p -PoisonWith|5.009004||p -Poison|5.008000||p -RETVAL|||n -Renewc||| -Renew||| -SAVECLEARSV||| -SAVECOMPPAD||| -SAVEPADSV||| -SAVETMPS||| -SAVE_DEFSV|5.004050||p -SPAGAIN||| -SP||| -START_EXTERN_C|5.005000||p -START_MY_CXT|5.007003||p -STMT_END|||p -STMT_START|||p -STR_WITH_LEN|5.009003||p -ST||| -SV_CONST_RETURN|5.009003||p -SV_COW_DROP_PV|5.008001||p -SV_COW_SHARED_HASH_KEYS|5.009005||p -SV_GMAGIC|5.007002||p -SV_HAS_TRAILING_NUL|5.009004||p -SV_IMMEDIATE_UNREF|5.007001||p -SV_MUTABLE_RETURN|5.009003||p -SV_NOSTEAL|5.009002||p -SV_SMAGIC|5.009003||p -SV_UTF8_NO_ENCODING|5.008001||p -SVfARG|5.009005||p -SVf_UTF8|5.006000||p -SVf|5.006000||p -SVt_IV||| -SVt_NV||| -SVt_PVAV||| -SVt_PVCV||| -SVt_PVHV||| -SVt_PVMG||| -SVt_PV||| -Safefree||| -Slab_Alloc||| -Slab_Free||| -Slab_to_rw||| -StructCopy||| -SvCUR_set||| -SvCUR||| -SvEND||| -SvGAMAGIC||5.006001| -SvGETMAGIC|5.004050||p -SvGROW||| -SvIOK_UV||5.006000| -SvIOK_notUV||5.006000| -SvIOK_off||| -SvIOK_only_UV||5.006000| -SvIOK_only||| -SvIOK_on||| -SvIOKp||| -SvIOK||| -SvIVX||| -SvIV_nomg|5.009001||p -SvIV_set||| -SvIVx||| -SvIV||| -SvIsCOW_shared_hash||5.008003| -SvIsCOW||5.008003| -SvLEN_set||| -SvLEN||| -SvLOCK||5.007003| -SvMAGIC_set|5.009003||p -SvNIOK_off||| -SvNIOKp||| -SvNIOK||| -SvNOK_off||| -SvNOK_only||| -SvNOK_on||| -SvNOKp||| -SvNOK||| -SvNVX||| -SvNV_set||| -SvNVx||| -SvNV||| -SvOK||| -SvOOK_offset||5.011000| -SvOOK||| -SvPOK_off||| -SvPOK_only_UTF8||5.006000| -SvPOK_only||| -SvPOK_on||| -SvPOKp||| -SvPOK||| -SvPVX_const|5.009003||p -SvPVX_mutable|5.009003||p -SvPVX||| -SvPV_const|5.009003||p -SvPV_flags_const_nolen|5.009003||p -SvPV_flags_const|5.009003||p -SvPV_flags_mutable|5.009003||p -SvPV_flags|5.007002||p -SvPV_force_flags_mutable|5.009003||p -SvPV_force_flags_nolen|5.009003||p -SvPV_force_flags|5.007002||p -SvPV_force_mutable|5.009003||p -SvPV_force_nolen|5.009003||p -SvPV_force_nomg_nolen|5.009003||p -SvPV_force_nomg|5.007002||p -SvPV_force|||p -SvPV_mutable|5.009003||p -SvPV_nolen_const|5.009003||p -SvPV_nolen|5.006000||p -SvPV_nomg_const_nolen|5.009003||p -SvPV_nomg_const|5.009003||p -SvPV_nomg|5.007002||p -SvPV_renew|5.009003||p -SvPV_set||| -SvPVbyte_force||5.009002| -SvPVbyte_nolen||5.006000| -SvPVbytex_force||5.006000| -SvPVbytex||5.006000| -SvPVbyte|5.006000||p -SvPVutf8_force||5.006000| -SvPVutf8_nolen||5.006000| -SvPVutf8x_force||5.006000| -SvPVutf8x||5.006000| -SvPVutf8||5.006000| -SvPVx||| -SvPV||| -SvREFCNT_dec||| -SvREFCNT_inc_NN|5.009004||p -SvREFCNT_inc_simple_NN|5.009004||p -SvREFCNT_inc_simple_void_NN|5.009004||p -SvREFCNT_inc_simple_void|5.009004||p -SvREFCNT_inc_simple|5.009004||p -SvREFCNT_inc_void_NN|5.009004||p -SvREFCNT_inc_void|5.009004||p -SvREFCNT_inc|||p -SvREFCNT||| -SvROK_off||| -SvROK_on||| -SvROK||| -SvRV_set|5.009003||p -SvRV||| -SvRXOK||5.009005| -SvRX||5.009005| -SvSETMAGIC||| -SvSHARED_HASH|5.009003||p -SvSHARE||5.007003| -SvSTASH_set|5.009003||p -SvSTASH||| -SvSetMagicSV_nosteal||5.004000| -SvSetMagicSV||5.004000| -SvSetSV_nosteal||5.004000| -SvSetSV||| -SvTAINTED_off||5.004000| -SvTAINTED_on||5.004000| -SvTAINTED||5.004000| -SvTAINT||| -SvTRUE||| -SvTYPE||| -SvUNLOCK||5.007003| -SvUOK|5.007001|5.006000|p -SvUPGRADE||| -SvUTF8_off||5.006000| -SvUTF8_on||5.006000| -SvUTF8||5.006000| -SvUVXx|5.004000||p -SvUVX|5.004000||p -SvUV_nomg|5.009001||p -SvUV_set|5.009003||p -SvUVx|5.004000||p -SvUV|5.004000||p -SvVOK||5.008001| -SvVSTRING_mg|5.009004||p -THIS|||n -UNDERBAR|5.009002||p -UTF8_MAXBYTES|5.009002||p -UVSIZE|5.006000||p -UVTYPE|5.006000||p -UVXf|5.007001||p -UVof|5.006000||p -UVuf|5.006000||p -UVxf|5.006000||p -WARN_ALL|5.006000||p -WARN_AMBIGUOUS|5.006000||p -WARN_ASSERTIONS|5.011000||p -WARN_BAREWORD|5.006000||p -WARN_CLOSED|5.006000||p -WARN_CLOSURE|5.006000||p -WARN_DEBUGGING|5.006000||p -WARN_DEPRECATED|5.006000||p -WARN_DIGIT|5.006000||p -WARN_EXEC|5.006000||p -WARN_EXITING|5.006000||p -WARN_GLOB|5.006000||p -WARN_INPLACE|5.006000||p -WARN_INTERNAL|5.006000||p -WARN_IO|5.006000||p -WARN_LAYER|5.008000||p -WARN_MALLOC|5.006000||p -WARN_MISC|5.006000||p -WARN_NEWLINE|5.006000||p -WARN_NUMERIC|5.006000||p -WARN_ONCE|5.006000||p -WARN_OVERFLOW|5.006000||p -WARN_PACK|5.006000||p -WARN_PARENTHESIS|5.006000||p -WARN_PIPE|5.006000||p -WARN_PORTABLE|5.006000||p -WARN_PRECEDENCE|5.006000||p -WARN_PRINTF|5.006000||p -WARN_PROTOTYPE|5.006000||p -WARN_QW|5.006000||p -WARN_RECURSION|5.006000||p -WARN_REDEFINE|5.006000||p -WARN_REGEXP|5.006000||p -WARN_RESERVED|5.006000||p -WARN_SEMICOLON|5.006000||p -WARN_SEVERE|5.006000||p -WARN_SIGNAL|5.006000||p -WARN_SUBSTR|5.006000||p -WARN_SYNTAX|5.006000||p -WARN_TAINT|5.006000||p -WARN_THREADS|5.008000||p -WARN_UNINITIALIZED|5.006000||p -WARN_UNOPENED|5.006000||p -WARN_UNPACK|5.006000||p -WARN_UNTIE|5.006000||p -WARN_UTF8|5.006000||p -WARN_VOID|5.006000||p -XCPT_CATCH|5.009002||p -XCPT_RETHROW|5.009002||p -XCPT_TRY_END|5.009002||p -XCPT_TRY_START|5.009002||p -XPUSHi||| -XPUSHmortal|5.009002||p -XPUSHn||| -XPUSHp||| -XPUSHs||| -XPUSHu|5.004000||p -XSPROTO|5.010000||p -XSRETURN_EMPTY||| -XSRETURN_IV||| -XSRETURN_NO||| -XSRETURN_NV||| -XSRETURN_PV||| -XSRETURN_UNDEF||| -XSRETURN_UV|5.008001||p -XSRETURN_YES||| -XSRETURN|||p -XST_mIV||| -XST_mNO||| -XST_mNV||| -XST_mPV||| -XST_mUNDEF||| -XST_mUV|5.008001||p -XST_mYES||| -XS_VERSION_BOOTCHECK||| -XS_VERSION||| -XSprePUSH|5.006000||p -XS||| -ZeroD|5.009002||p -Zero||| -_aMY_CXT|5.007003||p -_pMY_CXT|5.007003||p -aMY_CXT_|5.007003||p -aMY_CXT|5.007003||p -aTHXR_|5.011000||p -aTHXR|5.011000||p -aTHX_|5.006000||p -aTHX|5.006000||p -add_data|||n -addmad||| -allocmy||| -amagic_call||| -amagic_cmp_locale||| -amagic_cmp||| -amagic_i_ncmp||| -amagic_ncmp||| -any_dup||| -ao||| -append_elem||| -append_list||| -append_madprops||| -apply_attrs_my||| -apply_attrs_string||5.006001| -apply_attrs||| -apply||| -atfork_lock||5.007003|n -atfork_unlock||5.007003|n -av_arylen_p||5.009003| -av_clear||| -av_create_and_push||5.009005| -av_create_and_unshift_one||5.009005| -av_delete||5.006000| -av_exists||5.006000| -av_extend||| -av_fetch||| -av_fill||| -av_iter_p||5.011000| -av_len||| -av_make||| -av_pop||| -av_push||| -av_reify||| -av_shift||| -av_store||| -av_undef||| -av_unshift||| -ax|||n -bad_type||| -bind_match||| -block_end||| -block_gimme||5.004000| -block_start||| -boolSV|5.004000||p -boot_core_PerlIO||| -boot_core_UNIVERSAL||| -boot_core_mro||| -bytes_from_utf8||5.007001| -bytes_to_uni|||n -bytes_to_utf8||5.006001| -call_argv|5.006000||p -call_atexit||5.006000| -call_list||5.004000| -call_method|5.006000||p -call_pv|5.006000||p -call_sv|5.006000||p -calloc||5.007002|n -cando||| -cast_i32||5.006000| -cast_iv||5.006000| -cast_ulong||5.006000| -cast_uv||5.006000| -check_type_and_open||| -check_uni||| -checkcomma||| -checkposixcc||| -ckWARN|5.006000||p -ck_anoncode||| -ck_bitop||| -ck_concat||| -ck_defined||| -ck_delete||| -ck_die||| -ck_each||| -ck_eof||| -ck_eval||| -ck_exec||| -ck_exists||| -ck_exit||| -ck_ftst||| -ck_fun||| -ck_glob||| -ck_grep||| -ck_index||| -ck_join||| -ck_lfun||| -ck_listiob||| -ck_match||| -ck_method||| -ck_null||| -ck_open||| -ck_readline||| -ck_repeat||| -ck_require||| -ck_return||| -ck_rfun||| -ck_rvconst||| -ck_sassign||| -ck_select||| -ck_shift||| -ck_sort||| -ck_spair||| -ck_split||| -ck_subr||| -ck_substr||| -ck_svconst||| -ck_trunc||| -ck_unpack||| -ckwarn_d||5.009003| -ckwarn||5.009003| -cl_and|||n -cl_anything|||n -cl_init_zero|||n -cl_init|||n -cl_is_anything|||n -cl_or|||n -clear_placeholders||| -closest_cop||| -convert||| -cop_free||| -cr_textfilter||| -create_eval_scope||| -croak_nocontext|||vn -croak_xs_usage||5.011000| -croak|||v -csighandler||5.009003|n -curmad||| -custom_op_desc||5.007003| -custom_op_name||5.007003| -cv_ckproto_len||| -cv_clone||| -cv_const_sv||5.004000| -cv_dump||| -cv_undef||| -cx_dump||5.005000| -cx_dup||| -cxinc||| -dAXMARK|5.009003||p -dAX|5.007002||p -dITEMS|5.007002||p -dMARK||| -dMULTICALL||5.009003| -dMY_CXT_SV|5.007003||p -dMY_CXT|5.007003||p -dNOOP|5.006000||p -dORIGMARK||| -dSP||| -dTHR|5.004050||p -dTHXR|5.011000||p -dTHXa|5.006000||p -dTHXoa|5.006000||p -dTHX|5.006000||p -dUNDERBAR|5.009002||p -dVAR|5.009003||p -dXCPT|5.009002||p -dXSARGS||| -dXSI32||| -dXSTARG|5.006000||p -deb_curcv||| -deb_nocontext|||vn -deb_stack_all||| -deb_stack_n||| -debop||5.005000| -debprofdump||5.005000| -debprof||| -debstackptrs||5.007003| -debstack||5.007003| -debug_start_match||| -deb||5.007003|v -del_sv||| -delete_eval_scope||| -delimcpy||5.004000| -deprecate_old||| -deprecate||| -despatch_signals||5.007001| -destroy_matcher||| -die_nocontext|||vn -die_where||| -die|||v -dirp_dup||| -div128||| -djSP||| -do_aexec5||| -do_aexec||| -do_aspawn||| -do_binmode||5.004050| -do_chomp||| -do_chop||| -do_close||| -do_dump_pad||| -do_eof||| -do_exec3||| -do_execfree||| -do_exec||| -do_gv_dump||5.006000| -do_gvgv_dump||5.006000| -do_hv_dump||5.006000| -do_ipcctl||| -do_ipcget||| -do_join||| -do_kv||| -do_magic_dump||5.006000| -do_msgrcv||| -do_msgsnd||| -do_oddball||| -do_op_dump||5.006000| -do_op_xmldump||| -do_open9||5.006000| -do_openn||5.007001| -do_open||5.004000| -do_pmop_dump||5.006000| -do_pmop_xmldump||| -do_print||| -do_readline||| -do_seek||| -do_semop||| -do_shmio||| -do_smartmatch||| -do_spawn_nowait||| -do_spawn||| -do_sprintf||| -do_sv_dump||5.006000| -do_sysseek||| -do_tell||| -do_trans_complex_utf8||| -do_trans_complex||| -do_trans_count_utf8||| -do_trans_count||| -do_trans_simple_utf8||| -do_trans_simple||| -do_trans||| -do_vecget||| -do_vecset||| -do_vop||| -docatch||| -doeval||| -dofile||| -dofindlabel||| -doform||| -doing_taint||5.008001|n -dooneliner||| -doopen_pm||| -doparseform||| -dopoptoeval||| -dopoptogiven||| -dopoptolabel||| -dopoptoloop||| -dopoptosub_at||| -dopoptowhen||| -doref||5.009003| -dounwind||| -dowantarray||| -dump_all||5.006000| -dump_eval||5.006000| -dump_exec_pos||| -dump_fds||| -dump_form||5.006000| -dump_indent||5.006000|v -dump_mstats||| -dump_packsubs||5.006000| -dump_sub||5.006000| -dump_sv_child||| -dump_trie_interim_list||| -dump_trie_interim_table||| -dump_trie||| -dump_vindent||5.006000| -dumpuntil||| -dup_attrlist||| -emulate_cop_io||| -eval_pv|5.006000||p -eval_sv|5.006000||p -exec_failed||| -expect_number||| -fbm_compile||5.005000| -fbm_instr||5.005000| -feature_is_enabled||| -fetch_cop_label||5.011000| -filter_add||| -filter_del||| -filter_gets||| -filter_read||| -find_and_forget_pmops||| -find_array_subscript||| -find_beginning||| -find_byclass||| -find_hash_subscript||| -find_in_my_stash||| -find_runcv||5.008001| -find_rundefsvoffset||5.009002| -find_script||| -find_uninit_var||| -first_symbol|||n -fold_constants||| -forbid_setid||| -force_ident||| -force_list||| -force_next||| -force_version||| -force_word||| -forget_pmop||| -form_nocontext|||vn -form||5.004000|v -fp_dup||| -fprintf_nocontext|||vn -free_global_struct||| -free_tied_hv_pool||| -free_tmps||| -gen_constant_list||| -get_arena||| -get_aux_mg||| -get_av|5.006000||p -get_context||5.006000|n -get_cvn_flags||5.009005| -get_cv|5.006000||p -get_db_sub||| -get_debug_opts||| -get_hash_seed||| -get_hv|5.006000||p -get_isa_hash||| -get_mstats||| -get_no_modify||| -get_num||| -get_op_descs||5.005000| -get_op_names||5.005000| -get_opargs||| -get_ppaddr||5.006000| -get_re_arg||| -get_sv|5.006000||p -get_vtbl||5.005030| -getcwd_sv||5.007002| -getenv_len||| -glob_2number||| -glob_assign_glob||| -glob_assign_ref||| -gp_dup||| -gp_free||| -gp_ref||| -grok_bin|5.007003||p -grok_hex|5.007003||p -grok_number|5.007002||p -grok_numeric_radix|5.007002||p -grok_oct|5.007003||p -group_end||| -gv_AVadd||| -gv_HVadd||| -gv_IOadd||| -gv_SVadd||| -gv_autoload4||5.004000| -gv_check||| -gv_const_sv||5.009003| -gv_dump||5.006000| -gv_efullname3||5.004000| -gv_efullname4||5.006001| -gv_efullname||| -gv_ename||| -gv_fetchfile_flags||5.009005| -gv_fetchfile||| -gv_fetchmeth_autoload||5.007003| -gv_fetchmethod_autoload||5.004000| -gv_fetchmethod_flags||5.011000| -gv_fetchmethod||| -gv_fetchmeth||| -gv_fetchpvn_flags|5.009002||p -gv_fetchpvs|5.009004||p -gv_fetchpv||| -gv_fetchsv||5.009002| -gv_fullname3||5.004000| -gv_fullname4||5.006001| -gv_fullname||| -gv_get_super_pkg||| -gv_handler||5.007001| -gv_init_sv||| -gv_init||| -gv_name_set||5.009004| -gv_stashpvn|5.004000||p -gv_stashpvs|5.009003||p -gv_stashpv||| -gv_stashsv||| -he_dup||| -hek_dup||| -hfreeentries||| -hsplit||| -hv_assert||5.011000| -hv_auxinit|||n -hv_backreferences_p||| -hv_clear_placeholders||5.009001| -hv_clear||| -hv_common_key_len||5.010000| -hv_common||5.010000| -hv_copy_hints_hv||| -hv_delayfree_ent||5.004000| -hv_delete_common||| -hv_delete_ent||5.004000| -hv_delete||| -hv_eiter_p||5.009003| -hv_eiter_set||5.009003| -hv_exists_ent||5.004000| -hv_exists||| -hv_fetch_ent||5.004000| -hv_fetchs|5.009003||p -hv_fetch||| -hv_free_ent||5.004000| -hv_iterinit||| -hv_iterkeysv||5.004000| -hv_iterkey||| -hv_iternext_flags||5.008000| -hv_iternextsv||| -hv_iternext||| -hv_iterval||| -hv_kill_backrefs||| -hv_ksplit||5.004000| -hv_magic_check|||n -hv_magic||| -hv_name_set||5.009003| -hv_notallowed||| -hv_placeholders_get||5.009003| -hv_placeholders_p||5.009003| -hv_placeholders_set||5.009003| -hv_riter_p||5.009003| -hv_riter_set||5.009003| -hv_scalar||5.009001| -hv_store_ent||5.004000| -hv_store_flags||5.008000| -hv_stores|5.009004||p -hv_store||| -hv_undef||| -ibcmp_locale||5.004000| -ibcmp_utf8||5.007003| -ibcmp||| -incline||| -incpush_if_exists||| -incpush_use_sep||| -incpush||| -ingroup||| -init_argv_symbols||| -init_debugger||| -init_global_struct||| -init_i18nl10n||5.006000| -init_i18nl14n||5.006000| -init_ids||| -init_interp||| -init_main_stash||| -init_perllib||| -init_postdump_symbols||| -init_predump_symbols||| -init_stacks||5.005000| -init_tm||5.007002| -instr||| -intro_my||| -intuit_method||| -intuit_more||| -invert||| -io_close||| -isALNUMC|5.006000||p -isALNUM||| -isALPHA||| -isASCII|5.006000||p -isBLANK|5.006001||p -isCNTRL|5.006000||p -isDIGIT||| -isGRAPH|5.006000||p -isGV_with_GP|5.009004||p -isLOWER||| -isPRINT|5.004000||p -isPSXSPC|5.006001||p -isPUNCT|5.006000||p -isSPACE||| -isUPPER||| -isXDIGIT|5.006000||p -is_an_int||| -is_gv_magical_sv||| -is_handle_constructor|||n -is_list_assignment||| -is_lvalue_sub||5.007001| -is_uni_alnum_lc||5.006000| -is_uni_alnumc_lc||5.006000| -is_uni_alnumc||5.006000| -is_uni_alnum||5.006000| -is_uni_alpha_lc||5.006000| -is_uni_alpha||5.006000| -is_uni_ascii_lc||5.006000| -is_uni_ascii||5.006000| -is_uni_cntrl_lc||5.006000| -is_uni_cntrl||5.006000| -is_uni_digit_lc||5.006000| -is_uni_digit||5.006000| -is_uni_graph_lc||5.006000| -is_uni_graph||5.006000| -is_uni_idfirst_lc||5.006000| -is_uni_idfirst||5.006000| -is_uni_lower_lc||5.006000| -is_uni_lower||5.006000| -is_uni_print_lc||5.006000| -is_uni_print||5.006000| -is_uni_punct_lc||5.006000| -is_uni_punct||5.006000| -is_uni_space_lc||5.006000| -is_uni_space||5.006000| -is_uni_upper_lc||5.006000| -is_uni_upper||5.006000| -is_uni_xdigit_lc||5.006000| -is_uni_xdigit||5.006000| -is_utf8_alnumc||5.006000| -is_utf8_alnum||5.006000| -is_utf8_alpha||5.006000| -is_utf8_ascii||5.006000| -is_utf8_char_slow|||n -is_utf8_char||5.006000| -is_utf8_cntrl||5.006000| -is_utf8_common||| -is_utf8_digit||5.006000| -is_utf8_graph||5.006000| -is_utf8_idcont||5.008000| -is_utf8_idfirst||5.006000| -is_utf8_lower||5.006000| -is_utf8_mark||5.006000| -is_utf8_print||5.006000| -is_utf8_punct||5.006000| -is_utf8_space||5.006000| -is_utf8_string_loclen||5.009003| -is_utf8_string_loc||5.008001| -is_utf8_string||5.006001| -is_utf8_upper||5.006000| -is_utf8_xdigit||5.006000| -isa_lookup||| -items|||n -ix|||n -jmaybe||| -join_exact||| -keyword||| -leave_scope||| -lex_end||| -lex_start||| -linklist||| -listkids||| -list||| -load_module_nocontext|||vn -load_module|5.006000||pv -localize||| -looks_like_bool||| -looks_like_number||| -lop||| -mPUSHi|5.009002||p -mPUSHn|5.009002||p -mPUSHp|5.009002||p -mPUSHs|5.011000||p -mPUSHu|5.009002||p -mXPUSHi|5.009002||p -mXPUSHn|5.009002||p -mXPUSHp|5.009002||p -mXPUSHs|5.011000||p -mXPUSHu|5.009002||p -mad_free||| -madlex||| -madparse||| -magic_clear_all_env||| -magic_clearenv||| -magic_clearhint||| -magic_clearisa||| -magic_clearpack||| -magic_clearsig||| -magic_dump||5.006000| -magic_existspack||| -magic_freearylen_p||| -magic_freeovrld||| -magic_getarylen||| -magic_getdefelem||| -magic_getnkeys||| -magic_getpack||| -magic_getpos||| -magic_getsig||| -magic_getsubstr||| -magic_gettaint||| -magic_getuvar||| -magic_getvec||| -magic_get||| -magic_killbackrefs||| -magic_len||| -magic_methcall||| -magic_methpack||| -magic_nextpack||| -magic_regdata_cnt||| -magic_regdatum_get||| -magic_regdatum_set||| -magic_scalarpack||| -magic_set_all_env||| -magic_setamagic||| -magic_setarylen||| -magic_setcollxfrm||| -magic_setdbline||| -magic_setdefelem||| -magic_setenv||| -magic_sethint||| -magic_setisa||| -magic_setmglob||| -magic_setnkeys||| -magic_setpack||| -magic_setpos||| -magic_setregexp||| -magic_setsig||| -magic_setsubstr||| -magic_settaint||| -magic_setutf8||| -magic_setuvar||| -magic_setvec||| -magic_set||| -magic_sizepack||| -magic_wipepack||| -make_matcher||| -make_trie_failtable||| -make_trie||| -malloc_good_size|||n -malloced_size|||n -malloc||5.007002|n -markstack_grow||| -matcher_matches_sv||| -measure_struct||| -memEQ|5.004000||p -memNE|5.004000||p -mem_collxfrm||| -mem_log_common|||n -mess_alloc||| -mess_nocontext|||vn -mess||5.006000|v -method_common||| -mfree||5.007002|n -mg_clear||| -mg_copy||| -mg_dup||| -mg_find||| -mg_free||| -mg_get||| -mg_length||5.005000| -mg_localize||| -mg_magical||| -mg_set||| -mg_size||5.005000| -mini_mktime||5.007002| -missingterm||| -mode_from_discipline||| -modkids||| -mod||| -more_bodies||| -more_sv||| -moreswitches||| -mro_get_from_name||5.011000| -mro_get_linear_isa_dfs||| -mro_get_linear_isa||5.009005| -mro_get_private_data||5.011000| -mro_isa_changed_in||| -mro_meta_dup||| -mro_meta_init||| -mro_method_changed_in||5.009005| -mro_register||5.011000| -mro_set_mro||5.011000| -mro_set_private_data||5.011000| -mul128||| -mulexp10|||n -my_atof2||5.007002| -my_atof||5.006000| -my_attrs||| -my_bcopy|||n -my_betoh16|||n -my_betoh32|||n -my_betoh64|||n -my_betohi|||n -my_betohl|||n -my_betohs|||n -my_bzero|||n -my_chsize||| -my_clearenv||| -my_cxt_index||| -my_cxt_init||| -my_dirfd||5.009005| -my_exit_jump||| -my_exit||| -my_failure_exit||5.004000| -my_fflush_all||5.006000| -my_fork||5.007003|n -my_htobe16|||n -my_htobe32|||n -my_htobe64|||n -my_htobei|||n -my_htobel|||n -my_htobes|||n -my_htole16|||n -my_htole32|||n -my_htole64|||n -my_htolei|||n -my_htolel|||n -my_htoles|||n -my_htonl||| -my_kid||| -my_letoh16|||n -my_letoh32|||n -my_letoh64|||n -my_letohi|||n -my_letohl|||n -my_letohs|||n -my_lstat||| -my_memcmp||5.004000|n -my_memset|||n -my_ntohl||| -my_pclose||5.004000| -my_popen_list||5.007001| -my_popen||5.004000| -my_setenv||| -my_snprintf|5.009004||pvn -my_socketpair||5.007003|n -my_sprintf|5.009003||pvn -my_stat||| -my_strftime||5.007002| -my_strlcat|5.009004||pn -my_strlcpy|5.009004||pn -my_swabn|||n -my_swap||| -my_unexec||| -my_vsnprintf||5.009004|n -need_utf8|||n -newANONATTRSUB||5.006000| -newANONHASH||| -newANONLIST||| -newANONSUB||| -newASSIGNOP||| -newATTRSUB||5.006000| -newAVREF||| -newAV||| -newBINOP||| -newCONDOP||| -newCONSTSUB|5.004050||p -newCVREF||| -newDEFSVOP||| -newFORM||| -newFOROP||| -newGIVENOP||5.009003| -newGIVWHENOP||| -newGP||| -newGVOP||| -newGVREF||| -newGVgen||| -newHVREF||| -newHVhv||5.005000| -newHV||| -newIO||| -newLISTOP||| -newLOGOP||| -newLOOPEX||| -newLOOPOP||| -newMADPROP||| -newMADsv||| -newMYSUB||| -newNULLLIST||| -newOP||| -newPADOP||| -newPMOP||| -newPROG||| -newPVOP||| -newRANGE||| -newRV_inc|5.004000||p -newRV_noinc|5.004000||p -newRV||| -newSLICEOP||| -newSTATEOP||| -newSUB||| -newSVOP||| -newSVREF||| -newSV_type|5.009005||p -newSVhek||5.009003| -newSViv||| -newSVnv||| -newSVpvf_nocontext|||vn -newSVpvf||5.004000|v -newSVpvn_flags|5.011000||p -newSVpvn_share|5.007001||p -newSVpvn_utf8|5.011000||p -newSVpvn|5.004050||p -newSVpvs_flags|5.011000||p -newSVpvs_share||5.009003| -newSVpvs|5.009003||p -newSVpv||| -newSVrv||| -newSVsv||| -newSVuv|5.006000||p -newSV||| -newTOKEN||| -newUNOP||| -newWHENOP||5.009003| -newWHILEOP||5.009003| -newXS_flags||5.009004| -newXSproto||5.006000| -newXS||5.006000| -new_collate||5.006000| -new_constant||| -new_ctype||5.006000| -new_he||| -new_logop||| -new_numeric||5.006000| -new_stackinfo||5.005000| -new_version||5.009000| -new_warnings_bitfield||| -next_symbol||| -nextargv||| -nextchar||| -ninstr||| -no_bareword_allowed||| -no_fh_allowed||| -no_op||| -not_a_number||| -nothreadhook||5.008000| -nuke_stacks||| -num_overflow|||n -offer_nice_chunk||| -oopsAV||| -oopsHV||| -op_clear||| -op_const_sv||| -op_dump||5.006000| -op_free||| -op_getmad_weak||| -op_getmad||| -op_null||5.007002| -op_refcnt_dec||| -op_refcnt_inc||| -op_refcnt_lock||5.009002| -op_refcnt_unlock||5.009002| -op_xmldump||| -open_script||| -pMY_CXT_|5.007003||p -pMY_CXT|5.007003||p -pTHX_|5.006000||p -pTHX|5.006000||p -packWARN|5.007003||p -pack_cat||5.007003| -pack_rec||| -package||| -packlist||5.008001| -pad_add_anon||| -pad_add_name||| -pad_alloc||| -pad_block_start||| -pad_check_dup||| -pad_compname_type||| -pad_findlex||| -pad_findmy||| -pad_fixup_inner_anons||| -pad_free||| -pad_leavemy||| -pad_new||| -pad_peg|||n -pad_push||| -pad_reset||| -pad_setsv||| -pad_sv||5.011000| -pad_swipe||| -pad_tidy||| -pad_undef||| -parse_body||| -parse_unicode_opts||| -parser_dup||| -parser_free||| -path_is_absolute|||n -peep||| -pending_Slabs_to_ro||| -perl_alloc_using|||n -perl_alloc|||n -perl_clone_using|||n -perl_clone|||n -perl_construct|||n -perl_destruct||5.007003|n -perl_free|||n -perl_parse||5.006000|n -perl_run|||n -pidgone||| -pm_description||| -pmflag||| -pmop_dump||5.006000| -pmop_xmldump||| -pmruntime||| -pmtrans||| -pop_scope||| -pregcomp||5.009005| -pregexec||| -pregfree2||5.011000| -pregfree||| -prepend_elem||| -prepend_madprops||| -printbuf||| -printf_nocontext|||vn -process_special_blocks||| -ptr_table_clear||5.009005| -ptr_table_fetch||5.009005| -ptr_table_find|||n -ptr_table_free||5.009005| -ptr_table_new||5.009005| -ptr_table_split||5.009005| -ptr_table_store||5.009005| -push_scope||| -put_byte||| -pv_display|5.006000||p -pv_escape|5.009004||p -pv_pretty|5.009004||p -pv_uni_display||5.007003| -qerror||| -qsortsvu||| -re_compile||5.009005| -re_croak2||| -re_dup_guts||| -re_intuit_start||5.009005| -re_intuit_string||5.006000| -readpipe_override||| -realloc||5.007002|n -reentrant_free||| -reentrant_init||| -reentrant_retry|||vn -reentrant_size||| -ref_array_or_hash||| -refcounted_he_chain_2hv||| -refcounted_he_fetch||| -refcounted_he_free||| -refcounted_he_new_common||| -refcounted_he_new||| -refcounted_he_value||| -refkids||| -refto||| -ref||5.011000| -reg_check_named_buff_matched||| -reg_named_buff_all||5.009005| -reg_named_buff_exists||5.009005| -reg_named_buff_fetch||5.009005| -reg_named_buff_firstkey||5.009005| -reg_named_buff_iter||| -reg_named_buff_nextkey||5.009005| -reg_named_buff_scalar||5.009005| -reg_named_buff||| -reg_namedseq||| -reg_node||| -reg_numbered_buff_fetch||| -reg_numbered_buff_length||| -reg_numbered_buff_store||| -reg_qr_package||| -reg_recode||| -reg_scan_name||| -reg_skipcomment||| -reg_temp_copy||| -reganode||| -regatom||| -regbranch||| -regclass_swash||5.009004| -regclass||| -regcppop||| -regcppush||| -regcurly|||n -regdump_extflags||| -regdump||5.005000| -regdupe_internal||| -regexec_flags||5.005000| -regfree_internal||5.009005| -reghop3|||n -reghop4|||n -reghopmaybe3|||n -reginclass||| -reginitcolors||5.006000| -reginsert||| -regmatch||| -regnext||5.005000| -regpiece||| -regpposixcc||| -regprop||| -regrepeat||| -regtail_study||| -regtail||| -regtry||| -reguni||| -regwhite|||n -reg||| -repeatcpy||| -report_evil_fh||| -report_uninit||| -require_pv||5.006000| -require_tie_mod||| -restore_magic||| -rninstr||| -rsignal_restore||| -rsignal_save||| -rsignal_state||5.004000| -rsignal||5.004000| -run_body||| -run_user_filter||| -runops_debug||5.005000| -runops_standard||5.005000| -rvpv_dup||| -rxres_free||| -rxres_restore||| -rxres_save||| -safesyscalloc||5.006000|n -safesysfree||5.006000|n -safesysmalloc||5.006000|n -safesysrealloc||5.006000|n -same_dirent||| -save_I16||5.004000| -save_I32||| -save_I8||5.006000| -save_adelete||5.011000| -save_aelem||5.004050| -save_alloc||5.006000| -save_aptr||| -save_ary||| -save_bool||5.008001| -save_clearsv||| -save_delete||| -save_destructor_x||5.006000| -save_destructor||5.006000| -save_freeop||| -save_freepv||| -save_freesv||| -save_generic_pvref||5.006001| -save_generic_svref||5.005030| -save_gp||5.004000| -save_hash||| -save_hek_flags|||n -save_helem_flags||5.011000| -save_helem||5.004050| -save_hints||| -save_hptr||| -save_int||| -save_item||| -save_iv||5.005000| -save_lines||| -save_list||| -save_long||| -save_magic||| -save_mortalizesv||5.007001| -save_nogv||| -save_op||| -save_padsv_and_mortalize||5.011000| -save_pptr||| -save_pushi32ptr||| -save_pushptri32ptr||| -save_pushptrptr||| -save_pushptr||5.011000| -save_re_context||5.006000| -save_scalar_at||| -save_scalar||| -save_set_svflags||5.009000| -save_shared_pvref||5.007003| -save_sptr||| -save_svref||| -save_vptr||5.006000| -savepvn||| -savepvs||5.009003| -savepv||| -savesharedpvn||5.009005| -savesharedpv||5.007003| -savestack_grow_cnt||5.008001| -savestack_grow||| -savesvpv||5.009002| -sawparens||| -scalar_mod_type|||n -scalarboolean||| -scalarkids||| -scalarseq||| -scalarvoid||| -scalar||| -scan_bin||5.006000| -scan_commit||| -scan_const||| -scan_formline||| -scan_heredoc||| -scan_hex||| -scan_ident||| -scan_inputsymbol||| -scan_num||5.007001| -scan_oct||| -scan_pat||| -scan_str||| -scan_subst||| -scan_trans||| -scan_version||5.009001| -scan_vstring||5.009005| -scan_word||| -scope||| -screaminstr||5.005000| -search_const||| -seed||5.008001| -sequence_num||| -sequence_tail||| -sequence||| -set_context||5.006000|n -set_numeric_local||5.006000| -set_numeric_radix||5.006000| -set_numeric_standard||5.006000| -setdefout||| -share_hek_flags||| -share_hek||5.004000| -si_dup||| -sighandler|||n -simplify_sort||| -skipspace0||| -skipspace1||| -skipspace2||| -skipspace||| -softref2xv||| -sortcv_stacked||| -sortcv_xsub||| -sortcv||| -sortsv_flags||5.009003| -sortsv||5.007003| -space_join_names_mortal||| -ss_dup||| -stack_grow||| -start_force||| -start_glob||| -start_subparse||5.004000| -stashpv_hvname_match||5.011000| -stdize_locale||| -store_cop_label||| -strEQ||| -strGE||| -strGT||| -strLE||| -strLT||| -strNE||| -str_to_version||5.006000| -strip_return||| -strnEQ||| -strnNE||| -study_chunk||| -sub_crush_depth||| -sublex_done||| -sublex_push||| -sublex_start||| -sv_2bool||| -sv_2cv||| -sv_2io||| -sv_2iuv_common||| -sv_2iuv_non_preserve||| -sv_2iv_flags||5.009001| -sv_2iv||| -sv_2mortal||| -sv_2num||| -sv_2nv||| -sv_2pv_flags|5.007002||p -sv_2pv_nolen|5.006000||p -sv_2pvbyte_nolen|5.006000||p -sv_2pvbyte|5.006000||p -sv_2pvutf8_nolen||5.006000| -sv_2pvutf8||5.006000| -sv_2pv||| -sv_2uv_flags||5.009001| -sv_2uv|5.004000||p -sv_add_arena||| -sv_add_backref||| -sv_backoff||| -sv_bless||| -sv_cat_decode||5.008001| -sv_catpv_mg|5.004050||p -sv_catpvf_mg_nocontext|||pvn -sv_catpvf_mg|5.006000|5.004000|pv -sv_catpvf_nocontext|||vn -sv_catpvf||5.004000|v -sv_catpvn_flags||5.007002| -sv_catpvn_mg|5.004050||p -sv_catpvn_nomg|5.007002||p -sv_catpvn||| -sv_catpvs|5.009003||p -sv_catpv||| -sv_catsv_flags||5.007002| -sv_catsv_mg|5.004050||p -sv_catsv_nomg|5.007002||p -sv_catsv||| -sv_catxmlpvn||| -sv_catxmlsv||| -sv_chop||| -sv_clean_all||| -sv_clean_objs||| -sv_clear||| -sv_cmp_locale||5.004000| -sv_cmp||| -sv_collxfrm||| -sv_compile_2op||5.008001| -sv_copypv||5.007003| -sv_dec||| -sv_del_backref||| -sv_derived_from||5.004000| -sv_destroyable||5.010000| -sv_does||5.009004| -sv_dump||| -sv_dup_inc_multiple||| -sv_dup||| -sv_eq||| -sv_exp_grow||| -sv_force_normal_flags||5.007001| -sv_force_normal||5.006000| -sv_free2||| -sv_free_arenas||| -sv_free||| -sv_gets||5.004000| -sv_grow||| -sv_i_ncmp||| -sv_inc||| -sv_insert_flags||5.011000| -sv_insert||| -sv_isa||| -sv_isobject||| -sv_iv||5.005000| -sv_kill_backrefs||| -sv_len_utf8||5.006000| -sv_len||| -sv_magic_portable|5.011000|5.004000|p -sv_magicext||5.007003| -sv_magic||| -sv_mortalcopy||| -sv_ncmp||| -sv_newmortal||| -sv_newref||| -sv_nolocking||5.007003| -sv_nosharing||5.007003| -sv_nounlocking||| -sv_nv||5.005000| -sv_peek||5.005000| -sv_pos_b2u_midway||| -sv_pos_b2u||5.006000| -sv_pos_u2b_cached||| -sv_pos_u2b_forwards|||n -sv_pos_u2b_midway|||n -sv_pos_u2b||5.006000| -sv_pvbyten_force||5.006000| -sv_pvbyten||5.006000| -sv_pvbyte||5.006000| -sv_pvn_force_flags|5.007002||p -sv_pvn_force||| -sv_pvn_nomg|5.007003|5.005000|p -sv_pvn||5.005000| -sv_pvutf8n_force||5.006000| -sv_pvutf8n||5.006000| -sv_pvutf8||5.006000| -sv_pv||5.006000| -sv_recode_to_utf8||5.007003| -sv_reftype||| -sv_release_COW||| -sv_replace||| -sv_report_used||| -sv_reset||| -sv_rvweaken||5.006000| -sv_setiv_mg|5.004050||p -sv_setiv||| -sv_setnv_mg|5.006000||p -sv_setnv||| -sv_setpv_mg|5.004050||p -sv_setpvf_mg_nocontext|||pvn -sv_setpvf_mg|5.006000|5.004000|pv -sv_setpvf_nocontext|||vn -sv_setpvf||5.004000|v -sv_setpviv_mg||5.008001| -sv_setpviv||5.008001| -sv_setpvn_mg|5.004050||p -sv_setpvn||| -sv_setpvs|5.009004||p -sv_setpv||| -sv_setref_iv||| -sv_setref_nv||| -sv_setref_pvn||| -sv_setref_pv||| -sv_setref_uv||5.007001| -sv_setsv_cow||| -sv_setsv_flags||5.007002| -sv_setsv_mg|5.004050||p -sv_setsv_nomg|5.007002||p -sv_setsv||| -sv_setuv_mg|5.004050||p -sv_setuv|5.004000||p -sv_tainted||5.004000| -sv_taint||5.004000| -sv_true||5.005000| -sv_unglob||| -sv_uni_display||5.007003| -sv_unmagic||| -sv_unref_flags||5.007001| -sv_unref||| -sv_untaint||5.004000| -sv_upgrade||| -sv_usepvn_flags||5.009004| -sv_usepvn_mg|5.004050||p -sv_usepvn||| -sv_utf8_decode||5.006000| -sv_utf8_downgrade||5.006000| -sv_utf8_encode||5.006000| -sv_utf8_upgrade_flags_grow||5.011000| -sv_utf8_upgrade_flags||5.007002| -sv_utf8_upgrade_nomg||5.007002| -sv_utf8_upgrade||5.007001| -sv_uv|5.005000||p -sv_vcatpvf_mg|5.006000|5.004000|p -sv_vcatpvfn||5.004000| -sv_vcatpvf|5.006000|5.004000|p -sv_vsetpvf_mg|5.006000|5.004000|p -sv_vsetpvfn||5.004000| -sv_vsetpvf|5.006000|5.004000|p -sv_xmlpeek||| -svtype||| -swallow_bom||| -swap_match_buff||| -swash_fetch||5.007002| -swash_get||| -swash_init||5.006000| -sys_init3||5.010000|n -sys_init||5.010000|n -sys_intern_clear||| -sys_intern_dup||| -sys_intern_init||| -sys_term||5.010000|n -taint_env||| -taint_proper||| -tmps_grow||5.006000| -toLOWER||| -toUPPER||| -to_byte_substr||| -to_uni_fold||5.007003| -to_uni_lower_lc||5.006000| -to_uni_lower||5.007003| -to_uni_title_lc||5.006000| -to_uni_title||5.007003| -to_uni_upper_lc||5.006000| -to_uni_upper||5.007003| -to_utf8_case||5.007003| -to_utf8_fold||5.007003| -to_utf8_lower||5.007003| -to_utf8_substr||| -to_utf8_title||5.007003| -to_utf8_upper||5.007003| -token_free||| -token_getmad||| -tokenize_use||| -tokeq||| -tokereport||| -too_few_arguments||| -too_many_arguments||| -uiv_2buf|||n -unlnk||| -unpack_rec||| -unpack_str||5.007003| -unpackstring||5.008001| -unshare_hek_or_pvn||| -unshare_hek||| -unsharepvn||5.004000| -unwind_handler_stack||| -update_debugger_info||| -upg_version||5.009005| -usage||| -utf16_to_utf8_reversed||5.006001| -utf16_to_utf8||5.006001| -utf8_distance||5.006000| -utf8_hop||5.006000| -utf8_length||5.007001| -utf8_mg_pos_cache_update||| -utf8_to_bytes||5.006001| -utf8_to_uvchr||5.007001| -utf8_to_uvuni||5.007001| -utf8n_to_uvchr||| -utf8n_to_uvuni||5.007001| -utilize||| -uvchr_to_utf8_flags||5.007003| -uvchr_to_utf8||| -uvuni_to_utf8_flags||5.007003| -uvuni_to_utf8||5.007001| -validate_suid||| -varname||| -vcmp||5.009000| -vcroak||5.006000| -vdeb||5.007003| -vdie_common||| -vdie_croak_common||| -vdie||| -vform||5.006000| -visit||| -vivify_defelem||| -vivify_ref||| -vload_module|5.006000||p -vmess||5.006000| -vnewSVpvf|5.006000|5.004000|p -vnormal||5.009002| -vnumify||5.009000| -vstringify||5.009000| -vverify||5.009003| -vwarner||5.006000| -vwarn||5.006000| -wait4pid||| -warn_nocontext|||vn -warner_nocontext|||vn -warner|5.006000|5.004000|pv -warn|||v -watch||| -whichsig||| -write_no_mem||| -write_to_stderr||| -xmldump_all||| -xmldump_attr||| -xmldump_eval||| -xmldump_form||| -xmldump_indent|||v -xmldump_packsubs||| -xmldump_sub||| -xmldump_vindent||| -yyerror||| -yylex||| -yyparse||| -yywarn||| -); - -if (exists $opt{'list-unsupported'}) { - my $f; - for $f (sort { lc $a cmp lc $b } keys %API) { - next unless $API{$f}{todo}; - print "$f ", '.'x(40-length($f)), " ", format_version($API{$f}{todo}), "\n"; - } - exit 0; -} - -# Scan for possible replacement candidates - -my(%replace, %need, %hints, %warnings, %depends); -my $replace = 0; -my($hint, $define, $function); - -sub find_api -{ - my $code = shift; - $code =~ s{ - / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]*) - | "[^"\\]*(?:\\.[^"\\]*)*" - | '[^'\\]*(?:\\.[^'\\]*)*' }{}egsx; - grep { exists $API{$_} } $code =~ /(\w+)/mg; -} - -while () { - if ($hint) { - my $h = $hint->[0] eq 'Hint' ? \%hints : \%warnings; - if (m{^\s*\*\s(.*?)\s*$}) { - for (@{$hint->[1]}) { - $h->{$_} ||= ''; # suppress warning with older perls - $h->{$_} .= "$1\n"; - } - } - else { undef $hint } - } - - $hint = [$1, [split /,?\s+/, $2]] - if m{^\s*$rccs\s+(Hint|Warning):\s+(\w+(?:,?\s+\w+)*)\s*$}; - - if ($define) { - if ($define->[1] =~ /\\$/) { - $define->[1] .= $_; - } - else { - if (exists $API{$define->[0]} && $define->[1] !~ /^DPPP_\(/) { - my @n = find_api($define->[1]); - push @{$depends{$define->[0]}}, @n if @n - } - undef $define; - } - } - - $define = [$1, $2] if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(.*)}; - - if ($function) { - if (/^}/) { - if (exists $API{$function->[0]}) { - my @n = find_api($function->[1]); - push @{$depends{$function->[0]}}, @n if @n - } - undef $function; - } - else { - $function->[1] .= $_; - } - } - - $function = [$1, ''] if m{^DPPP_\(my_(\w+)\)}; - - $replace = $1 if m{^\s*$rccs\s+Replace:\s+(\d+)\s+$rcce\s*$}; - $replace{$2} = $1 if $replace and m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+)}; - $replace{$2} = $1 if m{^\s*#\s*define\s+(\w+)(?:\([^)]*\))?\s+(\w+).*$rccs\s+Replace\s+$rcce}; - $replace{$1} = $2 if m{^\s*$rccs\s+Replace (\w+) with (\w+)\s+$rcce\s*$}; - - if (m{^\s*$rccs\s+(\w+(\s*,\s*\w+)*)\s+depends\s+on\s+(\w+(\s*,\s*\w+)*)\s+$rcce\s*$}) { - my @deps = map { s/\s+//g; $_ } split /,/, $3; - my $d; - for $d (map { s/\s+//g; $_ } split /,/, $1) { - push @{$depends{$d}}, @deps; - } - } - - $need{$1} = 1 if m{^#if\s+defined\(NEED_(\w+)(?:_GLOBAL)?\)}; -} - -for (values %depends) { - my %s; - $_ = [sort grep !$s{$_}++, @$_]; -} - -if (exists $opt{'api-info'}) { - my $f; - my $count = 0; - my $match = $opt{'api-info'} =~ m!^/(.*)/$! ? $1 : "^\Q$opt{'api-info'}\E\$"; - for $f (sort { lc $a cmp lc $b } keys %API) { - next unless $f =~ /$match/; - print "\n=== $f ===\n\n"; - my $info = 0; - if ($API{$f}{base} || $API{$f}{todo}) { - my $base = format_version($API{$f}{base} || $API{$f}{todo}); - print "Supported at least starting from perl-$base.\n"; - $info++; - } - if ($API{$f}{provided}) { - my $todo = $API{$f}{todo} ? format_version($API{$f}{todo}) : "5.003"; - print "Support by $ppport provided back to perl-$todo.\n"; - print "Support needs to be explicitly requested by NEED_$f.\n" if exists $need{$f}; - print "Depends on: ", join(', ', @{$depends{$f}}), ".\n" if exists $depends{$f}; - print "\n$hints{$f}" if exists $hints{$f}; - print "\nWARNING:\n$warnings{$f}" if exists $warnings{$f}; - $info++; - } - print "No portability information available.\n" unless $info; - $count++; - } - $count or print "Found no API matching '$opt{'api-info'}'."; - print "\n"; - exit 0; -} - -if (exists $opt{'list-provided'}) { - my $f; - for $f (sort { lc $a cmp lc $b } keys %API) { - next unless $API{$f}{provided}; - my @flags; - push @flags, 'explicit' if exists $need{$f}; - push @flags, 'depend' if exists $depends{$f}; - push @flags, 'hint' if exists $hints{$f}; - push @flags, 'warning' if exists $warnings{$f}; - my $flags = @flags ? ' ['.join(', ', @flags).']' : ''; - print "$f$flags\n"; - } - exit 0; -} - -my @files; -my @srcext = qw( .xs .c .h .cc .cpp -c.inc -xs.inc ); -my $srcext = join '|', map { quotemeta $_ } @srcext; - -if (@ARGV) { - my %seen; - for (@ARGV) { - if (-e) { - if (-f) { - push @files, $_ unless $seen{$_}++; - } - else { warn "'$_' is not a file.\n" } - } - else { - my @new = grep { -f } glob $_ - or warn "'$_' does not exist.\n"; - push @files, grep { !$seen{$_}++ } @new; - } - } -} -else { - eval { - require File::Find; - File::Find::find(sub { - $File::Find::name =~ /($srcext)$/i - and push @files, $File::Find::name; - }, '.'); - }; - if ($@) { - @files = map { glob "*$_" } @srcext; - } -} - -if (!@ARGV || $opt{filter}) { - my(@in, @out); - my %xsc = map { /(.*)\.xs$/ ? ("$1.c" => 1, "$1.cc" => 1) : () } @files; - for (@files) { - my $out = exists $xsc{$_} || /\b\Q$ppport\E$/i || !/($srcext)$/i; - push @{ $out ? \@out : \@in }, $_; - } - if (@ARGV && @out) { - warning("Skipping the following files (use --nofilter to avoid this):\n| ", join "\n| ", @out); - } - @files = @in; -} - -die "No input files given!\n" unless @files; - -my(%files, %global, %revreplace); -%revreplace = reverse %replace; -my $filename; -my $patch_opened = 0; - -for $filename (@files) { - unless (open IN, "<$filename") { - warn "Unable to read from $filename: $!\n"; - next; - } - - info("Scanning $filename ..."); - - my $c = do { local $/; }; - close IN; - - my %file = (orig => $c, changes => 0); - - # Temporarily remove C/XS comments and strings from the code - my @ccom; - - $c =~ s{ - ( ^$HS*\#$HS*include\b[^\r\n]+\b(?:\Q$ppport\E|XSUB\.h)\b[^\r\n]* - | ^$HS*\#$HS*(?:define|elif|if(?:def)?)\b[^\r\n]* ) - | ( ^$HS*\#[^\r\n]* - | "[^"\\]*(?:\\.[^"\\]*)*" - | '[^'\\]*(?:\\.[^'\\]*)*' - | / (?: \*[^*]*\*+(?:[^$ccs][^*]*\*+)* / | /[^\r\n]* ) ) - }{ defined $2 and push @ccom, $2; - defined $1 ? $1 : "$ccs$#ccom$cce" }mgsex; - - $file{ccom} = \@ccom; - $file{code} = $c; - $file{has_inc_ppport} = $c =~ /^$HS*#$HS*include[^\r\n]+\b\Q$ppport\E\b/m; - - my $func; - - for $func (keys %API) { - my $match = $func; - $match .= "|$revreplace{$func}" if exists $revreplace{$func}; - if ($c =~ /\b(?:Perl_)?($match)\b/) { - $file{uses_replace}{$1}++ if exists $revreplace{$func} && $1 eq $revreplace{$func}; - $file{uses_Perl}{$func}++ if $c =~ /\bPerl_$func\b/; - if (exists $API{$func}{provided}) { - $file{uses_provided}{$func}++; - if (!exists $API{$func}{base} || $API{$func}{base} > $opt{'compat-version'}) { - $file{uses}{$func}++; - my @deps = rec_depend($func); - if (@deps) { - $file{uses_deps}{$func} = \@deps; - for (@deps) { - $file{uses}{$_} = 0 unless exists $file{uses}{$_}; - } - } - for ($func, @deps) { - $file{needs}{$_} = 'static' if exists $need{$_}; - } - } - } - if (exists $API{$func}{todo} && $API{$func}{todo} > $opt{'compat-version'}) { - if ($c =~ /\b$func\b/) { - $file{uses_todo}{$func}++; - } - } - } - } - - while ($c =~ /^$HS*#$HS*define$HS+(NEED_(\w+?)(_GLOBAL)?)\b/mg) { - if (exists $need{$2}) { - $file{defined $3 ? 'needed_global' : 'needed_static'}{$2}++; - } - else { warning("Possibly wrong #define $1 in $filename") } - } - - for (qw(uses needs uses_todo needed_global needed_static)) { - for $func (keys %{$file{$_}}) { - push @{$global{$_}{$func}}, $filename; - } - } - - $files{$filename} = \%file; -} - -# Globally resolve NEED_'s -my $need; -for $need (keys %{$global{needs}}) { - if (@{$global{needs}{$need}} > 1) { - my @targets = @{$global{needs}{$need}}; - my @t = grep $files{$_}{needed_global}{$need}, @targets; - @targets = @t if @t; - @t = grep /\.xs$/i, @targets; - @targets = @t if @t; - my $target = shift @targets; - $files{$target}{needs}{$need} = 'global'; - for (@{$global{needs}{$need}}) { - $files{$_}{needs}{$need} = 'extern' if $_ ne $target; - } - } -} - -for $filename (@files) { - exists $files{$filename} or next; - - info("=== Analyzing $filename ==="); - - my %file = %{$files{$filename}}; - my $func; - my $c = $file{code}; - my $warnings = 0; - - for $func (sort keys %{$file{uses_Perl}}) { - if ($API{$func}{varargs}) { - unless ($API{$func}{nothxarg}) { - my $changes = ($c =~ s{\b(Perl_$func\s*\(\s*)(?!aTHX_?)(\)|[^\s)]*\))} - { $1 . ($2 eq ')' ? 'aTHX' : 'aTHX_ ') . $2 }ge); - if ($changes) { - warning("Doesn't pass interpreter argument aTHX to Perl_$func"); - $file{changes} += $changes; - } - } - } - else { - warning("Uses Perl_$func instead of $func"); - $file{changes} += ($c =~ s{\bPerl_$func(\s*)\((\s*aTHX_?)?\s*} - {$func$1(}g); - } - } - - for $func (sort keys %{$file{uses_replace}}) { - warning("Uses $func instead of $replace{$func}"); - $file{changes} += ($c =~ s/\b$func\b/$replace{$func}/g); - } - - for $func (sort keys %{$file{uses_provided}}) { - if ($file{uses}{$func}) { - if (exists $file{uses_deps}{$func}) { - diag("Uses $func, which depends on ", join(', ', @{$file{uses_deps}{$func}})); - } - else { - diag("Uses $func"); - } - } - $warnings += hint($func); - } - - unless ($opt{quiet}) { - for $func (sort keys %{$file{uses_todo}}) { - print "*** WARNING: Uses $func, which may not be portable below perl ", - format_version($API{$func}{todo}), ", even with '$ppport'\n"; - $warnings++; - } - } - - for $func (sort keys %{$file{needed_static}}) { - my $message = ''; - if (not exists $file{uses}{$func}) { - $message = "No need to define NEED_$func if $func is never used"; - } - elsif (exists $file{needs}{$func} && $file{needs}{$func} ne 'static') { - $message = "No need to define NEED_$func when already needed globally"; - } - if ($message) { - diag($message); - $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_$func\b.*$LF//mg); - } - } - - for $func (sort keys %{$file{needed_global}}) { - my $message = ''; - if (not exists $global{uses}{$func}) { - $message = "No need to define NEED_${func}_GLOBAL if $func is never used"; - } - elsif (exists $file{needs}{$func}) { - if ($file{needs}{$func} eq 'extern') { - $message = "No need to define NEED_${func}_GLOBAL when already needed globally"; - } - elsif ($file{needs}{$func} eq 'static') { - $message = "No need to define NEED_${func}_GLOBAL when only used in this file"; - } - } - if ($message) { - diag($message); - $file{changes} += ($c =~ s/^$HS*#$HS*define$HS+NEED_${func}_GLOBAL\b.*$LF//mg); - } - } - - $file{needs_inc_ppport} = keys %{$file{uses}}; - - if ($file{needs_inc_ppport}) { - my $pp = ''; - - for $func (sort keys %{$file{needs}}) { - my $type = $file{needs}{$func}; - next if $type eq 'extern'; - my $suffix = $type eq 'global' ? '_GLOBAL' : ''; - unless (exists $file{"needed_$type"}{$func}) { - if ($type eq 'global') { - diag("Files [@{$global{needs}{$func}}] need $func, adding global request"); - } - else { - diag("File needs $func, adding static request"); - } - $pp .= "#define NEED_$func$suffix\n"; - } - } - - if ($pp && ($c =~ s/^(?=$HS*#$HS*define$HS+NEED_\w+)/$pp/m)) { - $pp = ''; - $file{changes}++; - } - - unless ($file{has_inc_ppport}) { - diag("Needs to include '$ppport'"); - $pp .= qq(#include "$ppport"\n) - } - - if ($pp) { - $file{changes} += ($c =~ s/^($HS*#$HS*define$HS+NEED_\w+.*?)^/$1$pp/ms) - || ($c =~ s/^(?=$HS*#$HS*include.*\Q$ppport\E)/$pp/m) - || ($c =~ s/^($HS*#$HS*include.*XSUB.*\s*?)^/$1$pp/m) - || ($c =~ s/^/$pp/); - } - } - else { - if ($file{has_inc_ppport}) { - diag("No need to include '$ppport'"); - $file{changes} += ($c =~ s/^$HS*?#$HS*include.*\Q$ppport\E.*?$LF//m); - } - } - - # put back in our C comments - my $ix; - my $cppc = 0; - my @ccom = @{$file{ccom}}; - for $ix (0 .. $#ccom) { - if (!$opt{cplusplus} && $ccom[$ix] =~ s!^//!!) { - $cppc++; - $file{changes} += $c =~ s/$rccs$ix$rcce/$ccs$ccom[$ix] $cce/; - } - else { - $c =~ s/$rccs$ix$rcce/$ccom[$ix]/; - } - } - - if ($cppc) { - my $s = $cppc != 1 ? 's' : ''; - warning("Uses $cppc C++ style comment$s, which is not portable"); - } - - my $s = $warnings != 1 ? 's' : ''; - my $warn = $warnings ? " ($warnings warning$s)" : ''; - info("Analysis completed$warn"); - - if ($file{changes}) { - if (exists $opt{copy}) { - my $newfile = "$filename$opt{copy}"; - if (-e $newfile) { - error("'$newfile' already exists, refusing to write copy of '$filename'"); - } - else { - local *F; - if (open F, ">$newfile") { - info("Writing copy of '$filename' with changes to '$newfile'"); - print F $c; - close F; - } - else { - error("Cannot open '$newfile' for writing: $!"); - } - } - } - elsif (exists $opt{patch} || $opt{changes}) { - if (exists $opt{patch}) { - unless ($patch_opened) { - if (open PATCH, ">$opt{patch}") { - $patch_opened = 1; - } - else { - error("Cannot open '$opt{patch}' for writing: $!"); - delete $opt{patch}; - $opt{changes} = 1; - goto fallback; - } - } - mydiff(\*PATCH, $filename, $c); - } - else { -fallback: - info("Suggested changes:"); - mydiff(\*STDOUT, $filename, $c); - } - } - else { - my $s = $file{changes} == 1 ? '' : 's'; - info("$file{changes} potentially required change$s detected"); - } - } - else { - info("Looks good"); - } -} - -close PATCH if $patch_opened; - -exit 0; - - -sub try_use { eval "use @_;"; return $@ eq '' } - -sub mydiff -{ - local *F = shift; - my($file, $str) = @_; - my $diff; - - if (exists $opt{diff}) { - $diff = run_diff($opt{diff}, $file, $str); - } - - if (!defined $diff and try_use('Text::Diff')) { - $diff = Text::Diff::diff($file, \$str, { STYLE => 'Unified' }); - $diff = <
$tmp") { - print F $str; - close F; - - if (open F, "$prog $file $tmp |") { - while () { - s/\Q$tmp\E/$file.patched/; - $diff .= $_; - } - close F; - unlink $tmp; - return $diff; - } - - unlink $tmp; - } - else { - error("Cannot open '$tmp' for writing: $!"); - } - - return undef; -} - -sub rec_depend -{ - my($func, $seen) = @_; - return () unless exists $depends{$func}; - $seen = {%{$seen||{}}}; - return () if $seen->{$func}++; - my %s; - grep !$s{$_}++, map { ($_, rec_depend($_, $seen)) } @{$depends{$func}}; -} - -sub parse_version -{ - my $ver = shift; - - if ($ver =~ /^(\d+)\.(\d+)\.(\d+)$/) { - return ($1, $2, $3); - } - elsif ($ver !~ /^\d+\.[\d_]+$/) { - die "cannot parse version '$ver'\n"; - } - - $ver =~ s/_//g; - $ver =~ s/$/000000/; - - my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; - - $v = int $v; - $s = int $s; - - if ($r < 5 || ($r == 5 && $v < 6)) { - if ($s % 10) { - die "cannot parse version '$ver'\n"; - } - } - - return ($r, $v, $s); -} - -sub format_version -{ - my $ver = shift; - - $ver =~ s/$/000000/; - my($r,$v,$s) = $ver =~ /(\d+)\.(\d{3})(\d{3})/; - - $v = int $v; - $s = int $s; - - if ($r < 5 || ($r == 5 && $v < 6)) { - if ($s % 10) { - die "invalid version '$ver'\n"; - } - $s /= 10; - - $ver = sprintf "%d.%03d", $r, $v; - $s > 0 and $ver .= sprintf "_%02d", $s; - - return $ver; - } - - return sprintf "%d.%d.%d", $r, $v, $s; -} - -sub info -{ - $opt{quiet} and return; - print @_, "\n"; -} - -sub diag -{ - $opt{quiet} and return; - $opt{diag} and print @_, "\n"; -} - -sub warning -{ - $opt{quiet} and return; - print "*** ", @_, "\n"; -} - -sub error -{ - print "*** ERROR: ", @_, "\n"; -} - -my %given_hints; -my %given_warnings; -sub hint -{ - $opt{quiet} and return; - my $func = shift; - my $rv = 0; - if (exists $warnings{$func} && !$given_warnings{$func}++) { - my $warn = $warnings{$func}; - $warn =~ s!^!*** !mg; - print "*** WARNING: $func\n", $warn; - $rv++; - } - if ($opt{hints} && exists $hints{$func} && !$given_hints{$func}++) { - my $hint = $hints{$func}; - $hint =~ s/^/ /mg; - print " --- hint for $func ---\n", $hint; - } - $rv; -} - -sub usage -{ - my($usage) = do { local(@ARGV,$/)=($0); <> } =~ /^=head\d$HS+SYNOPSIS\s*^(.*?)\s*^=/ms; - my %M = ( 'I' => '*' ); - $usage =~ s/^\s*perl\s+\S+/$^X $0/; - $usage =~ s/([A-Z])<([^>]+)>/$M{$1}$2$M{$1}/g; - - print < }; - my($copy) = $self =~ /^=head\d\s+COPYRIGHT\s*^(.*?)^=\w+/ms; - $copy =~ s/^(?=\S+)/ /gms; - $self =~ s/^$HS+Do NOT edit.*?(?=^-)/$copy/ms; - $self =~ s/^SKIP.*(?=^__DATA__)/SKIP -if (\@ARGV && \$ARGV[0] eq '--unstrip') { - eval { require Devel::PPPort }; - \$@ and die "Cannot require Devel::PPPort, please install.\\n"; - if (eval \$Devel::PPPort::VERSION < $VERSION) { - die "$0 was originally generated with Devel::PPPort $VERSION.\\n" - . "Your Devel::PPPort is only version \$Devel::PPPort::VERSION.\\n" - . "Please install a newer version, or --unstrip will not work.\\n"; - } - Devel::PPPort::WriteFile(\$0); - exit 0; -} -print <$0" or die "cannot strip $0: $!\n"; - print OUT "$pl$c\n"; - - exit 0; -} - -__DATA__ -*/ - -#ifndef _P_P_PORTABILITY_H_ -#define _P_P_PORTABILITY_H_ - -#ifndef DPPP_NAMESPACE -# define DPPP_NAMESPACE DPPP_ -#endif - -#define DPPP_CAT2(x,y) CAT2(x,y) -#define DPPP_(name) DPPP_CAT2(DPPP_NAMESPACE, name) - -#ifndef PERL_REVISION -# if !defined(__PATCHLEVEL_H_INCLUDED__) && !(defined(PATCHLEVEL) && defined(SUBVERSION)) -# define PERL_PATCHLEVEL_H_IMPLICIT -# include -# endif -# if !(defined(PERL_VERSION) || (defined(SUBVERSION) && defined(PATCHLEVEL))) -# include -# endif -# ifndef PERL_REVISION -# define PERL_REVISION (5) - /* Replace: 1 */ -# define PERL_VERSION PATCHLEVEL -# define PERL_SUBVERSION SUBVERSION - /* Replace PERL_PATCHLEVEL with PERL_VERSION */ - /* Replace: 0 */ -# endif -#endif - -#define _dpppDEC2BCD(dec) ((((dec)/100)<<8)|((((dec)%100)/10)<<4)|((dec)%10)) -#define PERL_BCDVERSION ((_dpppDEC2BCD(PERL_REVISION)<<24)|(_dpppDEC2BCD(PERL_VERSION)<<12)|_dpppDEC2BCD(PERL_SUBVERSION)) - -/* It is very unlikely that anyone will try to use this with Perl 6 - (or greater), but who knows. - */ -#if PERL_REVISION != 5 -# error ppport.h only works with Perl version 5 -#endif /* PERL_REVISION != 5 */ -#ifndef dTHR -# define dTHR dNOOP -#endif -#ifndef dTHX -# define dTHX dNOOP -#endif - -#ifndef dTHXa -# define dTHXa(x) dNOOP -#endif -#ifndef pTHX -# define pTHX void -#endif - -#ifndef pTHX_ -# define pTHX_ -#endif - -#ifndef aTHX -# define aTHX -#endif - -#ifndef aTHX_ -# define aTHX_ -#endif - -#if (PERL_BCDVERSION < 0x5006000) -# ifdef USE_THREADS -# define aTHXR thr -# define aTHXR_ thr, -# else -# define aTHXR -# define aTHXR_ -# endif -# define dTHXR dTHR -#else -# define aTHXR aTHX -# define aTHXR_ aTHX_ -# define dTHXR dTHX -#endif -#ifndef dTHXoa -# define dTHXoa(x) dTHXa(x) -#endif - -#ifdef I_LIMITS -# include -#endif - -#ifndef PERL_UCHAR_MIN -# define PERL_UCHAR_MIN ((unsigned char)0) -#endif - -#ifndef PERL_UCHAR_MAX -# ifdef UCHAR_MAX -# define PERL_UCHAR_MAX ((unsigned char)UCHAR_MAX) -# else -# ifdef MAXUCHAR -# define PERL_UCHAR_MAX ((unsigned char)MAXUCHAR) -# else -# define PERL_UCHAR_MAX ((unsigned char)~(unsigned)0) -# endif -# endif -#endif - -#ifndef PERL_USHORT_MIN -# define PERL_USHORT_MIN ((unsigned short)0) -#endif - -#ifndef PERL_USHORT_MAX -# ifdef USHORT_MAX -# define PERL_USHORT_MAX ((unsigned short)USHORT_MAX) -# else -# ifdef MAXUSHORT -# define PERL_USHORT_MAX ((unsigned short)MAXUSHORT) -# else -# ifdef USHRT_MAX -# define PERL_USHORT_MAX ((unsigned short)USHRT_MAX) -# else -# define PERL_USHORT_MAX ((unsigned short)~(unsigned)0) -# endif -# endif -# endif -#endif - -#ifndef PERL_SHORT_MAX -# ifdef SHORT_MAX -# define PERL_SHORT_MAX ((short)SHORT_MAX) -# else -# ifdef MAXSHORT /* Often used in */ -# define PERL_SHORT_MAX ((short)MAXSHORT) -# else -# ifdef SHRT_MAX -# define PERL_SHORT_MAX ((short)SHRT_MAX) -# else -# define PERL_SHORT_MAX ((short) (PERL_USHORT_MAX >> 1)) -# endif -# endif -# endif -#endif - -#ifndef PERL_SHORT_MIN -# ifdef SHORT_MIN -# define PERL_SHORT_MIN ((short)SHORT_MIN) -# else -# ifdef MINSHORT -# define PERL_SHORT_MIN ((short)MINSHORT) -# else -# ifdef SHRT_MIN -# define PERL_SHORT_MIN ((short)SHRT_MIN) -# else -# define PERL_SHORT_MIN (-PERL_SHORT_MAX - ((3 & -1) == 3)) -# endif -# endif -# endif -#endif - -#ifndef PERL_UINT_MAX -# ifdef UINT_MAX -# define PERL_UINT_MAX ((unsigned int)UINT_MAX) -# else -# ifdef MAXUINT -# define PERL_UINT_MAX ((unsigned int)MAXUINT) -# else -# define PERL_UINT_MAX (~(unsigned int)0) -# endif -# endif -#endif - -#ifndef PERL_UINT_MIN -# define PERL_UINT_MIN ((unsigned int)0) -#endif - -#ifndef PERL_INT_MAX -# ifdef INT_MAX -# define PERL_INT_MAX ((int)INT_MAX) -# else -# ifdef MAXINT /* Often used in */ -# define PERL_INT_MAX ((int)MAXINT) -# else -# define PERL_INT_MAX ((int)(PERL_UINT_MAX >> 1)) -# endif -# endif -#endif - -#ifndef PERL_INT_MIN -# ifdef INT_MIN -# define PERL_INT_MIN ((int)INT_MIN) -# else -# ifdef MININT -# define PERL_INT_MIN ((int)MININT) -# else -# define PERL_INT_MIN (-PERL_INT_MAX - ((3 & -1) == 3)) -# endif -# endif -#endif - -#ifndef PERL_ULONG_MAX -# ifdef ULONG_MAX -# define PERL_ULONG_MAX ((unsigned long)ULONG_MAX) -# else -# ifdef MAXULONG -# define PERL_ULONG_MAX ((unsigned long)MAXULONG) -# else -# define PERL_ULONG_MAX (~(unsigned long)0) -# endif -# endif -#endif - -#ifndef PERL_ULONG_MIN -# define PERL_ULONG_MIN ((unsigned long)0L) -#endif - -#ifndef PERL_LONG_MAX -# ifdef LONG_MAX -# define PERL_LONG_MAX ((long)LONG_MAX) -# else -# ifdef MAXLONG -# define PERL_LONG_MAX ((long)MAXLONG) -# else -# define PERL_LONG_MAX ((long) (PERL_ULONG_MAX >> 1)) -# endif -# endif -#endif - -#ifndef PERL_LONG_MIN -# ifdef LONG_MIN -# define PERL_LONG_MIN ((long)LONG_MIN) -# else -# ifdef MINLONG -# define PERL_LONG_MIN ((long)MINLONG) -# else -# define PERL_LONG_MIN (-PERL_LONG_MAX - ((3 & -1) == 3)) -# endif -# endif -#endif - -#if defined(HAS_QUAD) && (defined(convex) || defined(uts)) -# ifndef PERL_UQUAD_MAX -# ifdef ULONGLONG_MAX -# define PERL_UQUAD_MAX ((unsigned long long)ULONGLONG_MAX) -# else -# ifdef MAXULONGLONG -# define PERL_UQUAD_MAX ((unsigned long long)MAXULONGLONG) -# else -# define PERL_UQUAD_MAX (~(unsigned long long)0) -# endif -# endif -# endif - -# ifndef PERL_UQUAD_MIN -# define PERL_UQUAD_MIN ((unsigned long long)0L) -# endif - -# ifndef PERL_QUAD_MAX -# ifdef LONGLONG_MAX -# define PERL_QUAD_MAX ((long long)LONGLONG_MAX) -# else -# ifdef MAXLONGLONG -# define PERL_QUAD_MAX ((long long)MAXLONGLONG) -# else -# define PERL_QUAD_MAX ((long long) (PERL_UQUAD_MAX >> 1)) -# endif -# endif -# endif - -# ifndef PERL_QUAD_MIN -# ifdef LONGLONG_MIN -# define PERL_QUAD_MIN ((long long)LONGLONG_MIN) -# else -# ifdef MINLONGLONG -# define PERL_QUAD_MIN ((long long)MINLONGLONG) -# else -# define PERL_QUAD_MIN (-PERL_QUAD_MAX - ((3 & -1) == 3)) -# endif -# endif -# endif -#endif - -/* This is based on code from 5.003 perl.h */ -#ifdef HAS_QUAD -# ifdef cray -#ifndef IVTYPE -# define IVTYPE int -#endif - -#ifndef IV_MIN -# define IV_MIN PERL_INT_MIN -#endif - -#ifndef IV_MAX -# define IV_MAX PERL_INT_MAX -#endif - -#ifndef UV_MIN -# define UV_MIN PERL_UINT_MIN -#endif - -#ifndef UV_MAX -# define UV_MAX PERL_UINT_MAX -#endif - -# ifdef INTSIZE -#ifndef IVSIZE -# define IVSIZE INTSIZE -#endif - -# endif -# else -# if defined(convex) || defined(uts) -#ifndef IVTYPE -# define IVTYPE long long -#endif - -#ifndef IV_MIN -# define IV_MIN PERL_QUAD_MIN -#endif - -#ifndef IV_MAX -# define IV_MAX PERL_QUAD_MAX -#endif - -#ifndef UV_MIN -# define UV_MIN PERL_UQUAD_MIN -#endif - -#ifndef UV_MAX -# define UV_MAX PERL_UQUAD_MAX -#endif - -# ifdef LONGLONGSIZE -#ifndef IVSIZE -# define IVSIZE LONGLONGSIZE -#endif - -# endif -# else -#ifndef IVTYPE -# define IVTYPE long -#endif - -#ifndef IV_MIN -# define IV_MIN PERL_LONG_MIN -#endif - -#ifndef IV_MAX -# define IV_MAX PERL_LONG_MAX -#endif - -#ifndef UV_MIN -# define UV_MIN PERL_ULONG_MIN -#endif - -#ifndef UV_MAX -# define UV_MAX PERL_ULONG_MAX -#endif - -# ifdef LONGSIZE -#ifndef IVSIZE -# define IVSIZE LONGSIZE -#endif - -# endif -# endif -# endif -#ifndef IVSIZE -# define IVSIZE 8 -#endif - -#ifndef PERL_QUAD_MIN -# define PERL_QUAD_MIN IV_MIN -#endif - -#ifndef PERL_QUAD_MAX -# define PERL_QUAD_MAX IV_MAX -#endif - -#ifndef PERL_UQUAD_MIN -# define PERL_UQUAD_MIN UV_MIN -#endif - -#ifndef PERL_UQUAD_MAX -# define PERL_UQUAD_MAX UV_MAX -#endif - -#else -#ifndef IVTYPE -# define IVTYPE long -#endif - -#ifndef IV_MIN -# define IV_MIN PERL_LONG_MIN -#endif - -#ifndef IV_MAX -# define IV_MAX PERL_LONG_MAX -#endif - -#ifndef UV_MIN -# define UV_MIN PERL_ULONG_MIN -#endif - -#ifndef UV_MAX -# define UV_MAX PERL_ULONG_MAX -#endif - -#endif - -#ifndef IVSIZE -# ifdef LONGSIZE -# define IVSIZE LONGSIZE -# else -# define IVSIZE 4 /* A bold guess, but the best we can make. */ -# endif -#endif -#ifndef UVTYPE -# define UVTYPE unsigned IVTYPE -#endif - -#ifndef UVSIZE -# define UVSIZE IVSIZE -#endif -#ifndef sv_setuv -# define sv_setuv(sv, uv) \ - STMT_START { \ - UV TeMpUv = uv; \ - if (TeMpUv <= IV_MAX) \ - sv_setiv(sv, TeMpUv); \ - else \ - sv_setnv(sv, (double)TeMpUv); \ - } STMT_END -#endif -#ifndef newSVuv -# define newSVuv(uv) ((uv) <= IV_MAX ? newSViv((IV)uv) : newSVnv((NV)uv)) -#endif -#ifndef sv_2uv -# define sv_2uv(sv) ((PL_Sv = (sv)), (UV) (SvNOK(PL_Sv) ? SvNV(PL_Sv) : sv_2nv(PL_Sv))) -#endif - -#ifndef SvUVX -# define SvUVX(sv) ((UV)SvIVX(sv)) -#endif - -#ifndef SvUVXx -# define SvUVXx(sv) SvUVX(sv) -#endif - -#ifndef SvUV -# define SvUV(sv) (SvIOK(sv) ? SvUVX(sv) : sv_2uv(sv)) -#endif - -#ifndef SvUVx -# define SvUVx(sv) ((PL_Sv = (sv)), SvUV(PL_Sv)) -#endif - -/* Hint: sv_uv - * Always use the SvUVx() macro instead of sv_uv(). - */ -#ifndef sv_uv -# define sv_uv(sv) SvUVx(sv) -#endif - -#if !defined(SvUOK) && defined(SvIOK_UV) -# define SvUOK(sv) SvIOK_UV(sv) -#endif -#ifndef XST_mUV -# define XST_mUV(i,v) (ST(i) = sv_2mortal(newSVuv(v)) ) -#endif - -#ifndef XSRETURN_UV -# define XSRETURN_UV(v) STMT_START { XST_mUV(0,v); XSRETURN(1); } STMT_END -#endif -#ifndef PUSHu -# define PUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); PUSHTARG; } STMT_END -#endif - -#ifndef XPUSHu -# define XPUSHu(u) STMT_START { sv_setuv(TARG, (UV)(u)); XPUSHTARG; } STMT_END -#endif - -#ifdef HAS_MEMCMP -#ifndef memNE -# define memNE(s1,s2,l) (memcmp(s1,s2,l)) -#endif - -#ifndef memEQ -# define memEQ(s1,s2,l) (!memcmp(s1,s2,l)) -#endif - -#else -#ifndef memNE -# define memNE(s1,s2,l) (bcmp(s1,s2,l)) -#endif - -#ifndef memEQ -# define memEQ(s1,s2,l) (!bcmp(s1,s2,l)) -#endif - -#endif -#ifndef MoveD -# define MoveD(s,d,n,t) memmove((char*)(d),(char*)(s), (n) * sizeof(t)) -#endif - -#ifndef CopyD -# define CopyD(s,d,n,t) memcpy((char*)(d),(char*)(s), (n) * sizeof(t)) -#endif - -#ifdef HAS_MEMSET -#ifndef ZeroD -# define ZeroD(d,n,t) memzero((char*)(d), (n) * sizeof(t)) -#endif - -#else -#ifndef ZeroD -# define ZeroD(d,n,t) ((void)memzero((char*)(d), (n) * sizeof(t)), d) -#endif - -#endif -#ifndef PoisonWith -# define PoisonWith(d,n,t,b) (void)memset((char*)(d), (U8)(b), (n) * sizeof(t)) -#endif - -#ifndef PoisonNew -# define PoisonNew(d,n,t) PoisonWith(d,n,t,0xAB) -#endif - -#ifndef PoisonFree -# define PoisonFree(d,n,t) PoisonWith(d,n,t,0xEF) -#endif - -#ifndef Poison -# define Poison(d,n,t) PoisonFree(d,n,t) -#endif -#ifndef Newx -# define Newx(v,n,t) New(0,v,n,t) -#endif - -#ifndef Newxc -# define Newxc(v,n,t,c) Newc(0,v,n,t,c) -#endif - -#ifndef Newxz -# define Newxz(v,n,t) Newz(0,v,n,t) -#endif - -#ifndef PERL_UNUSED_DECL -# ifdef HASATTRIBUTE -# if (defined(__GNUC__) && defined(__cplusplus)) || defined(__INTEL_COMPILER) -# define PERL_UNUSED_DECL -# else -# define PERL_UNUSED_DECL __attribute__((unused)) -# endif -# else -# define PERL_UNUSED_DECL -# endif -#endif - -#ifndef PERL_UNUSED_ARG -# if defined(lint) && defined(S_SPLINT_S) /* www.splint.org */ -# include -# define PERL_UNUSED_ARG(x) NOTE(ARGUNUSED(x)) -# else -# define PERL_UNUSED_ARG(x) ((void)x) -# endif -#endif - -#ifndef PERL_UNUSED_VAR -# define PERL_UNUSED_VAR(x) ((void)x) -#endif - -#ifndef PERL_UNUSED_CONTEXT -# ifdef USE_ITHREADS -# define PERL_UNUSED_CONTEXT PERL_UNUSED_ARG(my_perl) -# else -# define PERL_UNUSED_CONTEXT -# endif -#endif -#ifndef NOOP -# define NOOP /*EMPTY*/(void)0 -#endif - -#ifndef dNOOP -# define dNOOP extern int /*@unused@*/ Perl___notused PERL_UNUSED_DECL -#endif - -#ifndef NVTYPE -# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) -# define NVTYPE long double -# else -# define NVTYPE double -# endif -typedef NVTYPE NV; -#endif - -#ifndef INT2PTR -# if (IVSIZE == PTRSIZE) && (UVSIZE == PTRSIZE) -# define PTRV UV -# define INT2PTR(any,d) (any)(d) -# else -# if PTRSIZE == LONGSIZE -# define PTRV unsigned long -# else -# define PTRV unsigned -# endif -# define INT2PTR(any,d) (any)(PTRV)(d) -# endif -#endif - -#ifndef PTR2ul -# if PTRSIZE == LONGSIZE -# define PTR2ul(p) (unsigned long)(p) -# else -# define PTR2ul(p) INT2PTR(unsigned long,p) -# endif -#endif -#ifndef PTR2nat -# define PTR2nat(p) (PTRV)(p) -#endif - -#ifndef NUM2PTR -# define NUM2PTR(any,d) (any)PTR2nat(d) -#endif - -#ifndef PTR2IV -# define PTR2IV(p) INT2PTR(IV,p) -#endif - -#ifndef PTR2UV -# define PTR2UV(p) INT2PTR(UV,p) -#endif - -#ifndef PTR2NV -# define PTR2NV(p) NUM2PTR(NV,p) -#endif - -#undef START_EXTERN_C -#undef END_EXTERN_C -#undef EXTERN_C -#ifdef __cplusplus -# define START_EXTERN_C extern "C" { -# define END_EXTERN_C } -# define EXTERN_C extern "C" -#else -# define START_EXTERN_C -# define END_EXTERN_C -# define EXTERN_C extern -#endif - -#if defined(PERL_GCC_PEDANTIC) -# ifndef PERL_GCC_BRACE_GROUPS_FORBIDDEN -# define PERL_GCC_BRACE_GROUPS_FORBIDDEN -# endif -#endif - -#if defined(__GNUC__) && !defined(PERL_GCC_BRACE_GROUPS_FORBIDDEN) && !defined(__cplusplus) -# ifndef PERL_USE_GCC_BRACE_GROUPS -# define PERL_USE_GCC_BRACE_GROUPS -# endif -#endif - -#undef STMT_START -#undef STMT_END -#ifdef PERL_USE_GCC_BRACE_GROUPS -# define STMT_START (void)( /* gcc supports ``({ STATEMENTS; })'' */ -# define STMT_END ) -#else -# if defined(VOIDFLAGS) && (VOIDFLAGS) && (defined(sun) || defined(__sun__)) && !defined(__GNUC__) -# define STMT_START if (1) -# define STMT_END else (void)0 -# else -# define STMT_START do -# define STMT_END while (0) -# endif -#endif -#ifndef boolSV -# define boolSV(b) ((b) ? &PL_sv_yes : &PL_sv_no) -#endif - -/* DEFSV appears first in 5.004_56 */ -#ifndef DEFSV -# define DEFSV GvSV(PL_defgv) -#endif - -#ifndef SAVE_DEFSV -# define SAVE_DEFSV SAVESPTR(GvSV(PL_defgv)) -#endif - -#ifndef DEFSV_set -# define DEFSV_set(sv) (DEFSV = (sv)) -#endif - -/* Older perls (<=5.003) lack AvFILLp */ -#ifndef AvFILLp -# define AvFILLp AvFILL -#endif -#ifndef ERRSV -# define ERRSV get_sv("@",FALSE) -#endif - -/* Hint: gv_stashpvn - * This function's backport doesn't support the length parameter, but - * rather ignores it. Portability can only be ensured if the length - * parameter is used for speed reasons, but the length can always be - * correctly computed from the string argument. - */ -#ifndef gv_stashpvn -# define gv_stashpvn(str,len,create) gv_stashpv(str,create) -#endif - -/* Replace: 1 */ -#ifndef get_cv -# define get_cv perl_get_cv -#endif - -#ifndef get_sv -# define get_sv perl_get_sv -#endif - -#ifndef get_av -# define get_av perl_get_av -#endif - -#ifndef get_hv -# define get_hv perl_get_hv -#endif - -/* Replace: 0 */ -#ifndef dUNDERBAR -# define dUNDERBAR dNOOP -#endif - -#ifndef UNDERBAR -# define UNDERBAR DEFSV -#endif -#ifndef dAX -# define dAX I32 ax = MARK - PL_stack_base + 1 -#endif - -#ifndef dITEMS -# define dITEMS I32 items = SP - MARK -#endif -#ifndef dXSTARG -# define dXSTARG SV * targ = sv_newmortal() -#endif -#ifndef dAXMARK -# define dAXMARK I32 ax = POPMARK; \ - register SV ** const mark = PL_stack_base + ax++ -#endif -#ifndef XSprePUSH -# define XSprePUSH (sp = PL_stack_base + ax - 1) -#endif - -#if (PERL_BCDVERSION < 0x5005000) -# undef XSRETURN -# define XSRETURN(off) \ - STMT_START { \ - PL_stack_sp = PL_stack_base + ax + ((off) - 1); \ - return; \ - } STMT_END -#endif -#ifndef XSPROTO -# define XSPROTO(name) void name(pTHX_ CV* cv) -#endif - -#ifndef SVfARG -# define SVfARG(p) ((void*)(p)) -#endif -#ifndef PERL_ABS -# define PERL_ABS(x) ((x) < 0 ? -(x) : (x)) -#endif -#ifndef dVAR -# define dVAR dNOOP -#endif -#ifndef SVf -# define SVf "_" -#endif -#ifndef UTF8_MAXBYTES -# define UTF8_MAXBYTES UTF8_MAXLEN -#endif -#ifndef CPERLscope -# define CPERLscope(x) x -#endif -#ifndef PERL_HASH -# define PERL_HASH(hash,str,len) \ - STMT_START { \ - const char *s_PeRlHaSh = str; \ - I32 i_PeRlHaSh = len; \ - U32 hash_PeRlHaSh = 0; \ - while (i_PeRlHaSh--) \ - hash_PeRlHaSh = hash_PeRlHaSh * 33 + *s_PeRlHaSh++; \ - (hash) = hash_PeRlHaSh; \ - } STMT_END -#endif - -#ifndef PERLIO_FUNCS_DECL -# ifdef PERLIO_FUNCS_CONST -# define PERLIO_FUNCS_DECL(funcs) const PerlIO_funcs funcs -# define PERLIO_FUNCS_CAST(funcs) (PerlIO_funcs*)(funcs) -# else -# define PERLIO_FUNCS_DECL(funcs) PerlIO_funcs funcs -# define PERLIO_FUNCS_CAST(funcs) (funcs) -# endif -#endif - -/* provide these typedefs for older perls */ -#if (PERL_BCDVERSION < 0x5009003) - -# ifdef ARGSproto -typedef OP* (CPERLscope(*Perl_ppaddr_t))(ARGSproto); -# else -typedef OP* (CPERLscope(*Perl_ppaddr_t))(pTHX); -# endif - -typedef OP* (CPERLscope(*Perl_check_t)) (pTHX_ OP*); - -#endif -#ifndef isPSXSPC -# define isPSXSPC(c) (isSPACE(c) || (c) == '\v') -#endif - -#ifndef isBLANK -# define isBLANK(c) ((c) == ' ' || (c) == '\t') -#endif - -#ifdef EBCDIC -#ifndef isALNUMC -# define isALNUMC(c) isalnum(c) -#endif - -#ifndef isASCII -# define isASCII(c) isascii(c) -#endif - -#ifndef isCNTRL -# define isCNTRL(c) iscntrl(c) -#endif - -#ifndef isGRAPH -# define isGRAPH(c) isgraph(c) -#endif - -#ifndef isPRINT -# define isPRINT(c) isprint(c) -#endif - -#ifndef isPUNCT -# define isPUNCT(c) ispunct(c) -#endif - -#ifndef isXDIGIT -# define isXDIGIT(c) isxdigit(c) -#endif - -#else -# if (PERL_BCDVERSION < 0x5010000) -/* Hint: isPRINT - * The implementation in older perl versions includes all of the - * isSPACE() characters, which is wrong. The version provided by - * Devel::PPPort always overrides a present buggy version. - */ -# undef isPRINT -# endif -#ifndef isALNUMC -# define isALNUMC(c) (isALPHA(c) || isDIGIT(c)) -#endif - -#ifndef isASCII -# define isASCII(c) ((c) <= 127) -#endif - -#ifndef isCNTRL -# define isCNTRL(c) ((c) < ' ' || (c) == 127) -#endif - -#ifndef isGRAPH -# define isGRAPH(c) (isALNUM(c) || isPUNCT(c)) -#endif - -#ifndef isPRINT -# define isPRINT(c) (((c) >= 32 && (c) < 127)) -#endif - -#ifndef isPUNCT -# define isPUNCT(c) (((c) >= 33 && (c) <= 47) || ((c) >= 58 && (c) <= 64) || ((c) >= 91 && (c) <= 96) || ((c) >= 123 && (c) <= 126)) -#endif - -#ifndef isXDIGIT -# define isXDIGIT(c) (isDIGIT(c) || ((c) >= 'a' && (c) <= 'f') || ((c) >= 'A' && (c) <= 'F')) -#endif - -#endif - -#ifndef PERL_SIGNALS_UNSAFE_FLAG - -#define PERL_SIGNALS_UNSAFE_FLAG 0x0001 - -#if (PERL_BCDVERSION < 0x5008000) -# define D_PPP_PERL_SIGNALS_INIT PERL_SIGNALS_UNSAFE_FLAG -#else -# define D_PPP_PERL_SIGNALS_INIT 0 -#endif - -#if defined(NEED_PL_signals) -static U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; -#elif defined(NEED_PL_signals_GLOBAL) -U32 DPPP_(my_PL_signals) = D_PPP_PERL_SIGNALS_INIT; -#else -extern U32 DPPP_(my_PL_signals); -#endif -#define PL_signals DPPP_(my_PL_signals) - -#endif - -/* Hint: PL_ppaddr - * Calling an op via PL_ppaddr requires passing a context argument - * for threaded builds. Since the context argument is different for - * 5.005 perls, you can use aTHXR (supplied by ppport.h), which will - * automatically be defined as the correct argument. - */ - -#if (PERL_BCDVERSION <= 0x5005005) -/* Replace: 1 */ -# define PL_ppaddr ppaddr -# define PL_no_modify no_modify -/* Replace: 0 */ -#endif - -#if (PERL_BCDVERSION <= 0x5004005) -/* Replace: 1 */ -# define PL_DBsignal DBsignal -# define PL_DBsingle DBsingle -# define PL_DBsub DBsub -# define PL_DBtrace DBtrace -# define PL_Sv Sv -# define PL_bufend bufend -# define PL_bufptr bufptr -# define PL_compiling compiling -# define PL_copline copline -# define PL_curcop curcop -# define PL_curstash curstash -# define PL_debstash debstash -# define PL_defgv defgv -# define PL_diehook diehook -# define PL_dirty dirty -# define PL_dowarn dowarn -# define PL_errgv errgv -# define PL_error_count error_count -# define PL_expect expect -# define PL_hexdigit hexdigit -# define PL_hints hints -# define PL_in_my in_my -# define PL_laststatval laststatval -# define PL_lex_state lex_state -# define PL_lex_stuff lex_stuff -# define PL_linestr linestr -# define PL_na na -# define PL_perl_destruct_level perl_destruct_level -# define PL_perldb perldb -# define PL_rsfp_filters rsfp_filters -# define PL_rsfp rsfp -# define PL_stack_base stack_base -# define PL_stack_sp stack_sp -# define PL_statcache statcache -# define PL_stdingv stdingv -# define PL_sv_arenaroot sv_arenaroot -# define PL_sv_no sv_no -# define PL_sv_undef sv_undef -# define PL_sv_yes sv_yes -# define PL_tainted tainted -# define PL_tainting tainting -# define PL_tokenbuf tokenbuf -/* Replace: 0 */ -#endif - -/* Warning: PL_parser - * For perl versions earlier than 5.9.5, this is an always - * non-NULL dummy. Also, it cannot be dereferenced. Don't - * use it if you can avoid is and unless you absolutely know - * what you're doing. - * If you always check that PL_parser is non-NULL, you can - * define DPPP_PL_parser_NO_DUMMY to avoid the creation of - * a dummy parser structure. - */ - -#if (PERL_BCDVERSION >= 0x5009005) -# ifdef DPPP_PL_parser_NO_DUMMY -# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ - (croak("panic: PL_parser == NULL in %s:%d", \ - __FILE__, __LINE__), (yy_parser *) NULL))->var) -# else -# ifdef DPPP_PL_parser_NO_DUMMY_WARNING -# define D_PPP_parser_dummy_warning(var) -# else -# define D_PPP_parser_dummy_warning(var) \ - warn("warning: dummy PL_" #var " used in %s:%d", __FILE__, __LINE__), -# endif -# define D_PPP_my_PL_parser_var(var) ((PL_parser ? PL_parser : \ - (D_PPP_parser_dummy_warning(var) &DPPP_(dummy_PL_parser)))->var) -#if defined(NEED_PL_parser) -static yy_parser DPPP_(dummy_PL_parser); -#elif defined(NEED_PL_parser_GLOBAL) -yy_parser DPPP_(dummy_PL_parser); -#else -extern yy_parser DPPP_(dummy_PL_parser); -#endif - -# endif - -/* PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf depends on PL_parser */ -/* Warning: PL_expect, PL_copline, PL_rsfp, PL_rsfp_filters, PL_linestr, PL_bufptr, PL_bufend, PL_lex_state, PL_lex_stuff, PL_tokenbuf - * Do not use this variable unless you know exactly what you're - * doint. It is internal to the perl parser and may change or even - * be removed in the future. As of perl 5.9.5, you have to check - * for (PL_parser != NULL) for this variable to have any effect. - * An always non-NULL PL_parser dummy is provided for earlier - * perl versions. - * If PL_parser is NULL when you try to access this variable, a - * dummy is being accessed instead and a warning is issued unless - * you define DPPP_PL_parser_NO_DUMMY_WARNING. - * If DPPP_PL_parser_NO_DUMMY is defined, the code trying to access - * this variable will croak with a panic message. - */ - -# define PL_expect D_PPP_my_PL_parser_var(expect) -# define PL_copline D_PPP_my_PL_parser_var(copline) -# define PL_rsfp D_PPP_my_PL_parser_var(rsfp) -# define PL_rsfp_filters D_PPP_my_PL_parser_var(rsfp_filters) -# define PL_linestr D_PPP_my_PL_parser_var(linestr) -# define PL_bufptr D_PPP_my_PL_parser_var(bufptr) -# define PL_bufend D_PPP_my_PL_parser_var(bufend) -# define PL_lex_state D_PPP_my_PL_parser_var(lex_state) -# define PL_lex_stuff D_PPP_my_PL_parser_var(lex_stuff) -# define PL_tokenbuf D_PPP_my_PL_parser_var(tokenbuf) -# define PL_in_my D_PPP_my_PL_parser_var(in_my) -# define PL_in_my_stash D_PPP_my_PL_parser_var(in_my_stash) -# define PL_error_count D_PPP_my_PL_parser_var(error_count) - - -#else - -/* ensure that PL_parser != NULL and cannot be dereferenced */ -# define PL_parser ((void *) 1) - -#endif -#ifndef mPUSHs -# define mPUSHs(s) PUSHs(sv_2mortal(s)) -#endif - -#ifndef PUSHmortal -# define PUSHmortal PUSHs(sv_newmortal()) -#endif - -#ifndef mPUSHp -# define mPUSHp(p,l) sv_setpvn(PUSHmortal, (p), (l)) -#endif - -#ifndef mPUSHn -# define mPUSHn(n) sv_setnv(PUSHmortal, (NV)(n)) -#endif - -#ifndef mPUSHi -# define mPUSHi(i) sv_setiv(PUSHmortal, (IV)(i)) -#endif - -#ifndef mPUSHu -# define mPUSHu(u) sv_setuv(PUSHmortal, (UV)(u)) -#endif -#ifndef mXPUSHs -# define mXPUSHs(s) XPUSHs(sv_2mortal(s)) -#endif - -#ifndef XPUSHmortal -# define XPUSHmortal XPUSHs(sv_newmortal()) -#endif - -#ifndef mXPUSHp -# define mXPUSHp(p,l) STMT_START { EXTEND(sp,1); sv_setpvn(PUSHmortal, (p), (l)); } STMT_END -#endif - -#ifndef mXPUSHn -# define mXPUSHn(n) STMT_START { EXTEND(sp,1); sv_setnv(PUSHmortal, (NV)(n)); } STMT_END -#endif - -#ifndef mXPUSHi -# define mXPUSHi(i) STMT_START { EXTEND(sp,1); sv_setiv(PUSHmortal, (IV)(i)); } STMT_END -#endif - -#ifndef mXPUSHu -# define mXPUSHu(u) STMT_START { EXTEND(sp,1); sv_setuv(PUSHmortal, (UV)(u)); } STMT_END -#endif - -/* Replace: 1 */ -#ifndef call_sv -# define call_sv perl_call_sv -#endif - -#ifndef call_pv -# define call_pv perl_call_pv -#endif - -#ifndef call_argv -# define call_argv perl_call_argv -#endif - -#ifndef call_method -# define call_method perl_call_method -#endif -#ifndef eval_sv -# define eval_sv perl_eval_sv -#endif - -/* Replace: 0 */ -#ifndef PERL_LOADMOD_DENY -# define PERL_LOADMOD_DENY 0x1 -#endif - -#ifndef PERL_LOADMOD_NOIMPORT -# define PERL_LOADMOD_NOIMPORT 0x2 -#endif - -#ifndef PERL_LOADMOD_IMPORT_OPS -# define PERL_LOADMOD_IMPORT_OPS 0x4 -#endif - -#ifndef G_METHOD -# define G_METHOD 64 -# ifdef call_sv -# undef call_sv -# endif -# if (PERL_BCDVERSION < 0x5006000) -# define call_sv(sv, flags) ((flags) & G_METHOD ? perl_call_method((char *) SvPV_nolen_const(sv), \ - (flags) & ~G_METHOD) : perl_call_sv(sv, flags)) -# else -# define call_sv(sv, flags) ((flags) & G_METHOD ? Perl_call_method(aTHX_ (char *) SvPV_nolen_const(sv), \ - (flags) & ~G_METHOD) : Perl_call_sv(aTHX_ sv, flags)) -# endif -#endif - -/* Replace perl_eval_pv with eval_pv */ - -#ifndef eval_pv -#if defined(NEED_eval_pv) -static SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); -static -#else -extern SV* DPPP_(my_eval_pv)(char *p, I32 croak_on_error); -#endif - -#ifdef eval_pv -# undef eval_pv -#endif -#define eval_pv(a,b) DPPP_(my_eval_pv)(aTHX_ a,b) -#define Perl_eval_pv DPPP_(my_eval_pv) - -#if defined(NEED_eval_pv) || defined(NEED_eval_pv_GLOBAL) - -SV* -DPPP_(my_eval_pv)(char *p, I32 croak_on_error) -{ - dSP; - SV* sv = newSVpv(p, 0); - - PUSHMARK(sp); - eval_sv(sv, G_SCALAR); - SvREFCNT_dec(sv); - - SPAGAIN; - sv = POPs; - PUTBACK; - - if (croak_on_error && SvTRUE(GvSV(errgv))) - croak(SvPVx(GvSV(errgv), na)); - - return sv; -} - -#endif -#endif - -#ifndef vload_module -#if defined(NEED_vload_module) -static void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); -static -#else -extern void DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args); -#endif - -#ifdef vload_module -# undef vload_module -#endif -#define vload_module(a,b,c,d) DPPP_(my_vload_module)(aTHX_ a,b,c,d) -#define Perl_vload_module DPPP_(my_vload_module) - -#if defined(NEED_vload_module) || defined(NEED_vload_module_GLOBAL) - -void -DPPP_(my_vload_module)(U32 flags, SV *name, SV *ver, va_list *args) -{ - dTHR; - dVAR; - OP *veop, *imop; - - OP * const modname = newSVOP(OP_CONST, 0, name); - /* 5.005 has a somewhat hacky force_normal that doesn't croak on - SvREADONLY() if PL_compling is true. Current perls take care in - ck_require() to correctly turn off SvREADONLY before calling - force_normal_flags(). This seems a better fix than fudging PL_compling - */ - SvREADONLY_off(((SVOP*)modname)->op_sv); - modname->op_private |= OPpCONST_BARE; - if (ver) { - veop = newSVOP(OP_CONST, 0, ver); - } - else - veop = NULL; - if (flags & PERL_LOADMOD_NOIMPORT) { - imop = sawparens(newNULLLIST()); - } - else if (flags & PERL_LOADMOD_IMPORT_OPS) { - imop = va_arg(*args, OP*); - } - else { - SV *sv; - imop = NULL; - sv = va_arg(*args, SV*); - while (sv) { - imop = append_elem(OP_LIST, imop, newSVOP(OP_CONST, 0, sv)); - sv = va_arg(*args, SV*); - } - } - { - const line_t ocopline = PL_copline; - COP * const ocurcop = PL_curcop; - const int oexpect = PL_expect; - -#if (PERL_BCDVERSION >= 0x5004000) - utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(FALSE, 0), - veop, modname, imop); -#else - utilize(!(flags & PERL_LOADMOD_DENY), start_subparse(), - modname, imop); -#endif - PL_expect = oexpect; - PL_copline = ocopline; - PL_curcop = ocurcop; - } -} - -#endif -#endif - -#ifndef load_module -#if defined(NEED_load_module) -static void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); -static -#else -extern void DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...); -#endif - -#ifdef load_module -# undef load_module -#endif -#define load_module DPPP_(my_load_module) -#define Perl_load_module DPPP_(my_load_module) - -#if defined(NEED_load_module) || defined(NEED_load_module_GLOBAL) - -void -DPPP_(my_load_module)(U32 flags, SV *name, SV *ver, ...) -{ - va_list args; - va_start(args, ver); - vload_module(flags, name, ver, &args); - va_end(args); -} - -#endif -#endif -#ifndef newRV_inc -# define newRV_inc(sv) newRV(sv) /* Replace */ -#endif - -#ifndef newRV_noinc -#if defined(NEED_newRV_noinc) -static SV * DPPP_(my_newRV_noinc)(SV *sv); -static -#else -extern SV * DPPP_(my_newRV_noinc)(SV *sv); -#endif - -#ifdef newRV_noinc -# undef newRV_noinc -#endif -#define newRV_noinc(a) DPPP_(my_newRV_noinc)(aTHX_ a) -#define Perl_newRV_noinc DPPP_(my_newRV_noinc) - -#if defined(NEED_newRV_noinc) || defined(NEED_newRV_noinc_GLOBAL) -SV * -DPPP_(my_newRV_noinc)(SV *sv) -{ - SV *rv = (SV *)newRV(sv); - SvREFCNT_dec(sv); - return rv; -} -#endif -#endif - -/* Hint: newCONSTSUB - * Returns a CV* as of perl-5.7.1. This return value is not supported - * by Devel::PPPort. - */ - -/* newCONSTSUB from IO.xs is in the core starting with 5.004_63 */ -#if (PERL_BCDVERSION < 0x5004063) && (PERL_BCDVERSION != 0x5004005) -#if defined(NEED_newCONSTSUB) -static void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); -static -#else -extern void DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv); -#endif - -#ifdef newCONSTSUB -# undef newCONSTSUB -#endif -#define newCONSTSUB(a,b,c) DPPP_(my_newCONSTSUB)(aTHX_ a,b,c) -#define Perl_newCONSTSUB DPPP_(my_newCONSTSUB) - -#if defined(NEED_newCONSTSUB) || defined(NEED_newCONSTSUB_GLOBAL) - -/* This is just a trick to avoid a dependency of newCONSTSUB on PL_parser */ -/* (There's no PL_parser in perl < 5.005, so this is completely safe) */ -#define D_PPP_PL_copline PL_copline - -void -DPPP_(my_newCONSTSUB)(HV *stash, const char *name, SV *sv) -{ - U32 oldhints = PL_hints; - HV *old_cop_stash = PL_curcop->cop_stash; - HV *old_curstash = PL_curstash; - line_t oldline = PL_curcop->cop_line; - PL_curcop->cop_line = D_PPP_PL_copline; - - PL_hints &= ~HINT_BLOCK_SCOPE; - if (stash) - PL_curstash = PL_curcop->cop_stash = stash; - - newSUB( - -#if (PERL_BCDVERSION < 0x5003022) - start_subparse(), -#elif (PERL_BCDVERSION == 0x5003022) - start_subparse(0), -#else /* 5.003_23 onwards */ - start_subparse(FALSE, 0), -#endif - - newSVOP(OP_CONST, 0, newSVpv((char *) name, 0)), - newSVOP(OP_CONST, 0, &PL_sv_no), /* SvPV(&PL_sv_no) == "" -- GMB */ - newSTATEOP(0, Nullch, newSVOP(OP_CONST, 0, sv)) - ); - - PL_hints = oldhints; - PL_curcop->cop_stash = old_cop_stash; - PL_curstash = old_curstash; - PL_curcop->cop_line = oldline; -} -#endif -#endif - -/* - * Boilerplate macros for initializing and accessing interpreter-local - * data from C. All statics in extensions should be reworked to use - * this, if you want to make the extension thread-safe. See ext/re/re.xs - * for an example of the use of these macros. - * - * Code that uses these macros is responsible for the following: - * 1. #define MY_CXT_KEY to a unique string, e.g. "DynaLoader_guts" - * 2. Declare a typedef named my_cxt_t that is a structure that contains - * all the data that needs to be interpreter-local. - * 3. Use the START_MY_CXT macro after the declaration of my_cxt_t. - * 4. Use the MY_CXT_INIT macro such that it is called exactly once - * (typically put in the BOOT: section). - * 5. Use the members of the my_cxt_t structure everywhere as - * MY_CXT.member. - * 6. Use the dMY_CXT macro (a declaration) in all the functions that - * access MY_CXT. - */ - -#if defined(MULTIPLICITY) || defined(PERL_OBJECT) || \ - defined(PERL_CAPI) || defined(PERL_IMPLICIT_CONTEXT) - -#ifndef START_MY_CXT - -/* This must appear in all extensions that define a my_cxt_t structure, - * right after the definition (i.e. at file scope). The non-threads - * case below uses it to declare the data as static. */ -#define START_MY_CXT - -#if (PERL_BCDVERSION < 0x5004068) -/* Fetches the SV that keeps the per-interpreter data. */ -#define dMY_CXT_SV \ - SV *my_cxt_sv = get_sv(MY_CXT_KEY, FALSE) -#else /* >= perl5.004_68 */ -#define dMY_CXT_SV \ - SV *my_cxt_sv = *hv_fetch(PL_modglobal, MY_CXT_KEY, \ - sizeof(MY_CXT_KEY)-1, TRUE) -#endif /* < perl5.004_68 */ - -/* This declaration should be used within all functions that use the - * interpreter-local data. */ -#define dMY_CXT \ - dMY_CXT_SV; \ - my_cxt_t *my_cxtp = INT2PTR(my_cxt_t*,SvUV(my_cxt_sv)) - -/* Creates and zeroes the per-interpreter data. - * (We allocate my_cxtp in a Perl SV so that it will be released when - * the interpreter goes away.) */ -#define MY_CXT_INIT \ - dMY_CXT_SV; \ - /* newSV() allocates one more than needed */ \ - my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ - Zero(my_cxtp, 1, my_cxt_t); \ - sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) - -/* This macro must be used to access members of the my_cxt_t structure. - * e.g. MYCXT.some_data */ -#define MY_CXT (*my_cxtp) - -/* Judicious use of these macros can reduce the number of times dMY_CXT - * is used. Use is similar to pTHX, aTHX etc. */ -#define pMY_CXT my_cxt_t *my_cxtp -#define pMY_CXT_ pMY_CXT, -#define _pMY_CXT ,pMY_CXT -#define aMY_CXT my_cxtp -#define aMY_CXT_ aMY_CXT, -#define _aMY_CXT ,aMY_CXT - -#endif /* START_MY_CXT */ - -#ifndef MY_CXT_CLONE -/* Clones the per-interpreter data. */ -#define MY_CXT_CLONE \ - dMY_CXT_SV; \ - my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ - Copy(INT2PTR(my_cxt_t*, SvUV(my_cxt_sv)), my_cxtp, 1, my_cxt_t);\ - sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) -#endif - -#else /* single interpreter */ - -#ifndef START_MY_CXT - -#define START_MY_CXT static my_cxt_t my_cxt; -#define dMY_CXT_SV dNOOP -#define dMY_CXT dNOOP -#define MY_CXT_INIT NOOP -#define MY_CXT my_cxt - -#define pMY_CXT void -#define pMY_CXT_ -#define _pMY_CXT -#define aMY_CXT -#define aMY_CXT_ -#define _aMY_CXT - -#endif /* START_MY_CXT */ - -#ifndef MY_CXT_CLONE -#define MY_CXT_CLONE NOOP -#endif - -#endif - -#ifndef IVdf -# if IVSIZE == LONGSIZE -# define IVdf "ld" -# define UVuf "lu" -# define UVof "lo" -# define UVxf "lx" -# define UVXf "lX" -# else -# if IVSIZE == INTSIZE -# define IVdf "d" -# define UVuf "u" -# define UVof "o" -# define UVxf "x" -# define UVXf "X" -# endif -# endif -#endif - -#ifndef NVef -# if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) && \ - defined(PERL_PRIfldbl) && (PERL_BCDVERSION != 0x5006000) - /* Not very likely, but let's try anyway. */ -# define NVef PERL_PRIeldbl -# define NVff PERL_PRIfldbl -# define NVgf PERL_PRIgldbl -# else -# define NVef "e" -# define NVff "f" -# define NVgf "g" -# endif -#endif - -#ifndef SvREFCNT_inc -# ifdef PERL_USE_GCC_BRACE_GROUPS -# define SvREFCNT_inc(sv) \ - ({ \ - SV * const _sv = (SV*)(sv); \ - if (_sv) \ - (SvREFCNT(_sv))++; \ - _sv; \ - }) -# else -# define SvREFCNT_inc(sv) \ - ((PL_Sv=(SV*)(sv)) ? (++(SvREFCNT(PL_Sv)),PL_Sv) : NULL) -# endif -#endif - -#ifndef SvREFCNT_inc_simple -# ifdef PERL_USE_GCC_BRACE_GROUPS -# define SvREFCNT_inc_simple(sv) \ - ({ \ - if (sv) \ - (SvREFCNT(sv))++; \ - (SV *)(sv); \ - }) -# else -# define SvREFCNT_inc_simple(sv) \ - ((sv) ? (SvREFCNT(sv)++,(SV*)(sv)) : NULL) -# endif -#endif - -#ifndef SvREFCNT_inc_NN -# ifdef PERL_USE_GCC_BRACE_GROUPS -# define SvREFCNT_inc_NN(sv) \ - ({ \ - SV * const _sv = (SV*)(sv); \ - SvREFCNT(_sv)++; \ - _sv; \ - }) -# else -# define SvREFCNT_inc_NN(sv) \ - (PL_Sv=(SV*)(sv),++(SvREFCNT(PL_Sv)),PL_Sv) -# endif -#endif - -#ifndef SvREFCNT_inc_void -# ifdef PERL_USE_GCC_BRACE_GROUPS -# define SvREFCNT_inc_void(sv) \ - ({ \ - SV * const _sv = (SV*)(sv); \ - if (_sv) \ - (void)(SvREFCNT(_sv)++); \ - }) -# else -# define SvREFCNT_inc_void(sv) \ - (void)((PL_Sv=(SV*)(sv)) ? ++(SvREFCNT(PL_Sv)) : 0) -# endif -#endif -#ifndef SvREFCNT_inc_simple_void -# define SvREFCNT_inc_simple_void(sv) STMT_START { if (sv) SvREFCNT(sv)++; } STMT_END -#endif - -#ifndef SvREFCNT_inc_simple_NN -# define SvREFCNT_inc_simple_NN(sv) (++SvREFCNT(sv), (SV*)(sv)) -#endif - -#ifndef SvREFCNT_inc_void_NN -# define SvREFCNT_inc_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) -#endif - -#ifndef SvREFCNT_inc_simple_void_NN -# define SvREFCNT_inc_simple_void_NN(sv) (void)(++SvREFCNT((SV*)(sv))) -#endif - -#ifndef newSV_type - -#if defined(NEED_newSV_type) -static SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); -static -#else -extern SV* DPPP_(my_newSV_type)(pTHX_ svtype const t); -#endif - -#ifdef newSV_type -# undef newSV_type -#endif -#define newSV_type(a) DPPP_(my_newSV_type)(aTHX_ a) -#define Perl_newSV_type DPPP_(my_newSV_type) - -#if defined(NEED_newSV_type) || defined(NEED_newSV_type_GLOBAL) - -SV* -DPPP_(my_newSV_type)(pTHX_ svtype const t) -{ - SV* const sv = newSV(0); - sv_upgrade(sv, t); - return sv; -} - -#endif - -#endif - -#if (PERL_BCDVERSION < 0x5006000) -# define D_PPP_CONSTPV_ARG(x) ((char *) (x)) -#else -# define D_PPP_CONSTPV_ARG(x) (x) -#endif -#ifndef newSVpvn -# define newSVpvn(data,len) ((data) \ - ? ((len) ? newSVpv((data), (len)) : newSVpv("", 0)) \ - : newSV(0)) -#endif -#ifndef newSVpvn_utf8 -# define newSVpvn_utf8(s, len, u) newSVpvn_flags((s), (len), (u) ? SVf_UTF8 : 0) -#endif -#ifndef SVf_UTF8 -# define SVf_UTF8 0 -#endif - -#ifndef newSVpvn_flags - -#if defined(NEED_newSVpvn_flags) -static SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); -static -#else -extern SV * DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags); -#endif - -#ifdef newSVpvn_flags -# undef newSVpvn_flags -#endif -#define newSVpvn_flags(a,b,c) DPPP_(my_newSVpvn_flags)(aTHX_ a,b,c) -#define Perl_newSVpvn_flags DPPP_(my_newSVpvn_flags) - -#if defined(NEED_newSVpvn_flags) || defined(NEED_newSVpvn_flags_GLOBAL) - -SV * -DPPP_(my_newSVpvn_flags)(pTHX_ const char *s, STRLEN len, U32 flags) -{ - SV *sv = newSVpvn(D_PPP_CONSTPV_ARG(s), len); - SvFLAGS(sv) |= (flags & SVf_UTF8); - return (flags & SVs_TEMP) ? sv_2mortal(sv) : sv; -} - -#endif - -#endif - -/* Backwards compatibility stuff... :-( */ -#if !defined(NEED_sv_2pv_flags) && defined(NEED_sv_2pv_nolen) -# define NEED_sv_2pv_flags -#endif -#if !defined(NEED_sv_2pv_flags_GLOBAL) && defined(NEED_sv_2pv_nolen_GLOBAL) -# define NEED_sv_2pv_flags_GLOBAL -#endif - -/* Hint: sv_2pv_nolen - * Use the SvPV_nolen() or SvPV_nolen_const() macros instead of sv_2pv_nolen(). - */ -#ifndef sv_2pv_nolen -# define sv_2pv_nolen(sv) SvPV_nolen(sv) -#endif - -#ifdef SvPVbyte - -/* Hint: SvPVbyte - * Does not work in perl-5.6.1, ppport.h implements a version - * borrowed from perl-5.7.3. - */ - -#if (PERL_BCDVERSION < 0x5007000) - -#if defined(NEED_sv_2pvbyte) -static char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); -static -#else -extern char * DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp); -#endif - -#ifdef sv_2pvbyte -# undef sv_2pvbyte -#endif -#define sv_2pvbyte(a,b) DPPP_(my_sv_2pvbyte)(aTHX_ a,b) -#define Perl_sv_2pvbyte DPPP_(my_sv_2pvbyte) - -#if defined(NEED_sv_2pvbyte) || defined(NEED_sv_2pvbyte_GLOBAL) - -char * -DPPP_(my_sv_2pvbyte)(pTHX_ SV *sv, STRLEN *lp) -{ - sv_utf8_downgrade(sv,0); - return SvPV(sv,*lp); -} - -#endif - -/* Hint: sv_2pvbyte - * Use the SvPVbyte() macro instead of sv_2pvbyte(). - */ - -#undef SvPVbyte - -#define SvPVbyte(sv, lp) \ - ((SvFLAGS(sv) & (SVf_POK|SVf_UTF8)) == (SVf_POK) \ - ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pvbyte(sv, &lp)) - -#endif - -#else - -# define SvPVbyte SvPV -# define sv_2pvbyte sv_2pv - -#endif -#ifndef sv_2pvbyte_nolen -# define sv_2pvbyte_nolen(sv) sv_2pv_nolen(sv) -#endif - -/* Hint: sv_pvn - * Always use the SvPV() macro instead of sv_pvn(). - */ - -/* Hint: sv_pvn_force - * Always use the SvPV_force() macro instead of sv_pvn_force(). - */ - -/* If these are undefined, they're not handled by the core anyway */ -#ifndef SV_IMMEDIATE_UNREF -# define SV_IMMEDIATE_UNREF 0 -#endif - -#ifndef SV_GMAGIC -# define SV_GMAGIC 0 -#endif - -#ifndef SV_COW_DROP_PV -# define SV_COW_DROP_PV 0 -#endif - -#ifndef SV_UTF8_NO_ENCODING -# define SV_UTF8_NO_ENCODING 0 -#endif - -#ifndef SV_NOSTEAL -# define SV_NOSTEAL 0 -#endif - -#ifndef SV_CONST_RETURN -# define SV_CONST_RETURN 0 -#endif - -#ifndef SV_MUTABLE_RETURN -# define SV_MUTABLE_RETURN 0 -#endif - -#ifndef SV_SMAGIC -# define SV_SMAGIC 0 -#endif - -#ifndef SV_HAS_TRAILING_NUL -# define SV_HAS_TRAILING_NUL 0 -#endif - -#ifndef SV_COW_SHARED_HASH_KEYS -# define SV_COW_SHARED_HASH_KEYS 0 -#endif - -#if (PERL_BCDVERSION < 0x5007002) - -#if defined(NEED_sv_2pv_flags) -static char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); -static -#else -extern char * DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); -#endif - -#ifdef sv_2pv_flags -# undef sv_2pv_flags -#endif -#define sv_2pv_flags(a,b,c) DPPP_(my_sv_2pv_flags)(aTHX_ a,b,c) -#define Perl_sv_2pv_flags DPPP_(my_sv_2pv_flags) - -#if defined(NEED_sv_2pv_flags) || defined(NEED_sv_2pv_flags_GLOBAL) - -char * -DPPP_(my_sv_2pv_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) -{ - STRLEN n_a = (STRLEN) flags; - return sv_2pv(sv, lp ? lp : &n_a); -} - -#endif - -#if defined(NEED_sv_pvn_force_flags) -static char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); -static -#else -extern char * DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags); -#endif - -#ifdef sv_pvn_force_flags -# undef sv_pvn_force_flags -#endif -#define sv_pvn_force_flags(a,b,c) DPPP_(my_sv_pvn_force_flags)(aTHX_ a,b,c) -#define Perl_sv_pvn_force_flags DPPP_(my_sv_pvn_force_flags) - -#if defined(NEED_sv_pvn_force_flags) || defined(NEED_sv_pvn_force_flags_GLOBAL) - -char * -DPPP_(my_sv_pvn_force_flags)(pTHX_ SV *sv, STRLEN *lp, I32 flags) -{ - STRLEN n_a = (STRLEN) flags; - return sv_pvn_force(sv, lp ? lp : &n_a); -} - -#endif - -#endif - -#if (PERL_BCDVERSION < 0x5008008) || ( (PERL_BCDVERSION >= 0x5009000) && (PERL_BCDVERSION < 0x5009003) ) -# define DPPP_SVPV_NOLEN_LP_ARG &PL_na -#else -# define DPPP_SVPV_NOLEN_LP_ARG 0 -#endif -#ifndef SvPV_const -# define SvPV_const(sv, lp) SvPV_flags_const(sv, lp, SV_GMAGIC) -#endif - -#ifndef SvPV_mutable -# define SvPV_mutable(sv, lp) SvPV_flags_mutable(sv, lp, SV_GMAGIC) -#endif -#ifndef SvPV_flags -# define SvPV_flags(sv, lp, flags) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_2pv_flags(sv, &lp, flags)) -#endif -#ifndef SvPV_flags_const -# define SvPV_flags_const(sv, lp, flags) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? ((lp = SvCUR(sv)), SvPVX_const(sv)) : \ - (const char*) sv_2pv_flags(sv, &lp, flags|SV_CONST_RETURN)) -#endif -#ifndef SvPV_flags_const_nolen -# define SvPV_flags_const_nolen(sv, flags) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? SvPVX_const(sv) : \ - (const char*) sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags|SV_CONST_RETURN)) -#endif -#ifndef SvPV_flags_mutable -# define SvPV_flags_mutable(sv, lp, flags) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) : \ - sv_2pv_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) -#endif -#ifndef SvPV_force -# define SvPV_force(sv, lp) SvPV_force_flags(sv, lp, SV_GMAGIC) -#endif - -#ifndef SvPV_force_nolen -# define SvPV_force_nolen(sv) SvPV_force_flags_nolen(sv, SV_GMAGIC) -#endif - -#ifndef SvPV_force_mutable -# define SvPV_force_mutable(sv, lp) SvPV_force_flags_mutable(sv, lp, SV_GMAGIC) -#endif - -#ifndef SvPV_force_nomg -# define SvPV_force_nomg(sv, lp) SvPV_force_flags(sv, lp, 0) -#endif - -#ifndef SvPV_force_nomg_nolen -# define SvPV_force_nomg_nolen(sv) SvPV_force_flags_nolen(sv, 0) -#endif -#ifndef SvPV_force_flags -# define SvPV_force_flags(sv, lp, flags) \ - ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ - ? ((lp = SvCUR(sv)), SvPVX(sv)) : sv_pvn_force_flags(sv, &lp, flags)) -#endif -#ifndef SvPV_force_flags_nolen -# define SvPV_force_flags_nolen(sv, flags) \ - ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ - ? SvPVX(sv) : sv_pvn_force_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, flags)) -#endif -#ifndef SvPV_force_flags_mutable -# define SvPV_force_flags_mutable(sv, lp, flags) \ - ((SvFLAGS(sv) & (SVf_POK|SVf_THINKFIRST)) == SVf_POK \ - ? ((lp = SvCUR(sv)), SvPVX_mutable(sv)) \ - : sv_pvn_force_flags(sv, &lp, flags|SV_MUTABLE_RETURN)) -#endif -#ifndef SvPV_nolen -# define SvPV_nolen(sv) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? SvPVX(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC)) -#endif -#ifndef SvPV_nolen_const -# define SvPV_nolen_const(sv) \ - ((SvFLAGS(sv) & (SVf_POK)) == SVf_POK \ - ? SvPVX_const(sv) : sv_2pv_flags(sv, DPPP_SVPV_NOLEN_LP_ARG, SV_GMAGIC|SV_CONST_RETURN)) -#endif -#ifndef SvPV_nomg -# define SvPV_nomg(sv, lp) SvPV_flags(sv, lp, 0) -#endif - -#ifndef SvPV_nomg_const -# define SvPV_nomg_const(sv, lp) SvPV_flags_const(sv, lp, 0) -#endif - -#ifndef SvPV_nomg_const_nolen -# define SvPV_nomg_const_nolen(sv) SvPV_flags_const_nolen(sv, 0) -#endif -#ifndef SvPV_renew -# define SvPV_renew(sv,n) STMT_START { SvLEN_set(sv, n); \ - SvPV_set((sv), (char *) saferealloc( \ - (Malloc_t)SvPVX(sv), (MEM_SIZE)((n)))); \ - } STMT_END -#endif -#ifndef SvMAGIC_set -# define SvMAGIC_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ - (((XPVMG*) SvANY(sv))->xmg_magic = (val)); } STMT_END -#endif - -#if (PERL_BCDVERSION < 0x5009003) -#ifndef SvPVX_const -# define SvPVX_const(sv) ((const char*) (0 + SvPVX(sv))) -#endif - -#ifndef SvPVX_mutable -# define SvPVX_mutable(sv) (0 + SvPVX(sv)) -#endif -#ifndef SvRV_set -# define SvRV_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ - (((XRV*) SvANY(sv))->xrv_rv = (val)); } STMT_END -#endif - -#else -#ifndef SvPVX_const -# define SvPVX_const(sv) ((const char*)((sv)->sv_u.svu_pv)) -#endif - -#ifndef SvPVX_mutable -# define SvPVX_mutable(sv) ((sv)->sv_u.svu_pv) -#endif -#ifndef SvRV_set -# define SvRV_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) >= SVt_RV); \ - ((sv)->sv_u.svu_rv = (val)); } STMT_END -#endif - -#endif -#ifndef SvSTASH_set -# define SvSTASH_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) >= SVt_PVMG); \ - (((XPVMG*) SvANY(sv))->xmg_stash = (val)); } STMT_END -#endif - -#if (PERL_BCDVERSION < 0x5004000) -#ifndef SvUV_set -# define SvUV_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ - (((XPVIV*) SvANY(sv))->xiv_iv = (IV) (val)); } STMT_END -#endif - -#else -#ifndef SvUV_set -# define SvUV_set(sv, val) \ - STMT_START { assert(SvTYPE(sv) == SVt_IV || SvTYPE(sv) >= SVt_PVIV); \ - (((XPVUV*) SvANY(sv))->xuv_uv = (val)); } STMT_END -#endif - -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(vnewSVpvf) -#if defined(NEED_vnewSVpvf) -static SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); -static -#else -extern SV * DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args); -#endif - -#ifdef vnewSVpvf -# undef vnewSVpvf -#endif -#define vnewSVpvf(a,b) DPPP_(my_vnewSVpvf)(aTHX_ a,b) -#define Perl_vnewSVpvf DPPP_(my_vnewSVpvf) - -#if defined(NEED_vnewSVpvf) || defined(NEED_vnewSVpvf_GLOBAL) - -SV * -DPPP_(my_vnewSVpvf)(pTHX_ const char *pat, va_list *args) -{ - register SV *sv = newSV(0); - sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); - return sv; -} - -#endif -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf) -# define sv_vcatpvf(sv, pat, args) sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf) -# define sv_vsetpvf(sv, pat, args) sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)) -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg) -#if defined(NEED_sv_catpvf_mg) -static void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); -static -#else -extern void DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...); -#endif - -#define Perl_sv_catpvf_mg DPPP_(my_sv_catpvf_mg) - -#if defined(NEED_sv_catpvf_mg) || defined(NEED_sv_catpvf_mg_GLOBAL) - -void -DPPP_(my_sv_catpvf_mg)(pTHX_ SV *sv, const char *pat, ...) -{ - va_list args; - va_start(args, pat); - sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); - SvSETMAGIC(sv); - va_end(args); -} - -#endif -#endif - -#ifdef PERL_IMPLICIT_CONTEXT -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_catpvf_mg_nocontext) -#if defined(NEED_sv_catpvf_mg_nocontext) -static void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); -static -#else -extern void DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...); -#endif - -#define sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) -#define Perl_sv_catpvf_mg_nocontext DPPP_(my_sv_catpvf_mg_nocontext) - -#if defined(NEED_sv_catpvf_mg_nocontext) || defined(NEED_sv_catpvf_mg_nocontext_GLOBAL) - -void -DPPP_(my_sv_catpvf_mg_nocontext)(SV *sv, const char *pat, ...) -{ - dTHX; - va_list args; - va_start(args, pat); - sv_vcatpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); - SvSETMAGIC(sv); - va_end(args); -} - -#endif -#endif -#endif - -/* sv_catpvf_mg depends on sv_catpvf_mg_nocontext */ -#ifndef sv_catpvf_mg -# ifdef PERL_IMPLICIT_CONTEXT -# define sv_catpvf_mg Perl_sv_catpvf_mg_nocontext -# else -# define sv_catpvf_mg Perl_sv_catpvf_mg -# endif -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vcatpvf_mg) -# define sv_vcatpvf_mg(sv, pat, args) \ - STMT_START { \ - sv_vcatpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ - SvSETMAGIC(sv); \ - } STMT_END -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg) -#if defined(NEED_sv_setpvf_mg) -static void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); -static -#else -extern void DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...); -#endif - -#define Perl_sv_setpvf_mg DPPP_(my_sv_setpvf_mg) - -#if defined(NEED_sv_setpvf_mg) || defined(NEED_sv_setpvf_mg_GLOBAL) - -void -DPPP_(my_sv_setpvf_mg)(pTHX_ SV *sv, const char *pat, ...) -{ - va_list args; - va_start(args, pat); - sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); - SvSETMAGIC(sv); - va_end(args); -} - -#endif -#endif - -#ifdef PERL_IMPLICIT_CONTEXT -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_setpvf_mg_nocontext) -#if defined(NEED_sv_setpvf_mg_nocontext) -static void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); -static -#else -extern void DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...); -#endif - -#define sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) -#define Perl_sv_setpvf_mg_nocontext DPPP_(my_sv_setpvf_mg_nocontext) - -#if defined(NEED_sv_setpvf_mg_nocontext) || defined(NEED_sv_setpvf_mg_nocontext_GLOBAL) - -void -DPPP_(my_sv_setpvf_mg_nocontext)(SV *sv, const char *pat, ...) -{ - dTHX; - va_list args; - va_start(args, pat); - sv_vsetpvfn(sv, pat, strlen(pat), &args, Null(SV**), 0, Null(bool*)); - SvSETMAGIC(sv); - va_end(args); -} - -#endif -#endif -#endif - -/* sv_setpvf_mg depends on sv_setpvf_mg_nocontext */ -#ifndef sv_setpvf_mg -# ifdef PERL_IMPLICIT_CONTEXT -# define sv_setpvf_mg Perl_sv_setpvf_mg_nocontext -# else -# define sv_setpvf_mg Perl_sv_setpvf_mg -# endif -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(sv_vsetpvf_mg) -# define sv_vsetpvf_mg(sv, pat, args) \ - STMT_START { \ - sv_vsetpvfn(sv, pat, strlen(pat), args, Null(SV**), 0, Null(bool*)); \ - SvSETMAGIC(sv); \ - } STMT_END -#endif - -#ifndef newSVpvn_share - -#if defined(NEED_newSVpvn_share) -static SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); -static -#else -extern SV * DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash); -#endif - -#ifdef newSVpvn_share -# undef newSVpvn_share -#endif -#define newSVpvn_share(a,b,c) DPPP_(my_newSVpvn_share)(aTHX_ a,b,c) -#define Perl_newSVpvn_share DPPP_(my_newSVpvn_share) - -#if defined(NEED_newSVpvn_share) || defined(NEED_newSVpvn_share_GLOBAL) - -SV * -DPPP_(my_newSVpvn_share)(pTHX_ const char *src, I32 len, U32 hash) -{ - SV *sv; - if (len < 0) - len = -len; - if (!hash) - PERL_HASH(hash, (char*) src, len); - sv = newSVpvn((char *) src, len); - sv_upgrade(sv, SVt_PVIV); - SvIVX(sv) = hash; - SvREADONLY_on(sv); - SvPOK_on(sv); - return sv; -} - -#endif - -#endif -#ifndef SvSHARED_HASH -# define SvSHARED_HASH(sv) (0 + SvUVX(sv)) -#endif -#ifndef HvNAME_get -# define HvNAME_get(hv) HvNAME(hv) -#endif -#ifndef HvNAMELEN_get -# define HvNAMELEN_get(hv) (HvNAME_get(hv) ? (I32)strlen(HvNAME_get(hv)) : 0) -#endif -#ifndef GvSVn -# define GvSVn(gv) GvSV(gv) -#endif - -#ifndef isGV_with_GP -# define isGV_with_GP(gv) isGV(gv) -#endif -#ifndef WARN_ALL -# define WARN_ALL 0 -#endif - -#ifndef WARN_CLOSURE -# define WARN_CLOSURE 1 -#endif - -#ifndef WARN_DEPRECATED -# define WARN_DEPRECATED 2 -#endif - -#ifndef WARN_EXITING -# define WARN_EXITING 3 -#endif - -#ifndef WARN_GLOB -# define WARN_GLOB 4 -#endif - -#ifndef WARN_IO -# define WARN_IO 5 -#endif - -#ifndef WARN_CLOSED -# define WARN_CLOSED 6 -#endif - -#ifndef WARN_EXEC -# define WARN_EXEC 7 -#endif - -#ifndef WARN_LAYER -# define WARN_LAYER 8 -#endif - -#ifndef WARN_NEWLINE -# define WARN_NEWLINE 9 -#endif - -#ifndef WARN_PIPE -# define WARN_PIPE 10 -#endif - -#ifndef WARN_UNOPENED -# define WARN_UNOPENED 11 -#endif - -#ifndef WARN_MISC -# define WARN_MISC 12 -#endif - -#ifndef WARN_NUMERIC -# define WARN_NUMERIC 13 -#endif - -#ifndef WARN_ONCE -# define WARN_ONCE 14 -#endif - -#ifndef WARN_OVERFLOW -# define WARN_OVERFLOW 15 -#endif - -#ifndef WARN_PACK -# define WARN_PACK 16 -#endif - -#ifndef WARN_PORTABLE -# define WARN_PORTABLE 17 -#endif - -#ifndef WARN_RECURSION -# define WARN_RECURSION 18 -#endif - -#ifndef WARN_REDEFINE -# define WARN_REDEFINE 19 -#endif - -#ifndef WARN_REGEXP -# define WARN_REGEXP 20 -#endif - -#ifndef WARN_SEVERE -# define WARN_SEVERE 21 -#endif - -#ifndef WARN_DEBUGGING -# define WARN_DEBUGGING 22 -#endif - -#ifndef WARN_INPLACE -# define WARN_INPLACE 23 -#endif - -#ifndef WARN_INTERNAL -# define WARN_INTERNAL 24 -#endif - -#ifndef WARN_MALLOC -# define WARN_MALLOC 25 -#endif - -#ifndef WARN_SIGNAL -# define WARN_SIGNAL 26 -#endif - -#ifndef WARN_SUBSTR -# define WARN_SUBSTR 27 -#endif - -#ifndef WARN_SYNTAX -# define WARN_SYNTAX 28 -#endif - -#ifndef WARN_AMBIGUOUS -# define WARN_AMBIGUOUS 29 -#endif - -#ifndef WARN_BAREWORD -# define WARN_BAREWORD 30 -#endif - -#ifndef WARN_DIGIT -# define WARN_DIGIT 31 -#endif - -#ifndef WARN_PARENTHESIS -# define WARN_PARENTHESIS 32 -#endif - -#ifndef WARN_PRECEDENCE -# define WARN_PRECEDENCE 33 -#endif - -#ifndef WARN_PRINTF -# define WARN_PRINTF 34 -#endif - -#ifndef WARN_PROTOTYPE -# define WARN_PROTOTYPE 35 -#endif - -#ifndef WARN_QW -# define WARN_QW 36 -#endif - -#ifndef WARN_RESERVED -# define WARN_RESERVED 37 -#endif - -#ifndef WARN_SEMICOLON -# define WARN_SEMICOLON 38 -#endif - -#ifndef WARN_TAINT -# define WARN_TAINT 39 -#endif - -#ifndef WARN_THREADS -# define WARN_THREADS 40 -#endif - -#ifndef WARN_UNINITIALIZED -# define WARN_UNINITIALIZED 41 -#endif - -#ifndef WARN_UNPACK -# define WARN_UNPACK 42 -#endif - -#ifndef WARN_UNTIE -# define WARN_UNTIE 43 -#endif - -#ifndef WARN_UTF8 -# define WARN_UTF8 44 -#endif - -#ifndef WARN_VOID -# define WARN_VOID 45 -#endif - -#ifndef WARN_ASSERTIONS -# define WARN_ASSERTIONS 46 -#endif -#ifndef packWARN -# define packWARN(a) (a) -#endif - -#ifndef ckWARN -# ifdef G_WARN_ON -# define ckWARN(a) (PL_dowarn & G_WARN_ON) -# else -# define ckWARN(a) PL_dowarn -# endif -#endif - -#if (PERL_BCDVERSION >= 0x5004000) && !defined(warner) -#if defined(NEED_warner) -static void DPPP_(my_warner)(U32 err, const char *pat, ...); -static -#else -extern void DPPP_(my_warner)(U32 err, const char *pat, ...); -#endif - -#define Perl_warner DPPP_(my_warner) - -#if defined(NEED_warner) || defined(NEED_warner_GLOBAL) - -void -DPPP_(my_warner)(U32 err, const char *pat, ...) -{ - SV *sv; - va_list args; - - PERL_UNUSED_ARG(err); - - va_start(args, pat); - sv = vnewSVpvf(pat, &args); - va_end(args); - sv_2mortal(sv); - warn("%s", SvPV_nolen(sv)); -} - -#define warner Perl_warner - -#define Perl_warner_nocontext Perl_warner - -#endif -#endif - -/* concatenating with "" ensures that only literal strings are accepted as argument - * note that STR_WITH_LEN() can't be used as argument to macros or functions that - * under some configurations might be macros - */ -#ifndef STR_WITH_LEN -# define STR_WITH_LEN(s) (s ""), (sizeof(s)-1) -#endif -#ifndef newSVpvs -# define newSVpvs(str) newSVpvn(str "", sizeof(str) - 1) -#endif - -#ifndef newSVpvs_flags -# define newSVpvs_flags(str, flags) newSVpvn_flags(str "", sizeof(str) - 1, flags) -#endif - -#ifndef sv_catpvs -# define sv_catpvs(sv, str) sv_catpvn(sv, str "", sizeof(str) - 1) -#endif - -#ifndef sv_setpvs -# define sv_setpvs(sv, str) sv_setpvn(sv, str "", sizeof(str) - 1) -#endif - -#ifndef hv_fetchs -# define hv_fetchs(hv, key, lval) hv_fetch(hv, key "", sizeof(key) - 1, lval) -#endif - -#ifndef hv_stores -# define hv_stores(hv, key, val) hv_store(hv, key "", sizeof(key) - 1, val, 0) -#endif -#ifndef gv_fetchpvn_flags -# define gv_fetchpvn_flags(name, len, flags, svt) gv_fetchpv(name, flags, svt) -#endif - -#ifndef gv_fetchpvs -# define gv_fetchpvs(name, flags, svt) gv_fetchpvn_flags(name "", sizeof(name) - 1, flags, svt) -#endif - -#ifndef gv_stashpvs -# define gv_stashpvs(name, flags) gv_stashpvn(name "", sizeof(name) - 1, flags) -#endif -#ifndef SvGETMAGIC -# define SvGETMAGIC(x) STMT_START { if (SvGMAGICAL(x)) mg_get(x); } STMT_END -#endif -#ifndef PERL_MAGIC_sv -# define PERL_MAGIC_sv '\0' -#endif - -#ifndef PERL_MAGIC_overload -# define PERL_MAGIC_overload 'A' -#endif - -#ifndef PERL_MAGIC_overload_elem -# define PERL_MAGIC_overload_elem 'a' -#endif - -#ifndef PERL_MAGIC_overload_table -# define PERL_MAGIC_overload_table 'c' -#endif - -#ifndef PERL_MAGIC_bm -# define PERL_MAGIC_bm 'B' -#endif - -#ifndef PERL_MAGIC_regdata -# define PERL_MAGIC_regdata 'D' -#endif - -#ifndef PERL_MAGIC_regdatum -# define PERL_MAGIC_regdatum 'd' -#endif - -#ifndef PERL_MAGIC_env -# define PERL_MAGIC_env 'E' -#endif - -#ifndef PERL_MAGIC_envelem -# define PERL_MAGIC_envelem 'e' -#endif - -#ifndef PERL_MAGIC_fm -# define PERL_MAGIC_fm 'f' -#endif - -#ifndef PERL_MAGIC_regex_global -# define PERL_MAGIC_regex_global 'g' -#endif - -#ifndef PERL_MAGIC_isa -# define PERL_MAGIC_isa 'I' -#endif - -#ifndef PERL_MAGIC_isaelem -# define PERL_MAGIC_isaelem 'i' -#endif - -#ifndef PERL_MAGIC_nkeys -# define PERL_MAGIC_nkeys 'k' -#endif - -#ifndef PERL_MAGIC_dbfile -# define PERL_MAGIC_dbfile 'L' -#endif - -#ifndef PERL_MAGIC_dbline -# define PERL_MAGIC_dbline 'l' -#endif - -#ifndef PERL_MAGIC_mutex -# define PERL_MAGIC_mutex 'm' -#endif - -#ifndef PERL_MAGIC_shared -# define PERL_MAGIC_shared 'N' -#endif - -#ifndef PERL_MAGIC_shared_scalar -# define PERL_MAGIC_shared_scalar 'n' -#endif - -#ifndef PERL_MAGIC_collxfrm -# define PERL_MAGIC_collxfrm 'o' -#endif - -#ifndef PERL_MAGIC_tied -# define PERL_MAGIC_tied 'P' -#endif - -#ifndef PERL_MAGIC_tiedelem -# define PERL_MAGIC_tiedelem 'p' -#endif - -#ifndef PERL_MAGIC_tiedscalar -# define PERL_MAGIC_tiedscalar 'q' -#endif - -#ifndef PERL_MAGIC_qr -# define PERL_MAGIC_qr 'r' -#endif - -#ifndef PERL_MAGIC_sig -# define PERL_MAGIC_sig 'S' -#endif - -#ifndef PERL_MAGIC_sigelem -# define PERL_MAGIC_sigelem 's' -#endif - -#ifndef PERL_MAGIC_taint -# define PERL_MAGIC_taint 't' -#endif - -#ifndef PERL_MAGIC_uvar -# define PERL_MAGIC_uvar 'U' -#endif - -#ifndef PERL_MAGIC_uvar_elem -# define PERL_MAGIC_uvar_elem 'u' -#endif - -#ifndef PERL_MAGIC_vstring -# define PERL_MAGIC_vstring 'V' -#endif - -#ifndef PERL_MAGIC_vec -# define PERL_MAGIC_vec 'v' -#endif - -#ifndef PERL_MAGIC_utf8 -# define PERL_MAGIC_utf8 'w' -#endif - -#ifndef PERL_MAGIC_substr -# define PERL_MAGIC_substr 'x' -#endif - -#ifndef PERL_MAGIC_defelem -# define PERL_MAGIC_defelem 'y' -#endif - -#ifndef PERL_MAGIC_glob -# define PERL_MAGIC_glob '*' -#endif - -#ifndef PERL_MAGIC_arylen -# define PERL_MAGIC_arylen '#' -#endif - -#ifndef PERL_MAGIC_pos -# define PERL_MAGIC_pos '.' -#endif - -#ifndef PERL_MAGIC_backref -# define PERL_MAGIC_backref '<' -#endif - -#ifndef PERL_MAGIC_ext -# define PERL_MAGIC_ext '~' -#endif - -/* That's the best we can do... */ -#ifndef sv_catpvn_nomg -# define sv_catpvn_nomg sv_catpvn -#endif - -#ifndef sv_catsv_nomg -# define sv_catsv_nomg sv_catsv -#endif - -#ifndef sv_setsv_nomg -# define sv_setsv_nomg sv_setsv -#endif - -#ifndef sv_pvn_nomg -# define sv_pvn_nomg sv_pvn -#endif - -#ifndef SvIV_nomg -# define SvIV_nomg SvIV -#endif - -#ifndef SvUV_nomg -# define SvUV_nomg SvUV -#endif - -#ifndef sv_catpv_mg -# define sv_catpv_mg(sv, ptr) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_catpv(TeMpSv,ptr); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_catpvn_mg -# define sv_catpvn_mg(sv, ptr, len) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_catpvn(TeMpSv,ptr,len); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_catsv_mg -# define sv_catsv_mg(dsv, ssv) \ - STMT_START { \ - SV *TeMpSv = dsv; \ - sv_catsv(TeMpSv,ssv); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setiv_mg -# define sv_setiv_mg(sv, i) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_setiv(TeMpSv,i); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setnv_mg -# define sv_setnv_mg(sv, num) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_setnv(TeMpSv,num); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setpv_mg -# define sv_setpv_mg(sv, ptr) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_setpv(TeMpSv,ptr); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setpvn_mg -# define sv_setpvn_mg(sv, ptr, len) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_setpvn(TeMpSv,ptr,len); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setsv_mg -# define sv_setsv_mg(dsv, ssv) \ - STMT_START { \ - SV *TeMpSv = dsv; \ - sv_setsv(TeMpSv,ssv); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_setuv_mg -# define sv_setuv_mg(sv, i) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_setuv(TeMpSv,i); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif - -#ifndef sv_usepvn_mg -# define sv_usepvn_mg(sv, ptr, len) \ - STMT_START { \ - SV *TeMpSv = sv; \ - sv_usepvn(TeMpSv,ptr,len); \ - SvSETMAGIC(TeMpSv); \ - } STMT_END -#endif -#ifndef SvVSTRING_mg -# define SvVSTRING_mg(sv) (SvMAGICAL(sv) ? mg_find(sv, PERL_MAGIC_vstring) : NULL) -#endif - -/* Hint: sv_magic_portable - * This is a compatibility function that is only available with - * Devel::PPPort. It is NOT in the perl core. - * Its purpose is to mimic the 5.8.0 behaviour of sv_magic() when - * it is being passed a name pointer with namlen == 0. In that - * case, perl 5.8.0 and later store the pointer, not a copy of it. - * The compatibility can be provided back to perl 5.004. With - * earlier versions, the code will not compile. - */ - -#if (PERL_BCDVERSION < 0x5004000) - - /* code that uses sv_magic_portable will not compile */ - -#elif (PERL_BCDVERSION < 0x5008000) - -# define sv_magic_portable(sv, obj, how, name, namlen) \ - STMT_START { \ - SV *SvMp_sv = (sv); \ - char *SvMp_name = (char *) (name); \ - I32 SvMp_namlen = (namlen); \ - if (SvMp_name && SvMp_namlen == 0) \ - { \ - MAGIC *mg; \ - sv_magic(SvMp_sv, obj, how, 0, 0); \ - mg = SvMAGIC(SvMp_sv); \ - mg->mg_len = -42; /* XXX: this is the tricky part */ \ - mg->mg_ptr = SvMp_name; \ - } \ - else \ - { \ - sv_magic(SvMp_sv, obj, how, SvMp_name, SvMp_namlen); \ - } \ - } STMT_END - -#else - -# define sv_magic_portable(a, b, c, d, e) sv_magic(a, b, c, d, e) - -#endif - -#ifdef USE_ITHREADS -#ifndef CopFILE -# define CopFILE(c) ((c)->cop_file) -#endif - -#ifndef CopFILEGV -# define CopFILEGV(c) (CopFILE(c) ? gv_fetchfile(CopFILE(c)) : Nullgv) -#endif - -#ifndef CopFILE_set -# define CopFILE_set(c,pv) ((c)->cop_file = savepv(pv)) -#endif - -#ifndef CopFILESV -# define CopFILESV(c) (CopFILE(c) ? GvSV(gv_fetchfile(CopFILE(c))) : Nullsv) -#endif - -#ifndef CopFILEAV -# define CopFILEAV(c) (CopFILE(c) ? GvAV(gv_fetchfile(CopFILE(c))) : Nullav) -#endif - -#ifndef CopSTASHPV -# define CopSTASHPV(c) ((c)->cop_stashpv) -#endif - -#ifndef CopSTASHPV_set -# define CopSTASHPV_set(c,pv) ((c)->cop_stashpv = ((pv) ? savepv(pv) : Nullch)) -#endif - -#ifndef CopSTASH -# define CopSTASH(c) (CopSTASHPV(c) ? gv_stashpv(CopSTASHPV(c),GV_ADD) : Nullhv) -#endif - -#ifndef CopSTASH_set -# define CopSTASH_set(c,hv) CopSTASHPV_set(c, (hv) ? HvNAME(hv) : Nullch) -#endif - -#ifndef CopSTASH_eq -# define CopSTASH_eq(c,hv) ((hv) && (CopSTASHPV(c) == HvNAME(hv) \ - || (CopSTASHPV(c) && HvNAME(hv) \ - && strEQ(CopSTASHPV(c), HvNAME(hv))))) -#endif - -#else -#ifndef CopFILEGV -# define CopFILEGV(c) ((c)->cop_filegv) -#endif - -#ifndef CopFILEGV_set -# define CopFILEGV_set(c,gv) ((c)->cop_filegv = (GV*)SvREFCNT_inc(gv)) -#endif - -#ifndef CopFILE_set -# define CopFILE_set(c,pv) CopFILEGV_set((c), gv_fetchfile(pv)) -#endif - -#ifndef CopFILESV -# define CopFILESV(c) (CopFILEGV(c) ? GvSV(CopFILEGV(c)) : Nullsv) -#endif - -#ifndef CopFILEAV -# define CopFILEAV(c) (CopFILEGV(c) ? GvAV(CopFILEGV(c)) : Nullav) -#endif - -#ifndef CopFILE -# define CopFILE(c) (CopFILESV(c) ? SvPVX(CopFILESV(c)) : Nullch) -#endif - -#ifndef CopSTASH -# define CopSTASH(c) ((c)->cop_stash) -#endif - -#ifndef CopSTASH_set -# define CopSTASH_set(c,hv) ((c)->cop_stash = (hv)) -#endif - -#ifndef CopSTASHPV -# define CopSTASHPV(c) (CopSTASH(c) ? HvNAME(CopSTASH(c)) : Nullch) -#endif - -#ifndef CopSTASHPV_set -# define CopSTASHPV_set(c,pv) CopSTASH_set((c), gv_stashpv(pv,GV_ADD)) -#endif - -#ifndef CopSTASH_eq -# define CopSTASH_eq(c,hv) (CopSTASH(c) == (hv)) -#endif - -#endif /* USE_ITHREADS */ -#ifndef IN_PERL_COMPILETIME -# define IN_PERL_COMPILETIME (PL_curcop == &PL_compiling) -#endif - -#ifndef IN_LOCALE_RUNTIME -# define IN_LOCALE_RUNTIME (PL_curcop->op_private & HINT_LOCALE) -#endif - -#ifndef IN_LOCALE_COMPILETIME -# define IN_LOCALE_COMPILETIME (PL_hints & HINT_LOCALE) -#endif - -#ifndef IN_LOCALE -# define IN_LOCALE (IN_PERL_COMPILETIME ? IN_LOCALE_COMPILETIME : IN_LOCALE_RUNTIME) -#endif -#ifndef IS_NUMBER_IN_UV -# define IS_NUMBER_IN_UV 0x01 -#endif - -#ifndef IS_NUMBER_GREATER_THAN_UV_MAX -# define IS_NUMBER_GREATER_THAN_UV_MAX 0x02 -#endif - -#ifndef IS_NUMBER_NOT_INT -# define IS_NUMBER_NOT_INT 0x04 -#endif - -#ifndef IS_NUMBER_NEG -# define IS_NUMBER_NEG 0x08 -#endif - -#ifndef IS_NUMBER_INFINITY -# define IS_NUMBER_INFINITY 0x10 -#endif - -#ifndef IS_NUMBER_NAN -# define IS_NUMBER_NAN 0x20 -#endif -#ifndef GROK_NUMERIC_RADIX -# define GROK_NUMERIC_RADIX(sp, send) grok_numeric_radix(sp, send) -#endif -#ifndef PERL_SCAN_GREATER_THAN_UV_MAX -# define PERL_SCAN_GREATER_THAN_UV_MAX 0x02 -#endif - -#ifndef PERL_SCAN_SILENT_ILLDIGIT -# define PERL_SCAN_SILENT_ILLDIGIT 0x04 -#endif - -#ifndef PERL_SCAN_ALLOW_UNDERSCORES -# define PERL_SCAN_ALLOW_UNDERSCORES 0x01 -#endif - -#ifndef PERL_SCAN_DISALLOW_PREFIX -# define PERL_SCAN_DISALLOW_PREFIX 0x02 -#endif - -#ifndef grok_numeric_radix -#if defined(NEED_grok_numeric_radix) -static bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); -static -#else -extern bool DPPP_(my_grok_numeric_radix)(pTHX_ const char ** sp, const char * send); -#endif - -#ifdef grok_numeric_radix -# undef grok_numeric_radix -#endif -#define grok_numeric_radix(a,b) DPPP_(my_grok_numeric_radix)(aTHX_ a,b) -#define Perl_grok_numeric_radix DPPP_(my_grok_numeric_radix) - -#if defined(NEED_grok_numeric_radix) || defined(NEED_grok_numeric_radix_GLOBAL) -bool -DPPP_(my_grok_numeric_radix)(pTHX_ const char **sp, const char *send) -{ -#ifdef USE_LOCALE_NUMERIC -#ifdef PL_numeric_radix_sv - if (PL_numeric_radix_sv && IN_LOCALE) { - STRLEN len; - char* radix = SvPV(PL_numeric_radix_sv, len); - if (*sp + len <= send && memEQ(*sp, radix, len)) { - *sp += len; - return TRUE; - } - } -#else - /* older perls don't have PL_numeric_radix_sv so the radix - * must manually be requested from locale.h - */ -#include - dTHR; /* needed for older threaded perls */ - struct lconv *lc = localeconv(); - char *radix = lc->decimal_point; - if (radix && IN_LOCALE) { - STRLEN len = strlen(radix); - if (*sp + len <= send && memEQ(*sp, radix, len)) { - *sp += len; - return TRUE; - } - } -#endif -#endif /* USE_LOCALE_NUMERIC */ - /* always try "." if numeric radix didn't match because - * we may have data from different locales mixed */ - if (*sp < send && **sp == '.') { - ++*sp; - return TRUE; - } - return FALSE; -} -#endif -#endif - -#ifndef grok_number -#if defined(NEED_grok_number) -static int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); -static -#else -extern int DPPP_(my_grok_number)(pTHX_ const char * pv, STRLEN len, UV * valuep); -#endif - -#ifdef grok_number -# undef grok_number -#endif -#define grok_number(a,b,c) DPPP_(my_grok_number)(aTHX_ a,b,c) -#define Perl_grok_number DPPP_(my_grok_number) - -#if defined(NEED_grok_number) || defined(NEED_grok_number_GLOBAL) -int -DPPP_(my_grok_number)(pTHX_ const char *pv, STRLEN len, UV *valuep) -{ - const char *s = pv; - const char *send = pv + len; - const UV max_div_10 = UV_MAX / 10; - const char max_mod_10 = UV_MAX % 10; - int numtype = 0; - int sawinf = 0; - int sawnan = 0; - - while (s < send && isSPACE(*s)) - s++; - if (s == send) { - return 0; - } else if (*s == '-') { - s++; - numtype = IS_NUMBER_NEG; - } - else if (*s == '+') - s++; - - if (s == send) - return 0; - - /* next must be digit or the radix separator or beginning of infinity */ - if (isDIGIT(*s)) { - /* UVs are at least 32 bits, so the first 9 decimal digits cannot - overflow. */ - UV value = *s - '0'; - /* This construction seems to be more optimiser friendly. - (without it gcc does the isDIGIT test and the *s - '0' separately) - With it gcc on arm is managing 6 instructions (6 cycles) per digit. - In theory the optimiser could deduce how far to unroll the loop - before checking for overflow. */ - if (++s < send) { - int digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - digit = *s - '0'; - if (digit >= 0 && digit <= 9) { - value = value * 10 + digit; - if (++s < send) { - /* Now got 9 digits, so need to check - each time for overflow. */ - digit = *s - '0'; - while (digit >= 0 && digit <= 9 - && (value < max_div_10 - || (value == max_div_10 - && digit <= max_mod_10))) { - value = value * 10 + digit; - if (++s < send) - digit = *s - '0'; - else - break; - } - if (digit >= 0 && digit <= 9 - && (s < send)) { - /* value overflowed. - skip the remaining digits, don't - worry about setting *valuep. */ - do { - s++; - } while (s < send && isDIGIT(*s)); - numtype |= - IS_NUMBER_GREATER_THAN_UV_MAX; - goto skip_value; - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - numtype |= IS_NUMBER_IN_UV; - if (valuep) - *valuep = value; - - skip_value: - if (GROK_NUMERIC_RADIX(&s, send)) { - numtype |= IS_NUMBER_NOT_INT; - while (s < send && isDIGIT(*s)) /* optional digits after the radix */ - s++; - } - } - else if (GROK_NUMERIC_RADIX(&s, send)) { - numtype |= IS_NUMBER_NOT_INT | IS_NUMBER_IN_UV; /* valuep assigned below */ - /* no digits before the radix means we need digits after it */ - if (s < send && isDIGIT(*s)) { - do { - s++; - } while (s < send && isDIGIT(*s)); - if (valuep) { - /* integer approximation is valid - it's 0. */ - *valuep = 0; - } - } - else - return 0; - } else if (*s == 'I' || *s == 'i') { - s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; - s++; if (s == send || (*s != 'F' && *s != 'f')) return 0; - s++; if (s < send && (*s == 'I' || *s == 'i')) { - s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; - s++; if (s == send || (*s != 'I' && *s != 'i')) return 0; - s++; if (s == send || (*s != 'T' && *s != 't')) return 0; - s++; if (s == send || (*s != 'Y' && *s != 'y')) return 0; - s++; - } - sawinf = 1; - } else if (*s == 'N' || *s == 'n') { - /* XXX TODO: There are signaling NaNs and quiet NaNs. */ - s++; if (s == send || (*s != 'A' && *s != 'a')) return 0; - s++; if (s == send || (*s != 'N' && *s != 'n')) return 0; - s++; - sawnan = 1; - } else - return 0; - - if (sawinf) { - numtype &= IS_NUMBER_NEG; /* Keep track of sign */ - numtype |= IS_NUMBER_INFINITY | IS_NUMBER_NOT_INT; - } else if (sawnan) { - numtype &= IS_NUMBER_NEG; /* Keep track of sign */ - numtype |= IS_NUMBER_NAN | IS_NUMBER_NOT_INT; - } else if (s < send) { - /* we can have an optional exponent part */ - if (*s == 'e' || *s == 'E') { - /* The only flag we keep is sign. Blow away any "it's UV" */ - numtype &= IS_NUMBER_NEG; - numtype |= IS_NUMBER_NOT_INT; - s++; - if (s < send && (*s == '-' || *s == '+')) - s++; - if (s < send && isDIGIT(*s)) { - do { - s++; - } while (s < send && isDIGIT(*s)); - } - else - return 0; - } - } - while (s < send && isSPACE(*s)) - s++; - if (s >= send) - return numtype; - if (len == 10 && memEQ(pv, "0 but true", 10)) { - if (valuep) - *valuep = 0; - return IS_NUMBER_IN_UV; - } - return 0; -} -#endif -#endif - -/* - * The grok_* routines have been modified to use warn() instead of - * Perl_warner(). Also, 'hexdigit' was the former name of PL_hexdigit, - * which is why the stack variable has been renamed to 'xdigit'. - */ - -#ifndef grok_bin -#if defined(NEED_grok_bin) -static UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -static -#else -extern UV DPPP_(my_grok_bin)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -#endif - -#ifdef grok_bin -# undef grok_bin -#endif -#define grok_bin(a,b,c,d) DPPP_(my_grok_bin)(aTHX_ a,b,c,d) -#define Perl_grok_bin DPPP_(my_grok_bin) - -#if defined(NEED_grok_bin) || defined(NEED_grok_bin_GLOBAL) -UV -DPPP_(my_grok_bin)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) -{ - const char *s = start; - STRLEN len = *len_p; - UV value = 0; - NV value_nv = 0; - - const UV max_div_2 = UV_MAX / 2; - bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; - bool overflowed = FALSE; - - if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { - /* strip off leading b or 0b. - for compatibility silently suffer "b" and "0b" as valid binary - numbers. */ - if (len >= 1) { - if (s[0] == 'b') { - s++; - len--; - } - else if (len >= 2 && s[0] == '0' && s[1] == 'b') { - s+=2; - len-=2; - } - } - } - - for (; len-- && *s; s++) { - char bit = *s; - if (bit == '0' || bit == '1') { - /* Write it in this wonky order with a goto to attempt to get the - compiler to make the common case integer-only loop pretty tight. - With gcc seems to be much straighter code than old scan_bin. */ - redo: - if (!overflowed) { - if (value <= max_div_2) { - value = (value << 1) | (bit - '0'); - continue; - } - /* Bah. We're just overflowed. */ - warn("Integer overflow in binary number"); - overflowed = TRUE; - value_nv = (NV) value; - } - value_nv *= 2.0; - /* If an NV has not enough bits in its mantissa to - * represent a UV this summing of small low-order numbers - * is a waste of time (because the NV cannot preserve - * the low-order bits anyway): we could just remember when - * did we overflow and in the end just multiply value_nv by the - * right amount. */ - value_nv += (NV)(bit - '0'); - continue; - } - if (bit == '_' && len && allow_underscores && (bit = s[1]) - && (bit == '0' || bit == '1')) - { - --len; - ++s; - goto redo; - } - if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) - warn("Illegal binary digit '%c' ignored", *s); - break; - } - - if ( ( overflowed && value_nv > 4294967295.0) -#if UVSIZE > 4 - || (!overflowed && value > 0xffffffff ) -#endif - ) { - warn("Binary number > 0b11111111111111111111111111111111 non-portable"); - } - *len_p = s - start; - if (!overflowed) { - *flags = 0; - return value; - } - *flags = PERL_SCAN_GREATER_THAN_UV_MAX; - if (result) - *result = value_nv; - return UV_MAX; -} -#endif -#endif - -#ifndef grok_hex -#if defined(NEED_grok_hex) -static UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -static -#else -extern UV DPPP_(my_grok_hex)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -#endif - -#ifdef grok_hex -# undef grok_hex -#endif -#define grok_hex(a,b,c,d) DPPP_(my_grok_hex)(aTHX_ a,b,c,d) -#define Perl_grok_hex DPPP_(my_grok_hex) - -#if defined(NEED_grok_hex) || defined(NEED_grok_hex_GLOBAL) -UV -DPPP_(my_grok_hex)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) -{ - const char *s = start; - STRLEN len = *len_p; - UV value = 0; - NV value_nv = 0; - - const UV max_div_16 = UV_MAX / 16; - bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; - bool overflowed = FALSE; - const char *xdigit; - - if (!(*flags & PERL_SCAN_DISALLOW_PREFIX)) { - /* strip off leading x or 0x. - for compatibility silently suffer "x" and "0x" as valid hex numbers. - */ - if (len >= 1) { - if (s[0] == 'x') { - s++; - len--; - } - else if (len >= 2 && s[0] == '0' && s[1] == 'x') { - s+=2; - len-=2; - } - } - } - - for (; len-- && *s; s++) { - xdigit = strchr((char *) PL_hexdigit, *s); - if (xdigit) { - /* Write it in this wonky order with a goto to attempt to get the - compiler to make the common case integer-only loop pretty tight. - With gcc seems to be much straighter code than old scan_hex. */ - redo: - if (!overflowed) { - if (value <= max_div_16) { - value = (value << 4) | ((xdigit - PL_hexdigit) & 15); - continue; - } - warn("Integer overflow in hexadecimal number"); - overflowed = TRUE; - value_nv = (NV) value; - } - value_nv *= 16.0; - /* If an NV has not enough bits in its mantissa to - * represent a UV this summing of small low-order numbers - * is a waste of time (because the NV cannot preserve - * the low-order bits anyway): we could just remember when - * did we overflow and in the end just multiply value_nv by the - * right amount of 16-tuples. */ - value_nv += (NV)((xdigit - PL_hexdigit) & 15); - continue; - } - if (*s == '_' && len && allow_underscores && s[1] - && (xdigit = strchr((char *) PL_hexdigit, s[1]))) - { - --len; - ++s; - goto redo; - } - if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) - warn("Illegal hexadecimal digit '%c' ignored", *s); - break; - } - - if ( ( overflowed && value_nv > 4294967295.0) -#if UVSIZE > 4 - || (!overflowed && value > 0xffffffff ) -#endif - ) { - warn("Hexadecimal number > 0xffffffff non-portable"); - } - *len_p = s - start; - if (!overflowed) { - *flags = 0; - return value; - } - *flags = PERL_SCAN_GREATER_THAN_UV_MAX; - if (result) - *result = value_nv; - return UV_MAX; -} -#endif -#endif - -#ifndef grok_oct -#if defined(NEED_grok_oct) -static UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -static -#else -extern UV DPPP_(my_grok_oct)(pTHX_ const char * start, STRLEN * len_p, I32 * flags, NV * result); -#endif - -#ifdef grok_oct -# undef grok_oct -#endif -#define grok_oct(a,b,c,d) DPPP_(my_grok_oct)(aTHX_ a,b,c,d) -#define Perl_grok_oct DPPP_(my_grok_oct) - -#if defined(NEED_grok_oct) || defined(NEED_grok_oct_GLOBAL) -UV -DPPP_(my_grok_oct)(pTHX_ const char *start, STRLEN *len_p, I32 *flags, NV *result) -{ - const char *s = start; - STRLEN len = *len_p; - UV value = 0; - NV value_nv = 0; - - const UV max_div_8 = UV_MAX / 8; - bool allow_underscores = *flags & PERL_SCAN_ALLOW_UNDERSCORES; - bool overflowed = FALSE; - - for (; len-- && *s; s++) { - /* gcc 2.95 optimiser not smart enough to figure that this subtraction - out front allows slicker code. */ - int digit = *s - '0'; - if (digit >= 0 && digit <= 7) { - /* Write it in this wonky order with a goto to attempt to get the - compiler to make the common case integer-only loop pretty tight. - */ - redo: - if (!overflowed) { - if (value <= max_div_8) { - value = (value << 3) | digit; - continue; - } - /* Bah. We're just overflowed. */ - warn("Integer overflow in octal number"); - overflowed = TRUE; - value_nv = (NV) value; - } - value_nv *= 8.0; - /* If an NV has not enough bits in its mantissa to - * represent a UV this summing of small low-order numbers - * is a waste of time (because the NV cannot preserve - * the low-order bits anyway): we could just remember when - * did we overflow and in the end just multiply value_nv by the - * right amount of 8-tuples. */ - value_nv += (NV)digit; - continue; - } - if (digit == ('_' - '0') && len && allow_underscores - && (digit = s[1] - '0') && (digit >= 0 && digit <= 7)) - { - --len; - ++s; - goto redo; - } - /* Allow \octal to work the DWIM way (that is, stop scanning - * as soon as non-octal characters are seen, complain only iff - * someone seems to want to use the digits eight and nine). */ - if (digit == 8 || digit == 9) { - if (!(*flags & PERL_SCAN_SILENT_ILLDIGIT)) - warn("Illegal octal digit '%c' ignored", *s); - } - break; - } - - if ( ( overflowed && value_nv > 4294967295.0) -#if UVSIZE > 4 - || (!overflowed && value > 0xffffffff ) -#endif - ) { - warn("Octal number > 037777777777 non-portable"); - } - *len_p = s - start; - if (!overflowed) { - *flags = 0; - return value; - } - *flags = PERL_SCAN_GREATER_THAN_UV_MAX; - if (result) - *result = value_nv; - return UV_MAX; -} -#endif -#endif - -#if !defined(my_snprintf) -#if defined(NEED_my_snprintf) -static int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); -static -#else -extern int DPPP_(my_my_snprintf)(char * buffer, const Size_t len, const char * format, ...); -#endif - -#define my_snprintf DPPP_(my_my_snprintf) -#define Perl_my_snprintf DPPP_(my_my_snprintf) - -#if defined(NEED_my_snprintf) || defined(NEED_my_snprintf_GLOBAL) - -int -DPPP_(my_my_snprintf)(char *buffer, const Size_t len, const char *format, ...) -{ - dTHX; - int retval; - va_list ap; - va_start(ap, format); -#ifdef HAS_VSNPRINTF - retval = vsnprintf(buffer, len, format, ap); -#else - retval = vsprintf(buffer, format, ap); -#endif - va_end(ap); - if (retval < 0 || (len > 0 && (Size_t)retval >= len)) - Perl_croak(aTHX_ "panic: my_snprintf buffer overflow"); - return retval; -} - -#endif -#endif - -#if !defined(my_sprintf) -#if defined(NEED_my_sprintf) -static int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); -static -#else -extern int DPPP_(my_my_sprintf)(char * buffer, const char * pat, ...); -#endif - -#define my_sprintf DPPP_(my_my_sprintf) -#define Perl_my_sprintf DPPP_(my_my_sprintf) - -#if defined(NEED_my_sprintf) || defined(NEED_my_sprintf_GLOBAL) - -int -DPPP_(my_my_sprintf)(char *buffer, const char* pat, ...) -{ - va_list args; - va_start(args, pat); - vsprintf(buffer, pat, args); - va_end(args); - return strlen(buffer); -} - -#endif -#endif - -#ifdef NO_XSLOCKS -# ifdef dJMPENV -# define dXCPT dJMPENV; int rEtV = 0 -# define XCPT_TRY_START JMPENV_PUSH(rEtV); if (rEtV == 0) -# define XCPT_TRY_END JMPENV_POP; -# define XCPT_CATCH if (rEtV != 0) -# define XCPT_RETHROW JMPENV_JUMP(rEtV) -# else -# define dXCPT Sigjmp_buf oldTOP; int rEtV = 0 -# define XCPT_TRY_START Copy(top_env, oldTOP, 1, Sigjmp_buf); rEtV = Sigsetjmp(top_env, 1); if (rEtV == 0) -# define XCPT_TRY_END Copy(oldTOP, top_env, 1, Sigjmp_buf); -# define XCPT_CATCH if (rEtV != 0) -# define XCPT_RETHROW Siglongjmp(top_env, rEtV) -# endif -#endif - -#if !defined(my_strlcat) -#if defined(NEED_my_strlcat) -static Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); -static -#else -extern Size_t DPPP_(my_my_strlcat)(char * dst, const char * src, Size_t size); -#endif - -#define my_strlcat DPPP_(my_my_strlcat) -#define Perl_my_strlcat DPPP_(my_my_strlcat) - -#if defined(NEED_my_strlcat) || defined(NEED_my_strlcat_GLOBAL) - -Size_t -DPPP_(my_my_strlcat)(char *dst, const char *src, Size_t size) -{ - Size_t used, length, copy; - - used = strlen(dst); - length = strlen(src); - if (size > 0 && used < size - 1) { - copy = (length >= size - used) ? size - used - 1 : length; - memcpy(dst + used, src, copy); - dst[used + copy] = '\0'; - } - return used + length; -} -#endif -#endif - -#if !defined(my_strlcpy) -#if defined(NEED_my_strlcpy) -static Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); -static -#else -extern Size_t DPPP_(my_my_strlcpy)(char * dst, const char * src, Size_t size); -#endif - -#define my_strlcpy DPPP_(my_my_strlcpy) -#define Perl_my_strlcpy DPPP_(my_my_strlcpy) - -#if defined(NEED_my_strlcpy) || defined(NEED_my_strlcpy_GLOBAL) - -Size_t -DPPP_(my_my_strlcpy)(char *dst, const char *src, Size_t size) -{ - Size_t length, copy; - - length = strlen(src); - if (size > 0) { - copy = (length >= size) ? size - 1 : length; - memcpy(dst, src, copy); - dst[copy] = '\0'; - } - return length; -} - -#endif -#endif -#ifndef PERL_PV_ESCAPE_QUOTE -# define PERL_PV_ESCAPE_QUOTE 0x0001 -#endif - -#ifndef PERL_PV_PRETTY_QUOTE -# define PERL_PV_PRETTY_QUOTE PERL_PV_ESCAPE_QUOTE -#endif - -#ifndef PERL_PV_PRETTY_ELLIPSES -# define PERL_PV_PRETTY_ELLIPSES 0x0002 -#endif - -#ifndef PERL_PV_PRETTY_LTGT -# define PERL_PV_PRETTY_LTGT 0x0004 -#endif - -#ifndef PERL_PV_ESCAPE_FIRSTCHAR -# define PERL_PV_ESCAPE_FIRSTCHAR 0x0008 -#endif - -#ifndef PERL_PV_ESCAPE_UNI -# define PERL_PV_ESCAPE_UNI 0x0100 -#endif - -#ifndef PERL_PV_ESCAPE_UNI_DETECT -# define PERL_PV_ESCAPE_UNI_DETECT 0x0200 -#endif - -#ifndef PERL_PV_ESCAPE_ALL -# define PERL_PV_ESCAPE_ALL 0x1000 -#endif - -#ifndef PERL_PV_ESCAPE_NOBACKSLASH -# define PERL_PV_ESCAPE_NOBACKSLASH 0x2000 -#endif - -#ifndef PERL_PV_ESCAPE_NOCLEAR -# define PERL_PV_ESCAPE_NOCLEAR 0x4000 -#endif - -#ifndef PERL_PV_ESCAPE_RE -# define PERL_PV_ESCAPE_RE 0x8000 -#endif - -#ifndef PERL_PV_PRETTY_NOCLEAR -# define PERL_PV_PRETTY_NOCLEAR PERL_PV_ESCAPE_NOCLEAR -#endif -#ifndef PERL_PV_PRETTY_DUMP -# define PERL_PV_PRETTY_DUMP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_QUOTE -#endif - -#ifndef PERL_PV_PRETTY_REGPROP -# define PERL_PV_PRETTY_REGPROP PERL_PV_PRETTY_ELLIPSES|PERL_PV_PRETTY_LTGT|PERL_PV_ESCAPE_RE -#endif - -/* Hint: pv_escape - * Note that unicode functionality is only backported to - * those perl versions that support it. For older perl - * versions, the implementation will fall back to bytes. - */ - -#ifndef pv_escape -#if defined(NEED_pv_escape) -static char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); -static -#else -extern char * DPPP_(my_pv_escape)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, STRLEN * const escaped, const U32 flags); -#endif - -#ifdef pv_escape -# undef pv_escape -#endif -#define pv_escape(a,b,c,d,e,f) DPPP_(my_pv_escape)(aTHX_ a,b,c,d,e,f) -#define Perl_pv_escape DPPP_(my_pv_escape) - -#if defined(NEED_pv_escape) || defined(NEED_pv_escape_GLOBAL) - -char * -DPPP_(my_pv_escape)(pTHX_ SV *dsv, char const * const str, - const STRLEN count, const STRLEN max, - STRLEN * const escaped, const U32 flags) -{ - const char esc = flags & PERL_PV_ESCAPE_RE ? '%' : '\\'; - const char dq = flags & PERL_PV_ESCAPE_QUOTE ? '"' : esc; - char octbuf[32] = "%123456789ABCDF"; - STRLEN wrote = 0; - STRLEN chsize = 0; - STRLEN readsize = 1; -#if defined(is_utf8_string) && defined(utf8_to_uvchr) - bool isuni = flags & PERL_PV_ESCAPE_UNI ? 1 : 0; -#endif - const char *pv = str; - const char * const end = pv + count; - octbuf[0] = esc; - - if (!(flags & PERL_PV_ESCAPE_NOCLEAR)) - sv_setpvs(dsv, ""); - -#if defined(is_utf8_string) && defined(utf8_to_uvchr) - if ((flags & PERL_PV_ESCAPE_UNI_DETECT) && is_utf8_string((U8*)pv, count)) - isuni = 1; -#endif - - for (; pv < end && (!max || wrote < max) ; pv += readsize) { - const UV u = -#if defined(is_utf8_string) && defined(utf8_to_uvchr) - isuni ? utf8_to_uvchr((U8*)pv, &readsize) : -#endif - (U8)*pv; - const U8 c = (U8)u & 0xFF; - - if (u > 255 || (flags & PERL_PV_ESCAPE_ALL)) { - if (flags & PERL_PV_ESCAPE_FIRSTCHAR) - chsize = my_snprintf(octbuf, sizeof octbuf, - "%"UVxf, u); - else - chsize = my_snprintf(octbuf, sizeof octbuf, - "%cx{%"UVxf"}", esc, u); - } else if (flags & PERL_PV_ESCAPE_NOBACKSLASH) { - chsize = 1; - } else { - if (c == dq || c == esc || !isPRINT(c)) { - chsize = 2; - switch (c) { - case '\\' : /* fallthrough */ - case '%' : if (c == esc) - octbuf[1] = esc; - else - chsize = 1; - break; - case '\v' : octbuf[1] = 'v'; break; - case '\t' : octbuf[1] = 't'; break; - case '\r' : octbuf[1] = 'r'; break; - case '\n' : octbuf[1] = 'n'; break; - case '\f' : octbuf[1] = 'f'; break; - case '"' : if (dq == '"') - octbuf[1] = '"'; - else - chsize = 1; - break; - default: chsize = my_snprintf(octbuf, sizeof octbuf, - pv < end && isDIGIT((U8)*(pv+readsize)) - ? "%c%03o" : "%c%o", esc, c); - } - } else { - chsize = 1; - } - } - if (max && wrote + chsize > max) { - break; - } else if (chsize > 1) { - sv_catpvn(dsv, octbuf, chsize); - wrote += chsize; - } else { - char tmp[2]; - my_snprintf(tmp, sizeof tmp, "%c", c); - sv_catpvn(dsv, tmp, 1); - wrote++; - } - if (flags & PERL_PV_ESCAPE_FIRSTCHAR) - break; - } - if (escaped != NULL) - *escaped= pv - str; - return SvPVX(dsv); -} - -#endif -#endif - -#ifndef pv_pretty -#if defined(NEED_pv_pretty) -static char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); -static -#else -extern char * DPPP_(my_pv_pretty)(pTHX_ SV * dsv, char const * const str, const STRLEN count, const STRLEN max, char const * const start_color, char const * const end_color, const U32 flags); -#endif - -#ifdef pv_pretty -# undef pv_pretty -#endif -#define pv_pretty(a,b,c,d,e,f,g) DPPP_(my_pv_pretty)(aTHX_ a,b,c,d,e,f,g) -#define Perl_pv_pretty DPPP_(my_pv_pretty) - -#if defined(NEED_pv_pretty) || defined(NEED_pv_pretty_GLOBAL) - -char * -DPPP_(my_pv_pretty)(pTHX_ SV *dsv, char const * const str, const STRLEN count, - const STRLEN max, char const * const start_color, char const * const end_color, - const U32 flags) -{ - const U8 dq = (flags & PERL_PV_PRETTY_QUOTE) ? '"' : '%'; - STRLEN escaped; - - if (!(flags & PERL_PV_PRETTY_NOCLEAR)) - sv_setpvs(dsv, ""); - - if (dq == '"') - sv_catpvs(dsv, "\""); - else if (flags & PERL_PV_PRETTY_LTGT) - sv_catpvs(dsv, "<"); - - if (start_color != NULL) - sv_catpv(dsv, D_PPP_CONSTPV_ARG(start_color)); - - pv_escape(dsv, str, count, max, &escaped, flags | PERL_PV_ESCAPE_NOCLEAR); - - if (end_color != NULL) - sv_catpv(dsv, D_PPP_CONSTPV_ARG(end_color)); - - if (dq == '"') - sv_catpvs(dsv, "\""); - else if (flags & PERL_PV_PRETTY_LTGT) - sv_catpvs(dsv, ">"); - - if ((flags & PERL_PV_PRETTY_ELLIPSES) && escaped < count) - sv_catpvs(dsv, "..."); - - return SvPVX(dsv); -} - -#endif -#endif - -#ifndef pv_display -#if defined(NEED_pv_display) -static char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); -static -#else -extern char * DPPP_(my_pv_display)(pTHX_ SV * dsv, const char * pv, STRLEN cur, STRLEN len, STRLEN pvlim); -#endif - -#ifdef pv_display -# undef pv_display -#endif -#define pv_display(a,b,c,d,e) DPPP_(my_pv_display)(aTHX_ a,b,c,d,e) -#define Perl_pv_display DPPP_(my_pv_display) - -#if defined(NEED_pv_display) || defined(NEED_pv_display_GLOBAL) - -char * -DPPP_(my_pv_display)(pTHX_ SV *dsv, const char *pv, STRLEN cur, STRLEN len, STRLEN pvlim) -{ - pv_pretty(dsv, pv, cur, pvlim, NULL, NULL, PERL_PV_PRETTY_DUMP); - if (len > cur && pv[cur] == '\0') - sv_catpvs(dsv, "\\0"); - return SvPVX(dsv); -} - -#endif -#endif - -#endif /* _P_P_PORTABILITY_H_ */ - -/* End of File ppport.h */ From da9b75f5533dd83efb8ba3e1fb388daf374ab18d Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 30 Jul 2009 16:52:13 +0900 Subject: [PATCH 0269/1648] Checking in changes prior to tagging of version 0.08. Changelog diff is: diff --git a/perl/Changes b/perl/Changes index 4e7f1d7..c6e370c 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.08 + + - fixed PVNV issue... + 0.07 - do not use switch (SvTYPE(val)). --- perl/Changes | 4 ++++ perl/MANIFEST.SKIP | 1 + perl/lib/Data/MessagePack.pm | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 4e7f1d7..c6e370c 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.08 + + - fixed PVNV issue... + 0.07 - do not use switch (SvTYPE(val)). diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index 1b268ef..f524216 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -20,3 +20,4 @@ ^\.git/ \.sw[pon]$ ^\.gitignore$ +ppport.h diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 38431eb..2ab33f1 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.07'; +our $VERSION = '0.08'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From aaef612a057908bc4227154ba15aee2bb66f642f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 6 Aug 2009 13:20:43 +0900 Subject: [PATCH 0270/1648] Makefile.am: don't package perl and ruby files --- Makefile.am | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/Makefile.am b/Makefile.am index 9e36092..3144972 100644 --- a/Makefile.am +++ b/Makefile.am @@ -11,38 +11,3 @@ nobase_include_HEADERS = \ msgpack/unpack_define.h \ msgpack/unpack_template.h -EXTRA_DIST = \ - ruby/bench.rb \ - ruby/extconf.rb \ - ruby/gem/README \ - ruby/gem/Rakefile \ - ruby/gem/test/test_helper.rb \ - ruby/gengem.sh \ - ruby/msgpack.gemspec \ - ruby/pack.c \ - ruby/pack.h \ - ruby/rbinit.c \ - ruby/test_case.rb \ - ruby/test_format.rb \ - ruby/test_pack.rb \ - ruby/unpack.c \ - ruby/unpack.h \ - perl/Makefile.PL \ - perl/MessagePack.c \ - perl/benchmark/deserialize.pl \ - perl/benchmark/serialize.pl \ - perl/lib/Data/MessagePack.pm \ - perl/lib/Data/MessagePack \ - perl/lib/Data/MessagePack/Unpacker.pod \ - perl/pack.c \ - perl/ppport.h \ - perl/t/00_compile.t \ - perl/t/01_pack.t \ - perl/t/02_unpack.t \ - perl/t/03_stream_unpack.t \ - perl/t/04_invert.t \ - perl/t/Util.pm \ - perl/t/data.pl \ - perl/unpack.c \ - perl/xt/99_pod.t - From f7a9805f7b3b6413c307863f1f16b320abb665ec Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 6 Aug 2009 13:26:34 +0900 Subject: [PATCH 0271/1648] operator>> (object, std::vector): return reference; don't copy --- cpp/type/array.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/type/array.hpp b/cpp/type/array.hpp index 6027251..5b80dc1 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/array.hpp @@ -25,7 +25,7 @@ namespace msgpack { template -inline std::vector operator>> (object o, std::vector& v) +inline std::vector& operator>> (object o, std::vector& v) { if(o.type != type::ARRAY) { throw type_error(); } v.resize(o.via.array.size); From ec8932d6a11977136516fb2b7a3e3015cab0c3ac Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 6 Aug 2009 13:28:41 +0900 Subject: [PATCH 0272/1648] fix missing tuple<>::value_type --- cpp/type/tuple.hpp.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 586d84c..0105feb 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -103,6 +103,7 @@ tuple, A<%=j%><%}%>> make_tuple(typename tuple_type::tr template <> struct tuple<> { + typedef tuple<> value_type; }; <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> From 3afa9f265ebb283f26cb516ceca52cc083916cf7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 6 Aug 2009 13:51:49 +0900 Subject: [PATCH 0273/1648] cpp: add msgpack::type::define --- configure.in | 2 +- cpp/Makefile.am | 7 ++- cpp/type.hpp | 1 + cpp/type/define.hpp.erb | 102 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 cpp/type/define.hpp.erb diff --git a/configure.in b/configure.in index 45a04a9..8646d1e 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.3) +AM_INIT_AUTOMAKE(msgpack, 0.3.4) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index b9126f8..5efac3f 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -19,7 +19,8 @@ nobase_include_HEADERS = \ msgpack/type/map.hpp \ msgpack/type/nil.hpp \ msgpack/type/raw.hpp \ - msgpack/type/tuple.hpp + msgpack/type/tuple.hpp \ + msgpack/type/define.hpp # FIXME object.lo: msgpack/type/tuple.hpp msgpack/zone.hpp @@ -28,6 +29,10 @@ msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp mv $@.tmp $@ +msgpack/type/define.hpp: msgpack/type/define.hpp.erb + $(ERB) $< > $@.tmp + mv $@.tmp $@ + msgpack/zone.hpp: msgpack/zone.hpp.erb $(ERB) $< > $@.tmp mv $@.tmp $@ diff --git a/cpp/type.hpp b/cpp/type.hpp index 1dfd414..8ffe3bf 100644 --- a/cpp/type.hpp +++ b/cpp/type.hpp @@ -6,4 +6,5 @@ #include "msgpack/type/nil.hpp" #include "msgpack/type/raw.hpp" #include "msgpack/type/tuple.hpp" +#include "msgpack/type/define.hpp" diff --git a/cpp/type/define.hpp.erb b/cpp/type/define.hpp.erb new file mode 100644 index 0000000..9be4f77 --- /dev/null +++ b/cpp/type/define.hpp.erb @@ -0,0 +1,102 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_DEFINE_HPP__ +#define MSGPACK_TYPE_DEFINE_HPP__ + +#define MSGPACK_DEFINE(...) \ + template \ + void msgpack_pack(Packer& pk) const \ + { \ + msgpack::type::make_define(__VA_ARGS__).msgpack_pack(pk); \ + } \ + \ + void msgpack_unpack(msgpack::object o) \ + { \ + msgpack::type::make_define(__VA_ARGS__).msgpack_unpack(o); \ + } + +namespace msgpack { +namespace type { + + +<% GENERATION_LIMIT = 15 %> +template , typename A<%=i%> = void<%}%>> +struct define; + + +template <> +struct define<> { + typedef define<> value_type; + typedef tuple<> tuple_type; + template + void msgpack_pack(Packer& pk) const + { + pk.pack_array(1); + } + void msgpack_unpack(msgpack::object o) + { + if(o.type != type::ARRAY) { throw type_error(); } + } +}; + +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +struct define, A<%=j%><%}%>> { + typedef define, A<%=j%><%}%>> value_type; + typedef tuple, A<%=j%><%}%>> tuple_type; + define(A0& _a0<%1.upto(i) {|j|%>, A<%=j%>& _a<%=j%><%}%>) : + a0(_a0)<%1.upto(i) {|j|%>, a<%=j%>(_a<%=j%>)<%}%> {} + template + void msgpack_pack(Packer& pk) const + { + pk.pack_array(<%=i+1%>); + <%0.upto(i) {|j|%> + pk.pack(a<%=j%>);<%}%> + } + void msgpack_unpack(msgpack::object o) + { + if(o.type != type::ARRAY) { throw type_error(); } + const size_t size = o.via.array.size; + <%0.upto(i) {|j|%> + if(size <= <%=j%>) { return; } o.via.array.ptr[<%=j%>].convert(&a<%=j%>);<%}%> + } + <%0.upto(i) {|j|%> + A<%=j%>& a<%=j%>;<%}%> +}; +<%}%> + +define<> make_define() +{ + return define<>(); +} + +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +define, A<%=j%><%}%>> make_define(A0& a0<%1.upto(i) {|j|%>, A<%=j%>& a<%=j%><%}%>) +{ + return define, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>); +} +<%}%> + + +} // namespace type +} // namespace msgpack + + +#endif /* msgpack/type/define.hpp */ + From 7fbe845434937ac6dcf5b89e0d80661443eafd90 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 6 Aug 2009 13:59:48 +0900 Subject: [PATCH 0274/1648] cpp: msgpack::define is obsolete --- cpp/object.hpp | 1 + cpp/type/define.hpp.erb | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/object.hpp b/cpp/object.hpp index 07917c2..9ee575f 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -129,6 +129,7 @@ private: }; +// obsolete template class define : public Type { public: diff --git a/cpp/type/define.hpp.erb b/cpp/type/define.hpp.erb index 9be4f77..c13aa16 100644 --- a/cpp/type/define.hpp.erb +++ b/cpp/type/define.hpp.erb @@ -24,7 +24,6 @@ { \ msgpack::type::make_define(__VA_ARGS__).msgpack_pack(pk); \ } \ - \ void msgpack_unpack(msgpack::object o) \ { \ msgpack::type::make_define(__VA_ARGS__).msgpack_unpack(o); \ From 1375732c805264cf43baa112a5a45e4c0c43b8e6 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 7 Aug 2009 11:15:33 +0900 Subject: [PATCH 0275/1648] add --- example/custom.cc | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 example/custom.cc diff --git a/example/custom.cc b/example/custom.cc new file mode 100644 index 0000000..835ebed --- /dev/null +++ b/example/custom.cc @@ -0,0 +1,58 @@ +#include +#include +#include + +class old_class { +public: + old_class() : value("default") { } + + std::string value; + + MSGPACK_DEFINE(value); +}; + +class new_class { +public: + new_class() : value("default"), flag(-1) { } + + std::string value; + int flag; + + MSGPACK_DEFINE(value, flag); +}; + +int main(void) +{ + { + old_class oc; + new_class nc; + + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, oc); + + msgpack::zone zone; + msgpack::object obj; + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &zone, &obj); + + obj.convert(&nc); + + std::cout << obj << " value=" << nc.value << " flag=" << nc.flag << std::endl; + } + + { + new_class nc; + old_class oc; + + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, nc); + + msgpack::zone zone; + msgpack::object obj; + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &zone, &obj); + + obj.convert(&oc); + + std::cout << obj << " value=" << oc.value << std::endl; + } +} + From b4cb5e23c0a235d61d4a63acf4624f96f20f620b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 7 Aug 2009 11:28:23 +0900 Subject: [PATCH 0276/1648] fix Makefile.am --- cpp/Makefile.am | 2 ++ example/stream.cc | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 5efac3f..f630832 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -39,10 +39,12 @@ msgpack/zone.hpp: msgpack/zone.hpp.erb MOSTLYCLEANFILES = \ msgpack/type/tuple.hpp \ + msgpack/type/define.hpp \ msgpack/zone.hpp EXTRA_DIST = \ msgpack/type/tuple.hpp.erb \ + msgpack/type/define.hpp.erb \ msgpack/zone.hpp.erb libmsgpack_la_LIBADD = -L../c -lmsgpackc diff --git a/example/stream.cc b/example/stream.cc index 1e2e733..2241935 100644 --- a/example/stream.cc +++ b/example/stream.cc @@ -22,14 +22,14 @@ public: ssize_t count = read(m_sock, m_pac.buffer(), m_pac.buffer_capacity()); - if(count < 0) { + if(count <= 0) { + if(count == 0) { + throw std::runtime_error("connection closed"); + } if(errno == EAGAIN || errno == EINTR) { return; - } else { - throw std::runtime_error(strerror(errno)); } - } else if(count == 0) { - throw std::runtime_error("connection closed"); + throw std::runtime_error(strerror(errno)); } m_pac.buffer_consumed(count); From be32e3b1fb3fc45747c8df13f9a2eed5626e7c47 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 7 Aug 2009 13:31:42 +0900 Subject: [PATCH 0277/1648] cpp: add missing msgpack::type::make_tuple() --- cpp/type/tuple.hpp.erb | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 0105feb..d8ddbcd 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -64,7 +64,6 @@ struct tuple_type { typedef const T& transparent_reference; }; - <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> template , typename A<%=k%><%}%>> @@ -78,7 +77,6 @@ private: <%}%> <%}%> - <%0.upto(GENERATION_LIMIT) {|i|%> <%0.upto(i) {|j|%> template , typename A<%=k%><%}%>> @@ -91,16 +89,6 @@ private: <%}%> <%}%> - -<%0.upto(GENERATION_LIMIT) {|i|%> -template , typename A<%=j%><%}%>> -tuple, A<%=j%><%}%>> make_tuple(typename tuple_type::transparent_reference a0<%1.upto(i) {|j|%>, typename tuple_type>::transparent_reference a<%=j%><%}%>) -{ - return tuple, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>); -} -<%}%> - - template <> struct tuple<> { typedef tuple<> value_type; @@ -122,6 +110,18 @@ struct tuple, A<%=j%><%}%>> { }; <%}%> +inline tuple<> make_tuple() +{ + return tuple<>(); +} +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +tuple, A<%=j%><%}%>> make_tuple(typename tuple_type::transparent_reference a0<%1.upto(i) {|j|%>, typename tuple_type>::transparent_reference a<%=j%><%}%>) +{ + return tuple, A<%=j%><%}%>>(a0<%1.upto(i) {|j|%>, a<%=j%><%}%>); +} +<%}%> + } // namespace type @@ -144,7 +144,6 @@ type::tuple, A<%=j%><%}%>>& operator>> ( } <%}%> - template const packer& operator<< ( packer& o, @@ -164,7 +163,6 @@ const packer& operator<< ( } <%}%> - } // namespace msgpack #endif /* msgpack/type/tuple.hpp */ From 95a6316cc7df5617090a948f347bdf2c33d45b79 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 7 Aug 2009 13:32:07 +0900 Subject: [PATCH 0278/1648] cpp: fix msgpack::type::make_define() --- cpp/type/define.hpp.erb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cpp/type/define.hpp.erb b/cpp/type/define.hpp.erb index c13aa16..d9d3fb1 100644 --- a/cpp/type/define.hpp.erb +++ b/cpp/type/define.hpp.erb @@ -52,7 +52,6 @@ struct define<> { if(o.type != type::ARRAY) { throw type_error(); } } }; - <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> struct define, A<%=j%><%}%>> { @@ -79,11 +78,10 @@ struct define, A<%=j%><%}%>> { }; <%}%> -define<> make_define() +inline define<> make_define() { return define<>(); } - <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> define, A<%=j%><%}%>> make_define(A0& a0<%1.upto(i) {|j|%>, A<%=j%>& a<%=j%><%}%>) @@ -92,7 +90,6 @@ define, A<%=j%><%}%>> make_define(A0& a0<%1.upto(i) {|j|%>, } <%}%> - } // namespace type } // namespace msgpack From a5705183d6dc2c64ca01716bb5b06d574e4c2a5f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 7 Aug 2009 13:32:32 +0900 Subject: [PATCH 0279/1648] 0.3.5 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 8646d1e..202ca0d 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.4) +AM_INIT_AUTOMAKE(msgpack, 0.3.5) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From 0491768fb2a9b2ba0c52de838253981ed90111cd Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 10 Aug 2009 15:58:43 +0900 Subject: [PATCH 0280/1648] c++: fix Makefile.am --- cpp/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index f630832..b7bf069 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -23,7 +23,7 @@ nobase_include_HEADERS = \ msgpack/type/define.hpp # FIXME -object.lo: msgpack/type/tuple.hpp msgpack/zone.hpp +object.lo: msgpack/type/tuple.hpp msgpack/type/define.hpp msgpack/zone.hpp msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb $(ERB) $< > $@.tmp From 394331cd4ece745b9c5b5eb155560aaece8dc2bc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 10 Aug 2009 17:34:21 +0900 Subject: [PATCH 0281/1648] c++: add std::pair and std::set serializer --- cpp/Makefile.am | 2 ++ cpp/type.hpp | 2 ++ cpp/type/pair.hpp | 51 ++++++++++++++++++++++++++++++++++++++++++ cpp/type/set.hpp | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 cpp/type/pair.hpp create mode 100644 cpp/type/set.hpp diff --git a/cpp/Makefile.am b/cpp/Makefile.am index b7bf069..edbeb28 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -18,7 +18,9 @@ nobase_include_HEADERS = \ msgpack/type/integer.hpp \ msgpack/type/map.hpp \ msgpack/type/nil.hpp \ + msgpack/type/pair.hpp \ msgpack/type/raw.hpp \ + msgpack/type/set.hpp \ msgpack/type/tuple.hpp \ msgpack/type/define.hpp diff --git a/cpp/type.hpp b/cpp/type.hpp index 8ffe3bf..2a6aa62 100644 --- a/cpp/type.hpp +++ b/cpp/type.hpp @@ -4,7 +4,9 @@ #include "msgpack/type/integer.hpp" #include "msgpack/type/map.hpp" #include "msgpack/type/nil.hpp" +#include "msgpack/type/pair.hpp" #include "msgpack/type/raw.hpp" +#include "msgpack/type/set.hpp" #include "msgpack/type/tuple.hpp" #include "msgpack/type/define.hpp" diff --git a/cpp/type/pair.hpp b/cpp/type/pair.hpp new file mode 100644 index 0000000..316b9fe --- /dev/null +++ b/cpp/type/pair.hpp @@ -0,0 +1,51 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_PAIR_HPP__ +#define MSGPACK_TYPE_PAIR_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::pair& operator>> (object o, std::pair& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + if(o.via.array.size != 2) { throw type_error(); } + o.via.array.ptr[0].convert(&v.first); + o.via.array.ptr[1].convert(&v.second); + return v; +} + + +template +inline packer& operator<< (packer& o, const std::pair& v) +{ + o.pack_array(2); + o.pack(v.first); + o.pack(v.second); + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/pair.hpp */ + diff --git a/cpp/type/set.hpp b/cpp/type/set.hpp new file mode 100644 index 0000000..3f1920a --- /dev/null +++ b/cpp/type/set.hpp @@ -0,0 +1,56 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_SET_HPP__ +#define MSGPACK_TYPE_SET_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::set& operator>> (object o, std::set& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + object* p = o.via.array.ptr + o.via.array.size; + object* const pbegin = o.via.array.ptr; + while(p > pbegin) { + --p; + v.insert(p->as()); + } + return v; +} + + +template +inline packer& operator<< (packer& o, const std::set& v) +{ + o.pack_array(v.size()); + for(typename std::set::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/set.hpp */ + From 0627324da62922d332c75ec51525141329187beb Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 10 Aug 2009 18:26:01 +0900 Subject: [PATCH 0282/1648] c++: rebuild type/*.hpp --- cpp/Makefile.am | 8 +++-- cpp/type.hpp | 8 +++-- cpp/type/{boolean.hpp => bool.hpp} | 4 +-- cpp/type/deque.hpp | 56 ++++++++++++++++++++++++++++++ cpp/type/float.hpp | 2 -- cpp/type/{integer.hpp => int.hpp} | 6 ++-- cpp/type/list.hpp | 56 ++++++++++++++++++++++++++++++ cpp/type/pair.hpp | 1 - cpp/type/raw.hpp | 19 ---------- cpp/type/string.hpp | 47 +++++++++++++++++++++++++ cpp/type/{array.hpp => vector.hpp} | 7 ++-- 11 files changed, 177 insertions(+), 37 deletions(-) rename cpp/type/{boolean.hpp => bool.hpp} (94%) create mode 100644 cpp/type/deque.hpp rename cpp/type/{integer.hpp => int.hpp} (97%) create mode 100644 cpp/type/list.hpp create mode 100644 cpp/type/string.hpp rename cpp/type/{array.hpp => vector.hpp} (93%) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index edbeb28..45cc13c 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -12,15 +12,17 @@ nobase_include_HEADERS = \ msgpack/object.hpp \ msgpack/zone.hpp \ msgpack/type.hpp \ - msgpack/type/array.hpp \ - msgpack/type/boolean.hpp \ + msgpack/type/bool.hpp \ msgpack/type/float.hpp \ - msgpack/type/integer.hpp \ + msgpack/type/int.hpp \ + msgpack/type/list.hpp \ msgpack/type/map.hpp \ msgpack/type/nil.hpp \ msgpack/type/pair.hpp \ msgpack/type/raw.hpp \ msgpack/type/set.hpp \ + msgpack/type/string.hpp \ + msgpack/type/vector.hpp \ msgpack/type/tuple.hpp \ msgpack/type/define.hpp diff --git a/cpp/type.hpp b/cpp/type.hpp index 2a6aa62..2bd805d 100644 --- a/cpp/type.hpp +++ b/cpp/type.hpp @@ -1,12 +1,14 @@ -#include "msgpack/type/array.hpp" -#include "msgpack/type/boolean.hpp" +#include "msgpack/type/bool.hpp" #include "msgpack/type/float.hpp" -#include "msgpack/type/integer.hpp" +#include "msgpack/type/int.hpp" +#include "msgpack/type/list.hpp" #include "msgpack/type/map.hpp" #include "msgpack/type/nil.hpp" #include "msgpack/type/pair.hpp" #include "msgpack/type/raw.hpp" #include "msgpack/type/set.hpp" +#include "msgpack/type/string.hpp" +#include "msgpack/type/vector.hpp" #include "msgpack/type/tuple.hpp" #include "msgpack/type/define.hpp" diff --git a/cpp/type/boolean.hpp b/cpp/type/bool.hpp similarity index 94% rename from cpp/type/boolean.hpp rename to cpp/type/bool.hpp index 86bd697..f3ac6fa 100644 --- a/cpp/type/boolean.hpp +++ b/cpp/type/bool.hpp @@ -15,8 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. // -#ifndef MSGPACK_TYPE_BOOLEAN_HPP__ -#define MSGPACK_TYPE_BOOLEAN_HPP__ +#ifndef MSGPACK_TYPE_BOOL_HPP__ +#define MSGPACK_TYPE_BOOL_HPP__ #include "msgpack/object.hpp" #include diff --git a/cpp/type/deque.hpp b/cpp/type/deque.hpp new file mode 100644 index 0000000..d34d243 --- /dev/null +++ b/cpp/type/deque.hpp @@ -0,0 +1,56 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_DEQUE_HPP__ +#define MSGPACK_TYPE_DEQUE_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::deque& operator>> (object o, std::deque& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + v.resize(o.via.array.size); + object* p = o.via.array.ptr; + object* const pend = o.via.array.ptr + o.via.array.size; + typename std::deque::iterator it = v.begin(); + for(; p < pend; ++p, ++it) { + p->convert(&*it); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::deque& v) +{ + o.pack_array(v.size()); + for(typename std::deque::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/deque.hpp */ + diff --git a/cpp/type/float.hpp b/cpp/type/float.hpp index 108709d..390e340 100644 --- a/cpp/type/float.hpp +++ b/cpp/type/float.hpp @@ -34,7 +34,6 @@ inline float& operator>> (object o, float& v) return v; } - template inline packer& operator<< (packer& o, const float& v) { @@ -50,7 +49,6 @@ inline double& operator>> (object o, double& v) return v; } - template inline packer& operator<< (packer& o, const double& v) { diff --git a/cpp/type/integer.hpp b/cpp/type/int.hpp similarity index 97% rename from cpp/type/integer.hpp rename to cpp/type/int.hpp index ecb7b89..8fdc386 100644 --- a/cpp/type/integer.hpp +++ b/cpp/type/int.hpp @@ -15,8 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. // -#ifndef MSGPACK_TYPE_INTEGER_HPP__ -#define MSGPACK_TYPE_INTEGER_HPP__ +#ifndef MSGPACK_TYPE_INT_HPP__ +#define MSGPACK_TYPE_INT_HPP__ #include "msgpack/object.hpp" #include @@ -143,5 +143,5 @@ inline packer& operator<< (packer& o, const unsigned long long& } // namespace msgpack -#endif /* msgpack/type/integer.hpp */ +#endif /* msgpack/type/int.hpp */ diff --git a/cpp/type/list.hpp b/cpp/type/list.hpp new file mode 100644 index 0000000..6ecc02f --- /dev/null +++ b/cpp/type/list.hpp @@ -0,0 +1,56 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_LIST_HPP__ +#define MSGPACK_TYPE_LIST_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::list& operator>> (object o, std::list& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + v.resize(o.via.array.size); + object* p = o.via.array.ptr; + object* const pend = o.via.array.ptr + o.via.array.size; + typename std::list::iterator it = v.begin(); + for(; p < pend; ++p, ++it) { + p->convert(&*it); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::list& v) +{ + o.pack_array(v.size()); + for(typename std::list::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/list.hpp */ + diff --git a/cpp/type/pair.hpp b/cpp/type/pair.hpp index 316b9fe..ba72c1f 100644 --- a/cpp/type/pair.hpp +++ b/cpp/type/pair.hpp @@ -34,7 +34,6 @@ inline std::pair& operator>> (object o, std::pair& v) return v; } - template inline packer& operator<< (packer& o, const std::pair& v) { diff --git a/cpp/type/raw.hpp b/cpp/type/raw.hpp index b6ace3f..8c68aba 100644 --- a/cpp/type/raw.hpp +++ b/cpp/type/raw.hpp @@ -69,16 +69,6 @@ inline type::raw_ref& operator>> (object o, type::raw_ref& v) return v; } - -inline std::string& operator>> (object o, std::string& v) -{ - type::raw_ref r; - o >> r; - v.assign(r.ptr, r.size); - return v; -} - - template inline packer& operator<< (packer& o, const type::raw_ref& v) { @@ -88,15 +78,6 @@ inline packer& operator<< (packer& o, const type::raw_ref& v) } -template -inline packer& operator<< (packer& o, const std::string& v) -{ - o.pack_raw(v.size()); - o.pack_raw_body(v.data(), v.size()); - return o; -} - - } // namespace msgpack #endif /* msgpack/type/raw.hpp */ diff --git a/cpp/type/string.hpp b/cpp/type/string.hpp new file mode 100644 index 0000000..2a23058 --- /dev/null +++ b/cpp/type/string.hpp @@ -0,0 +1,47 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_STRING_HPP__ +#define MSGPACK_TYPE_STRING_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +inline std::string& operator>> (object o, std::string& v) +{ + type::raw_ref r; + o >> r; + v.assign(r.ptr, r.size); + return v; +} + +template +inline packer& operator<< (packer& o, const std::string& v) +{ + o.pack_raw(v.size()); + o.pack_raw_body(v.data(), v.size()); + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/string.hpp */ + diff --git a/cpp/type/array.hpp b/cpp/type/vector.hpp similarity index 93% rename from cpp/type/array.hpp rename to cpp/type/vector.hpp index 5b80dc1..754cdc0 100644 --- a/cpp/type/array.hpp +++ b/cpp/type/vector.hpp @@ -15,8 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. // -#ifndef MSGPACK_TYPE_ARRAY_HPP__ -#define MSGPACK_TYPE_ARRAY_HPP__ +#ifndef MSGPACK_TYPE_VECTOR_HPP__ +#define MSGPACK_TYPE_VECTOR_HPP__ #include "msgpack/object.hpp" #include @@ -38,7 +38,6 @@ inline std::vector& operator>> (object o, std::vector& v) return v; } - template inline packer& operator<< (packer& o, const std::vector& v) { @@ -53,5 +52,5 @@ inline packer& operator<< (packer& o, const std::vector& v) } // namespace msgpack -#endif /* msgpack/type/array.hpp */ +#endif /* msgpack/type/vector.hpp */ From a62a5d6c69837e474f9cc7260017de3fe61a92eb Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 10 Aug 2009 18:33:35 +0900 Subject: [PATCH 0283/1648] c++: fix type.hpp --- cpp/Makefile.am | 1 + cpp/type.hpp | 1 + 2 files changed, 2 insertions(+) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 45cc13c..f9d1fec 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -16,6 +16,7 @@ nobase_include_HEADERS = \ msgpack/type/float.hpp \ msgpack/type/int.hpp \ msgpack/type/list.hpp \ + msgpack/type/deque.hpp \ msgpack/type/map.hpp \ msgpack/type/nil.hpp \ msgpack/type/pair.hpp \ diff --git a/cpp/type.hpp b/cpp/type.hpp index 2bd805d..fafa674 100644 --- a/cpp/type.hpp +++ b/cpp/type.hpp @@ -2,6 +2,7 @@ #include "msgpack/type/float.hpp" #include "msgpack/type/int.hpp" #include "msgpack/type/list.hpp" +#include "msgpack/type/deque.hpp" #include "msgpack/type/map.hpp" #include "msgpack/type/nil.hpp" #include "msgpack/type/pair.hpp" From 92952f656691f5907e00c0a5ab45e3423d5429cb Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 10 Aug 2009 22:16:44 +0900 Subject: [PATCH 0284/1648] c++: fix object >> std::string --- configure.in | 2 +- cpp/type/bool.hpp | 1 - cpp/type/set.hpp | 1 - cpp/type/string.hpp | 5 ++--- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/configure.in b/configure.in index 202ca0d..7cdb99d 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.5) +AM_INIT_AUTOMAKE(msgpack, 0.3.6) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/type/bool.hpp b/cpp/type/bool.hpp index f3ac6fa..b945d85 100644 --- a/cpp/type/bool.hpp +++ b/cpp/type/bool.hpp @@ -31,7 +31,6 @@ inline bool& operator>> (object o, bool& v) return v; } - template inline packer& operator<< (packer& o, const bool& v) { diff --git a/cpp/type/set.hpp b/cpp/type/set.hpp index 3f1920a..11db2b3 100644 --- a/cpp/type/set.hpp +++ b/cpp/type/set.hpp @@ -37,7 +37,6 @@ inline std::set& operator>> (object o, std::set& v) return v; } - template inline packer& operator<< (packer& o, const std::set& v) { diff --git a/cpp/type/string.hpp b/cpp/type/string.hpp index 2a23058..a085d53 100644 --- a/cpp/type/string.hpp +++ b/cpp/type/string.hpp @@ -26,9 +26,8 @@ namespace msgpack { inline std::string& operator>> (object o, std::string& v) { - type::raw_ref r; - o >> r; - v.assign(r.ptr, r.size); + if(o.type != type::RAW) { throw type_error(); } + v.assign(o.via.raw.ptr, o.via.raw.size); return v; } From b2381d0513e3ec9a874643b7faf7afeb4490a1e7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 19 Aug 2009 15:10:10 +0900 Subject: [PATCH 0285/1648] cpp: fix overflow check for minimum number of signed integer type --- cpp/type/int.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/type/int.hpp b/cpp/type/int.hpp index 8fdc386..03500f7 100644 --- a/cpp/type/int.hpp +++ b/cpp/type/int.hpp @@ -37,7 +37,7 @@ namespace detail { { throw type_error(); } return o.via.u64; } else if(o.type == type::NEGATIVE_INTEGER) { - if(o.via.i64 < (int64_t)-std::numeric_limits::max()) + if(o.via.i64 < (int64_t)std::numeric_limits::min()) { throw type_error(); } return o.via.i64; } From 387eca6fbf8c18bf32f35eeb7dab53f2be670c33 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 19 Aug 2009 17:36:05 +0900 Subject: [PATCH 0286/1648] cpp-0.3.7 --- configure.in | 2 +- cpp/object.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/configure.in b/configure.in index 7cdb99d..c241ecf 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.6) +AM_INIT_AUTOMAKE(msgpack, 0.3.7) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/object.cpp b/cpp/object.cpp index eb45f77..a7adb71 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -16,7 +16,6 @@ // limitations under the License. // #include "msgpack/object.hpp" - #include namespace msgpack { From 3a39accb0b9e4fa6cebc1bafc5086366b5014778 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 19 Aug 2009 17:47:22 +0900 Subject: [PATCH 0287/1648] cpp: preprocess eruby templates on ./bootstrap; released package doesn't require erb --- Makefile.am | 1 - README | 4 +--- bootstrap | 5 +---- configure.in | 5 ----- cpp/Makefile.am | 25 ------------------------- 5 files changed, 2 insertions(+), 38 deletions(-) diff --git a/Makefile.am b/Makefile.am index 3144972..be3d75f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,4 @@ if ENABLE_CXX -export ERB SUBDIRS = c cpp else SUBDIRS = c diff --git a/README b/README index 2a1a3e0..66ca276 100644 --- a/README +++ b/README @@ -8,9 +8,7 @@ Binary-based efficient data interchange format. MessagePack is only tested on Linux and Mac OS X, but it may run on other UNIX-like platforms. - Following programs are required to build: - - gcc >= 4.1 with C++ support - - ruby >= 1.8 (ruby is used as a preprocessor) + gcc >= 4.1 is required to build. *Installation diff --git a/bootstrap b/bootstrap index 9ddb3da..8ac504b 100755 --- a/bootstrap +++ b/bootstrap @@ -31,11 +31,8 @@ if test x"$1" = x"--help"; then fi -#if [ -z "$NO_NEST" ];then -# cd c && ./bootstrap $@; cd .. -# cd cpp && ./bootstrap $@; cd .. -#fi mkdir -p ac +(cd cpp && ./preprocess.sh $@; cd ..) ACLOCAL="aclocal" diff --git a/configure.in b/configure.in index c241ecf..de71fc9 100644 --- a/configure.in +++ b/configure.in @@ -23,11 +23,6 @@ if test "$enable_cxx" != "no"; then if test "" = "$CXXFLAGS"; then CXXFLAGS="-g -O4" fi - - AC_CHECK_PROG(ERB, erb, erb) - if test "x$ERB" = x; then - AC_MSG_ERROR([cannot find erb. Ruby is needed to build.]) - fi fi # FIXME diff --git a/cpp/Makefile.am b/cpp/Makefile.am index f9d1fec..2c11535 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -27,31 +27,6 @@ nobase_include_HEADERS = \ msgpack/type/tuple.hpp \ msgpack/type/define.hpp -# FIXME -object.lo: msgpack/type/tuple.hpp msgpack/type/define.hpp msgpack/zone.hpp - -msgpack/type/tuple.hpp: msgpack/type/tuple.hpp.erb - $(ERB) $< > $@.tmp - mv $@.tmp $@ - -msgpack/type/define.hpp: msgpack/type/define.hpp.erb - $(ERB) $< > $@.tmp - mv $@.tmp $@ - -msgpack/zone.hpp: msgpack/zone.hpp.erb - $(ERB) $< > $@.tmp - mv $@.tmp $@ - -MOSTLYCLEANFILES = \ - msgpack/type/tuple.hpp \ - msgpack/type/define.hpp \ - msgpack/zone.hpp - -EXTRA_DIST = \ - msgpack/type/tuple.hpp.erb \ - msgpack/type/define.hpp.erb \ - msgpack/zone.hpp.erb - libmsgpack_la_LIBADD = -L../c -lmsgpackc # -version-info CURRENT:REVISION:AGE From dba7f480edc6302eb9938bd3388a5042ba3c499a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 19 Aug 2009 17:50:15 +0900 Subject: [PATCH 0288/1648] cpp: add missing preprocess.sh file --- cpp/preprocess.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 cpp/preprocess.sh diff --git a/cpp/preprocess.sh b/cpp/preprocess.sh new file mode 100755 index 0000000..69d3b96 --- /dev/null +++ b/cpp/preprocess.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +function preprocess() { + erb $1.erb > $1.tmp + mv $1.tmp $1 +} + +preprocess msgpack/type/tuple.hpp +preprocess msgpack/type/define.hpp +preprocess msgpack/zone.hpp + From 1ba330c4735082d917099fd8eb5fa1f89626cf93 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 25 Aug 2009 18:25:58 +0900 Subject: [PATCH 0289/1648] fix cpp/preprocess.sh --- cpp/preprocess.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/preprocess.sh b/cpp/preprocess.sh index 69d3b96..332dd9c 100755 --- a/cpp/preprocess.sh +++ b/cpp/preprocess.sh @@ -1,6 +1,6 @@ #!/bin/sh -function preprocess() { +preprocess() { erb $1.erb > $1.tmp mv $1.tmp $1 } From d3f9ab7deced474d88bb5b5e85bdb54b312ad86c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 25 Aug 2009 18:35:26 +0900 Subject: [PATCH 0290/1648] add c/test.cpp cpp/test.cpp --- c/Makefile.am | 8 ++ c/test.cpp | 28 +++++++ cpp/Makefile.am | 8 ++ cpp/test.cpp | 203 ++++-------------------------------------------- 4 files changed, 57 insertions(+), 190 deletions(-) create mode 100644 c/test.cpp diff --git a/c/Makefile.am b/c/Makefile.am index daa8a76..0a52446 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -18,3 +18,11 @@ nobase_include_HEADERS = \ # -version-info CURRENT:REVISION:AGE libmsgpackc_la_LDFLAGS = -version-info 1:0:0 +check_PROGRAMS = \ + msgpackc_test + +msgpackc_test_SOURCES = test.cpp +msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c +msgpackc_test_LDFLAGS = libmsgpackc.la -lgtest_main + +TESTS = $(check_PROGRAMS) diff --git a/c/test.cpp b/c/test.cpp new file mode 100644 index 0000000..b42d931 --- /dev/null +++ b/c/test.cpp @@ -0,0 +1,28 @@ +#include "msgpack.h" + +#include + +TEST(MSGPACKC, simple_buffer) +{ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + + msgpack_pack_int(pk, 1); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + + msgpack_object obj; + + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + + EXPECT_EQ(ret, MSGPACK_UNPACK_SUCCESS); + + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); + EXPECT_EQ(1, obj.via.u64); +} + diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 2c11535..af7e6e2 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -32,3 +32,11 @@ libmsgpack_la_LIBADD = -L../c -lmsgpackc # -version-info CURRENT:REVISION:AGE libmsgpack_la_LDFLAGS = -version-info 1:0:0 +check_PROGRAMS = \ + msgpack_test + +msgpackc_test_SOURCES = test.cpp +msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp +msgpackc_test_LDFLAGS = libmsgpack.la -lgtest_main + +TESTS = $(check_PROGRAMS) diff --git a/cpp/test.cpp b/cpp/test.cpp index dd0b1fd..1c5e6c8 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -1,201 +1,24 @@ -#include -#include -#include -#include -#include +#include "msgpack.hpp" -using namespace msgpack; +#include -class checker { -public: - template - void check(const char* d, size_t len, T should) { - try { - std::cout << "----" << std::endl; - - object o; - try { - o = unpack(d, len, m_zone); - } catch (std::runtime_error& e) { - std::cout << o << std::endl; - std::cout << "**" << e.what() << "**" << std::endl; - return; - } - - std::cout << o << std::endl; - - try { - std::stringstream s; - pack(s, should); - std::string str(s.str()); - object ro = unpack(str.data(), str.size(), m_zone); - std::cout << ro << std::endl; - if(ro != o) { throw std::runtime_error("NOT MATCH"); } - } catch (std::runtime_error& e) { - std::cout << "** REUNPACK FAILED **" << std::endl; - std::cout << e.what() << std::endl; - } catch (...) { - std::cout << "** REUNPACK FAILED **" << std::endl; - std::cout << "unknown error" << std::endl; - } - - } catch (...) { m_zone.clear(); throw; } - m_zone.clear(); - } -private: - zone m_zone; -}; - -int main(void) +TEST(MSGPACKC, simple_buffer) { - checker c; + msgpack::sbuffer sbuf; -#if 0 - { // SimpleValue - const char d[] = { - 0x93, 0xc0, 0xc2, 0xc3, - }; - c.check(d, sizeof(d), - type::make_tuple( - type::nil(), false, true - ) - ); - } + int v = 0; - { // Fixnum - const char d[] = { - 0x92, - 0x93, 0x00, 0x40, 0x7f, - 0x93, 0xe0, 0xf0, 0xff, - }; - c.check(d, sizeof(d), - type::make_tuple( - type::make_tuple( - 0, 64, 127 - ), - type::make_tuple( - -32, -16, -1 - ) - ) - ); - } + msgpack::pack(sbuf, v); - { // FixArray - const char d[] = { - 0x92, - 0x90, - 0x91, - 0x91, 0xc0, - }; - std::vector empty; - c.check(d, sizeof(d), - type::make_tuple( - empty, - type::make_tuple( - type::make_tuple( - type::nil() - ) - ) - ) - ); - } + msgpack::zone z; + msgpack::object obj; - { // FixRaw - const char d[] = { - 0x94, - 0xa0, - 0xa1, 'a', - 0xa2, 'b', 'c', - 0xa3, 'd', 'e', 'f', - }; - c.check(d, sizeof(d), - type::make_tuple( - std::string(""), - std::string("a"), - std::string("bc"), - type::raw_ref("def", 3) - ) - ); - } -#endif + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); - static const unsigned TASK_ARRAY = 1000; - static const unsigned TASK_REPEAT = 10; - std::vector task; + obj.convert(&v); - // create task - { - static char traw[64]; - memset(traw, 'a', sizeof(traw)); - - task.resize(TASK_ARRAY); - for(unsigned i=0; i < TASK_ARRAY; ++i) { - task[i] = std::string(traw, sizeof(traw)); - } - } - - - std::stringstream stream; - - // send message - { - for(unsigned i=0; i < TASK_REPEAT; ++i) { - pack(stream, task); - } - std::cout << "send " << stream.str().size() << " bytes" << std::endl; - } - - ssize_t total_bytes = stream.str().size(); - stream.seekg(0); - - // reserive message - { - unsigned num_msg = 0; - static const size_t RESERVE_SIZE = 32;//*1024; - - unpacker pac; - - while(stream.good() && total_bytes > 0) { - - // 1. reserve buffer - pac.reserve_buffer(RESERVE_SIZE); - - // 2. read data to buffer() up to buffer_capacity() bytes - size_t sz = stream.readsome( - pac.buffer(), - pac.buffer_capacity()); - - total_bytes -= sz; - std::cout << "read " << sz << " bytes to capacity " - << pac.buffer_capacity() << " bytes" - << std::endl; - - // 3. specify the number of bytes actually copied - pac.buffer_consumed(sz); - - // 4. repeat execute() until it returns false - while( pac.execute() ) { - // 5.1. take out the parsed object - object o = pac.data(); - - // 5.2 release the zone - std::auto_ptr olife( pac.release_zone() ); - - // 5.3 re-initialize the unpacker */ - pac.reset(); - - // do some with the o and olife - std::cout << "message parsed: " << o << std::endl; - ++num_msg; - } - - } - - std::cout << "stream finished" << std::endl; - std::cout << num_msg << " messages reached" << std::endl; - } - - return 0; + EXPECT_EQ(0, v); } - From 51e435d46c481755a8ffca50a0cffa038bd5db08 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Tue, 25 Aug 2009 18:40:06 +0900 Subject: [PATCH 0291/1648] fix build --- c/test.cpp | 2 +- cpp/Makefile.am | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index b42d931..bc418ed 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -10,7 +10,7 @@ TEST(MSGPACKC, simple_buffer) msgpack_packer pk; msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); - msgpack_pack_int(pk, 1); + msgpack_pack_int(&pk, 1); msgpack_zone z; msgpack_zone_init(&z, 2048); diff --git a/cpp/Makefile.am b/cpp/Makefile.am index af7e6e2..42d6d2a 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -35,8 +35,8 @@ libmsgpack_la_LDFLAGS = -version-info 1:0:0 check_PROGRAMS = \ msgpack_test -msgpackc_test_SOURCES = test.cpp -msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp -msgpackc_test_LDFLAGS = libmsgpack.la -lgtest_main +msgpack_test_SOURCES = test.cpp +msgpack_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp +msgpack_test_LDFLAGS = libmsgpack.la -lgtest_main TESTS = $(check_PROGRAMS) From 5b8777026a288a1d7606f0a2beac3dbc62915dbc Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Tue, 25 Aug 2009 19:04:21 +0900 Subject: [PATCH 0292/1648] c: add sometests for serialization --- c/test.cpp | 129 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 113 insertions(+), 16 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index bc418ed..68c4f22 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -1,28 +1,125 @@ #include "msgpack.h" +#include +#include + #include -TEST(MSGPACKC, simple_buffer) +using namespace std; + +#define LOOP 10000 + +TEST(MSGPACKC, simple_buffer_short) { - msgpack_sbuffer sbuf; - msgpack_sbuffer_init(&sbuf); + vector v; + v.push_back(0); + v.push_back(1); + v.push_back(-1); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + for (unsigned int i = 0; i < LOOP; i++) + v.push_back(rand()); - msgpack_packer pk; - msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + for (unsigned int i = 0; i < v.size(); i++) { + short val = v[i]; + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); - msgpack_pack_int(&pk, 1); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_short(&pk, val); - msgpack_zone z; - msgpack_zone_init(&z, 2048); + msgpack_zone z; + msgpack_zone_init(&z, 2048); - msgpack_object obj; + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); - msgpack_unpack_return ret = - msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); - - EXPECT_EQ(ret, MSGPACK_UNPACK_SUCCESS); - - EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); - EXPECT_EQ(1, obj.via.u64); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + if (val < 0) { + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.i64); + } else { + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.u64); + } + } } +TEST(MSGPACKC, simple_buffer_int) +{ + vector v; + v.push_back(0); + v.push_back(1); + v.push_back(-1); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + for (unsigned int i = 0; i < LOOP; i++) + v.push_back(rand()); + + for (unsigned int i = 0; i < v.size(); i++) { + int val = v[i]; + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_int(&pk, val); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + if (val < 0) { + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.i64); + } else { + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.u64); + } + } +} + +TEST(MSGPACKC, simple_buffer_long) +{ + vector v; + v.push_back(0); + v.push_back(1); + v.push_back(-1); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + for (unsigned int i = 0; i < LOOP; i++) + v.push_back(rand()); + + for (unsigned int i = 0; i < v.size() ; i++) { + long val = v[i]; + + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_int(&pk, val); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + if (val < 0) { + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.i64); + } else { + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); + EXPECT_EQ(val, obj.via.u64); + } + } +} From c232e91f831c1934c8c805d51cfd41f94852778a Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Tue, 25 Aug 2009 19:26:54 +0900 Subject: [PATCH 0293/1648] c: fix bugs in c/test.cpp and add more tests --- c/test.cpp | 235 +++++++++++++++++++++++++++++------------------------ 1 file changed, 131 insertions(+), 104 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index 68c4f22..a0fc7e4 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -7,119 +7,146 @@ using namespace std; -#define LOOP 10000 +const unsigned int kLoop = 10000; + +#define GEN_TEST_SIGNED(test_type, func_type) \ + do { \ + vector v; \ + v.push_back(0); \ + v.push_back(1); \ + v.push_back(-1); \ + v.push_back(numeric_limits::min()); \ + v.push_back(numeric_limits::max()); \ + for (unsigned int i = 0; i < kLoop; i++) \ + v.push_back(rand()); \ + for (unsigned int i = 0; i < v.size() ; i++) { \ + test_type val = v[i]; \ + msgpack_sbuffer sbuf; \ + msgpack_sbuffer_init(&sbuf); \ + msgpack_packer pk; \ + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); \ + msgpack_pack_##func_type(&pk, val); \ + msgpack_zone z; \ + msgpack_zone_init(&z, 2048); \ + msgpack_object obj; \ + msgpack_unpack_return ret = \ + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); \ + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); \ + if (val < 0) { \ + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); \ + EXPECT_EQ(val, obj.via.i64); \ + } else { \ + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); \ + EXPECT_EQ(val, obj.via.u64); \ + } \ + EXPECT_EQ(val, obj.via.u64); \ + } \ + } while(0) + +#define GEN_TEST_UNSIGNED(test_type, func_type) \ + do { \ + vector v; \ + v.push_back(0); \ + v.push_back(1); \ + v.push_back(-1); \ + v.push_back(numeric_limits::min()); \ + v.push_back(numeric_limits::max()); \ + for (unsigned int i = 0; i < kLoop; i++) \ + v.push_back(rand()); \ + for (unsigned int i = 0; i < v.size() ; i++) { \ + test_type val = v[i]; \ + msgpack_sbuffer sbuf; \ + msgpack_sbuffer_init(&sbuf); \ + msgpack_packer pk; \ + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); \ + msgpack_pack_##func_type(&pk, val); \ + msgpack_zone z; \ + msgpack_zone_init(&z, 2048); \ + msgpack_object obj; \ + msgpack_unpack_return ret = \ + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); \ + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); \ + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); \ + EXPECT_EQ(val, obj.via.u64); \ + } \ + } while(0) TEST(MSGPACKC, simple_buffer_short) { - vector v; - v.push_back(0); - v.push_back(1); - v.push_back(-1); - v.push_back(numeric_limits::min()); - v.push_back(numeric_limits::max()); - for (unsigned int i = 0; i < LOOP; i++) - v.push_back(rand()); - - for (unsigned int i = 0; i < v.size(); i++) { - short val = v[i]; - msgpack_sbuffer sbuf; - msgpack_sbuffer_init(&sbuf); - - msgpack_packer pk; - msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); - msgpack_pack_short(&pk, val); - - msgpack_zone z; - msgpack_zone_init(&z, 2048); - - msgpack_object obj; - msgpack_unpack_return ret = - msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); - - EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); - if (val < 0) { - EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.i64); - } else { - EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.u64); - } - } + GEN_TEST_SIGNED(short, short); } TEST(MSGPACKC, simple_buffer_int) { - vector v; - v.push_back(0); - v.push_back(1); - v.push_back(-1); - v.push_back(numeric_limits::min()); - v.push_back(numeric_limits::max()); - for (unsigned int i = 0; i < LOOP; i++) - v.push_back(rand()); - - for (unsigned int i = 0; i < v.size(); i++) { - int val = v[i]; - msgpack_sbuffer sbuf; - msgpack_sbuffer_init(&sbuf); - - msgpack_packer pk; - msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); - msgpack_pack_int(&pk, val); - - msgpack_zone z; - msgpack_zone_init(&z, 2048); - - msgpack_object obj; - msgpack_unpack_return ret = - msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); - - EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); - if (val < 0) { - EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.i64); - } else { - EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.u64); - } - } + GEN_TEST_SIGNED(int, int); } TEST(MSGPACKC, simple_buffer_long) { - vector v; - v.push_back(0); - v.push_back(1); - v.push_back(-1); - v.push_back(numeric_limits::min()); - v.push_back(numeric_limits::max()); - for (unsigned int i = 0; i < LOOP; i++) - v.push_back(rand()); - - for (unsigned int i = 0; i < v.size() ; i++) { - long val = v[i]; - - msgpack_sbuffer sbuf; - msgpack_sbuffer_init(&sbuf); - - msgpack_packer pk; - msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); - msgpack_pack_int(&pk, val); - - msgpack_zone z; - msgpack_zone_init(&z, 2048); - - msgpack_object obj; - msgpack_unpack_return ret = - msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); - - EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); - if (val < 0) { - EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.i64); - } else { - EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); - EXPECT_EQ(val, obj.via.u64); - } - } + GEN_TEST_SIGNED(long, long); +} + +TEST(MSGPACKC, simple_buffer_long_long) +{ + GEN_TEST_SIGNED(long long, long_long); +} + +TEST(MSGPACKC, simple_buffer_unsigned_short) +{ + GEN_TEST_UNSIGNED(unsigned short, unsigned_short); +} + +TEST(MSGPACKC, simple_buffer_unsigned_int) +{ + GEN_TEST_UNSIGNED(unsigned int, unsigned_int); +} + +TEST(MSGPACKC, simple_buffer_unsigned_long) +{ + GEN_TEST_UNSIGNED(unsigned long, unsigned_long); +} + +TEST(MSGPACKC, simple_buffer_unsigned_long_long) +{ + GEN_TEST_UNSIGNED(unsigned long long, unsigned_long_long); +} + +TEST(MSGPACKC, simple_buffer_uint8) +{ + GEN_TEST_UNSIGNED(uint8_t, uint8); +} + +TEST(MSGPACKC, simple_buffer_uint16) +{ + GEN_TEST_UNSIGNED(uint16_t, uint16); +} + +TEST(MSGPACKC, simple_buffer_uint32) +{ + GEN_TEST_UNSIGNED(uint32_t, uint32); +} + +TEST(MSGPACKC, simple_buffer_uint64) +{ + GEN_TEST_UNSIGNED(uint64_t, uint64); +} + +TEST(MSGPACKC, simple_buffer_int8) +{ + GEN_TEST_SIGNED(int8_t, int8); +} + +TEST(MSGPACKC, simple_buffer_int16) +{ + GEN_TEST_SIGNED(int16_t, int16); +} + +TEST(MSGPACKC, simple_buffer_int32) +{ + GEN_TEST_SIGNED(int32_t, int32); +} + +TEST(MSGPACKC, simple_buffer_int64) +{ + GEN_TEST_SIGNED(int64_t, int64); } From 93745710569a9c0051ddce62da7002dbde3c742b Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Wed, 26 Aug 2009 09:24:07 +0900 Subject: [PATCH 0294/1648] C: add more tests (float, double, nil, true, false, array, raw) --- c/test.cpp | 203 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 201 insertions(+), 2 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index a0fc7e4..faeb698 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -1,5 +1,6 @@ #include "msgpack.h" +#include #include #include @@ -8,6 +9,7 @@ using namespace std; const unsigned int kLoop = 10000; +const double kEPS = 10e-10; #define GEN_TEST_SIGNED(test_type, func_type) \ do { \ @@ -39,7 +41,6 @@ const unsigned int kLoop = 10000; EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); \ EXPECT_EQ(val, obj.via.u64); \ } \ - EXPECT_EQ(val, obj.via.u64); \ } \ } while(0) @@ -48,7 +49,7 @@ const unsigned int kLoop = 10000; vector v; \ v.push_back(0); \ v.push_back(1); \ - v.push_back(-1); \ + v.push_back(2); \ v.push_back(numeric_limits::min()); \ v.push_back(numeric_limits::max()); \ for (unsigned int i = 0; i < kLoop; i++) \ @@ -150,3 +151,201 @@ TEST(MSGPACKC, simple_buffer_int64) { GEN_TEST_SIGNED(int64_t, int64); } + +TEST(MSGPACKC, simple_buffer_float) +{ + vector v; + v.push_back(0.0); + v.push_back(1); + v.push_back(-1); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + v.push_back(nanf("tag")); + v.push_back(1.0/0.0); // inf + for (unsigned int i = 0; i < kLoop; i++) + v.push_back(drand48()); + + for (unsigned int i = 0; i < v.size() ; i++) { + float val = v[i]; + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_float(&pk, val); + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_DOUBLE, obj.type); + if (isnan(val)) + EXPECT_TRUE(isnan(obj.via.dec)); + else if (isinf(val)) + EXPECT_TRUE(isinf(obj.via.dec)); + else + EXPECT_TRUE(fabs(obj.via.dec - val) <= kEPS); + } +} + +TEST(MSGPACKC, simple_buffer_double) +{ + vector v; + v.push_back(0.0); + v.push_back(1); + v.push_back(-1); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + v.push_back(nan("tag")); + v.push_back(1.0/0.0); // inf + for (unsigned int i = 0; i < kLoop; i++) + v.push_back(drand48()); + + for (unsigned int i = 0; i < v.size() ; i++) { + double val = v[i]; + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_double(&pk, val); + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_DOUBLE, obj.type); + if (isnan(val)) + EXPECT_TRUE(isnan(obj.via.dec)); + else if (isinf(val)) + EXPECT_TRUE(isinf(obj.via.dec)); + else + EXPECT_TRUE(fabs(obj.via.dec - val) <= kEPS); + } +} + +TEST(MSGPACKC, simple_buffer_nil) +{ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_nil(&pk); + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_NIL, obj.type); +} + +TEST(MSGPACKC, simple_buffer_true) +{ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_true(&pk); + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, obj.type); + EXPECT_EQ(true, obj.via.boolean); +} + +TEST(MSGPACKC, simple_buffer_false) +{ + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_false(&pk); + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret = + msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, obj.type); + EXPECT_EQ(false, obj.via.boolean); +} + +TEST(MSGPACKC, simple_buffer_array) +{ + unsigned int array_size = 5; + + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_array(&pk, array_size); + msgpack_pack_nil(&pk); + msgpack_pack_true(&pk); + msgpack_pack_false(&pk); + msgpack_pack_int(&pk, 10); + msgpack_pack_int(&pk, -10); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret; + ret = msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_ARRAY, obj.type); + EXPECT_EQ(array_size, obj.via.array.size); + + for (unsigned int i = 0; i < obj.via.array.size; i++) { + msgpack_object o = obj.via.array.ptr[i]; + switch (i) { + case 0: + EXPECT_EQ(MSGPACK_OBJECT_NIL, o.type); + break; + case 1: + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, o.type); + EXPECT_EQ(true, o.via.boolean); + break; + case 2: + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, o.type); + EXPECT_EQ(false, o.via.boolean); + break; + case 3: + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, o.type); + EXPECT_EQ(10, o.via.u64); + break; + case 4: + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, o.type); + EXPECT_EQ(-10, o.via.u64); + break; + } + } +} + +TEST(MSGPACKC, simple_buffer_raw) +{ + unsigned int raw_size = 7; + + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_raw(&pk, raw_size); + msgpack_pack_raw_body(&pk, "fr", 2); + msgpack_pack_raw_body(&pk, "syuki", 5); + // invalid data + msgpack_pack_raw_body(&pk, "", 0); + msgpack_pack_raw_body(&pk, "kzk", 0); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret; + ret = msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_RAW, obj.type); + EXPECT_EQ(raw_size, obj.via.raw.size); + EXPECT_EQ(0, memcmp("frsyuki", obj.via.raw.ptr, raw_size)); +} From c94772104d70290f438cf629ff693a8d7e69f827 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Wed, 26 Aug 2009 11:28:54 +0900 Subject: [PATCH 0295/1648] C: add test for map C++: add tests for primitive types, stl types, user-defined type --- c/test.cpp | 47 +++++- cpp/test.cpp | 455 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 482 insertions(+), 20 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index faeb698..28a3d9e 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -9,7 +9,7 @@ using namespace std; const unsigned int kLoop = 10000; -const double kEPS = 10e-10; +const double kEPS = 1e-10; #define GEN_TEST_SIGNED(test_type, func_type) \ do { \ @@ -277,7 +277,7 @@ TEST(MSGPACKC, simple_buffer_false) TEST(MSGPACKC, simple_buffer_array) { unsigned int array_size = 5; - + msgpack_sbuffer sbuf; msgpack_sbuffer_init(&sbuf); msgpack_packer pk; @@ -324,6 +324,49 @@ TEST(MSGPACKC, simple_buffer_array) } } +TEST(MSGPACKC, simple_buffer_map) +{ + unsigned int map_size = 2; + + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_map(&pk, map_size); + msgpack_pack_true(&pk); + msgpack_pack_false(&pk); + msgpack_pack_int(&pk, 10); + msgpack_pack_int(&pk, -10); + + msgpack_zone z; + msgpack_zone_init(&z, 2048); + msgpack_object obj; + msgpack_unpack_return ret; + ret = msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); + EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); + EXPECT_EQ(MSGPACK_OBJECT_MAP, obj.type); + EXPECT_EQ(map_size, obj.via.map.size); + + for (unsigned int i = 0; i < map_size; i++) { + msgpack_object key = obj.via.map.ptr[i].key; + msgpack_object val = obj.via.map.ptr[i].val; + switch (i) { + case 0: + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, key.type); + EXPECT_EQ(true, key.via.boolean); + EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, val.type); + EXPECT_EQ(false, val.via.boolean); + break; + case 1: + EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, key.type); + EXPECT_EQ(10, key.via.u64); + EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, val.type); + EXPECT_EQ(-10, val.via.i64); + break; + } + } +} + TEST(MSGPACKC, simple_buffer_raw) { unsigned int raw_size = 7; diff --git a/cpp/test.cpp b/cpp/test.cpp index 1c5e6c8..f2ff1ab 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -1,24 +1,443 @@ #include "msgpack.hpp" +#include +#include +#include +#include +#include +#include +#include + #include -TEST(MSGPACKC, simple_buffer) +using namespace std; + +const unsigned int kLoop = 10000; +const unsigned int kElements = 100; +const double kEPS = 1e-10; + +#define GEN_TEST(test_type) \ + do { \ + vector v; \ + v.push_back(0); \ + v.push_back(1); \ + v.push_back(2); \ + v.push_back(numeric_limits::min()); \ + v.push_back(numeric_limits::max()); \ + for (unsigned int i = 0; i < kLoop; i++) \ + v.push_back(rand()); \ + for (unsigned int i = 0; i < v.size() ; i++) { \ + msgpack::sbuffer sbuf; \ + test_type val1 = v[i]; \ + msgpack::pack(sbuf, val1); \ + msgpack::zone z; \ + msgpack::object obj; \ + msgpack::unpack_return ret = \ + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); \ + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); \ + test_type val2; \ + obj.convert(&val2); \ + EXPECT_EQ(val1, val2); \ + } \ +} while(0) + +TEST(MSGPACK, simple_buffer_short) { - msgpack::sbuffer sbuf; - - int v = 0; - - msgpack::pack(sbuf, v); - - msgpack::zone z; - msgpack::object obj; - - msgpack::unpack_return ret = - msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); - - obj.convert(&v); - - EXPECT_EQ(0, v); + GEN_TEST(short); +} + +TEST(MSGPACK, simple_buffer_int) +{ + GEN_TEST(int); +} + +TEST(MSGPACK, simple_buffer_long) +{ + GEN_TEST(long); +} + +TEST(MSGPACK, simple_buffer_long_long) +{ + GEN_TEST(long long); +} + +TEST(MSGPACK, simple_buffer_unsigned_short) +{ + GEN_TEST(unsigned short); +} + +TEST(MSGPACK, simple_buffer_unsigned_int) +{ + GEN_TEST(unsigned int); +} + +TEST(MSGPACK, simple_buffer_unsigned_long) +{ + GEN_TEST(unsigned long); +} + +TEST(MSGPACK, simple_buffer_unsigned_long_long) +{ + GEN_TEST(unsigned long long); +} + +TEST(MSGPACK, simple_buffer_uint8) +{ + GEN_TEST(uint8_t); +} + +TEST(MSGPACK, simple_buffer_uint16) +{ + GEN_TEST(uint16_t); +} + +TEST(MSGPACK, simple_buffer_uint32) +{ + GEN_TEST(uint32_t); +} + +TEST(MSGPACK, simple_buffer_uint64) +{ + GEN_TEST(uint64_t); +} + +TEST(MSGPACK, simple_buffer_int8) +{ + GEN_TEST(int8_t); +} + +TEST(MSGPACK, simple_buffer_int16) +{ + GEN_TEST(int16_t); +} + +TEST(MSGPACK, simple_buffer_int32) +{ + GEN_TEST(int32_t); +} + +TEST(MSGPACK, simple_buffer_int64) +{ + GEN_TEST(int64_t); +} + +TEST(MSGPACK, simple_buffer_float) +{ + vector v; + v.push_back(0); + v.push_back(1); + v.push_back(2); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + v.push_back(nanf("tag")); + v.push_back(1.0/0.0); // inf + for (unsigned int i = 0; i < kLoop; i++) + v.push_back(drand48()); + for (unsigned int i = 0; i < v.size() ; i++) { + msgpack::sbuffer sbuf; + float val1 = v[i]; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + float val2; + obj.convert(&val2); + + if (isnan(val1)) + EXPECT_TRUE(isnan(val2)); + else if (isinf(val1)) + EXPECT_TRUE(isinf(val2)); + else + EXPECT_TRUE(fabs(val2 - val1) <= kEPS); + } +} + +TEST(MSGPACK, simple_buffer_double) +{ + vector v; + v.push_back(0); + v.push_back(1); + v.push_back(2); + v.push_back(numeric_limits::min()); + v.push_back(numeric_limits::max()); + v.push_back(nanf("tag")); + v.push_back(1.0/0.0); // inf + for (unsigned int i = 0; i < kLoop; i++) + v.push_back(drand48()); + for (unsigned int i = 0; i < v.size() ; i++) { + msgpack::sbuffer sbuf; + double val1 = v[i]; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + double val2; + obj.convert(&val2); + + if (isnan(val1)) + EXPECT_TRUE(isnan(val2)); + else if (isinf(val1)) + EXPECT_TRUE(isinf(val2)); + else + EXPECT_TRUE(fabs(val2 - val1) <= kEPS); + } +} + +TEST(MSGPACK, simple_buffer_true) +{ + msgpack::sbuffer sbuf; + bool val1 = true; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + bool val2; + obj.convert(&val2); + EXPECT_EQ(val1, val2); +} + +TEST(MSGPACK, simple_buffer_false) +{ + msgpack::sbuffer sbuf; + bool val1 = false; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + bool val2; + obj.convert(&val2); + EXPECT_EQ(val1, val2); +} + +//----------------------------------------------------------------------------- + +TEST(MSGPACK_STL, simple_buffer_string) +{ + for (unsigned int k = 0; k < kLoop; k++) { + string val1; + for (unsigned int i = 0; i < kElements; i++) + val1 += 'a' + rand() % 26; + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + string val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_EQ(val1, val2); + } +} + +TEST(MSGPACK_STL, simple_buffer_vector) +{ + for (unsigned int k = 0; k < kLoop; k++) { + vector val1; + for (unsigned int i = 0; i < kElements; i++) + val1.push_back(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + vector val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_TRUE(equal(val1.begin(), val1.end(), val2.begin())); + } +} + +TEST(MSGPACK_STL, simple_buffer_map) +{ + for (unsigned int k = 0; k < kLoop; k++) { + map val1; + for (unsigned int i = 0; i < kElements; i++) + val1[rand()] = rand(); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + map val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_TRUE(equal(val1.begin(), val1.end(), val2.begin())); + } +} + +TEST(MSGPACK_STL, simple_buffer_deque) +{ + for (unsigned int k = 0; k < kLoop; k++) { + deque val1; + for (unsigned int i = 0; i < kElements; i++) + val1.push_back(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + deque val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_TRUE(equal(val1.begin(), val1.end(), val2.begin())); + } +} + +TEST(MSGPACK_STL, simple_buffer_list) +{ + for (unsigned int k = 0; k < kLoop; k++) { + list val1; + for (unsigned int i = 0; i < kElements; i++) + val1.push_back(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + list val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_TRUE(equal(val1.begin(), val1.end(), val2.begin())); + } +} + +TEST(MSGPACK_STL, simple_buffer_set) +{ + for (unsigned int k = 0; k < kLoop; k++) { + set val1; + for (unsigned int i = 0; i < kElements; i++) + val1.insert(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + set val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_TRUE(equal(val1.begin(), val1.end(), val2.begin())); + } +} + +TEST(MSGPACK_STL, simple_buffer_pair) +{ + for (unsigned int k = 0; k < kLoop; k++) { + pair val1 = make_pair(rand(), rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + pair val2; + obj.convert(&val2); + EXPECT_EQ(val1.first, val2.first); + EXPECT_EQ(val1.second, val2.second); + } +} + +class TestClass +{ +public: + TestClass() : i(0), s("kzk") {} + int i; + string s; + MSGPACK_DEFINE(i, s); +}; + +TEST(MSGPACK_USER_DEFINED, simple_buffer_class) +{ + for (unsigned int k = 0; k < kLoop; k++) { + TestClass val1; + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestClass val2; + val2.i = -1; + val2.s = ""; + obj.convert(&val2); + EXPECT_EQ(val1.i, val2.i); + EXPECT_EQ(val1.s, val2.s); + } +} + +class TestClass2 +{ +public: + TestClass2() : i(0), s("kzk") { + for (unsigned int i = 0; i < kElements; i++) + v.push_back(rand()); + } + int i; + string s; + vector v; + MSGPACK_DEFINE(i, s, v); +}; + +TEST(MSGPACK_USER_DEFINED, simple_buffer_class_old_to_new) +{ + for (unsigned int k = 0; k < kLoop; k++) { + TestClass val1; + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestClass2 val2; + val2.i = -1; + val2.s = ""; + val2.v = vector(); + obj.convert(&val2); + EXPECT_EQ(val1.i, val2.i); + EXPECT_EQ(val1.s, val2.s); + EXPECT_FALSE(val2.s.empty()); + } +} + +TEST(MSGPACK_USER_DEFINED, simple_buffer_class_new_to_old) +{ + for (unsigned int k = 0; k < kLoop; k++) { + TestClass2 val1; + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestClass val2; + val2.i = -1; + val2.s = ""; + obj.convert(&val2); + EXPECT_EQ(val1.i, val2.i); + EXPECT_EQ(val1.s, val2.s); + EXPECT_FALSE(val2.s.empty()); + } } From 7186edc45e057aff9f10511547c3f94c16c9c773 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Wed, 26 Aug 2009 11:52:12 +0900 Subject: [PATCH 0296/1648] C: more strict tests for float, double C++: more strict tests for float, double & enum, union member --- c/test.cpp | 19 ++++--- cpp/test.cpp | 141 +++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 145 insertions(+), 15 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index 28a3d9e..0277206 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -156,14 +156,17 @@ TEST(MSGPACKC, simple_buffer_float) { vector v; v.push_back(0.0); - v.push_back(1); - v.push_back(-1); + v.push_back(1.0); + v.push_back(-1.0); v.push_back(numeric_limits::min()); v.push_back(numeric_limits::max()); v.push_back(nanf("tag")); v.push_back(1.0/0.0); // inf - for (unsigned int i = 0; i < kLoop; i++) + v.push_back(-(1.0/0.0)); // -inf + for (unsigned int i = 0; i < kLoop; i++) { v.push_back(drand48()); + v.push_back(-drand48()); + } for (unsigned int i = 0; i < v.size() ; i++) { float val = v[i]; @@ -192,14 +195,18 @@ TEST(MSGPACKC, simple_buffer_double) { vector v; v.push_back(0.0); - v.push_back(1); - v.push_back(-1); + v.push_back(-0.0); + v.push_back(1.0); + v.push_back(-1.0); v.push_back(numeric_limits::min()); v.push_back(numeric_limits::max()); v.push_back(nan("tag")); v.push_back(1.0/0.0); // inf - for (unsigned int i = 0; i < kLoop; i++) + v.push_back(-(1.0/0.0)); // -inf + for (unsigned int i = 0; i < kLoop; i++) { v.push_back(drand48()); + v.push_back(-drand48()); + } for (unsigned int i = 0; i < v.size() ; i++) { double val = v[i]; diff --git a/cpp/test.cpp b/cpp/test.cpp index f2ff1ab..fc3f9b4 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -124,15 +124,19 @@ TEST(MSGPACK, simple_buffer_int64) TEST(MSGPACK, simple_buffer_float) { vector v; - v.push_back(0); - v.push_back(1); - v.push_back(2); + v.push_back(0.0); + v.push_back(-0.0); + v.push_back(1.0); + v.push_back(-1.0); v.push_back(numeric_limits::min()); v.push_back(numeric_limits::max()); v.push_back(nanf("tag")); v.push_back(1.0/0.0); // inf - for (unsigned int i = 0; i < kLoop; i++) + v.push_back(-(1.0/0.0)); // -inf + for (unsigned int i = 0; i < kLoop; i++) { v.push_back(drand48()); + v.push_back(-drand48()); + } for (unsigned int i = 0; i < v.size() ; i++) { msgpack::sbuffer sbuf; float val1 = v[i]; @@ -157,15 +161,19 @@ TEST(MSGPACK, simple_buffer_float) TEST(MSGPACK, simple_buffer_double) { vector v; - v.push_back(0); - v.push_back(1); - v.push_back(2); + v.push_back(0.0); + v.push_back(-0.0); + v.push_back(1.0); + v.push_back(-1.0); v.push_back(numeric_limits::min()); v.push_back(numeric_limits::max()); v.push_back(nanf("tag")); v.push_back(1.0/0.0); // inf - for (unsigned int i = 0; i < kLoop; i++) + v.push_back(-(1.0/0.0)); // -inf + for (unsigned int i = 0; i < kLoop; i++) { v.push_back(drand48()); + v.push_back(-drand48()); + } for (unsigned int i = 0; i < v.size() ; i++) { msgpack::sbuffer sbuf; double val1 = v[i]; @@ -383,7 +391,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_class) obj.convert(&val2); EXPECT_EQ(val1.i, val2.i); EXPECT_EQ(val1.s, val2.s); - } + } } class TestClass2 @@ -441,3 +449,118 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_class_new_to_old) EXPECT_FALSE(val2.s.empty()); } } + +class TestEnumMemberClass +{ +public: + TestEnumMemberClass() + : t1(STATE_A), t2(STATE_B), t3(STATE_C) {} + + enum TestEnumType { + STATE_INVALID = 0, + STATE_A = 1, + STATE_B = 2, + STATE_C = 3 + }; + TestEnumType t1; + TestEnumType t2; + TestEnumType t3; + + MSGPACK_DEFINE((int&)t1, (int&)t2, (int&)t3); +}; + +TEST(MSGPACK_USER_DEFINED, simple_buffer_enum_member) +{ + TestEnumMemberClass val1; + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestEnumMemberClass val2; + val2.t1 = TestEnumMemberClass::STATE_INVALID; + val2.t2 = TestEnumMemberClass::STATE_INVALID; + val2.t3 = TestEnumMemberClass::STATE_INVALID; + obj.convert(&val2); + EXPECT_EQ(val1.t1, val2.t1); + EXPECT_EQ(val1.t2, val2.t2); + EXPECT_EQ(val1.t3, val2.t3); +} + +class TestUnionMemberClass +{ +public: + TestUnionMemberClass() {} + TestUnionMemberClass(double f) { + is_double = true; + value.f = f; + } + TestUnionMemberClass(int i) { + is_double = false; + value.i = i; + } + + union { + double f; + int i; + } value; + bool is_double; + + template + void msgpack_pack(Packer& pk) const + { + if (is_double) + pk.pack(msgpack::type::tuple(true, value.f)); + else + pk.pack(msgpack::type::tuple(false, value.i)); + } + + void msgpack_unpack(msgpack::object o) + { + msgpack::type::tuple tuple; + o.convert(&tuple); + + is_double = tuple.get<0>(); + if (is_double) + tuple.get<1>().convert(&value.f); + else + tuple.get<1>().convert(&value.i); + } +}; + +TEST(MSGPACK_USER_DEFINED, simple_buffer_union_member) +{ + { + // double + TestUnionMemberClass val1(1.0); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestUnionMemberClass val2; + obj.convert(&val2); + EXPECT_EQ(val1.is_double, val2.is_double); + EXPECT_TRUE(fabs(val1.value.f - val2.value.f) < kEPS); + } + { + // int + TestUnionMemberClass val1(1); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + TestUnionMemberClass val2; + obj.convert(&val2); + EXPECT_EQ(val1.is_double, val2.is_double); + EXPECT_EQ(val1.value.i, 1); + EXPECT_EQ(val1.value.i, val2.value.i); + } +} From c12d5b8461fdd3f3238b37607bf8b9b08af98a1c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 Aug 2009 12:27:28 +0900 Subject: [PATCH 0297/1648] c: test.cpp: add msgpack_sbuffer_destroy and msgpack_zone_destroy --- c/test.cpp | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/c/test.cpp b/c/test.cpp index 0277206..f5646ea 100644 --- a/c/test.cpp +++ b/c/test.cpp @@ -41,6 +41,8 @@ const double kEPS = 1e-10; EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); \ EXPECT_EQ(val, obj.via.u64); \ } \ + msgpack_zone_destroy(&z); \ + msgpack_sbuffer_destroy(&sbuf); \ } \ } while(0) @@ -69,6 +71,8 @@ const double kEPS = 1e-10; EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); \ EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, obj.type); \ EXPECT_EQ(val, obj.via.u64); \ + msgpack_zone_destroy(&z); \ + msgpack_sbuffer_destroy(&sbuf); \ } \ } while(0) @@ -188,6 +192,8 @@ TEST(MSGPACKC, simple_buffer_float) EXPECT_TRUE(isinf(obj.via.dec)); else EXPECT_TRUE(fabs(obj.via.dec - val) <= kEPS); + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } } @@ -228,6 +234,8 @@ TEST(MSGPACKC, simple_buffer_double) EXPECT_TRUE(isinf(obj.via.dec)); else EXPECT_TRUE(fabs(obj.via.dec - val) <= kEPS); + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } } @@ -245,6 +253,8 @@ TEST(MSGPACKC, simple_buffer_nil) msgpack_unpack(sbuf.data, sbuf.size, NULL, &z, &obj); EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); EXPECT_EQ(MSGPACK_OBJECT_NIL, obj.type); + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } TEST(MSGPACKC, simple_buffer_true) @@ -262,6 +272,8 @@ TEST(MSGPACKC, simple_buffer_true) EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, obj.type); EXPECT_EQ(true, obj.via.boolean); + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } TEST(MSGPACKC, simple_buffer_false) @@ -279,6 +291,8 @@ TEST(MSGPACKC, simple_buffer_false) EXPECT_EQ(MSGPACK_UNPACK_SUCCESS, ret); EXPECT_EQ(MSGPACK_OBJECT_BOOLEAN, obj.type); EXPECT_EQ(false, obj.via.boolean); + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } TEST(MSGPACKC, simple_buffer_array) @@ -325,10 +339,13 @@ TEST(MSGPACKC, simple_buffer_array) break; case 4: EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, o.type); - EXPECT_EQ(-10, o.via.u64); + EXPECT_EQ(-10, o.via.i64); break; } } + + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } TEST(MSGPACKC, simple_buffer_map) @@ -369,9 +386,12 @@ TEST(MSGPACKC, simple_buffer_map) EXPECT_EQ(10, key.via.u64); EXPECT_EQ(MSGPACK_OBJECT_NEGATIVE_INTEGER, val.type); EXPECT_EQ(-10, val.via.i64); - break; + break; } } + + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } TEST(MSGPACKC, simple_buffer_raw) @@ -398,4 +418,7 @@ TEST(MSGPACKC, simple_buffer_raw) EXPECT_EQ(MSGPACK_OBJECT_RAW, obj.type); EXPECT_EQ(raw_size, obj.via.raw.size); EXPECT_EQ(0, memcmp("frsyuki", obj.via.raw.ptr, raw_size)); + + msgpack_zone_destroy(&z); + msgpack_sbuffer_destroy(&sbuf); } From ebf64d9892b383aa8ddef1e09ef8c904dc8f1e58 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 Aug 2009 12:28:08 +0900 Subject: [PATCH 0298/1648] cpp: test.cpp: fixes EXPECT_EQ rule --- cpp/test.cpp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cpp/test.cpp b/cpp/test.cpp index fc3f9b4..d9f51da 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -34,7 +34,7 @@ const double kEPS = 1e-10; msgpack::object obj; \ msgpack::unpack_return ret = \ msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); \ - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); \ + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); \ test_type val2; \ obj.convert(&val2); \ EXPECT_EQ(val1, val2); \ @@ -145,7 +145,7 @@ TEST(MSGPACK, simple_buffer_float) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); float val2; obj.convert(&val2); @@ -182,7 +182,7 @@ TEST(MSGPACK, simple_buffer_double) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); double val2; obj.convert(&val2); @@ -204,7 +204,7 @@ TEST(MSGPACK, simple_buffer_true) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); bool val2; obj.convert(&val2); EXPECT_EQ(val1, val2); @@ -219,7 +219,7 @@ TEST(MSGPACK, simple_buffer_false) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); bool val2; obj.convert(&val2); EXPECT_EQ(val1, val2); @@ -239,7 +239,7 @@ TEST(MSGPACK_STL, simple_buffer_string) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); string val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -259,7 +259,7 @@ TEST(MSGPACK_STL, simple_buffer_vector) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); vector val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -279,7 +279,7 @@ TEST(MSGPACK_STL, simple_buffer_map) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); map val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -299,7 +299,7 @@ TEST(MSGPACK_STL, simple_buffer_deque) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); deque val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -319,7 +319,7 @@ TEST(MSGPACK_STL, simple_buffer_list) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); list val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -339,7 +339,7 @@ TEST(MSGPACK_STL, simple_buffer_set) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); set val2; obj.convert(&val2); EXPECT_EQ(val1.size(), val2.size()); @@ -357,7 +357,7 @@ TEST(MSGPACK_STL, simple_buffer_pair) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); pair val2; obj.convert(&val2); EXPECT_EQ(val1.first, val2.first); @@ -384,7 +384,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_class) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestClass val2; val2.i = -1; val2.s = ""; @@ -417,7 +417,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_class_old_to_new) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestClass2 val2; val2.i = -1; val2.s = ""; @@ -439,7 +439,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_class_new_to_old) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestClass val2; val2.i = -1; val2.s = ""; @@ -478,7 +478,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_enum_member) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestEnumMemberClass val2; val2.t1 = TestEnumMemberClass::STATE_INVALID; val2.t2 = TestEnumMemberClass::STATE_INVALID; @@ -541,7 +541,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_union_member) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestUnionMemberClass val2; obj.convert(&val2); EXPECT_EQ(val1.is_double, val2.is_double); @@ -556,7 +556,7 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_union_member) msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); - EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); TestUnionMemberClass val2; obj.convert(&val2); EXPECT_EQ(val1.is_double, val2.is_double); From 320c79db49fbb19b1a5097b0d254ece70cbf2c54 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Fri, 4 Sep 2009 16:20:29 +0900 Subject: [PATCH 0299/1648] c++: add vrefbuffer test --- cpp/test.cpp | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/cpp/test.cpp b/cpp/test.cpp index d9f51da..3907c47 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -564,3 +564,109 @@ TEST(MSGPACK_USER_DEFINED, simple_buffer_union_member) EXPECT_EQ(val1.value.i, val2.value.i); } } + +//----------------------------------------------------------------------------- + +#define GEN_TEST_VREF(test_type) \ + do { \ + vector v; \ + v.push_back(0); \ + for (unsigned int i = 0; i < v.size(); i++) { \ + test_type val1 = v[i]; \ + msgpack::vrefbuffer vbuf; \ + msgpack::pack(vbuf, val1); \ + msgpack::sbuffer sbuf; \ + const struct iovec* cur = vbuf.vector(); \ + const struct iovec* end = cur + vbuf.vector_size(); \ + for(; cur != end; ++cur) \ + sbuf.write((const char*)cur->iov_base, cur->iov_len); \ + msgpack::zone z; \ + msgpack::object obj; \ + msgpack::unpack_return ret = \ + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); \ + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); \ + test_type val2; \ + obj.convert(&val2); \ + EXPECT_EQ(val1, val2); \ + } \ + } while(0); + +TEST(MSGPACK, vrefbuffer_short) +{ + GEN_TEST_VREF(short); +} + +TEST(MSGPACK, vrefbuffer_int) +{ + GEN_TEST_VREF(int); +} + +TEST(MSGPACK, vrefbuffer_long) +{ + GEN_TEST_VREF(long); +} + +TEST(MSGPACK, vrefbuffer_long_long) +{ + GEN_TEST_VREF(long long); +} + +TEST(MSGPACK, vrefbuffer_unsigned_short) +{ + GEN_TEST_VREF(unsigned short); +} + +TEST(MSGPACK, vrefbuffer_unsigned_int) +{ + GEN_TEST_VREF(unsigned int); +} + +TEST(MSGPACK, vrefbuffer_unsigned_long) +{ + GEN_TEST_VREF(unsigned long); +} + +TEST(MSGPACK, vrefbuffer_unsigned_long_long) +{ + GEN_TEST_VREF(unsigned long long); +} + +TEST(MSGPACK, vrefbuffer_uint8) +{ + GEN_TEST_VREF(uint8_t); +} + +TEST(MSGPACK, vrefbuffer_uint16) +{ + GEN_TEST_VREF(uint16_t); +} + +TEST(MSGPACK, vrefbuffer_uint32) +{ + GEN_TEST_VREF(uint32_t); +} + +TEST(MSGPACK, vrefbuffer_uint64) +{ + GEN_TEST_VREF(uint64_t); +} + +TEST(MSGPACK, vrefbuffer_int8) +{ + GEN_TEST_VREF(int8_t); +} + +TEST(MSGPACK, vrefbuffer_int16) +{ + GEN_TEST_VREF(int16_t); +} + +TEST(MSGPACK, vrefbuffer_int32) +{ + GEN_TEST_VREF(int32_t); +} + +TEST(MSGPACK, vrefbuffer_int64) +{ + GEN_TEST_VREF(int64_t); +} From 8da7b692f662ef0b65974864bf70d7437070b254 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Sep 2009 13:50:19 +0900 Subject: [PATCH 0300/1648] cpp: define, type::tuple: GENERATION_LIMIT = 31 --- cpp/type/define.hpp.erb | 2 +- cpp/type/tuple.hpp.erb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/type/define.hpp.erb b/cpp/type/define.hpp.erb index d9d3fb1..2eac4f4 100644 --- a/cpp/type/define.hpp.erb +++ b/cpp/type/define.hpp.erb @@ -33,7 +33,7 @@ namespace msgpack { namespace type { -<% GENERATION_LIMIT = 15 %> +<% GENERATION_LIMIT = 31 %> template , typename A<%=i%> = void<%}%>> struct define; diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index d8ddbcd..501a0f1 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -26,7 +26,7 @@ namespace type { // FIXME operator== // FIXME operator!= -<% GENERATION_LIMIT = 15 %> +<% GENERATION_LIMIT = 31 %> template , typename A<%=i%> = void<%}%>> struct tuple; From bf3cb63d468c333d1455d21012aaa679d110af3e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Sep 2009 13:53:12 +0900 Subject: [PATCH 0301/1648] cpp: version 0.3.8 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index de71fc9..c2ca872 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.7) +AM_INIT_AUTOMAKE(msgpack, 0.3.8) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From d0b76814b026d18a13fe67d99505634f8878250d Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 12 Sep 2009 03:12:53 +0900 Subject: [PATCH 0302/1648] cpp: add tests for stream unpacker api --- cpp/test.cpp | 120 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/cpp/test.cpp b/cpp/test.cpp index 3907c47..4447890 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -670,3 +670,123 @@ TEST(MSGPACK, vrefbuffer_int64) { GEN_TEST_VREF(int64_t); } + +//----------------------------------------------------------------------------- + +#define GEN_TEST_STREAM(test_type) \ + for (unsigned int k = 0; k < kLoop; k++) { \ + msgpack::sbuffer sbuf; \ + msgpack::packer pk(sbuf); \ + typedef std::vector vec_type; \ + vec_type vec; \ + for(unsigned int i = 0; i < rand() % kLoop; ++i) { \ + vec_type::value_type r = rand(); \ + vec.push_back(r); \ + pk.pack(r); \ + } \ + msgpack::unpacker pac; \ + vec_type::const_iterator it = vec.begin(); \ + const char *p = sbuf.data(); \ + const char * const pend = p + sbuf.size(); \ + while (p < pend) { \ + const size_t sz = std::min(pend - p, rand() % 128); \ + pac.reserve_buffer(sz); \ + memcpy(pac.buffer(), p, sz); \ + pac.buffer_consumed(sz); \ + while (pac.execute()) { \ + if (it == vec.end()) goto out; \ + msgpack::object obj = pac.data(); \ + msgpack::zone *life = pac.release_zone(); \ + EXPECT_TRUE(life != NULL); \ + pac.reset(); \ + vec_type::value_type val; \ + obj.convert(&val); \ + EXPECT_EQ(*it, val); \ + ++it; \ + delete life; \ + } \ + p += sz; \ + } \ + out: \ + ; \ + } + +TEST(MSGPACK, stream_short) +{ + GEN_TEST_STREAM(short); +} + +TEST(MSGPACK, stream_int) +{ + GEN_TEST_STREAM(int); +} + +TEST(MSGPACK, stream_long) +{ + GEN_TEST_STREAM(long); +} + +TEST(MSGPACK, stream_long_long) +{ + GEN_TEST_STREAM(long long); +} + +TEST(MSGPACK, stream_unsigned_short) +{ + GEN_TEST_STREAM(unsigned short); +} + +TEST(MSGPACK, stream_unsigned_int) +{ + GEN_TEST_STREAM(unsigned int); +} + +TEST(MSGPACK, stream_unsigned_long) +{ + GEN_TEST_STREAM(unsigned long); +} + +TEST(MSGPACK, stream_unsigned_long_long) +{ + GEN_TEST_STREAM(unsigned long long); +} + +TEST(MSGPACK, stream_uint8) +{ + GEN_TEST_STREAM(uint8_t); +} + +TEST(MSGPACK, stream_uint16) +{ + GEN_TEST_STREAM(uint16_t); +} + +TEST(MSGPACK, stream_uint32) +{ + GEN_TEST_STREAM(uint32_t); +} + +TEST(MSGPACK, stream_uint64) +{ + GEN_TEST_STREAM(uint64_t); +} + +TEST(MSGPACK, stream_int8) +{ + GEN_TEST_STREAM(int8_t); +} + +TEST(MSGPACK, stream_int16) +{ + GEN_TEST_STREAM(int16_t); +} + +TEST(MSGPACK, stream_int32) +{ + GEN_TEST_STREAM(int32_t); +} + +TEST(MSGPACK, stream_int64) +{ + GEN_TEST_STREAM(int64_t); +} From c8ad32a39ed19828fe98c9d6838002653861d3e5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 5 Oct 2009 23:31:08 +0900 Subject: [PATCH 0303/1648] cpp: preprocess.sh: more verbose --- cpp/preprocess.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cpp/preprocess.sh b/cpp/preprocess.sh index 332dd9c..2e06c10 100755 --- a/cpp/preprocess.sh +++ b/cpp/preprocess.sh @@ -1,8 +1,14 @@ #!/bin/sh preprocess() { - erb $1.erb > $1.tmp - mv $1.tmp $1 + ruby -r erb -e 'puts ERB.new(ARGF.read).result' $1.erb > $1.tmp + if [ "$?" != 0 ]; then + echo "" + echo "** preprocess failed **" + echo "" + else + mv $1.tmp $1 + fi } preprocess msgpack/type/tuple.hpp From 3424dc916c27e7e82d73285e468d111449e34460 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 9 Oct 2009 17:49:30 +0900 Subject: [PATCH 0304/1648] cpp: add missing type::tuple::tuple(object o) --- cpp/type/tuple.hpp.erb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/type/tuple.hpp.erb b/cpp/type/tuple.hpp.erb index 501a0f1..2930ae0 100644 --- a/cpp/type/tuple.hpp.erb +++ b/cpp/type/tuple.hpp.erb @@ -91,6 +91,8 @@ private: template <> struct tuple<> { + tuple() {} + tuple(object o) { o.convert(this); } typedef tuple<> value_type; }; <%0.upto(GENERATION_LIMIT) {|i|%> From c6a2569af8839ce68f96cca1d3aa61c0ac6ed4a6 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 22 Oct 2009 14:32:39 +0900 Subject: [PATCH 0305/1648] - generate README automatically - added LICENSE term --- perl/Makefile.PL | 1 + perl/lib/Data/MessagePack.pm | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index dc74d23..ac83f72 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -1,6 +1,7 @@ use inc::Module::Install; name 'Data-MessagePack'; all_from 'lib/Data/MessagePack.pm'; +readme_from 'lib/Data/MessagePack.pm'; perl_version '5.008005'; license 'perl'; diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 2ab33f1..dde7302 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -44,6 +44,12 @@ Pack the string as int when the value looks like int(EXPERIMENTAL). Tokuhiro Matsuno +=head1 LICENSE + +This library is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + + =head1 SEE ALSO L From 68176e10f5266978a06249049f8c37c7930093c7 Mon Sep 17 00:00:00 2001 From: Tokuhiro Matsuno Date: Thu, 22 Oct 2009 14:34:53 +0900 Subject: [PATCH 0306/1648] added some pattern to MANIFEST.SKIP --- perl/MANIFEST.SKIP | 2 ++ 1 file changed, 2 insertions(+) diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index f524216..a318e47 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -21,3 +21,5 @@ \.sw[pon]$ ^\.gitignore$ ppport.h +\.o$ +\.bs$ From 5393a0df16f3bbdf296222b6337db5d57fe7c3a6 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Oct 2009 01:27:09 +0900 Subject: [PATCH 0307/1648] import MessagePack for Java implementation plan 1 --- java-plan1/MessagePack.java | 31 +++ java-plan1/Packer.java | 266 +++++++++++++++++++++++++ java-plan1/Unpacker.java | 385 ++++++++++++++++++++++++++++++++++++ java-plan1/run.sh | 3 + java-plan1/test.java | 27 +++ 5 files changed, 712 insertions(+) create mode 100644 java-plan1/MessagePack.java create mode 100644 java-plan1/Packer.java create mode 100644 java-plan1/Unpacker.java create mode 100755 java-plan1/run.sh create mode 100644 java-plan1/test.java diff --git a/java-plan1/MessagePack.java b/java-plan1/MessagePack.java new file mode 100644 index 0000000..143f6b5 --- /dev/null +++ b/java-plan1/MessagePack.java @@ -0,0 +1,31 @@ +import java.nio.ByteBuffer; +import java.io.InputStream; +import java.io.IOException; + +public class MessagePack { + + static public Object unpack(InputStream source) + { + // FIXME not implemented yet + return null; + } + + static public Object unpack(byte[] source, int len) throws IOException + { + // FIXME not implemented yet + return null; + } + + static public Object unpack(ByteBuffer source) throws IOException + { + // FIXME not implemented yet + return null; + } + + static public Object unpack(ByteBuffer[] source) throws IOException + { + // FIXME not implemented yet + return null; + } +} + diff --git a/java-plan1/Packer.java b/java-plan1/Packer.java new file mode 100644 index 0000000..ea6b2d4 --- /dev/null +++ b/java-plan1/Packer.java @@ -0,0 +1,266 @@ +import java.io.*; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.math.BigInteger; + +public class Packer { + protected byte[] castBytes = new byte[9]; + protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); + protected OutputStream out; + + public Packer(OutputStream out) + { + this.out = out; + } + + public Packer packByte(byte d) throws IOException + { + if(d < -(1<<5)) { + castBytes[0] = (byte)0xd1; + castBytes[1] = d; + out.write(castBytes, 0, 2); + } else { + out.write(d); + } + return this; + } + + public Packer packShort(short d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } + } + return this; + } + + public Packer packInt(int d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<15)) { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } else if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else if(d < (1<<16)) { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } + } + return this; + } + + public Packer packLong(long d) throws IOException + { + if(d < -(1L<<5)) { + if(d < -(1L<<15)) { + if(d < -(1L<<31)) { + // signed 64 + castBytes[0] = (byte)0xd3; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } else { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } + } else { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1L<<16)) { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + //System.out.println("pack uint 16 "+(short)d); + } + } else { + if(d < (1L<<32)) { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } else { + // unsigned 64 + castBytes[0] = (byte)0xcf; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } + } + } + return this; + } + + public Packer packFloat(float d) throws IOException + { + castBytes[0] = (byte)0xca; + castBuffer.putFloat(1, d); + out.write(castBytes, 0, 5); + return this; + } + + public Packer packDouble(double d) throws IOException + { + castBytes[0] = (byte)0xcb; + castBuffer.putDouble(1, d); + out.write(castBytes, 0, 9); + return this; + } + + public Packer packNil() throws IOException + { + out.write((byte)0xc0); + return this; + } + + public Packer packTrue() throws IOException + { + out.write((byte)0xc3); + return this; + } + + public Packer packFalse() throws IOException + { + out.write((byte)0xc2); + return this; + } + + public Packer packArray(int n) throws IOException + { + if(n < 16) { + final int d = 0x90 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xdc; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdd; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packMap(int n) throws IOException + { + if(n < 16) { + final int d = 0x80 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xde; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdf; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRaw(int n) throws IOException + { + if(n < 32) { + final int d = 0xa0 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xda; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdb; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRawBody(byte[] b) throws IOException + { + out.write(b); + return this; + } + + public Packer packRawBody(byte[] b, int off, int length) throws IOException + { + out.write(b, off, length); + return this; + } + + //public Packer pack(Object o) throws IOException +} + diff --git a/java-plan1/Unpacker.java b/java-plan1/Unpacker.java new file mode 100644 index 0000000..be86344 --- /dev/null +++ b/java-plan1/Unpacker.java @@ -0,0 +1,385 @@ +import java.io.*; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.math.BigInteger; + +public class Unpacker { + protected static final int CS_HEADER = 0x00; + protected static final int CS_FLOAT = 0x0a; + protected static final int CS_DOUBLE = 0x0b; + protected static final int CS_UINT_8 = 0x0c; + protected static final int CS_UINT_16 = 0x0d; + protected static final int CS_UINT_32 = 0x0e; + protected static final int CS_UINT_64 = 0x0f; + protected static final int CS_INT_8 = 0x10; + protected static final int CS_INT_16 = 0x11; + protected static final int CS_INT_32 = 0x12; + protected static final int CS_INT_64 = 0x13; + protected static final int CS_RAW_16 = 0x1a; + protected static final int CS_RAW_32 = 0x1b; + protected static final int CS_ARRAY_16 = 0x1c; + protected static final int CS_ARRAY_32 = 0x1d; + protected static final int CS_MAP_16 = 0x1e; + protected static final int CS_MAP_32 = 0x1f; + protected static final int ACS_RAW_VALUE = 0x20; + protected static final int CT_ARRAY_ITEM = 0x00; + protected static final int CT_MAP_KEY = 0x01; + protected static final int CT_MAP_VALUE = 0x02; + + protected static final int MAX_STACK_SIZE = 16; + + protected int cs = CS_HEADER; + protected int trail = 0; + protected int top = -1; + protected boolean finished = false; + protected int[] stack_ct = new int[MAX_STACK_SIZE]; + protected int[] stack_count = new int[MAX_STACK_SIZE]; + protected Object[] stack_obj = new Object[MAX_STACK_SIZE]; + protected Object[] stack_map_key = new Object[MAX_STACK_SIZE]; + //protected Object user; + protected ByteBuffer castBuffer = ByteBuffer.allocate(8); + + public Object getData() + { + return stack_obj[0]; + } + + public boolean isFinished() + { + return finished; + } + + + @SuppressWarnings("unchecked") + public int execute(byte[] src, int off, int length) throws IOException + { + if(off >= length) { return off; } + + int limit = off + length; + int i = off; + int count; + + Object obj = null; + + _out: do { + _header_again: { + //System.out.println("while i:"+i); + + int b = src[i]; + + _push: { + _fixed_trail_again: + if(cs == CS_HEADER) { + + if((b & 0x80) == 0) { // Positive Fixnum + //System.out.println("positive fixnum "+b); + obj = b; + break _push; + } + + if((b & 0xe0) == 0xe0) { // Negative Fixnum + //System.out.println("negative fixnum "+b); + obj = b; + break _push; + } + + if((b & 0xe0) == 0xa0) { // FixRaw + trail = b & 0x1f; + if(trail == 0) { + obj = new byte[0]; + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + } + + if((b & 0xf0) == 0x90) { // FixArray + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + count = b & 0x0f; + ++top; + stack_obj[top] = new ArrayList(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + //System.out.println("fixarray count:"+count); + break _header_again; + } + + if((b & 0xf0) == 0x80) { // FixMap + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + count = b & 0x0f; + ++top; + stack_obj[top] = new HashMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + } + + switch(b & 0xff) { // FIXME + case 0xc0: // nil + obj = null; + break _push; + case 0xc2: // false + obj = false; + break _push; + case 0xc3: // true + obj = true; + break _push; + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + trail = 1 << (b & 0x03); + cs = b & 0x1f; + //System.out.println("a trail "+trail+" cs:"+cs); + break _fixed_trail_again; + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + trail = 2 << (b & 0x01); + cs = b & 0x1f; + //System.out.println("b trail "+trail+" cs:"+cs); + break _fixed_trail_again; + default: + //System.out.println("unknown b "+(b&0xff)); + throw new IOException("parse error"); + } + + } // _fixed_trail_again + + do { + _fixed_trail_again: { + + if(limit - i <= trail) { break _out; } + int n = i + 1; + i += trail; + + switch(cs) { + case CS_FLOAT: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = castBuffer.getFloat(0); + //System.out.println("float "+obj); + break _push; + case CS_DOUBLE: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = castBuffer.getDouble(0); + //System.out.println("double "+obj); + break _push; + case CS_UINT_8: + //System.out.println(n); + //System.out.println(src[n]); + //System.out.println(src[n+1]); + //System.out.println(src[n-1]); + obj = ((int)src[n]) & 0xff; + //System.out.println("uint8 "+obj); + break _push; + case CS_UINT_16: + //System.out.println(src[n]); + //System.out.println(src[n+1]); + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = ((int)castBuffer.getShort(0)) & 0xffff; + //System.out.println("uint 16 "+obj); + break _push; + case CS_UINT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + { + // FIXME always long? + int o = castBuffer.getInt(0); + if(o < 0) { + obj = ((long)o) & 0xffffffffL; + } else { + obj = o; + } + } + //System.out.println("uint 32 "+obj); + break _push; + case CS_UINT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + // FIXME always BigInteger? + long o = castBuffer.getLong(0); + if(o < 0) { + obj = BigInteger.valueOf(o & 0x7fffffffL).setBit(31); + } else { + obj = o; + } + } + throw new IOException("uint 64 is not supported"); + case CS_INT_8: + obj = (int)src[n]; + break _push; + case CS_INT_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = (int)castBuffer.getShort(0); + break _push; + case CS_INT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = (int)castBuffer.getInt(0); + break _push; + case CS_INT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + // FIXME always long? + long o = castBuffer.getLong(0); + if(o <= 0x7fffffffL && o > -0x80000000L) { + obj = (int)o; + } else { + obj = o; + } + } + //System.out.println("long "+obj); + break _push; + case CS_RAW_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + trail = ((int)castBuffer.getShort(0)) & 0xffff; + if(trail == 0) { + obj = new byte[0]; + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + case CS_RAW_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + trail = castBuffer.getInt(0) & 0x7fffffff; + if(trail == 0) { + obj = new byte[0]; + break _push; + } + cs = ACS_RAW_VALUE; + case ACS_RAW_VALUE: + obj = ByteBuffer.wrap(src, n, trail); // FIXME プール? コピー + break _push; + case CS_ARRAY_16: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = new ArrayList(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_ARRAY_32: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = new ArrayList(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_MAP_16: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = new HashMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + case CS_MAP_32: + if(top >= MAX_STACK_SIZE) { + throw new IOException("parse error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = new HashMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + default: + throw new IOException("parse error"); + } + + } // _fixed_trail_again + } while(true); + } // _push + + do { + _push: { + //System.out.println("push top:"+top); + if(top == -1) { + stack_obj[0] = obj; + finished = true; + break _out; + } + + switch(stack_ct[top]) { + case CT_ARRAY_ITEM: + //System.out.println("array item "+obj); + ((ArrayList)(stack_obj[top])).add(obj); + if(--stack_count[top] == 0) { + obj = stack_obj[top]; + --top; + break _push; + } + break _header_again; + case CT_MAP_KEY: + //System.out.println("map key:"+top+" "+obj); + stack_map_key[top] = obj; + stack_ct[top] = CT_MAP_VALUE; + break _header_again; + case CT_MAP_VALUE: + //System.out.println("map value:"+top+" "+obj); + ((HashMap)(stack_obj[top])).put(stack_map_key[top], obj); + if(--stack_count[top] == 0) { + obj = stack_obj[top]; + --top; + break _push; + } + stack_ct[top] = CT_MAP_KEY; + break _header_again; + default: + throw new IOException("parse error"); + } + } // _push + } while(true); + + } // _header_again + cs = CS_HEADER; + ++i; + } while(i < limit); // _out + + return i - off; + } +} + diff --git a/java-plan1/run.sh b/java-plan1/run.sh new file mode 100755 index 0000000..1539a37 --- /dev/null +++ b/java-plan1/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh +javac MessagePack.java Packer.java Unpacker.java test.java +java test diff --git a/java-plan1/test.java b/java-plan1/test.java new file mode 100644 index 0000000..938a687 --- /dev/null +++ b/java-plan1/test.java @@ -0,0 +1,27 @@ +import java.util.*; +import java.io.*; + +class OpenByteArrayOutputStream extends ByteArrayOutputStream { + int getCount() { return count; } + byte[] getBuffer() { return buf; } +} + +public class test { + public static void main(String[] args) throws IOException + { + OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); + + Packer pk = new Packer(out); + pk.packArray(3) + .packInt(0) + .packByte((byte)1) + .packDouble(0.1); + + Unpacker pac = new Unpacker(); + int nlen = pac.execute(out.getBuffer(), 0, out.getCount()); + if(pac.isFinished()) { + System.out.println(pac.getData()); + } + } +} + From d39c016e1d84765a50ae3c7cbfc3d6869faa9c07 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Oct 2009 02:41:55 +0900 Subject: [PATCH 0308/1648] java: fix streaming de/serializer --- java-plan1/Unpacker.java | 22 ++++++++++-- java-plan1/test.java | 75 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/java-plan1/Unpacker.java b/java-plan1/Unpacker.java index be86344..32945de 100644 --- a/java-plan1/Unpacker.java +++ b/java-plan1/Unpacker.java @@ -50,13 +50,26 @@ public class Unpacker { return finished; } + public void reset() + { + for(int i=0; i <= top; ++top) { + stack_ct[top] = 0; + stack_count[top] = 0; + stack_obj[top] = null; + stack_map_key[top] = null; + } + cs = CS_HEADER; + trail = 0; + top = -1; + finished = false; + } @SuppressWarnings("unchecked") public int execute(byte[] src, int off, int length) throws IOException { if(off >= length) { return off; } - int limit = off + length; + int limit = length; int i = off; int count; @@ -64,7 +77,7 @@ public class Unpacker { _out: do { _header_again: { - //System.out.println("while i:"+i); + //System.out.println("while i:"+i+" limit:"+limit); int b = src[i]; @@ -349,6 +362,7 @@ public class Unpacker { ((ArrayList)(stack_obj[top])).add(obj); if(--stack_count[top] == 0) { obj = stack_obj[top]; + stack_obj[top] = null; --top; break _push; } @@ -363,6 +377,8 @@ public class Unpacker { ((HashMap)(stack_obj[top])).put(stack_map_key[top], obj); if(--stack_count[top] == 0) { obj = stack_obj[top]; + stack_map_key[top] = null; + stack_obj[top] = null; --top; break _push; } @@ -379,7 +395,7 @@ public class Unpacker { ++i; } while(i < limit); // _out - return i - off; + return i; } } diff --git a/java-plan1/test.java b/java-plan1/test.java index 938a687..5b2349f 100644 --- a/java-plan1/test.java +++ b/java-plan1/test.java @@ -6,21 +6,88 @@ class OpenByteArrayOutputStream extends ByteArrayOutputStream { byte[] getBuffer() { return buf; } } + public class test { + public static void main(String[] args) throws IOException + { + testSimple(); + testStreaming(); + } + + + public static void testSimple() throws IOException { OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); Packer pk = new Packer(out); pk.packArray(3) - .packInt(0) - .packByte((byte)1) - .packDouble(0.1); + .packInt(1) + .packByte((byte)2) + .packDouble(0.3); Unpacker pac = new Unpacker(); int nlen = pac.execute(out.getBuffer(), 0, out.getCount()); + if(pac.isFinished()) { - System.out.println(pac.getData()); + System.out.println("testSimple: "+pac.getData()); + } + } + + + public static void testStreaming() throws IOException + { + OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); + + //// + // sender + // + // initialize the streaming serializer + Packer pk = new Packer(out); + + // serialize 2 objects + pk.packArray(3) + .packInt(0) + .packByte((byte)1) + .packDouble(0.2); + pk.packArray(3) + .packInt(3) + .packByte((byte)4) + .packDouble(0.5); + + // send it through the network + InputStream sock = new ByteArrayInputStream(out.getBuffer(), 0, out.getCount()); + + + //// + // receiver + // + // initialize the streaming deserializer + Unpacker pac = new Unpacker(); + int parsed = 0; + + byte[] buf = new byte[1024]; + int buflen = 0; + + while(true) { + // receive data from the network + int c = sock.read(); + if(c < 0) { return; } + + buf[buflen++] = (byte)c; + + // deserialize + parsed = pac.execute(buf, parsed, buflen); + if(pac.isFinished()) { + // get an object + Object msg = pac.getData(); + System.out.println("testStreaming: "+msg); + + // reset the streaming deserializer + pac.reset(); + buflen = 0; + parsed = 0; + } } } } From 4758d9f04b115bbccb568e262e49d53c5549edb5 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 5 Nov 2009 14:24:47 +0900 Subject: [PATCH 0309/1648] Fix to use MANIEFST.in. --- python/MANIFEST | 9 --------- python/MANIFEST.in | 2 ++ 2 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 python/MANIFEST create mode 100644 python/MANIFEST.in diff --git a/python/MANIFEST b/python/MANIFEST deleted file mode 100644 index f6c43b0..0000000 --- a/python/MANIFEST +++ /dev/null @@ -1,9 +0,0 @@ -setup.py -msgpack/pack.h -msgpack/unpack.h -msgpack/_msgpack.pyx -msgpack/__init__.py -msgpack/pack_define.h -msgpack/pack_template.h -msgpack/unpack_define.h -msgpack/unpack_template.h diff --git a/python/MANIFEST.in b/python/MANIFEST.in new file mode 100644 index 0000000..6841ffe --- /dev/null +++ b/python/MANIFEST.in @@ -0,0 +1,2 @@ +include setup.py +recursive-include msgpack *.h *.c *.pyx From fb7f2ed4ada6188f0ea5e0b3e1b88a6eccc3457a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 5 Nov 2009 14:24:47 +0900 Subject: [PATCH 0310/1648] Fix to use MANIEFST.in. --- MANIFEST | 9 --------- MANIFEST.in | 2 ++ 2 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 MANIFEST create mode 100644 MANIFEST.in diff --git a/MANIFEST b/MANIFEST deleted file mode 100644 index f6c43b0..0000000 --- a/MANIFEST +++ /dev/null @@ -1,9 +0,0 @@ -setup.py -msgpack/pack.h -msgpack/unpack.h -msgpack/_msgpack.pyx -msgpack/__init__.py -msgpack/pack_define.h -msgpack/pack_template.h -msgpack/unpack_define.h -msgpack/unpack_template.h diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..6841ffe --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include setup.py +recursive-include msgpack *.h *.c *.pyx From 93a95725fc45d7d0047578ecdc5b2ae4e900970f Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 5 Nov 2009 14:26:12 +0900 Subject: [PATCH 0311/1648] Fix Makefile --- python/Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/Makefile b/python/Makefile index d90789a..267415c 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,7 +1,6 @@ all: - python setup.py build_ext -i -f - python setup.py build - python setup.py sdist + python setup_dev.py build_ext -i -f + python setup.py build sdist .PHONY: test test: From e72aa01fe5192960618544ec0394112904b24c7d Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 5 Nov 2009 14:26:12 +0900 Subject: [PATCH 0312/1648] Fix Makefile --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d90789a..267415c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ all: - python setup.py build_ext -i -f - python setup.py build - python setup.py sdist + python setup_dev.py build_ext -i -f + python setup.py build sdist .PHONY: test test: From e39e1d4f602b0202b830f8e672e2116bdb8b9f34 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 12 Nov 2009 01:10:36 +0900 Subject: [PATCH 0313/1648] import MessagePack for Java implementation plan 2 --- java-plan1/MessagePack.java | 6 + java-plan2/build.xml | 15 + java-plan2/src/org/msgpack/GenericArray.java | 25 ++ .../src/org/msgpack/GenericBoolean.java | 17 + java-plan2/src/org/msgpack/GenericLong.java | 17 + java-plan2/src/org/msgpack/GenericMap.java | 25 ++ java-plan2/src/org/msgpack/GenericObject.java | 52 +++ java-plan2/src/org/msgpack/GenericRaw.java | 59 +++ java-plan2/src/org/msgpack/GenericRawRef.java | 55 +++ java-plan2/src/org/msgpack/GenericShort.java | 29 ++ .../src/org/msgpack/MessageConvertable.java | 7 + .../src/org/msgpack/MessageMergeable.java | 7 + .../src/org/msgpack/MessagePackException.java | 10 + .../src/org/msgpack/MessagePackable.java | 9 + java-plan2/src/org/msgpack/Packer.java | 332 +++++++++++++++ .../src/org/msgpack/UnbufferedUnpacker.java | 74 ++++ .../src/org/msgpack/UnpackException.java | 35 ++ .../src/org/msgpack/UnpackIterator.java | 53 +++ java-plan2/src/org/msgpack/Unpacker.java | 269 ++++++++++++ .../src/org/msgpack/impl/ArrayBuilder.java | 7 + .../msgpack/impl/GenericObjectBuilder.java | 127 ++++++ .../impl/GenericZeroCopyObjectBuilder.java | 12 + .../src/org/msgpack/impl/MapBuilder.java | 8 + .../src/org/msgpack/impl/ObjectBuilder.java | 16 + .../msgpack/impl/SpecificObjectBuilder.java | 203 +++++++++ .../src/org/msgpack/impl/UnpackerImpl.java | 395 ++++++++++++++++++ .../src/org/msgpack/schema/ArraySchema.java | 61 +++ .../org/msgpack/schema/ClassGenerator.java | 230 ++++++++++ .../src/org/msgpack/schema/ClassSchema.java | 39 ++ .../src/org/msgpack/schema/FieldSchema.java | 31 ++ .../msgpack/schema/GenericClassSchema.java | 89 ++++ .../msgpack/schema/GenericFieldSchema.java | 25 ++ .../src/org/msgpack/schema/IntSchema.java | 58 +++ .../src/org/msgpack/schema/LongSchema.java | 58 +++ .../src/org/msgpack/schema/MapSchema.java | 70 ++++ .../src/org/msgpack/schema/ObjectSchema.java | 33 ++ .../org/msgpack/schema/PrimitiveSchema.java | 21 + .../src/org/msgpack/schema/RawSchema.java | 44 ++ .../src/org/msgpack/schema/SSchemaParser.java | 246 +++++++++++ java-plan2/src/org/msgpack/schema/Schema.java | 93 +++++ .../src/org/msgpack/schema/ShortSchema.java | 46 ++ .../msgpack/schema/SpecificClassSchema.java | 156 +++++++ .../msgpack/schema/SpecificFieldSchema.java | 78 ++++ .../src/org/msgpack/schema/StringSchema.java | 48 +++ java-plan2/test/Generate.java | 18 + .../src/serializers/msgpack/MediaContent.java | 277 ++++++++++++ .../serializers/msgpack/MediaContent.mpacs | 21 + .../msgpack/MessagePackSerializer.java | 70 ++++ 48 files changed, 3676 insertions(+) create mode 100644 java-plan2/build.xml create mode 100644 java-plan2/src/org/msgpack/GenericArray.java create mode 100644 java-plan2/src/org/msgpack/GenericBoolean.java create mode 100644 java-plan2/src/org/msgpack/GenericLong.java create mode 100644 java-plan2/src/org/msgpack/GenericMap.java create mode 100644 java-plan2/src/org/msgpack/GenericObject.java create mode 100644 java-plan2/src/org/msgpack/GenericRaw.java create mode 100644 java-plan2/src/org/msgpack/GenericRawRef.java create mode 100644 java-plan2/src/org/msgpack/GenericShort.java create mode 100644 java-plan2/src/org/msgpack/MessageConvertable.java create mode 100644 java-plan2/src/org/msgpack/MessageMergeable.java create mode 100644 java-plan2/src/org/msgpack/MessagePackException.java create mode 100644 java-plan2/src/org/msgpack/MessagePackable.java create mode 100644 java-plan2/src/org/msgpack/Packer.java create mode 100644 java-plan2/src/org/msgpack/UnbufferedUnpacker.java create mode 100644 java-plan2/src/org/msgpack/UnpackException.java create mode 100644 java-plan2/src/org/msgpack/UnpackIterator.java create mode 100644 java-plan2/src/org/msgpack/Unpacker.java create mode 100644 java-plan2/src/org/msgpack/impl/ArrayBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/MapBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/ObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java create mode 100644 java-plan2/src/org/msgpack/impl/UnpackerImpl.java create mode 100644 java-plan2/src/org/msgpack/schema/ArraySchema.java create mode 100644 java-plan2/src/org/msgpack/schema/ClassGenerator.java create mode 100644 java-plan2/src/org/msgpack/schema/ClassSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/FieldSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/GenericClassSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/GenericFieldSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/IntSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/LongSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/MapSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/ObjectSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/PrimitiveSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/RawSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/SSchemaParser.java create mode 100644 java-plan2/src/org/msgpack/schema/Schema.java create mode 100644 java-plan2/src/org/msgpack/schema/ShortSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/SpecificClassSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java create mode 100644 java-plan2/src/org/msgpack/schema/StringSchema.java create mode 100644 java-plan2/test/Generate.java create mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java create mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs create mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java diff --git a/java-plan1/MessagePack.java b/java-plan1/MessagePack.java index 143f6b5..0a4f442 100644 --- a/java-plan1/MessagePack.java +++ b/java-plan1/MessagePack.java @@ -29,3 +29,9 @@ public class MessagePack { } } +/* +public interface MessagePackable { + public void toMessagePack(Packer pk); +} +*/ + diff --git a/java-plan2/build.xml b/java-plan2/build.xml new file mode 100644 index 0000000..d382ce0 --- /dev/null +++ b/java-plan2/build.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/java-plan2/src/org/msgpack/GenericArray.java b/java-plan2/src/org/msgpack/GenericArray.java new file mode 100644 index 0000000..53799ca --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericArray.java @@ -0,0 +1,25 @@ +package org.msgpack; + +import java.util.List; +import java.util.ArrayList; + +public class GenericArray extends GenericObject { + private ArrayList array; + + public GenericArray(int length) + { + this.array = new ArrayList(length); + } + + public void add(GenericObject element) + { + array.add(element); + } + + @Override + public List asArray() + { + return array; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericBoolean.java b/java-plan2/src/org/msgpack/GenericBoolean.java new file mode 100644 index 0000000..908128f --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericBoolean.java @@ -0,0 +1,17 @@ +package org.msgpack; + +public class GenericBoolean extends GenericObject { + boolean value; + + public GenericBoolean(boolean value) + { + this.value = value; + } + + @Override + public boolean asBoolean() + { + return value; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericLong.java b/java-plan2/src/org/msgpack/GenericLong.java new file mode 100644 index 0000000..7cc9110 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericLong.java @@ -0,0 +1,17 @@ +package org.msgpack; + +public class GenericLong extends GenericObject { + long value; + + public GenericLong(long value) + { + this.value = value; + } + + @Override + public long asLong() + { + return value; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericMap.java b/java-plan2/src/org/msgpack/GenericMap.java new file mode 100644 index 0000000..0a5512a --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericMap.java @@ -0,0 +1,25 @@ +package org.msgpack; + +import java.util.Map; +import java.util.HashMap; + +public class GenericMap extends GenericObject { + private HashMap map; + + public GenericMap(int length) + { + this.map = new HashMap(length); + } + + public void put(GenericObject key, GenericObject value) + { + map.put(key, value); + } + + @Override + public Map asMap() + { + return map; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericObject.java b/java-plan2/src/org/msgpack/GenericObject.java new file mode 100644 index 0000000..32eacd3 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericObject.java @@ -0,0 +1,52 @@ +package org.msgpack; + +import java.util.List; +import java.util.Map; + +public abstract class GenericObject { + public boolean asBoolean() + { + throw new RuntimeException("type error"); + } + + public byte asByte() + { + throw new RuntimeException("type error"); + } + + public short asShort() + { + throw new RuntimeException("type error"); + } + + public int asInt() + { + throw new RuntimeException("type error"); + } + + public long asLong() + { + throw new RuntimeException("type error"); + } + + public String asString() + { + throw new RuntimeException("type error"); + } + + public byte[] asBytes() + { + throw new RuntimeException("type error"); + } + + public List asArray() + { + throw new RuntimeException("type error"); + } + + public Map asMap() + { + throw new RuntimeException("type error"); + } +} + diff --git a/java-plan2/src/org/msgpack/GenericRaw.java b/java-plan2/src/org/msgpack/GenericRaw.java new file mode 100644 index 0000000..6de03b2 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericRaw.java @@ -0,0 +1,59 @@ +package org.msgpack; + +import java.nio.charset.Charset; + +public class GenericRaw extends GenericObject { + byte[] bytes; + String string; + + public GenericRaw() + { + this.bytes = new byte[0]; + this.string = null; + } + + public GenericRaw(byte[] bytes) + { + this.bytes = bytes; + this.string = null; + } + + public GenericRaw(String string) + { + this.bytes = null; + this.string = string; + } + + public synchronized void setString(String string) + { + this.string = string; + this.bytes = null; + } + + public synchronized void setBytes(byte[] bytes) + { + this.bytes = bytes; + this.string = null; + } + + private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); + + @Override + public synchronized String asString() + { + if(string == null) { + return string = new String(bytes, UTF8_CHARSET); + } + return string; + } + + @Override + public synchronized byte[] asBytes() + { + if(bytes == null) { + return bytes = string.getBytes(UTF8_CHARSET); + } + return bytes; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericRawRef.java b/java-plan2/src/org/msgpack/GenericRawRef.java new file mode 100644 index 0000000..7007810 --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericRawRef.java @@ -0,0 +1,55 @@ +package org.msgpack; + +import java.nio.charset.Charset; + +public class GenericRawRef extends GenericRaw { + int offset; + int length; + + public GenericRawRef(byte[] bytes, int offset, int length) + { + this.bytes = bytes; + this.offset = offset; + this.length = length; + this.string = null; + } + + public GenericRawRef(String string) + { + this.bytes = null; + this.string = string; + } + + public synchronized void setString(String string) + { + this.string = string; + this.bytes = null; + } + + public synchronized void setBytes(byte[] bytes) + { + this.bytes = bytes; + this.string = null; + } + + private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); + + @Override + public synchronized String asString() + { + if(string == null) { + return string = new String(bytes, UTF8_CHARSET); + } + return string; + } + + @Override + public synchronized byte[] asBytes() + { + if(bytes == null) { + return bytes = string.getBytes(UTF8_CHARSET); + } + return bytes; + } +} + diff --git a/java-plan2/src/org/msgpack/GenericShort.java b/java-plan2/src/org/msgpack/GenericShort.java new file mode 100644 index 0000000..eb2463e --- /dev/null +++ b/java-plan2/src/org/msgpack/GenericShort.java @@ -0,0 +1,29 @@ +package org.msgpack; + +public class GenericShort extends GenericObject { + short value; + + public GenericShort(short value) + { + this.value = value; + } + + @Override + public short asShort() + { + return value; + } + + @Override + public int asInt() + { + return value; + } + + @Override + public long asLong() + { + return value; + } +} + diff --git a/java-plan2/src/org/msgpack/MessageConvertable.java b/java-plan2/src/org/msgpack/MessageConvertable.java new file mode 100644 index 0000000..68c123e --- /dev/null +++ b/java-plan2/src/org/msgpack/MessageConvertable.java @@ -0,0 +1,7 @@ +package org.msgpack; + +public interface MessageConvertable +{ + public void messageConvert(GenericObject obj); +} + diff --git a/java-plan2/src/org/msgpack/MessageMergeable.java b/java-plan2/src/org/msgpack/MessageMergeable.java new file mode 100644 index 0000000..dc50749 --- /dev/null +++ b/java-plan2/src/org/msgpack/MessageMergeable.java @@ -0,0 +1,7 @@ +package org.msgpack; + +public interface MessageMergeable { + public void setField(int index, Object value); + public Object getField(int index); +} + diff --git a/java-plan2/src/org/msgpack/MessagePackException.java b/java-plan2/src/org/msgpack/MessagePackException.java new file mode 100644 index 0000000..dc37989 --- /dev/null +++ b/java-plan2/src/org/msgpack/MessagePackException.java @@ -0,0 +1,10 @@ +package org.msgpack; + +public class MessagePackException extends RuntimeException +{ + public MessagePackException(String message) + { + super(message); + } +} + diff --git a/java-plan2/src/org/msgpack/MessagePackable.java b/java-plan2/src/org/msgpack/MessagePackable.java new file mode 100644 index 0000000..1efff3f --- /dev/null +++ b/java-plan2/src/org/msgpack/MessagePackable.java @@ -0,0 +1,9 @@ +package org.msgpack; + +import java.io.IOException; + +public interface MessagePackable +{ + public void messagePack(Packer pk) throws IOException; +} + diff --git a/java-plan2/src/org/msgpack/Packer.java b/java-plan2/src/org/msgpack/Packer.java new file mode 100644 index 0000000..4545849 --- /dev/null +++ b/java-plan2/src/org/msgpack/Packer.java @@ -0,0 +1,332 @@ +package org.msgpack; + +import java.io.OutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Map; + +public class Packer { + protected byte[] castBytes = new byte[9]; + protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); + protected OutputStream out; + + public Packer(OutputStream out) + { + this.out = out; + } + + public Packer packByte(byte d) throws IOException + { + if(d < -(1<<5)) { + castBytes[0] = (byte)0xd1; + castBytes[1] = d; + out.write(castBytes, 0, 2); + } else { + out.write(d); + } + return this; + } + + public Packer packShort(short d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } + } + return this; + } + + public Packer packInt(int d) throws IOException + { + if(d < -(1<<5)) { + if(d < -(1<<15)) { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } else if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else if(d < (1<<16)) { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } + } + return this; + } + + public Packer packLong(long d) throws IOException + { + if(d < -(1L<<5)) { + if(d < -(1L<<15)) { + if(d < -(1L<<31)) { + // signed 64 + castBytes[0] = (byte)0xd3; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } else { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } + } else { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1L<<16)) { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + //System.out.println("pack uint 16 "+(short)d); + } + } else { + if(d < (1L<<32)) { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } else { + // unsigned 64 + castBytes[0] = (byte)0xcf; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } + } + } + return this; + } + + public Packer packFloat(float d) throws IOException + { + castBytes[0] = (byte)0xca; + castBuffer.putFloat(1, d); + out.write(castBytes, 0, 5); + return this; + } + + public Packer packDouble(double d) throws IOException + { + castBytes[0] = (byte)0xcb; + castBuffer.putDouble(1, d); + out.write(castBytes, 0, 9); + return this; + } + + public Packer packNil() throws IOException + { + out.write((byte)0xc0); + return this; + } + + public Packer packTrue() throws IOException + { + out.write((byte)0xc3); + return this; + } + + public Packer packFalse() throws IOException + { + out.write((byte)0xc2); + return this; + } + + public Packer packArray(int n) throws IOException + { + if(n < 16) { + final int d = 0x90 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xdc; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdd; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packMap(int n) throws IOException + { + if(n < 16) { + final int d = 0x80 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xde; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdf; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRaw(int n) throws IOException + { + if(n < 32) { + final int d = 0xa0 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xda; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdb; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRawBody(byte[] b) throws IOException + { + out.write(b); + return this; + } + + public Packer packRawBody(byte[] b, int off, int length) throws IOException + { + out.write(b, off, length); + return this; + } + + private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); + + public Packer packString(String s) throws IOException + { + byte[] b = ((String)s).getBytes(UTF8_CHARSET); + packRaw(b.length); + packRawBody(b); + return this; + } + + public Packer pack(MessagePackable o) throws IOException + { + o.messagePack(this); + return this; + } + + public Packer pack(Object o) throws IOException + { + if(o == null) { + packNil(); + } else if(o instanceof String) { + byte[] b = ((String)o).getBytes(UTF8_CHARSET); + packRaw(b.length); + packRawBody(b); + } else if(o instanceof byte[]) { + byte[] b = (byte[])o; + packRaw(b.length); + packRawBody(b); + } else if(o instanceof List) { + List l = (List)o; + packArray(l.size()); + for(Object i : l) { pack(i); } + } else if(o instanceof Map) { + Map m = (Map)o; + packMap(m.size()); + for(Map.Entry e : m.entrySet()) { + pack(e.getKey()); + pack(e.getValue()); + } + } else if(o instanceof Boolean) { + if((Boolean)o) { + packTrue(); + } else { + packFalse(); + } + } else if(o instanceof Integer) { + packInt((Integer)o); + } else if(o instanceof Long) { + packLong((Long)o); + } else if(o instanceof Short) { + packShort((Short)o); + } else if(o instanceof Byte) { + packByte((Byte)o); + } else if(o instanceof Float) { + packFloat((Float)o); + } else if(o instanceof Double) { + packDouble((Double)o); + } else if(o instanceof MessagePackable) { + ((MessagePackable)o).messagePack(this); + } else { + throw new IOException("unknown object "+o+" ("+o.getClass()+")"); + } + return this; + } +} + diff --git a/java-plan2/src/org/msgpack/UnbufferedUnpacker.java b/java-plan2/src/org/msgpack/UnbufferedUnpacker.java new file mode 100644 index 0000000..5b5eb97 --- /dev/null +++ b/java-plan2/src/org/msgpack/UnbufferedUnpacker.java @@ -0,0 +1,74 @@ +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.*; +import org.msgpack.schema.Schema; + +public class UnbufferedUnpacker extends UnpackerImpl { + private int offset; + private boolean finished; + private Object data; + + public UnbufferedUnpacker() + { + super(new GenericObjectBuilder()); + this.offset = 0; + this.finished = false; + } + + public UnbufferedUnpacker useSchema(Schema s) + { + super.setBuilder(new SpecificObjectBuilder(s)); + return this; + } + + public Object getData() + { + return data; + } + + public boolean isFinished() + { + return finished; + } + + public void reset() + { + super.reset(); + this.offset = 0; + } + + int getOffset() + { + return offset; + } + + void setOffset(int offset) + { + this.offset = offset; + } + + public int execute(byte[] buffer) throws UnpackException + { + return execute(buffer, 0, buffer.length); + } + + // FIXME + public int execute(byte[] buffer, int offset, int length) throws UnpackException + { + int noffset = super.execute(buffer, offset + this.offset, length); + this.offset = noffset - offset; + if(super.isFinished()) { + this.data = super.getData(); + this.finished = true; + super.reset(); + } else { + this.finished = false; + } + return noffset; + } +} + diff --git a/java-plan2/src/org/msgpack/UnpackException.java b/java-plan2/src/org/msgpack/UnpackException.java new file mode 100644 index 0000000..2081faf --- /dev/null +++ b/java-plan2/src/org/msgpack/UnpackException.java @@ -0,0 +1,35 @@ +package org.msgpack; + +public class UnpackException extends MessagePackException +{ + public static final int EXTRA_BYTES = 1; + public static final int INSUFFICIENT_BYTES = 0; + public static final int PARSE_ERROR = -1; + + private int errorCode; + private Object data; + + public UnpackException(String message, int errorCode) + { + super(message); + this.errorCode = errorCode; + } + + public UnpackException(String message, int errorCode, Object data) + { + super(message); + this.errorCode = errorCode; + this.data = data; + } + + public int getErrorCode() + { + return errorCode; + } + + public Object getData() + { + return data; + } +} + diff --git a/java-plan2/src/org/msgpack/UnpackIterator.java b/java-plan2/src/org/msgpack/UnpackIterator.java new file mode 100644 index 0000000..9010101 --- /dev/null +++ b/java-plan2/src/org/msgpack/UnpackIterator.java @@ -0,0 +1,53 @@ +package org.msgpack; + +import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class UnpackIterator implements Iterator { + private Unpacker pac; + private boolean have; + private Object data; + + UnpackIterator(Unpacker pac) + { + this.pac = pac; + this.have = false; + } + + public boolean hasNext() + { + if(have) { return true; } + try { + while(true) { + if(pac.execute()) { + data = pac.getData(); + pac.reset(); + have = true; + return true; + } + + if(!pac.fill()) { + return false; + } + } + } catch (IOException e) { + return false; + } + } + + public Object next() + { + if(!have) { + throw new NoSuchElementException(); + } + have = false; + return data; + } + + public void remove() + { + throw new UnsupportedOperationException(); + } +} + diff --git a/java-plan2/src/org/msgpack/Unpacker.java b/java-plan2/src/org/msgpack/Unpacker.java new file mode 100644 index 0000000..5a58ce8 --- /dev/null +++ b/java-plan2/src/org/msgpack/Unpacker.java @@ -0,0 +1,269 @@ +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.*; +import org.msgpack.schema.Schema; + +public class Unpacker extends UnpackerImpl implements Iterable { + + public static final int DEFAULT_BUFFER_SIZE = 32*1024; + + private int used; + private int offset; + private int parsed; + private byte[] buffer; + private int bufferReserveSize; + private InputStream stream; + + public Unpacker() + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[DEFAULT_BUFFER_SIZE]; + this.bufferReserveSize = DEFAULT_BUFFER_SIZE/2; + this.stream = null; + } + + public Unpacker(int bufferReserveSize) + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[bufferReserveSize]; + this.bufferReserveSize = bufferReserveSize/2; + this.stream = null; + } + + public Unpacker(InputStream stream) + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[DEFAULT_BUFFER_SIZE]; + this.bufferReserveSize = DEFAULT_BUFFER_SIZE/2; + this.stream = stream; + } + + public Unpacker(InputStream stream, int bufferReserveSize) + { + super(new GenericZeroCopyObjectBuilder()); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[bufferReserveSize]; + this.bufferReserveSize = bufferReserveSize/2; + this.stream = stream; + } + + public Unpacker useSchema(Schema s) + { + super.setBuilder(new SpecificObjectBuilder(s)); + return this; + } + + public void reserveBuffer(int size) + { + if(buffer.length - used >= size) { + return; + } + /* + if(used == parsed && buffer.length >= size) { + // rewind buffer + used = 0; + offset = 0; + return; + } + */ + + int nextSize = buffer.length * 2; + while(nextSize < size + used) { + nextSize *= 2; + } + + byte[] tmp = new byte[nextSize]; + System.arraycopy(buffer, offset, tmp, 0, used - offset); + + buffer = tmp; + used -= offset; + offset = 0; + } + + public byte[] getBuffer() + { + return buffer; + } + + public int getBufferOffset() + { + return used; + } + + public int getBufferCapacity() + { + return buffer.length - used; + } + + public void bufferConsumed(int size) + { + used += size; + } + + public void feed(byte[] buffer) + { + feed(buffer, 0, buffer.length); + } + + public void feed(byte[] buffer, int offset, int length) + { + reserveBuffer(length); + System.arraycopy(buffer, offset, this.buffer, this.offset, length); + bufferConsumed(length); + } + + public boolean fill() throws IOException + { + if(stream == null) { + return false; + } + reserveBuffer(bufferReserveSize); + int rl = stream.read(getBuffer(), getBufferOffset(), getBufferCapacity()); + if(rl <= 0) { + return false; + } + bufferConsumed(rl); + return true; + } + + public Iterator iterator() + { + return new UnpackIterator(this); + } + + public boolean execute() throws UnpackException + { + int noffset = super.execute(buffer, offset, used); + if(noffset <= offset) { + return false; + } + parsed += noffset - offset; + offset = noffset; + return super.isFinished(); + } + + public Object getData() + { + return super.getData(); + } + + public void reset() + { + super.reset(); + parsed = 0; + } + + public int getMessageSize() + { + return parsed - offset + used; + } + + public int getParsedSize() + { + return parsed; + } + + public int getNonParsedSize() + { + return used - offset; + } + + public void skipNonparsedBuffer(int size) + { + offset += size; + } + + public void removeNonparsedBuffer() + { + used = offset; + } + + /* + public static class Context { + private boolean finished; + private Object data; + private int offset; + private UnpackerImpl impl; + + public Context() + { + this.finished = false; + this.impl = new UnpackerImpl(); + } + + public boolean isFinished() + { + return finished; + } + + public Object getData() + { + return data; + } + + int getOffset() + { + return offset; + } + + void setFinished(boolean finished) + { + this.finished = finished; + } + + void setData(Object data) + { + this.data = data; + } + + void setOffset(int offset) + { + this.offset = offset; + } + + UnpackerImpl getImpl() + { + return impl; + } + } + + public static int unpack(Context ctx, byte[] buffer) throws UnpackException + { + return unpack(ctx, buffer, 0, buffer.length); + } + + public static int unpack(Context ctx, byte[] buffer, int offset, int length) throws UnpackException + { + UnpackerImpl impl = ctx.getImpl(); + int noffset = impl.execute(buffer, offset + ctx.getOffset(), length); + ctx.setOffset(noffset - offset); + if(impl.isFinished()) { + ctx.setData(impl.getData()); + ctx.setFinished(false); + impl.reset(); + } else { + ctx.setData(null); + ctx.setFinished(true); + } + int parsed = noffset - offset; + ctx.setOffset(parsed); + return noffset; + } + */ +} + diff --git a/java-plan2/src/org/msgpack/impl/ArrayBuilder.java b/java-plan2/src/org/msgpack/impl/ArrayBuilder.java new file mode 100644 index 0000000..9bb099b --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/ArrayBuilder.java @@ -0,0 +1,7 @@ +package org.msgpack.impl; + +public interface ArrayBuilder { + public void add(Object element); + public Object finish(); +} + diff --git a/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java b/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java new file mode 100644 index 0000000..814e302 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java @@ -0,0 +1,127 @@ +package org.msgpack.impl; + +import org.msgpack.*; + +public class GenericObjectBuilder implements ObjectBuilder { + public Object createNil() + { + return null; + } + + @Override + public Object createBoolean(boolean v) + { + return new GenericBoolean(v); + } + + @Override + public Object createByte(byte v) + { + //return new GenericByte(v); + return null; // FIXME + } + + @Override + public Object createShort(short v) + { + return new GenericShort(v); + } + + @Override + public Object createInt(int v) + { + //return new GenericInt(v); + return null; // FIXME + } + + @Override + public Object createLong(long v) + { + return new GenericLong(v); + } + + @Override + public Object createFloat(float v) + { + //return new GenericFloat(v); + return null; // FIXME + } + + @Override + public Object createDouble(double v) + { + //return new GenericDouble(v); + return null; // FIXME + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + byte[] copy = new byte[length]; + System.arraycopy(b, offset, copy, 0, length); + return new GenericRaw(copy); + } + + @Override + public ArrayBuilder createArray(int length) + { + return new GenericArrayBuilder(length); + } + + @Override + public MapBuilder createMap(int length) + { + return new GenericMapBuilder(length); + } +} + +final class GenericArrayBuilder implements ArrayBuilder { + private GenericArray a; + + GenericArrayBuilder(int length) + { + this.a = new GenericArray(length); + } + + @Override + public void add(Object element) + { + a.add((GenericObject)element); + } + + @Override + public Object finish() + { + return a; + } +} + +final class GenericMapBuilder implements MapBuilder { + private GenericMap m; + private GenericObject key; + + GenericMapBuilder(int length) + { + this.m = new GenericMap(length); + } + + @Override + public void putKey(Object key) + { + this.key = (GenericObject)key; + } + + @Override + public void putValue(Object value) + { + m.put(this.key, (GenericObject)value); + this.key = null; + } + + @Override + public Object finish() + { + return m; + } +} + diff --git a/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java b/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java new file mode 100644 index 0000000..5569fbf --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java @@ -0,0 +1,12 @@ +package org.msgpack.impl; + +import org.msgpack.*; + +public class GenericZeroCopyObjectBuilder extends GenericObjectBuilder { + @Override + public Object createRaw(byte[] b, int offset, int length) + { + return new GenericRawRef(b, offset, length); + } +} + diff --git a/java-plan2/src/org/msgpack/impl/MapBuilder.java b/java-plan2/src/org/msgpack/impl/MapBuilder.java new file mode 100644 index 0000000..57859a6 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/MapBuilder.java @@ -0,0 +1,8 @@ +package org.msgpack.impl; + +public interface MapBuilder { + public void putKey(Object key); + public void putValue(Object value); + public Object finish(); +} + diff --git a/java-plan2/src/org/msgpack/impl/ObjectBuilder.java b/java-plan2/src/org/msgpack/impl/ObjectBuilder.java new file mode 100644 index 0000000..3268903 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/ObjectBuilder.java @@ -0,0 +1,16 @@ +package org.msgpack.impl; + +public interface ObjectBuilder { + public Object createNil(); + public Object createBoolean(boolean v); + public Object createByte(byte v); + public Object createShort(short v); + public Object createInt(int v); + public Object createLong(long v); + public Object createFloat(float v); + public Object createDouble(double v); + public Object createRaw(byte[] b, int offset, int length); + public ArrayBuilder createArray(int length); + public MapBuilder createMap(int length); +} + diff --git a/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java b/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java new file mode 100644 index 0000000..7748844 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java @@ -0,0 +1,203 @@ +package org.msgpack.impl; + +import java.util.List; +import java.util.ArrayList; +import java.util.HashMap; +import org.msgpack.schema.*; + +public final class SpecificObjectBuilder implements ObjectBuilder { + private int top; + private Schema schema; + + public SpecificObjectBuilder(Schema schema) + { + this.top = 0; + this.schema = schema; + } + + void setSchema(Schema s) + { + schema = s; + } + + Schema swapSchema(Schema s) + { + Schema old = schema; + schema = s; + return old; + } + + @Override + public Object createNil() + { + return schema.createNil(); + } + + @Override + public Object createBoolean(boolean v) + { + return schema.createBoolean(v); + } + + @Override + public Object createByte(byte v) + { + return schema.createByte(v); + } + + @Override + public Object createShort(short v) + { + return schema.createShort(v); + } + + @Override + public Object createInt(int v) + { + return schema.createInt(v); + } + + @Override + public Object createLong(long v) + { + return schema.createLong(v); + } + + @Override + public Object createFloat(float v) + { + return schema.createFloat(v); + } + + @Override + public Object createDouble(double v) + { + return schema.createDouble(v); + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + return schema.createRaw(b, offset, length); + } + + @Override + public ArrayBuilder createArray(int length) + { + if(schema instanceof ClassSchema) { + return new ClassBuilder(length, (ClassSchema)schema, this); + } + ArraySchema as = (ArraySchema)schema; + return new SpecificArrayBuilder(length, as.getElementType(), this); + } + + @Override + public MapBuilder createMap(int length) + { + MapSchema ms = (MapSchema)schema; + return new SpecificMapBuilder(length, ms.getKeyType(), ms.getValueType(), this); + } +} + +final class SpecificArrayBuilder implements ArrayBuilder { + private ArrayList a; + private SpecificObjectBuilder builder; + private Schema parentSchema; + + SpecificArrayBuilder(int length, Schema elementSchema, SpecificObjectBuilder builder) + { + this.a = new ArrayList(length); + this.builder = builder; + this.parentSchema = builder.swapSchema(elementSchema); + } + + public void add(Object element) + { + a.add(element); + } + + public Object finish() + { + builder.swapSchema(parentSchema); + return a; + } +} + +final class SpecificMapBuilder implements MapBuilder { + private HashMap m; + private Object key; + private Schema keySchema; + private Schema valueSchema; + private SpecificObjectBuilder builder; + private Schema parentSchema; + + SpecificMapBuilder(int length, Schema keySchema, Schema valueSchema, SpecificObjectBuilder builder) + { + this.m = new HashMap(length); + this.keySchema = keySchema; + this.valueSchema = valueSchema; + this.builder = builder; + this.parentSchema = builder.swapSchema(keySchema); + } + + @Override + public void putKey(Object key) + { + this.key = key; + this.builder.setSchema(valueSchema); + } + + @Override + public void putValue(Object value) + { + m.put(this.key, value); + this.key = null; + this.builder.setSchema(keySchema); + } + + @Override + public Object finish() + { + builder.swapSchema(parentSchema); + return m; + } +} + +final class ClassBuilder implements ArrayBuilder { + private Object object; + private int index; + private List fields; + private SpecificObjectBuilder builder; + private Schema parentSchema; + + ClassBuilder(int length, ClassSchema schema, SpecificObjectBuilder builder) + { + this.object = schema.newInstance(); + this.index = 0; + this.fields = schema.getFields(); + this.builder = builder; + this.parentSchema = builder.swapSchema(fields.get(0).getType()); + // FIXME check length + } + + @Override + public void add(Object element) + { + FieldSchema f = fields.get(index++); // FIXME check fields.size + f.setFieldValue(object, element); // XXX FIXME debug + if(fields.size() > index) { + builder.setSchema( fields.get(index).getType() ); + } else { + builder.setSchema( null ); + // FIXME: builder.setSchema(new InvalidFieldSchema); + } + } + + @Override + public Object finish() + { + builder.swapSchema(parentSchema); + return object; + } +} + diff --git a/java-plan2/src/org/msgpack/impl/UnpackerImpl.java b/java-plan2/src/org/msgpack/impl/UnpackerImpl.java new file mode 100644 index 0000000..9f88072 --- /dev/null +++ b/java-plan2/src/org/msgpack/impl/UnpackerImpl.java @@ -0,0 +1,395 @@ +package org.msgpack.impl; + +import java.nio.ByteBuffer; +//import java.math.BigInteger; +import org.msgpack.UnpackException; + +public class UnpackerImpl { + static final int CS_HEADER = 0x00; + static final int CS_FLOAT = 0x0a; + static final int CS_DOUBLE = 0x0b; + static final int CS_UINT_8 = 0x0c; + static final int CS_UINT_16 = 0x0d; + static final int CS_UINT_32 = 0x0e; + static final int CS_UINT_64 = 0x0f; + static final int CS_INT_8 = 0x10; + static final int CS_INT_16 = 0x11; + static final int CS_INT_32 = 0x12; + static final int CS_INT_64 = 0x13; + static final int CS_RAW_16 = 0x1a; + static final int CS_RAW_32 = 0x1b; + static final int CS_ARRAY_16 = 0x1c; + static final int CS_ARRAY_32 = 0x1d; + static final int CS_MAP_16 = 0x1e; + static final int CS_MAP_32 = 0x1f; + static final int ACS_RAW_VALUE = 0x20; + static final int CT_ARRAY_ITEM = 0x00; + static final int CT_MAP_KEY = 0x01; + static final int CT_MAP_VALUE = 0x02; + + static final int MAX_STACK_SIZE = 16; + + protected int cs = CS_HEADER; + protected int trail = 0; + protected int top = -1; + protected boolean finished = false; + protected Object data = null; + protected int[] stack_ct = new int[MAX_STACK_SIZE]; + protected int[] stack_count = new int[MAX_STACK_SIZE]; + protected Object[] stack_obj = new Object[MAX_STACK_SIZE]; + protected ByteBuffer castBuffer = ByteBuffer.allocate(8); + protected ObjectBuilder builder; + + protected UnpackerImpl(ObjectBuilder builder) + { + this.builder = builder; + } + + protected void setBuilder(ObjectBuilder builder) + { + this.builder = builder; + } + + protected Object getData() + { + return data; + } + + protected boolean isFinished() + { + return finished; + } + + protected void reset() + { + for(int i=0; i <= top; ++top) { + stack_ct[top] = 0; + stack_count[top] = 0; + stack_obj[top] = null; + } + cs = CS_HEADER; + trail = 0; + top = -1; + finished = false; + data = null; + } + + @SuppressWarnings("unchecked") + protected int execute(byte[] src, int off, int length) throws UnpackException + { + if(off >= length) { return off; } + + int limit = length; + int i = off; + int count; + + Object obj = null; + + _out: do { + _header_again: { + //System.out.println("while i:"+i+" limit:"+limit); + + int b = src[i]; + + _push: { + _fixed_trail_again: + if(cs == CS_HEADER) { + + if((b & 0x80) == 0) { // Positive Fixnum + //System.out.println("positive fixnum "+b); + obj = builder.createByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xe0) { // Negative Fixnum + //System.out.println("negative fixnum "+b); + obj = builder.createByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xa0) { // FixRaw + trail = b & 0x1f; + if(trail == 0) { + obj = builder.createRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + } + + if((b & 0xf0) == 0x90) { // FixArray + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + count = b & 0x0f; + ++top; + stack_obj[top] = builder.createArray(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + //System.out.println("fixarray count:"+count); + break _header_again; + } + + if((b & 0xf0) == 0x80) { // FixMap + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + count = b & 0x0f; + ++top; + stack_obj[top] = builder.createMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + } + + switch(b & 0xff) { // FIXME + case 0xc0: // nil + obj = builder.createNil(); + break _push; + case 0xc2: // false + obj = builder.createBoolean(false); + break _push; + case 0xc3: // true + obj = builder.createBoolean(true); + break _push; + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + trail = 1 << (b & 0x03); + cs = b & 0x1f; + //System.out.println("a trail "+trail+" cs:"+cs); + break _fixed_trail_again; + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + trail = 2 << (b & 0x01); + cs = b & 0x1f; + //System.out.println("b trail "+trail+" cs:"+cs); + break _fixed_trail_again; + default: + //System.out.println("unknown b "+(b&0xff)); + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + + } // _fixed_trail_again + + do { + _fixed_trail_again: { + + if(limit - i <= trail) { break _out; } + int n = i + 1; + i += trail; + + switch(cs) { + case CS_FLOAT: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = builder.createFloat( castBuffer.getFloat(0) ); + //System.out.println("float "+obj); + break _push; + case CS_DOUBLE: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = builder.createDouble( castBuffer.getDouble(0) ); + //System.out.println("double "+obj); + break _push; + case CS_UINT_8: + //System.out.println(n); + //System.out.println(src[n]); + //System.out.println(src[n+1]); + //System.out.println(src[n-1]); + obj = builder.createShort( (short)((src[n]) & 0xff) ); + //System.out.println("uint8 "+obj); + break _push; + case CS_UINT_16: + //System.out.println(src[n]); + //System.out.println(src[n+1]); + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = builder.createInt( ((int)castBuffer.getShort(0)) & 0xffff ); + //System.out.println("uint 16 "+obj); + break _push; + case CS_UINT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = builder.createLong( ((long)castBuffer.getInt(0)) & 0xffffffffL ); + //System.out.println("uint 32 "+obj); + break _push; + case CS_UINT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + long o = castBuffer.getLong(0); + if(o < 0) { + // FIXME + //obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31); + } else { + obj = builder.createLong( o ); + } + } + throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported", UnpackException.PARSE_ERROR); + case CS_INT_8: + obj = builder.createByte( src[n] ); + break _push; + case CS_INT_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = builder.createShort( castBuffer.getShort(0) ); + break _push; + case CS_INT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = builder.createInt( castBuffer.getInt(0) ); + break _push; + case CS_INT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = builder.createLong( castBuffer.getLong(0) ); + break _push; + case CS_RAW_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + trail = ((int)castBuffer.getShort(0)) & 0xffff; + if(trail == 0) { + obj = builder.createRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + case CS_RAW_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + trail = castBuffer.getInt(0) & 0x7fffffff; + if(trail == 0) { + obj = builder.createRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + case ACS_RAW_VALUE: + obj = builder.createRaw(src, n, trail); + break _push; + case CS_ARRAY_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = builder.createArray(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_ARRAY_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = builder.createArray(count); + stack_ct[top] = CT_ARRAY_ITEM; + stack_count[top] = count; + break _header_again; + case CS_MAP_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + ++top; + stack_obj[top] = builder.createMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + case CS_MAP_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + ++top; + stack_obj[top] = builder.createMap(count); + stack_ct[top] = CT_MAP_KEY; + stack_count[top] = count; + //System.out.println("fixmap count:"+count); + break _header_again; + default: + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + + } // _fixed_trail_again + } while(true); + } // _push + + do { + _push: { + //System.out.println("push top:"+top); + if(top == -1) { + ++i; + data = obj; + finished = true; + break _out; + } + + switch(stack_ct[top]) { + case CT_ARRAY_ITEM: + //System.out.println("array item "+obj); + ((ArrayBuilder)stack_obj[top]).add(obj); + if(--stack_count[top] == 0) { + obj = ((ArrayBuilder)stack_obj[top]).finish(); + stack_obj[top] = null; + --top; + break _push; + } + break _header_again; + case CT_MAP_KEY: + //System.out.println("map key:"+top+" "+obj); + MapBuilder mb = (MapBuilder)stack_obj[top]; + mb.putKey(obj); + stack_ct[top] = CT_MAP_VALUE; + break _header_again; + case CT_MAP_VALUE: + //System.out.println("map value:"+top+" "+obj); + ((MapBuilder)stack_obj[top]).putValue(obj); + if(--stack_count[top] == 0) { + obj = ((MapBuilder)stack_obj[top]).finish(); + stack_obj[top] = null; + --top; + break _push; + } + stack_ct[top] = CT_MAP_KEY; + break _header_again; + default: + throw new UnpackException("parse error", UnpackException.PARSE_ERROR); + } + } // _push + } while(true); + + } // _header_again + cs = CS_HEADER; + ++i; + } while(i < limit); // _out + + return i; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ArraySchema.java b/java-plan2/src/org/msgpack/schema/ArraySchema.java new file mode 100644 index 0000000..4b05190 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ArraySchema.java @@ -0,0 +1,61 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.ArrayList; +import java.io.IOException; +import org.msgpack.*; + +public class ArraySchema extends Schema { + private Schema elementType; + + public ArraySchema(Schema elementType) + { + super("array"); + this.elementType = elementType; + } + + public Schema getElementType() + { + return elementType; + } + + @Override + public String getFullName() + { + return "ArrayList<"+elementType.getFullName()+">"; + } + + @Override + public String getExpression() + { + return "(array "+elementType.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + List d = (List)obj; + pk.packArray(d.size()); + for(Object e : d) { + elementType.pack(pk, e); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + List d = obj.asArray(); + List g = new ArrayList(); + for(GenericObject o : d) { + g.add( elementType.convert(o) ); + } + return g; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ClassGenerator.java b/java-plan2/src/org/msgpack/schema/ClassGenerator.java new file mode 100644 index 0000000..25a9620 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ClassGenerator.java @@ -0,0 +1,230 @@ +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.List; +import java.io.IOException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.Writer; + +public class ClassGenerator { + private ClassSchema schema; + private Writer writer; + private int indent; + + private ClassGenerator(Writer writer) + { + this.writer = writer; + this.indent = 0; + } + + public static void write(Schema schema, Writer dest) throws IOException + { + if(!(schema instanceof ClassSchema)) { + throw new RuntimeException("schema is not class schema"); + } + ClassSchema cs = (ClassSchema)schema; + new ClassGenerator(dest).run(cs); + } + + private void run(ClassSchema cs) throws IOException + { + List subclasses = new ArrayList(); + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(subclasses, f.getType()); + } + + for(ClassSchema sub : subclasses) { + sub.setNamespace(cs.getNamespace()); + sub.setImports(cs.getImports()); + } + + this.schema = cs; + + writeHeader(); + + writeClass(); + + for(ClassSchema sub : subclasses) { + this.schema = sub; + writeSubclass(); + } + + writeFooter(); + + this.schema = null; + writer.flush(); + } + + private void findSubclassSchema(List dst, Schema s) + { + if(s instanceof ClassSchema) { + ClassSchema cs = (ClassSchema)s; + if(!dst.contains(cs)) { dst.add(cs); } + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(dst, f.getType()); + } + } else if(s instanceof ArraySchema) { + ArraySchema as = (ArraySchema)s; + findSubclassSchema(dst, as.getElementType()); + } else if(s instanceof MapSchema) { + MapSchema as = (MapSchema)s; + findSubclassSchema(dst, as.getKeyType()); + findSubclassSchema(dst, as.getValueType()); + } + } + + private void writeHeader() throws IOException + { + if(schema.getNamespace() != null) { + line("package "+schema.getNamespace()+";"); + line(); + } + line("import java.util.*;"); + line("import java.io.*;"); + line("import org.msgpack.*;"); + line("import org.msgpack.schema.*;"); + } + + private void writeFooter() throws IOException + { + } + + private void writeClass() throws IOException + { + line(); + line("public final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSubclass() throws IOException + { + line(); + line("final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSchema() throws IOException + { + line("private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load(\""+schema.getExpression()+"\");"); + line("public static ClassSchema getSchema() { return _SCHEMA; }"); + } + + private void writeMemberVariables() throws IOException + { + line(); + for(FieldSchema f : schema.getFields()) { + line("public "+f.getType().getFullName()+" "+f.getName()+";"); + } + } + + private void writeMemberFunctions() throws IOException + { + // void messagePack(Packer pk) + // boolean equals(Object obj) + // int hashCode() + // void set(int _index, Object _value) + // Object get(int _index); + // getXxx() + // setXxx(Xxx xxx) + writeConstructors(); + writeAccessors(); + writePackFunction(); + writeConvertFunction(); + } + + private void writeConstructors() throws IOException + { + line(); + line("public "+schema.getName()+"() { }"); + } + + private void writeAccessors() throws IOException + { + // FIXME + //line(); + //for(FieldSchema f : schema.getFields()) { + // line(""); + //} + } + + private void writePackFunction() throws IOException + { + line(); + line("@Override"); + line("public void messagePack(Packer pk) throws IOException"); + line("{"); + pushIndent(); + line("List _f = _SCHEMA.getFields();"); + line("pk.packArray("+schema.getFields().size()+");"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("_f.get("+i+").getType().pack(pk, "+f.getName()+");"); + ++i; + } + popIndent(); + line("}"); + } + + private void writeConvertFunction() throws IOException + { + line(); + line("@Override"); + line("@SuppressWarnings(\"unchecked\")"); + line("public void messageConvert(GenericObject obj)"); + line("{"); + pushIndent(); + line("List _l = obj.asArray();"); + line("List _f = _SCHEMA.getFields();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("if(_l.size() <= "+i+") { return; } "+f.getName()+" = ("+f.getType().getFullName()+")_f.get("+i+").getType().convert(_l.get("+i+"));"); + ++i; + } + popIndent(); + line("}"); + line(); + line("public static "+schema.getName()+" convert(GenericObject obj)"); + line("{"); + pushIndent(); + line("return ("+schema.getName()+")_SCHEMA.convert(obj);"); + popIndent(); + line("}"); + } + + private void line(String str) throws IOException + { + for(int i=0; i < indent; ++i) { + writer.write("\t"); + } + writer.write(str+"\n"); + } + + private void line() throws IOException + { + writer.write("\n"); + } + + private void pushIndent() + { + indent += 1; + } + + private void popIndent() + { + indent -= 1; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ClassSchema.java b/java-plan2/src/org/msgpack/schema/ClassSchema.java new file mode 100644 index 0000000..3343fca --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ClassSchema.java @@ -0,0 +1,39 @@ +package org.msgpack.schema; + +import java.util.List; + +public abstract class ClassSchema extends Schema { + protected String namespace; + protected List imports; + + public ClassSchema(String name, String namespace, List imports) + { + super(name); + this.namespace = namespace; + this.imports = imports; + } + + public String getNamespace() + { + return namespace; + } + + public List getImports() + { + return imports; + } + + void setNamespace(String namespace) + { + this.namespace = namespace; + } + + void setImports(List imports) + { + this.imports = imports; + } + + public abstract List getFields(); + public abstract Object newInstance(); +} + diff --git a/java-plan2/src/org/msgpack/schema/FieldSchema.java b/java-plan2/src/org/msgpack/schema/FieldSchema.java new file mode 100644 index 0000000..31a132c --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/FieldSchema.java @@ -0,0 +1,31 @@ +package org.msgpack.schema; + +public abstract class FieldSchema { + private String name; + private Schema type; + + public FieldSchema(String name, Schema type) + { + this.name = name; + this.type = type; + } + + public String getName() + { + return this.name; + } + + public Schema getType() + { + return type; + } + + public String getExpression() + { + return "(field "+name+" "+type.getExpression()+")"; + } + + public abstract Object getFieldValue(Object obj); + public abstract void setFieldValue(Object obj, Object value); +} + diff --git a/java-plan2/src/org/msgpack/schema/GenericClassSchema.java b/java-plan2/src/org/msgpack/schema/GenericClassSchema.java new file mode 100644 index 0000000..f1e2b44 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/GenericClassSchema.java @@ -0,0 +1,89 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import org.msgpack.*; + + +public class GenericClassSchema extends ClassSchema { + private List fields; + + private String fqdn; + private Constructor constructorCache; + + public GenericClassSchema(String name, List fields, String namespace, List imports) + { + super(name, namespace, imports); + this.fields = fields; + if(namespace == null) { + this.fqdn = name; + } else { + this.fqdn = namespace+"."+name; + } + } + + //@Override + //public String getFullName() + //{ + // if(namespace == null) { + // return getName(); + // } else { + // return namespace+"."+getName(); + // } + //} + + public List getFields() + { + return fields; + } + + public String getNamespace() + { + return namespace; + } + + @Override + public String getExpression() + { + StringBuffer b = new StringBuffer(); + b.append("(class "); + b.append(getName()); + if(namespace != null) { + b.append(" (package "+namespace+")"); + } + for(GenericFieldSchema f : fields) { + b.append(" "+f.getExpression()); + } + b.append(")"); + return b.toString(); + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + // FIXME + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + // FIXME + return obj; + } + + @Override + public Object newInstance() + { + return new HashMap(fields.size()); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java b/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java new file mode 100644 index 0000000..507ee18 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java @@ -0,0 +1,25 @@ +package org.msgpack.schema; + +import java.util.Map; +import java.lang.reflect.Field; + +public final class GenericFieldSchema extends FieldSchema { + public GenericFieldSchema(String name, Schema type) + { + super(name, type); + } + + @Override + public Object getFieldValue(Object obj) + { + return ((Map)obj).get(getName()); + } + + @Override + @SuppressWarnings("unchecked") + public void setFieldValue(Object obj, Object value) + { + ((Map)obj).put(getName(), value); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/IntSchema.java b/java-plan2/src/org/msgpack/schema/IntSchema.java new file mode 100644 index 0000000..69771c3 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/IntSchema.java @@ -0,0 +1,58 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class IntSchema extends Schema { + public IntSchema() + { + super("Integer"); + } + + @Override + public String getExpression() + { + return "int"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.packInt((Integer)obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asInt(); + } + + @Override + public Object createByte(byte v) + { + return (int)v; + } + + @Override + public Object createShort(short v) + { + return (int)v; + } + + @Override + public Object createInt(int v) + { + return (int)v; + } + + @Override + public Object createLong(long v) + { + return (int)v; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/LongSchema.java b/java-plan2/src/org/msgpack/schema/LongSchema.java new file mode 100644 index 0000000..0ba3057 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/LongSchema.java @@ -0,0 +1,58 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class LongSchema extends Schema { + public LongSchema() + { + super("Long"); + } + + @Override + public String getExpression() + { + return "long"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.packLong((Long)obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asLong(); + } + + @Override + public Object createByte(byte v) + { + return (long)v; + } + + @Override + public Object createShort(short v) + { + return (long)v; + } + + @Override + public Object createInt(int v) + { + return (long)v; + } + + @Override + public Object createLong(long v) + { + return (long)v; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/MapSchema.java b/java-plan2/src/org/msgpack/schema/MapSchema.java new file mode 100644 index 0000000..e72cf63 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/MapSchema.java @@ -0,0 +1,70 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; + +public class MapSchema extends Schema { + private Schema keyType; + private Schema valueType; + + public MapSchema(Schema keyType, Schema valueType) + { + super("map"); + this.keyType = keyType; + this.valueType = valueType; + } + + public Schema getKeyType() + { + return keyType; + } + + public Schema getValueType() + { + return valueType; + } + + @Override + public String getFullName() + { + return "HashList<"+keyType.getFullName()+", "+valueType.getFullName()+">"; + } + + @Override + public String getExpression() + { + return "(map "+keyType.getExpression()+" "+valueType.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + Map d = (Map)obj; + pk.packMap(d.size()); + for(Map.Entry e : d.entrySet()) { + keyType.pack(pk, e.getKey()); + valueType.pack(pk, e.getValue()); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + Map d = obj.asMap(); + Map g = new HashMap(); + for(Map.Entry e : d.entrySet()) { + g.put(keyType.convert(e.getKey()), valueType.convert(e.getValue())); + } + return g; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ObjectSchema.java b/java-plan2/src/org/msgpack/schema/ObjectSchema.java new file mode 100644 index 0000000..a9f30f5 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ObjectSchema.java @@ -0,0 +1,33 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ObjectSchema extends Schema { + public ObjectSchema() + { + super("object"); + } + + public String getFullName() + { + return "GenericObject"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.pack(obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java b/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java new file mode 100644 index 0000000..023d81b --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java @@ -0,0 +1,21 @@ +package org.msgpack.schema; + +public abstract class PrimitiveSchema extends Schema { + public static enum PrimitiveType { + BYTE, + SHORT, + INT, + LONG, + FLOAT, + DOUBLE, + } + + public final PrimitiveType type; + + protected PrimitiveSchema(String name, PrimitiveType type) + { + super(name); + this.type = type; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/RawSchema.java b/java-plan2/src/org/msgpack/schema/RawSchema.java new file mode 100644 index 0000000..847ad29 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/RawSchema.java @@ -0,0 +1,44 @@ +package org.msgpack.schema; + +import java.io.IOException; +import java.nio.charset.Charset; +import org.msgpack.*; + +public class RawSchema extends Schema { + public RawSchema() + { + super("raw"); + } + + public String getFullName() + { + return "byte[]"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + byte[] d = (byte[])obj; + pk.packRaw(d.length); + pk.packRawBody(d); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asBytes(); + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + byte[] d = new byte[length]; + System.arraycopy(b, offset, d, 0, length); + return d; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/SSchemaParser.java b/java-plan2/src/org/msgpack/schema/SSchemaParser.java new file mode 100644 index 0000000..bfe912f --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/SSchemaParser.java @@ -0,0 +1,246 @@ +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Stack; +import java.util.regex.Pattern; +import java.util.regex.Matcher; + +// FIXME exception class + +class SSchemaParser { + public static Schema parse(String source) + { + return new SSchemaParser(false).run(source); + } + + public static Schema load(String source) + { + return new SSchemaParser(true).run(source); + } + + private static abstract class SExp { + boolean isAtom() { return false; } + public String getAtom() { return null; } + + boolean isTuple() { return false; } + public SExp getTuple(int i) { return null; } + public int size() { return 0; } + public boolean empty() { return size() == 0; } + Iterator iterator(int offset) { return null; } + } + + private static class SAtom extends SExp { + private String atom; + + SAtom(String atom) { this.atom = atom; } + + boolean isAtom() { return true; } + public String getAtom() { return atom; } + + public String toString() { return atom; } + } + + private static class STuple extends SExp { + private List tuple; + + STuple() { this.tuple = new ArrayList(); } + + public void add(SExp e) { tuple.add(e); } + + boolean isTuple() { return true; } + public SExp getTuple(int i) { return tuple.get(i); } + public int size() { return tuple.size(); } + + Iterator iterator(int skip) { + Iterator i = tuple.iterator(); + for(int s=0; s < skip; ++s) { i.next(); } + return i; + } + + public String toString() { + if(tuple.isEmpty()) { return "()"; } + Iterator i = tuple.iterator(); + StringBuffer o = new StringBuffer(); + o.append("(").append(i.next()); + while(i.hasNext()) { o.append(" ").append(i.next()); } + o.append(")"); + return o.toString(); + } + } + + boolean specificClass; + + private SSchemaParser(boolean specificClass) + { + this.specificClass = specificClass; + } + + private static Pattern pattern = Pattern.compile( + "(?:\\s+)|([\\(\\)]|[\\d\\w\\.]+)"); + + private Schema run(String source) + { + Matcher m = pattern.matcher(source); + + Stack stack = new Stack(); + String token; + + while(true) { + while(true) { + if(!m.find()) { throw new RuntimeException("unexpected end of file"); } + token = m.group(1); + if(token != null) { break; } + } + + if(token.equals("(")) { + stack.push(new STuple()); + } else if(token.equals(")")) { + STuple top = stack.pop(); + if(stack.empty()) { + stack.push(top); + break; + } + stack.peek().add(top); + } else { + if(stack.empty()) { + throw new RuntimeException("unexpected token '"+token+"'"); + } + stack.peek().add(new SAtom(token)); + } + } + + while(true) { + if(!m.find()) { break; } + token = m.group(1); + if(token != null) { throw new RuntimeException("unexpected token '"+token+"'"); } + } + + return readType( stack.pop() ); + } + + private Schema readType(SExp exp) + { + if(exp.isAtom()) { + String type = exp.getAtom(); + // FIXME + if(type.equals("string")) { + return new StringSchema(); + } else if(type.equals("raw")) { + return new RawSchema(); + } else if(type.equals("short")) { + return new ShortSchema(); + } else if(type.equals("int")) { + return new IntSchema(); + } else if(type.equals("long")) { + return new LongSchema(); + } else if(type.equals("object")) { + return new ObjectSchema(); + } else { + throw new RuntimeException("byte, short, int, long, float, double, raw, string or object is expected but got '"+type+"': "+exp); + } + } else { + String type = exp.getTuple(0).getAtom(); + if(type.equals("class")) { + return parseClass(exp); + } else if(type.equals("array")) { + return parseArray(exp); + } else if(type.equals("map")) { + return parseMap(exp); + } else { + throw new RuntimeException("class, array or map is expected but got '"+type+"': "+exp); + } + } + } + + private ClassSchema parseClass(SExp exp) + { + if(exp.size() < 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("class is (class NAME CLASS_BODY): "+exp); + } + + String namespace = null; + List imports = new ArrayList(); + String name = exp.getTuple(1).getAtom(); + List fields = new ArrayList(); + + for(Iterator i=exp.iterator(2); i.hasNext();) { + SExp subexp = i.next(); + if(!subexp.isTuple() || subexp.empty() || !subexp.getTuple(0).isAtom()) { + throw new RuntimeException("field, package or import is expected: "+subexp); + } + String type = subexp.getTuple(0).getAtom(); + if(type.equals("field")) { + fields.add( parseField(subexp) ); + } else if(type.equals("package")) { + if(namespace != null) { + throw new RuntimeException("duplicated package definition: "+subexp); + } + namespace = parseNamespace(subexp); + } else if(type.equals("import")) { + imports.add( parseImport(subexp) ); + } else { + throw new RuntimeException("field, package or import is expected but got '"+type+"': "+subexp); + } + } + + if(specificClass) { + return new SpecificClassSchema(name, fields, namespace, imports); + } else { + return new GenericClassSchema(name, fields, namespace, imports); + } + } + + private ArraySchema parseArray(SExp exp) + { + if(exp.size() != 2) { + throw new RuntimeException("array is (array ELEMENT_TYPE): "+exp); + } + Schema elementType = readType(exp.getTuple(1)); + return new ArraySchema(elementType); + } + + private MapSchema parseMap(SExp exp) + { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("map is (map KEY_TYPE VALUE_TYPE): "+exp); + } + Schema keyType = readType(exp.getTuple(1)); + Schema valueType = readType(exp.getTuple(2)); + return new MapSchema(keyType, valueType); + } + + private String parseNamespace(SExp exp) + { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("package is (package NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private String parseImport(SExp exp) + { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("import is (import NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private FieldSchema parseField(SExp exp) + { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("field is (field NAME TYPE): "+exp); + } + String name = exp.getTuple(1).getAtom(); + Schema type = readType(exp.getTuple(2)); + if(specificClass) { + return new SpecificFieldSchema(name, type); + } else { + return new GenericFieldSchema(name, type); + } + } +} + diff --git a/java-plan2/src/org/msgpack/schema/Schema.java b/java-plan2/src/org/msgpack/schema/Schema.java new file mode 100644 index 0000000..15b7e72 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/Schema.java @@ -0,0 +1,93 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.impl.*; +import org.msgpack.Packer; +import org.msgpack.GenericObject; + +public abstract class Schema { + private String expression; + private String name; + + public Schema(String name) + { + this.expression = expression; + this.name = name; + } + + public String getName() + { + return name; + } + + public String getFullName() + { + return name; + } + + public String getExpression() + { + return name; + } + + public static Schema parse(String source) + { + return SSchemaParser.parse(source); + } + + public static Schema load(String source) + { + return SSchemaParser.load(source); + } + + public abstract void pack(Packer pk, Object obj) throws IOException; + public abstract Object convert(GenericObject obj); + //public abstract Object convertGeneric(GenericObject obj); + + + public Object createNil() + { + return null; + } + + public Object createBoolean(boolean v) + { + throw new RuntimeException("type error"); + } + + public Object createByte(byte v) + { + throw new RuntimeException("type error"); + } + + public Object createShort(short v) + { + throw new RuntimeException("type error"); + } + + public Object createInt(int v) + { + throw new RuntimeException("type error"); + } + + public Object createLong(long v) + { + throw new RuntimeException("type error"); + } + + public Object createFloat(float v) + { + throw new RuntimeException("type error"); + } + + public Object createDouble(double v) + { + throw new RuntimeException("type error"); + } + + public Object createRaw(byte[] b, int offset, int length) + { + throw new RuntimeException("type error"); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/ShortSchema.java b/java-plan2/src/org/msgpack/schema/ShortSchema.java new file mode 100644 index 0000000..aa95f51 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/ShortSchema.java @@ -0,0 +1,46 @@ +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ShortSchema extends Schema { + public ShortSchema() + { + super("Short"); + } + + @Override + public String getExpression() + { + return "short"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + pk.packShort((Short)obj); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asShort(); + } + + @Override + public Object createByte(byte v) + { + return (int)v; + } + + @Override + public Object createShort(short v) + { + return (int)v; + } +} + diff --git a/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java b/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java new file mode 100644 index 0000000..75c474a --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java @@ -0,0 +1,156 @@ +package org.msgpack.schema; + +import java.util.List; +import java.util.Map; +import java.io.IOException; +import java.util.Iterator; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import org.msgpack.*; + +public class SpecificClassSchema extends ClassSchema { + private List fields; + private String namespace; + private List imports; + + private String fqdn; + private Constructor constructorCache; + + public SpecificClassSchema(String name, List fields, String namespace, List imports) + { + super(name, namespace, imports); + this.fields = fields; + if(namespace == null) { + this.fqdn = name; + } else { + this.fqdn = namespace+"."+name; + } + } + + //@Override + //public String getFullName() + //{ + // if(namespace == null) { + // return getName(); + // } else { + // return namespace+"."+getName(); + // } + //} + + public List getFields() + { + return fields; + } + + @Override + public String getExpression() + { + StringBuffer b = new StringBuffer(); + b.append("(class "); + b.append(getName()); + if(namespace != null) { + b.append(" (package "+namespace+")"); + } + for(SpecificFieldSchema f : fields) { + b.append(" "+f.getExpression()); + } + b.append(")"); + return b.toString(); + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + + if(constructorCache == null) { + cacheConstructor(); + } + + pk.packArray(fields.size()); + for(SpecificFieldSchema f : fields) { + f.getType().pack(pk, f.getFieldValue(obj)); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(GenericObject obj) + { + if(constructorCache == null) { + cacheConstructor(); + } + + List d = obj.asArray(); + + try { + Object g = constructorCache.newInstance((Object[])null); + + Iterator vi = d.iterator(); + Iterator fi = fields.iterator(); + while(fi.hasNext() && vi.hasNext()) { + SpecificFieldSchema f = fi.next(); + GenericObject v = vi.next(); + f.setFieldValue(g, f.getType().convert(v)); + } + // leave it as uninitialized + //while(fi.hasNext()) { + // SpecificFieldSchema f = fi.next(); + // g.put(f.getName(), null); + //} + + return g; + + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + private void cacheConstructor() + { + try { + Class c = Class.forName(fqdn); + int index = 0; + for(SpecificFieldSchema f : fields) { + f.cacheField(c, index++); + } + constructorCache = c.getDeclaredConstructor((Class[])null); + constructorCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } + + @Override + public Object newInstance() + { + if(constructorCache == null) { + cacheConstructor(); + } + try { + return constructorCache.newInstance((Object[])null); + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + public boolean equals(SpecificClassSchema o) + { + return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) && + getName().equals(o.getName()); + } +} + diff --git a/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java b/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java new file mode 100644 index 0000000..297df27 --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java @@ -0,0 +1,78 @@ +package org.msgpack.schema; + +import java.util.Map; +import java.util.Arrays; +import java.lang.reflect.Field; +import org.msgpack.*; + +public class SpecificFieldSchema extends FieldSchema { + public Field fieldCache; + private int index; + + public SpecificFieldSchema(String name, Schema type) + { + super(name, type); + this.index = -1; + } + + @Override + public Object getFieldValue(Object obj) + { + if(index >= 0) { + return ((MessageMergeable)obj).getField(index); + } + + try { + return fieldCache.get(obj); + } catch(IllegalArgumentException e) { + throw new RuntimeException("can't get value from '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } catch(IllegalAccessException e) { + throw new RuntimeException("can't get value from '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } + } + + @Override + public void setFieldValue(Object obj, Object value) + { + if(index >= 0) { + ((MessageMergeable)obj).setField(index, value); + return; + } + + try { + fieldCache.set(obj, value); + } catch(IllegalArgumentException e) { + throw new RuntimeException("can't set value into '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } catch(IllegalAccessException e) { + throw new RuntimeException("can't set value into '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); + } + } + + void cacheField(Class c, int index) + { + for(Class i : c.getInterfaces()) { + if(i.equals(MessageMergeable.class)) { + this.index = index; + return; + } + } + + try { + fieldCache = c.getDeclaredField(getName()); + if(!fieldCache.isAccessible()) { + fieldCache.setAccessible(true); + } + } catch(NoSuchFieldException e) { + throw new RuntimeException("can't get '"+getName()+"' field of '"+c.getName()+"' class: "+e.getMessage()); + } catch(SecurityException e) { + throw new RuntimeException("can't get '"+getName()+"' field of '"+c.getName()+"' class: "+e.getMessage()); + } + } + + //public void setFieldInt(Object obj, int value) + //{ + // if(type instanceof PrimitiveSchema) { + // } + //} +} + diff --git a/java-plan2/src/org/msgpack/schema/StringSchema.java b/java-plan2/src/org/msgpack/schema/StringSchema.java new file mode 100644 index 0000000..fc6855b --- /dev/null +++ b/java-plan2/src/org/msgpack/schema/StringSchema.java @@ -0,0 +1,48 @@ +package org.msgpack.schema; + +import java.io.IOException; +import java.nio.charset.Charset; +import org.msgpack.*; + +public class StringSchema extends Schema { + public StringSchema() + { + super("string"); + } + + public String getFullName() + { + return "String"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException + { + if(obj == null) { + pk.packNil(); + return; + } + String s = (String)obj; + byte[] d = s.getBytes("UTF-8"); + pk.packRaw(d.length); + pk.packRawBody(d); + } + + @Override + public Object convert(GenericObject obj) + { + return obj.asString(); + } + + @Override + public Object createRaw(byte[] b, int offset, int length) + { + try { + return new String(b, offset, length, "UTF-8"); // XXX FIXME debug + } catch (Exception e) { + // FIXME + throw new RuntimeException(e.getMessage()); + } + } +} + diff --git a/java-plan2/test/Generate.java b/java-plan2/test/Generate.java new file mode 100644 index 0000000..2ac5878 --- /dev/null +++ b/java-plan2/test/Generate.java @@ -0,0 +1,18 @@ +import java.io.*; +import java.util.*; +import org.msgpack.*; +import org.msgpack.schema.*; + +public class Generate { + public static void main(String[] args) throws IOException + { + Writer output = new OutputStreamWriter(System.out); + + Schema s1 = Schema.parse("(class Test (field uri raw) (field width int))"); + ClassGenerator.write(s1, output); + + Schema s1 = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + ClassGenerator.write(s2, output); + } +} + diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java new file mode 100644 index 0000000..ecb64fd --- /dev/null +++ b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java @@ -0,0 +1,277 @@ +package serializers.msgpack; + +import java.util.*; +import java.io.*; +import org.msgpack.*; +import org.msgpack.schema.*; + +public final class MediaContent implements MessagePackable, MessageConvertable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public ArrayList image; + public Media media; + + public MediaContent() { } + + @Override + public void messagePack(Packer pk) throws IOException + { + List _f = _SCHEMA.getFields(); + pk.packArray(2); + _f.get(0).getType().pack(pk, image); + _f.get(1).getType().pack(pk, media); + } + + @Override + @SuppressWarnings("unchecked") + public void messageConvert(GenericObject obj) + { + List _l = obj.asArray(); + List _f = _SCHEMA.getFields(); + if(_l.size() <= 0) { return; } image = (ArrayList)_f.get(0).getType().convert(_l.get(0)); + if(_l.size() <= 1) { return; } media = (Media)_f.get(1).getType().convert(_l.get(1)); + } + + public static MediaContent convert(GenericObject obj) + { + return (MediaContent)_SCHEMA.convert(obj); + } + + public void setField(int index, Object value) + { + switch(index) { + case 0: + image = (ArrayList)value; + break; + case 1: + media = (Media)value; + break; + } + } + + public Object getField(int index) + { + switch(index) { + case 0: + return image; + case 1: + return media; + } + return null; + } +} + +final class Image implements MessagePackable, MessageConvertable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public Integer size; + + public Image() { } + + @Override + public void messagePack(Packer pk) throws IOException + { + List _f = _SCHEMA.getFields(); + pk.packArray(5); + _f.get(0).getType().pack(pk, uri); + _f.get(1).getType().pack(pk, title); + _f.get(2).getType().pack(pk, width); + _f.get(3).getType().pack(pk, height); + _f.get(4).getType().pack(pk, size); + } + + @Override + @SuppressWarnings("unchecked") + public void messageConvert(GenericObject obj) + { + List _l = obj.asArray(); + List _f = _SCHEMA.getFields(); + if(_l.size() <= 0) { return; } uri = (String)_f.get(0).getType().convert(_l.get(0)); + if(_l.size() <= 1) { return; } title = (String)_f.get(1).getType().convert(_l.get(1)); + if(_l.size() <= 2) { return; } width = (Integer)_f.get(2).getType().convert(_l.get(2)); + if(_l.size() <= 3) { return; } height = (Integer)_f.get(3).getType().convert(_l.get(3)); + if(_l.size() <= 4) { return; } size = (Integer)_f.get(4).getType().convert(_l.get(4)); + } + + public static Image convert(GenericObject obj) + { + return (Image)_SCHEMA.convert(obj); + } + + public void setField(int index, Object value) + { + switch(index) { + case 0: + uri = (String)value; + break; + case 1: + title = (String)value; + break; + case 2: + width = (Integer)value; + break; + case 3: + height = (Integer)value; + break; + case 4: + size = (Integer)value; + break; + } + } + + public Object getField(int index) + { + switch(index) { + case 0: + return uri; + case 1: + return title; + case 2: + return width; + case 3: + return height; + case 4: + return size; + } + return null; + } +} + +final class Media implements MessagePackable, MessageConvertable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public String format; + public Long duration; + public Long size; + public Integer bitrate; + public ArrayList person; + public Integer player; + public String copyright; + + public Media() { } + + @Override + public void messagePack(Packer pk) throws IOException + { + List _f = _SCHEMA.getFields(); + pk.packArray(11); + _f.get(0).getType().pack(pk, uri); + _f.get(1).getType().pack(pk, title); + _f.get(2).getType().pack(pk, width); + _f.get(3).getType().pack(pk, height); + _f.get(4).getType().pack(pk, format); + _f.get(5).getType().pack(pk, duration); + _f.get(6).getType().pack(pk, size); + _f.get(7).getType().pack(pk, bitrate); + _f.get(8).getType().pack(pk, person); + _f.get(9).getType().pack(pk, player); + _f.get(10).getType().pack(pk, copyright); + } + + @Override + @SuppressWarnings("unchecked") + public void messageConvert(GenericObject obj) + { + List _l = obj.asArray(); + List _f = _SCHEMA.getFields(); + if(_l.size() <= 0) { return; } uri = (String)_f.get(0).getType().convert(_l.get(0)); + if(_l.size() <= 1) { return; } title = (String)_f.get(1).getType().convert(_l.get(1)); + if(_l.size() <= 2) { return; } width = (Integer)_f.get(2).getType().convert(_l.get(2)); + if(_l.size() <= 3) { return; } height = (Integer)_f.get(3).getType().convert(_l.get(3)); + if(_l.size() <= 4) { return; } format = (String)_f.get(4).getType().convert(_l.get(4)); + if(_l.size() <= 5) { return; } duration = (Long)_f.get(5).getType().convert(_l.get(5)); + if(_l.size() <= 6) { return; } size = (Long)_f.get(6).getType().convert(_l.get(6)); + if(_l.size() <= 7) { return; } bitrate = (Integer)_f.get(7).getType().convert(_l.get(7)); + if(_l.size() <= 8) { return; } person = (ArrayList)_f.get(8).getType().convert(_l.get(8)); + if(_l.size() <= 9) { return; } player = (Integer)_f.get(9).getType().convert(_l.get(9)); + if(_l.size() <= 10) { return; } copyright = (String)_f.get(10).getType().convert(_l.get(10)); + } + + public static Media convert(GenericObject obj) + { + return (Media)_SCHEMA.convert(obj); + } + + public void setField(int index, Object value) + { + switch(index) { + case 0: + uri = (String)value; + break; + case 1: + title = (String)value; + break; + case 2: + width = (Integer)value; + break; + case 3: + height = (Integer)value; + break; + case 4: + format = (String)value; + break; + case 5: + duration = (Long)value; + break; + case 6: + size = (Long)value; + break; + case 7: + bitrate = (Integer)value; + break; + case 8: + person = (ArrayList)value; + break; + case 9: + player = (Integer)value; + break; + case 10: + copyright = (String)value; + break; + } + } + + public Object getField(int index) + { + switch(index) { + case 0: + return uri; + case 1: + return title; + case 2: + return width; + case 3: + return height; + case 4: + return format; + case 5: + return duration; + case 6: + return size; + case 7: + return bitrate; + case 8: + return person; + case 9: + return player; + case 10: + return copyright; + } + return null; + } + +} diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs new file mode 100644 index 0000000..547ba48 --- /dev/null +++ b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs @@ -0,0 +1,21 @@ +(class MediaContent + (package serializers.msgpack) + (field image (array (class Image + (field uri string) + (field title string) + (field width int) + (field height int) + (field size int)))) + (field media (class Media + (field uri string) + (field title string) + (field width int) + (field height int) + (field format string) + (field duration long) + (field size long) + (field bitrate int) + (field person (array string)) + (field player int) + (field copyright string))) + ) diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java new file mode 100644 index 0000000..acb5580 --- /dev/null +++ b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java @@ -0,0 +1,70 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import java.nio.charset.Charset; + +import org.msgpack.*; +import org.msgpack.schema.*; +import serializers.msgpack.*; + +import serializers.ObjectSerializer; + +public class MessagePackSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-specific"; + } + + public MediaContent create() throws Exception { + Media media = new Media(); + media.uri = "http://javaone.com/keynote.mpg"; + media.format = "video/mpg4"; + media.title = "Javaone Keynote"; + media.duration = 1234567L; + media.bitrate = 0; + media.person = new ArrayList(2); + media.person.add("Bill Gates"); + media.person.add("Steve Jobs"); + media.player = 0; + media.height = 0; + media.width = 0; + media.size = 123L; + media.copyright = ""; + + Image image1 = new Image(); + image1.uri = "http://javaone.com/keynote_large.jpg"; + image1.width = 0; + image1.height = 0; + image1.size = 2; + image1.title = "Javaone Keynote"; + + Image image2 = new Image(); + image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; + image2.width = 0; + image2.height = 0; + image2.size = 1; + image2.title = "Javaone Keynote"; + + MediaContent content = new MediaContent(); + content.media = media; + content.image = new ArrayList(2); + content.image.add(image1); + content.image.add(image2); + return content; + } + + public MediaContent deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MediaContent.getSchema()); + pac.execute(array); + return (MediaContent)pac.getData(); + } + + public byte[] serialize(MediaContent content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } +} + From eb9e89249137cbed0ace62de9902e69bd082b2a3 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 26 Nov 2009 11:22:08 +0900 Subject: [PATCH 0314/1648] import MessagePack for Java implementation plan 3 --- java-plan2/test/Generate.java | 2 +- java-plan3/build.xml | 15 + .../src/org/msgpack/MessageMergeable.java | 23 + .../src/org/msgpack/MessagePackable.java | 25 + .../src/org/msgpack/MessageTypeException.java | 39 ++ java-plan3/src/org/msgpack/Packer.java | 409 +++++++++++++++ java-plan3/src/org/msgpack/Schema.java | 133 +++++ .../src/org/msgpack/UnbufferedUnpacker.java | 82 +++ .../src/org/msgpack/UnpackException.java | 29 ++ .../src/org/msgpack/UnpackIterator.java | 66 +++ java-plan3/src/org/msgpack/Unpacker.java | 245 +++++++++ .../src/org/msgpack/impl/UnpackerImpl.java | 483 ++++++++++++++++++ .../src/org/msgpack/schema/ArraySchema.java | 125 +++++ .../src/org/msgpack/schema/ByteSchema.java | 89 ++++ .../org/msgpack/schema/ClassGenerator.java | 241 +++++++++ .../src/org/msgpack/schema/ClassSchema.java | 95 ++++ .../src/org/msgpack/schema/DoubleSchema.java | 84 +++ .../src/org/msgpack/schema/FieldSchema.java | 43 ++ .../src/org/msgpack/schema/FloatSchema.java | 84 +++ .../msgpack/schema/GenericClassSchema.java | 91 ++++ .../src/org/msgpack/schema/GenericSchema.java | 192 +++++++ .../src/org/msgpack/schema/IArraySchema.java | 26 + .../src/org/msgpack/schema/IMapSchema.java | 27 + .../src/org/msgpack/schema/IntSchema.java | 89 ++++ .../src/org/msgpack/schema/LongSchema.java | 89 ++++ .../src/org/msgpack/schema/MapSchema.java | 102 ++++ .../src/org/msgpack/schema/RawSchema.java | 105 ++++ .../msgpack/schema/ReflectionClassSchema.java | 64 +++ .../src/org/msgpack/schema/SSchemaParser.java | 254 +++++++++ .../src/org/msgpack/schema/ShortSchema.java | 89 ++++ .../msgpack/schema/SpecificClassSchema.java | 122 +++++ .../src/org/msgpack/schema/StringSchema.java | 111 ++++ java-plan3/test/Generate.java | 38 ++ .../src/serializers/msgpack/MediaContent.java | 173 +++++++ .../serializers/msgpack/MediaContent.mpacs | 21 + .../msgpack/MessagePackDynamicSerializer.java | 68 +++ .../msgpack/MessagePackGenericSerializer.java | 70 +++ .../MessagePackIndirectSerializer.java | 67 +++ .../MessagePackSpecificSerializer.java | 66 +++ 39 files changed, 4175 insertions(+), 1 deletion(-) create mode 100644 java-plan3/build.xml create mode 100644 java-plan3/src/org/msgpack/MessageMergeable.java create mode 100644 java-plan3/src/org/msgpack/MessagePackable.java create mode 100644 java-plan3/src/org/msgpack/MessageTypeException.java create mode 100644 java-plan3/src/org/msgpack/Packer.java create mode 100644 java-plan3/src/org/msgpack/Schema.java create mode 100644 java-plan3/src/org/msgpack/UnbufferedUnpacker.java create mode 100644 java-plan3/src/org/msgpack/UnpackException.java create mode 100644 java-plan3/src/org/msgpack/UnpackIterator.java create mode 100644 java-plan3/src/org/msgpack/Unpacker.java create mode 100644 java-plan3/src/org/msgpack/impl/UnpackerImpl.java create mode 100644 java-plan3/src/org/msgpack/schema/ArraySchema.java create mode 100644 java-plan3/src/org/msgpack/schema/ByteSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/ClassGenerator.java create mode 100644 java-plan3/src/org/msgpack/schema/ClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/DoubleSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/FieldSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/FloatSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/GenericClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/GenericSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/IArraySchema.java create mode 100644 java-plan3/src/org/msgpack/schema/IMapSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/IntSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/LongSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/MapSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/RawSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/SSchemaParser.java create mode 100644 java-plan3/src/org/msgpack/schema/ShortSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/SpecificClassSchema.java create mode 100644 java-plan3/src/org/msgpack/schema/StringSchema.java create mode 100644 java-plan3/test/Generate.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java create mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java diff --git a/java-plan2/test/Generate.java b/java-plan2/test/Generate.java index 2ac5878..6b7800b 100644 --- a/java-plan2/test/Generate.java +++ b/java-plan2/test/Generate.java @@ -11,7 +11,7 @@ public class Generate { Schema s1 = Schema.parse("(class Test (field uri raw) (field width int))"); ClassGenerator.write(s1, output); - Schema s1 = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + Schema s2 = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); ClassGenerator.write(s2, output); } } diff --git a/java-plan3/build.xml b/java-plan3/build.xml new file mode 100644 index 0000000..598a853 --- /dev/null +++ b/java-plan3/build.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/java-plan3/src/org/msgpack/MessageMergeable.java b/java-plan3/src/org/msgpack/MessageMergeable.java new file mode 100644 index 0000000..e11119c --- /dev/null +++ b/java-plan3/src/org/msgpack/MessageMergeable.java @@ -0,0 +1,23 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +public interface MessageMergeable { + public void messageMerge(Object obj) throws MessageTypeException; +} + diff --git a/java-plan3/src/org/msgpack/MessagePackable.java b/java-plan3/src/org/msgpack/MessagePackable.java new file mode 100644 index 0000000..d8a7db9 --- /dev/null +++ b/java-plan3/src/org/msgpack/MessagePackable.java @@ -0,0 +1,25 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; + +public interface MessagePackable { + public void messagePack(Packer pk) throws IOException; +} + diff --git a/java-plan3/src/org/msgpack/MessageTypeException.java b/java-plan3/src/org/msgpack/MessageTypeException.java new file mode 100644 index 0000000..09031b2 --- /dev/null +++ b/java-plan3/src/org/msgpack/MessageTypeException.java @@ -0,0 +1,39 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; + +public class MessageTypeException extends IOException { + public MessageTypeException() { } + + public MessageTypeException(String s) { + super(s); + } + + public static MessageTypeException invalidConvert(Object from, Schema to) { + return new MessageTypeException(from.getClass().getName()+" cannot be convert to "+to.getExpression()); + } + + /* FIXME + public static MessageTypeException schemaMismatch(Schema to) { + return new MessageTypeException("schema mismatch "+to.getExpression()); + } + */ +} + diff --git a/java-plan3/src/org/msgpack/Packer.java b/java-plan3/src/org/msgpack/Packer.java new file mode 100644 index 0000000..7f2508c --- /dev/null +++ b/java-plan3/src/org/msgpack/Packer.java @@ -0,0 +1,409 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.OutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Map; + +public class Packer { + protected byte[] castBytes = new byte[9]; + protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); + protected OutputStream out; + + public Packer(OutputStream out) { + this.out = out; + } + + public Packer packByte(byte d) throws IOException { + if(d < -(1<<5)) { + castBytes[0] = (byte)0xd1; + castBytes[1] = d; + out.write(castBytes, 0, 2); + } else { + out.write(d); + } + return this; + } + + public Packer packShort(short d) throws IOException { + if(d < -(1<<5)) { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, d); + out.write(castBytes, 0, 3); + } + } + return this; + } + + public Packer packInt(int d) throws IOException { + if(d < -(1<<5)) { + if(d < -(1<<15)) { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } else if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else if(d < (1<<16)) { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, d); + out.write(castBytes, 0, 5); + } + } + return this; + } + + public Packer packLong(long d) throws IOException { + if(d < -(1L<<5)) { + if(d < -(1L<<15)) { + if(d < -(1L<<31)) { + // signed 64 + castBytes[0] = (byte)0xd3; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } else { + // signed 32 + castBytes[0] = (byte)0xd2; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } + } else { + if(d < -(1<<7)) { + // signed 16 + castBytes[0] = (byte)0xd1; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + } else { + // signed 8 + castBytes[0] = (byte)0xd0; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } + } + } else if(d < (1<<7)) { + // fixnum + out.write((byte)d); + } else { + if(d < (1L<<16)) { + if(d < (1<<8)) { + // unsigned 8 + castBytes[0] = (byte)0xcc; + castBytes[1] = (byte)d; + out.write(castBytes, 0, 2); + } else { + // unsigned 16 + castBytes[0] = (byte)0xcd; + castBuffer.putShort(1, (short)d); + out.write(castBytes, 0, 3); + //System.out.println("pack uint 16 "+(short)d); + } + } else { + if(d < (1L<<32)) { + // unsigned 32 + castBytes[0] = (byte)0xce; + castBuffer.putInt(1, (int)d); + out.write(castBytes, 0, 5); + } else { + // unsigned 64 + castBytes[0] = (byte)0xcf; + castBuffer.putLong(1, d); + out.write(castBytes, 0, 9); + } + } + } + return this; + } + + public Packer packFloat(float d) throws IOException { + castBytes[0] = (byte)0xca; + castBuffer.putFloat(1, d); + out.write(castBytes, 0, 5); + return this; + } + + public Packer packDouble(double d) throws IOException { + castBytes[0] = (byte)0xcb; + castBuffer.putDouble(1, d); + out.write(castBytes, 0, 9); + return this; + } + + public Packer packNil() throws IOException { + out.write((byte)0xc0); + return this; + } + + public Packer packTrue() throws IOException { + out.write((byte)0xc3); + return this; + } + + public Packer packFalse() throws IOException { + out.write((byte)0xc2); + return this; + } + + public Packer packArray(int n) throws IOException { + if(n < 16) { + final int d = 0x90 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xdc; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdd; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packMap(int n) throws IOException { + if(n < 16) { + final int d = 0x80 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xde; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdf; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRaw(int n) throws IOException { + if(n < 32) { + final int d = 0xa0 | n; + out.write((byte)d); + } else if(n < 65536) { + castBytes[0] = (byte)0xda; + castBuffer.putShort(1, (short)n); + out.write(castBytes, 0, 3); + } else { + castBytes[0] = (byte)0xdb; + castBuffer.putInt(1, n); + out.write(castBytes, 0, 5); + } + return this; + } + + public Packer packRawBody(byte[] b) throws IOException { + out.write(b); + return this; + } + + public Packer packRawBody(byte[] b, int off, int length) throws IOException { + out.write(b, off, length); + return this; + } + + + public Packer packWithSchema(Object o, Schema s) throws IOException { + s.pack(this, o); + return this; + } + + + public Packer packString(String s) throws IOException { + byte[] b = ((String)s).getBytes("UTF-8"); + packRaw(b.length); + return packRawBody(b); + } + + + public Packer pack(String o) throws IOException { + if(o == null) { return packNil(); } + return packString(o); + } + + public Packer pack(MessagePackable o) throws IOException { + if(o == null) { return packNil(); } + o.messagePack(this); + return this; + } + + public Packer pack(byte[] o) throws IOException { + if(o == null) { return packNil(); } + packRaw(o.length); + return packRawBody(o); + } + + public Packer pack(List o) throws IOException { + if(o == null) { return packNil(); } + packArray(o.size()); + for(Object i : o) { pack(i); } + return this; + } + + @SuppressWarnings("unchecked") + public Packer pack(Map o) throws IOException { + if(o == null) { return packNil(); } + packMap(o.size()); + for(Map.Entry e : ((Map)o).entrySet()) { + pack(e.getKey()); + pack(e.getValue()); + } + return this; + } + + public Packer pack(Boolean o) throws IOException { + if(o == null) { return packNil(); } + if(o) { + return packTrue(); + } else { + return packFalse(); + } + } + + public Packer pack(Byte o) throws IOException { + if(o == null) { return packNil(); } + return packByte(o); + } + + public Packer pack(Short o) throws IOException { + if(o == null) { return packNil(); } + return packShort(o); + } + + public Packer pack(Integer o) throws IOException { + if(o == null) { return packNil(); } + return packInt(o); + } + + public Packer pack(Long o) throws IOException { + if(o == null) { return packNil(); } + return packLong(o); + } + + public Packer pack(Float o) throws IOException { + if(o == null) { return packNil(); } + return packFloat(o); + } + + public Packer pack(Double o) throws IOException { + if(o == null) { return packNil(); } + return packDouble(o); + } + + + @SuppressWarnings("unchecked") + public Packer pack(Object o) throws IOException { + if(o == null) { + return packNil(); + } else if(o instanceof String) { + byte[] b = ((String)o).getBytes("UTF-8"); + packRaw(b.length); + return packRawBody(b); + } else if(o instanceof MessagePackable) { + ((MessagePackable)o).messagePack(this); + return this; + } else if(o instanceof byte[]) { + byte[] b = (byte[])o; + packRaw(b.length); + return packRawBody(b); + } else if(o instanceof List) { + List l = (List)o; + packArray(l.size()); + for(Object i : l) { pack(i); } + return this; + } else if(o instanceof Map) { + Map m = (Map)o; + packMap(m.size()); + for(Map.Entry e : m.entrySet()) { + pack(e.getKey()); + pack(e.getValue()); + } + return this; + } else if(o instanceof Boolean) { + if((Boolean)o) { + return packTrue(); + } else { + return packFalse(); + } + } else if(o instanceof Integer) { + return packInt((Integer)o); + } else if(o instanceof Long) { + return packLong((Long)o); + } else if(o instanceof Short) { + return packShort((Short)o); + } else if(o instanceof Byte) { + return packByte((Byte)o); + } else if(o instanceof Float) { + return packFloat((Float)o); + } else if(o instanceof Double) { + return packDouble((Double)o); + } else { + throw new IOException("unknown object "+o+" ("+o.getClass()+")"); + } + } +} + diff --git a/java-plan3/src/org/msgpack/Schema.java b/java-plan3/src/org/msgpack/Schema.java new file mode 100644 index 0000000..f99b3d0 --- /dev/null +++ b/java-plan3/src/org/msgpack/Schema.java @@ -0,0 +1,133 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.Writer; +import java.io.IOException; +import org.msgpack.schema.SSchemaParser; +import org.msgpack.schema.ClassGenerator; + +public abstract class Schema { + private String expression; + private String name; + + public Schema(String name) { + this.expression = expression; + this.name = name; + } + + public String getName() { + return name; + } + + public String getFullName() { + return name; + } + + public String getExpression() { + return name; + } + + public static Schema parse(String source) { + return SSchemaParser.parse(source); + } + + public static Schema load(String source) { + return SSchemaParser.load(source); + } + + public void write(Writer output) throws IOException { + ClassGenerator.write(this, output); + } + + public abstract void pack(Packer pk, Object obj) throws IOException; + + public abstract Object convert(Object obj) throws MessageTypeException; + + + public Object createFromNil() { + return null; + } + + public Object createFromBoolean(boolean v) { + throw new RuntimeException("type error"); + } + + public Object createFromByte(byte v) { + throw new RuntimeException("type error"); + } + + public Object createFromShort(short v) { + throw new RuntimeException("type error"); + } + + public Object createFromInt(int v) { + throw new RuntimeException("type error"); + } + + public Object createFromLong(long v) { + throw new RuntimeException("type error"); + } + + public Object createFromFloat(float v) { + throw new RuntimeException("type error"); + } + + public Object createFromDouble(double v) { + throw new RuntimeException("type error"); + } + + public Object createFromRaw(byte[] b, int offset, int length) { + throw new RuntimeException("type error"); + } + + /* FIXME + public Object createFromBoolean(boolean v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromByte(byte v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromShort(short v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromInt(int v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromLong(long v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromFloat(float v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromDouble(double v) { + throw MessageTypeException.schemaMismatch(this); + } + + public Object createFromRaw(byte[] b, int offset, int length) { + throw MessageTypeException.schemaMismatch(this); + } + */ +} + diff --git a/java-plan3/src/org/msgpack/UnbufferedUnpacker.java b/java-plan3/src/org/msgpack/UnbufferedUnpacker.java new file mode 100644 index 0000000..471605f --- /dev/null +++ b/java-plan3/src/org/msgpack/UnbufferedUnpacker.java @@ -0,0 +1,82 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.UnpackerImpl; + +public class UnbufferedUnpacker extends UnpackerImpl { + private int offset; + private boolean finished; + private Object data; + + public UnbufferedUnpacker() { + super(); + this.offset = 0; + this.finished = false; + } + + public UnbufferedUnpacker useSchema(Schema s) { + super.setSchema(s); + return this; + } + + public Object getData() { + return data; + } + + public boolean isFinished() { + return finished; + } + + public void reset() { + super.reset(); + this.offset = 0; + } + + int getOffset() { + return offset; + } + + void setOffset(int offset) { + this.offset = offset; + } + + public int execute(byte[] buffer) throws UnpackException { + return execute(buffer, 0, buffer.length); + } + + // FIXME + public int execute(byte[] buffer, int offset, int length) throws UnpackException + { + int noffset = super.execute(buffer, offset + this.offset, length); + this.offset = noffset - offset; + if(super.isFinished()) { + this.data = super.getData(); + this.finished = true; + super.reset(); + } else { + this.finished = false; + } + return noffset; + } +} + diff --git a/java-plan3/src/org/msgpack/UnpackException.java b/java-plan3/src/org/msgpack/UnpackException.java new file mode 100644 index 0000000..db08d95 --- /dev/null +++ b/java-plan3/src/org/msgpack/UnpackException.java @@ -0,0 +1,29 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; + +public class UnpackException extends IOException { + public UnpackException() { } + + public UnpackException(String s) { + super(s); + } +} + diff --git a/java-plan3/src/org/msgpack/UnpackIterator.java b/java-plan3/src/org/msgpack/UnpackIterator.java new file mode 100644 index 0000000..9975b68 --- /dev/null +++ b/java-plan3/src/org/msgpack/UnpackIterator.java @@ -0,0 +1,66 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class UnpackIterator implements Iterator { + private Unpacker pac; + private boolean have; + private Object data; + + UnpackIterator(Unpacker pac) { + this.pac = pac; + this.have = false; + } + + public boolean hasNext() { + if(have) { return true; } + try { + while(true) { + if(pac.execute()) { + data = pac.getData(); + pac.reset(); + have = true; + return true; + } + + if(!pac.fill()) { + return false; + } + } + } catch (IOException e) { + return false; + } + } + + public Object next() { + if(!have && !hasNext()) { + throw new NoSuchElementException(); + } + have = false; + return data; + } + + public void remove() { + throw new UnsupportedOperationException(); + } +} + diff --git a/java-plan3/src/org/msgpack/Unpacker.java b/java-plan3/src/org/msgpack/Unpacker.java new file mode 100644 index 0000000..af211c6 --- /dev/null +++ b/java-plan3/src/org/msgpack/Unpacker.java @@ -0,0 +1,245 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.lang.Iterable; +import java.io.InputStream; +import java.io.IOException; +import java.util.Iterator; +import org.msgpack.impl.UnpackerImpl; + +public class Unpacker extends UnpackerImpl implements Iterable { + + public static final int DEFAULT_BUFFER_SIZE = 32*1024; + + private int used; + private int offset; + private int parsed; + private byte[] buffer; + private int bufferReserveSize; + private InputStream stream; + + public Unpacker() { + this(DEFAULT_BUFFER_SIZE); + } + + public Unpacker(int bufferReserveSize) { + this(null, bufferReserveSize); + } + + public Unpacker(InputStream stream) { + this(stream, DEFAULT_BUFFER_SIZE); + } + + public Unpacker(InputStream stream, int bufferReserveSize) { + super(); + this.used = 0; + this.offset = 0; + this.parsed = 0; + this.buffer = new byte[bufferReserveSize]; + this.bufferReserveSize = bufferReserveSize/2; + this.stream = stream; + } + + public Unpacker useSchema(Schema s) { + super.setSchema(s); + return this; + } + + public void reserveBuffer(int size) { + if(buffer.length - used >= size) { + return; + } + /* + if(used == parsed && buffer.length >= size) { + // rewind buffer + used = 0; + offset = 0; + return; + } + */ + + int nextSize = buffer.length * 2; + while(nextSize < size + used) { + nextSize *= 2; + } + + byte[] tmp = new byte[nextSize]; + System.arraycopy(buffer, offset, tmp, 0, used - offset); + + buffer = tmp; + used -= offset; + offset = 0; + } + + public byte[] getBuffer() { + return buffer; + } + + public int getBufferOffset() { + return used; + } + + public int getBufferCapacity() { + return buffer.length - used; + } + + public void bufferConsumed(int size) { + used += size; + } + + public void feed(byte[] buffer) { + feed(buffer, 0, buffer.length); + } + + public void feed(byte[] buffer, int offset, int length) { + reserveBuffer(length); + System.arraycopy(buffer, offset, this.buffer, this.offset, length); + bufferConsumed(length); + } + + public boolean fill() throws IOException { + if(stream == null) { + return false; + } + reserveBuffer(bufferReserveSize); + int rl = stream.read(getBuffer(), getBufferOffset(), getBufferCapacity()); + if(rl <= 0) { + return false; + } + bufferConsumed(rl); + return true; + } + + public Iterator iterator() { + return new UnpackIterator(this); + } + + public boolean execute() throws UnpackException { + int noffset = super.execute(buffer, offset, used); + if(noffset <= offset) { + return false; + } + parsed += noffset - offset; + offset = noffset; + return super.isFinished(); + } + + public Object getData() { + return super.getData(); + } + + public void reset() { + super.reset(); + parsed = 0; + } + + public int getMessageSize() { + return parsed - offset + used; + } + + public int getParsedSize() { + return parsed; + } + + public int getNonParsedSize() { + return used - offset; + } + + public void skipNonparsedBuffer(int size) { + offset += size; + } + + public void removeNonparsedBuffer() { + used = offset; + } + + /* + public static class Context { + private boolean finished; + private Object data; + private int offset; + private UnpackerImpl impl; + + public Context() + { + this.finished = false; + this.impl = new UnpackerImpl(); + } + + public boolean isFinished() + { + return finished; + } + + public Object getData() + { + return data; + } + + int getOffset() + { + return offset; + } + + void setFinished(boolean finished) + { + this.finished = finished; + } + + void setData(Object data) + { + this.data = data; + } + + void setOffset(int offset) + { + this.offset = offset; + } + + UnpackerImpl getImpl() + { + return impl; + } + } + + public static int unpack(Context ctx, byte[] buffer) throws UnpackException + { + return unpack(ctx, buffer, 0, buffer.length); + } + + public static int unpack(Context ctx, byte[] buffer, int offset, int length) throws UnpackException + { + UnpackerImpl impl = ctx.getImpl(); + int noffset = impl.execute(buffer, offset + ctx.getOffset(), length); + ctx.setOffset(noffset - offset); + if(impl.isFinished()) { + ctx.setData(impl.getData()); + ctx.setFinished(false); + impl.reset(); + } else { + ctx.setData(null); + ctx.setFinished(true); + } + int parsed = noffset - offset; + ctx.setOffset(parsed); + return noffset; + } + */ +} + diff --git a/java-plan3/src/org/msgpack/impl/UnpackerImpl.java b/java-plan3/src/org/msgpack/impl/UnpackerImpl.java new file mode 100644 index 0000000..47a1800 --- /dev/null +++ b/java-plan3/src/org/msgpack/impl/UnpackerImpl.java @@ -0,0 +1,483 @@ +package org.msgpack.impl; + +import java.nio.ByteBuffer; +//import java.math.BigInteger; +import org.msgpack.*; +import org.msgpack.schema.GenericSchema; +import org.msgpack.schema.IMapSchema; +import org.msgpack.schema.IArraySchema; + +public class UnpackerImpl { + static final int CS_HEADER = 0x00; + static final int CS_FLOAT = 0x0a; + static final int CS_DOUBLE = 0x0b; + static final int CS_UINT_8 = 0x0c; + static final int CS_UINT_16 = 0x0d; + static final int CS_UINT_32 = 0x0e; + static final int CS_UINT_64 = 0x0f; + static final int CS_INT_8 = 0x10; + static final int CS_INT_16 = 0x11; + static final int CS_INT_32 = 0x12; + static final int CS_INT_64 = 0x13; + static final int CS_RAW_16 = 0x1a; + static final int CS_RAW_32 = 0x1b; + static final int CS_ARRAY_16 = 0x1c; + static final int CS_ARRAY_32 = 0x1d; + static final int CS_MAP_16 = 0x1e; + static final int CS_MAP_32 = 0x1f; + static final int ACS_RAW_VALUE = 0x20; + static final int CT_ARRAY_ITEM = 0x00; + static final int CT_MAP_KEY = 0x01; + static final int CT_MAP_VALUE = 0x02; + + static final int MAX_STACK_SIZE = 16; + + private int cs; + private int trail; + private int top; + private int[] stack_ct = new int[MAX_STACK_SIZE]; + private int[] stack_count = new int[MAX_STACK_SIZE]; + private Object[] stack_obj = new Object[MAX_STACK_SIZE]; + private Schema[] stack_schema = new Schema[MAX_STACK_SIZE]; + private int top_ct; + private int top_count; + private Object top_obj; + private Schema top_schema; + private ByteBuffer castBuffer = ByteBuffer.allocate(8); + private boolean finished = false; + private Object data = null; + + private static final Schema GENERIC_SCHEMA = new GenericSchema(); + private Schema rootSchema; + + protected UnpackerImpl() + { + setSchema(GENERIC_SCHEMA); + } + + protected void setSchema(Schema schema) + { + this.rootSchema = schema; + reset(); + } + + protected Object getData() + { + return data; + } + + protected boolean isFinished() + { + return finished; + } + + protected void reset() + { + cs = CS_HEADER; + top = -1; + finished = false; + data = null; + top_ct = 0; + top_count = 0; + top_obj = null; + top_schema = rootSchema; + } + + @SuppressWarnings("unchecked") + protected int execute(byte[] src, int off, int length) throws UnpackException + { + if(off >= length) { return off; } + + int limit = length; + int i = off; + int count; + + Object obj = null; + + _out: do { + _header_again: { + //System.out.println("while i:"+i+" limit:"+limit); + + int b = src[i]; + + _push: { + _fixed_trail_again: + if(cs == CS_HEADER) { + + if((b & 0x80) == 0) { // Positive Fixnum + //System.out.println("positive fixnum "+b); + obj = top_schema.createFromByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xe0) { // Negative Fixnum + //System.out.println("negative fixnum "+b); + obj = top_schema.createFromByte((byte)b); + break _push; + } + + if((b & 0xe0) == 0xa0) { // FixRaw + trail = b & 0x1f; + if(trail == 0) { + obj = top_schema.createFromRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + } + + if((b & 0xf0) == 0x90) { // FixArray + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IArraySchema)) { + throw new RuntimeException("type error"); + } + count = b & 0x0f; + //System.out.println("fixarray count:"+count); + obj = new Object[count]; + if(count == 0) { break _push; } // FIXME check IArraySchema + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_ARRAY_ITEM; + top_count = count; + top_schema = ((IArraySchema)top_schema).getElementSchema(0); + break _header_again; + } + + if((b & 0xf0) == 0x80) { // FixMap + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IMapSchema)) { + throw new RuntimeException("type error"); + } + count = b & 0x0f; + obj = new Object[count*2]; + if(count == 0) { break _push; } // FIXME check IMapSchema + //System.out.println("fixmap count:"+count); + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_MAP_KEY; + top_count = count; + top_schema = ((IMapSchema)top_schema).getKeySchema(); + break _header_again; + } + + switch(b & 0xff) { // FIXME + case 0xc0: // nil + obj = top_schema.createFromNil(); + break _push; + case 0xc2: // false + obj = top_schema.createFromBoolean(false); + break _push; + case 0xc3: // true + obj = top_schema.createFromBoolean(true); + break _push; + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + trail = 1 << (b & 0x03); + cs = b & 0x1f; + //System.out.println("a trail "+trail+" cs:"+cs); + break _fixed_trail_again; + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + trail = 2 << (b & 0x01); + cs = b & 0x1f; + //System.out.println("b trail "+trail+" cs:"+cs); + break _fixed_trail_again; + default: + //System.out.println("unknown b "+(b&0xff)); + throw new UnpackException("parse error"); + } + + } // _fixed_trail_again + + do { + _fixed_trail_again: { + + if(limit - i <= trail) { break _out; } + int n = i + 1; + i += trail; + + switch(cs) { + case CS_FLOAT: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = top_schema.createFromFloat( castBuffer.getFloat(0) ); + //System.out.println("float "+obj); + break _push; + case CS_DOUBLE: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = top_schema.createFromDouble( castBuffer.getDouble(0) ); + //System.out.println("double "+obj); + break _push; + case CS_UINT_8: + //System.out.println(n); + //System.out.println(src[n]); + //System.out.println(src[n+1]); + //System.out.println(src[n-1]); + obj = top_schema.createFromShort( (short)((src[n]) & 0xff) ); + //System.out.println("uint8 "+obj); + break _push; + case CS_UINT_16: + //System.out.println(src[n]); + //System.out.println(src[n+1]); + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = top_schema.createFromInt( ((int)castBuffer.getShort(0)) & 0xffff ); + //System.out.println("uint 16 "+obj); + break _push; + case CS_UINT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = top_schema.createFromLong( ((long)castBuffer.getInt(0)) & 0xffffffffL ); + //System.out.println("uint 32 "+obj); + break _push; + case CS_UINT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + { + long o = castBuffer.getLong(0); + if(o < 0) { + // FIXME + //obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31); + throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported"); + } else { + obj = top_schema.createFromLong( o ); + } + } + break _push; + case CS_INT_8: + obj = top_schema.createFromByte( src[n] ); + break _push; + case CS_INT_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + obj = top_schema.createFromShort( castBuffer.getShort(0) ); + break _push; + case CS_INT_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + obj = top_schema.createFromInt( castBuffer.getInt(0) ); + break _push; + case CS_INT_64: + castBuffer.rewind(); + castBuffer.put(src, n, 8); + obj = top_schema.createFromLong( castBuffer.getLong(0) ); + break _push; + case CS_RAW_16: + castBuffer.rewind(); + castBuffer.put(src, n, 2); + trail = ((int)castBuffer.getShort(0)) & 0xffff; + if(trail == 0) { + obj = top_schema.createFromRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + break _fixed_trail_again; + case CS_RAW_32: + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + trail = castBuffer.getInt(0) & 0x7fffffff; + if(trail == 0) { + obj = top_schema.createFromRaw(new byte[0], 0, 0); + break _push; + } + cs = ACS_RAW_VALUE; + case ACS_RAW_VALUE: + obj = top_schema.createFromRaw(src, n, trail); + break _push; + case CS_ARRAY_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IArraySchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + obj = new Object[count]; + if(count == 0) { break _push; } // FIXME check IArraySchema + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_ARRAY_ITEM; + top_count = count; + top_schema = ((IArraySchema)top_schema).getElementSchema(0); + break _header_again; + case CS_ARRAY_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IArraySchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + obj = new Object[count]; + if(count == 0) { break _push; } // FIXME check IArraySchema + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_ARRAY_ITEM; + top_count = count; + top_schema = ((IArraySchema)top_schema).getElementSchema(0); + break _header_again; + case CS_MAP_16: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IMapSchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 2); + count = ((int)castBuffer.getShort(0)) & 0xffff; + obj = new Object[count*2]; + if(count == 0) { break _push; } // FIXME check IMapSchema + //System.out.println("fixmap count:"+count); + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + ++top; + top_obj = obj; + top_ct = CT_MAP_KEY; + top_count = count; + top_schema = ((IMapSchema)top_schema).getKeySchema(); + break _header_again; + case CS_MAP_32: + if(top >= MAX_STACK_SIZE) { + throw new UnpackException("parse error"); + } + if(!(top_schema instanceof IMapSchema)) { + throw new RuntimeException("type error"); + } + castBuffer.rewind(); + castBuffer.put(src, n, 4); + // FIXME overflow check + count = castBuffer.getInt(0) & 0x7fffffff; + obj = new Object[count*2]; + if(count == 0) { break _push; } // FIXME check IMapSchema + //System.out.println("fixmap count:"+count); + ++top; + stack_obj[top] = top_obj; + stack_ct[top] = top_ct; + stack_count[top] = top_count; + stack_schema[top] = top_schema; + top_obj = obj; + top_ct = CT_MAP_KEY; + top_count = count; + top_schema = ((IMapSchema)top_schema).getKeySchema(); + break _header_again; + default: + throw new UnpackException("parse error"); + } + + } // _fixed_trail_again + } while(true); + } // _push + + do { + _push: { + //System.out.println("push top:"+top); + if(top == -1) { + ++i; + data = obj; + finished = true; + break _out; + } + + switch(top_ct) { + case CT_ARRAY_ITEM: { + //System.out.println("array item "+obj); + Object[] ar = (Object[])top_obj; + ar[ar.length - top_count] = obj; + if(--top_count == 0) { + top_obj = stack_obj[top]; + top_ct = stack_ct[top]; + top_count = stack_count[top]; + top_schema = stack_schema[top]; + obj = ((IArraySchema)top_schema).createFromArray(ar); + stack_obj[top] = null; + stack_schema[top] = null; + --top; + break _push; + } else { + top_schema = ((IArraySchema)stack_schema[top]).getElementSchema(ar.length - top_count); + } + break _header_again; + } + case CT_MAP_KEY: { + //System.out.println("map key:"+top+" "+obj); + Object[] mp = (Object[])top_obj; + mp[mp.length - top_count*2] = obj; + top_ct = CT_MAP_VALUE; + top_schema = ((IMapSchema)stack_schema[top]).getValueSchema(); + break _header_again; + } + case CT_MAP_VALUE: { + //System.out.println("map value:"+top+" "+obj); + Object[] mp = (Object[])top_obj; + mp[mp.length - top_count*2 + 1] = obj; + if(--top_count == 0) { + top_obj = stack_obj[top]; + top_ct = stack_ct[top]; + top_count = stack_count[top]; + top_schema = stack_schema[top]; + obj = ((IMapSchema)top_schema).createFromMap(mp); + stack_obj[top] = null; + stack_schema[top] = null; + --top; + break _push; + } + top_ct = CT_MAP_KEY; + break _header_again; + } + default: + throw new UnpackException("parse error"); + } + } // _push + } while(true); + + } // _header_again + cs = CS_HEADER; + ++i; + } while(i < limit); // _out + + return i; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ArraySchema.java b/java-plan3/src/org/msgpack/schema/ArraySchema.java new file mode 100644 index 0000000..24fa758 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ArraySchema.java @@ -0,0 +1,125 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Set; +import java.util.List; +import java.util.ArrayList; +import java.util.RandomAccess; +import java.io.IOException; +import org.msgpack.*; + +public class ArraySchema extends Schema implements IArraySchema { + private Schema elementSchema; + + public ArraySchema(Schema elementSchema) + { + super("array"); + this.elementSchema = elementSchema; + } + + @Override + public String getFullName() + { + return "List<"+elementSchema.getFullName()+">"; + } + + @Override + public String getExpression() + { + return "(array "+elementSchema.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException + { + if(obj instanceof List) { + ArrayList d = (ArrayList)obj; + pk.packArray(d.size()); + if(obj instanceof RandomAccess) { + for(int i=0; i < d.size(); ++i) { + elementSchema.pack(pk, d.get(i)); + } + } else { + for(Object e : d) { + elementSchema.pack(pk, e); + } + } + + } else if(obj instanceof Set) { + Set d = (Set)obj; + pk.packArray(d.size()); + for(Object e : d) { + elementSchema.pack(pk, e); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(Object obj) throws MessageTypeException + { + if(obj instanceof List) { + List d = (List)obj; + ArrayList ar = new ArrayList(d.size()); + if(obj instanceof RandomAccess) { + for(int i=0; i < d.size(); ++i) { + ar.add( elementSchema.convert(d.get(i)) ); + } + } else { + for(Object e : d) { + ar.add( elementSchema.convert(e) ); + } + } + return ar; + + } else if(obj instanceof Collection) { + Collection d = (Collection)obj; + ArrayList ar = new ArrayList(d.size()); + for(Object e : (Collection)obj) { + ar.add( elementSchema.convert(e) ); + } + return ar; + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Schema getElementSchema(int index) + { + return elementSchema; + } + + @Override + public Object createFromArray(Object[] obj) + { + return Arrays.asList(obj); + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ByteSchema.java b/java-plan3/src/org/msgpack/schema/ByteSchema.java new file mode 100644 index 0000000..3bc7045 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ByteSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ByteSchema extends Schema { + public ByteSchema() { + super("Byte"); + } + + @Override + public String getExpression() { + return "byte"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packByte( ((Number)obj).byteValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Byte) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).byteValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (byte)v; + } + + @Override + public Object createFromShort(short v) { + return (byte)v; + } + + @Override + public Object createFromInt(int v) { + return (byte)v; + } + + @Override + public Object createFromLong(long v) { + return (byte)v; + } + + @Override + public Object createFromFloat(float v) { + return (byte)v; + } + + @Override + public Object createFromDouble(double v) { + return (byte)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ClassGenerator.java b/java-plan3/src/org/msgpack/schema/ClassGenerator.java new file mode 100644 index 0000000..65213aa --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ClassGenerator.java @@ -0,0 +1,241 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.List; +import java.io.IOException; +import java.io.File; +import java.io.Writer; +import org.msgpack.*; + +public class ClassGenerator { + private ClassSchema schema; + private Writer writer; + private int indent; + + private ClassGenerator(Writer writer) { + this.writer = writer; + this.indent = 0; + } + + public static void write(Schema schema, Writer dest) throws IOException { + if(!(schema instanceof ClassSchema)) { + throw new RuntimeException("schema is not class schema"); + } + ClassSchema cs = (ClassSchema)schema; + new ClassGenerator(dest).run(cs); + } + + private void run(ClassSchema cs) throws IOException { + List subclasses = new ArrayList(); + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(subclasses, f.getSchema()); + } + + for(ClassSchema sub : subclasses) { + sub.setNamespace(cs.getNamespace()); + sub.setImports(cs.getImports()); + } + + this.schema = cs; + + writeHeader(); + + writeClass(); + + for(ClassSchema sub : subclasses) { + this.schema = sub; + writeSubclass(); + } + + writeFooter(); + + this.schema = null; + writer.flush(); + } + + private void findSubclassSchema(List dst, Schema s) { + if(s instanceof ClassSchema) { + ClassSchema cs = (ClassSchema)s; + if(!dst.contains(cs)) { dst.add(cs); } + for(FieldSchema f : cs.getFields()) { + findSubclassSchema(dst, f.getSchema()); + } + } else if(s instanceof ArraySchema) { + ArraySchema as = (ArraySchema)s; + findSubclassSchema(dst, as.getElementSchema(0)); + } else if(s instanceof MapSchema) { + MapSchema as = (MapSchema)s; + findSubclassSchema(dst, as.getKeySchema()); + findSubclassSchema(dst, as.getValueSchema()); + } + } + + private void writeHeader() throws IOException { + if(schema.getNamespace() != null) { + line("package "+schema.getNamespace()+";"); + line(); + } + line("import java.util.*;"); + line("import java.io.*;"); + line("import org.msgpack.*;"); + line("import org.msgpack.schema.ClassSchema;"); + line("import org.msgpack.schema.FieldSchema;"); + } + + private void writeFooter() throws IOException { + line(); + } + + private void writeClass() throws IOException { + line(); + line("public final class "+schema.getName()+" implements MessagePackable, MessageMergeable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSubclass() throws IOException { + line(); + line("final class "+schema.getName()+" implements MessagePackable, MessageMergeable"); + line("{"); + pushIndent(); + writeSchema(); + writeMemberVariables(); + writeMemberFunctions(); + popIndent(); + line("}"); + } + + private void writeSchema() throws IOException { + line("private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load(\""+schema.getExpression()+"\");"); + line("public static ClassSchema getSchema() { return _SCHEMA; }"); + } + + private void writeMemberVariables() throws IOException { + line(); + for(FieldSchema f : schema.getFields()) { + line("public "+f.getSchema().getFullName()+" "+f.getName()+";"); + } + } + + private void writeMemberFunctions() throws IOException { + // void messagePack(Packer pk) + // boolean equals(Object obj) + // int hashCode() + // void set(int _index, Object _value) + // Object get(int _index); + // getXxx() + // setXxx(Xxx xxx) + writeConstructors(); + writeAccessors(); + writePackFunction(); + writeMergeFunction(); + writeFactoryFunction(); + } + + private void writeConstructors() throws IOException { + line(); + line("public "+schema.getName()+"() { }"); + } + + private void writeAccessors() throws IOException { + // FIXME + //line(); + //for(FieldSchema f : schema.getFields()) { + // line(""); + //} + } + + private void writePackFunction() throws IOException { + line(); + line("@Override"); + line("public void messagePack(Packer _pk) throws IOException"); + line("{"); + pushIndent(); + line("_pk.packArray("+schema.getFields().length+");"); + line("FieldSchema[] _fields = _SCHEMA.getFields();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("_fields["+i+"].getSchema().pack(_pk, "+f.getName()+");"); + ++i; + } + popIndent(); + line("}"); + } + + private void writeMergeFunction() throws IOException { + line(); + line("@Override"); + line("@SuppressWarnings(\"unchecked\")"); + line("public void messageMerge(Object obj) throws MessageTypeException"); + line("{"); + pushIndent(); + line("Object[] _source = ((List)obj).toArray();"); + line("FieldSchema[] _fields = _SCHEMA.getFields();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("if(_source.length <= "+i+") { return; } this."+f.getName()+" = ("+f.getSchema().getFullName()+")_fields["+i+"].getSchema().convert(_source["+i+"]);"); + ++i; + } + popIndent(); + line("}"); + } + + private void writeFactoryFunction() throws IOException { + line(); + line("@SuppressWarnings(\"unchecked\")"); + line("public static "+schema.getName()+" createFromMessage(Object[] _message)"); + line("{"); + pushIndent(); + line(schema.getName()+" _self = new "+schema.getName()+"();"); + int i = 0; + for(FieldSchema f : schema.getFields()) { + line("if(_message.length <= "+i+") { return _self; } _self."+f.getName()+" = ("+f.getSchema().getFullName()+")_message["+i+"];"); + ++i; + } + line("return _self;"); + popIndent(); + line("}"); + } + + private void line(String str) throws IOException { + for(int i=0; i < indent; ++i) { + writer.write("\t"); + } + writer.write(str+"\n"); + } + + private void line() throws IOException { + writer.write("\n"); + } + + private void pushIndent() { + indent += 1; + } + + private void popIndent() { + indent -= 1; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ClassSchema.java b/java-plan3/src/org/msgpack/schema/ClassSchema.java new file mode 100644 index 0000000..75315e7 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ClassSchema.java @@ -0,0 +1,95 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.List; +import org.msgpack.*; + +public abstract class ClassSchema extends Schema implements IArraySchema { + protected FieldSchema[] fields; + protected List imports; + protected String namespace; + protected String fqdn; + + public ClassSchema( + String name, String namespace, + List imports, List fields) { + super(name); + this.namespace = namespace; + this.imports = imports; // FIXME clone? + this.fields = new FieldSchema[fields.size()]; + System.arraycopy(fields.toArray(), 0, this.fields, 0, fields.size()); + if(namespace == null) { + this.fqdn = name; + } else { + this.fqdn = namespace+"."+name; + } + } + + public final FieldSchema[] getFields() { + return fields; + } + + String getNamespace() { + return namespace; + } + + List getImports() { + return imports; + } + + void setNamespace(String namespace) { + this.namespace = namespace; + } + + void setImports(List imports) { + this.imports = imports; // FIXME clone? + } + + //@Override + //public String getFullName() + //{ + // if(namespace == null) { + // return getName(); + // } else { + // return namespace+"."+getName(); + // } + //} + + @Override + public String getExpression() { + StringBuffer b = new StringBuffer(); + b.append("(class "); + b.append(getName()); + if(namespace != null) { + b.append(" (package "+namespace+")"); + } + for(FieldSchema f : fields) { + b.append(" "+f.getExpression()); + } + b.append(")"); + return b.toString(); + } + + public boolean equals(SpecificClassSchema o) { + return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) && + getName().equals(o.getName()); + } +} + diff --git a/java-plan3/src/org/msgpack/schema/DoubleSchema.java b/java-plan3/src/org/msgpack/schema/DoubleSchema.java new file mode 100644 index 0000000..feffbb9 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/DoubleSchema.java @@ -0,0 +1,84 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class DoubleSchema extends Schema { + public DoubleSchema() { + super("Double"); + } + + @Override + public String getExpression() { + return "double"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packDouble( ((Number)obj).doubleValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Double) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).doubleValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (double)v; + } + + @Override + public Object createFromShort(short v) { + return (double)v; + } + + @Override + public Object createFromInt(int v) { + return (double)v; + } + + @Override + public Object createFromFloat(float v) { + return (double)v; + } + + @Override + public Object createFromDouble(double v) { + return (double)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/FieldSchema.java b/java-plan3/src/org/msgpack/schema/FieldSchema.java new file mode 100644 index 0000000..3391f2b --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/FieldSchema.java @@ -0,0 +1,43 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import org.msgpack.Schema; + +public class FieldSchema { + private String name; + private Schema schema; + + public FieldSchema(String name, Schema schema) { + this.name = name; + this.schema = schema; + } + + public final String getName() { + return name; + } + + public final Schema getSchema() { + return schema; + } + + public String getExpression() { + return "(field "+name+" "+schema.getExpression()+")"; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/FloatSchema.java b/java-plan3/src/org/msgpack/schema/FloatSchema.java new file mode 100644 index 0000000..2f4240a --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/FloatSchema.java @@ -0,0 +1,84 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class FloatSchema extends Schema { + public FloatSchema() { + super("Float"); + } + + @Override + public String getExpression() { + return "float"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packFloat( ((Number)obj).floatValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Float) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).floatValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (float)v; + } + + @Override + public Object createFromShort(short v) { + return (float)v; + } + + @Override + public Object createFromInt(int v) { + return (float)v; + } + + @Override + public Object createFromFloat(float v) { + return (float)v; + } + + @Override + public Object createFromDouble(double v) { + return (float)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/GenericClassSchema.java b/java-plan3/src/org/msgpack/schema/GenericClassSchema.java new file mode 100644 index 0000000..736bdfa --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/GenericClassSchema.java @@ -0,0 +1,91 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; + +public class GenericClassSchema extends ClassSchema { + public GenericClassSchema( + String name, String namespace, + List imports, List fields) { + super(name, namespace, imports, fields); + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Map) { + Map d = (Map)obj; + pk.packArray(fields.length); + for(int i=0; i < fields.length; ++i) { + FieldSchema f = fields[i]; + f.getSchema().pack(pk, d.get(f.getName())); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Collection) { + // FIXME optimize + return createFromArray( ((Collection)obj).toArray() ); + + } else if(obj instanceof Map) { + HashMap m = new HashMap(fields.length); + Map d = (Map)obj; + for(int i=0; i < fields.length; ++i) { + FieldSchema f = fields[i]; + String fieldName = f.getName(); + m.put(fieldName, f.getSchema().convert(d.get(fieldName))); + } + return m; + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + public Schema getElementSchema(int index) { + // FIXME check index < fields.length + return fields[index].getSchema(); + } + + public Object createFromArray(Object[] obj) { + HashMap m = new HashMap(fields.length); + int i=0; + for(; i < obj.length; ++i) { + m.put(fields[i].getName(), obj[i]); + } + for(; i < fields.length; ++i) { + m.put(fields[i].getName(), null); + } + return m; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/GenericSchema.java b/java-plan3/src/org/msgpack/schema/GenericSchema.java new file mode 100644 index 0000000..52e0161 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/GenericSchema.java @@ -0,0 +1,192 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.List; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; +//import org.msgpack.generic.*; + +public class GenericSchema extends Schema implements IArraySchema, IMapSchema { + public GenericSchema() { + super("Object"); + } + + @Override + public String getExpression() { + return "object"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + pk.pack(obj); + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + return obj; + } + + @Override + public Schema getElementSchema(int index) { + return this; + } + + @Override + public Schema getKeySchema() { + return this; + } + + @Override + public Schema getValueSchema() { + return this; + } + + @Override + public Object createFromNil() { + return null; + } + + @Override + public Object createFromBoolean(boolean v) { + return v; + } + + @Override + public Object createFromByte(byte v) { + return v; + } + + @Override + public Object createFromShort(short v) { + return v; + } + + @Override + public Object createFromInt(int v) { + return v; + } + + @Override + public Object createFromLong(long v) { + return v; + } + + @Override + public Object createFromFloat(float v) { + return v; + } + + @Override + public Object createFromDouble(double v) { + return v; + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + byte[] bytes = new byte[length]; + System.arraycopy(b, offset, bytes, 0, length); + return bytes; + } + + @Override + public Object createFromArray(Object[] obj) { + return Arrays.asList(obj); + } + + @Override + @SuppressWarnings("unchecked") + public Object createFromMap(Object[] obj) { + HashMap m = new HashMap(obj.length / 2); + int i = 0; + while(i < obj.length) { + Object k = obj[i++]; + Object v = obj[i++]; + m.put(k, v); + } + return m; + } + + /* + @Override + public Object createFromNil() { + return null; + } + + @Override + public Object createFromBoolean(boolean v) { + return new GenericBoolean(v); + } + + @Override + public Object createFromFromByte(byte v) { + return new GenericByte(v); + } + + @Override + public Object createFromShort(short v) { + return new GenericShort(v); + } + + @Override + public Object createFromInt(int v) { + return new GenericInt(v); + } + + @Override + public Object createFromLong(long v) { + return new GenericLong(v); + } + + @Override + public Object createFromFloat(float v) { + return new GenericFloat(v); + } + + @Override + public Object createFromDouble(double v) { + return new GenericDouble(v); + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + return new GenericRaw(b, offset, length); + } + + @Override + public Object createFromArray(Object[] obj) { + // FIXME GenericArray + return Arrays.asList(obj); + } + + @Override + public Object createFromMap(Object[] obj) { + GenericMap m = new GenericMap(obj.length / 2); + int i = 0; + while(i < obj.length) { + Object k = obj[i++]; + Object v = obj[i++]; + m.put(k, v); + } + return m; + } + */ +} + diff --git a/java-plan3/src/org/msgpack/schema/IArraySchema.java b/java-plan3/src/org/msgpack/schema/IArraySchema.java new file mode 100644 index 0000000..ccc6d14 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/IArraySchema.java @@ -0,0 +1,26 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import org.msgpack.Schema; + +public interface IArraySchema { + public Schema getElementSchema(int index); + public Object createFromArray(Object[] obj); +} + diff --git a/java-plan3/src/org/msgpack/schema/IMapSchema.java b/java-plan3/src/org/msgpack/schema/IMapSchema.java new file mode 100644 index 0000000..60b3e8e --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/IMapSchema.java @@ -0,0 +1,27 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import org.msgpack.Schema; + +public interface IMapSchema { + public Schema getKeySchema(); + public Schema getValueSchema(); + public Object createFromMap(Object[] obj); +} + diff --git a/java-plan3/src/org/msgpack/schema/IntSchema.java b/java-plan3/src/org/msgpack/schema/IntSchema.java new file mode 100644 index 0000000..c54c0ae --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/IntSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class IntSchema extends Schema { + public IntSchema() { + super("Integer"); + } + + @Override + public String getExpression() { + return "int"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packInt( ((Number)obj).intValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Integer) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).intValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (int)v; + } + + @Override + public Object createFromShort(short v) { + return (int)v; + } + + @Override + public Object createFromInt(int v) { + return (int)v; + } + + @Override + public Object createFromLong(long v) { + return (int)v; + } + + @Override + public Object createFromFloat(float v) { + return (int)v; + } + + @Override + public Object createFromDouble(double v) { + return (int)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/LongSchema.java b/java-plan3/src/org/msgpack/schema/LongSchema.java new file mode 100644 index 0000000..ccf3043 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/LongSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class LongSchema extends Schema { + public LongSchema() { + super("Long"); + } + + @Override + public String getExpression() { + return "long"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packLong( ((Number)obj).longValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Long) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).longValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (long)v; + } + + @Override + public Object createFromShort(short v) { + return (long)v; + } + + @Override + public Object createFromInt(int v) { + return (long)v; + } + + @Override + public Object createFromLong(long v) { + return (long)v; + } + + @Override + public Object createFromFloat(float v) { + return (long)v; + } + + @Override + public Object createFromDouble(double v) { + return (long)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/MapSchema.java b/java-plan3/src/org/msgpack/schema/MapSchema.java new file mode 100644 index 0000000..71629b0 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/MapSchema.java @@ -0,0 +1,102 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Map; +import java.util.HashMap; +import java.io.IOException; +import org.msgpack.*; + +public class MapSchema extends Schema implements IMapSchema { + private Schema keySchema; + private Schema valueSchema; + + public MapSchema(Schema keySchema, Schema valueSchema) { + super("map"); + this.keySchema = keySchema; + this.valueSchema = valueSchema; + } + + @Override + public String getFullName() { + return "HashList<"+keySchema.getFullName()+", "+valueSchema.getFullName()+">"; + } + + @Override + public String getExpression() { + return "(map "+keySchema.getExpression()+" "+valueSchema.getExpression()+")"; + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Map) { + Map d = (Map)obj; + pk.packMap(d.size()); + for(Map.Entry e : d.entrySet()) { + keySchema.pack(pk, e.getKey()); + valueSchema.pack(pk, e.getValue()); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + @SuppressWarnings("unchecked") + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Map) { + Map d = (Map)obj; + Map m = new HashMap(); + for(Map.Entry e : d.entrySet()) { + m.put(keySchema.convert(e.getKey()), valueSchema.convert(e.getValue())); + } + return m; + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Schema getKeySchema() { + return keySchema; + } + + @Override + public Schema getValueSchema() { + return valueSchema; + } + + @Override + public Object createFromMap(Object[] obj) { + HashMap m = new HashMap(obj.length / 2); + int i = 0; + while(i < obj.length) { + Object k = obj[i++]; + Object v = obj[i++]; + m.put(k, v); + } + return m; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/RawSchema.java b/java-plan3/src/org/msgpack/schema/RawSchema.java new file mode 100644 index 0000000..582f766 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/RawSchema.java @@ -0,0 +1,105 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.nio.ByteBuffer; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import org.msgpack.*; + +public class RawSchema extends Schema { + public RawSchema() { + super("raw"); + } + + public String getFullName() { + return "byte[]"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + // FIXME instanceof GenericObject + if(obj instanceof byte[]) { + byte[] d = (byte[])obj; + pk.packRaw(d.length); + pk.packRawBody(d); + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + if(!d.hasArray()) { + throw MessageTypeException.invalidConvert(obj, this); + } + pk.packRaw(d.capacity()); + pk.packRawBody(d.array(), d.position(), d.capacity()); + + } else if(obj instanceof String) { + try { + byte[] d = ((String)obj).getBytes("UTF-8"); + pk.packRaw(d.length); + pk.packRawBody(d); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + // FIXME instanceof GenericObject + if(obj instanceof byte[]) { + // FIXME copy? + //byte[] d = (byte[])obj; + //byte[] v = new byte[d.length]; + //System.arraycopy(d, 0, v, 0, d.length); + //return v; + return obj; + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + byte[] v = new byte[d.capacity()]; + int pos = d.position(); + d.get(v); + d.position(pos); + return v; + + } else if(obj instanceof String) { + try { + return ((String)obj).getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + byte[] d = new byte[length]; + System.arraycopy(b, offset, d, 0, length); + return d; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java b/java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java new file mode 100644 index 0000000..fb94adf --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java @@ -0,0 +1,64 @@ +package org.msgpack.schema; + +import java.util.Arrays; +import java.util.List; +import java.lang.reflect.*; +import org.msgpack.*; + +// FIXME +public abstract class ReflectionClassSchema extends ClassSchema { + private Constructor constructorCache; + + public ReflectionClassSchema(String name, List fields, String namespace, List imports) { + super(name, namespace, imports, fields); + } + + /* + Schema getElementSchema(int index) + { + // FIXME check index < fields.length + fields[index].getSchema(); + } + + Object createFromArray(Object[] obj) + { + Object o = newInstance(); + ((MessageConvertable)o).messageConvert(obj); + return o; + } + + Object newInstance() + { + if(constructorCache == null) { + cacheConstructor(); + } + try { + return constructorCache.newInstance((Object[])null); + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + private void cacheConstructor() + { + try { + Class c = Class.forName(fqdn); + int index = 0; + for(SpecificFieldSchema f : fields) { + f.cacheField(c, index++); + } + constructorCache = c.getDeclaredConstructor((Class[])null); + constructorCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } + */ +} + diff --git a/java-plan3/src/org/msgpack/schema/SSchemaParser.java b/java-plan3/src/org/msgpack/schema/SSchemaParser.java new file mode 100644 index 0000000..c6bbc77 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/SSchemaParser.java @@ -0,0 +1,254 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Stack; +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import org.msgpack.*; + +// FIXME exception class + +public class SSchemaParser { + public static Schema parse(String source) { + return new SSchemaParser(false).run(source); + } + + public static Schema load(String source) { + return new SSchemaParser(true).run(source); + } + + private static abstract class SExp { + boolean isAtom() { return false; } + public String getAtom() { return null; } + + boolean isTuple() { return false; } + public SExp getTuple(int i) { return null; } + public int size() { return 0; } + public boolean empty() { return size() == 0; } + Iterator iterator(int offset) { return null; } + } + + private static class SAtom extends SExp { + private String atom; + + SAtom(String atom) { this.atom = atom; } + + boolean isAtom() { return true; } + public String getAtom() { return atom; } + + public String toString() { return atom; } + } + + private static class STuple extends SExp { + private List tuple; + + STuple() { this.tuple = new ArrayList(); } + + public void add(SExp e) { tuple.add(e); } + + boolean isTuple() { return true; } + public SExp getTuple(int i) { return tuple.get(i); } + public int size() { return tuple.size(); } + + Iterator iterator(int skip) { + Iterator i = tuple.iterator(); + for(int s=0; s < skip; ++s) { i.next(); } + return i; + } + + public String toString() { + if(tuple.isEmpty()) { return "()"; } + Iterator i = tuple.iterator(); + StringBuffer o = new StringBuffer(); + o.append("(").append(i.next()); + while(i.hasNext()) { o.append(" ").append(i.next()); } + o.append(")"); + return o.toString(); + } + } + + boolean specificClass; + + private SSchemaParser(boolean specificClass) { + this.specificClass = specificClass; + } + + private static Pattern pattern = Pattern.compile( + "(?:\\s+)|([\\(\\)]|[\\d\\w\\.]+)"); + + private Schema run(String source) { + Matcher m = pattern.matcher(source); + + Stack stack = new Stack(); + String token; + + while(true) { + while(true) { + if(!m.find()) { throw new RuntimeException("unexpected end of file"); } + token = m.group(1); + if(token != null) { break; } + } + + if(token.equals("(")) { + stack.push(new STuple()); + } else if(token.equals(")")) { + STuple top = stack.pop(); + if(stack.empty()) { + stack.push(top); + break; + } + stack.peek().add(top); + } else { + if(stack.empty()) { + throw new RuntimeException("unexpected token '"+token+"'"); + } + stack.peek().add(new SAtom(token)); + } + } + + while(true) { + if(!m.find()) { break; } + token = m.group(1); + if(token != null) { throw new RuntimeException("unexpected token '"+token+"'"); } + } + + return readType( stack.pop() ); + } + + private Schema readType(SExp exp) { + if(exp.isAtom()) { + String type = exp.getAtom(); + if(type.equals("string")) { + return new StringSchema(); + } else if(type.equals("raw")) { + return new RawSchema(); + } else if(type.equals("byte")) { + return new ByteSchema(); + } else if(type.equals("short")) { + return new ShortSchema(); + } else if(type.equals("int")) { + return new IntSchema(); + } else if(type.equals("long")) { + return new LongSchema(); + } else if(type.equals("float")) { + return new FloatSchema(); + } else if(type.equals("double")) { + return new DoubleSchema(); + } else if(type.equals("object")) { + return new GenericSchema(); + } else { + throw new RuntimeException("byte, short, int, long, float, double, raw, string or object is expected but got '"+type+"': "+exp); + } + } else { + String type = exp.getTuple(0).getAtom(); + if(type.equals("class")) { + return parseClass(exp); + } else if(type.equals("array")) { + return parseArray(exp); + } else if(type.equals("map")) { + return parseMap(exp); + } else { + throw new RuntimeException("class, array or map is expected but got '"+type+"': "+exp); + } + } + } + + private ClassSchema parseClass(SExp exp) { + if(exp.size() < 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("class is (class NAME CLASS_BODY): "+exp); + } + + String namespace = null; + List imports = new ArrayList(); + String name = exp.getTuple(1).getAtom(); + List fields = new ArrayList(); + + for(Iterator i=exp.iterator(2); i.hasNext();) { + SExp subexp = i.next(); + if(!subexp.isTuple() || subexp.empty() || !subexp.getTuple(0).isAtom()) { + throw new RuntimeException("field, package or import is expected: "+subexp); + } + String type = subexp.getTuple(0).getAtom(); + if(type.equals("field")) { + fields.add( parseField(subexp) ); + } else if(type.equals("package")) { + if(namespace != null) { + throw new RuntimeException("duplicated package definition: "+subexp); + } + namespace = parseNamespace(subexp); + } else if(type.equals("import")) { + imports.add( parseImport(subexp) ); + } else { + throw new RuntimeException("field, package or import is expected but got '"+type+"': "+subexp); + } + } + + if(specificClass) { + return new SpecificClassSchema(name, namespace, imports, fields); + } else { + return new GenericClassSchema(name, namespace, imports, fields); + } + } + + private ArraySchema parseArray(SExp exp) { + if(exp.size() != 2) { + throw new RuntimeException("array is (array ELEMENT_TYPE): "+exp); + } + Schema elementType = readType(exp.getTuple(1)); + return new ArraySchema(elementType); + } + + private MapSchema parseMap(SExp exp) { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("map is (map KEY_TYPE VALUE_TYPE): "+exp); + } + Schema keyType = readType(exp.getTuple(1)); + Schema valueType = readType(exp.getTuple(2)); + return new MapSchema(keyType, valueType); + } + + private String parseNamespace(SExp exp) { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("package is (package NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private String parseImport(SExp exp) { + if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("import is (import NAME): "+exp); + } + String name = exp.getTuple(1).getAtom(); + return name; + } + + private FieldSchema parseField(SExp exp) { + if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { + throw new RuntimeException("field is (field NAME TYPE): "+exp); + } + String name = exp.getTuple(1).getAtom(); + Schema type = readType(exp.getTuple(2)); + return new FieldSchema(name, type); + } +} + diff --git a/java-plan3/src/org/msgpack/schema/ShortSchema.java b/java-plan3/src/org/msgpack/schema/ShortSchema.java new file mode 100644 index 0000000..089a024 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/ShortSchema.java @@ -0,0 +1,89 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.io.IOException; +import org.msgpack.*; + +public class ShortSchema extends Schema { + public ShortSchema() { + super("Short"); + } + + @Override + public String getExpression() { + return "short"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + if(obj instanceof Number) { + pk.packShort( ((Number)obj).shortValue() ); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Short) { + return obj; + + } else if(obj instanceof Number) { + return ((Number)obj).shortValue(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromByte(byte v) { + return (short)v; + } + + @Override + public Object createFromShort(short v) { + return (short)v; + } + + @Override + public Object createFromInt(int v) { + return (short)v; + } + + @Override + public Object createFromLong(long v) { + return (short)v; + } + + @Override + public Object createFromFloat(float v) { + return (short)v; + } + + @Override + public Object createFromDouble(double v) { + return (short)v; + } +} + diff --git a/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java b/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java new file mode 100644 index 0000000..81e5e00 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java @@ -0,0 +1,122 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.util.Collection; +import java.util.List; +import java.lang.reflect.*; +import java.io.IOException; +import org.msgpack.*; + +public class SpecificClassSchema extends ClassSchema { + private Class classCache; + private Method factoryCache; + private Constructor constructorCache; + + public SpecificClassSchema( + String name, String namespace, + List imports, List fields) { + super(name, namespace, imports, fields); + } + + @Override + @SuppressWarnings("unchecked") + public void pack(Packer pk, Object obj) throws IOException { + if(obj == null) { + pk.packNil(); + return; + } + if(classCache == null) { + cacheFactory(); + } + if(classCache.isInstance(obj)) { + ((MessagePackable)obj).messagePack(pk); + } else { + // FIXME Map + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + if(obj instanceof Collection) { + if(constructorCache == null) { + cacheConstructor(); + } + try { + MessageMergeable o = (MessageMergeable)constructorCache.newInstance((Object[])null); + o.messageMerge(obj); + return o; + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (InstantiationException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + public Schema getElementSchema(int index) { + // FIXME check index < fields.length + return fields[index].getSchema(); + } + + public Object createFromArray(Object[] obj) { + if(factoryCache == null) { + cacheFactory(); + } + try { + return factoryCache.invoke(null, new Object[]{obj}); + } catch (InvocationTargetException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getCause()); + } catch (IllegalAccessException e) { + throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private void cacheFactory() { + try { + classCache = Class.forName(fqdn); + factoryCache = classCache.getDeclaredMethod("createFromMessage", new Class[]{Object[].class}); + factoryCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private void cacheConstructor() { + try { + classCache = Class.forName(fqdn); + constructorCache = classCache.getDeclaredConstructor((Class[])null); + constructorCache.setAccessible(true); + } catch(ClassNotFoundException e) { + throw new RuntimeException("class not found: "+fqdn); + } catch (NoSuchMethodException e) { + throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); + } + } +} + diff --git a/java-plan3/src/org/msgpack/schema/StringSchema.java b/java-plan3/src/org/msgpack/schema/StringSchema.java new file mode 100644 index 0000000..f5e0bf1 --- /dev/null +++ b/java-plan3/src/org/msgpack/schema/StringSchema.java @@ -0,0 +1,111 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack.schema; + +import java.nio.ByteBuffer; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import org.msgpack.*; + +public class StringSchema extends Schema { + public StringSchema() { + super("string"); + } + + @Override + public String getFullName() { + return "String"; + } + + @Override + public void pack(Packer pk, Object obj) throws IOException { + // FIXME instanceof GenericObject + if(obj instanceof String) { + try { + byte[] d = ((String)obj).getBytes("UTF-8"); + pk.packRaw(d.length); + pk.packRawBody(d); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else if(obj instanceof byte[]) { + byte[] d = (byte[])obj; + pk.packRaw(d.length); + pk.packRawBody(d); + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + if(!d.hasArray()) { + throw MessageTypeException.invalidConvert(obj, this); + } + pk.packRaw(d.capacity()); + pk.packRawBody(d.array(), d.position(), d.capacity()); + + } else if(obj == null) { + pk.packNil(); + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object convert(Object obj) throws MessageTypeException { + // FIXME instanceof GenericObject + if(obj instanceof String) { + return obj; + + } else if(obj instanceof byte[]) { + try { + return new String((byte[])obj, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else if(obj instanceof ByteBuffer) { + ByteBuffer d = (ByteBuffer)obj; + try { + if(d.hasArray()) { + return new String(d.array(), d.position(), d.capacity(), "UTF-8"); + } else { + byte[] v = new byte[d.capacity()]; + int pos = d.position(); + d.get(v); + d.position(pos); + return new String(v, "UTF-8"); + } + } catch (UnsupportedEncodingException e) { + throw MessageTypeException.invalidConvert(obj, this); + } + + } else { + throw MessageTypeException.invalidConvert(obj, this); + } + } + + @Override + public Object createFromRaw(byte[] b, int offset, int length) { + try { + return new String(b, offset, length, "UTF-8"); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } +} + diff --git a/java-plan3/test/Generate.java b/java-plan3/test/Generate.java new file mode 100644 index 0000000..1b72e90 --- /dev/null +++ b/java-plan3/test/Generate.java @@ -0,0 +1,38 @@ +import java.io.*; +import java.util.*; +import org.msgpack.*; +import org.msgpack.schema.*; + +public class Generate { + public static void main(String[] args) throws IOException + { + String source = + "(class MediaContent"+ + " (package serializers.msgpack)"+ + " (field image (array (class Image"+ + " (field uri string)"+ + " (field title string)"+ + " (field width int)"+ + " (field height int)"+ + " (field size int))))"+ + " (field media (class Media"+ + " (field uri string)"+ + " (field title string)"+ + " (field width int)"+ + " (field height int)"+ + " (field format string)"+ + " (field duration long)"+ + " (field size long)"+ + " (field bitrate int)"+ + " (field person (array string))"+ + " (field player int)"+ + " (field copyright string)))"+ + " )"; + + Schema schema = Schema.parse(source); + + Writer output = new OutputStreamWriter(System.out); + ClassGenerator.write(schema, output); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java new file mode 100644 index 0000000..5dfbc8d --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java @@ -0,0 +1,173 @@ +package serializers.msgpack; + +import java.util.*; +import java.io.*; +import org.msgpack.*; +import org.msgpack.schema.ClassSchema; +import org.msgpack.schema.FieldSchema; + +public final class MediaContent implements MessagePackable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public List image; + public Media media; + + public MediaContent() { } + + @Override + public void messagePack(Packer _pk) throws IOException + { + _pk.packArray(2); + FieldSchema[] _fields = _SCHEMA.getFields(); + _fields[0].getSchema().pack(_pk, image); + _fields[1].getSchema().pack(_pk, media); + } + + @Override + @SuppressWarnings("unchecked") + public void messageMerge(Object obj) throws MessageTypeException + { + Object[] _source = ((List)obj).toArray(); + FieldSchema[] _fields = _SCHEMA.getFields(); + if(_source.length <= 0) { return; } this.image = (List)_fields[0].getSchema().convert(_source[0]); + if(_source.length <= 1) { return; } this.media = (Media)_fields[1].getSchema().convert(_source[1]); + } + + @SuppressWarnings("unchecked") + public static MediaContent createFromMessage(Object[] _message) + { + MediaContent _self = new MediaContent(); + if(_message.length <= 0) { return _self; } _self.image = (List)_message[0]; + if(_message.length <= 1) { return _self; } _self.media = (Media)_message[1]; + return _self; + } +} + +final class Image implements MessagePackable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public Integer size; + + public Image() { } + + @Override + public void messagePack(Packer _pk) throws IOException + { + _pk.packArray(5); + FieldSchema[] _fields = _SCHEMA.getFields(); + _fields[0].getSchema().pack(_pk, uri); + _fields[1].getSchema().pack(_pk, title); + _fields[2].getSchema().pack(_pk, width); + _fields[3].getSchema().pack(_pk, height); + _fields[4].getSchema().pack(_pk, size); + } + + @Override + @SuppressWarnings("unchecked") + public void messageMerge(Object obj) throws MessageTypeException + { + Object[] _source = ((List)obj).toArray(); + FieldSchema[] _fields = _SCHEMA.getFields(); + if(_source.length <= 0) { return; } this.uri = (String)_fields[0].getSchema().convert(_source[0]); + if(_source.length <= 1) { return; } this.title = (String)_fields[1].getSchema().convert(_source[1]); + if(_source.length <= 2) { return; } this.width = (Integer)_fields[2].getSchema().convert(_source[2]); + if(_source.length <= 3) { return; } this.height = (Integer)_fields[3].getSchema().convert(_source[3]); + if(_source.length <= 4) { return; } this.size = (Integer)_fields[4].getSchema().convert(_source[4]); + } + + @SuppressWarnings("unchecked") + public static Image createFromMessage(Object[] _message) + { + Image _self = new Image(); + if(_message.length <= 0) { return _self; } _self.uri = (String)_message[0]; + if(_message.length <= 1) { return _self; } _self.title = (String)_message[1]; + if(_message.length <= 2) { return _self; } _self.width = (Integer)_message[2]; + if(_message.length <= 3) { return _self; } _self.height = (Integer)_message[3]; + if(_message.length <= 4) { return _self; } _self.size = (Integer)_message[4]; + return _self; + } +} + +final class Media implements MessagePackable, MessageMergeable +{ + private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))"); + public static ClassSchema getSchema() { return _SCHEMA; } + + public String uri; + public String title; + public Integer width; + public Integer height; + public String format; + public Long duration; + public Long size; + public Integer bitrate; + public List person; + public Integer player; + public String copyright; + + public Media() { } + + @Override + public void messagePack(Packer _pk) throws IOException + { + _pk.packArray(11); + FieldSchema[] _fields = _SCHEMA.getFields(); + _fields[0].getSchema().pack(_pk, uri); + _fields[1].getSchema().pack(_pk, title); + _fields[2].getSchema().pack(_pk, width); + _fields[3].getSchema().pack(_pk, height); + _fields[4].getSchema().pack(_pk, format); + _fields[5].getSchema().pack(_pk, duration); + _fields[6].getSchema().pack(_pk, size); + _fields[7].getSchema().pack(_pk, bitrate); + _fields[8].getSchema().pack(_pk, person); + _fields[9].getSchema().pack(_pk, player); + _fields[10].getSchema().pack(_pk, copyright); + } + + @Override + @SuppressWarnings("unchecked") + public void messageMerge(Object obj) throws MessageTypeException + { + Object[] _source = ((List)obj).toArray(); + FieldSchema[] _fields = _SCHEMA.getFields(); + if(_source.length <= 0) { return; } this.uri = (String)_fields[0].getSchema().convert(_source[0]); + if(_source.length <= 1) { return; } this.title = (String)_fields[1].getSchema().convert(_source[1]); + if(_source.length <= 2) { return; } this.width = (Integer)_fields[2].getSchema().convert(_source[2]); + if(_source.length <= 3) { return; } this.height = (Integer)_fields[3].getSchema().convert(_source[3]); + if(_source.length <= 4) { return; } this.format = (String)_fields[4].getSchema().convert(_source[4]); + if(_source.length <= 5) { return; } this.duration = (Long)_fields[5].getSchema().convert(_source[5]); + if(_source.length <= 6) { return; } this.size = (Long)_fields[6].getSchema().convert(_source[6]); + if(_source.length <= 7) { return; } this.bitrate = (Integer)_fields[7].getSchema().convert(_source[7]); + if(_source.length <= 8) { return; } this.person = (List)_fields[8].getSchema().convert(_source[8]); + if(_source.length <= 9) { return; } this.player = (Integer)_fields[9].getSchema().convert(_source[9]); + if(_source.length <= 10) { return; } this.copyright = (String)_fields[10].getSchema().convert(_source[10]); + } + + @SuppressWarnings("unchecked") + public static Media createFromMessage(Object[] _message) + { + Media _self = new Media(); + if(_message.length <= 0) { return _self; } _self.uri = (String)_message[0]; + if(_message.length <= 1) { return _self; } _self.title = (String)_message[1]; + if(_message.length <= 2) { return _self; } _self.width = (Integer)_message[2]; + if(_message.length <= 3) { return _self; } _self.height = (Integer)_message[3]; + if(_message.length <= 4) { return _self; } _self.format = (String)_message[4]; + if(_message.length <= 5) { return _self; } _self.duration = (Long)_message[5]; + if(_message.length <= 6) { return _self; } _self.size = (Long)_message[6]; + if(_message.length <= 7) { return _self; } _self.bitrate = (Integer)_message[7]; + if(_message.length <= 8) { return _self; } _self.person = (List)_message[8]; + if(_message.length <= 9) { return _self; } _self.player = (Integer)_message[9]; + if(_message.length <= 10) { return _self; } _self.copyright = (String)_message[10]; + return _self; + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs new file mode 100644 index 0000000..547ba48 --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs @@ -0,0 +1,21 @@ +(class MediaContent + (package serializers.msgpack) + (field image (array (class Image + (field uri string) + (field title string) + (field width int) + (field height int) + (field size int)))) + (field media (class Media + (field uri string) + (field title string) + (field width int) + (field height int) + (field format string) + (field duration long) + (field size long) + (field bitrate int) + (field person (array string)) + (field player int) + (field copyright string))) + ) diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java new file mode 100644 index 0000000..c8a88ac --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java @@ -0,0 +1,68 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackDynamicSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-dynamic"; + } + + public Object create() throws Exception { + ArrayList media = new ArrayList(11); + media.add("http://javaone.com/keynote.mpg"); + media.add("video/mpg4"); + media.add("Javaone Keynote"); + media.add(1234567L); + media.add(0); + ArrayList person = new ArrayList(2); + person.add("Bill Gates"); + person.add("Steve Jobs"); + media.add(person); + media.add(0); + media.add(0); + media.add(0); + media.add(123L); + media.add(""); + + ArrayList image1 = new ArrayList(5); + image1.add("http://javaone.com/keynote_large.jpg"); + image1.add(0); + image1.add(0); + image1.add(2); + image1.add("Javaone Keynote"); + + ArrayList image2 = new ArrayList(5); + image2.add("http://javaone.com/keynote_thumbnail.jpg"); + image2.add(0); + image2.add(0); + image2.add(1); + image2.add("Javaone Keynote"); + + ArrayList content = new ArrayList(2); + content.add(media); + ArrayList images = new ArrayList(2); + images.add(image1); + images.add(image2); + content.add(images); + + return content; + } + + public byte[] serialize(Object content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } + + public Object deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker(); + pac.execute(array); + return (Object)pac.getData(); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java new file mode 100644 index 0000000..4935899 --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java @@ -0,0 +1,70 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackGenericSerializer implements ObjectSerializer +{ + private static final Schema MEDIA_CONTENT_SCHEMA = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); + + public String getName() { + return "msgpack-generic"; + } + + public Object create() throws Exception { + HashMap media = new HashMap(11); + media.put("uri", "http://javaone.com/keynote.mpg"); + media.put("format", "video/mpg4"); + media.put("title", "Javaone Keynote"); + media.put("duration", 1234567L); + media.put("bitrate", 0); + ArrayList person = new ArrayList(2); + person.add("Bill Gates"); + person.add("Steve Jobs"); + media.put("person", person); + media.put("player", 0); + media.put("height", 0); + media.put("width", 0); + media.put("size", 123L); + media.put("copyright", ""); + + HashMap image1 = new HashMap(5); + image1.put("uri", "http://javaone.com/keynote_large.jpg"); + image1.put("width", 0); + image1.put("height", 0); + image1.put("size", 2); + image1.put("title", "Javaone Keynote"); + + HashMap image2 = new HashMap(5); + image2.put("uri", "http://javaone.com/keynote_thumbnail.jpg"); + image2.put("width", 0); + image2.put("height", 0); + image2.put("size", 1); + image2.put("title", "Javaone Keynote"); + + HashMap content = new HashMap(2); + content.put("media", media); + ArrayList images = new ArrayList(2); + images.add(image1); + images.add(image2); + content.put("image", images); + + return content; + } + + public byte[] serialize(Object content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.packWithSchema(content, MEDIA_CONTENT_SCHEMA); + return os.toByteArray(); + } + + public Object deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MEDIA_CONTENT_SCHEMA); + pac.execute(array); + return (Object)pac.getData(); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java new file mode 100644 index 0000000..2767474 --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java @@ -0,0 +1,67 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackIndirectSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-indirect"; + } + + public MediaContent create() throws Exception { + Media media = new Media(); + media.uri = "http://javaone.com/keynote.mpg"; + media.format = "video/mpg4"; + media.title = "Javaone Keynote"; + media.duration = 1234567L; + media.bitrate = 0; + media.person = new ArrayList(2); + media.person.add("Bill Gates"); + media.person.add("Steve Jobs"); + media.player = 0; + media.height = 0; + media.width = 0; + media.size = 123L; + media.copyright = ""; + + Image image1 = new Image(); + image1.uri = "http://javaone.com/keynote_large.jpg"; + image1.width = 0; + image1.height = 0; + image1.size = 2; + image1.title = "Javaone Keynote"; + + Image image2 = new Image(); + image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; + image2.width = 0; + image2.height = 0; + image2.size = 1; + image2.title = "Javaone Keynote"; + + MediaContent content = new MediaContent(); + content.media = media; + content.image = new ArrayList(2); + content.image.add(image1); + content.image.add(image2); + + return content; + } + + public byte[] serialize(MediaContent content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } + + public MediaContent deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker(); + pac.execute(array); + Object obj = pac.getData(); + return (MediaContent)MediaContent.getSchema().convert(obj); + } +} + diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java new file mode 100644 index 0000000..91ded5c --- /dev/null +++ b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java @@ -0,0 +1,66 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackSpecificSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-specific"; + } + + public MediaContent create() throws Exception { + Media media = new Media(); + media.uri = "http://javaone.com/keynote.mpg"; + media.format = "video/mpg4"; + media.title = "Javaone Keynote"; + media.duration = 1234567L; + media.bitrate = 0; + media.person = new ArrayList(2); + media.person.add("Bill Gates"); + media.person.add("Steve Jobs"); + media.player = 0; + media.height = 0; + media.width = 0; + media.size = 123L; + media.copyright = ""; + + Image image1 = new Image(); + image1.uri = "http://javaone.com/keynote_large.jpg"; + image1.width = 0; + image1.height = 0; + image1.size = 2; + image1.title = "Javaone Keynote"; + + Image image2 = new Image(); + image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; + image2.width = 0; + image2.height = 0; + image2.size = 1; + image2.title = "Javaone Keynote"; + + MediaContent content = new MediaContent(); + content.media = media; + content.image = new ArrayList(2); + content.image.add(image1); + content.image.add(image2); + + return content; + } + + public byte[] serialize(MediaContent content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } + + public MediaContent deserialize(byte[] array) throws Exception { + UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MediaContent.getSchema()); + pac.execute(array); + return (MediaContent)pac.getData(); + } +} + From 0ae1965f6b492efb71f7457ba1dec48ae6110399 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 04:32:33 +0900 Subject: [PATCH 0315/1648] ruby: fixes MessagePack_Unpacker_mark marks uninitialized map_key --- ruby/unpack.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index 411a94d..e9d6494 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -127,6 +127,15 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha static VALUE cUnpacker; static VALUE eUnpackError; +// FIXME slow operation +static void init_stack(msgpack_unpack_t* mp) +{ + size_t i; + for(i=0; i < MSGPACK_MAX_STACK_SIZE; ++i) { + mp->stack[i].map_key = Qnil; /* GC */ + } +} + static void MessagePack_Unpacker_free(void* data) { if(data) { free(data); } @@ -137,7 +146,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) unsigned int i; for(i=0; i < mp->top; ++i) { rb_gc_mark(mp->stack[i].obj); - rb_gc_mark(mp->stack[i].map_key); + rb_gc_mark(mp->stack[i].map_key); /* maybe map_key is not initialized */ } } @@ -154,6 +163,7 @@ static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); template_init(mp); + init_stack(mp); unpack_user u = {0, Qnil}; mp->user = u; return self; @@ -281,6 +291,7 @@ static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) msgpack_unpack_t mp; template_init(&mp); + init_stack(&mp); unpack_user u = {0, Qnil}; mp.user = u; From 7ce866ad7c2884f0b01ee77d99d2f9e01217fcdf Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 06:19:53 +0900 Subject: [PATCH 0316/1648] msgpack template: architecture specific endian conversion --- c/object.c | 10 ++ c/object.h | 4 +- c/pack.h | 4 +- c/unpack.c | 35 ++++-- c/unpack.h | 6 +- c/vrefbuffer.h | 9 ++ cpp/object.hpp | 1 - cpp/pack.hpp | 3 +- msgpack/pack_define.h | 3 +- msgpack/pack_template.h | 253 +++++++++++++++----------------------- msgpack/unpack_define.h | 39 +----- msgpack/unpack_template.h | 6 +- 12 files changed, 154 insertions(+), 219 deletions(-) diff --git a/c/object.c b/c/object.c index bcb2537..a22ce21 100644 --- a/c/object.c +++ b/c/object.c @@ -18,7 +18,17 @@ #include "msgpack/object.h" #include "msgpack/pack.h" #include + +#ifndef _MSC_VER #include +#else +#ifndef PRIu64 +#define PRIu64 "I64u" +#endif +#ifndef PRIi64 +#define PRIi64 "I64d" +#endif +#endif int msgpack_pack_object(msgpack_packer* pk, msgpack_object d) diff --git a/c/object.h b/c/object.h index 7c603b3..0aed0e4 100644 --- a/c/object.h +++ b/c/object.h @@ -19,9 +19,7 @@ #define MSGPACK_OBJECT_H__ #include "msgpack/zone.h" -#include -#include -#include +#include "msgpack/sys.h" #include #ifdef __cplusplus diff --git a/c/pack.h b/c/pack.h index 1a57ea4..1525e0f 100644 --- a/c/pack.h +++ b/c/pack.h @@ -18,11 +18,9 @@ #ifndef MSGPACK_PACK_H__ #define MSGPACK_PACK_H__ -#include -#include -#include #include "msgpack/pack_define.h" #include "msgpack/object.h" +#include #ifdef __cplusplus extern "C" { diff --git a/c/unpack.c b/c/unpack.c index 08fd6cb..4d9af9e 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -101,7 +101,7 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac { o->type = MSGPACK_OBJECT_ARRAY; o->via.array.size = 0; - o->via.array.ptr = msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); + o->via.array.ptr = (msgpack_object*)msgpack_zone_malloc(u->z, n*sizeof(msgpack_object)); if(o->via.array.ptr == NULL) { return -1; } return 0; } @@ -142,30 +142,47 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced -static const size_t COUNTER_SIZE = sizeof(unsigned int); +#ifndef _MSC_VER +typedef unsigned int counter_t; +#else +typedef long counter_t; +#endif + +#define COUNTER_SIZE (sizeof(volatile counter_t)) + static inline void init_count(void* buffer) { - *(volatile unsigned int*)buffer = 1; + *(volatile counter_t*)buffer = 1; } static inline void decl_count(void* buffer) { - //if(--*(unsigned int*)buffer == 0) { - if(__sync_sub_and_fetch((unsigned int*)buffer, 1) == 0) { + // atomic if(--*(counter_t*)buffer == 0) { free(buffer); } + if( +#ifndef _MSC_VER + __sync_sub_and_fetch((counter_t*)buffer, 1) == 0 +#else + InterlockedDecrement((volatile counter_t*)&buffer) == 0 +#endif + ) { free(buffer); } } static inline void incr_count(void* buffer) { - //++*(unsigned int*)buffer; - __sync_add_and_fetch((unsigned int*)buffer, 1); + // atomic ++*(counter_t*)buffer; +#ifndef _MSC_VER + __sync_add_and_fetch((counter_t*)buffer, 1); +#else + InterlockedIncrement((volatile counter_t*)&buffer); +#endif } -static inline unsigned int get_count(void* buffer) +static inline counter_t get_count(void* buffer) { - return *(volatile unsigned int*)buffer; + return *(volatile counter_t*)buffer; } diff --git a/c/unpack.h b/c/unpack.h index ef63774..e17d0d8 100644 --- a/c/unpack.h +++ b/c/unpack.h @@ -15,13 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef msgpack_unpacker_H__ -#define msgpack_unpacker_H__ +#ifndef MSGPACK_UNPACKER_H__ +#define MSGPACK_UNPACKER_H__ #include "msgpack/zone.h" #include "msgpack/object.h" -#include -#include #ifdef __cplusplus extern "C" { diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h index baa7c03..063075f 100644 --- a/c/vrefbuffer.h +++ b/c/vrefbuffer.h @@ -19,7 +19,16 @@ #define MSGPACK_VREFBUFFER_H__ #include "msgpack/zone.h" + +#ifndef _WIN32 #include +#else +struct iovec { + void *iov_base; + size_t iov_len; +}; +#endif + #ifdef __cplusplus extern "C" { diff --git a/cpp/object.hpp b/cpp/object.hpp index 9ee575f..ed2e290 100644 --- a/cpp/object.hpp +++ b/cpp/object.hpp @@ -20,7 +20,6 @@ #include "msgpack/object.h" #include "msgpack/pack.hpp" -#include #include #include #include diff --git a/cpp/pack.hpp b/cpp/pack.hpp index c8e37eb..9c291c1 100644 --- a/cpp/pack.hpp +++ b/cpp/pack.hpp @@ -18,10 +18,9 @@ #ifndef MSGPACK_PACK_HPP__ #define MSGPACK_PACK_HPP__ -#include // __BYTE_ORDER +#include "msgpack/pack_define.h" #include #include -#include "msgpack/pack_define.h" namespace msgpack { diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index 33408e5..aef9295 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -18,8 +18,7 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include -#include +#include "msgpack/sys.h" #include #endif /* msgpack/pack_define.h */ diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index aa620f5..ffbbbba 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -16,88 +16,16 @@ * limitations under the License. */ -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - - #ifdef __LITTLE_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[0] +#define TAKE8_32(d) ((uint8_t*)&d)[0] +#define TAKE8_64(d) ((uint8_t*)&d)[0] #elif __BIG_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[1] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[3] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[7] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[1] +#define TAKE8_32(d) ((uint8_t*)&d)[3] +#define TAKE8_64(d) ((uint8_t*)&d)[7] #endif #ifndef msgpack_pack_inline_func @@ -121,10 +49,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -133,14 +61,15 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -150,20 +79,22 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -174,24 +105,27 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -201,11 +135,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } \ } while(0) @@ -214,24 +148,26 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -242,32 +178,36 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -279,46 +219,52 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -330,49 +276,55 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -604,7 +556,8 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + unsigned char buf[5]; + buf[0] = 0xca; *(uint32_t*)&buf[1] = msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -612,7 +565,8 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + unsigned char buf[9]; + buf[0] = 0xcb; *(uint64_t*)&buf[1] = msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } @@ -655,12 +609,12 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) unsigned char d = 0x90 | n; msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xdc; *(uint16_t*)&buf[1] = msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdd; *(uint32_t*)&buf[1] = msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -674,14 +628,14 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xde; *(uint16_t*)&buf[1] = msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdf; *(uint32_t*)&buf[1] = msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -695,14 +649,14 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { - uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xda; *(uint16_t*)&buf[1] = msgpack_be16(l); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdb; *(uint32_t*)&buf[1] = msgpack_be32(l); msgpack_pack_append_buffer(x, buf, 5); } } @@ -716,19 +670,10 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE8_BE8 - -#undef STORE16_BE8 -#undef STORE16_BE16 - -#undef STORE32_BE8 -#undef STORE32_BE16 -#undef STORE32_BE32 - -#undef STORE64_BE8 -#undef STORE64_BE16 -#undef STORE64_BE32 -#undef STORE64_BE64 +#undef TAKE8_8 +#undef TAKE8_16 +#undef TAKE8_32 +#undef TAKE8_64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 63668c2..027d409 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -18,14 +18,10 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include -#include +#include "msgpack/sys.h" #include #include #include -#ifndef __WIN32__ -#include -#endif #ifdef __cplusplus extern "C" { @@ -37,39 +33,6 @@ extern "C" { #endif -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - -#define msgpack_betoh16(x) ntohs(x) -#define msgpack_betoh32(x) ntohl(x) - -#ifdef __LITTLE_ENDIAN__ -#if defined(__bswap_64) -# define msgpack_betoh64(x) __bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) -#else -static inline uint64_t msgpack_betoh64(uint64_t x) { - return ((x << 56) & 0xff00000000000000ULL ) | - ((x << 40) & 0x00ff000000000000ULL ) | - ((x << 24) & 0x0000ff0000000000ULL ) | - ((x << 8) & 0x000000ff00000000ULL ) | - ((x >> 8) & 0x00000000ff000000ULL ) | - ((x >> 24) & 0x0000000000ff0000ULL ) | - ((x >> 40) & 0x000000000000ff00ULL ) | - ((x >> 56) & 0x00000000000000ffULL ) ; -} -#endif -#else -#define msgpack_betoh64(x) (x) -#endif - - typedef enum { CS_HEADER = 0x00, // nil diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index d67fd1e..212a47e 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -126,9 +126,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_be16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_be32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_be64(*(uint64_t*)ptr) if(p == pe) { goto _out; } do { From ba3ba0367cefdb2e87425f257711865e2e540adc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 06:40:29 +0900 Subject: [PATCH 0317/1648] msgpack template: macros for compilers that doesn't not support Case Ranges --- msgpack/unpack_template.h | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 212a47e..6f99d54 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -40,6 +40,11 @@ #error msgpack_unpack_user type is not defined #endif +#ifndef USE_CASE_RANGE +#if !defined(_MSC_VER) +#define USE_CASE_RANGE +#endif +#endif msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; @@ -130,16 +135,28 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c #define PTR_CAST_32(ptr) msgpack_be32(*(uint32_t*)ptr) #define PTR_CAST_64(ptr) msgpack_be64(*(uint64_t*)ptr) +#ifdef USE_CASE_RANGE +#define SWITCH_RANGE_BEGIN switch(*p) { +#define SWITCH_RANGE(FROM, TO) case FROM ... TO: +#define SWITCH_RANGE_DEFAULT default: +#define SWITCH_RANGE_END } +#else +#define SWITCH_RANGE_BEGIN { if(0) { +#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) { +#define SWITCH_RANGE_DEFAULT } else { +#define SWITCH_RANGE_END } } +#endif + if(p == pe) { goto _out; } do { switch(cs) { case CS_HEADER: - switch(*p) { - case 0x00 ... 0x7f: // Positive Fixnum + SWITCH_RANGE_BEGIN + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum push_fixed_value(_uint8, *(uint8_t*)p); - case 0xe0 ... 0xff: // Negative Fixnum + SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum push_fixed_value(_int8, *(int8_t*)p); - case 0xc0 ... 0xdf: // Variable + SWITCH_RANGE(0xc0, 0xdf) // Variable switch(*p) { case 0xc0: // nil push_simple_value(_nil); @@ -182,16 +199,16 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c default: goto _failed; } - case 0xa0 ... 0xbf: // FixRaw + SWITCH_RANGE(0xa0, 0xbf) // FixRaw again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - case 0x90 ... 0x9f: // FixArray + SWITCH_RANGE(0x90, 0x9f) // FixArray start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - case 0x80 ... 0x8f: // FixMap + SWITCH_RANGE(0x80, 0x8f) // FixMap start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - default: + SWITCH_RANGE_DEFAULT goto _failed; - } + SWITCH_RANGE_END // end CS_HEADER From 35929b46ae1ed8cd001d8b0964ec3a1bb1c053e1 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 10 Dec 2009 07:22:39 +0900 Subject: [PATCH 0318/1648] add msgpack/sysdep.h --- Makefile.am | 3 +- c/object.h | 2 +- c/unpack.c | 33 ++++---------- configure.in | 2 +- msgpack/pack_define.h | 2 +- msgpack/pack_template.h | 64 +++++++++++++------------- msgpack/sysdep.h | 94 +++++++++++++++++++++++++++++++++++++++ msgpack/unpack_define.h | 2 +- msgpack/unpack_template.h | 6 +-- 9 files changed, 143 insertions(+), 65 deletions(-) create mode 100644 msgpack/sysdep.h diff --git a/Makefile.am b/Makefile.am index be3d75f..42fb233 100644 --- a/Makefile.am +++ b/Makefile.am @@ -8,5 +8,6 @@ nobase_include_HEADERS = \ msgpack/pack_define.h \ msgpack/pack_template.h \ msgpack/unpack_define.h \ - msgpack/unpack_template.h + msgpack/unpack_template.h \ + msgpack/sysdep.h diff --git a/c/object.h b/c/object.h index 0aed0e4..27b593e 100644 --- a/c/object.h +++ b/c/object.h @@ -19,7 +19,7 @@ #define MSGPACK_OBJECT_H__ #include "msgpack/zone.h" -#include "msgpack/sys.h" +#include "msgpack/sysdep.h" #include #ifdef __cplusplus diff --git a/c/unpack.c b/c/unpack.c index 4d9af9e..6a435ba 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -141,48 +141,31 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #define CTX_CAST(m) ((template_context*)(m)) #define CTX_REFERENCED(mpac) CTX_CAST((mpac)->ctx)->user.referenced - -#ifndef _MSC_VER -typedef unsigned int counter_t; -#else -typedef long counter_t; -#endif - -#define COUNTER_SIZE (sizeof(volatile counter_t)) +#define COUNTER_SIZE (sizeof(_msgpack_atomic_counter_t)) static inline void init_count(void* buffer) { - *(volatile counter_t*)buffer = 1; + *(volatile _msgpack_atomic_counter_t*)buffer = 1; } static inline void decl_count(void* buffer) { - // atomic if(--*(counter_t*)buffer == 0) { free(buffer); } - if( -#ifndef _MSC_VER - __sync_sub_and_fetch((counter_t*)buffer, 1) == 0 -#else - InterlockedDecrement((volatile counter_t*)&buffer) == 0 -#endif - ) { + // atomic if(--*(_msgpack_atomic_counter_t*)buffer == 0) { free(buffer); } + if(_msgpack_sync_decr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer)) { free(buffer); } } static inline void incr_count(void* buffer) { - // atomic ++*(counter_t*)buffer; -#ifndef _MSC_VER - __sync_add_and_fetch((counter_t*)buffer, 1); -#else - InterlockedIncrement((volatile counter_t*)&buffer); -#endif + // atomic ++*(_msgpack_atomic_counter_t*)buffer; + _msgpack_sync_incr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer); } -static inline counter_t get_count(void* buffer) +static inline _msgpack_atomic_counter_t get_count(void* buffer) { - return *(volatile counter_t*)buffer; + return *(volatile _msgpack_atomic_counter_t*)buffer; } diff --git a/configure.in b/configure.in index c2ca872..76bd7e4 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.8) +AM_INIT_AUTOMAKE(msgpack, 0.3.9) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index aef9295..692ef7d 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -18,7 +18,7 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include "msgpack/sys.h" +#include "msgpack/sysdep.h" #include #endif /* msgpack/pack_define.h */ diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index ffbbbba..de148bf 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -69,7 +69,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -89,12 +89,12 @@ do { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -115,17 +115,17 @@ do { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -149,7 +149,7 @@ do { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -167,7 +167,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -179,12 +179,12 @@ do { \ if(d < -(1<<15)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -202,12 +202,12 @@ do { \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -220,19 +220,19 @@ do { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); \ + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -252,19 +252,19 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -283,21 +283,21 @@ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { unsigned char buf[3]; - buf[0] = 0xcd; *(uint16_t*)&buf[1] = msgpack_be16(d); + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { unsigned char buf[5]; - buf[0] = 0xce; *(uint32_t*)&buf[1] = msgpack_be32(d); + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { unsigned char buf[9]; - buf[0] = 0xcf; *(uint64_t*)&buf[1] = msgpack_be64(d); + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -310,21 +310,21 @@ msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { unsigned char buf[3]; - buf[0] = 0xd1; *(uint16_t*)&buf[1] = msgpack_be16(d); + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { unsigned char buf[5]; - buf[0] = 0xd2; *(uint32_t*)&buf[1] = msgpack_be32(d); + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { unsigned char buf[9]; - buf[0] = 0xd3; *(uint64_t*)&buf[1] = msgpack_be64(d); + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -557,7 +557,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME unsigned char buf[5]; - buf[0] = 0xca; *(uint32_t*)&buf[1] = msgpack_be32(f.num); + buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -566,7 +566,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME unsigned char buf[9]; - buf[0] = 0xcb; *(uint64_t*)&buf[1] = msgpack_be64(f.num); + buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } @@ -610,11 +610,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xdc; *(uint16_t*)&buf[1] = msgpack_be16(n); + buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdd; *(uint32_t*)&buf[1] = msgpack_be32(n); + buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -631,11 +631,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xde; *(uint16_t*)&buf[1] = msgpack_be16(n); + buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdf; *(uint32_t*)&buf[1] = msgpack_be32(n); + buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -652,11 +652,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { unsigned char buf[3]; - buf[0] = 0xda; *(uint16_t*)&buf[1] = msgpack_be16(l); + buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdb; *(uint32_t*)&buf[1] = msgpack_be32(l); + buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l); msgpack_pack_append_buffer(x, buf, 5); } } diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h new file mode 100644 index 0000000..106158e --- /dev/null +++ b/msgpack/sysdep.h @@ -0,0 +1,94 @@ +/* + * MessagePack system dependencies + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_SYSDEP_H__ +#define MSGPACK_SYSDEP_H__ + + +#ifdef _MSC_VER +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else +#include +#include +#include +#endif + + +#ifdef _WIN32 +typedef long _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr) +#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr) +#else +typedef unsigned int _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1) +#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1) +#endif + + +#ifdef _WIN32 +#include +#else +#include /* __BYTE_ORDER */ +#endif + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#ifdef __LITTLE_ENDIAN__ + +#define _msgpack_be16(x) ntohs(x) +#define _msgpack_be32(x) ntohl(x) + +#if defined(_byteswap_uint64) +# define _msgpack_be64(x) (_byteswap_uint64(x)) +#elif defined(bswap_64) +# define _msgpack_be64(x) bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x) +#else +#define _msgpack_be64(x) \ + ( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \ + ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ + ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ + ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ + ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ + ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ + ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ + ((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) ) +#endif + +#else +#define _msgpack_be16(x) (x) +#define _msgpack_be32(x) (x) +#define _msgpack_be64(x) (x) +#endif + + +#endif /* msgpack/sysdep.h */ + diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 027d409..c36aa51 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -18,7 +18,7 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include "msgpack/sys.h" +#include "msgpack/sysdep.h" #include #include #include diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 6f99d54..3328ea3 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -131,9 +131,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) msgpack_be16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) msgpack_be32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) msgpack_be64(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr) #ifdef USE_CASE_RANGE #define SWITCH_RANGE_BEGIN switch(*p) { From 0d44348c7dabdbfee4ac1ee17bfb141dfe8706f2 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 11 Dec 2009 04:13:24 +0900 Subject: [PATCH 0319/1648] ruby: version 0.3.2 --- ruby/gem/Rakefile | 6 +++--- ruby/gengem.sh | 1 + ruby/msgpack.gemspec | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile index 5445906..f2fd684 100644 --- a/ruby/gem/Rakefile +++ b/ruby/gem/Rakefile @@ -15,9 +15,9 @@ AUTHOR = "FURUHASHI Sadayuki" EMAIL = "frsyuki _at_ users.sourceforge.jp" DESCRIPTION = "Binary-based efficient data interchange format." RUBYFORGE_PROJECT = "msgpack" -HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org" +HOMEPATH = "http://msgpack.sourceforge.jp/" BIN_FILES = %w( ) -VERS = "0.3.1" +VERS = "0.3.2" #REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil REV = nil @@ -44,7 +44,7 @@ spec = Gem::Specification.new do |s| s.name = NAME s.version = VERS s.platform = Gem::Platform::RUBY - s.has_rdoc = true + s.has_rdoc = false s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] s.rdoc_options += RDOC_OPTS + ['--exclude', '^(examples|extras)/'] s.summary = DESCRIPTION diff --git a/ruby/gengem.sh b/ruby/gengem.sh index 0afb8f5..359debf 100755 --- a/ruby/gengem.sh +++ b/ruby/gengem.sh @@ -15,6 +15,7 @@ cp ../msgpack/pack_define.h gem/msgpack/ cp ../msgpack/pack_template.h gem/msgpack/ cp ../msgpack/unpack_define.h gem/msgpack/ cp ../msgpack/unpack_template.h gem/msgpack/ +cp ../msgpack/sysdep.h gem/msgpack/ cd gem && rake --trace package diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index f0b5c44..59186a4 100755 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.3.1" + s.version = "0.3.2" s.summary = "MessagePack" s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From 5aa47d667783476277409b06d5829322a801df05 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 16 Dec 2009 03:52:14 +0900 Subject: [PATCH 0320/1648] cpp: zone::push_finalizer supports std::auto_ptr --- cpp/zone.hpp.erb | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index 8fd14a6..f1e4624 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -21,6 +21,7 @@ #include "msgpack/object.hpp" #include "msgpack/zone.h" #include +#include #include <% GENERATION_LIMIT = 15 %> @@ -38,6 +39,9 @@ public: void push_finalizer(void (*func)(void*), void* data); + template + void push_finalizer(std::auto_ptr obj); + void clear(); <%0.upto(GENERATION_LIMIT) {|i|%> @@ -94,6 +98,15 @@ inline void zone::push_finalizer(void (*func)(void*), void* data) } } +template +inline void zone::push_finalizer(std::auto_ptr obj) +{ + if(!msgpack_zone_push_finalizer(this, &zone::object_destructor, obj.get())) { + throw std::bad_alloc(); + } + obj.release(); +} + inline void zone::clear() { msgpack_zone_clear(this); From 686e8ca0f004004f4b8e10438fe91a48a95e6ff9 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 16 Dec 2009 04:08:36 +0900 Subject: [PATCH 0321/1648] c,cpp: fix unpacker --- c/unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c/unpack.c b/c/unpack.c index 6a435ba..d5bcb2d 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -152,7 +152,7 @@ static inline void init_count(void* buffer) static inline void decl_count(void* buffer) { // atomic if(--*(_msgpack_atomic_counter_t*)buffer == 0) { free(buffer); } - if(_msgpack_sync_decr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer)) { + if(_msgpack_sync_decr_and_fetch((volatile _msgpack_atomic_counter_t*)buffer) == 0) { free(buffer); } } From dd18402737bdd12fbf53fe0543e29509f9609f3f Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:05:31 +0900 Subject: [PATCH 0322/1648] Fix stream unpacker broken. --- python/msgpack/_msgpack.pyx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index cb95146..9ff0c57 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -233,7 +233,9 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=1024*1024): + def __init__(self, file_like=None, int read_size=0): + if read_size == 0: + read_size = 1024*1024 self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -309,6 +311,7 @@ cdef class Unpacker(object): self.fill_buffer() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: + template_init(&self.ctx) return template_data(&self.ctx) elif ret == 0: if self.file_like is not None: @@ -319,3 +322,10 @@ cdef class Unpacker(object): def __iter__(self): return UnpackIterator(self) + + # for debug. + #def _buf(self): + # return PyString_FromStringAndSize(self.buf, self.buf_tail) + + #def _off(self): + # return self.buf_head From 35a4d612c9f45c238f78f37013a4be4c9d74c13b Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:05:31 +0900 Subject: [PATCH 0323/1648] Fix stream unpacker broken. --- msgpack/_msgpack.pyx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index cb95146..9ff0c57 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -233,7 +233,9 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=1024*1024): + def __init__(self, file_like=None, int read_size=0): + if read_size == 0: + read_size = 1024*1024 self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -309,6 +311,7 @@ cdef class Unpacker(object): self.fill_buffer() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: + template_init(&self.ctx) return template_data(&self.ctx) elif ret == 0: if self.file_like is not None: @@ -319,3 +322,10 @@ cdef class Unpacker(object): def __iter__(self): return UnpackIterator(self) + + # for debug. + #def _buf(self): + # return PyString_FromStringAndSize(self.buf, self.buf_tail) + + #def _off(self): + # return self.buf_head From 5ff2c6be74cb61363995d43772c5a52566b19000 Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:14:13 +0900 Subject: [PATCH 0324/1648] Fix bug come from previous commit --- python/msgpack/_msgpack.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index 9ff0c57..dcabc0f 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -311,8 +311,9 @@ cdef class Unpacker(object): self.fill_buffer() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: + o = template_data(&self.ctx) template_init(&self.ctx) - return template_data(&self.ctx) + return o elif ret == 0: if self.file_like is not None: return self.unpack() From bb93f093b6a7c4bbba46fe73f08fd8e8c7325443 Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:14:13 +0900 Subject: [PATCH 0325/1648] Fix bug come from previous commit --- msgpack/_msgpack.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index 9ff0c57..dcabc0f 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -311,8 +311,9 @@ cdef class Unpacker(object): self.fill_buffer() ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head) if ret == 1: + o = template_data(&self.ctx) template_init(&self.ctx) - return template_data(&self.ctx) + return o elif ret == 0: if self.file_like is not None: return self.unpack() From 1ed4236bcf1f96e997dc453b2f88a7c4e385688c Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:18:17 +0900 Subject: [PATCH 0326/1648] Make new Python release. --- python/setup.py | 2 +- python/setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index b26b9e8..ee37098 100755 --- a/python/setup.py +++ b/python/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.1' +version = '0.1.2' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/python/setup_dev.py b/python/setup_dev.py index e28ef25..abed7cd 100755 --- a/python/setup_dev.py +++ b/python/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.1dev' +version = '0.1.2dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 52a1ace9eff2e8e7ce57a26ef331a05e1a75ad93 Mon Sep 17 00:00:00 2001 From: inada-n Date: Wed, 16 Dec 2009 22:18:17 +0900 Subject: [PATCH 0327/1648] Make new Python release. --- setup.py | 2 +- setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b26b9e8..ee37098 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.1' +version = '0.1.2' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/setup_dev.py b/setup_dev.py index e28ef25..abed7cd 100755 --- a/setup_dev.py +++ b/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.1dev' +version = '0.1.2dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 3a5f7f53ff26c73396a211157d3d5c713bfc1b8a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 10:43:01 +0900 Subject: [PATCH 0328/1648] Start 0.2.0 --- python/setup.py | 2 +- python/setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index ee37098..c48be14 100755 --- a/python/setup.py +++ b/python/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.2' +version = '0.2.0dev' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/python/setup_dev.py b/python/setup_dev.py index abed7cd..4efc769 100755 --- a/python/setup_dev.py +++ b/python/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.2dev' +version = '0.2.0dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From fecbeb6d075ab996040c5d1ced3f6b3b6ed9faba Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 10:43:01 +0900 Subject: [PATCH 0329/1648] Start 0.2.0 --- setup.py | 2 +- setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ee37098..c48be14 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension #from Cython.Distutils import build_ext import os -version = '0.1.2' +version = '0.2.0dev' msgpack_mod = Extension('msgpack._msgpack', #sources=['msgpack/_msgpack.pyx'] diff --git a/setup_dev.py b/setup_dev.py index abed7cd..4efc769 100755 --- a/setup_dev.py +++ b/setup_dev.py @@ -5,7 +5,7 @@ from distutils.core import setup, Extension from Cython.Distutils import build_ext import os -version = '0.1.2dev' +version = '0.2.0dev' msgpack_mod = Extension('msgpack._msgpack', sources=['msgpack/_msgpack.pyx'] From 4d33bd456cb31e0aeb3a88128e2cebe89d116ce1 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 10:43:22 +0900 Subject: [PATCH 0330/1648] Update new headers. --- python/msgpack/pack.h | 1 + python/msgpack/pack_define.h | 3 +- python/msgpack/pack_template.h | 253 ++++++++++++------------------- python/msgpack/sysdep.h | 94 ++++++++++++ python/msgpack/unpack_define.h | 52 +------ python/msgpack/unpack_template.h | 41 +++-- 6 files changed, 225 insertions(+), 219 deletions(-) create mode 100644 python/msgpack/sysdep.h diff --git a/python/msgpack/pack.h b/python/msgpack/pack.h index 58f021e..2ae95d1 100644 --- a/python/msgpack/pack.h +++ b/python/msgpack/pack.h @@ -18,6 +18,7 @@ #include #include +#include "sysdep.h" #include "pack_define.h" #ifdef __cplusplus diff --git a/python/msgpack/pack_define.h b/python/msgpack/pack_define.h index 33408e5..f72391b 100644 --- a/python/msgpack/pack_define.h +++ b/python/msgpack/pack_define.h @@ -18,8 +18,7 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include -#include +#include "sysdep.h" #include #endif /* msgpack/pack_define.h */ diff --git a/python/msgpack/pack_template.h b/python/msgpack/pack_template.h index aa620f5..de148bf 100644 --- a/python/msgpack/pack_template.h +++ b/python/msgpack/pack_template.h @@ -16,88 +16,16 @@ * limitations under the License. */ -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - - #ifdef __LITTLE_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[0] +#define TAKE8_32(d) ((uint8_t*)&d)[0] +#define TAKE8_64(d) ((uint8_t*)&d)[0] #elif __BIG_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[1] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[3] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[7] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[1] +#define TAKE8_32(d) ((uint8_t*)&d)[3] +#define TAKE8_64(d) ((uint8_t*)&d)[7] #endif #ifndef msgpack_pack_inline_func @@ -121,10 +49,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -133,14 +61,15 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -150,20 +79,22 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -174,24 +105,27 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -201,11 +135,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } \ } while(0) @@ -214,24 +148,26 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -242,32 +178,36 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -279,46 +219,52 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -330,49 +276,55 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -604,7 +556,8 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + unsigned char buf[5]; + buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -612,7 +565,8 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + unsigned char buf[9]; + buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } @@ -655,12 +609,12 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) unsigned char d = 0x90 | n; msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -674,14 +628,14 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -695,14 +649,14 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { - uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l); msgpack_pack_append_buffer(x, buf, 5); } } @@ -716,19 +670,10 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE8_BE8 - -#undef STORE16_BE8 -#undef STORE16_BE16 - -#undef STORE32_BE8 -#undef STORE32_BE16 -#undef STORE32_BE32 - -#undef STORE64_BE8 -#undef STORE64_BE16 -#undef STORE64_BE32 -#undef STORE64_BE64 +#undef TAKE8_8 +#undef TAKE8_16 +#undef TAKE8_32 +#undef TAKE8_64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 diff --git a/python/msgpack/sysdep.h b/python/msgpack/sysdep.h new file mode 100644 index 0000000..106158e --- /dev/null +++ b/python/msgpack/sysdep.h @@ -0,0 +1,94 @@ +/* + * MessagePack system dependencies + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_SYSDEP_H__ +#define MSGPACK_SYSDEP_H__ + + +#ifdef _MSC_VER +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else +#include +#include +#include +#endif + + +#ifdef _WIN32 +typedef long _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr) +#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr) +#else +typedef unsigned int _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1) +#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1) +#endif + + +#ifdef _WIN32 +#include +#else +#include /* __BYTE_ORDER */ +#endif + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#ifdef __LITTLE_ENDIAN__ + +#define _msgpack_be16(x) ntohs(x) +#define _msgpack_be32(x) ntohl(x) + +#if defined(_byteswap_uint64) +# define _msgpack_be64(x) (_byteswap_uint64(x)) +#elif defined(bswap_64) +# define _msgpack_be64(x) bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x) +#else +#define _msgpack_be64(x) \ + ( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \ + ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ + ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ + ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ + ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ + ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ + ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ + ((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) ) +#endif + +#else +#define _msgpack_be16(x) (x) +#define _msgpack_be32(x) (x) +#define _msgpack_be64(x) (x) +#endif + + +#endif /* msgpack/sysdep.h */ + diff --git a/python/msgpack/unpack_define.h b/python/msgpack/unpack_define.h index d997569..63d90a8 100644 --- a/python/msgpack/unpack_define.h +++ b/python/msgpack/unpack_define.h @@ -18,14 +18,10 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include -#include +#include "sysdep.h" #include #include #include -#ifndef __WIN32__ -#include -#endif #ifdef __cplusplus extern "C" { @@ -37,52 +33,6 @@ extern "C" { #endif -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - -#ifdef __WIN32__ -static inline uint16_t msgpack_betoh16(uint16_t x) { - return ((x << 8) & 0xff00U) | - ((x >> 8) & 0x00ffU); -} -static inline uint32_t msgpack_betoh32(uint32_t x) { - return ((x << 24) & 0xff000000UL ) | - ((x << 8) & 0x00ff0000UL ) | - ((x >> 8) & 0x0000ff00UL ) | - ((x >> 24) & 0x000000ffUL ); -} -#else -#define msgpack_betoh16(x) ntohs(x) -#define msgpack_betoh32(x) ntohl(x) -#endif - -#ifdef __LITTLE_ENDIAN__ -#if defined(__bswap_64) -# define msgpack_betoh64(x) __bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) -#else -static inline uint64_t msgpack_betoh64(uint64_t x) { - return ((x << 56) & 0xff00000000000000ULL ) | - ((x << 40) & 0x00ff000000000000ULL ) | - ((x << 24) & 0x0000ff0000000000ULL ) | - ((x << 8) & 0x000000ff00000000ULL ) | - ((x >> 8) & 0x00000000ff000000ULL ) | - ((x >> 24) & 0x0000000000ff0000ULL ) | - ((x >> 40) & 0x000000000000ff00ULL ) | - ((x >> 56) & 0x00000000000000ffULL ) ; -} -#endif -#else -#define msgpack_betoh64(x) (x) -#endif - - typedef enum { CS_HEADER = 0x00, // nil diff --git a/python/msgpack/unpack_template.h b/python/msgpack/unpack_template.h index c960c3a..ca6e1f3 100644 --- a/python/msgpack/unpack_template.h +++ b/python/msgpack/unpack_template.h @@ -40,6 +40,11 @@ #error msgpack_unpack_user type is not defined #endif +#ifndef USE_CASE_RANGE +#if !defined(_MSC_VER) +#define USE_CASE_RANGE +#endif +#endif msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; @@ -131,20 +136,32 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr) + +#ifdef USE_CASE_RANGE +#define SWITCH_RANGE_BEGIN switch(*p) { +#define SWITCH_RANGE(FROM, TO) case FROM ... TO: +#define SWITCH_RANGE_DEFAULT default: +#define SWITCH_RANGE_END } +#else +#define SWITCH_RANGE_BEGIN { if(0) { +#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) { +#define SWITCH_RANGE_DEFAULT } else { +#define SWITCH_RANGE_END } } +#endif if(p == pe) { goto _out; } do { switch(cs) { case CS_HEADER: - switch(*p) { - case 0x00 ... 0x7f: // Positive Fixnum + SWITCH_RANGE_BEGIN + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum push_fixed_value(_uint8, *(uint8_t*)p); - case 0xe0 ... 0xff: // Negative Fixnum + SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum push_fixed_value(_int8, *(int8_t*)p); - case 0xc0 ... 0xdf: // Variable + SWITCH_RANGE(0xc0, 0xdf) // Variable switch(*p) { case 0xc0: // nil push_simple_value(_nil); @@ -187,16 +204,16 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c default: goto _failed; } - case 0xa0 ... 0xbf: // FixRaw + SWITCH_RANGE(0xa0, 0xbf) // FixRaw again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - case 0x90 ... 0x9f: // FixArray + SWITCH_RANGE(0x90, 0x9f) // FixArray start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - case 0x80 ... 0x8f: // FixMap + SWITCH_RANGE(0x80, 0x8f) // FixMap start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - default: + SWITCH_RANGE_DEFAULT goto _failed; - } + SWITCH_RANGE_END // end CS_HEADER From af7c4d2a60086503ee9c124f8cc79a05c2fabe70 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 10:43:22 +0900 Subject: [PATCH 0331/1648] Update new headers. --- msgpack/pack.h | 1 + msgpack/pack_define.h | 3 +- msgpack/pack_template.h | 253 +++++++++++++++----------------------- msgpack/sysdep.h | 94 ++++++++++++++ msgpack/unpack_define.h | 52 +------- msgpack/unpack_template.h | 41 ++++-- 6 files changed, 225 insertions(+), 219 deletions(-) create mode 100644 msgpack/sysdep.h diff --git a/msgpack/pack.h b/msgpack/pack.h index 58f021e..2ae95d1 100644 --- a/msgpack/pack.h +++ b/msgpack/pack.h @@ -18,6 +18,7 @@ #include #include +#include "sysdep.h" #include "pack_define.h" #ifdef __cplusplus diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index 33408e5..f72391b 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -18,8 +18,7 @@ #ifndef MSGPACK_PACK_DEFINE_H__ #define MSGPACK_PACK_DEFINE_H__ -#include -#include +#include "sysdep.h" #include #endif /* msgpack/pack_define.h */ diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index aa620f5..de148bf 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -16,88 +16,16 @@ * limitations under the License. */ -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - - #ifdef __LITTLE_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[0] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ - ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] - - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[0] +#define TAKE8_32(d) ((uint8_t*)&d)[0] +#define TAKE8_64(d) ((uint8_t*)&d)[0] #elif __BIG_ENDIAN__ - -#define STORE8_BE8(d) \ - ((uint8_t*)&d)[0] - - -#define STORE16_BE8(d) \ - ((uint8_t*)&d)[1] - -#define STORE16_BE16(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] - - -#define STORE32_BE8(d) \ - ((uint8_t*)&d)[3] - -#define STORE32_BE16(d) \ - ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - -#define STORE32_BE32(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] - - -#define STORE64_BE8(d) \ - ((uint8_t*)&d)[7] - -#define STORE64_BE16(d) \ - ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE32(d) \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - -#define STORE64_BE64(d) \ - ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ - ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] - +#define TAKE8_8(d) ((uint8_t*)&d)[0] +#define TAKE8_16(d) ((uint8_t*)&d)[1] +#define TAKE8_32(d) ((uint8_t*)&d)[3] +#define TAKE8_64(d) ((uint8_t*)&d)[7] #endif #ifndef msgpack_pack_inline_func @@ -121,10 +49,10 @@ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } while(0) @@ -133,14 +61,15 @@ do { \ do { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -150,20 +79,22 @@ do { \ if(d < (1<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -174,24 +105,27 @@ do { \ if(d < (1ULL<<8)) { \ if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -201,11 +135,11 @@ do { \ do { \ if(d < -(1<<5)) { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \ } \ } while(0) @@ -214,24 +148,26 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -242,32 +178,36 @@ do { \ if(d < -(1<<5)) { \ if(d < -(1<<15)) { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \ } else { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -279,46 +219,52 @@ do { \ if(d < -(1LL<<15)) { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ - const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ - const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ - const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } \ } \ } else if(d < (1<<7)) { \ /* fixnum */ \ - msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \ } else { \ if(d < (1LL<<16)) { \ if(d < (1<<8)) { \ /* unsigned 8 */ \ - const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \ msgpack_pack_append_buffer(x, buf, 2); \ } else { \ /* unsigned 16 */ \ - const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + unsigned char buf[3]; \ + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ - const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + unsigned char buf[5]; \ + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + unsigned char buf[9]; \ + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -330,49 +276,55 @@ do { \ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) { - const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + unsigned char buf[2] = {0xcc, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { - const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { - const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { - const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) { - const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + unsigned char buf[2] = {0xd0, TAKE8_8(d)}; msgpack_pack_append_buffer(x, buf, 2); } msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { - const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { - const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { - const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + unsigned char buf[9]; + buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); msgpack_pack_append_buffer(x, buf, 9); } @@ -604,7 +556,8 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { union { char buf[4]; uint32_t num; } f; *((float*)&f.buf) = d; // FIXME - const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + unsigned char buf[5]; + buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); msgpack_pack_append_buffer(x, buf, 5); } @@ -612,7 +565,8 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { union { char buf[8]; uint64_t num; } f; *((double*)&f.buf) = d; // FIXME - const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + unsigned char buf[9]; + buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); msgpack_pack_append_buffer(x, buf, 9); } @@ -655,12 +609,12 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) unsigned char d = 0x90 | n; msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -674,14 +628,14 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) { if(n < 16) { unsigned char d = 0x80 | n; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { - uint16_t d = (uint16_t)n; - unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)n; - unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -695,14 +649,14 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) { if(l < 32) { unsigned char d = 0xa0 | l; - msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { - uint16_t d = (uint16_t)l; - unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + unsigned char buf[3]; + buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l); msgpack_pack_append_buffer(x, buf, 3); } else { - uint32_t d = (uint32_t)l; - unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + unsigned char buf[5]; + buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l); msgpack_pack_append_buffer(x, buf, 5); } } @@ -716,19 +670,10 @@ msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l #undef msgpack_pack_user #undef msgpack_pack_append_buffer -#undef STORE8_BE8 - -#undef STORE16_BE8 -#undef STORE16_BE16 - -#undef STORE32_BE8 -#undef STORE32_BE16 -#undef STORE32_BE32 - -#undef STORE64_BE8 -#undef STORE64_BE16 -#undef STORE64_BE32 -#undef STORE64_BE64 +#undef TAKE8_8 +#undef TAKE8_16 +#undef TAKE8_32 +#undef TAKE8_64 #undef msgpack_pack_real_uint8 #undef msgpack_pack_real_uint16 diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h new file mode 100644 index 0000000..106158e --- /dev/null +++ b/msgpack/sysdep.h @@ -0,0 +1,94 @@ +/* + * MessagePack system dependencies + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_SYSDEP_H__ +#define MSGPACK_SYSDEP_H__ + + +#ifdef _MSC_VER +typedef __int8 int8_t; +typedef unsigned __int8 uint8_t; +typedef __int16 int16_t; +typedef unsigned __int16 uint16_t; +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else +#include +#include +#include +#endif + + +#ifdef _WIN32 +typedef long _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr) +#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr) +#else +typedef unsigned int _msgpack_atomic_counter_t; +#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1) +#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1) +#endif + + +#ifdef _WIN32 +#include +#else +#include /* __BYTE_ORDER */ +#endif + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#ifdef __LITTLE_ENDIAN__ + +#define _msgpack_be16(x) ntohs(x) +#define _msgpack_be32(x) ntohl(x) + +#if defined(_byteswap_uint64) +# define _msgpack_be64(x) (_byteswap_uint64(x)) +#elif defined(bswap_64) +# define _msgpack_be64(x) bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define _msgpack_be64(x) __DARWIN_OSSwapInt64(x) +#else +#define _msgpack_be64(x) \ + ( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \ + ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \ + ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \ + ((((uint64_t)x) << 8) & 0x000000ff00000000ULL ) | \ + ((((uint64_t)x) >> 8) & 0x00000000ff000000ULL ) | \ + ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \ + ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \ + ((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) ) +#endif + +#else +#define _msgpack_be16(x) (x) +#define _msgpack_be32(x) (x) +#define _msgpack_be64(x) (x) +#endif + + +#endif /* msgpack/sysdep.h */ + diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index d997569..63d90a8 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -18,14 +18,10 @@ #ifndef MSGPACK_UNPACK_DEFINE_H__ #define MSGPACK_UNPACK_DEFINE_H__ -#include -#include +#include "sysdep.h" #include #include #include -#ifndef __WIN32__ -#include -#endif #ifdef __cplusplus extern "C" { @@ -37,52 +33,6 @@ extern "C" { #endif -#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN__ -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN__ -#endif -#endif - -#ifdef __WIN32__ -static inline uint16_t msgpack_betoh16(uint16_t x) { - return ((x << 8) & 0xff00U) | - ((x >> 8) & 0x00ffU); -} -static inline uint32_t msgpack_betoh32(uint32_t x) { - return ((x << 24) & 0xff000000UL ) | - ((x << 8) & 0x00ff0000UL ) | - ((x >> 8) & 0x0000ff00UL ) | - ((x >> 24) & 0x000000ffUL ); -} -#else -#define msgpack_betoh16(x) ntohs(x) -#define msgpack_betoh32(x) ntohl(x) -#endif - -#ifdef __LITTLE_ENDIAN__ -#if defined(__bswap_64) -# define msgpack_betoh64(x) __bswap_64(x) -#elif defined(__DARWIN_OSSwapInt64) -# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) -#else -static inline uint64_t msgpack_betoh64(uint64_t x) { - return ((x << 56) & 0xff00000000000000ULL ) | - ((x << 40) & 0x00ff000000000000ULL ) | - ((x << 24) & 0x0000ff0000000000ULL ) | - ((x << 8) & 0x000000ff00000000ULL ) | - ((x >> 8) & 0x00000000ff000000ULL ) | - ((x >> 24) & 0x0000000000ff0000ULL ) | - ((x >> 40) & 0x000000000000ff00ULL ) | - ((x >> 56) & 0x00000000000000ffULL ) ; -} -#endif -#else -#define msgpack_betoh64(x) (x) -#endif - - typedef enum { CS_HEADER = 0x00, // nil diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index c960c3a..ca6e1f3 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -40,6 +40,11 @@ #error msgpack_unpack_user type is not defined #endif +#ifndef USE_CASE_RANGE +#if !defined(_MSC_VER) +#define USE_CASE_RANGE +#endif +#endif msgpack_unpack_struct_decl(_stack) { msgpack_unpack_object obj; @@ -131,20 +136,32 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ((unsigned int)*p & 0x1f) #define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) +#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr) + +#ifdef USE_CASE_RANGE +#define SWITCH_RANGE_BEGIN switch(*p) { +#define SWITCH_RANGE(FROM, TO) case FROM ... TO: +#define SWITCH_RANGE_DEFAULT default: +#define SWITCH_RANGE_END } +#else +#define SWITCH_RANGE_BEGIN { if(0) { +#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) { +#define SWITCH_RANGE_DEFAULT } else { +#define SWITCH_RANGE_END } } +#endif if(p == pe) { goto _out; } do { switch(cs) { case CS_HEADER: - switch(*p) { - case 0x00 ... 0x7f: // Positive Fixnum + SWITCH_RANGE_BEGIN + SWITCH_RANGE(0x00, 0x7f) // Positive Fixnum push_fixed_value(_uint8, *(uint8_t*)p); - case 0xe0 ... 0xff: // Negative Fixnum + SWITCH_RANGE(0xe0, 0xff) // Negative Fixnum push_fixed_value(_int8, *(int8_t*)p); - case 0xc0 ... 0xdf: // Variable + SWITCH_RANGE(0xc0, 0xdf) // Variable switch(*p) { case 0xc0: // nil push_simple_value(_nil); @@ -187,16 +204,16 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c default: goto _failed; } - case 0xa0 ... 0xbf: // FixRaw + SWITCH_RANGE(0xa0, 0xbf) // FixRaw again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); - case 0x90 ... 0x9f: // FixArray + SWITCH_RANGE(0x90, 0x9f) // FixArray start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); - case 0x80 ... 0x8f: // FixMap + SWITCH_RANGE(0x80, 0x8f) // FixMap start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); - default: + SWITCH_RANGE_DEFAULT goto _failed; - } + SWITCH_RANGE_END // end CS_HEADER From f4c5b15cc6cf972695bb2281714f5b13b299fbd3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 11:13:47 +0900 Subject: [PATCH 0332/1648] Add `use_tuple` option that returns tuple for array object to Unpacker. --- python/msgpack/_msgpack.pyx | 10 +++++++++- python/msgpack/unpack.h | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index dcabc0f..b20ad7b 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -151,7 +151,11 @@ def packb(object o): packs = packb cdef extern from "unpack.h": + ctypedef struct msgpack_user: + int use_tuple + ctypedef struct template_context: + msgpack_user user PyObject* obj size_t count unsigned int ct @@ -170,6 +174,7 @@ def unpackb(object packed_bytes): cdef size_t off = 0 cdef int ret template_init(&ctx) + ctx.user.use_tuple = 0 ret = template_execute(&ctx, p, len(packed_bytes), &off) if ret == 1: return template_data(&ctx) @@ -225,6 +230,7 @@ cdef class Unpacker(object): cdef object file_like cdef int read_size cdef object waiting_bytes + cdef int use_tuple def __cinit__(self): self.buf = NULL @@ -233,9 +239,10 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0): + def __init__(self, file_like=None, int read_size=0, use_tuple=0): if read_size == 0: read_size = 1024*1024 + self.use_tuple = use_tuple self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -244,6 +251,7 @@ cdef class Unpacker(object): self.buf_head = 0 self.buf_tail = 0 template_init(&self.ctx) + self.ctx.user.use_tuple = use_tuple def feed(self, next_bytes): if not isinstance(next_bytes, str): diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 40058d0..0bf2568 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -20,6 +20,7 @@ #include "unpack_define.h" typedef struct unpack_user { + int use_tuple; } unpack_user; @@ -135,7 +136,8 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = PyList_New(n); + PyObject *p = u->use_tuple ? PyTuple_New(n) : PyList_New(n); + if (!p) return -1; *o = p; @@ -143,7 +145,15 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac } static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) -{ PyList_SET_ITEM(*c, current, o); return 0; } +{ + if (u->use_tuple) { + PyTuple_SET_ITEM(*c, current, o); + } + else { + PyList_SET_ITEM(*c, current, o); + } + return 0; +} static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { From 77e3e59620d65589451c833fa01e9c9e5f39624d Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 11:13:47 +0900 Subject: [PATCH 0333/1648] Add `use_tuple` option that returns tuple for array object to Unpacker. --- msgpack/_msgpack.pyx | 10 +++++++++- msgpack/unpack.h | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index dcabc0f..b20ad7b 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -151,7 +151,11 @@ def packb(object o): packs = packb cdef extern from "unpack.h": + ctypedef struct msgpack_user: + int use_tuple + ctypedef struct template_context: + msgpack_user user PyObject* obj size_t count unsigned int ct @@ -170,6 +174,7 @@ def unpackb(object packed_bytes): cdef size_t off = 0 cdef int ret template_init(&ctx) + ctx.user.use_tuple = 0 ret = template_execute(&ctx, p, len(packed_bytes), &off) if ret == 1: return template_data(&ctx) @@ -225,6 +230,7 @@ cdef class Unpacker(object): cdef object file_like cdef int read_size cdef object waiting_bytes + cdef int use_tuple def __cinit__(self): self.buf = NULL @@ -233,9 +239,10 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0): + def __init__(self, file_like=None, int read_size=0, use_tuple=0): if read_size == 0: read_size = 1024*1024 + self.use_tuple = use_tuple self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -244,6 +251,7 @@ cdef class Unpacker(object): self.buf_head = 0 self.buf_tail = 0 template_init(&self.ctx) + self.ctx.user.use_tuple = use_tuple def feed(self, next_bytes): if not isinstance(next_bytes, str): diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 40058d0..0bf2568 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -20,6 +20,7 @@ #include "unpack_define.h" typedef struct unpack_user { + int use_tuple; } unpack_user; @@ -135,7 +136,8 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = PyList_New(n); + PyObject *p = u->use_tuple ? PyTuple_New(n) : PyList_New(n); + if (!p) return -1; *o = p; @@ -143,7 +145,15 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac } static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) -{ PyList_SET_ITEM(*c, current, o); return 0; } +{ + if (u->use_tuple) { + PyTuple_SET_ITEM(*c, current, o); + } + else { + PyList_SET_ITEM(*c, current, o); + } + return 0; +} static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { From b9f78821d423c2883ddff4d8471cc4c93d032388 Mon Sep 17 00:00:00 2001 From: inada-n Date: Thu, 17 Dec 2009 15:19:18 +0900 Subject: [PATCH 0334/1648] Make tuple default. --- python/msgpack/_msgpack.pyx | 12 ++++++------ python/msgpack/unpack.h | 12 +++++------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index b20ad7b..eb83c85 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -152,7 +152,7 @@ packs = packb cdef extern from "unpack.h": ctypedef struct msgpack_user: - int use_tuple + int use_list ctypedef struct template_context: msgpack_user user @@ -174,7 +174,7 @@ def unpackb(object packed_bytes): cdef size_t off = 0 cdef int ret template_init(&ctx) - ctx.user.use_tuple = 0 + ctx.user.use_list = 0 ret = template_execute(&ctx, p, len(packed_bytes), &off) if ret == 1: return template_data(&ctx) @@ -230,7 +230,7 @@ cdef class Unpacker(object): cdef object file_like cdef int read_size cdef object waiting_bytes - cdef int use_tuple + cdef int use_list def __cinit__(self): self.buf = NULL @@ -239,10 +239,10 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0, use_tuple=0): + def __init__(self, file_like=None, int read_size=0, use_list=0): if read_size == 0: read_size = 1024*1024 - self.use_tuple = use_tuple + self.use_list = use_list self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -251,7 +251,7 @@ cdef class Unpacker(object): self.buf_head = 0 self.buf_tail = 0 template_init(&self.ctx) - self.ctx.user.use_tuple = use_tuple + self.ctx.user.use_list = use_list def feed(self, next_bytes): if not isinstance(next_bytes, str): diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h index 0bf2568..61a3786 100644 --- a/python/msgpack/unpack.h +++ b/python/msgpack/unpack.h @@ -20,7 +20,7 @@ #include "unpack_define.h" typedef struct unpack_user { - int use_tuple; + int use_list; } unpack_user; @@ -136,7 +136,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = u->use_tuple ? PyTuple_New(n) : PyList_New(n); + PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n); if (!p) return -1; @@ -146,12 +146,10 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { - if (u->use_tuple) { - PyTuple_SET_ITEM(*c, current, o); - } - else { + if (u->use_list) PyList_SET_ITEM(*c, current, o); - } + else + PyTuple_SET_ITEM(*c, current, o); return 0; } From 658c90f132d9582ec1e8ec68a30f9e67b8d7a572 Mon Sep 17 00:00:00 2001 From: inada-n Date: Thu, 17 Dec 2009 15:19:18 +0900 Subject: [PATCH 0335/1648] Make tuple default. --- msgpack/_msgpack.pyx | 12 ++++++------ msgpack/unpack.h | 12 +++++------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index b20ad7b..eb83c85 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -152,7 +152,7 @@ packs = packb cdef extern from "unpack.h": ctypedef struct msgpack_user: - int use_tuple + int use_list ctypedef struct template_context: msgpack_user user @@ -174,7 +174,7 @@ def unpackb(object packed_bytes): cdef size_t off = 0 cdef int ret template_init(&ctx) - ctx.user.use_tuple = 0 + ctx.user.use_list = 0 ret = template_execute(&ctx, p, len(packed_bytes), &off) if ret == 1: return template_data(&ctx) @@ -230,7 +230,7 @@ cdef class Unpacker(object): cdef object file_like cdef int read_size cdef object waiting_bytes - cdef int use_tuple + cdef int use_list def __cinit__(self): self.buf = NULL @@ -239,10 +239,10 @@ cdef class Unpacker(object): if self.buf: free(self.buf); - def __init__(self, file_like=None, int read_size=0, use_tuple=0): + def __init__(self, file_like=None, int read_size=0, use_list=0): if read_size == 0: read_size = 1024*1024 - self.use_tuple = use_tuple + self.use_list = use_list self.file_like = file_like self.read_size = read_size self.waiting_bytes = [] @@ -251,7 +251,7 @@ cdef class Unpacker(object): self.buf_head = 0 self.buf_tail = 0 template_init(&self.ctx) - self.ctx.user.use_tuple = use_tuple + self.ctx.user.use_list = use_list def feed(self, next_bytes): if not isinstance(next_bytes, str): diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 0bf2568..61a3786 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -20,7 +20,7 @@ #include "unpack_define.h" typedef struct unpack_user { - int use_tuple; + int use_list; } unpack_user; @@ -136,7 +136,7 @@ static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) { - PyObject *p = u->use_tuple ? PyTuple_New(n) : PyList_New(n); + PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n); if (!p) return -1; @@ -146,12 +146,10 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o) { - if (u->use_tuple) { - PyTuple_SET_ITEM(*c, current, o); - } - else { + if (u->use_list) PyList_SET_ITEM(*c, current, o); - } + else + PyTuple_SET_ITEM(*c, current, o); return 0; } From 5cf85a82d3a39dd9135de473d0fa4ed21aea8270 Mon Sep 17 00:00:00 2001 From: inada-n Date: Thu, 17 Dec 2009 15:20:20 +0900 Subject: [PATCH 0336/1648] Add .gitignore for Python. --- python/.gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 python/.gitignore diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 0000000..81df440 --- /dev/null +++ b/python/.gitignore @@ -0,0 +1,3 @@ +MANIFEST +build/* +dist/* From 8194212d5845d648cc35b677c71db61ceb2d11d3 Mon Sep 17 00:00:00 2001 From: inada-n Date: Thu, 17 Dec 2009 15:20:20 +0900 Subject: [PATCH 0337/1648] Add .gitignore for Python. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..81df440 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +MANIFEST +build/* +dist/* From 63b9a876b0b402709b7b8773b08a58f8fd0bc8b2 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 17:58:41 +0900 Subject: [PATCH 0338/1648] Fix tests. --- python/test/test_case.py | 9 +++++---- python/test/test_format.py | 21 +++++++++++---------- python/test/test_pack.py | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/python/test/test_case.py b/python/test/test_case.py index d754fb0..a08c6ce 100644 --- a/python/test/test_case.py +++ b/python/test/test_case.py @@ -5,6 +5,7 @@ from nose import main from nose.tools import * from msgpack import packs, unpacks + def check(length, obj): v = packs(obj) assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) @@ -54,7 +55,7 @@ def test_raw32(): def check_array(overhead, num): - check(num + overhead, [None] * num) + check(num + overhead, (None,) * num) def test_fixarray(): check_array(1, 0) @@ -86,9 +87,9 @@ def test_match(): (-129, '\xd1\xff\x7f'), ({1:1}, '\x81\x01\x01'), (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), - ([], '\x90'), - (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), - (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ((), '\x90'), + (tuple(range(15)),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (tuple(range(16)),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), ({}, '\x80'), (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), diff --git a/python/test/test_format.py b/python/test/test_format.py index 009a764..562ef54 100644 --- a/python/test/test_format.py +++ b/python/test/test_format.py @@ -10,21 +10,21 @@ def check(src, should): def testSimpleValue(): check("\x93\xc0\xc2\xc3", - [None, False, True]) + (None, False, True,)) def testFixnum(): check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", - [[0,64,127], [-32,-16,-1]] + ((0,64,127,), (-32,-16,-1,),) ) def testFixArray(): check("\x92\x90\x91\x91\xc0", - [[],[[None]]], + ((),((None,),),), ) def testFixRaw(): check("\x94\xa0\xa1a\xa2bc\xa3def", - ["", "a", "bc", "def"], + ("", "a", "bc", "def",), ) def testFixMap(): @@ -38,25 +38,26 @@ def testUnsignedInt(): "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" "\xce\xff\xff\xff\xff", - [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), ) def testSignedInt(): check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" "\xd2\xff\xff\xff\xff", - [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) + (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,)) def testRaw(): check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", - ["", "a", "ab", "", "a", "ab"]) + ("", "a", "ab", "", "a", "ab")) def testArray(): check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" "\xc2\xc3", - [[], [None], [False,True], [], [None], [False,True]]) + ((), (None,), (False,True), (), (None,), (False,True)) + ) def testMap(): check( @@ -67,8 +68,8 @@ def testMap(): "\xdf\x00\x00\x00\x00" "\xdf\x00\x00\x00\x01\xc0\xc2" "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", - [{}, {None: False}, {True: False, None: False}, {}, - {None: False}, {True: False, None: False}]) + ({}, {None: False}, {True: False, None: False}, {}, + {None: False}, {True: False, None: False})) if __name__ == '__main__': main() diff --git a/python/test/test_pack.py b/python/test/test_pack.py index 86badb5..5dec068 100644 --- a/python/test/test_pack.py +++ b/python/test/test_pack.py @@ -17,7 +17,7 @@ def testPack(): 1.0, "", "a", "a"*31, "a"*32, None, True, False, - [], [[]], [[], None], + (), ((),), ((), None,), {None: 0}, (1<<23), ] From 472f5be977e913632cc1d15e663a34b62dfbab5c Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 17:58:41 +0900 Subject: [PATCH 0339/1648] Fix tests. --- test/test_case.py | 9 +++++---- test/test_format.py | 21 +++++++++++---------- test/test_pack.py | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/test/test_case.py b/test/test_case.py index d754fb0..a08c6ce 100644 --- a/test/test_case.py +++ b/test/test_case.py @@ -5,6 +5,7 @@ from nose import main from nose.tools import * from msgpack import packs, unpacks + def check(length, obj): v = packs(obj) assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v))) @@ -54,7 +55,7 @@ def test_raw32(): def check_array(overhead, num): - check(num + overhead, [None] * num) + check(num + overhead, (None,) * num) def test_fixarray(): check_array(1, 0) @@ -86,9 +87,9 @@ def test_match(): (-129, '\xd1\xff\x7f'), ({1:1}, '\x81\x01\x01'), (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"), - ([], '\x90'), - (range(15),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), - (range(16),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), + ((), '\x90'), + (tuple(range(15)),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"), + (tuple(range(16)),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), ({}, '\x80'), (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'), (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'), diff --git a/test/test_format.py b/test/test_format.py index 009a764..562ef54 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -10,21 +10,21 @@ def check(src, should): def testSimpleValue(): check("\x93\xc0\xc2\xc3", - [None, False, True]) + (None, False, True,)) def testFixnum(): check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", - [[0,64,127], [-32,-16,-1]] + ((0,64,127,), (-32,-16,-1,),) ) def testFixArray(): check("\x92\x90\x91\x91\xc0", - [[],[[None]]], + ((),((None,),),), ) def testFixRaw(): check("\x94\xa0\xa1a\xa2bc\xa3def", - ["", "a", "bc", "def"], + ("", "a", "bc", "def",), ) def testFixMap(): @@ -38,25 +38,26 @@ def testUnsignedInt(): "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00" "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00" "\xce\xff\xff\xff\xff", - [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295], + (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,), ) def testSignedInt(): check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00" "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00" "\xd2\xff\xff\xff\xff", - [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) + (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,)) def testRaw(): check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00" "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab", - ["", "a", "ab", "", "a", "ab"]) + ("", "a", "ab", "", "a", "ab")) def testArray(): check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00" "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02" "\xc2\xc3", - [[], [None], [False,True], [], [None], [False,True]]) + ((), (None,), (False,True), (), (None,), (False,True)) + ) def testMap(): check( @@ -67,8 +68,8 @@ def testMap(): "\xdf\x00\x00\x00\x00" "\xdf\x00\x00\x00\x01\xc0\xc2" "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2", - [{}, {None: False}, {True: False, None: False}, {}, - {None: False}, {True: False, None: False}]) + ({}, {None: False}, {True: False, None: False}, {}, + {None: False}, {True: False, None: False})) if __name__ == '__main__': main() diff --git a/test/test_pack.py b/test/test_pack.py index 86badb5..5dec068 100644 --- a/test/test_pack.py +++ b/test/test_pack.py @@ -17,7 +17,7 @@ def testPack(): 1.0, "", "a", "a"*31, "a"*32, None, True, False, - [], [[]], [[], None], + (), ((),), ((), None,), {None: 0}, (1<<23), ] From 232aced926635a7054ac4081d472529cdf96f749 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 18:03:33 +0900 Subject: [PATCH 0340/1648] Update gitignore. --- python/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/.gitignore b/python/.gitignore index 81df440..430c633 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1,3 +1,5 @@ MANIFEST build/* dist/* +*.pyc +*.pyo From 9d168d2ae872f024f62e7e6653e6a8a17473c33a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 17 Dec 2009 18:03:33 +0900 Subject: [PATCH 0341/1648] Update gitignore. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 81df440..430c633 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ MANIFEST build/* dist/* +*.pyc +*.pyo From c2dd22ec102d12374b97f4ae32dccdaaca8630cd Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 19 Dec 2009 22:09:29 +0900 Subject: [PATCH 0342/1648] c,cpp: add msgpack_vrefbuffer_migrate, msgpack::vrefbuffer::migrate --- c/vrefbuffer.c | 29 ++++++++++++++++++++++++++++- c/vrefbuffer.h | 1 + cpp/vrefbuffer.hpp | 7 +++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c index bbaf61d..2bf97af 100644 --- a/c/vrefbuffer.c +++ b/c/vrefbuffer.c @@ -77,7 +77,7 @@ int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, const char* buf, unsigned int len) { if(vbuf->tail == vbuf->end) { - const size_t nused = vbuf->end - vbuf->array; + const size_t nused = vbuf->tail - vbuf->array; const size_t nnext = nused * 2; struct iovec* nvec = (struct iovec*)realloc( @@ -133,3 +133,30 @@ int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, } } +int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to) +{ + const size_t tosize = to->tail - to->array; + if(vbuf->tail + tosize < vbuf->end) { + const size_t nused = vbuf->tail - vbuf->array; + const size_t nsize = vbuf->end - vbuf->array; + const size_t reqsize = nused + tosize; + size_t nnext = nsize * 2; + while(nnext < reqsize) { + nnext *= 2; + } + + struct iovec* nvec = (struct iovec*)realloc( + vbuf->array, sizeof(struct iovec)*nnext); + if(nvec == NULL) { + return -1; + } + + vbuf->array = nvec; + vbuf->end = nvec + nnext; + vbuf->tail = nvec + nused; + } + + memcpy(vbuf->tail, vbuf->array, sizeof(struct iovec)*tosize); + return 0; +} + diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h index 063075f..9f24f14 100644 --- a/c/vrefbuffer.h +++ b/c/vrefbuffer.h @@ -76,6 +76,7 @@ int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, const char* buf, unsigned int len); +int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to); int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len) { diff --git a/cpp/vrefbuffer.hpp b/cpp/vrefbuffer.hpp index 549d77f..c8eca7b 100644 --- a/cpp/vrefbuffer.hpp +++ b/cpp/vrefbuffer.hpp @@ -71,6 +71,13 @@ public: return msgpack_vrefbuffer_veclen(this); } + void migrate(vrefbuffer* to) + { + if(msgpack_vrefbuffer_migrate(this, to) < 0) { + throw std::bad_alloc(); + } + } + private: typedef msgpack_vrefbuffer base; From 9817e9b18df85ed8f6afdc24f9f5c2f8228b609d Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 4 Jan 2010 11:59:52 +0900 Subject: [PATCH 0343/1648] Perl: support NVTYPE=="long double" or IVTYPE="long long" environment. --- perl/pack.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/perl/pack.c b/perl/pack.c index 01dd5b6..589cee8 100644 --- a/perl/pack.c +++ b/perl/pack.c @@ -37,10 +37,19 @@ static void need(enc_t *enc, STRLEN len); #include "msgpack/pack_template.h" -#define _PACK_WRAPPER(t) msgpack_pack_##t -#define PACK_WRAPPER(t) _PACK_WRAPPER(t) #define INIT_SIZE 32 /* initial scalar size to be allocated */ +#if IVSIZE == 8 +# define PACK_IV msgpack_pack_int64 +#elif IVSIZE == 4 +# define PACK_IV msgpack_pack_int32 +#elif IVSIZE == 2 +# define PACK_IV msgpack_pack_int16 +#else +# error "msgpack only supports IVSIZE = 8,4,2 environment." +#endif + + static void need(enc_t *enc, STRLEN len) { if (enc->cur + len >= enc->end) { @@ -155,11 +164,12 @@ static void _msgpack_pack_sv(enc_t *enc, SV* sv) { msgpack_pack_raw_body(enc, csv, len); } } else if (SvNOKp(sv)) { - PACK_WRAPPER(NVTYPE)(enc, SvNVX(sv)); + /* XXX long double is not supported yet. */ + msgpack_pack_double(enc, (double)SvNVX(sv)); } else if (SvIOK_UV(sv)) { msgpack_pack_uint32(enc, SvUV(sv)); } else if (SvIOKp(sv)) { - PACK_WRAPPER(IVTYPE)(enc, SvIV(sv)); + PACK_IV(enc, SvIV(sv)); } else if (SvROK(sv)) { _msgpack_pack_rv(enc, SvRV(sv)); } else if (!SvOK(sv)) { From 7873e41e00d3edc9cef567762c1e44a1627cab15 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 4 Jan 2010 12:10:46 +0900 Subject: [PATCH 0344/1648] Perl: change for release Data-MessagePack-0.09 --- perl/Changes | 5 +++++ perl/MANIFEST.SKIP | 2 +- perl/lib/Data/MessagePack.pm | 6 +++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/perl/Changes b/perl/Changes index c6e370c..4ff69c2 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,8 @@ +0.09 + + - support NVTYPE=="long double" or IVTYPE=="long long" environment + (thanks to Jun Kuriyama++) + 0.08 - fixed PVNV issue... diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP index a318e47..f634035 100644 --- a/perl/MANIFEST.SKIP +++ b/perl/MANIFEST.SKIP @@ -20,6 +20,6 @@ ^\.git/ \.sw[pon]$ ^\.gitignore$ -ppport.h \.o$ \.bs$ +^Data-MessagePack-[0-9.]+/ diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index dde7302..4a95e45 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.08'; +our $VERSION = '0.09'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; @@ -44,6 +44,10 @@ Pack the string as int when the value looks like int(EXPERIMENTAL). Tokuhiro Matsuno +=head1 THANKS TO + +Jun Kuriyama + =head1 LICENSE This library is free software; you can redistribute it and/or modify From d76093b148c1082a1465041263810595f3fd5b3b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 8 Jan 2010 17:34:14 +0900 Subject: [PATCH 0345/1648] import MessagePack for Java --- java-plan1/MessagePack.java | 37 -- java-plan1/Packer.java | 266 ----------- java-plan1/Unpacker.java | 401 ---------------- java-plan1/run.sh | 3 - java-plan1/test.java | 94 ---- java-plan2/build.xml | 15 - java-plan2/src/org/msgpack/GenericArray.java | 25 - .../src/org/msgpack/GenericBoolean.java | 17 - java-plan2/src/org/msgpack/GenericLong.java | 17 - java-plan2/src/org/msgpack/GenericMap.java | 25 - java-plan2/src/org/msgpack/GenericObject.java | 52 --- java-plan2/src/org/msgpack/GenericRaw.java | 59 --- java-plan2/src/org/msgpack/GenericRawRef.java | 55 --- java-plan2/src/org/msgpack/GenericShort.java | 29 -- .../src/org/msgpack/MessageConvertable.java | 7 - .../src/org/msgpack/MessageMergeable.java | 7 - .../src/org/msgpack/MessagePackException.java | 10 - .../src/org/msgpack/MessagePackable.java | 9 - java-plan2/src/org/msgpack/Packer.java | 332 -------------- .../src/org/msgpack/UnbufferedUnpacker.java | 74 --- .../src/org/msgpack/UnpackException.java | 35 -- .../src/org/msgpack/UnpackIterator.java | 53 --- java-plan2/src/org/msgpack/Unpacker.java | 269 ----------- .../src/org/msgpack/impl/ArrayBuilder.java | 7 - .../msgpack/impl/GenericObjectBuilder.java | 127 ----- .../impl/GenericZeroCopyObjectBuilder.java | 12 - .../src/org/msgpack/impl/MapBuilder.java | 8 - .../src/org/msgpack/impl/ObjectBuilder.java | 16 - .../msgpack/impl/SpecificObjectBuilder.java | 203 -------- .../src/org/msgpack/impl/UnpackerImpl.java | 395 ---------------- .../src/org/msgpack/schema/ArraySchema.java | 61 --- .../org/msgpack/schema/ClassGenerator.java | 230 ---------- .../src/org/msgpack/schema/ClassSchema.java | 39 -- .../src/org/msgpack/schema/FieldSchema.java | 31 -- .../msgpack/schema/GenericClassSchema.java | 89 ---- .../msgpack/schema/GenericFieldSchema.java | 25 - .../src/org/msgpack/schema/IntSchema.java | 58 --- .../src/org/msgpack/schema/LongSchema.java | 58 --- .../src/org/msgpack/schema/MapSchema.java | 70 --- .../src/org/msgpack/schema/ObjectSchema.java | 33 -- .../org/msgpack/schema/PrimitiveSchema.java | 21 - .../src/org/msgpack/schema/RawSchema.java | 44 -- .../src/org/msgpack/schema/SSchemaParser.java | 246 ---------- java-plan2/src/org/msgpack/schema/Schema.java | 93 ---- .../src/org/msgpack/schema/ShortSchema.java | 46 -- .../msgpack/schema/SpecificClassSchema.java | 156 ------- .../msgpack/schema/SpecificFieldSchema.java | 78 ---- .../src/org/msgpack/schema/StringSchema.java | 48 -- java-plan2/test/Generate.java | 18 - .../src/serializers/msgpack/MediaContent.java | 277 ----------- .../msgpack/MessagePackSerializer.java | 70 --- .../serializers/msgpack/MediaContent.mpacs | 21 - {java-plan3 => java}/build.xml | 3 + .../src/org/msgpack/MessageMergeable.java | 2 +- .../src/org/msgpack/MessagePackable.java | 2 +- .../src/org/msgpack/MessageTypeException.java | 2 +- .../src/org/msgpack/Packer.java | 2 +- .../src/org/msgpack/Schema.java | 2 +- .../src/org/msgpack/UnbufferedUnpacker.java | 2 +- .../src/org/msgpack/UnpackException.java | 2 +- .../src/org/msgpack/UnpackIterator.java | 2 +- .../src/org/msgpack/Unpacker.java | 2 +- .../src/org/msgpack/impl/UnpackerImpl.java | 17 + .../src/org/msgpack/schema/ArraySchema.java | 2 +- .../src/org/msgpack/schema/ByteSchema.java | 2 +- .../org/msgpack/schema/ClassGenerator.java | 2 +- .../src/org/msgpack/schema/ClassSchema.java | 2 +- .../src/org/msgpack/schema/DoubleSchema.java | 2 +- .../src/org/msgpack/schema/FieldSchema.java | 2 +- .../src/org/msgpack/schema/FloatSchema.java | 2 +- .../msgpack/schema/GenericClassSchema.java | 2 +- .../src/org/msgpack/schema/GenericSchema.java | 2 +- .../src/org/msgpack/schema/IArraySchema.java | 2 +- .../src/org/msgpack/schema/IMapSchema.java | 2 +- .../src/org/msgpack/schema/IntSchema.java | 2 +- .../src/org/msgpack/schema/LongSchema.java | 2 +- .../src/org/msgpack/schema/MapSchema.java | 3 +- .../src/org/msgpack/schema/RawSchema.java | 2 +- .../msgpack/schema/ReflectionClassSchema.java | 0 .../src/org/msgpack/schema/SSchemaParser.java | 2 +- .../src/org/msgpack/schema/ShortSchema.java | 2 +- .../msgpack/schema/SpecificClassSchema.java | 2 +- .../src/org/msgpack/schema/StringSchema.java | 2 +- {java-plan3 => java}/test/Generate.java | 0 java/test/README | 7 + .../tpc/src/serializers/BenchmarkRunner.java | 434 ++++++++++++++++++ .../src/serializers/msgpack/MediaContent.java | 0 .../serializers/msgpack/MediaContent.mpacs | 0 .../msgpack/MessagePackDynamicSerializer.java | 0 .../msgpack/MessagePackGenericSerializer.java | 0 .../MessagePackIndirectSerializer.java | 0 .../MessagePackSpecificSerializer.java | 0 92 files changed, 490 insertions(+), 4499 deletions(-) delete mode 100644 java-plan1/MessagePack.java delete mode 100644 java-plan1/Packer.java delete mode 100644 java-plan1/Unpacker.java delete mode 100755 java-plan1/run.sh delete mode 100644 java-plan1/test.java delete mode 100644 java-plan2/build.xml delete mode 100644 java-plan2/src/org/msgpack/GenericArray.java delete mode 100644 java-plan2/src/org/msgpack/GenericBoolean.java delete mode 100644 java-plan2/src/org/msgpack/GenericLong.java delete mode 100644 java-plan2/src/org/msgpack/GenericMap.java delete mode 100644 java-plan2/src/org/msgpack/GenericObject.java delete mode 100644 java-plan2/src/org/msgpack/GenericRaw.java delete mode 100644 java-plan2/src/org/msgpack/GenericRawRef.java delete mode 100644 java-plan2/src/org/msgpack/GenericShort.java delete mode 100644 java-plan2/src/org/msgpack/MessageConvertable.java delete mode 100644 java-plan2/src/org/msgpack/MessageMergeable.java delete mode 100644 java-plan2/src/org/msgpack/MessagePackException.java delete mode 100644 java-plan2/src/org/msgpack/MessagePackable.java delete mode 100644 java-plan2/src/org/msgpack/Packer.java delete mode 100644 java-plan2/src/org/msgpack/UnbufferedUnpacker.java delete mode 100644 java-plan2/src/org/msgpack/UnpackException.java delete mode 100644 java-plan2/src/org/msgpack/UnpackIterator.java delete mode 100644 java-plan2/src/org/msgpack/Unpacker.java delete mode 100644 java-plan2/src/org/msgpack/impl/ArrayBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/MapBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/ObjectBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java delete mode 100644 java-plan2/src/org/msgpack/impl/UnpackerImpl.java delete mode 100644 java-plan2/src/org/msgpack/schema/ArraySchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/ClassGenerator.java delete mode 100644 java-plan2/src/org/msgpack/schema/ClassSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/FieldSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/GenericClassSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/GenericFieldSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/IntSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/LongSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/MapSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/ObjectSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/PrimitiveSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/RawSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/SSchemaParser.java delete mode 100644 java-plan2/src/org/msgpack/schema/Schema.java delete mode 100644 java-plan2/src/org/msgpack/schema/ShortSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/SpecificClassSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java delete mode 100644 java-plan2/src/org/msgpack/schema/StringSchema.java delete mode 100644 java-plan2/test/Generate.java delete mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java delete mode 100644 java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java delete mode 100644 java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs rename {java-plan3 => java}/build.xml (78%) rename {java-plan3 => java}/src/org/msgpack/MessageMergeable.java (94%) rename {java-plan3 => java}/src/org/msgpack/MessagePackable.java (94%) rename {java-plan3 => java}/src/org/msgpack/MessageTypeException.java (96%) rename {java-plan3 => java}/src/org/msgpack/Packer.java (99%) rename {java-plan3 => java}/src/org/msgpack/Schema.java (98%) rename {java-plan3 => java}/src/org/msgpack/UnbufferedUnpacker.java (97%) rename {java-plan3 => java}/src/org/msgpack/UnpackException.java (94%) rename {java-plan3 => java}/src/org/msgpack/UnpackIterator.java (96%) rename {java-plan3 => java}/src/org/msgpack/Unpacker.java (99%) rename {java-plan3 => java}/src/org/msgpack/impl/UnpackerImpl.java (95%) rename {java-plan3 => java}/src/org/msgpack/schema/ArraySchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/ByteSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/ClassGenerator.java (99%) rename {java-plan3 => java}/src/org/msgpack/schema/ClassSchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/DoubleSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/FieldSchema.java (95%) rename {java-plan3 => java}/src/org/msgpack/schema/FloatSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/GenericClassSchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/GenericSchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/IArraySchema.java (94%) rename {java-plan3 => java}/src/org/msgpack/schema/IMapSchema.java (94%) rename {java-plan3 => java}/src/org/msgpack/schema/IntSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/LongSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/MapSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/RawSchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/ReflectionClassSchema.java (100%) rename {java-plan3 => java}/src/org/msgpack/schema/SSchemaParser.java (99%) rename {java-plan3 => java}/src/org/msgpack/schema/ShortSchema.java (97%) rename {java-plan3 => java}/src/org/msgpack/schema/SpecificClassSchema.java (98%) rename {java-plan3 => java}/src/org/msgpack/schema/StringSchema.java (98%) rename {java-plan3 => java}/test/Generate.java (100%) create mode 100644 java/test/README create mode 100644 java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java rename {java-plan3 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java (100%) rename {java-plan2 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs (100%) rename {java-plan3 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java (100%) rename {java-plan3 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java (100%) rename {java-plan3 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java (100%) rename {java-plan3 => java}/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java (100%) diff --git a/java-plan1/MessagePack.java b/java-plan1/MessagePack.java deleted file mode 100644 index 0a4f442..0000000 --- a/java-plan1/MessagePack.java +++ /dev/null @@ -1,37 +0,0 @@ -import java.nio.ByteBuffer; -import java.io.InputStream; -import java.io.IOException; - -public class MessagePack { - - static public Object unpack(InputStream source) - { - // FIXME not implemented yet - return null; - } - - static public Object unpack(byte[] source, int len) throws IOException - { - // FIXME not implemented yet - return null; - } - - static public Object unpack(ByteBuffer source) throws IOException - { - // FIXME not implemented yet - return null; - } - - static public Object unpack(ByteBuffer[] source) throws IOException - { - // FIXME not implemented yet - return null; - } -} - -/* -public interface MessagePackable { - public void toMessagePack(Packer pk); -} -*/ - diff --git a/java-plan1/Packer.java b/java-plan1/Packer.java deleted file mode 100644 index ea6b2d4..0000000 --- a/java-plan1/Packer.java +++ /dev/null @@ -1,266 +0,0 @@ -import java.io.*; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.math.BigInteger; - -public class Packer { - protected byte[] castBytes = new byte[9]; - protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); - protected OutputStream out; - - public Packer(OutputStream out) - { - this.out = out; - } - - public Packer packByte(byte d) throws IOException - { - if(d < -(1<<5)) { - castBytes[0] = (byte)0xd1; - castBytes[1] = d; - out.write(castBytes, 0, 2); - } else { - out.write(d); - } - return this; - } - - public Packer packShort(short d) throws IOException - { - if(d < -(1<<5)) { - if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, d); - out.write(castBytes, 0, 3); - } - } - return this; - } - - public Packer packInt(int d) throws IOException - { - if(d < -(1<<5)) { - if(d < -(1<<15)) { - // signed 32 - castBytes[0] = (byte)0xd2; - castBuffer.putInt(1, d); - out.write(castBytes, 0, 5); - } else if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else if(d < (1<<16)) { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // unsigned 32 - castBytes[0] = (byte)0xce; - castBuffer.putInt(1, d); - out.write(castBytes, 0, 5); - } - } - return this; - } - - public Packer packLong(long d) throws IOException - { - if(d < -(1L<<5)) { - if(d < -(1L<<15)) { - if(d < -(1L<<31)) { - // signed 64 - castBytes[0] = (byte)0xd3; - castBuffer.putLong(1, d); - out.write(castBytes, 0, 9); - } else { - // signed 32 - castBytes[0] = (byte)0xd2; - castBuffer.putInt(1, (int)d); - out.write(castBytes, 0, 5); - } - } else { - if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1L<<16)) { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - //System.out.println("pack uint 16 "+(short)d); - } - } else { - if(d < (1L<<32)) { - // unsigned 32 - castBytes[0] = (byte)0xce; - castBuffer.putInt(1, (int)d); - out.write(castBytes, 0, 5); - } else { - // unsigned 64 - castBytes[0] = (byte)0xcf; - castBuffer.putLong(1, d); - out.write(castBytes, 0, 9); - } - } - } - return this; - } - - public Packer packFloat(float d) throws IOException - { - castBytes[0] = (byte)0xca; - castBuffer.putFloat(1, d); - out.write(castBytes, 0, 5); - return this; - } - - public Packer packDouble(double d) throws IOException - { - castBytes[0] = (byte)0xcb; - castBuffer.putDouble(1, d); - out.write(castBytes, 0, 9); - return this; - } - - public Packer packNil() throws IOException - { - out.write((byte)0xc0); - return this; - } - - public Packer packTrue() throws IOException - { - out.write((byte)0xc3); - return this; - } - - public Packer packFalse() throws IOException - { - out.write((byte)0xc2); - return this; - } - - public Packer packArray(int n) throws IOException - { - if(n < 16) { - final int d = 0x90 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xdc; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdd; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packMap(int n) throws IOException - { - if(n < 16) { - final int d = 0x80 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xde; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdf; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packRaw(int n) throws IOException - { - if(n < 32) { - final int d = 0xa0 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xda; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdb; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packRawBody(byte[] b) throws IOException - { - out.write(b); - return this; - } - - public Packer packRawBody(byte[] b, int off, int length) throws IOException - { - out.write(b, off, length); - return this; - } - - //public Packer pack(Object o) throws IOException -} - diff --git a/java-plan1/Unpacker.java b/java-plan1/Unpacker.java deleted file mode 100644 index 32945de..0000000 --- a/java-plan1/Unpacker.java +++ /dev/null @@ -1,401 +0,0 @@ -import java.io.*; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.math.BigInteger; - -public class Unpacker { - protected static final int CS_HEADER = 0x00; - protected static final int CS_FLOAT = 0x0a; - protected static final int CS_DOUBLE = 0x0b; - protected static final int CS_UINT_8 = 0x0c; - protected static final int CS_UINT_16 = 0x0d; - protected static final int CS_UINT_32 = 0x0e; - protected static final int CS_UINT_64 = 0x0f; - protected static final int CS_INT_8 = 0x10; - protected static final int CS_INT_16 = 0x11; - protected static final int CS_INT_32 = 0x12; - protected static final int CS_INT_64 = 0x13; - protected static final int CS_RAW_16 = 0x1a; - protected static final int CS_RAW_32 = 0x1b; - protected static final int CS_ARRAY_16 = 0x1c; - protected static final int CS_ARRAY_32 = 0x1d; - protected static final int CS_MAP_16 = 0x1e; - protected static final int CS_MAP_32 = 0x1f; - protected static final int ACS_RAW_VALUE = 0x20; - protected static final int CT_ARRAY_ITEM = 0x00; - protected static final int CT_MAP_KEY = 0x01; - protected static final int CT_MAP_VALUE = 0x02; - - protected static final int MAX_STACK_SIZE = 16; - - protected int cs = CS_HEADER; - protected int trail = 0; - protected int top = -1; - protected boolean finished = false; - protected int[] stack_ct = new int[MAX_STACK_SIZE]; - protected int[] stack_count = new int[MAX_STACK_SIZE]; - protected Object[] stack_obj = new Object[MAX_STACK_SIZE]; - protected Object[] stack_map_key = new Object[MAX_STACK_SIZE]; - //protected Object user; - protected ByteBuffer castBuffer = ByteBuffer.allocate(8); - - public Object getData() - { - return stack_obj[0]; - } - - public boolean isFinished() - { - return finished; - } - - public void reset() - { - for(int i=0; i <= top; ++top) { - stack_ct[top] = 0; - stack_count[top] = 0; - stack_obj[top] = null; - stack_map_key[top] = null; - } - cs = CS_HEADER; - trail = 0; - top = -1; - finished = false; - } - - @SuppressWarnings("unchecked") - public int execute(byte[] src, int off, int length) throws IOException - { - if(off >= length) { return off; } - - int limit = length; - int i = off; - int count; - - Object obj = null; - - _out: do { - _header_again: { - //System.out.println("while i:"+i+" limit:"+limit); - - int b = src[i]; - - _push: { - _fixed_trail_again: - if(cs == CS_HEADER) { - - if((b & 0x80) == 0) { // Positive Fixnum - //System.out.println("positive fixnum "+b); - obj = b; - break _push; - } - - if((b & 0xe0) == 0xe0) { // Negative Fixnum - //System.out.println("negative fixnum "+b); - obj = b; - break _push; - } - - if((b & 0xe0) == 0xa0) { // FixRaw - trail = b & 0x1f; - if(trail == 0) { - obj = new byte[0]; - break _push; - } - cs = ACS_RAW_VALUE; - break _fixed_trail_again; - } - - if((b & 0xf0) == 0x90) { // FixArray - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - count = b & 0x0f; - ++top; - stack_obj[top] = new ArrayList(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - //System.out.println("fixarray count:"+count); - break _header_again; - } - - if((b & 0xf0) == 0x80) { // FixMap - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - count = b & 0x0f; - ++top; - stack_obj[top] = new HashMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - } - - switch(b & 0xff) { // FIXME - case 0xc0: // nil - obj = null; - break _push; - case 0xc2: // false - obj = false; - break _push; - case 0xc3: // true - obj = true; - break _push; - case 0xca: // float - case 0xcb: // double - case 0xcc: // unsigned int 8 - case 0xcd: // unsigned int 16 - case 0xce: // unsigned int 32 - case 0xcf: // unsigned int 64 - case 0xd0: // signed int 8 - case 0xd1: // signed int 16 - case 0xd2: // signed int 32 - case 0xd3: // signed int 64 - trail = 1 << (b & 0x03); - cs = b & 0x1f; - //System.out.println("a trail "+trail+" cs:"+cs); - break _fixed_trail_again; - case 0xda: // raw 16 - case 0xdb: // raw 32 - case 0xdc: // array 16 - case 0xdd: // array 32 - case 0xde: // map 16 - case 0xdf: // map 32 - trail = 2 << (b & 0x01); - cs = b & 0x1f; - //System.out.println("b trail "+trail+" cs:"+cs); - break _fixed_trail_again; - default: - //System.out.println("unknown b "+(b&0xff)); - throw new IOException("parse error"); - } - - } // _fixed_trail_again - - do { - _fixed_trail_again: { - - if(limit - i <= trail) { break _out; } - int n = i + 1; - i += trail; - - switch(cs) { - case CS_FLOAT: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - obj = castBuffer.getFloat(0); - //System.out.println("float "+obj); - break _push; - case CS_DOUBLE: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - obj = castBuffer.getDouble(0); - //System.out.println("double "+obj); - break _push; - case CS_UINT_8: - //System.out.println(n); - //System.out.println(src[n]); - //System.out.println(src[n+1]); - //System.out.println(src[n-1]); - obj = ((int)src[n]) & 0xff; - //System.out.println("uint8 "+obj); - break _push; - case CS_UINT_16: - //System.out.println(src[n]); - //System.out.println(src[n+1]); - castBuffer.rewind(); - castBuffer.put(src, n, 2); - obj = ((int)castBuffer.getShort(0)) & 0xffff; - //System.out.println("uint 16 "+obj); - break _push; - case CS_UINT_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - { - // FIXME always long? - int o = castBuffer.getInt(0); - if(o < 0) { - obj = ((long)o) & 0xffffffffL; - } else { - obj = o; - } - } - //System.out.println("uint 32 "+obj); - break _push; - case CS_UINT_64: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - { - // FIXME always BigInteger? - long o = castBuffer.getLong(0); - if(o < 0) { - obj = BigInteger.valueOf(o & 0x7fffffffL).setBit(31); - } else { - obj = o; - } - } - throw new IOException("uint 64 is not supported"); - case CS_INT_8: - obj = (int)src[n]; - break _push; - case CS_INT_16: - castBuffer.rewind(); - castBuffer.put(src, n, 2); - obj = (int)castBuffer.getShort(0); - break _push; - case CS_INT_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - obj = (int)castBuffer.getInt(0); - break _push; - case CS_INT_64: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - { - // FIXME always long? - long o = castBuffer.getLong(0); - if(o <= 0x7fffffffL && o > -0x80000000L) { - obj = (int)o; - } else { - obj = o; - } - } - //System.out.println("long "+obj); - break _push; - case CS_RAW_16: - castBuffer.rewind(); - castBuffer.put(src, n, 2); - trail = ((int)castBuffer.getShort(0)) & 0xffff; - if(trail == 0) { - obj = new byte[0]; - break _push; - } - cs = ACS_RAW_VALUE; - break _fixed_trail_again; - case CS_RAW_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - trail = castBuffer.getInt(0) & 0x7fffffff; - if(trail == 0) { - obj = new byte[0]; - break _push; - } - cs = ACS_RAW_VALUE; - case ACS_RAW_VALUE: - obj = ByteBuffer.wrap(src, n, trail); // FIXME プール? コピー - break _push; - case CS_ARRAY_16: - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - castBuffer.rewind(); - castBuffer.put(src, n, 2); - count = ((int)castBuffer.getShort(0)) & 0xffff; - ++top; - stack_obj[top] = new ArrayList(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - break _header_again; - case CS_ARRAY_32: - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - count = castBuffer.getInt(0) & 0x7fffffff; - ++top; - stack_obj[top] = new ArrayList(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - break _header_again; - case CS_MAP_16: - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - castBuffer.rewind(); - castBuffer.put(src, n, 2); - count = ((int)castBuffer.getShort(0)) & 0xffff; - ++top; - stack_obj[top] = new HashMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - case CS_MAP_32: - if(top >= MAX_STACK_SIZE) { - throw new IOException("parse error"); - } - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - count = castBuffer.getInt(0) & 0x7fffffff; - ++top; - stack_obj[top] = new HashMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - default: - throw new IOException("parse error"); - } - - } // _fixed_trail_again - } while(true); - } // _push - - do { - _push: { - //System.out.println("push top:"+top); - if(top == -1) { - stack_obj[0] = obj; - finished = true; - break _out; - } - - switch(stack_ct[top]) { - case CT_ARRAY_ITEM: - //System.out.println("array item "+obj); - ((ArrayList)(stack_obj[top])).add(obj); - if(--stack_count[top] == 0) { - obj = stack_obj[top]; - stack_obj[top] = null; - --top; - break _push; - } - break _header_again; - case CT_MAP_KEY: - //System.out.println("map key:"+top+" "+obj); - stack_map_key[top] = obj; - stack_ct[top] = CT_MAP_VALUE; - break _header_again; - case CT_MAP_VALUE: - //System.out.println("map value:"+top+" "+obj); - ((HashMap)(stack_obj[top])).put(stack_map_key[top], obj); - if(--stack_count[top] == 0) { - obj = stack_obj[top]; - stack_map_key[top] = null; - stack_obj[top] = null; - --top; - break _push; - } - stack_ct[top] = CT_MAP_KEY; - break _header_again; - default: - throw new IOException("parse error"); - } - } // _push - } while(true); - - } // _header_again - cs = CS_HEADER; - ++i; - } while(i < limit); // _out - - return i; - } -} - diff --git a/java-plan1/run.sh b/java-plan1/run.sh deleted file mode 100755 index 1539a37..0000000 --- a/java-plan1/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -javac MessagePack.java Packer.java Unpacker.java test.java -java test diff --git a/java-plan1/test.java b/java-plan1/test.java deleted file mode 100644 index 5b2349f..0000000 --- a/java-plan1/test.java +++ /dev/null @@ -1,94 +0,0 @@ -import java.util.*; -import java.io.*; - -class OpenByteArrayOutputStream extends ByteArrayOutputStream { - int getCount() { return count; } - byte[] getBuffer() { return buf; } -} - - -public class test { - - public static void main(String[] args) throws IOException - { - testSimple(); - testStreaming(); - } - - - public static void testSimple() throws IOException - { - OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); - - Packer pk = new Packer(out); - pk.packArray(3) - .packInt(1) - .packByte((byte)2) - .packDouble(0.3); - - Unpacker pac = new Unpacker(); - int nlen = pac.execute(out.getBuffer(), 0, out.getCount()); - - if(pac.isFinished()) { - System.out.println("testSimple: "+pac.getData()); - } - } - - - public static void testStreaming() throws IOException - { - OpenByteArrayOutputStream out = new OpenByteArrayOutputStream(); - - //// - // sender - // - // initialize the streaming serializer - Packer pk = new Packer(out); - - // serialize 2 objects - pk.packArray(3) - .packInt(0) - .packByte((byte)1) - .packDouble(0.2); - pk.packArray(3) - .packInt(3) - .packByte((byte)4) - .packDouble(0.5); - - // send it through the network - InputStream sock = new ByteArrayInputStream(out.getBuffer(), 0, out.getCount()); - - - //// - // receiver - // - // initialize the streaming deserializer - Unpacker pac = new Unpacker(); - int parsed = 0; - - byte[] buf = new byte[1024]; - int buflen = 0; - - while(true) { - // receive data from the network - int c = sock.read(); - if(c < 0) { return; } - - buf[buflen++] = (byte)c; - - // deserialize - parsed = pac.execute(buf, parsed, buflen); - if(pac.isFinished()) { - // get an object - Object msg = pac.getData(); - System.out.println("testStreaming: "+msg); - - // reset the streaming deserializer - pac.reset(); - buflen = 0; - parsed = 0; - } - } - } -} - diff --git a/java-plan2/build.xml b/java-plan2/build.xml deleted file mode 100644 index d382ce0..0000000 --- a/java-plan2/build.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/java-plan2/src/org/msgpack/GenericArray.java b/java-plan2/src/org/msgpack/GenericArray.java deleted file mode 100644 index 53799ca..0000000 --- a/java-plan2/src/org/msgpack/GenericArray.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.msgpack; - -import java.util.List; -import java.util.ArrayList; - -public class GenericArray extends GenericObject { - private ArrayList array; - - public GenericArray(int length) - { - this.array = new ArrayList(length); - } - - public void add(GenericObject element) - { - array.add(element); - } - - @Override - public List asArray() - { - return array; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericBoolean.java b/java-plan2/src/org/msgpack/GenericBoolean.java deleted file mode 100644 index 908128f..0000000 --- a/java-plan2/src/org/msgpack/GenericBoolean.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.msgpack; - -public class GenericBoolean extends GenericObject { - boolean value; - - public GenericBoolean(boolean value) - { - this.value = value; - } - - @Override - public boolean asBoolean() - { - return value; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericLong.java b/java-plan2/src/org/msgpack/GenericLong.java deleted file mode 100644 index 7cc9110..0000000 --- a/java-plan2/src/org/msgpack/GenericLong.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.msgpack; - -public class GenericLong extends GenericObject { - long value; - - public GenericLong(long value) - { - this.value = value; - } - - @Override - public long asLong() - { - return value; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericMap.java b/java-plan2/src/org/msgpack/GenericMap.java deleted file mode 100644 index 0a5512a..0000000 --- a/java-plan2/src/org/msgpack/GenericMap.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.msgpack; - -import java.util.Map; -import java.util.HashMap; - -public class GenericMap extends GenericObject { - private HashMap map; - - public GenericMap(int length) - { - this.map = new HashMap(length); - } - - public void put(GenericObject key, GenericObject value) - { - map.put(key, value); - } - - @Override - public Map asMap() - { - return map; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericObject.java b/java-plan2/src/org/msgpack/GenericObject.java deleted file mode 100644 index 32eacd3..0000000 --- a/java-plan2/src/org/msgpack/GenericObject.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.msgpack; - -import java.util.List; -import java.util.Map; - -public abstract class GenericObject { - public boolean asBoolean() - { - throw new RuntimeException("type error"); - } - - public byte asByte() - { - throw new RuntimeException("type error"); - } - - public short asShort() - { - throw new RuntimeException("type error"); - } - - public int asInt() - { - throw new RuntimeException("type error"); - } - - public long asLong() - { - throw new RuntimeException("type error"); - } - - public String asString() - { - throw new RuntimeException("type error"); - } - - public byte[] asBytes() - { - throw new RuntimeException("type error"); - } - - public List asArray() - { - throw new RuntimeException("type error"); - } - - public Map asMap() - { - throw new RuntimeException("type error"); - } -} - diff --git a/java-plan2/src/org/msgpack/GenericRaw.java b/java-plan2/src/org/msgpack/GenericRaw.java deleted file mode 100644 index 6de03b2..0000000 --- a/java-plan2/src/org/msgpack/GenericRaw.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.msgpack; - -import java.nio.charset.Charset; - -public class GenericRaw extends GenericObject { - byte[] bytes; - String string; - - public GenericRaw() - { - this.bytes = new byte[0]; - this.string = null; - } - - public GenericRaw(byte[] bytes) - { - this.bytes = bytes; - this.string = null; - } - - public GenericRaw(String string) - { - this.bytes = null; - this.string = string; - } - - public synchronized void setString(String string) - { - this.string = string; - this.bytes = null; - } - - public synchronized void setBytes(byte[] bytes) - { - this.bytes = bytes; - this.string = null; - } - - private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); - - @Override - public synchronized String asString() - { - if(string == null) { - return string = new String(bytes, UTF8_CHARSET); - } - return string; - } - - @Override - public synchronized byte[] asBytes() - { - if(bytes == null) { - return bytes = string.getBytes(UTF8_CHARSET); - } - return bytes; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericRawRef.java b/java-plan2/src/org/msgpack/GenericRawRef.java deleted file mode 100644 index 7007810..0000000 --- a/java-plan2/src/org/msgpack/GenericRawRef.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.msgpack; - -import java.nio.charset.Charset; - -public class GenericRawRef extends GenericRaw { - int offset; - int length; - - public GenericRawRef(byte[] bytes, int offset, int length) - { - this.bytes = bytes; - this.offset = offset; - this.length = length; - this.string = null; - } - - public GenericRawRef(String string) - { - this.bytes = null; - this.string = string; - } - - public synchronized void setString(String string) - { - this.string = string; - this.bytes = null; - } - - public synchronized void setBytes(byte[] bytes) - { - this.bytes = bytes; - this.string = null; - } - - private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); - - @Override - public synchronized String asString() - { - if(string == null) { - return string = new String(bytes, UTF8_CHARSET); - } - return string; - } - - @Override - public synchronized byte[] asBytes() - { - if(bytes == null) { - return bytes = string.getBytes(UTF8_CHARSET); - } - return bytes; - } -} - diff --git a/java-plan2/src/org/msgpack/GenericShort.java b/java-plan2/src/org/msgpack/GenericShort.java deleted file mode 100644 index eb2463e..0000000 --- a/java-plan2/src/org/msgpack/GenericShort.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.msgpack; - -public class GenericShort extends GenericObject { - short value; - - public GenericShort(short value) - { - this.value = value; - } - - @Override - public short asShort() - { - return value; - } - - @Override - public int asInt() - { - return value; - } - - @Override - public long asLong() - { - return value; - } -} - diff --git a/java-plan2/src/org/msgpack/MessageConvertable.java b/java-plan2/src/org/msgpack/MessageConvertable.java deleted file mode 100644 index 68c123e..0000000 --- a/java-plan2/src/org/msgpack/MessageConvertable.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.msgpack; - -public interface MessageConvertable -{ - public void messageConvert(GenericObject obj); -} - diff --git a/java-plan2/src/org/msgpack/MessageMergeable.java b/java-plan2/src/org/msgpack/MessageMergeable.java deleted file mode 100644 index dc50749..0000000 --- a/java-plan2/src/org/msgpack/MessageMergeable.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.msgpack; - -public interface MessageMergeable { - public void setField(int index, Object value); - public Object getField(int index); -} - diff --git a/java-plan2/src/org/msgpack/MessagePackException.java b/java-plan2/src/org/msgpack/MessagePackException.java deleted file mode 100644 index dc37989..0000000 --- a/java-plan2/src/org/msgpack/MessagePackException.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.msgpack; - -public class MessagePackException extends RuntimeException -{ - public MessagePackException(String message) - { - super(message); - } -} - diff --git a/java-plan2/src/org/msgpack/MessagePackable.java b/java-plan2/src/org/msgpack/MessagePackable.java deleted file mode 100644 index 1efff3f..0000000 --- a/java-plan2/src/org/msgpack/MessagePackable.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.msgpack; - -import java.io.IOException; - -public interface MessagePackable -{ - public void messagePack(Packer pk) throws IOException; -} - diff --git a/java-plan2/src/org/msgpack/Packer.java b/java-plan2/src/org/msgpack/Packer.java deleted file mode 100644 index 4545849..0000000 --- a/java-plan2/src/org/msgpack/Packer.java +++ /dev/null @@ -1,332 +0,0 @@ -package org.msgpack; - -import java.io.OutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.List; -import java.util.Map; - -public class Packer { - protected byte[] castBytes = new byte[9]; - protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); - protected OutputStream out; - - public Packer(OutputStream out) - { - this.out = out; - } - - public Packer packByte(byte d) throws IOException - { - if(d < -(1<<5)) { - castBytes[0] = (byte)0xd1; - castBytes[1] = d; - out.write(castBytes, 0, 2); - } else { - out.write(d); - } - return this; - } - - public Packer packShort(short d) throws IOException - { - if(d < -(1<<5)) { - if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, d); - out.write(castBytes, 0, 3); - } - } - return this; - } - - public Packer packInt(int d) throws IOException - { - if(d < -(1<<5)) { - if(d < -(1<<15)) { - // signed 32 - castBytes[0] = (byte)0xd2; - castBuffer.putInt(1, d); - out.write(castBytes, 0, 5); - } else if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else if(d < (1<<16)) { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // unsigned 32 - castBytes[0] = (byte)0xce; - castBuffer.putInt(1, d); - out.write(castBytes, 0, 5); - } - } - return this; - } - - public Packer packLong(long d) throws IOException - { - if(d < -(1L<<5)) { - if(d < -(1L<<15)) { - if(d < -(1L<<31)) { - // signed 64 - castBytes[0] = (byte)0xd3; - castBuffer.putLong(1, d); - out.write(castBytes, 0, 9); - } else { - // signed 32 - castBytes[0] = (byte)0xd2; - castBuffer.putInt(1, (int)d); - out.write(castBytes, 0, 5); - } - } else { - if(d < -(1<<7)) { - // signed 16 - castBytes[0] = (byte)0xd1; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - } else { - // signed 8 - castBytes[0] = (byte)0xd0; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } - } - } else if(d < (1<<7)) { - // fixnum - out.write((byte)d); - } else { - if(d < (1L<<16)) { - if(d < (1<<8)) { - // unsigned 8 - castBytes[0] = (byte)0xcc; - castBytes[1] = (byte)d; - out.write(castBytes, 0, 2); - } else { - // unsigned 16 - castBytes[0] = (byte)0xcd; - castBuffer.putShort(1, (short)d); - out.write(castBytes, 0, 3); - //System.out.println("pack uint 16 "+(short)d); - } - } else { - if(d < (1L<<32)) { - // unsigned 32 - castBytes[0] = (byte)0xce; - castBuffer.putInt(1, (int)d); - out.write(castBytes, 0, 5); - } else { - // unsigned 64 - castBytes[0] = (byte)0xcf; - castBuffer.putLong(1, d); - out.write(castBytes, 0, 9); - } - } - } - return this; - } - - public Packer packFloat(float d) throws IOException - { - castBytes[0] = (byte)0xca; - castBuffer.putFloat(1, d); - out.write(castBytes, 0, 5); - return this; - } - - public Packer packDouble(double d) throws IOException - { - castBytes[0] = (byte)0xcb; - castBuffer.putDouble(1, d); - out.write(castBytes, 0, 9); - return this; - } - - public Packer packNil() throws IOException - { - out.write((byte)0xc0); - return this; - } - - public Packer packTrue() throws IOException - { - out.write((byte)0xc3); - return this; - } - - public Packer packFalse() throws IOException - { - out.write((byte)0xc2); - return this; - } - - public Packer packArray(int n) throws IOException - { - if(n < 16) { - final int d = 0x90 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xdc; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdd; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packMap(int n) throws IOException - { - if(n < 16) { - final int d = 0x80 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xde; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdf; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packRaw(int n) throws IOException - { - if(n < 32) { - final int d = 0xa0 | n; - out.write((byte)d); - } else if(n < 65536) { - castBytes[0] = (byte)0xda; - castBuffer.putShort(1, (short)n); - out.write(castBytes, 0, 3); - } else { - castBytes[0] = (byte)0xdb; - castBuffer.putInt(1, n); - out.write(castBytes, 0, 5); - } - return this; - } - - public Packer packRawBody(byte[] b) throws IOException - { - out.write(b); - return this; - } - - public Packer packRawBody(byte[] b, int off, int length) throws IOException - { - out.write(b, off, length); - return this; - } - - private static Charset UTF8_CHARSET = Charset.forName("UTF-8"); - - public Packer packString(String s) throws IOException - { - byte[] b = ((String)s).getBytes(UTF8_CHARSET); - packRaw(b.length); - packRawBody(b); - return this; - } - - public Packer pack(MessagePackable o) throws IOException - { - o.messagePack(this); - return this; - } - - public Packer pack(Object o) throws IOException - { - if(o == null) { - packNil(); - } else if(o instanceof String) { - byte[] b = ((String)o).getBytes(UTF8_CHARSET); - packRaw(b.length); - packRawBody(b); - } else if(o instanceof byte[]) { - byte[] b = (byte[])o; - packRaw(b.length); - packRawBody(b); - } else if(o instanceof List) { - List l = (List)o; - packArray(l.size()); - for(Object i : l) { pack(i); } - } else if(o instanceof Map) { - Map m = (Map)o; - packMap(m.size()); - for(Map.Entry e : m.entrySet()) { - pack(e.getKey()); - pack(e.getValue()); - } - } else if(o instanceof Boolean) { - if((Boolean)o) { - packTrue(); - } else { - packFalse(); - } - } else if(o instanceof Integer) { - packInt((Integer)o); - } else if(o instanceof Long) { - packLong((Long)o); - } else if(o instanceof Short) { - packShort((Short)o); - } else if(o instanceof Byte) { - packByte((Byte)o); - } else if(o instanceof Float) { - packFloat((Float)o); - } else if(o instanceof Double) { - packDouble((Double)o); - } else if(o instanceof MessagePackable) { - ((MessagePackable)o).messagePack(this); - } else { - throw new IOException("unknown object "+o+" ("+o.getClass()+")"); - } - return this; - } -} - diff --git a/java-plan2/src/org/msgpack/UnbufferedUnpacker.java b/java-plan2/src/org/msgpack/UnbufferedUnpacker.java deleted file mode 100644 index 5b5eb97..0000000 --- a/java-plan2/src/org/msgpack/UnbufferedUnpacker.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.msgpack; - -import java.lang.Iterable; -import java.io.InputStream; -import java.io.IOException; -import java.util.Iterator; -import org.msgpack.impl.*; -import org.msgpack.schema.Schema; - -public class UnbufferedUnpacker extends UnpackerImpl { - private int offset; - private boolean finished; - private Object data; - - public UnbufferedUnpacker() - { - super(new GenericObjectBuilder()); - this.offset = 0; - this.finished = false; - } - - public UnbufferedUnpacker useSchema(Schema s) - { - super.setBuilder(new SpecificObjectBuilder(s)); - return this; - } - - public Object getData() - { - return data; - } - - public boolean isFinished() - { - return finished; - } - - public void reset() - { - super.reset(); - this.offset = 0; - } - - int getOffset() - { - return offset; - } - - void setOffset(int offset) - { - this.offset = offset; - } - - public int execute(byte[] buffer) throws UnpackException - { - return execute(buffer, 0, buffer.length); - } - - // FIXME - public int execute(byte[] buffer, int offset, int length) throws UnpackException - { - int noffset = super.execute(buffer, offset + this.offset, length); - this.offset = noffset - offset; - if(super.isFinished()) { - this.data = super.getData(); - this.finished = true; - super.reset(); - } else { - this.finished = false; - } - return noffset; - } -} - diff --git a/java-plan2/src/org/msgpack/UnpackException.java b/java-plan2/src/org/msgpack/UnpackException.java deleted file mode 100644 index 2081faf..0000000 --- a/java-plan2/src/org/msgpack/UnpackException.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.msgpack; - -public class UnpackException extends MessagePackException -{ - public static final int EXTRA_BYTES = 1; - public static final int INSUFFICIENT_BYTES = 0; - public static final int PARSE_ERROR = -1; - - private int errorCode; - private Object data; - - public UnpackException(String message, int errorCode) - { - super(message); - this.errorCode = errorCode; - } - - public UnpackException(String message, int errorCode, Object data) - { - super(message); - this.errorCode = errorCode; - this.data = data; - } - - public int getErrorCode() - { - return errorCode; - } - - public Object getData() - { - return data; - } -} - diff --git a/java-plan2/src/org/msgpack/UnpackIterator.java b/java-plan2/src/org/msgpack/UnpackIterator.java deleted file mode 100644 index 9010101..0000000 --- a/java-plan2/src/org/msgpack/UnpackIterator.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.msgpack; - -import java.io.IOException; -import java.util.Iterator; -import java.util.NoSuchElementException; - -public class UnpackIterator implements Iterator { - private Unpacker pac; - private boolean have; - private Object data; - - UnpackIterator(Unpacker pac) - { - this.pac = pac; - this.have = false; - } - - public boolean hasNext() - { - if(have) { return true; } - try { - while(true) { - if(pac.execute()) { - data = pac.getData(); - pac.reset(); - have = true; - return true; - } - - if(!pac.fill()) { - return false; - } - } - } catch (IOException e) { - return false; - } - } - - public Object next() - { - if(!have) { - throw new NoSuchElementException(); - } - have = false; - return data; - } - - public void remove() - { - throw new UnsupportedOperationException(); - } -} - diff --git a/java-plan2/src/org/msgpack/Unpacker.java b/java-plan2/src/org/msgpack/Unpacker.java deleted file mode 100644 index 5a58ce8..0000000 --- a/java-plan2/src/org/msgpack/Unpacker.java +++ /dev/null @@ -1,269 +0,0 @@ -package org.msgpack; - -import java.lang.Iterable; -import java.io.InputStream; -import java.io.IOException; -import java.util.Iterator; -import org.msgpack.impl.*; -import org.msgpack.schema.Schema; - -public class Unpacker extends UnpackerImpl implements Iterable { - - public static final int DEFAULT_BUFFER_SIZE = 32*1024; - - private int used; - private int offset; - private int parsed; - private byte[] buffer; - private int bufferReserveSize; - private InputStream stream; - - public Unpacker() - { - super(new GenericZeroCopyObjectBuilder()); - this.used = 0; - this.offset = 0; - this.parsed = 0; - this.buffer = new byte[DEFAULT_BUFFER_SIZE]; - this.bufferReserveSize = DEFAULT_BUFFER_SIZE/2; - this.stream = null; - } - - public Unpacker(int bufferReserveSize) - { - super(new GenericZeroCopyObjectBuilder()); - this.used = 0; - this.offset = 0; - this.parsed = 0; - this.buffer = new byte[bufferReserveSize]; - this.bufferReserveSize = bufferReserveSize/2; - this.stream = null; - } - - public Unpacker(InputStream stream) - { - super(new GenericZeroCopyObjectBuilder()); - this.used = 0; - this.offset = 0; - this.parsed = 0; - this.buffer = new byte[DEFAULT_BUFFER_SIZE]; - this.bufferReserveSize = DEFAULT_BUFFER_SIZE/2; - this.stream = stream; - } - - public Unpacker(InputStream stream, int bufferReserveSize) - { - super(new GenericZeroCopyObjectBuilder()); - this.used = 0; - this.offset = 0; - this.parsed = 0; - this.buffer = new byte[bufferReserveSize]; - this.bufferReserveSize = bufferReserveSize/2; - this.stream = stream; - } - - public Unpacker useSchema(Schema s) - { - super.setBuilder(new SpecificObjectBuilder(s)); - return this; - } - - public void reserveBuffer(int size) - { - if(buffer.length - used >= size) { - return; - } - /* - if(used == parsed && buffer.length >= size) { - // rewind buffer - used = 0; - offset = 0; - return; - } - */ - - int nextSize = buffer.length * 2; - while(nextSize < size + used) { - nextSize *= 2; - } - - byte[] tmp = new byte[nextSize]; - System.arraycopy(buffer, offset, tmp, 0, used - offset); - - buffer = tmp; - used -= offset; - offset = 0; - } - - public byte[] getBuffer() - { - return buffer; - } - - public int getBufferOffset() - { - return used; - } - - public int getBufferCapacity() - { - return buffer.length - used; - } - - public void bufferConsumed(int size) - { - used += size; - } - - public void feed(byte[] buffer) - { - feed(buffer, 0, buffer.length); - } - - public void feed(byte[] buffer, int offset, int length) - { - reserveBuffer(length); - System.arraycopy(buffer, offset, this.buffer, this.offset, length); - bufferConsumed(length); - } - - public boolean fill() throws IOException - { - if(stream == null) { - return false; - } - reserveBuffer(bufferReserveSize); - int rl = stream.read(getBuffer(), getBufferOffset(), getBufferCapacity()); - if(rl <= 0) { - return false; - } - bufferConsumed(rl); - return true; - } - - public Iterator iterator() - { - return new UnpackIterator(this); - } - - public boolean execute() throws UnpackException - { - int noffset = super.execute(buffer, offset, used); - if(noffset <= offset) { - return false; - } - parsed += noffset - offset; - offset = noffset; - return super.isFinished(); - } - - public Object getData() - { - return super.getData(); - } - - public void reset() - { - super.reset(); - parsed = 0; - } - - public int getMessageSize() - { - return parsed - offset + used; - } - - public int getParsedSize() - { - return parsed; - } - - public int getNonParsedSize() - { - return used - offset; - } - - public void skipNonparsedBuffer(int size) - { - offset += size; - } - - public void removeNonparsedBuffer() - { - used = offset; - } - - /* - public static class Context { - private boolean finished; - private Object data; - private int offset; - private UnpackerImpl impl; - - public Context() - { - this.finished = false; - this.impl = new UnpackerImpl(); - } - - public boolean isFinished() - { - return finished; - } - - public Object getData() - { - return data; - } - - int getOffset() - { - return offset; - } - - void setFinished(boolean finished) - { - this.finished = finished; - } - - void setData(Object data) - { - this.data = data; - } - - void setOffset(int offset) - { - this.offset = offset; - } - - UnpackerImpl getImpl() - { - return impl; - } - } - - public static int unpack(Context ctx, byte[] buffer) throws UnpackException - { - return unpack(ctx, buffer, 0, buffer.length); - } - - public static int unpack(Context ctx, byte[] buffer, int offset, int length) throws UnpackException - { - UnpackerImpl impl = ctx.getImpl(); - int noffset = impl.execute(buffer, offset + ctx.getOffset(), length); - ctx.setOffset(noffset - offset); - if(impl.isFinished()) { - ctx.setData(impl.getData()); - ctx.setFinished(false); - impl.reset(); - } else { - ctx.setData(null); - ctx.setFinished(true); - } - int parsed = noffset - offset; - ctx.setOffset(parsed); - return noffset; - } - */ -} - diff --git a/java-plan2/src/org/msgpack/impl/ArrayBuilder.java b/java-plan2/src/org/msgpack/impl/ArrayBuilder.java deleted file mode 100644 index 9bb099b..0000000 --- a/java-plan2/src/org/msgpack/impl/ArrayBuilder.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.msgpack.impl; - -public interface ArrayBuilder { - public void add(Object element); - public Object finish(); -} - diff --git a/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java b/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java deleted file mode 100644 index 814e302..0000000 --- a/java-plan2/src/org/msgpack/impl/GenericObjectBuilder.java +++ /dev/null @@ -1,127 +0,0 @@ -package org.msgpack.impl; - -import org.msgpack.*; - -public class GenericObjectBuilder implements ObjectBuilder { - public Object createNil() - { - return null; - } - - @Override - public Object createBoolean(boolean v) - { - return new GenericBoolean(v); - } - - @Override - public Object createByte(byte v) - { - //return new GenericByte(v); - return null; // FIXME - } - - @Override - public Object createShort(short v) - { - return new GenericShort(v); - } - - @Override - public Object createInt(int v) - { - //return new GenericInt(v); - return null; // FIXME - } - - @Override - public Object createLong(long v) - { - return new GenericLong(v); - } - - @Override - public Object createFloat(float v) - { - //return new GenericFloat(v); - return null; // FIXME - } - - @Override - public Object createDouble(double v) - { - //return new GenericDouble(v); - return null; // FIXME - } - - @Override - public Object createRaw(byte[] b, int offset, int length) - { - byte[] copy = new byte[length]; - System.arraycopy(b, offset, copy, 0, length); - return new GenericRaw(copy); - } - - @Override - public ArrayBuilder createArray(int length) - { - return new GenericArrayBuilder(length); - } - - @Override - public MapBuilder createMap(int length) - { - return new GenericMapBuilder(length); - } -} - -final class GenericArrayBuilder implements ArrayBuilder { - private GenericArray a; - - GenericArrayBuilder(int length) - { - this.a = new GenericArray(length); - } - - @Override - public void add(Object element) - { - a.add((GenericObject)element); - } - - @Override - public Object finish() - { - return a; - } -} - -final class GenericMapBuilder implements MapBuilder { - private GenericMap m; - private GenericObject key; - - GenericMapBuilder(int length) - { - this.m = new GenericMap(length); - } - - @Override - public void putKey(Object key) - { - this.key = (GenericObject)key; - } - - @Override - public void putValue(Object value) - { - m.put(this.key, (GenericObject)value); - this.key = null; - } - - @Override - public Object finish() - { - return m; - } -} - diff --git a/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java b/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java deleted file mode 100644 index 5569fbf..0000000 --- a/java-plan2/src/org/msgpack/impl/GenericZeroCopyObjectBuilder.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.msgpack.impl; - -import org.msgpack.*; - -public class GenericZeroCopyObjectBuilder extends GenericObjectBuilder { - @Override - public Object createRaw(byte[] b, int offset, int length) - { - return new GenericRawRef(b, offset, length); - } -} - diff --git a/java-plan2/src/org/msgpack/impl/MapBuilder.java b/java-plan2/src/org/msgpack/impl/MapBuilder.java deleted file mode 100644 index 57859a6..0000000 --- a/java-plan2/src/org/msgpack/impl/MapBuilder.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.msgpack.impl; - -public interface MapBuilder { - public void putKey(Object key); - public void putValue(Object value); - public Object finish(); -} - diff --git a/java-plan2/src/org/msgpack/impl/ObjectBuilder.java b/java-plan2/src/org/msgpack/impl/ObjectBuilder.java deleted file mode 100644 index 3268903..0000000 --- a/java-plan2/src/org/msgpack/impl/ObjectBuilder.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.msgpack.impl; - -public interface ObjectBuilder { - public Object createNil(); - public Object createBoolean(boolean v); - public Object createByte(byte v); - public Object createShort(short v); - public Object createInt(int v); - public Object createLong(long v); - public Object createFloat(float v); - public Object createDouble(double v); - public Object createRaw(byte[] b, int offset, int length); - public ArrayBuilder createArray(int length); - public MapBuilder createMap(int length); -} - diff --git a/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java b/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java deleted file mode 100644 index 7748844..0000000 --- a/java-plan2/src/org/msgpack/impl/SpecificObjectBuilder.java +++ /dev/null @@ -1,203 +0,0 @@ -package org.msgpack.impl; - -import java.util.List; -import java.util.ArrayList; -import java.util.HashMap; -import org.msgpack.schema.*; - -public final class SpecificObjectBuilder implements ObjectBuilder { - private int top; - private Schema schema; - - public SpecificObjectBuilder(Schema schema) - { - this.top = 0; - this.schema = schema; - } - - void setSchema(Schema s) - { - schema = s; - } - - Schema swapSchema(Schema s) - { - Schema old = schema; - schema = s; - return old; - } - - @Override - public Object createNil() - { - return schema.createNil(); - } - - @Override - public Object createBoolean(boolean v) - { - return schema.createBoolean(v); - } - - @Override - public Object createByte(byte v) - { - return schema.createByte(v); - } - - @Override - public Object createShort(short v) - { - return schema.createShort(v); - } - - @Override - public Object createInt(int v) - { - return schema.createInt(v); - } - - @Override - public Object createLong(long v) - { - return schema.createLong(v); - } - - @Override - public Object createFloat(float v) - { - return schema.createFloat(v); - } - - @Override - public Object createDouble(double v) - { - return schema.createDouble(v); - } - - @Override - public Object createRaw(byte[] b, int offset, int length) - { - return schema.createRaw(b, offset, length); - } - - @Override - public ArrayBuilder createArray(int length) - { - if(schema instanceof ClassSchema) { - return new ClassBuilder(length, (ClassSchema)schema, this); - } - ArraySchema as = (ArraySchema)schema; - return new SpecificArrayBuilder(length, as.getElementType(), this); - } - - @Override - public MapBuilder createMap(int length) - { - MapSchema ms = (MapSchema)schema; - return new SpecificMapBuilder(length, ms.getKeyType(), ms.getValueType(), this); - } -} - -final class SpecificArrayBuilder implements ArrayBuilder { - private ArrayList a; - private SpecificObjectBuilder builder; - private Schema parentSchema; - - SpecificArrayBuilder(int length, Schema elementSchema, SpecificObjectBuilder builder) - { - this.a = new ArrayList(length); - this.builder = builder; - this.parentSchema = builder.swapSchema(elementSchema); - } - - public void add(Object element) - { - a.add(element); - } - - public Object finish() - { - builder.swapSchema(parentSchema); - return a; - } -} - -final class SpecificMapBuilder implements MapBuilder { - private HashMap m; - private Object key; - private Schema keySchema; - private Schema valueSchema; - private SpecificObjectBuilder builder; - private Schema parentSchema; - - SpecificMapBuilder(int length, Schema keySchema, Schema valueSchema, SpecificObjectBuilder builder) - { - this.m = new HashMap(length); - this.keySchema = keySchema; - this.valueSchema = valueSchema; - this.builder = builder; - this.parentSchema = builder.swapSchema(keySchema); - } - - @Override - public void putKey(Object key) - { - this.key = key; - this.builder.setSchema(valueSchema); - } - - @Override - public void putValue(Object value) - { - m.put(this.key, value); - this.key = null; - this.builder.setSchema(keySchema); - } - - @Override - public Object finish() - { - builder.swapSchema(parentSchema); - return m; - } -} - -final class ClassBuilder implements ArrayBuilder { - private Object object; - private int index; - private List fields; - private SpecificObjectBuilder builder; - private Schema parentSchema; - - ClassBuilder(int length, ClassSchema schema, SpecificObjectBuilder builder) - { - this.object = schema.newInstance(); - this.index = 0; - this.fields = schema.getFields(); - this.builder = builder; - this.parentSchema = builder.swapSchema(fields.get(0).getType()); - // FIXME check length - } - - @Override - public void add(Object element) - { - FieldSchema f = fields.get(index++); // FIXME check fields.size - f.setFieldValue(object, element); // XXX FIXME debug - if(fields.size() > index) { - builder.setSchema( fields.get(index).getType() ); - } else { - builder.setSchema( null ); - // FIXME: builder.setSchema(new InvalidFieldSchema); - } - } - - @Override - public Object finish() - { - builder.swapSchema(parentSchema); - return object; - } -} - diff --git a/java-plan2/src/org/msgpack/impl/UnpackerImpl.java b/java-plan2/src/org/msgpack/impl/UnpackerImpl.java deleted file mode 100644 index 9f88072..0000000 --- a/java-plan2/src/org/msgpack/impl/UnpackerImpl.java +++ /dev/null @@ -1,395 +0,0 @@ -package org.msgpack.impl; - -import java.nio.ByteBuffer; -//import java.math.BigInteger; -import org.msgpack.UnpackException; - -public class UnpackerImpl { - static final int CS_HEADER = 0x00; - static final int CS_FLOAT = 0x0a; - static final int CS_DOUBLE = 0x0b; - static final int CS_UINT_8 = 0x0c; - static final int CS_UINT_16 = 0x0d; - static final int CS_UINT_32 = 0x0e; - static final int CS_UINT_64 = 0x0f; - static final int CS_INT_8 = 0x10; - static final int CS_INT_16 = 0x11; - static final int CS_INT_32 = 0x12; - static final int CS_INT_64 = 0x13; - static final int CS_RAW_16 = 0x1a; - static final int CS_RAW_32 = 0x1b; - static final int CS_ARRAY_16 = 0x1c; - static final int CS_ARRAY_32 = 0x1d; - static final int CS_MAP_16 = 0x1e; - static final int CS_MAP_32 = 0x1f; - static final int ACS_RAW_VALUE = 0x20; - static final int CT_ARRAY_ITEM = 0x00; - static final int CT_MAP_KEY = 0x01; - static final int CT_MAP_VALUE = 0x02; - - static final int MAX_STACK_SIZE = 16; - - protected int cs = CS_HEADER; - protected int trail = 0; - protected int top = -1; - protected boolean finished = false; - protected Object data = null; - protected int[] stack_ct = new int[MAX_STACK_SIZE]; - protected int[] stack_count = new int[MAX_STACK_SIZE]; - protected Object[] stack_obj = new Object[MAX_STACK_SIZE]; - protected ByteBuffer castBuffer = ByteBuffer.allocate(8); - protected ObjectBuilder builder; - - protected UnpackerImpl(ObjectBuilder builder) - { - this.builder = builder; - } - - protected void setBuilder(ObjectBuilder builder) - { - this.builder = builder; - } - - protected Object getData() - { - return data; - } - - protected boolean isFinished() - { - return finished; - } - - protected void reset() - { - for(int i=0; i <= top; ++top) { - stack_ct[top] = 0; - stack_count[top] = 0; - stack_obj[top] = null; - } - cs = CS_HEADER; - trail = 0; - top = -1; - finished = false; - data = null; - } - - @SuppressWarnings("unchecked") - protected int execute(byte[] src, int off, int length) throws UnpackException - { - if(off >= length) { return off; } - - int limit = length; - int i = off; - int count; - - Object obj = null; - - _out: do { - _header_again: { - //System.out.println("while i:"+i+" limit:"+limit); - - int b = src[i]; - - _push: { - _fixed_trail_again: - if(cs == CS_HEADER) { - - if((b & 0x80) == 0) { // Positive Fixnum - //System.out.println("positive fixnum "+b); - obj = builder.createByte((byte)b); - break _push; - } - - if((b & 0xe0) == 0xe0) { // Negative Fixnum - //System.out.println("negative fixnum "+b); - obj = builder.createByte((byte)b); - break _push; - } - - if((b & 0xe0) == 0xa0) { // FixRaw - trail = b & 0x1f; - if(trail == 0) { - obj = builder.createRaw(new byte[0], 0, 0); - break _push; - } - cs = ACS_RAW_VALUE; - break _fixed_trail_again; - } - - if((b & 0xf0) == 0x90) { // FixArray - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - count = b & 0x0f; - ++top; - stack_obj[top] = builder.createArray(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - //System.out.println("fixarray count:"+count); - break _header_again; - } - - if((b & 0xf0) == 0x80) { // FixMap - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - count = b & 0x0f; - ++top; - stack_obj[top] = builder.createMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - } - - switch(b & 0xff) { // FIXME - case 0xc0: // nil - obj = builder.createNil(); - break _push; - case 0xc2: // false - obj = builder.createBoolean(false); - break _push; - case 0xc3: // true - obj = builder.createBoolean(true); - break _push; - case 0xca: // float - case 0xcb: // double - case 0xcc: // unsigned int 8 - case 0xcd: // unsigned int 16 - case 0xce: // unsigned int 32 - case 0xcf: // unsigned int 64 - case 0xd0: // signed int 8 - case 0xd1: // signed int 16 - case 0xd2: // signed int 32 - case 0xd3: // signed int 64 - trail = 1 << (b & 0x03); - cs = b & 0x1f; - //System.out.println("a trail "+trail+" cs:"+cs); - break _fixed_trail_again; - case 0xda: // raw 16 - case 0xdb: // raw 32 - case 0xdc: // array 16 - case 0xdd: // array 32 - case 0xde: // map 16 - case 0xdf: // map 32 - trail = 2 << (b & 0x01); - cs = b & 0x1f; - //System.out.println("b trail "+trail+" cs:"+cs); - break _fixed_trail_again; - default: - //System.out.println("unknown b "+(b&0xff)); - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - - } // _fixed_trail_again - - do { - _fixed_trail_again: { - - if(limit - i <= trail) { break _out; } - int n = i + 1; - i += trail; - - switch(cs) { - case CS_FLOAT: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - obj = builder.createFloat( castBuffer.getFloat(0) ); - //System.out.println("float "+obj); - break _push; - case CS_DOUBLE: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - obj = builder.createDouble( castBuffer.getDouble(0) ); - //System.out.println("double "+obj); - break _push; - case CS_UINT_8: - //System.out.println(n); - //System.out.println(src[n]); - //System.out.println(src[n+1]); - //System.out.println(src[n-1]); - obj = builder.createShort( (short)((src[n]) & 0xff) ); - //System.out.println("uint8 "+obj); - break _push; - case CS_UINT_16: - //System.out.println(src[n]); - //System.out.println(src[n+1]); - castBuffer.rewind(); - castBuffer.put(src, n, 2); - obj = builder.createInt( ((int)castBuffer.getShort(0)) & 0xffff ); - //System.out.println("uint 16 "+obj); - break _push; - case CS_UINT_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - obj = builder.createLong( ((long)castBuffer.getInt(0)) & 0xffffffffL ); - //System.out.println("uint 32 "+obj); - break _push; - case CS_UINT_64: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - { - long o = castBuffer.getLong(0); - if(o < 0) { - // FIXME - //obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31); - } else { - obj = builder.createLong( o ); - } - } - throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported", UnpackException.PARSE_ERROR); - case CS_INT_8: - obj = builder.createByte( src[n] ); - break _push; - case CS_INT_16: - castBuffer.rewind(); - castBuffer.put(src, n, 2); - obj = builder.createShort( castBuffer.getShort(0) ); - break _push; - case CS_INT_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - obj = builder.createInt( castBuffer.getInt(0) ); - break _push; - case CS_INT_64: - castBuffer.rewind(); - castBuffer.put(src, n, 8); - obj = builder.createLong( castBuffer.getLong(0) ); - break _push; - case CS_RAW_16: - castBuffer.rewind(); - castBuffer.put(src, n, 2); - trail = ((int)castBuffer.getShort(0)) & 0xffff; - if(trail == 0) { - obj = builder.createRaw(new byte[0], 0, 0); - break _push; - } - cs = ACS_RAW_VALUE; - break _fixed_trail_again; - case CS_RAW_32: - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - trail = castBuffer.getInt(0) & 0x7fffffff; - if(trail == 0) { - obj = builder.createRaw(new byte[0], 0, 0); - break _push; - } - cs = ACS_RAW_VALUE; - case ACS_RAW_VALUE: - obj = builder.createRaw(src, n, trail); - break _push; - case CS_ARRAY_16: - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - castBuffer.rewind(); - castBuffer.put(src, n, 2); - count = ((int)castBuffer.getShort(0)) & 0xffff; - ++top; - stack_obj[top] = builder.createArray(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - break _header_again; - case CS_ARRAY_32: - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - count = castBuffer.getInt(0) & 0x7fffffff; - ++top; - stack_obj[top] = builder.createArray(count); - stack_ct[top] = CT_ARRAY_ITEM; - stack_count[top] = count; - break _header_again; - case CS_MAP_16: - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - castBuffer.rewind(); - castBuffer.put(src, n, 2); - count = ((int)castBuffer.getShort(0)) & 0xffff; - ++top; - stack_obj[top] = builder.createMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - case CS_MAP_32: - if(top >= MAX_STACK_SIZE) { - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - castBuffer.rewind(); - castBuffer.put(src, n, 4); - // FIXME overflow check - count = castBuffer.getInt(0) & 0x7fffffff; - ++top; - stack_obj[top] = builder.createMap(count); - stack_ct[top] = CT_MAP_KEY; - stack_count[top] = count; - //System.out.println("fixmap count:"+count); - break _header_again; - default: - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - - } // _fixed_trail_again - } while(true); - } // _push - - do { - _push: { - //System.out.println("push top:"+top); - if(top == -1) { - ++i; - data = obj; - finished = true; - break _out; - } - - switch(stack_ct[top]) { - case CT_ARRAY_ITEM: - //System.out.println("array item "+obj); - ((ArrayBuilder)stack_obj[top]).add(obj); - if(--stack_count[top] == 0) { - obj = ((ArrayBuilder)stack_obj[top]).finish(); - stack_obj[top] = null; - --top; - break _push; - } - break _header_again; - case CT_MAP_KEY: - //System.out.println("map key:"+top+" "+obj); - MapBuilder mb = (MapBuilder)stack_obj[top]; - mb.putKey(obj); - stack_ct[top] = CT_MAP_VALUE; - break _header_again; - case CT_MAP_VALUE: - //System.out.println("map value:"+top+" "+obj); - ((MapBuilder)stack_obj[top]).putValue(obj); - if(--stack_count[top] == 0) { - obj = ((MapBuilder)stack_obj[top]).finish(); - stack_obj[top] = null; - --top; - break _push; - } - stack_ct[top] = CT_MAP_KEY; - break _header_again; - default: - throw new UnpackException("parse error", UnpackException.PARSE_ERROR); - } - } // _push - } while(true); - - } // _header_again - cs = CS_HEADER; - ++i; - } while(i < limit); // _out - - return i; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/ArraySchema.java b/java-plan2/src/org/msgpack/schema/ArraySchema.java deleted file mode 100644 index 4b05190..0000000 --- a/java-plan2/src/org/msgpack/schema/ArraySchema.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.msgpack.schema; - -import java.util.List; -import java.util.ArrayList; -import java.io.IOException; -import org.msgpack.*; - -public class ArraySchema extends Schema { - private Schema elementType; - - public ArraySchema(Schema elementType) - { - super("array"); - this.elementType = elementType; - } - - public Schema getElementType() - { - return elementType; - } - - @Override - public String getFullName() - { - return "ArrayList<"+elementType.getFullName()+">"; - } - - @Override - public String getExpression() - { - return "(array "+elementType.getExpression()+")"; - } - - @Override - @SuppressWarnings("unchecked") - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - List d = (List)obj; - pk.packArray(d.size()); - for(Object e : d) { - elementType.pack(pk, e); - } - } - - @Override - @SuppressWarnings("unchecked") - public Object convert(GenericObject obj) - { - List d = obj.asArray(); - List g = new ArrayList(); - for(GenericObject o : d) { - g.add( elementType.convert(o) ); - } - return g; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/ClassGenerator.java b/java-plan2/src/org/msgpack/schema/ClassGenerator.java deleted file mode 100644 index 25a9620..0000000 --- a/java-plan2/src/org/msgpack/schema/ClassGenerator.java +++ /dev/null @@ -1,230 +0,0 @@ -package org.msgpack.schema; - -import java.util.ArrayList; -import java.util.List; -import java.io.IOException; -import java.io.File; -import java.io.FileOutputStream; -import java.io.Writer; - -public class ClassGenerator { - private ClassSchema schema; - private Writer writer; - private int indent; - - private ClassGenerator(Writer writer) - { - this.writer = writer; - this.indent = 0; - } - - public static void write(Schema schema, Writer dest) throws IOException - { - if(!(schema instanceof ClassSchema)) { - throw new RuntimeException("schema is not class schema"); - } - ClassSchema cs = (ClassSchema)schema; - new ClassGenerator(dest).run(cs); - } - - private void run(ClassSchema cs) throws IOException - { - List subclasses = new ArrayList(); - for(FieldSchema f : cs.getFields()) { - findSubclassSchema(subclasses, f.getType()); - } - - for(ClassSchema sub : subclasses) { - sub.setNamespace(cs.getNamespace()); - sub.setImports(cs.getImports()); - } - - this.schema = cs; - - writeHeader(); - - writeClass(); - - for(ClassSchema sub : subclasses) { - this.schema = sub; - writeSubclass(); - } - - writeFooter(); - - this.schema = null; - writer.flush(); - } - - private void findSubclassSchema(List dst, Schema s) - { - if(s instanceof ClassSchema) { - ClassSchema cs = (ClassSchema)s; - if(!dst.contains(cs)) { dst.add(cs); } - for(FieldSchema f : cs.getFields()) { - findSubclassSchema(dst, f.getType()); - } - } else if(s instanceof ArraySchema) { - ArraySchema as = (ArraySchema)s; - findSubclassSchema(dst, as.getElementType()); - } else if(s instanceof MapSchema) { - MapSchema as = (MapSchema)s; - findSubclassSchema(dst, as.getKeyType()); - findSubclassSchema(dst, as.getValueType()); - } - } - - private void writeHeader() throws IOException - { - if(schema.getNamespace() != null) { - line("package "+schema.getNamespace()+";"); - line(); - } - line("import java.util.*;"); - line("import java.io.*;"); - line("import org.msgpack.*;"); - line("import org.msgpack.schema.*;"); - } - - private void writeFooter() throws IOException - { - } - - private void writeClass() throws IOException - { - line(); - line("public final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); - line("{"); - pushIndent(); - writeSchema(); - writeMemberVariables(); - writeMemberFunctions(); - popIndent(); - line("}"); - } - - private void writeSubclass() throws IOException - { - line(); - line("final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); - line("{"); - pushIndent(); - writeSchema(); - writeMemberVariables(); - writeMemberFunctions(); - popIndent(); - line("}"); - } - - private void writeSchema() throws IOException - { - line("private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load(\""+schema.getExpression()+"\");"); - line("public static ClassSchema getSchema() { return _SCHEMA; }"); - } - - private void writeMemberVariables() throws IOException - { - line(); - for(FieldSchema f : schema.getFields()) { - line("public "+f.getType().getFullName()+" "+f.getName()+";"); - } - } - - private void writeMemberFunctions() throws IOException - { - // void messagePack(Packer pk) - // boolean equals(Object obj) - // int hashCode() - // void set(int _index, Object _value) - // Object get(int _index); - // getXxx() - // setXxx(Xxx xxx) - writeConstructors(); - writeAccessors(); - writePackFunction(); - writeConvertFunction(); - } - - private void writeConstructors() throws IOException - { - line(); - line("public "+schema.getName()+"() { }"); - } - - private void writeAccessors() throws IOException - { - // FIXME - //line(); - //for(FieldSchema f : schema.getFields()) { - // line(""); - //} - } - - private void writePackFunction() throws IOException - { - line(); - line("@Override"); - line("public void messagePack(Packer pk) throws IOException"); - line("{"); - pushIndent(); - line("List _f = _SCHEMA.getFields();"); - line("pk.packArray("+schema.getFields().size()+");"); - int i = 0; - for(FieldSchema f : schema.getFields()) { - line("_f.get("+i+").getType().pack(pk, "+f.getName()+");"); - ++i; - } - popIndent(); - line("}"); - } - - private void writeConvertFunction() throws IOException - { - line(); - line("@Override"); - line("@SuppressWarnings(\"unchecked\")"); - line("public void messageConvert(GenericObject obj)"); - line("{"); - pushIndent(); - line("List _l = obj.asArray();"); - line("List _f = _SCHEMA.getFields();"); - int i = 0; - for(FieldSchema f : schema.getFields()) { - line("if(_l.size() <= "+i+") { return; } "+f.getName()+" = ("+f.getType().getFullName()+")_f.get("+i+").getType().convert(_l.get("+i+"));"); - ++i; - } - popIndent(); - line("}"); - line(); - line("public static "+schema.getName()+" convert(GenericObject obj)"); - line("{"); - pushIndent(); - line("return ("+schema.getName()+")_SCHEMA.convert(obj);"); - popIndent(); - line("}"); - } - - private void line(String str) throws IOException - { - for(int i=0; i < indent; ++i) { - writer.write("\t"); - } - writer.write(str+"\n"); - } - - private void line() throws IOException - { - writer.write("\n"); - } - - private void pushIndent() - { - indent += 1; - } - - private void popIndent() - { - indent -= 1; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/ClassSchema.java b/java-plan2/src/org/msgpack/schema/ClassSchema.java deleted file mode 100644 index 3343fca..0000000 --- a/java-plan2/src/org/msgpack/schema/ClassSchema.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.msgpack.schema; - -import java.util.List; - -public abstract class ClassSchema extends Schema { - protected String namespace; - protected List imports; - - public ClassSchema(String name, String namespace, List imports) - { - super(name); - this.namespace = namespace; - this.imports = imports; - } - - public String getNamespace() - { - return namespace; - } - - public List getImports() - { - return imports; - } - - void setNamespace(String namespace) - { - this.namespace = namespace; - } - - void setImports(List imports) - { - this.imports = imports; - } - - public abstract List getFields(); - public abstract Object newInstance(); -} - diff --git a/java-plan2/src/org/msgpack/schema/FieldSchema.java b/java-plan2/src/org/msgpack/schema/FieldSchema.java deleted file mode 100644 index 31a132c..0000000 --- a/java-plan2/src/org/msgpack/schema/FieldSchema.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.msgpack.schema; - -public abstract class FieldSchema { - private String name; - private Schema type; - - public FieldSchema(String name, Schema type) - { - this.name = name; - this.type = type; - } - - public String getName() - { - return this.name; - } - - public Schema getType() - { - return type; - } - - public String getExpression() - { - return "(field "+name+" "+type.getExpression()+")"; - } - - public abstract Object getFieldValue(Object obj); - public abstract void setFieldValue(Object obj, Object value); -} - diff --git a/java-plan2/src/org/msgpack/schema/GenericClassSchema.java b/java-plan2/src/org/msgpack/schema/GenericClassSchema.java deleted file mode 100644 index f1e2b44..0000000 --- a/java-plan2/src/org/msgpack/schema/GenericClassSchema.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.msgpack.schema; - -import java.util.List; -import java.util.Map; -import java.util.HashMap; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import org.msgpack.*; - - -public class GenericClassSchema extends ClassSchema { - private List fields; - - private String fqdn; - private Constructor constructorCache; - - public GenericClassSchema(String name, List fields, String namespace, List imports) - { - super(name, namespace, imports); - this.fields = fields; - if(namespace == null) { - this.fqdn = name; - } else { - this.fqdn = namespace+"."+name; - } - } - - //@Override - //public String getFullName() - //{ - // if(namespace == null) { - // return getName(); - // } else { - // return namespace+"."+getName(); - // } - //} - - public List getFields() - { - return fields; - } - - public String getNamespace() - { - return namespace; - } - - @Override - public String getExpression() - { - StringBuffer b = new StringBuffer(); - b.append("(class "); - b.append(getName()); - if(namespace != null) { - b.append(" (package "+namespace+")"); - } - for(GenericFieldSchema f : fields) { - b.append(" "+f.getExpression()); - } - b.append(")"); - return b.toString(); - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - // FIXME - } - - @Override - @SuppressWarnings("unchecked") - public Object convert(GenericObject obj) - { - // FIXME - return obj; - } - - @Override - public Object newInstance() - { - return new HashMap(fields.size()); - } -} - diff --git a/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java b/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java deleted file mode 100644 index 507ee18..0000000 --- a/java-plan2/src/org/msgpack/schema/GenericFieldSchema.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.msgpack.schema; - -import java.util.Map; -import java.lang.reflect.Field; - -public final class GenericFieldSchema extends FieldSchema { - public GenericFieldSchema(String name, Schema type) - { - super(name, type); - } - - @Override - public Object getFieldValue(Object obj) - { - return ((Map)obj).get(getName()); - } - - @Override - @SuppressWarnings("unchecked") - public void setFieldValue(Object obj, Object value) - { - ((Map)obj).put(getName(), value); - } -} - diff --git a/java-plan2/src/org/msgpack/schema/IntSchema.java b/java-plan2/src/org/msgpack/schema/IntSchema.java deleted file mode 100644 index 69771c3..0000000 --- a/java-plan2/src/org/msgpack/schema/IntSchema.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class IntSchema extends Schema { - public IntSchema() - { - super("Integer"); - } - - @Override - public String getExpression() - { - return "int"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - pk.packInt((Integer)obj); - } - - @Override - public Object convert(GenericObject obj) - { - return obj.asInt(); - } - - @Override - public Object createByte(byte v) - { - return (int)v; - } - - @Override - public Object createShort(short v) - { - return (int)v; - } - - @Override - public Object createInt(int v) - { - return (int)v; - } - - @Override - public Object createLong(long v) - { - return (int)v; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/LongSchema.java b/java-plan2/src/org/msgpack/schema/LongSchema.java deleted file mode 100644 index 0ba3057..0000000 --- a/java-plan2/src/org/msgpack/schema/LongSchema.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class LongSchema extends Schema { - public LongSchema() - { - super("Long"); - } - - @Override - public String getExpression() - { - return "long"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - pk.packLong((Long)obj); - } - - @Override - public Object convert(GenericObject obj) - { - return obj.asLong(); - } - - @Override - public Object createByte(byte v) - { - return (long)v; - } - - @Override - public Object createShort(short v) - { - return (long)v; - } - - @Override - public Object createInt(int v) - { - return (long)v; - } - - @Override - public Object createLong(long v) - { - return (long)v; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/MapSchema.java b/java-plan2/src/org/msgpack/schema/MapSchema.java deleted file mode 100644 index e72cf63..0000000 --- a/java-plan2/src/org/msgpack/schema/MapSchema.java +++ /dev/null @@ -1,70 +0,0 @@ -package org.msgpack.schema; - -import java.util.List; -import java.util.Map; -import java.util.HashMap; -import java.io.IOException; -import org.msgpack.*; - -public class MapSchema extends Schema { - private Schema keyType; - private Schema valueType; - - public MapSchema(Schema keyType, Schema valueType) - { - super("map"); - this.keyType = keyType; - this.valueType = valueType; - } - - public Schema getKeyType() - { - return keyType; - } - - public Schema getValueType() - { - return valueType; - } - - @Override - public String getFullName() - { - return "HashList<"+keyType.getFullName()+", "+valueType.getFullName()+">"; - } - - @Override - public String getExpression() - { - return "(map "+keyType.getExpression()+" "+valueType.getExpression()+")"; - } - - @Override - @SuppressWarnings("unchecked") - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - Map d = (Map)obj; - pk.packMap(d.size()); - for(Map.Entry e : d.entrySet()) { - keyType.pack(pk, e.getKey()); - valueType.pack(pk, e.getValue()); - } - } - - @Override - @SuppressWarnings("unchecked") - public Object convert(GenericObject obj) - { - Map d = obj.asMap(); - Map g = new HashMap(); - for(Map.Entry e : d.entrySet()) { - g.put(keyType.convert(e.getKey()), valueType.convert(e.getValue())); - } - return g; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/ObjectSchema.java b/java-plan2/src/org/msgpack/schema/ObjectSchema.java deleted file mode 100644 index a9f30f5..0000000 --- a/java-plan2/src/org/msgpack/schema/ObjectSchema.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class ObjectSchema extends Schema { - public ObjectSchema() - { - super("object"); - } - - public String getFullName() - { - return "GenericObject"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - pk.pack(obj); - } - - @Override - public Object convert(GenericObject obj) - { - return obj; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java b/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java deleted file mode 100644 index 023d81b..0000000 --- a/java-plan2/src/org/msgpack/schema/PrimitiveSchema.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.msgpack.schema; - -public abstract class PrimitiveSchema extends Schema { - public static enum PrimitiveType { - BYTE, - SHORT, - INT, - LONG, - FLOAT, - DOUBLE, - } - - public final PrimitiveType type; - - protected PrimitiveSchema(String name, PrimitiveType type) - { - super(name); - this.type = type; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/RawSchema.java b/java-plan2/src/org/msgpack/schema/RawSchema.java deleted file mode 100644 index 847ad29..0000000 --- a/java-plan2/src/org/msgpack/schema/RawSchema.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import java.nio.charset.Charset; -import org.msgpack.*; - -public class RawSchema extends Schema { - public RawSchema() - { - super("raw"); - } - - public String getFullName() - { - return "byte[]"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - byte[] d = (byte[])obj; - pk.packRaw(d.length); - pk.packRawBody(d); - } - - @Override - public Object convert(GenericObject obj) - { - return obj.asBytes(); - } - - @Override - public Object createRaw(byte[] b, int offset, int length) - { - byte[] d = new byte[length]; - System.arraycopy(b, offset, d, 0, length); - return d; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/SSchemaParser.java b/java-plan2/src/org/msgpack/schema/SSchemaParser.java deleted file mode 100644 index bfe912f..0000000 --- a/java-plan2/src/org/msgpack/schema/SSchemaParser.java +++ /dev/null @@ -1,246 +0,0 @@ -package org.msgpack.schema; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Stack; -import java.util.regex.Pattern; -import java.util.regex.Matcher; - -// FIXME exception class - -class SSchemaParser { - public static Schema parse(String source) - { - return new SSchemaParser(false).run(source); - } - - public static Schema load(String source) - { - return new SSchemaParser(true).run(source); - } - - private static abstract class SExp { - boolean isAtom() { return false; } - public String getAtom() { return null; } - - boolean isTuple() { return false; } - public SExp getTuple(int i) { return null; } - public int size() { return 0; } - public boolean empty() { return size() == 0; } - Iterator iterator(int offset) { return null; } - } - - private static class SAtom extends SExp { - private String atom; - - SAtom(String atom) { this.atom = atom; } - - boolean isAtom() { return true; } - public String getAtom() { return atom; } - - public String toString() { return atom; } - } - - private static class STuple extends SExp { - private List tuple; - - STuple() { this.tuple = new ArrayList(); } - - public void add(SExp e) { tuple.add(e); } - - boolean isTuple() { return true; } - public SExp getTuple(int i) { return tuple.get(i); } - public int size() { return tuple.size(); } - - Iterator iterator(int skip) { - Iterator i = tuple.iterator(); - for(int s=0; s < skip; ++s) { i.next(); } - return i; - } - - public String toString() { - if(tuple.isEmpty()) { return "()"; } - Iterator i = tuple.iterator(); - StringBuffer o = new StringBuffer(); - o.append("(").append(i.next()); - while(i.hasNext()) { o.append(" ").append(i.next()); } - o.append(")"); - return o.toString(); - } - } - - boolean specificClass; - - private SSchemaParser(boolean specificClass) - { - this.specificClass = specificClass; - } - - private static Pattern pattern = Pattern.compile( - "(?:\\s+)|([\\(\\)]|[\\d\\w\\.]+)"); - - private Schema run(String source) - { - Matcher m = pattern.matcher(source); - - Stack stack = new Stack(); - String token; - - while(true) { - while(true) { - if(!m.find()) { throw new RuntimeException("unexpected end of file"); } - token = m.group(1); - if(token != null) { break; } - } - - if(token.equals("(")) { - stack.push(new STuple()); - } else if(token.equals(")")) { - STuple top = stack.pop(); - if(stack.empty()) { - stack.push(top); - break; - } - stack.peek().add(top); - } else { - if(stack.empty()) { - throw new RuntimeException("unexpected token '"+token+"'"); - } - stack.peek().add(new SAtom(token)); - } - } - - while(true) { - if(!m.find()) { break; } - token = m.group(1); - if(token != null) { throw new RuntimeException("unexpected token '"+token+"'"); } - } - - return readType( stack.pop() ); - } - - private Schema readType(SExp exp) - { - if(exp.isAtom()) { - String type = exp.getAtom(); - // FIXME - if(type.equals("string")) { - return new StringSchema(); - } else if(type.equals("raw")) { - return new RawSchema(); - } else if(type.equals("short")) { - return new ShortSchema(); - } else if(type.equals("int")) { - return new IntSchema(); - } else if(type.equals("long")) { - return new LongSchema(); - } else if(type.equals("object")) { - return new ObjectSchema(); - } else { - throw new RuntimeException("byte, short, int, long, float, double, raw, string or object is expected but got '"+type+"': "+exp); - } - } else { - String type = exp.getTuple(0).getAtom(); - if(type.equals("class")) { - return parseClass(exp); - } else if(type.equals("array")) { - return parseArray(exp); - } else if(type.equals("map")) { - return parseMap(exp); - } else { - throw new RuntimeException("class, array or map is expected but got '"+type+"': "+exp); - } - } - } - - private ClassSchema parseClass(SExp exp) - { - if(exp.size() < 3 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("class is (class NAME CLASS_BODY): "+exp); - } - - String namespace = null; - List imports = new ArrayList(); - String name = exp.getTuple(1).getAtom(); - List fields = new ArrayList(); - - for(Iterator i=exp.iterator(2); i.hasNext();) { - SExp subexp = i.next(); - if(!subexp.isTuple() || subexp.empty() || !subexp.getTuple(0).isAtom()) { - throw new RuntimeException("field, package or import is expected: "+subexp); - } - String type = subexp.getTuple(0).getAtom(); - if(type.equals("field")) { - fields.add( parseField(subexp) ); - } else if(type.equals("package")) { - if(namespace != null) { - throw new RuntimeException("duplicated package definition: "+subexp); - } - namespace = parseNamespace(subexp); - } else if(type.equals("import")) { - imports.add( parseImport(subexp) ); - } else { - throw new RuntimeException("field, package or import is expected but got '"+type+"': "+subexp); - } - } - - if(specificClass) { - return new SpecificClassSchema(name, fields, namespace, imports); - } else { - return new GenericClassSchema(name, fields, namespace, imports); - } - } - - private ArraySchema parseArray(SExp exp) - { - if(exp.size() != 2) { - throw new RuntimeException("array is (array ELEMENT_TYPE): "+exp); - } - Schema elementType = readType(exp.getTuple(1)); - return new ArraySchema(elementType); - } - - private MapSchema parseMap(SExp exp) - { - if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("map is (map KEY_TYPE VALUE_TYPE): "+exp); - } - Schema keyType = readType(exp.getTuple(1)); - Schema valueType = readType(exp.getTuple(2)); - return new MapSchema(keyType, valueType); - } - - private String parseNamespace(SExp exp) - { - if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("package is (package NAME): "+exp); - } - String name = exp.getTuple(1).getAtom(); - return name; - } - - private String parseImport(SExp exp) - { - if(exp.size() != 2 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("import is (import NAME): "+exp); - } - String name = exp.getTuple(1).getAtom(); - return name; - } - - private FieldSchema parseField(SExp exp) - { - if(exp.size() != 3 || !exp.getTuple(1).isAtom()) { - throw new RuntimeException("field is (field NAME TYPE): "+exp); - } - String name = exp.getTuple(1).getAtom(); - Schema type = readType(exp.getTuple(2)); - if(specificClass) { - return new SpecificFieldSchema(name, type); - } else { - return new GenericFieldSchema(name, type); - } - } -} - diff --git a/java-plan2/src/org/msgpack/schema/Schema.java b/java-plan2/src/org/msgpack/schema/Schema.java deleted file mode 100644 index 15b7e72..0000000 --- a/java-plan2/src/org/msgpack/schema/Schema.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.impl.*; -import org.msgpack.Packer; -import org.msgpack.GenericObject; - -public abstract class Schema { - private String expression; - private String name; - - public Schema(String name) - { - this.expression = expression; - this.name = name; - } - - public String getName() - { - return name; - } - - public String getFullName() - { - return name; - } - - public String getExpression() - { - return name; - } - - public static Schema parse(String source) - { - return SSchemaParser.parse(source); - } - - public static Schema load(String source) - { - return SSchemaParser.load(source); - } - - public abstract void pack(Packer pk, Object obj) throws IOException; - public abstract Object convert(GenericObject obj); - //public abstract Object convertGeneric(GenericObject obj); - - - public Object createNil() - { - return null; - } - - public Object createBoolean(boolean v) - { - throw new RuntimeException("type error"); - } - - public Object createByte(byte v) - { - throw new RuntimeException("type error"); - } - - public Object createShort(short v) - { - throw new RuntimeException("type error"); - } - - public Object createInt(int v) - { - throw new RuntimeException("type error"); - } - - public Object createLong(long v) - { - throw new RuntimeException("type error"); - } - - public Object createFloat(float v) - { - throw new RuntimeException("type error"); - } - - public Object createDouble(double v) - { - throw new RuntimeException("type error"); - } - - public Object createRaw(byte[] b, int offset, int length) - { - throw new RuntimeException("type error"); - } -} - diff --git a/java-plan2/src/org/msgpack/schema/ShortSchema.java b/java-plan2/src/org/msgpack/schema/ShortSchema.java deleted file mode 100644 index aa95f51..0000000 --- a/java-plan2/src/org/msgpack/schema/ShortSchema.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import org.msgpack.*; - -public class ShortSchema extends Schema { - public ShortSchema() - { - super("Short"); - } - - @Override - public String getExpression() - { - return "short"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - pk.packShort((Short)obj); - } - - @Override - public Object convert(GenericObject obj) - { - return obj.asShort(); - } - - @Override - public Object createByte(byte v) - { - return (int)v; - } - - @Override - public Object createShort(short v) - { - return (int)v; - } -} - diff --git a/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java b/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java deleted file mode 100644 index 75c474a..0000000 --- a/java-plan2/src/org/msgpack/schema/SpecificClassSchema.java +++ /dev/null @@ -1,156 +0,0 @@ -package org.msgpack.schema; - -import java.util.List; -import java.util.Map; -import java.io.IOException; -import java.util.Iterator; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import org.msgpack.*; - -public class SpecificClassSchema extends ClassSchema { - private List fields; - private String namespace; - private List imports; - - private String fqdn; - private Constructor constructorCache; - - public SpecificClassSchema(String name, List fields, String namespace, List imports) - { - super(name, namespace, imports); - this.fields = fields; - if(namespace == null) { - this.fqdn = name; - } else { - this.fqdn = namespace+"."+name; - } - } - - //@Override - //public String getFullName() - //{ - // if(namespace == null) { - // return getName(); - // } else { - // return namespace+"."+getName(); - // } - //} - - public List getFields() - { - return fields; - } - - @Override - public String getExpression() - { - StringBuffer b = new StringBuffer(); - b.append("(class "); - b.append(getName()); - if(namespace != null) { - b.append(" (package "+namespace+")"); - } - for(SpecificFieldSchema f : fields) { - b.append(" "+f.getExpression()); - } - b.append(")"); - return b.toString(); - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - - if(constructorCache == null) { - cacheConstructor(); - } - - pk.packArray(fields.size()); - for(SpecificFieldSchema f : fields) { - f.getType().pack(pk, f.getFieldValue(obj)); - } - } - - @Override - @SuppressWarnings("unchecked") - public Object convert(GenericObject obj) - { - if(constructorCache == null) { - cacheConstructor(); - } - - List d = obj.asArray(); - - try { - Object g = constructorCache.newInstance((Object[])null); - - Iterator vi = d.iterator(); - Iterator fi = fields.iterator(); - while(fi.hasNext() && vi.hasNext()) { - SpecificFieldSchema f = fi.next(); - GenericObject v = vi.next(); - f.setFieldValue(g, f.getType().convert(v)); - } - // leave it as uninitialized - //while(fi.hasNext()) { - // SpecificFieldSchema f = fi.next(); - // g.put(f.getName(), null); - //} - - return g; - - } catch (InvocationTargetException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (InstantiationException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (IllegalAccessException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } - } - - private void cacheConstructor() - { - try { - Class c = Class.forName(fqdn); - int index = 0; - for(SpecificFieldSchema f : fields) { - f.cacheField(c, index++); - } - constructorCache = c.getDeclaredConstructor((Class[])null); - constructorCache.setAccessible(true); - } catch(ClassNotFoundException e) { - throw new RuntimeException("class not found: "+fqdn); - } catch (NoSuchMethodException e) { - throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage()); - } - } - - @Override - public Object newInstance() - { - if(constructorCache == null) { - cacheConstructor(); - } - try { - return constructorCache.newInstance((Object[])null); - } catch (InvocationTargetException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (InstantiationException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } catch (IllegalAccessException e) { - throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); - } - } - - public boolean equals(SpecificClassSchema o) - { - return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) && - getName().equals(o.getName()); - } -} - diff --git a/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java b/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java deleted file mode 100644 index 297df27..0000000 --- a/java-plan2/src/org/msgpack/schema/SpecificFieldSchema.java +++ /dev/null @@ -1,78 +0,0 @@ -package org.msgpack.schema; - -import java.util.Map; -import java.util.Arrays; -import java.lang.reflect.Field; -import org.msgpack.*; - -public class SpecificFieldSchema extends FieldSchema { - public Field fieldCache; - private int index; - - public SpecificFieldSchema(String name, Schema type) - { - super(name, type); - this.index = -1; - } - - @Override - public Object getFieldValue(Object obj) - { - if(index >= 0) { - return ((MessageMergeable)obj).getField(index); - } - - try { - return fieldCache.get(obj); - } catch(IllegalArgumentException e) { - throw new RuntimeException("can't get value from '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); - } catch(IllegalAccessException e) { - throw new RuntimeException("can't get value from '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); - } - } - - @Override - public void setFieldValue(Object obj, Object value) - { - if(index >= 0) { - ((MessageMergeable)obj).setField(index, value); - return; - } - - try { - fieldCache.set(obj, value); - } catch(IllegalArgumentException e) { - throw new RuntimeException("can't set value into '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); - } catch(IllegalAccessException e) { - throw new RuntimeException("can't set value into '"+getName()+"' field of '"+obj.getClass().getName()+"' class: "+e.getMessage()); - } - } - - void cacheField(Class c, int index) - { - for(Class i : c.getInterfaces()) { - if(i.equals(MessageMergeable.class)) { - this.index = index; - return; - } - } - - try { - fieldCache = c.getDeclaredField(getName()); - if(!fieldCache.isAccessible()) { - fieldCache.setAccessible(true); - } - } catch(NoSuchFieldException e) { - throw new RuntimeException("can't get '"+getName()+"' field of '"+c.getName()+"' class: "+e.getMessage()); - } catch(SecurityException e) { - throw new RuntimeException("can't get '"+getName()+"' field of '"+c.getName()+"' class: "+e.getMessage()); - } - } - - //public void setFieldInt(Object obj, int value) - //{ - // if(type instanceof PrimitiveSchema) { - // } - //} -} - diff --git a/java-plan2/src/org/msgpack/schema/StringSchema.java b/java-plan2/src/org/msgpack/schema/StringSchema.java deleted file mode 100644 index fc6855b..0000000 --- a/java-plan2/src/org/msgpack/schema/StringSchema.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.msgpack.schema; - -import java.io.IOException; -import java.nio.charset.Charset; -import org.msgpack.*; - -public class StringSchema extends Schema { - public StringSchema() - { - super("string"); - } - - public String getFullName() - { - return "String"; - } - - @Override - public void pack(Packer pk, Object obj) throws IOException - { - if(obj == null) { - pk.packNil(); - return; - } - String s = (String)obj; - byte[] d = s.getBytes("UTF-8"); - pk.packRaw(d.length); - pk.packRawBody(d); - } - - @Override - public Object convert(GenericObject obj) - { - return obj.asString(); - } - - @Override - public Object createRaw(byte[] b, int offset, int length) - { - try { - return new String(b, offset, length, "UTF-8"); // XXX FIXME debug - } catch (Exception e) { - // FIXME - throw new RuntimeException(e.getMessage()); - } - } -} - diff --git a/java-plan2/test/Generate.java b/java-plan2/test/Generate.java deleted file mode 100644 index 6b7800b..0000000 --- a/java-plan2/test/Generate.java +++ /dev/null @@ -1,18 +0,0 @@ -import java.io.*; -import java.util.*; -import org.msgpack.*; -import org.msgpack.schema.*; - -public class Generate { - public static void main(String[] args) throws IOException - { - Writer output = new OutputStreamWriter(System.out); - - Schema s1 = Schema.parse("(class Test (field uri raw) (field width int))"); - ClassGenerator.write(s1, output); - - Schema s2 = Schema.parse("(class MediaContent (package serializers.msgpack) (field image (array (class Image (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); - ClassGenerator.write(s2, output); - } -} - diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java deleted file mode 100644 index ecb64fd..0000000 --- a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java +++ /dev/null @@ -1,277 +0,0 @@ -package serializers.msgpack; - -import java.util.*; -import java.io.*; -import org.msgpack.*; -import org.msgpack.schema.*; - -public final class MediaContent implements MessagePackable, MessageConvertable, MessageMergeable -{ - private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); - public static ClassSchema getSchema() { return _SCHEMA; } - - public ArrayList image; - public Media media; - - public MediaContent() { } - - @Override - public void messagePack(Packer pk) throws IOException - { - List _f = _SCHEMA.getFields(); - pk.packArray(2); - _f.get(0).getType().pack(pk, image); - _f.get(1).getType().pack(pk, media); - } - - @Override - @SuppressWarnings("unchecked") - public void messageConvert(GenericObject obj) - { - List _l = obj.asArray(); - List _f = _SCHEMA.getFields(); - if(_l.size() <= 0) { return; } image = (ArrayList)_f.get(0).getType().convert(_l.get(0)); - if(_l.size() <= 1) { return; } media = (Media)_f.get(1).getType().convert(_l.get(1)); - } - - public static MediaContent convert(GenericObject obj) - { - return (MediaContent)_SCHEMA.convert(obj); - } - - public void setField(int index, Object value) - { - switch(index) { - case 0: - image = (ArrayList)value; - break; - case 1: - media = (Media)value; - break; - } - } - - public Object getField(int index) - { - switch(index) { - case 0: - return image; - case 1: - return media; - } - return null; - } -} - -final class Image implements MessagePackable, MessageConvertable, MessageMergeable -{ - private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int))"); - public static ClassSchema getSchema() { return _SCHEMA; } - - public String uri; - public String title; - public Integer width; - public Integer height; - public Integer size; - - public Image() { } - - @Override - public void messagePack(Packer pk) throws IOException - { - List _f = _SCHEMA.getFields(); - pk.packArray(5); - _f.get(0).getType().pack(pk, uri); - _f.get(1).getType().pack(pk, title); - _f.get(2).getType().pack(pk, width); - _f.get(3).getType().pack(pk, height); - _f.get(4).getType().pack(pk, size); - } - - @Override - @SuppressWarnings("unchecked") - public void messageConvert(GenericObject obj) - { - List _l = obj.asArray(); - List _f = _SCHEMA.getFields(); - if(_l.size() <= 0) { return; } uri = (String)_f.get(0).getType().convert(_l.get(0)); - if(_l.size() <= 1) { return; } title = (String)_f.get(1).getType().convert(_l.get(1)); - if(_l.size() <= 2) { return; } width = (Integer)_f.get(2).getType().convert(_l.get(2)); - if(_l.size() <= 3) { return; } height = (Integer)_f.get(3).getType().convert(_l.get(3)); - if(_l.size() <= 4) { return; } size = (Integer)_f.get(4).getType().convert(_l.get(4)); - } - - public static Image convert(GenericObject obj) - { - return (Image)_SCHEMA.convert(obj); - } - - public void setField(int index, Object value) - { - switch(index) { - case 0: - uri = (String)value; - break; - case 1: - title = (String)value; - break; - case 2: - width = (Integer)value; - break; - case 3: - height = (Integer)value; - break; - case 4: - size = (Integer)value; - break; - } - } - - public Object getField(int index) - { - switch(index) { - case 0: - return uri; - case 1: - return title; - case 2: - return width; - case 3: - return height; - case 4: - return size; - } - return null; - } -} - -final class Media implements MessagePackable, MessageConvertable, MessageMergeable -{ - private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))"); - public static ClassSchema getSchema() { return _SCHEMA; } - - public String uri; - public String title; - public Integer width; - public Integer height; - public String format; - public Long duration; - public Long size; - public Integer bitrate; - public ArrayList person; - public Integer player; - public String copyright; - - public Media() { } - - @Override - public void messagePack(Packer pk) throws IOException - { - List _f = _SCHEMA.getFields(); - pk.packArray(11); - _f.get(0).getType().pack(pk, uri); - _f.get(1).getType().pack(pk, title); - _f.get(2).getType().pack(pk, width); - _f.get(3).getType().pack(pk, height); - _f.get(4).getType().pack(pk, format); - _f.get(5).getType().pack(pk, duration); - _f.get(6).getType().pack(pk, size); - _f.get(7).getType().pack(pk, bitrate); - _f.get(8).getType().pack(pk, person); - _f.get(9).getType().pack(pk, player); - _f.get(10).getType().pack(pk, copyright); - } - - @Override - @SuppressWarnings("unchecked") - public void messageConvert(GenericObject obj) - { - List _l = obj.asArray(); - List _f = _SCHEMA.getFields(); - if(_l.size() <= 0) { return; } uri = (String)_f.get(0).getType().convert(_l.get(0)); - if(_l.size() <= 1) { return; } title = (String)_f.get(1).getType().convert(_l.get(1)); - if(_l.size() <= 2) { return; } width = (Integer)_f.get(2).getType().convert(_l.get(2)); - if(_l.size() <= 3) { return; } height = (Integer)_f.get(3).getType().convert(_l.get(3)); - if(_l.size() <= 4) { return; } format = (String)_f.get(4).getType().convert(_l.get(4)); - if(_l.size() <= 5) { return; } duration = (Long)_f.get(5).getType().convert(_l.get(5)); - if(_l.size() <= 6) { return; } size = (Long)_f.get(6).getType().convert(_l.get(6)); - if(_l.size() <= 7) { return; } bitrate = (Integer)_f.get(7).getType().convert(_l.get(7)); - if(_l.size() <= 8) { return; } person = (ArrayList)_f.get(8).getType().convert(_l.get(8)); - if(_l.size() <= 9) { return; } player = (Integer)_f.get(9).getType().convert(_l.get(9)); - if(_l.size() <= 10) { return; } copyright = (String)_f.get(10).getType().convert(_l.get(10)); - } - - public static Media convert(GenericObject obj) - { - return (Media)_SCHEMA.convert(obj); - } - - public void setField(int index, Object value) - { - switch(index) { - case 0: - uri = (String)value; - break; - case 1: - title = (String)value; - break; - case 2: - width = (Integer)value; - break; - case 3: - height = (Integer)value; - break; - case 4: - format = (String)value; - break; - case 5: - duration = (Long)value; - break; - case 6: - size = (Long)value; - break; - case 7: - bitrate = (Integer)value; - break; - case 8: - person = (ArrayList)value; - break; - case 9: - player = (Integer)value; - break; - case 10: - copyright = (String)value; - break; - } - } - - public Object getField(int index) - { - switch(index) { - case 0: - return uri; - case 1: - return title; - case 2: - return width; - case 3: - return height; - case 4: - return format; - case 5: - return duration; - case 6: - return size; - case 7: - return bitrate; - case 8: - return person; - case 9: - return player; - case 10: - return copyright; - } - return null; - } - -} diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java b/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java deleted file mode 100644 index acb5580..0000000 --- a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSerializer.java +++ /dev/null @@ -1,70 +0,0 @@ -package serializers.msgpack; - -import java.io.*; -import java.util.*; -import java.nio.charset.Charset; - -import org.msgpack.*; -import org.msgpack.schema.*; -import serializers.msgpack.*; - -import serializers.ObjectSerializer; - -public class MessagePackSerializer implements ObjectSerializer -{ - public String getName() { - return "msgpack-specific"; - } - - public MediaContent create() throws Exception { - Media media = new Media(); - media.uri = "http://javaone.com/keynote.mpg"; - media.format = "video/mpg4"; - media.title = "Javaone Keynote"; - media.duration = 1234567L; - media.bitrate = 0; - media.person = new ArrayList(2); - media.person.add("Bill Gates"); - media.person.add("Steve Jobs"); - media.player = 0; - media.height = 0; - media.width = 0; - media.size = 123L; - media.copyright = ""; - - Image image1 = new Image(); - image1.uri = "http://javaone.com/keynote_large.jpg"; - image1.width = 0; - image1.height = 0; - image1.size = 2; - image1.title = "Javaone Keynote"; - - Image image2 = new Image(); - image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; - image2.width = 0; - image2.height = 0; - image2.size = 1; - image2.title = "Javaone Keynote"; - - MediaContent content = new MediaContent(); - content.media = media; - content.image = new ArrayList(2); - content.image.add(image1); - content.image.add(image2); - return content; - } - - public MediaContent deserialize(byte[] array) throws Exception { - UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MediaContent.getSchema()); - pac.execute(array); - return (MediaContent)pac.getData(); - } - - public byte[] serialize(MediaContent content) throws Exception { - ByteArrayOutputStream os = new ByteArrayOutputStream(); - Packer pk = new Packer(os); - pk.pack(content); - return os.toByteArray(); - } -} - diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs b/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs deleted file mode 100644 index 547ba48..0000000 --- a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs +++ /dev/null @@ -1,21 +0,0 @@ -(class MediaContent - (package serializers.msgpack) - (field image (array (class Image - (field uri string) - (field title string) - (field width int) - (field height int) - (field size int)))) - (field media (class Media - (field uri string) - (field title string) - (field width int) - (field height int) - (field format string) - (field duration long) - (field size long) - (field bitrate int) - (field person (array string)) - (field player int) - (field copyright string))) - ) diff --git a/java-plan3/build.xml b/java/build.xml similarity index 78% rename from java-plan3/build.xml rename to java/build.xml index 598a853..40d7d61 100644 --- a/java-plan3/build.xml +++ b/java/build.xml @@ -12,4 +12,7 @@ + + + diff --git a/java-plan3/src/org/msgpack/MessageMergeable.java b/java/src/org/msgpack/MessageMergeable.java similarity index 94% rename from java-plan3/src/org/msgpack/MessageMergeable.java rename to java/src/org/msgpack/MessageMergeable.java index e11119c..e5a5b45 100644 --- a/java-plan3/src/org/msgpack/MessageMergeable.java +++ b/java/src/org/msgpack/MessageMergeable.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/MessagePackable.java b/java/src/org/msgpack/MessagePackable.java similarity index 94% rename from java-plan3/src/org/msgpack/MessagePackable.java rename to java/src/org/msgpack/MessagePackable.java index d8a7db9..9e9852a 100644 --- a/java-plan3/src/org/msgpack/MessagePackable.java +++ b/java/src/org/msgpack/MessagePackable.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/MessageTypeException.java b/java/src/org/msgpack/MessageTypeException.java similarity index 96% rename from java-plan3/src/org/msgpack/MessageTypeException.java rename to java/src/org/msgpack/MessageTypeException.java index 09031b2..feb6c08 100644 --- a/java-plan3/src/org/msgpack/MessageTypeException.java +++ b/java/src/org/msgpack/MessageTypeException.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/Packer.java b/java/src/org/msgpack/Packer.java similarity index 99% rename from java-plan3/src/org/msgpack/Packer.java rename to java/src/org/msgpack/Packer.java index 7f2508c..ebd8402 100644 --- a/java-plan3/src/org/msgpack/Packer.java +++ b/java/src/org/msgpack/Packer.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/Schema.java b/java/src/org/msgpack/Schema.java similarity index 98% rename from java-plan3/src/org/msgpack/Schema.java rename to java/src/org/msgpack/Schema.java index f99b3d0..f191f7a 100644 --- a/java-plan3/src/org/msgpack/Schema.java +++ b/java/src/org/msgpack/Schema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/UnbufferedUnpacker.java b/java/src/org/msgpack/UnbufferedUnpacker.java similarity index 97% rename from java-plan3/src/org/msgpack/UnbufferedUnpacker.java rename to java/src/org/msgpack/UnbufferedUnpacker.java index 471605f..b427973 100644 --- a/java-plan3/src/org/msgpack/UnbufferedUnpacker.java +++ b/java/src/org/msgpack/UnbufferedUnpacker.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/UnpackException.java b/java/src/org/msgpack/UnpackException.java similarity index 94% rename from java-plan3/src/org/msgpack/UnpackException.java rename to java/src/org/msgpack/UnpackException.java index db08d95..35e3e44 100644 --- a/java-plan3/src/org/msgpack/UnpackException.java +++ b/java/src/org/msgpack/UnpackException.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/UnpackIterator.java b/java/src/org/msgpack/UnpackIterator.java similarity index 96% rename from java-plan3/src/org/msgpack/UnpackIterator.java rename to java/src/org/msgpack/UnpackIterator.java index 9975b68..0a78e83 100644 --- a/java-plan3/src/org/msgpack/UnpackIterator.java +++ b/java/src/org/msgpack/UnpackIterator.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/Unpacker.java b/java/src/org/msgpack/Unpacker.java similarity index 99% rename from java-plan3/src/org/msgpack/Unpacker.java rename to java/src/org/msgpack/Unpacker.java index af211c6..458ffee 100644 --- a/java-plan3/src/org/msgpack/Unpacker.java +++ b/java/src/org/msgpack/Unpacker.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/impl/UnpackerImpl.java b/java/src/org/msgpack/impl/UnpackerImpl.java similarity index 95% rename from java-plan3/src/org/msgpack/impl/UnpackerImpl.java rename to java/src/org/msgpack/impl/UnpackerImpl.java index 47a1800..ddf9940 100644 --- a/java-plan3/src/org/msgpack/impl/UnpackerImpl.java +++ b/java/src/org/msgpack/impl/UnpackerImpl.java @@ -1,3 +1,20 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// package org.msgpack.impl; import java.nio.ByteBuffer; diff --git a/java-plan3/src/org/msgpack/schema/ArraySchema.java b/java/src/org/msgpack/schema/ArraySchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/ArraySchema.java rename to java/src/org/msgpack/schema/ArraySchema.java index 24fa758..fd47143 100644 --- a/java-plan3/src/org/msgpack/schema/ArraySchema.java +++ b/java/src/org/msgpack/schema/ArraySchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/ByteSchema.java b/java/src/org/msgpack/schema/ByteSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/ByteSchema.java rename to java/src/org/msgpack/schema/ByteSchema.java index 3bc7045..9ee6a82 100644 --- a/java-plan3/src/org/msgpack/schema/ByteSchema.java +++ b/java/src/org/msgpack/schema/ByteSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/ClassGenerator.java b/java/src/org/msgpack/schema/ClassGenerator.java similarity index 99% rename from java-plan3/src/org/msgpack/schema/ClassGenerator.java rename to java/src/org/msgpack/schema/ClassGenerator.java index 65213aa..061dcbb 100644 --- a/java-plan3/src/org/msgpack/schema/ClassGenerator.java +++ b/java/src/org/msgpack/schema/ClassGenerator.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/ClassSchema.java b/java/src/org/msgpack/schema/ClassSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/ClassSchema.java rename to java/src/org/msgpack/schema/ClassSchema.java index 75315e7..cd5c008 100644 --- a/java-plan3/src/org/msgpack/schema/ClassSchema.java +++ b/java/src/org/msgpack/schema/ClassSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/DoubleSchema.java b/java/src/org/msgpack/schema/DoubleSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/DoubleSchema.java rename to java/src/org/msgpack/schema/DoubleSchema.java index feffbb9..d53e47d 100644 --- a/java-plan3/src/org/msgpack/schema/DoubleSchema.java +++ b/java/src/org/msgpack/schema/DoubleSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/FieldSchema.java b/java/src/org/msgpack/schema/FieldSchema.java similarity index 95% rename from java-plan3/src/org/msgpack/schema/FieldSchema.java rename to java/src/org/msgpack/schema/FieldSchema.java index 3391f2b..66c2ff2 100644 --- a/java-plan3/src/org/msgpack/schema/FieldSchema.java +++ b/java/src/org/msgpack/schema/FieldSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/FloatSchema.java b/java/src/org/msgpack/schema/FloatSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/FloatSchema.java rename to java/src/org/msgpack/schema/FloatSchema.java index 2f4240a..2777521 100644 --- a/java-plan3/src/org/msgpack/schema/FloatSchema.java +++ b/java/src/org/msgpack/schema/FloatSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/GenericClassSchema.java b/java/src/org/msgpack/schema/GenericClassSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/GenericClassSchema.java rename to java/src/org/msgpack/schema/GenericClassSchema.java index 736bdfa..ffdd4ab 100644 --- a/java-plan3/src/org/msgpack/schema/GenericClassSchema.java +++ b/java/src/org/msgpack/schema/GenericClassSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/GenericSchema.java b/java/src/org/msgpack/schema/GenericSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/GenericSchema.java rename to java/src/org/msgpack/schema/GenericSchema.java index 52e0161..0adf898 100644 --- a/java-plan3/src/org/msgpack/schema/GenericSchema.java +++ b/java/src/org/msgpack/schema/GenericSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/IArraySchema.java b/java/src/org/msgpack/schema/IArraySchema.java similarity index 94% rename from java-plan3/src/org/msgpack/schema/IArraySchema.java rename to java/src/org/msgpack/schema/IArraySchema.java index ccc6d14..67e9f55 100644 --- a/java-plan3/src/org/msgpack/schema/IArraySchema.java +++ b/java/src/org/msgpack/schema/IArraySchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/IMapSchema.java b/java/src/org/msgpack/schema/IMapSchema.java similarity index 94% rename from java-plan3/src/org/msgpack/schema/IMapSchema.java rename to java/src/org/msgpack/schema/IMapSchema.java index 60b3e8e..3a2f556 100644 --- a/java-plan3/src/org/msgpack/schema/IMapSchema.java +++ b/java/src/org/msgpack/schema/IMapSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/IntSchema.java b/java/src/org/msgpack/schema/IntSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/IntSchema.java rename to java/src/org/msgpack/schema/IntSchema.java index c54c0ae..5a7e281 100644 --- a/java-plan3/src/org/msgpack/schema/IntSchema.java +++ b/java/src/org/msgpack/schema/IntSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/LongSchema.java b/java/src/org/msgpack/schema/LongSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/LongSchema.java rename to java/src/org/msgpack/schema/LongSchema.java index ccf3043..83a30e3 100644 --- a/java-plan3/src/org/msgpack/schema/LongSchema.java +++ b/java/src/org/msgpack/schema/LongSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/MapSchema.java b/java/src/org/msgpack/schema/MapSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/MapSchema.java rename to java/src/org/msgpack/schema/MapSchema.java index 71629b0..ba75993 100644 --- a/java-plan3/src/org/msgpack/schema/MapSchema.java +++ b/java/src/org/msgpack/schema/MapSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -88,6 +88,7 @@ public class MapSchema extends Schema implements IMapSchema { } @Override + @SuppressWarnings("unchecked") public Object createFromMap(Object[] obj) { HashMap m = new HashMap(obj.length / 2); int i = 0; diff --git a/java-plan3/src/org/msgpack/schema/RawSchema.java b/java/src/org/msgpack/schema/RawSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/RawSchema.java rename to java/src/org/msgpack/schema/RawSchema.java index 582f766..f621e4c 100644 --- a/java-plan3/src/org/msgpack/schema/RawSchema.java +++ b/java/src/org/msgpack/schema/RawSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java b/java/src/org/msgpack/schema/ReflectionClassSchema.java similarity index 100% rename from java-plan3/src/org/msgpack/schema/ReflectionClassSchema.java rename to java/src/org/msgpack/schema/ReflectionClassSchema.java diff --git a/java-plan3/src/org/msgpack/schema/SSchemaParser.java b/java/src/org/msgpack/schema/SSchemaParser.java similarity index 99% rename from java-plan3/src/org/msgpack/schema/SSchemaParser.java rename to java/src/org/msgpack/schema/SSchemaParser.java index c6bbc77..4ae8a4b 100644 --- a/java-plan3/src/org/msgpack/schema/SSchemaParser.java +++ b/java/src/org/msgpack/schema/SSchemaParser.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/ShortSchema.java b/java/src/org/msgpack/schema/ShortSchema.java similarity index 97% rename from java-plan3/src/org/msgpack/schema/ShortSchema.java rename to java/src/org/msgpack/schema/ShortSchema.java index 089a024..f32ab41 100644 --- a/java-plan3/src/org/msgpack/schema/ShortSchema.java +++ b/java/src/org/msgpack/schema/ShortSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java b/java/src/org/msgpack/schema/SpecificClassSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/SpecificClassSchema.java rename to java/src/org/msgpack/schema/SpecificClassSchema.java index 81e5e00..30bd9e1 100644 --- a/java-plan3/src/org/msgpack/schema/SpecificClassSchema.java +++ b/java/src/org/msgpack/schema/SpecificClassSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/src/org/msgpack/schema/StringSchema.java b/java/src/org/msgpack/schema/StringSchema.java similarity index 98% rename from java-plan3/src/org/msgpack/schema/StringSchema.java rename to java/src/org/msgpack/schema/StringSchema.java index f5e0bf1..46d515b 100644 --- a/java-plan3/src/org/msgpack/schema/StringSchema.java +++ b/java/src/org/msgpack/schema/StringSchema.java @@ -1,7 +1,7 @@ // // MessagePack for Java // -// Copyright (C) 2009 FURUHASHI Sadayuki +// Copyright (C) 2009-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/java-plan3/test/Generate.java b/java/test/Generate.java similarity index 100% rename from java-plan3/test/Generate.java rename to java/test/Generate.java diff --git a/java/test/README b/java/test/README new file mode 100644 index 0000000..4e16454 --- /dev/null +++ b/java/test/README @@ -0,0 +1,7 @@ +#!/bin/sh +svn checkout -r114 http://thrift-protobuf-compare.googlecode.com/svn/trunk/ thrift-protobuf-compare-base +cp -rf thrift-protobuf-compare/tpc thrift-protobuf-compare-base +cp ../dist/msgpack.jar thrift-protobuf-compare-base/tpc/lib/ +cd thrift-protobuf-compare-base/tpc/ +ant compile +./run-benchmark.sh diff --git a/java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java new file mode 100644 index 0000000..b17dfb2 --- /dev/null +++ b/java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java @@ -0,0 +1,434 @@ +package serializers; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; + +import serializers.msgpack.MessagePackSpecificSerializer; +import serializers.msgpack.MessagePackIndirectSerializer; +import serializers.msgpack.MessagePackDynamicSerializer; +import serializers.msgpack.MessagePackGenericSerializer; +import serializers.avro.AvroGenericSerializer; +import serializers.avro.specific.AvroSpecificSerializer; +import serializers.kryo.KryoOptimizedSerializer; +import serializers.kryo.KryoSerializer; + +public class BenchmarkRunner +{ + public final static int ITERATIONS = 2000; + public final static int TRIALS = 20; + + /** + * Number of milliseconds to warm up for each operation type for each serializer. Let's + * start with 3 seconds. + */ + final static long WARMUP_MSECS = 3000; + + @SuppressWarnings("unchecked") + private Set _serializers = new LinkedHashSet(); + + public static void main(String... args) throws Exception + { + BenchmarkRunner runner = new BenchmarkRunner(); + + // binary codecs first + runner.addObjectSerializer(new MessagePackSpecificSerializer()); + runner.addObjectSerializer(new MessagePackIndirectSerializer()); + runner.addObjectSerializer(new MessagePackDynamicSerializer()); + runner.addObjectSerializer(new MessagePackGenericSerializer()); + runner.addObjectSerializer(new AvroGenericSerializer()); + runner.addObjectSerializer(new AvroSpecificSerializer()); + runner.addObjectSerializer(new ActiveMQProtobufSerializer()); + runner.addObjectSerializer(new ProtobufSerializer()); + runner.addObjectSerializer(new ThriftSerializer()); + runner.addObjectSerializer(new HessianSerializer()); + runner.addObjectSerializer(new KryoSerializer()); + runner.addObjectSerializer(new KryoOptimizedSerializer()); + + // None of the other serializers use compression, so we'll leave this out. + // runner.addObjectSerializer(new KryoCompressedSerializer()); + + // then language default serializers + runner.addObjectSerializer(new JavaSerializer()); + + runner.addObjectSerializer(new JavaExtSerializer()); + runner.addObjectSerializer(new ScalaSerializer()); + + // then Json + runner.addObjectSerializer(new JsonSerializer()); + runner.addObjectSerializer(new JsonDataBindingSerializer()); + runner.addObjectSerializer(new JsonMarshallerSerializer()); + runner.addObjectSerializer(new ProtostuffJsonSerializer()); + runner.addObjectSerializer(new ProtostuffNumericJsonSerializer()); + // this is pretty slow; so slow that it's almost not worth keeping but: + runner.addObjectSerializer(new GsonSerializer()); + + // then xml via stax, textual and binary + runner.addObjectSerializer(new StaxSerializer("stax/woodstox", + new com.ctc.wstx.stax.WstxInputFactory(), + new com.ctc.wstx.stax.WstxOutputFactory())); + runner.addObjectSerializer(new StaxSerializer("stax/aalto", + new com.fasterxml.aalto.stax.InputFactoryImpl(), + new com.fasterxml.aalto.stax.OutputFactoryImpl())); + + runner.addObjectSerializer(new StaxSerializer("binaryxml/FI", + new com.sun.xml.fastinfoset.stax.factory.StAXInputFactory(), + new com.sun.xml.fastinfoset.stax.factory.StAXOutputFactory())); + + // No point in running all 4 variants: let's just use fastest one: + //runner.addObjectSerializer(new XStreamSerializer("xstream (xpp)", false, null, null)); + //runner.addObjectSerializer(new XStreamSerializer("xstream (xpp with conv)", true, null, null)); + //runner.addObjectSerializer(new XStreamSerializer("xstream (stax)", false, new com.ctc.wstx.stax.WstxInputFactory(), new com.ctc.wstx.stax.WstxOutputFactory())); + runner.addObjectSerializer(new XStreamSerializer("xstream (stax with conv)", + true, + new com.ctc.wstx.stax.WstxInputFactory(), + new com.ctc.wstx.stax.WstxOutputFactory())); + runner.addObjectSerializer(new JavolutionXMLFormatSerializer()); + + runner.addObjectSerializer(new SbinarySerializer()); + // broken? Does not correctly round-trip: + // runner.addObjectSerializer(new YamlSerializer()); + + System.out.println("Starting"); + + runner.start(); + } + + @SuppressWarnings("unchecked") + private void addObjectSerializer(ObjectSerializer serializer) + { + _serializers.add(serializer); + } + + private double createObjects(ObjectSerializer serializer, int iterations) throws Exception + { + long start = System.nanoTime(); + for (int i = 0; i < iterations; i++) + { + serializer.create(); + } + return iterationTime(System.nanoTime() - start, iterations); + } + + private double iterationTime(long delta, int iterations) + { + return (double) delta / (double) (iterations); + } + + private double serializeDifferentObjects(ObjectSerializer serializer, int iterations) throws Exception + { + long start = System.nanoTime(); + for (int i = 0; i < iterations; i++) + { + T obj = serializer.create(); + serializer.serialize(obj); + } + return iterationTime(System.nanoTime()-start, iterations); + } + + + private double serializeSameObject(ObjectSerializer serializer, int iterations) throws Exception + { + // let's reuse same instance to reduce overhead + T obj = serializer.create(); + long delta = 0; + for (int i = 0; i < iterations; i++) + { + long start = System.nanoTime(); + serializer.serialize(obj); + delta += System.nanoTime() - start; + if (i % 1000 == 0) + doGc(); + } + return iterationTime(delta, iterations); + } + + private double deserializeNoFieldAccess(ObjectSerializer serializer, int iterations) throws Exception + { + byte[] array = serializer.serialize(serializer.create()); + long start = System.nanoTime(); + T result = null; + for (int i = 0; i < iterations; i++) + { + result = serializer.deserialize(array); + } + return iterationTime(System.nanoTime()-start, iterations); + } + + private double deserializeAndCheckAllFields(CheckingObjectSerializer serializer, int iterations) throws Exception + { + byte[] array = serializer.serialize(serializer.create()); + long delta = 0; + for (int i = 0; i < iterations; i++) + { + long start = System.nanoTime(); + T obj = serializer.deserialize(array); + serializer.checkAllFields(obj); + delta += System.nanoTime() - start; + } + return iterationTime(delta, iterations); + } + + private double deserializeAndCheckMediaField(CheckingObjectSerializer serializer, int iterations) throws Exception + { + byte[] array = serializer.serialize(serializer.create()); + long delta = 0; + for (int i = 0; i < iterations; i++) + { + long start = System.nanoTime(); + T obj = serializer.deserialize(array); + serializer.checkMediaField(obj); + delta += System.nanoTime() - start; + } + return iterationTime(delta, iterations); + } + + /** + * JVM is not required to honor GC requests, but adding bit of sleep around request is + * most likely to give it a chance to do it. + */ + private void doGc() + { + try { + Thread.sleep(50L); + } catch (InterruptedException ie) { } + System.gc(); + try { // longer sleep afterwards (not needed by GC, but may help with scheduling) + Thread.sleep(200L); + } catch (InterruptedException ie) { } + } + + enum measurements + { + timeCreate, timeSerializeDifferentObjects, timeSerializeSameObject, timeDeserializeNoFieldAccess, timeDeserializeAndCheckMediaField, timeDeserializeAndCheckAllFields, totalTime, length + } + + @SuppressWarnings("unchecked") + private void start() throws Exception + { + System.out.printf("%-24s, %15s, %15s, %15s, %15s, %15s, %15s, %15s, %10s\n", + " ", + "Object create", + "Serialize", + "/w Same Object", + "Deserialize", + "and Check Media", + "and Check All", + "Total Time", + "Serialized Size"); + EnumMap> values = new EnumMap>(measurements.class); + for (measurements m : measurements.values()) + values.put(m, new HashMap()); + + for (ObjectSerializer serializer : _serializers) + { + /* + * Should only warm things for the serializer that we test next: HotSpot JIT will + * otherwise spent most of its time optimizing slower ones... Use + * -XX:CompileThreshold=1 to hint the JIT to start immediately + * + * Actually: 1 is often not a good value -- threshold is the number + * of samples needed to trigger inlining, and there's no point in + * inlining everything. Default value is in thousands, so lowering + * it to, say, 1000 is usually better. + */ + warmCreation(serializer); + doGc(); + double timeCreate = Double.MAX_VALUE; + // do more iteration for object creation because of its short time + for (int i = 0; i < TRIALS; i++) + timeCreate = Math.min(timeCreate, createObjects(serializer, ITERATIONS * 100)); + + warmSerialization(serializer); + + // actually: let's verify serializer actually works now: + checkCorrectness(serializer); + + doGc(); + double timeSerializeDifferentObjects = Double.MAX_VALUE; + for (int i = 0; i < TRIALS; i++) + timeSerializeDifferentObjects = Math.min(timeSerializeDifferentObjects, serializeDifferentObjects(serializer, ITERATIONS)); + + doGc(); + double timeSerializeSameObject = Double.MAX_VALUE; + for (int i = 0; i < TRIALS; i++) + timeSerializeSameObject = Math.min(timeSerializeSameObject, serializeSameObject(serializer, ITERATIONS)); + + warmDeserialization(serializer); + + doGc(); + double timeDeserializeNoFieldAccess = Double.MAX_VALUE; + for (int i = 0; i < TRIALS; i++) + timeDeserializeNoFieldAccess = Math.min(timeDeserializeNoFieldAccess, deserializeNoFieldAccess(serializer, ITERATIONS)); + + double timeDeserializeAndCheckAllFields = Double.NaN; + double timeDeserializeAndCheckMediaField = Double.NaN; + + double totalTime = timeSerializeDifferentObjects + timeDeserializeNoFieldAccess; + + if( serializer instanceof CheckingObjectSerializer) { + CheckingObjectSerializer checkingSerializer = (CheckingObjectSerializer)serializer; + + timeDeserializeAndCheckMediaField = Double.MAX_VALUE; + doGc(); + for (int i = 0; i < TRIALS; i++) + timeDeserializeAndCheckMediaField = Math.min(timeDeserializeAndCheckMediaField, deserializeAndCheckMediaField(checkingSerializer, ITERATIONS)); + + timeDeserializeAndCheckAllFields = Double.MAX_VALUE; + doGc(); + for (int i = 0; i < TRIALS; i++) + timeDeserializeAndCheckAllFields = Math.min(timeDeserializeAndCheckAllFields, deserializeAndCheckAllFields(checkingSerializer, ITERATIONS)); + + totalTime = timeSerializeDifferentObjects + timeDeserializeAndCheckAllFields; + } + + + byte[] array = serializer.serialize(serializer.create()); + System.out.printf("%-24s, %15.5f, %15.5f, %15.5f, %15.5f, %15.5f, %15.5f, %15.5f, %10d\n", + serializer.getName(), + timeCreate, + timeSerializeDifferentObjects, + timeSerializeSameObject, + timeDeserializeNoFieldAccess, + timeDeserializeAndCheckMediaField, + timeDeserializeAndCheckAllFields, + totalTime, + array.length); + + addValue(values, serializer.getName(), timeCreate, timeSerializeDifferentObjects, timeSerializeSameObject, + timeDeserializeNoFieldAccess, timeDeserializeAndCheckMediaField, timeDeserializeAndCheckAllFields, totalTime, array.length); + } + printImages(values); + } + + /** + * Method that tries to validate correctness of serializer, using + * round-trip (construct, serializer, deserialize; compare objects + * after steps 1 and 3). + * Currently only done for StdMediaDeserializer... + */ + private void checkCorrectness(ObjectSerializer serializer) + throws Exception + { + Object input = serializer.create(); + byte[] array = serializer.serialize(input); + Object output = serializer.deserialize(array); + + if (!input.equals(output)) { + /* Should throw an exception; but for now (that we have a few + * failures) let's just whine... + */ + String msg = "serializer '"+serializer.getName()+"' failed round-trip test (ser+deser produces Object different from input), input="+input+", output="+output; + //throw new Exception("Error: "+msg); + System.err.println("WARN: "+msg); + } + } + + private void printImages(EnumMap> values) + { + for (measurements m : values.keySet()) { + Map map = values.get(m); + ArrayList list = new ArrayList(map.entrySet()); + Collections.sort(list, new Comparator() { + public int compare (Entry o1, Entry o2) { + double diff = (Double)o1.getValue() - (Double)o2.getValue(); + return diff > 0 ? 1 : (diff < 0 ? -1 : 0); + } + }); + LinkedHashMap sortedMap = new LinkedHashMap(); + for (Entry entry : list) { + if( !entry.getValue().isNaN() ) { + sortedMap.put(entry.getKey(), entry.getValue()); + } + } + printImage(sortedMap, m); + } + } + + private void printImage(Map map, measurements m) + { + StringBuilder valSb = new StringBuilder(); + String names = ""; + double max = Double.MIN_NORMAL; + for (Entry entry : map.entrySet()) + { + valSb.append(entry.getValue()).append(','); + max = Math.max(max, entry.getValue()); + names = entry.getKey() + '|' + names; + } + + int height = Math.min(30+map.size()*20, 430); + double scale = max * 1.1; + System.out.println(""); + + } + + private void addValue(EnumMap> values, + String name, + double timeCreate, + double timeSerializeDifferentObjects, + double timeSerializeSameObject, + double timeDeserializeNoFieldAccess, + double timeDeserializeAndCheckMediaField, + double timeDeserializeAndCheckAllFields, + double totalTime, + double length) + { + + values.get(measurements.timeCreate).put(name, timeCreate); + values.get(measurements.timeSerializeDifferentObjects).put(name, timeSerializeDifferentObjects); + values.get(measurements.timeSerializeSameObject).put(name, timeSerializeSameObject); + values.get(measurements.timeDeserializeNoFieldAccess).put(name, timeDeserializeNoFieldAccess); + values.get(measurements.timeDeserializeAndCheckMediaField).put(name, timeDeserializeAndCheckMediaField); + values.get(measurements.timeDeserializeAndCheckAllFields).put(name, timeDeserializeAndCheckAllFields); + values.get(measurements.totalTime).put(name, totalTime); + values.get(measurements.length).put(name, length); + } + + private void warmCreation(ObjectSerializer serializer) throws Exception + { + // Instead of fixed counts, let's try to prime by running for N seconds + long endTime = System.currentTimeMillis() + WARMUP_MSECS; + do + { + createObjects(serializer, 1); + } + while (System.currentTimeMillis() < endTime); + } + + private void warmSerialization(ObjectSerializer serializer) throws Exception + { + // Instead of fixed counts, let's try to prime by running for N seconds + long endTime = System.currentTimeMillis() + WARMUP_MSECS; + do + { + serializeDifferentObjects(serializer, 1); + } + while (System.currentTimeMillis() < endTime); + } + + private void warmDeserialization(ObjectSerializer serializer) throws Exception + { + // Instead of fixed counts, let's try to prime by running for N seconds + long endTime = System.currentTimeMillis() + WARMUP_MSECS; + do + { + deserializeNoFieldAccess(serializer, 1); + } + while (System.currentTimeMillis() < endTime); + } +} diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java similarity index 100% rename from java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java diff --git a/java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs similarity index 100% rename from java-plan2/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.mpacs diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java similarity index 100% rename from java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java similarity index 100% rename from java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java similarity index 100% rename from java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java diff --git a/java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java similarity index 100% rename from java-plan3/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java rename to java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackSpecificSerializer.java From 55cfbf378e5fcf91d0dffe73f9cab3f3f7414233 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 14 Jan 2010 21:20:32 +0900 Subject: [PATCH 0346/1648] cpp: add msgpack/type/tr1/unordered_{map,set}.hpp --- configure.in | 2 +- cpp/Makefile.am | 4 +- cpp/type/set.hpp | 25 ++++++++++ cpp/type/tr1/unordered_map.hpp | 85 ++++++++++++++++++++++++++++++++++ cpp/type/tr1/unordered_set.hpp | 80 ++++++++++++++++++++++++++++++++ 5 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 cpp/type/tr1/unordered_map.hpp create mode 100644 cpp/type/tr1/unordered_set.hpp diff --git a/configure.in b/configure.in index 76bd7e4..32d906e 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.9) +AM_INIT_AUTOMAKE(msgpack, 0.3.10) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 42d6d2a..0923362 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -25,7 +25,9 @@ nobase_include_HEADERS = \ msgpack/type/string.hpp \ msgpack/type/vector.hpp \ msgpack/type/tuple.hpp \ - msgpack/type/define.hpp + msgpack/type/define.hpp \ + msgpack/type/tr1/unordered_map.hpp \ + msgpack/type/tr1/unordered_set.hpp libmsgpack_la_LIBADD = -L../c -lmsgpackc diff --git a/cpp/type/set.hpp b/cpp/type/set.hpp index 11db2b3..f2c5bfb 100644 --- a/cpp/type/set.hpp +++ b/cpp/type/set.hpp @@ -49,6 +49,31 @@ inline packer& operator<< (packer& o, const std::set& v) } +template +inline std::multiset& operator>> (object o, std::multiset& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + object* p = o.via.array.ptr + o.via.array.size; + object* const pbegin = o.via.array.ptr; + while(p > pbegin) { + --p; + v.insert(p->as()); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::multiset& v) +{ + o.pack_array(v.size()); + for(typename std::multiset::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + } // namespace msgpack #endif /* msgpack/type/set.hpp */ diff --git a/cpp/type/tr1/unordered_map.hpp b/cpp/type/tr1/unordered_map.hpp new file mode 100644 index 0000000..1996cfd --- /dev/null +++ b/cpp/type/tr1/unordered_map.hpp @@ -0,0 +1,85 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_TR1_UNORDERED_MAP_HPP__ +#define MSGPACK_TYPE_TR1_UNORDERED_MAP_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::tr1::unordered_map operator>> (object o, std::tr1::unordered_map& v) +{ + if(o.type != type::MAP) { throw type_error(); } + object_kv* p(o.via.map.ptr); + object_kv* const pend(o.via.map.ptr + o.via.map.size); + for(; p != pend; ++p) { + K key; + p->key.convert(&key); + p->val.convert(&v[key]); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::tr1::unordered_map& v) +{ + o.pack_map(v.size()); + for(typename std::tr1::unordered_map::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(it->first); + o.pack(it->second); + } + return o; +} + + +template +inline std::tr1::unordered_multimap operator>> (object o, std::tr1::unordered_multimap& v) +{ + if(o.type != type::MAP) { throw type_error(); } + object_kv* p(o.via.map.ptr); + object_kv* const pend(o.via.map.ptr + o.via.map.size); + for(; p != pend; ++p) { + std::pair value; + p->key.convert(&value.first); + p->val.convert(&value.second); + v.insert(value); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::tr1::unordered_multimap& v) +{ + o.pack_map(v.size()); + for(typename std::tr1::unordered_multimap::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(it->first); + o.pack(it->second); + } + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/map.hpp */ + diff --git a/cpp/type/tr1/unordered_set.hpp b/cpp/type/tr1/unordered_set.hpp new file mode 100644 index 0000000..eb127b5 --- /dev/null +++ b/cpp/type/tr1/unordered_set.hpp @@ -0,0 +1,80 @@ +// +// MessagePack for C++ static resolution routine +// +// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_TYPE_TR1_UNORDERED_SET_HPP__ +#define MSGPACK_TYPE_TR1_UNORDERED_SET_HPP__ + +#include "msgpack/object.hpp" +#include + +namespace msgpack { + + +template +inline std::tr1::unordered_set& operator>> (object o, std::tr1::unordered_set& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + object* p = o.via.array.ptr + o.via.array.size; + object* const pbegin = o.via.array.ptr; + while(p > pbegin) { + --p; + v.insert(p->as()); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::tr1::unordered_set& v) +{ + o.pack_array(v.size()); + for(typename std::tr1::unordered_set::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + +template +inline std::tr1::unordered_multiset& operator>> (object o, std::tr1::unordered_multiset& v) +{ + if(o.type != type::ARRAY) { throw type_error(); } + object* p = o.via.array.ptr + o.via.array.size; + object* const pbegin = o.via.array.ptr; + while(p > pbegin) { + --p; + v.insert(p->as()); + } + return v; +} + +template +inline packer& operator<< (packer& o, const std::tr1::unordered_multiset& v) +{ + o.pack_array(v.size()); + for(typename std::tr1::unordered_multiset::const_iterator it(v.begin()), it_end(v.end()); + it != it_end; ++it) { + o.pack(*it); + } + return o; +} + + +} // namespace msgpack + +#endif /* msgpack/type/set.hpp */ + From 8d365458d56cc3ac9ce74d8cc1470ede30465522 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 15 Jan 2010 05:27:44 +0900 Subject: [PATCH 0347/1648] c,cpp: optimize msgpack_zone: variable-length array of chunk -> list of chunk --- c/Makefile.am | 2 +- c/zone.c | 153 +++++++++++++++++++++-------------------------- c/zone.h | 35 +++++------ cpp/Makefile.am | 2 +- cpp/zone.hpp.erb | 5 +- 5 files changed, 88 insertions(+), 109 deletions(-) diff --git a/c/Makefile.am b/c/Makefile.am index 0a52446..23995a5 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -16,7 +16,7 @@ nobase_include_HEADERS = \ msgpack/zone.h # -version-info CURRENT:REVISION:AGE -libmsgpackc_la_LDFLAGS = -version-info 1:0:0 +libmsgpackc_la_LDFLAGS = -version-info 2:0:0 check_PROGRAMS = \ msgpackc_test diff --git a/c/zone.c b/c/zone.c index 79e90dc..2d8bc58 100644 --- a/c/zone.c +++ b/c/zone.c @@ -19,83 +19,61 @@ #include #include -static inline bool init_chunk_array(msgpack_zone_chunk_array* ca, size_t chunk_size) -{ - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ - const size_t nfirst = (sizeof(msgpack_zone_chunk) < 72/2) ? - 72 / sizeof(msgpack_zone_chunk) : 8; +struct msgpack_zone_chunk { + struct msgpack_zone_chunk* next; + /* data ... */ +}; - msgpack_zone_chunk* array = (msgpack_zone_chunk*)malloc( - sizeof(msgpack_zone_chunk) * nfirst); - if(array == NULL) { +static inline bool init_chunk_list(msgpack_zone_chunk_list* cl, size_t chunk_size) +{ + msgpack_zone_chunk* chunk = (msgpack_zone_chunk*)malloc( + sizeof(msgpack_zone_chunk) + chunk_size); + if(chunk == NULL) { return false; } - const size_t sz = chunk_size; - - char* ptr = (char*)malloc(sz); - if(ptr == NULL) { - free(array); - return NULL; - } - - ca->tail = array; - ca->end = array + nfirst; - ca->array = array; - - array[0].free = sz; - array[0].ptr = ptr; - array[0].alloc = ptr; + cl->head = chunk; + cl->free = chunk_size; + cl->ptr = ((char*)chunk) + sizeof(msgpack_zone_chunk); + chunk->next = NULL; return true; } -static inline void destroy_chunk_array(msgpack_zone_chunk_array* ca) +static inline void destroy_chunk_list(msgpack_zone_chunk_list* cl) { - msgpack_zone_chunk* chunk = ca->array; - for(; chunk != ca->tail+1; ++chunk) { - free(chunk->alloc); + msgpack_zone_chunk* c = cl->head; + while(true) { + msgpack_zone_chunk* n = c->next; + free(c); + if(n != NULL) { + c = n; + } else { + break; + } } - - free(ca->array); } -static inline void clear_chunk_array(msgpack_zone_chunk_array* ca) +static inline void clear_chunk_list(msgpack_zone_chunk_list* cl, size_t chunk_size) { - msgpack_zone_chunk* chunk = ca->array + 1; - for(; chunk != ca->tail+1; ++chunk) { - free(chunk->alloc); + msgpack_zone_chunk* c = cl->head; + while(true) { + msgpack_zone_chunk* n = c->next; + if(n != NULL) { + free(c); + c = n; + } else { + break; + } } - - ca->tail = ca->array; - - ca->array[0].free += ca->array[0].ptr - (char*)ca->array[0].alloc; - ca->array[0].ptr = (char*)ca->array[0].alloc; + cl->head->next = NULL; + cl->free = chunk_size; + cl->ptr = ((char*)cl->head) + sizeof(msgpack_zone_chunk); } void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) { - msgpack_zone_chunk_array* const ca = &zone->chunk_array; - - msgpack_zone_chunk* chunk = ++ca->tail; - - if(chunk == ca->end) { - // ca->arrayã«ç©ºããŒãªã„ - // ca->arrayã‚’æ‹¡å¼µã™ã‚‹ - - const size_t nused = ca->end - ca->array; - const size_t nnext = (ca->end - ca->array) * 2; - - chunk = (msgpack_zone_chunk*)realloc(ca->array, - sizeof(msgpack_zone_chunk) * nnext); - if(chunk == NULL) { - return NULL; - } - - ca->array = chunk; - ca->end = chunk + nnext; - chunk = ca->tail = chunk + nused; - } + msgpack_zone_chunk_list* const cl = &zone->chunk_list; size_t sz = zone->chunk_size; @@ -103,14 +81,15 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size) sz *= 2; } - char* ptr = (char*)malloc(sz); - if(ptr == NULL) { - return NULL; - } + msgpack_zone_chunk* chunk = (msgpack_zone_chunk*)malloc( + sizeof(msgpack_zone_chunk) + sz); - chunk->free = sz - size; - chunk->ptr = ptr + size; - chunk->alloc = ptr; + char* ptr = ((char*)chunk) + sizeof(msgpack_zone_chunk); + + chunk->next = cl->head; + cl->head = chunk; + cl->free = sz - size; + cl->ptr = ptr + size; return ptr; } @@ -185,19 +164,30 @@ bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, bool msgpack_zone_is_empty(msgpack_zone* zone) { - msgpack_zone_chunk_array* const ca = &zone->chunk_array; + msgpack_zone_chunk_list* const cl = &zone->chunk_list; msgpack_zone_finalizer_array* const fa = &zone->finalizer_array; - return ca->array[0].ptr == ca->array[0].alloc && - ca->tail == ca->array && + return cl->free == zone->chunk_size && cl->head->next == NULL && fa->tail == fa->array; } +void msgpack_zone_destroy(msgpack_zone* zone) +{ + destroy_finalizer_array(&zone->finalizer_array); + destroy_chunk_list(&zone->chunk_list); +} + +void msgpack_zone_clear(msgpack_zone* zone) +{ + clear_finalizer_array(&zone->finalizer_array); + clear_chunk_list(&zone->chunk_list, zone->chunk_size); +} + bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size) { zone->chunk_size = chunk_size; - if(!init_chunk_array(&zone->chunk_array, chunk_size)) { + if(!init_chunk_list(&zone->chunk_list, chunk_size)) { return false; } @@ -206,30 +196,23 @@ bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size) return true; } -void msgpack_zone_destroy(msgpack_zone* zone) -{ - destroy_finalizer_array(&zone->finalizer_array); - destroy_chunk_array(&zone->chunk_array); -} - -void msgpack_zone_clear(msgpack_zone* zone) -{ - clear_finalizer_array(&zone->finalizer_array); - clear_chunk_array(&zone->chunk_array); -} - msgpack_zone* msgpack_zone_new(size_t chunk_size) { - msgpack_zone* zone = (msgpack_zone*)malloc(sizeof(msgpack_zone)); + msgpack_zone* zone = (msgpack_zone*)malloc( + sizeof(msgpack_zone) + chunk_size); if(zone == NULL) { return NULL; } - if(!msgpack_zone_init(zone, chunk_size)) { + zone->chunk_size = chunk_size; + + if(!init_chunk_list(&zone->chunk_list, chunk_size)) { free(zone); - return NULL; + return false; } + init_finalizer_array(&zone->finalizer_array); + return zone; } diff --git a/c/zone.h b/c/zone.h index 79b51f8..e07e762 100644 --- a/c/zone.h +++ b/c/zone.h @@ -26,37 +26,34 @@ extern "C" { #endif -typedef struct msgpack_zone_chunk { - size_t free; - char* ptr; - void* alloc; -} msgpack_zone_chunk; - typedef struct msgpack_zone_finalizer { void (*func)(void* data); void* data; } msgpack_zone_finalizer; -typedef struct msgpack_zone_chunk_array { - msgpack_zone_chunk* tail; - msgpack_zone_chunk* end; - msgpack_zone_chunk* array; -} msgpack_zone_chunk_array; - typedef struct msgpack_zone_finalizer_array { msgpack_zone_finalizer* tail; msgpack_zone_finalizer* end; msgpack_zone_finalizer* array; } msgpack_zone_finalizer_array; +struct msgpack_zone_chunk; +typedef struct msgpack_zone_chunk msgpack_zone_chunk; + +typedef struct msgpack_zone_chunk_list { + size_t free; + char* ptr; + msgpack_zone_chunk* head; +} msgpack_zone_chunk_list; + typedef struct msgpack_zone { - msgpack_zone_chunk_array chunk_array; + msgpack_zone_chunk_list chunk_list; msgpack_zone_finalizer_array finalizer_array; size_t chunk_size; } msgpack_zone; #ifndef MSGPACK_ZONE_CHUNK_SIZE -#define MSGPACK_ZONE_CHUNK_SIZE 2048 +#define MSGPACK_ZONE_CHUNK_SIZE 8192 #endif bool msgpack_zone_init(msgpack_zone* zone, size_t chunk_size); @@ -85,15 +82,15 @@ void* msgpack_zone_malloc_expand(msgpack_zone* zone, size_t size); void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size) { - msgpack_zone_chunk* chunk = zone->chunk_array.tail; + msgpack_zone_chunk_list* cl = &zone->chunk_list; - if(chunk->free < size) { + if(zone->chunk_list.free < size) { return msgpack_zone_malloc_expand(zone, size); } - char* ptr = chunk->ptr; - chunk->ptr += size; - chunk->free -= size; + char* ptr = cl->ptr; + cl->free -= size; + cl->ptr += size; return ptr; } diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 0923362..99f4a32 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -32,7 +32,7 @@ nobase_include_HEADERS = \ libmsgpack_la_LIBADD = -L../c -lmsgpackc # -version-info CURRENT:REVISION:AGE -libmsgpack_la_LDFLAGS = -version-info 1:0:0 +libmsgpack_la_LDFLAGS = -version-info 2:0:0 check_PROGRAMS = \ msgpack_test diff --git a/cpp/zone.hpp.erb b/cpp/zone.hpp.erb index f1e4624..48988ab 100644 --- a/cpp/zone.hpp.erb +++ b/cpp/zone.hpp.erb @@ -120,9 +120,8 @@ void zone::object_destructor(void* obj) inline void zone::undo_malloc(size_t size) { - msgpack_zone_chunk* chunk = base::chunk_array.tail; - chunk->ptr -= size; - chunk->free += size; + base::chunk_list.ptr -= size; + base::chunk_list.free += size; } <%0.upto(GENERATION_LIMIT) {|i|%> From f145129f6ec91a13fbec99d4dc5b76654a951f79 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 15 Jan 2010 05:29:41 +0900 Subject: [PATCH 0348/1648] c,cpp: optimize msgpack_vrefbuffer --- c/vrefbuffer.c | 112 +++++++++++++++++++++++++++++++++---------------- c/vrefbuffer.h | 21 ++++++---- 2 files changed, 88 insertions(+), 45 deletions(-) diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c index 2bf97af..b970ece 100644 --- a/c/vrefbuffer.c +++ b/c/vrefbuffer.c @@ -19,15 +19,14 @@ #include #include +struct msgpack_vrefbuffer_chunk { + struct msgpack_vrefbuffer_chunk* next; + /* data ... */ +}; + bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, size_t ref_size, size_t chunk_size) { - if(chunk_size < sizeof(msgpack_vrefbuffer_chunk)+72) { - chunk_size = 72; - } else { - chunk_size -= sizeof(msgpack_vrefbuffer_chunk); - } - vbuf->chunk_size = chunk_size; vbuf->ref_size = ref_size; @@ -45,26 +44,30 @@ bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, vbuf->end = array + nfirst; vbuf->array = array; - vbuf->chunk = (msgpack_vrefbuffer_chunk*)malloc( - chunk_size + sizeof(msgpack_vrefbuffer_chunk)); - if(vbuf->chunk == NULL) { + msgpack_vrefbuffer_chunk* chunk = (msgpack_vrefbuffer_chunk*)malloc( + sizeof(msgpack_vrefbuffer_chunk) + chunk_size); + if(chunk == NULL) { free(array); return false; } - vbuf->chunk->next = NULL; - vbuf->chunk->free = chunk_size; + msgpack_vrefbuffer_inner_buffer* const ib = &vbuf->inner_buffer; + + ib->free = chunk_size; + ib->ptr = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk); + ib->head = chunk; + chunk->next = NULL; return true; } void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf) { - msgpack_vrefbuffer_chunk* c = vbuf->chunk; + msgpack_vrefbuffer_chunk* c = vbuf->inner_buffer.head; while(true) { msgpack_vrefbuffer_chunk* n = c->next; free(c); - if(n) { + if(n != NULL) { c = n; } else { break; @@ -101,28 +104,30 @@ int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, const char* buf, unsigned int len) { - msgpack_vrefbuffer_chunk* chunk = vbuf->chunk; - size_t cur_size = vbuf->chunk_size; + msgpack_vrefbuffer_inner_buffer* const ib = &vbuf->inner_buffer; - if(chunk->free < len) { - cur_size = (cur_size > len) ? cur_size : len; + if(ib->free < len) { + size_t sz = vbuf->chunk_size; + if(sz < len) { + sz = len; + } - chunk = (msgpack_vrefbuffer_chunk*)malloc( - cur_size + sizeof(msgpack_vrefbuffer_chunk)); + msgpack_vrefbuffer_chunk* chunk = (msgpack_vrefbuffer_chunk*)malloc( + sizeof(msgpack_vrefbuffer_chunk) + sz); if(chunk == NULL) { return -1; } - chunk->free = cur_size; - chunk->next = vbuf->chunk; - vbuf->chunk = chunk; + chunk->next = ib->head; + ib->head = chunk; + ib->free = sz; + ib->ptr = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk); } - char* m = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk) - + (cur_size - chunk->free); - + char* m = ib->ptr; memcpy(m, buf, len); - chunk->free -= len; + ib->free -= len; + ib->ptr += len; if(vbuf->tail != vbuf->array && m == (const char*)((vbuf->tail-1)->iov_base) + (vbuf->tail-1)->iov_len) { @@ -135,28 +140,63 @@ int msgpack_vrefbuffer_append_copy(msgpack_vrefbuffer* vbuf, int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to) { - const size_t tosize = to->tail - to->array; - if(vbuf->tail + tosize < vbuf->end) { - const size_t nused = vbuf->tail - vbuf->array; - const size_t nsize = vbuf->end - vbuf->array; + size_t sz = vbuf->chunk_size; + + msgpack_vrefbuffer_chunk* empty = (msgpack_vrefbuffer_chunk*)malloc( + sizeof(msgpack_vrefbuffer_chunk) + sz); + if(empty == NULL) { + return -1; + } + + empty->next = NULL; + + + const size_t nused = vbuf->tail - vbuf->array; + if(to->tail + nused < vbuf->end) { + const size_t tosize = to->tail - to->array; const size_t reqsize = nused + tosize; - size_t nnext = nsize * 2; + size_t nnext = (to->end - to->array) * 2; while(nnext < reqsize) { nnext *= 2; } struct iovec* nvec = (struct iovec*)realloc( - vbuf->array, sizeof(struct iovec)*nnext); + to->array, sizeof(struct iovec)*nnext); if(nvec == NULL) { + free(empty); return -1; } - vbuf->array = nvec; - vbuf->end = nvec + nnext; - vbuf->tail = nvec + nused; + to->array = nvec; + to->end = nvec + nnext; + to->tail = nvec + tosize; } - memcpy(vbuf->tail, vbuf->array, sizeof(struct iovec)*tosize); + memcpy(to->tail, vbuf->array, sizeof(struct iovec)*nused); + + to->tail += nused; + vbuf->tail = vbuf->array; + + + msgpack_vrefbuffer_inner_buffer* const ib = &vbuf->inner_buffer; + msgpack_vrefbuffer_inner_buffer* const toib = &to->inner_buffer; + + msgpack_vrefbuffer_chunk* last = ib->head; + while(last->next != NULL) { + last = last->next; + } + last->next = toib->head; + toib->head = ib->head; + + if(toib->free < ib->free) { + toib->free = ib->free; + toib->ptr = ib->ptr; + } + + ib->head = empty; + ib->free = sz; + ib->ptr = ((char*)empty) + sizeof(msgpack_vrefbuffer_chunk); + return 0; } diff --git a/c/vrefbuffer.h b/c/vrefbuffer.h index 9f24f14..ffc52bf 100644 --- a/c/vrefbuffer.h +++ b/c/vrefbuffer.h @@ -40,24 +40,27 @@ extern "C" { #endif #ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE -#define MSGPACK_VREFBUFFER_CHUNK_SIZE 2048 +#define MSGPACK_VREFBUFFER_CHUNK_SIZE 8192 #endif -typedef struct msgpack_vrefbuffer_chunk { +struct msgpack_vrefbuffer_chunk; +typedef struct msgpack_vrefbuffer_chunk msgpack_vrefbuffer_chunk; + +typedef struct msgpack_vrefbuffer_inner_buffer { size_t free; - struct msgpack_vrefbuffer_chunk* next; - /* data ... */ -} msgpack_vrefbuffer_chunk; + char* ptr; + msgpack_vrefbuffer_chunk* head; +} msgpack_vrefbuffer_inner_buffer; typedef struct msgpack_vrefbuffer { - size_t chunk_size; - size_t ref_size; - struct iovec* tail; struct iovec* end; struct iovec* array; - msgpack_vrefbuffer_chunk* chunk; + size_t chunk_size; + size_t ref_size; + + msgpack_vrefbuffer_inner_buffer inner_buffer; } msgpack_vrefbuffer; From 2bf3f1856f23d0759686ab74b8abf7f46841564a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 15 Jan 2010 14:45:52 +0900 Subject: [PATCH 0349/1648] c,cpp: configure.in: version 0.4.0 --- configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 32d906e..3d2247e 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.3.10) +AM_INIT_AUTOMAKE(msgpack, 0.4.0) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From 1066bb38a87df13e89d891682e39df2ef9a08d7c Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 16 Jan 2010 02:03:11 +0900 Subject: [PATCH 0350/1648] add tests for multi[map, set] and tr1::unordered_[multi][map, set] --- configure.in | 5 +- cpp/test.cpp | 190 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 3d2247e..99c4352 100644 --- a/configure.in +++ b/configure.in @@ -12,7 +12,6 @@ AC_PROG_CC CFLAGS="-O4 -Wall $CFLAGS -I.." - AC_MSG_CHECKING([if c++ api is enabled]) AC_ARG_ENABLE(cxx, AS_HELP_STRING([--disable-cxx], @@ -30,6 +29,10 @@ AC_PROG_CXX CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" +AC_LANG_PUSH([C++]) +AC_CHECK_HEADERS(tr1/unordered_map) +AC_CHECK_HEADERS(tr1/unordered_set) +AC_LANG_POP([C++]) AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") diff --git a/cpp/test.cpp b/cpp/test.cpp index 4447890..b152e6d 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -10,6 +10,10 @@ #include +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + using namespace std; const unsigned int kLoop = 10000; @@ -227,6 +231,8 @@ TEST(MSGPACK, simple_buffer_false) //----------------------------------------------------------------------------- +// STL + TEST(MSGPACK_STL, simple_buffer_string) { for (unsigned int k = 0; k < kLoop; k++) { @@ -365,6 +371,190 @@ TEST(MSGPACK_STL, simple_buffer_pair) } } +TEST(MSGPACK_STL, simple_buffer_multimap) +{ + for (unsigned int k = 0; k < kLoop; k++) { + multimap val1; + for (unsigned int i = 0; i < kElements; i++) { + int i1 = rand(); + val1.insert(make_pair(i1, rand())); + val1.insert(make_pair(i1, rand())); + } + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + multimap val2; + obj.convert(&val2); + + vector > v1, v2; + multimap::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) + v1.push_back(make_pair(it->first, it->second)); + for (it = val2.begin(); it != val2.end(); ++it) + v2.push_back(make_pair(it->first, it->second)); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_EQ(v1.size(), v2.size()); + sort(v1.begin(), v1.end()); + sort(v2.begin(), v2.end()); + EXPECT_TRUE(v1 == v2); + } +} + +TEST(MSGPACK_STL, simple_buffer_multiset) +{ + for (unsigned int k = 0; k < kLoop; k++) { + multiset val1; + for (unsigned int i = 0; i < kElements; i++) + val1.insert(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + multiset val2; + obj.convert(&val2); + + vector v1, v2; + multiset::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) + v1.push_back(*it); + for (it = val2.begin(); it != val2.end(); ++it) + v2.push_back(*it); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_EQ(v1.size(), v2.size()); + sort(v1.begin(), v1.end()); + sort(v2.begin(), v2.end()); + EXPECT_TRUE(v1 == v2); + } +} + +// TR1 + +#ifdef HAVE_TR1_UNORDERED_MAP +#include +#include "cpp/type/tr1/unordered_map.hpp" +TEST(MSGPACK_TR1, simple_buffer_unordered_map) +{ + for (unsigned int k = 0; k < kLoop; k++) { + tr1::unordered_map val1; + for (unsigned int i = 0; i < kElements; i++) + val1[rand()] = rand(); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + tr1::unordered_map val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + tr1::unordered_map::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) { + EXPECT_TRUE(val2.find(it->first) != val2.end()); + EXPECT_EQ(it->second, val2.find(it->first)->second); + } + } +} + +TEST(MSGPACK_TR1, simple_buffer_unordered_multimap) +{ + for (unsigned int k = 0; k < kLoop; k++) { + tr1::unordered_multimap val1; + for (unsigned int i = 0; i < kElements; i++) { + int i1 = rand(); + val1.insert(make_pair(i1, rand())); + val1.insert(make_pair(i1, rand())); + } + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + tr1::unordered_multimap val2; + obj.convert(&val2); + + vector > v1, v2; + tr1::unordered_multimap::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) + v1.push_back(make_pair(it->first, it->second)); + for (it = val2.begin(); it != val2.end(); ++it) + v2.push_back(make_pair(it->first, it->second)); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_EQ(v1.size(), v2.size()); + sort(v1.begin(), v1.end()); + sort(v2.begin(), v2.end()); + EXPECT_TRUE(v1 == v2); + } +} +#endif + +#ifdef HAVE_TR1_UNORDERED_SET +#include +#include "cpp/type/tr1/unordered_set.hpp" +TEST(MSGPACK_TR1, simple_buffer_unordered_set) +{ + for (unsigned int k = 0; k < kLoop; k++) { + tr1::unordered_set val1; + for (unsigned int i = 0; i < kElements; i++) + val1.insert(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + tr1::unordered_set val2; + obj.convert(&val2); + EXPECT_EQ(val1.size(), val2.size()); + tr1::unordered_set::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) + EXPECT_TRUE(val2.find(*it) != val2.end()); + } +} + +TEST(MSGPACK_TR1, simple_buffer_unordered_multiset) +{ + for (unsigned int k = 0; k < kLoop; k++) { + tr1::unordered_multiset val1; + for (unsigned int i = 0; i < kElements; i++) + val1.insert(rand()); + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, val1); + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + EXPECT_EQ(msgpack::UNPACK_SUCCESS, ret); + tr1::unordered_multiset val2; + obj.convert(&val2); + + vector v1, v2; + tr1::unordered_multiset::const_iterator it; + for (it = val1.begin(); it != val1.end(); ++it) + v1.push_back(*it); + for (it = val2.begin(); it != val2.end(); ++it) + v2.push_back(*it); + EXPECT_EQ(val1.size(), val2.size()); + EXPECT_EQ(v1.size(), v2.size()); + sort(v1.begin(), v1.end()); + sort(v2.begin(), v2.end()); + EXPECT_TRUE(v1 == v2); + } +} +#endif + +// User-Defined Structures + class TestClass { public: From d8212ad6200db16ca49c5c8198ba0955272763c8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 20 Jan 2010 14:46:54 +0900 Subject: [PATCH 0351/1648] strict-aliasing rule --- msgpack/pack_template.h | 12 ++++++------ msgpack/unpack_template.h | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index de148bf..b314d6d 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -554,19 +554,19 @@ if(sizeof(unsigned long long) == 2) { msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) { - union { char buf[4]; uint32_t num; } f; - *((float*)&f.buf) = d; // FIXME + union { float f; uint32_t i; } mem; + mem.f = d; unsigned char buf[5]; - buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num); + buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(mem.i); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) { - union { char buf[8]; uint64_t num; } f; - *((double*)&f.buf) = d; // FIXME + union { double f; uint64_t i; } mem; + mem.f = d; unsigned char buf[9]; - buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num); + buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(mem.i); msgpack_pack_append_buffer(x, buf, 9); } diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 3328ea3..cf2e74c 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -222,13 +222,13 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c //case CS_ //case CS_ case CS_FLOAT: { - union { uint32_t num; char buf[4]; } f; - f.num = PTR_CAST_32(n); // FIXME - push_fixed_value(_float, *((float*)f.buf)); } + union { uint32_t i; float f; } mem; + mem.i = PTR_CAST_32(n); + push_fixed_value(_float, mem.f); } case CS_DOUBLE: { - union { uint64_t num; char buf[8]; } f; - f.num = PTR_CAST_64(n); // FIXME - push_fixed_value(_double, *((double*)f.buf)); } + union { uint64_t i; double f; } mem; + mem.i = PTR_CAST_64(n); + push_fixed_value(_double, mem.f); } case CS_UINT_8: push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); case CS_UINT_16: From 2b72f35c324e3a0623894f9c678bd08341973a32 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 20 Jan 2010 14:49:51 +0900 Subject: [PATCH 0352/1648] configure.in: show error message if __sync_* atomic operations are not supported --- configure.in | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 99c4352..6bf1b48 100644 --- a/configure.in +++ b/configure.in @@ -24,7 +24,7 @@ if test "$enable_cxx" != "no"; then fi fi -# FIXME +# FIXME enable_cxx AC_PROG_CXX CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" @@ -34,6 +34,22 @@ AC_CHECK_HEADERS(tr1/unordered_map) AC_CHECK_HEADERS(tr1/unordered_set) AC_LANG_POP([C++]) +AC_CACHE_CHECK([for __sync_* atomic operations], msgpack_cv_atomic_ops, [ + AC_TRY_LINK([ + int atomic_sub(int i) { return __sync_sub_and_fetch(&i, 1); } + int atomic_add(int i) { return __sync_add_and_fetch(&i, 1); } + ], [], msgpack_cv_atomic_ops="yes") + ]) +if test "$msgpack_cv_atomic_ops" != "yes"; then + AC_MSG_ERROR([__sync_* atomic operations are not supported. +Note that gcc < 4.1 is not supported. +If you are using gcc-4.1 and the CPU architecture is x86, try to add +CFLAGS"--march=i686" and CXXFLAGS="-march=i668" options to ./configure as follows: + + $ ./configure CFLAGS="-march=i686" CXXFLAGS="-march=i686" +]) +fi + AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") AC_PROG_LIBTOOL From ebc0eeac7911891c5794891e48ab8db8605e5ab7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 20 Jan 2010 15:17:33 +0900 Subject: [PATCH 0353/1648] c,cpp: 0.4.1 --- configure.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.in b/configure.in index 6bf1b48..f9f2918 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.4.0) +AM_INIT_AUTOMAKE(msgpack, 0.4.1) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) @@ -29,6 +29,7 @@ AC_PROG_CXX CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" +# FIXME enable_cxx AC_LANG_PUSH([C++]) AC_CHECK_HEADERS(tr1/unordered_map) AC_CHECK_HEADERS(tr1/unordered_set) From c69092e110b82f4a6b37759a37cd43dbe8e486ed Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 21 Jan 2010 17:25:32 +0900 Subject: [PATCH 0354/1648] configure.in: fix message --- configure.in | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/configure.in b/configure.in index 6bf1b48..00211cc 100644 --- a/configure.in +++ b/configure.in @@ -35,16 +35,18 @@ AC_CHECK_HEADERS(tr1/unordered_set) AC_LANG_POP([C++]) AC_CACHE_CHECK([for __sync_* atomic operations], msgpack_cv_atomic_ops, [ - AC_TRY_LINK([ - int atomic_sub(int i) { return __sync_sub_and_fetch(&i, 1); } - int atomic_add(int i) { return __sync_add_and_fetch(&i, 1); } - ], [], msgpack_cv_atomic_ops="yes") - ]) + AC_TRY_LINK([ + int atomic_sub(int i) { return __sync_sub_and_fetch(&i, 1); } + int atomic_add(int i) { return __sync_add_and_fetch(&i, 1); } + ], [], msgpack_cv_atomic_ops="yes") + ]) if test "$msgpack_cv_atomic_ops" != "yes"; then - AC_MSG_ERROR([__sync_* atomic operations are not supported. + AC_MSG_ERROR([__sync_* atomic operations are not supported. + Note that gcc < 4.1 is not supported. -If you are using gcc-4.1 and the CPU architecture is x86, try to add -CFLAGS"--march=i686" and CXXFLAGS="-march=i668" options to ./configure as follows: + +If you are using gcc >= 4.1 and the default target CPU architecture is "i386", try to +add CFLAGS="--march=i686" and CXXFLAGS="-march=i668" options to ./configure as follows: $ ./configure CFLAGS="-march=i686" CXXFLAGS="-march=i686" ]) From 404a3933154dc57b2e650cdabb51f80e4aeb1dbe Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Thu, 21 Jan 2010 21:55:06 +0900 Subject: [PATCH 0355/1648] c, cpp: fix test problem - using LDADD instead of LDFLAGS - fix include path in test.cpp --- c/Makefile.am | 2 +- cpp/Makefile.am | 2 +- cpp/test.cpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/c/Makefile.am b/c/Makefile.am index 23995a5..f84759a 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -23,6 +23,6 @@ check_PROGRAMS = \ msgpackc_test_SOURCES = test.cpp msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -msgpackc_test_LDFLAGS = libmsgpackc.la -lgtest_main +msgpackc_test_LDADD = libmsgpackc.la -lgtest_main TESTS = $(check_PROGRAMS) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 99f4a32..d4de161 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -39,6 +39,6 @@ check_PROGRAMS = \ msgpack_test_SOURCES = test.cpp msgpack_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp -msgpack_test_LDFLAGS = libmsgpack.la -lgtest_main +msgpack_test_LDADD = libmsgpack.la -lgtest_main TESTS = $(check_PROGRAMS) diff --git a/cpp/test.cpp b/cpp/test.cpp index b152e6d..113914a 100644 --- a/cpp/test.cpp +++ b/cpp/test.cpp @@ -438,7 +438,7 @@ TEST(MSGPACK_STL, simple_buffer_multiset) #ifdef HAVE_TR1_UNORDERED_MAP #include -#include "cpp/type/tr1/unordered_map.hpp" +#include "cpp/msgpack/type/tr1/unordered_map.hpp" TEST(MSGPACK_TR1, simple_buffer_unordered_map) { for (unsigned int k = 0; k < kLoop; k++) { @@ -499,7 +499,7 @@ TEST(MSGPACK_TR1, simple_buffer_unordered_multimap) #ifdef HAVE_TR1_UNORDERED_SET #include -#include "cpp/type/tr1/unordered_set.hpp" +#include "cpp/msgpack/type/tr1/unordered_set.hpp" TEST(MSGPACK_TR1, simple_buffer_unordered_set) { for (unsigned int k = 0; k < kLoop; k++) { From 9e2c9d7812565bfff8004990aef5e3f99f36c2b7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 12:21:46 +0900 Subject: [PATCH 0356/1648] Tiny fix in README --- python/README | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/README b/python/README index 2b2ebe9..a7e8c53 100644 --- a/python/README +++ b/python/README @@ -2,7 +2,7 @@ MessagePack Python Binding =========================== -:author: Naoki INADA +:author: INADA Naoki :version: 0.1.0 :date: 2009-07-12 @@ -32,7 +32,11 @@ MinGW GCC. TEST ---- -MessagePack uses nosetest for testing. +MessagePack uses `nosetest` for testing. Run test with following command: $ nosetests test + + +.. + vim: filetype=rst From 6f8e7f2709b811068988055c956790cb0537b41b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 12:21:46 +0900 Subject: [PATCH 0357/1648] Tiny fix in README --- README | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README b/README index 2b2ebe9..a7e8c53 100644 --- a/README +++ b/README @@ -2,7 +2,7 @@ MessagePack Python Binding =========================== -:author: Naoki INADA +:author: INADA Naoki :version: 0.1.0 :date: 2009-07-12 @@ -32,7 +32,11 @@ MinGW GCC. TEST ---- -MessagePack uses nosetest for testing. +MessagePack uses `nosetest` for testing. Run test with following command: $ nosetests test + + +.. + vim: filetype=rst From 1a11608f1f5ed4f9ae6dd302b31327145529bf38 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 20:51:55 +0900 Subject: [PATCH 0358/1648] cythoning *.pyx when sdist. --- python/setup_dev.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/python/setup_dev.py b/python/setup_dev.py index 4efc769..66cf27e 100755 --- a/python/setup_dev.py +++ b/python/setup_dev.py @@ -1,15 +1,41 @@ #!/usr/bin/env python # coding: utf-8 -from distutils.core import setup, Extension -from Cython.Distutils import build_ext import os +from glob import glob +from distutils.core import setup, Extension +from distutils.command.sdist import sdist + +try: + from Cython.Distutils import build_ext + import Cython.Compiler.Main as cython_compiler + have_cython = True +except ImportError: + from distutils.command.build_ext import build_ext + have_cython = False version = '0.2.0dev' +# take care of extension modules. +if have_cython: + sources = ['msgpack/_msgpack.pyx'] + + class Sdist(sdist): + def __init__(self, *args, **kwargs): + for src in glob('msgpack/*.pyx'): + cython_compiler.compile(glob('msgpack/*.pyx'), + cython_compiler.default_options) + sdist.__init__(self, *args, **kwargs) +else: + sources = ['msgpack/_msgpack.c'] + + Sdist = sdist + msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'] + sources=sources, ) +del sources + desc = 'MessagePack (de)serializer.' long_desc = desc + """ @@ -25,14 +51,15 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) """ setup(name='msgpack', - author='Naoki INADA', + author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, - cmdclass={'build_ext': build_ext}, + cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, long_description=long_desc, + url="http://msgpack.sourceforge.jp/", classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', From b6a4568b0fcb4433c4e1d033015c881a24bf32a3 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 20:51:55 +0900 Subject: [PATCH 0359/1648] cythoning *.pyx when sdist. --- setup_dev.py | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/setup_dev.py b/setup_dev.py index 4efc769..66cf27e 100755 --- a/setup_dev.py +++ b/setup_dev.py @@ -1,15 +1,41 @@ #!/usr/bin/env python # coding: utf-8 -from distutils.core import setup, Extension -from Cython.Distutils import build_ext import os +from glob import glob +from distutils.core import setup, Extension +from distutils.command.sdist import sdist + +try: + from Cython.Distutils import build_ext + import Cython.Compiler.Main as cython_compiler + have_cython = True +except ImportError: + from distutils.command.build_ext import build_ext + have_cython = False version = '0.2.0dev' +# take care of extension modules. +if have_cython: + sources = ['msgpack/_msgpack.pyx'] + + class Sdist(sdist): + def __init__(self, *args, **kwargs): + for src in glob('msgpack/*.pyx'): + cython_compiler.compile(glob('msgpack/*.pyx'), + cython_compiler.default_options) + sdist.__init__(self, *args, **kwargs) +else: + sources = ['msgpack/_msgpack.c'] + + Sdist = sdist + msgpack_mod = Extension('msgpack._msgpack', - sources=['msgpack/_msgpack.pyx'] + sources=sources, ) +del sources + desc = 'MessagePack (de)serializer.' long_desc = desc + """ @@ -25,14 +51,15 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) """ setup(name='msgpack', - author='Naoki INADA', + author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, - cmdclass={'build_ext': build_ext}, + cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, long_description=long_desc, + url="http://msgpack.sourceforge.jp/", classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', From e02d20dd747ce70e5e0de71bce244bd86666c6f0 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 20:52:48 +0900 Subject: [PATCH 0360/1648] replace setup by setup_dev. --- python/setup.py | 38 +++++++++++++++++++++---- python/setup_dev.py | 68 --------------------------------------------- 2 files changed, 32 insertions(+), 74 deletions(-) delete mode 100755 python/setup_dev.py diff --git a/python/setup.py b/python/setup.py index c48be14..66cf27e 100755 --- a/python/setup.py +++ b/python/setup.py @@ -1,16 +1,41 @@ #!/usr/bin/env python # coding: utf-8 -from distutils.core import setup, Extension -#from Cython.Distutils import build_ext import os +from glob import glob +from distutils.core import setup, Extension +from distutils.command.sdist import sdist + +try: + from Cython.Distutils import build_ext + import Cython.Compiler.Main as cython_compiler + have_cython = True +except ImportError: + from distutils.command.build_ext import build_ext + have_cython = False version = '0.2.0dev' +# take care of extension modules. +if have_cython: + sources = ['msgpack/_msgpack.pyx'] + + class Sdist(sdist): + def __init__(self, *args, **kwargs): + for src in glob('msgpack/*.pyx'): + cython_compiler.compile(glob('msgpack/*.pyx'), + cython_compiler.default_options) + sdist.__init__(self, *args, **kwargs) +else: + sources = ['msgpack/_msgpack.c'] + + Sdist = sdist + msgpack_mod = Extension('msgpack._msgpack', - #sources=['msgpack/_msgpack.pyx'] - sources=['msgpack/_msgpack.c'] + sources=sources, ) +del sources + desc = 'MessagePack (de)serializer.' long_desc = desc + """ @@ -26,14 +51,15 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) """ setup(name='msgpack', - author='Naoki INADA', + author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, - #cmdclass={'build_ext': build_ext}, + cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, long_description=long_desc, + url="http://msgpack.sourceforge.jp/", classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', diff --git a/python/setup_dev.py b/python/setup_dev.py deleted file mode 100755 index 66cf27e..0000000 --- a/python/setup_dev.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import os -from glob import glob -from distutils.core import setup, Extension -from distutils.command.sdist import sdist - -try: - from Cython.Distutils import build_ext - import Cython.Compiler.Main as cython_compiler - have_cython = True -except ImportError: - from distutils.command.build_ext import build_ext - have_cython = False - -version = '0.2.0dev' - -# take care of extension modules. -if have_cython: - sources = ['msgpack/_msgpack.pyx'] - - class Sdist(sdist): - def __init__(self, *args, **kwargs): - for src in glob('msgpack/*.pyx'): - cython_compiler.compile(glob('msgpack/*.pyx'), - cython_compiler.default_options) - sdist.__init__(self, *args, **kwargs) -else: - sources = ['msgpack/_msgpack.c'] - - Sdist = sdist - -msgpack_mod = Extension('msgpack._msgpack', - sources=sources, - ) -del sources - - -desc = 'MessagePack (de)serializer.' -long_desc = desc + """ - -MessagePack_ (de)serializer for Python. - -.. _MessagePack: http://msgpack.sourceforge.jp/ - -What's MessagePack? (from http://msgpack.sourceforge.jp/) - - MessagePack is a binary-based efficient data interchange format that is - focused on high performance. It is like JSON, but very fast and small. -""" - -setup(name='msgpack', - author='INADA Naoki', - author_email='songofacandy@gmail.com', - version=version, - cmdclass={'build_ext': build_ext, 'sdist': Sdist}, - ext_modules=[msgpack_mod], - packages=['msgpack'], - description=desc, - long_description=long_desc, - url="http://msgpack.sourceforge.jp/", - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - ] - ) From de508dcc1047c7bf065fc5476a48f94f725b059a Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Mon, 25 Jan 2010 20:52:48 +0900 Subject: [PATCH 0361/1648] replace setup by setup_dev. --- setup.py | 38 ++++++++++++++++++++++++----- setup_dev.py | 68 ---------------------------------------------------- 2 files changed, 32 insertions(+), 74 deletions(-) delete mode 100755 setup_dev.py diff --git a/setup.py b/setup.py index c48be14..66cf27e 100755 --- a/setup.py +++ b/setup.py @@ -1,16 +1,41 @@ #!/usr/bin/env python # coding: utf-8 -from distutils.core import setup, Extension -#from Cython.Distutils import build_ext import os +from glob import glob +from distutils.core import setup, Extension +from distutils.command.sdist import sdist + +try: + from Cython.Distutils import build_ext + import Cython.Compiler.Main as cython_compiler + have_cython = True +except ImportError: + from distutils.command.build_ext import build_ext + have_cython = False version = '0.2.0dev' +# take care of extension modules. +if have_cython: + sources = ['msgpack/_msgpack.pyx'] + + class Sdist(sdist): + def __init__(self, *args, **kwargs): + for src in glob('msgpack/*.pyx'): + cython_compiler.compile(glob('msgpack/*.pyx'), + cython_compiler.default_options) + sdist.__init__(self, *args, **kwargs) +else: + sources = ['msgpack/_msgpack.c'] + + Sdist = sdist + msgpack_mod = Extension('msgpack._msgpack', - #sources=['msgpack/_msgpack.pyx'] - sources=['msgpack/_msgpack.c'] + sources=sources, ) +del sources + desc = 'MessagePack (de)serializer.' long_desc = desc + """ @@ -26,14 +51,15 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) """ setup(name='msgpack', - author='Naoki INADA', + author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, - #cmdclass={'build_ext': build_ext}, + cmdclass={'build_ext': build_ext, 'sdist': Sdist}, ext_modules=[msgpack_mod], packages=['msgpack'], description=desc, long_description=long_desc, + url="http://msgpack.sourceforge.jp/", classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', diff --git a/setup_dev.py b/setup_dev.py deleted file mode 100755 index 66cf27e..0000000 --- a/setup_dev.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import os -from glob import glob -from distutils.core import setup, Extension -from distutils.command.sdist import sdist - -try: - from Cython.Distutils import build_ext - import Cython.Compiler.Main as cython_compiler - have_cython = True -except ImportError: - from distutils.command.build_ext import build_ext - have_cython = False - -version = '0.2.0dev' - -# take care of extension modules. -if have_cython: - sources = ['msgpack/_msgpack.pyx'] - - class Sdist(sdist): - def __init__(self, *args, **kwargs): - for src in glob('msgpack/*.pyx'): - cython_compiler.compile(glob('msgpack/*.pyx'), - cython_compiler.default_options) - sdist.__init__(self, *args, **kwargs) -else: - sources = ['msgpack/_msgpack.c'] - - Sdist = sdist - -msgpack_mod = Extension('msgpack._msgpack', - sources=sources, - ) -del sources - - -desc = 'MessagePack (de)serializer.' -long_desc = desc + """ - -MessagePack_ (de)serializer for Python. - -.. _MessagePack: http://msgpack.sourceforge.jp/ - -What's MessagePack? (from http://msgpack.sourceforge.jp/) - - MessagePack is a binary-based efficient data interchange format that is - focused on high performance. It is like JSON, but very fast and small. -""" - -setup(name='msgpack', - author='INADA Naoki', - author_email='songofacandy@gmail.com', - version=version, - cmdclass={'build_ext': build_ext, 'sdist': Sdist}, - ext_modules=[msgpack_mod], - packages=['msgpack'], - description=desc, - long_description=long_desc, - url="http://msgpack.sourceforge.jp/", - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - ] - ) From 34b3bbc8839f63e3e624dc0ce42525657cdcdfa7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 29 Jan 2010 17:23:48 +0900 Subject: [PATCH 0362/1648] ruby: add Unpacker#each --- ruby/unpack.c | 127 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 120 insertions(+), 7 deletions(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index e9d6494..0f1af38 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -18,10 +18,16 @@ #include "ruby.h" #include "msgpack/unpack_define.h" +static ID s_sysread; typedef struct { int finished; VALUE source; + size_t offset; + size_t parsed; + VALUE buffer; + VALUE stream; + VALUE streambuf; } unpack_user; @@ -144,6 +150,9 @@ static void MessagePack_Unpacker_free(void* data) static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) { unsigned int i; + rb_gc_mark(mp->user.buffer); + rb_gc_mark(mp->user.stream); + rb_gc_mark(mp->user.streambuf); for(i=0; i < mp->top; ++i) { rb_gc_mark(mp->stack[i].obj); rb_gc_mark(mp->stack[i].map_key); /* maybe map_key is not initialized */ @@ -164,14 +173,32 @@ static VALUE MessagePack_Unpacker_reset(VALUE self) UNPACKER(self, mp); template_init(mp); init_stack(mp); - unpack_user u = {0, Qnil}; - mp->user = u; + mp->user.finished = 0; return self; } -static VALUE MessagePack_Unpacker_initialize(VALUE self) +static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) { - return MessagePack_Unpacker_reset(self); + VALUE stream; + switch(argc) { + case 0: + stream = Qnil; + break; + case 1: + stream = argv[0]; + break; + default: + rb_raise(rb_eArgError, "wrong number of arguments (%d for 0)", argc); + } + + MessagePack_Unpacker_reset(self); + UNPACKER(self, mp); + mp->user.offset = 0; + mp->user.parsed = 0; + mp->user.buffer = rb_str_new("",0); + mp->user.stream = stream; + mp->user.streambuf = rb_str_new("",0); + return self; } @@ -249,6 +276,87 @@ static VALUE MessagePack_Unpacker_data(VALUE self) } +static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) +{ + UNPACKER(self, mp); + StringValue(data); + rb_str_cat(mp->user.buffer, RSTRING_PTR(data), RSTRING_LEN(data)); + return Qnil; +} + +static VALUE MessagePack_Unpacker_stream_get(VALUE self) +{ + UNPACKER(self, mp); + return mp->user.stream; +} + +static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) +{ + UNPACKER(self, mp); + return mp->user.stream = val; +} + +static VALUE MessagePack_Unpacker_fill(VALUE self) +{ + UNPACKER(self, mp); + + if(mp->user.stream == Qnil) { + return Qnil; + } + + size_t len; + if(RSTRING_LEN(mp->user.buffer) == 0) { + rb_funcall(mp->user.stream, s_sysread, 2, LONG2FIX(64*1024), mp->user.buffer); + len = RSTRING_LEN(mp->user.buffer); + } else { + rb_funcall(mp->user.stream, s_sysread, 2, LONG2FIX(64*1024), mp->user.streambuf); + len = RSTRING_LEN(mp->user.streambuf); + rb_str_cat(mp->user.buffer, RSTRING_PTR(mp->user.streambuf), RSTRING_LEN(mp->user.streambuf)); + } + + return LONG2FIX(len); +} + +static VALUE MessagePack_Unpacker_each(VALUE self) +{ + UNPACKER(self, mp); + int ret; + +#ifdef RETURN_ENUMERATOR + RETURN_ENUMERATOR(self, 0, 0); +#endif + + while(1) { + if(RSTRING_LEN(mp->user.buffer) <= mp->user.offset) { + do_fill: + { + VALUE len = MessagePack_Unpacker_fill(self); + if(len == Qnil || FIX2LONG(len) == 0) { + break; + } + } + } + + mp->user.source = mp->user.buffer; + ret = template_execute(mp, RSTRING_PTR(mp->user.buffer), RSTRING_LEN(mp->user.buffer), &mp->user.offset); + mp->user.source = Qnil; + + if(ret < 0) { + rb_raise(eUnpackError, "parse error."); + } else if(ret > 0) { + VALUE data = template_data(mp); + template_init(mp); + init_stack(mp); + rb_yield(data); + } else { + goto do_fill; + } + } + + return Qnil; +} + + static VALUE MessagePack_unpack_impl(VALUE args) { msgpack_unpack_t* mp = (msgpack_unpack_t*)((VALUE*)args)[0]; @@ -292,7 +400,7 @@ static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) msgpack_unpack_t mp; template_init(&mp); init_stack(&mp); - unpack_user u = {0, Qnil}; + unpack_user u = {0, Qnil, 0, 0, Qnil, Qnil, Qnil}; mp.user = u; rb_gc_disable(); @@ -313,17 +421,22 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) void Init_msgpack_unpack(VALUE mMessagePack) { + s_sysread = rb_intern("sysread"); eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); - rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, 0); + rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, -1); rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); rb_define_method(cUnpacker, "execute_limit", MessagePack_Unpacker_execute_limit, 3); rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); + rb_define_method(cUnpacker, "feed", MessagePack_Unpacker_feed, 1); + rb_define_method(cUnpacker, "fill", MessagePack_Unpacker_fill, 0); + rb_define_method(cUnpacker, "each", MessagePack_Unpacker_each, 0); + rb_define_method(cUnpacker, "stream", MessagePack_Unpacker_stream_get, 0); + rb_define_method(cUnpacker, "stream=", MessagePack_Unpacker_stream_set, 1); rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2); } - From 7df60b259b55b6294c19315350ec312809107117 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 2 Feb 2010 10:52:42 +0900 Subject: [PATCH 0363/1648] MSVC2005 compatibility (@hotpepsi++) --- c/object.h | 1 - c/zone.c | 4 ---- c/zone.h | 1 + cpp/type/vector.hpp | 14 +++++++++----- msgpack/sysdep.h | 11 +++++++++++ 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/c/object.h b/c/object.h index 27b593e..9a014be 100644 --- a/c/object.h +++ b/c/object.h @@ -19,7 +19,6 @@ #define MSGPACK_OBJECT_H__ #include "msgpack/zone.h" -#include "msgpack/sysdep.h" #include #ifdef __cplusplus diff --git a/c/zone.c b/c/zone.c index 2d8bc58..3d0634e 100644 --- a/c/zone.c +++ b/c/zone.c @@ -104,7 +104,6 @@ static inline void init_finalizer_array(msgpack_zone_finalizer_array* fa) static inline void call_finalizer_array(msgpack_zone_finalizer_array* fa) { - // 逆順ã«å‘¼ã³å‡ºã— msgpack_zone_finalizer* fin = fa->tail; for(; fin != fa->array; --fin) { (*(fin-1)->func)((fin-1)->data); @@ -132,9 +131,6 @@ bool msgpack_zone_push_finalizer_expand(msgpack_zone* zone, size_t nnext; if(nused == 0) { - // åˆå›žã®å‘¼ã³å‡ºã—：fa->tail == fa->end == fa->array == NULL - - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ nnext = (sizeof(msgpack_zone_finalizer) < 72/2) ? 72 / sizeof(msgpack_zone_finalizer) : 8; diff --git a/c/zone.h b/c/zone.h index e07e762..475ae51 100644 --- a/c/zone.h +++ b/c/zone.h @@ -18,6 +18,7 @@ #ifndef MSGPACK_ZONE_H__ #define MSGPACK_ZONE_H__ +#include "msgpack/sysdep.h" #include #include diff --git a/cpp/type/vector.hpp b/cpp/type/vector.hpp index 754cdc0..385d070 100644 --- a/cpp/type/vector.hpp +++ b/cpp/type/vector.hpp @@ -29,11 +29,15 @@ inline std::vector& operator>> (object o, std::vector& v) { if(o.type != type::ARRAY) { throw type_error(); } v.resize(o.via.array.size); - object* p = o.via.array.ptr; - object* const pend = o.via.array.ptr + o.via.array.size; - T* it = &v.front(); - for(; p < pend; ++p, ++it) { - p->convert(it); + if(o.via.array.size > 0) { + object* p = o.via.array.ptr; + object* const pend = o.via.array.ptr + o.via.array.size; + T* it = &v[0]; + do { + p->convert(it); + ++p; + ++it; + } while(p < pend); } return v; } diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index 106158e..e9540d5 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -46,6 +46,17 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif +#ifdef __cplusplus +/* numeric_limits::min,max */ +#ifdef max +#undef max +#endif +#ifdef min +#undef min +#endif +#endif + + #ifdef _WIN32 #include #else From 66ef6c9e4cf66626ec5af498096b35abb8a050c0 Mon Sep 17 00:00:00 2001 From: firewood Date: Fri, 5 Feb 2010 22:49:00 +0900 Subject: [PATCH 0364/1648] undef after including winsock2.h. --- msgpack/sysdep.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index e9540d5..c69c69a 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -46,6 +46,9 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif +#ifdef _WIN32 +#include + #ifdef __cplusplus /* numeric_limits::min,max */ #ifdef max @@ -56,9 +59,6 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif #endif - -#ifdef _WIN32 -#include #else #include /* __BYTE_ORDER */ #endif From 9a034234d8955b2f6f3877d9cb39993ffd5981a9 Mon Sep 17 00:00:00 2001 From: firewood Date: Fri, 5 Feb 2010 22:49:28 +0900 Subject: [PATCH 0365/1648] add vc2005 project file. --- msgpack_vc8.vcproj | 275 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 msgpack_vc8.vcproj diff --git a/msgpack_vc8.vcproj b/msgpack_vc8.vcproj new file mode 100644 index 0000000..5daec01 --- /dev/null +++ b/msgpack_vc8.vcproj @@ -0,0 +1,275 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From ca0b085d156260b6d916139495a6d306a1e7f886 Mon Sep 17 00:00:00 2001 From: firewood Date: Fri, 5 Feb 2010 22:50:05 +0900 Subject: [PATCH 0366/1648] add a blanc after Japanese comment --- c/vrefbuffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c index b970ece..b528874 100644 --- a/c/vrefbuffer.c +++ b/c/vrefbuffer.c @@ -30,7 +30,7 @@ bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, vbuf->chunk_size = chunk_size; vbuf->ref_size = ref_size; - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ + // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ size_t nfirst = (sizeof(struct iovec) < 72/2) ? 72 / sizeof(struct iovec) : 8; From cd10fbc1fe39308045f0b7bf9cce5f8153f25f1d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 6 Feb 2010 20:09:41 +0900 Subject: [PATCH 0367/1648] removed symbolic links --- c/msgpack | 1 - c/{ => msgpack}/object.h | 0 c/{ => msgpack}/pack.h | 0 c/{ => msgpack}/sbuffer.h | 0 c/{ => msgpack}/unpack.h | 0 c/{ => msgpack}/vrefbuffer.h | 0 c/{ => msgpack}/zone.h | 0 cpp/msgpack | 1 - cpp/{ => msgpack}/object.hpp | 0 cpp/{ => msgpack}/pack.hpp | 0 cpp/{ => msgpack}/sbuffer.hpp | 0 cpp/{ => msgpack}/type.hpp | 0 cpp/{ => msgpack}/type/bool.hpp | 0 cpp/{ => msgpack}/type/define.hpp.erb | 0 cpp/{ => msgpack}/type/deque.hpp | 0 cpp/{ => msgpack}/type/float.hpp | 0 cpp/{ => msgpack}/type/int.hpp | 0 cpp/{ => msgpack}/type/list.hpp | 0 cpp/{ => msgpack}/type/map.hpp | 0 cpp/{ => msgpack}/type/nil.hpp | 0 cpp/{ => msgpack}/type/pair.hpp | 0 cpp/{ => msgpack}/type/raw.hpp | 0 cpp/{ => msgpack}/type/set.hpp | 0 cpp/{ => msgpack}/type/string.hpp | 0 cpp/{ => msgpack}/type/tr1/unordered_map.hpp | 0 cpp/{ => msgpack}/type/tr1/unordered_set.hpp | 0 cpp/{ => msgpack}/type/tuple.hpp.erb | 0 cpp/{ => msgpack}/type/vector.hpp | 0 cpp/{ => msgpack}/unpack.hpp | 0 cpp/{ => msgpack}/vrefbuffer.hpp | 0 cpp/{ => msgpack}/zone.hpp.erb | 0 31 files changed, 2 deletions(-) delete mode 120000 c/msgpack rename c/{ => msgpack}/object.h (100%) rename c/{ => msgpack}/pack.h (100%) rename c/{ => msgpack}/sbuffer.h (100%) rename c/{ => msgpack}/unpack.h (100%) rename c/{ => msgpack}/vrefbuffer.h (100%) rename c/{ => msgpack}/zone.h (100%) delete mode 120000 cpp/msgpack rename cpp/{ => msgpack}/object.hpp (100%) rename cpp/{ => msgpack}/pack.hpp (100%) rename cpp/{ => msgpack}/sbuffer.hpp (100%) rename cpp/{ => msgpack}/type.hpp (100%) rename cpp/{ => msgpack}/type/bool.hpp (100%) rename cpp/{ => msgpack}/type/define.hpp.erb (100%) rename cpp/{ => msgpack}/type/deque.hpp (100%) rename cpp/{ => msgpack}/type/float.hpp (100%) rename cpp/{ => msgpack}/type/int.hpp (100%) rename cpp/{ => msgpack}/type/list.hpp (100%) rename cpp/{ => msgpack}/type/map.hpp (100%) rename cpp/{ => msgpack}/type/nil.hpp (100%) rename cpp/{ => msgpack}/type/pair.hpp (100%) rename cpp/{ => msgpack}/type/raw.hpp (100%) rename cpp/{ => msgpack}/type/set.hpp (100%) rename cpp/{ => msgpack}/type/string.hpp (100%) rename cpp/{ => msgpack}/type/tr1/unordered_map.hpp (100%) rename cpp/{ => msgpack}/type/tr1/unordered_set.hpp (100%) rename cpp/{ => msgpack}/type/tuple.hpp.erb (100%) rename cpp/{ => msgpack}/type/vector.hpp (100%) rename cpp/{ => msgpack}/unpack.hpp (100%) rename cpp/{ => msgpack}/vrefbuffer.hpp (100%) rename cpp/{ => msgpack}/zone.hpp.erb (100%) diff --git a/c/msgpack b/c/msgpack deleted file mode 120000 index 945c9b4..0000000 --- a/c/msgpack +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/c/object.h b/c/msgpack/object.h similarity index 100% rename from c/object.h rename to c/msgpack/object.h diff --git a/c/pack.h b/c/msgpack/pack.h similarity index 100% rename from c/pack.h rename to c/msgpack/pack.h diff --git a/c/sbuffer.h b/c/msgpack/sbuffer.h similarity index 100% rename from c/sbuffer.h rename to c/msgpack/sbuffer.h diff --git a/c/unpack.h b/c/msgpack/unpack.h similarity index 100% rename from c/unpack.h rename to c/msgpack/unpack.h diff --git a/c/vrefbuffer.h b/c/msgpack/vrefbuffer.h similarity index 100% rename from c/vrefbuffer.h rename to c/msgpack/vrefbuffer.h diff --git a/c/zone.h b/c/msgpack/zone.h similarity index 100% rename from c/zone.h rename to c/msgpack/zone.h diff --git a/cpp/msgpack b/cpp/msgpack deleted file mode 120000 index 945c9b4..0000000 --- a/cpp/msgpack +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/cpp/object.hpp b/cpp/msgpack/object.hpp similarity index 100% rename from cpp/object.hpp rename to cpp/msgpack/object.hpp diff --git a/cpp/pack.hpp b/cpp/msgpack/pack.hpp similarity index 100% rename from cpp/pack.hpp rename to cpp/msgpack/pack.hpp diff --git a/cpp/sbuffer.hpp b/cpp/msgpack/sbuffer.hpp similarity index 100% rename from cpp/sbuffer.hpp rename to cpp/msgpack/sbuffer.hpp diff --git a/cpp/type.hpp b/cpp/msgpack/type.hpp similarity index 100% rename from cpp/type.hpp rename to cpp/msgpack/type.hpp diff --git a/cpp/type/bool.hpp b/cpp/msgpack/type/bool.hpp similarity index 100% rename from cpp/type/bool.hpp rename to cpp/msgpack/type/bool.hpp diff --git a/cpp/type/define.hpp.erb b/cpp/msgpack/type/define.hpp.erb similarity index 100% rename from cpp/type/define.hpp.erb rename to cpp/msgpack/type/define.hpp.erb diff --git a/cpp/type/deque.hpp b/cpp/msgpack/type/deque.hpp similarity index 100% rename from cpp/type/deque.hpp rename to cpp/msgpack/type/deque.hpp diff --git a/cpp/type/float.hpp b/cpp/msgpack/type/float.hpp similarity index 100% rename from cpp/type/float.hpp rename to cpp/msgpack/type/float.hpp diff --git a/cpp/type/int.hpp b/cpp/msgpack/type/int.hpp similarity index 100% rename from cpp/type/int.hpp rename to cpp/msgpack/type/int.hpp diff --git a/cpp/type/list.hpp b/cpp/msgpack/type/list.hpp similarity index 100% rename from cpp/type/list.hpp rename to cpp/msgpack/type/list.hpp diff --git a/cpp/type/map.hpp b/cpp/msgpack/type/map.hpp similarity index 100% rename from cpp/type/map.hpp rename to cpp/msgpack/type/map.hpp diff --git a/cpp/type/nil.hpp b/cpp/msgpack/type/nil.hpp similarity index 100% rename from cpp/type/nil.hpp rename to cpp/msgpack/type/nil.hpp diff --git a/cpp/type/pair.hpp b/cpp/msgpack/type/pair.hpp similarity index 100% rename from cpp/type/pair.hpp rename to cpp/msgpack/type/pair.hpp diff --git a/cpp/type/raw.hpp b/cpp/msgpack/type/raw.hpp similarity index 100% rename from cpp/type/raw.hpp rename to cpp/msgpack/type/raw.hpp diff --git a/cpp/type/set.hpp b/cpp/msgpack/type/set.hpp similarity index 100% rename from cpp/type/set.hpp rename to cpp/msgpack/type/set.hpp diff --git a/cpp/type/string.hpp b/cpp/msgpack/type/string.hpp similarity index 100% rename from cpp/type/string.hpp rename to cpp/msgpack/type/string.hpp diff --git a/cpp/type/tr1/unordered_map.hpp b/cpp/msgpack/type/tr1/unordered_map.hpp similarity index 100% rename from cpp/type/tr1/unordered_map.hpp rename to cpp/msgpack/type/tr1/unordered_map.hpp diff --git a/cpp/type/tr1/unordered_set.hpp b/cpp/msgpack/type/tr1/unordered_set.hpp similarity index 100% rename from cpp/type/tr1/unordered_set.hpp rename to cpp/msgpack/type/tr1/unordered_set.hpp diff --git a/cpp/type/tuple.hpp.erb b/cpp/msgpack/type/tuple.hpp.erb similarity index 100% rename from cpp/type/tuple.hpp.erb rename to cpp/msgpack/type/tuple.hpp.erb diff --git a/cpp/type/vector.hpp b/cpp/msgpack/type/vector.hpp similarity index 100% rename from cpp/type/vector.hpp rename to cpp/msgpack/type/vector.hpp diff --git a/cpp/unpack.hpp b/cpp/msgpack/unpack.hpp similarity index 100% rename from cpp/unpack.hpp rename to cpp/msgpack/unpack.hpp diff --git a/cpp/vrefbuffer.hpp b/cpp/msgpack/vrefbuffer.hpp similarity index 100% rename from cpp/vrefbuffer.hpp rename to cpp/msgpack/vrefbuffer.hpp diff --git a/cpp/zone.hpp.erb b/cpp/msgpack/zone.hpp.erb similarity index 100% rename from cpp/zone.hpp.erb rename to cpp/msgpack/zone.hpp.erb From d5609f3207ee8b98dc1e2032cd31a23380ad5402 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 6 Feb 2010 21:28:27 +0900 Subject: [PATCH 0368/1648] ruby: makegem.sh --- ruby/{gem => }/README | 0 ruby/gem/Rakefile | 133 ------------------------- ruby/gengem.sh | 21 ---- ruby/makegem.sh | 21 ++++ ruby/msgpack.gemspec | 26 +++-- ruby/{test_case.rb => msgpack_test.rb} | 0 ruby/{gem => }/test/test_helper.rb | 0 ruby/test_format.rb | 122 ----------------------- ruby/test_pack.rb | 58 ----------- 9 files changed, 36 insertions(+), 345 deletions(-) rename ruby/{gem => }/README (100%) delete mode 100644 ruby/gem/Rakefile delete mode 100755 ruby/gengem.sh create mode 100755 ruby/makegem.sh mode change 100755 => 100644 ruby/msgpack.gemspec rename ruby/{test_case.rb => msgpack_test.rb} (100%) rename ruby/{gem => }/test/test_helper.rb (100%) delete mode 100644 ruby/test_format.rb delete mode 100644 ruby/test_pack.rb diff --git a/ruby/gem/README b/ruby/README similarity index 100% rename from ruby/gem/README rename to ruby/README diff --git a/ruby/gem/Rakefile b/ruby/gem/Rakefile deleted file mode 100644 index f2fd684..0000000 --- a/ruby/gem/Rakefile +++ /dev/null @@ -1,133 +0,0 @@ -require 'rubygems' -require 'rake' -require 'rake/clean' -require 'rake/testtask' -require 'rake/packagetask' -require 'rake/gempackagetask' -require 'rake/rdoctask' -require 'rake/contrib/rubyforgepublisher' -require 'rake/contrib/sshpublisher' -require 'fileutils' -include FileUtils - -NAME = "msgpack" -AUTHOR = "FURUHASHI Sadayuki" -EMAIL = "frsyuki _at_ users.sourceforge.jp" -DESCRIPTION = "Binary-based efficient data interchange format." -RUBYFORGE_PROJECT = "msgpack" -HOMEPATH = "http://msgpack.sourceforge.jp/" -BIN_FILES = %w( ) -VERS = "0.3.2" - -#REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil -REV = nil -CLEAN.include ['**/.*.sw?', '*.gem', '.config'] -RDOC_OPTS = [ - '--title', "#{NAME} documentation", - "--charset", "utf-8", - "--opname", "index.html", - "--line-numbers", - "--main", "README", - "--inline-source", -] - -task :default => [:test] -task :package => [:clean] - -Rake::TestTask.new("test") do |t| - t.libs << "test" - t.pattern = "test/**/*_test.rb" - t.verbose = true -end - -spec = Gem::Specification.new do |s| - s.name = NAME - s.version = VERS - s.platform = Gem::Platform::RUBY - s.has_rdoc = false - s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] - s.rdoc_options += RDOC_OPTS + ['--exclude', '^(examples|extras)/'] - s.summary = DESCRIPTION - s.description = DESCRIPTION - s.author = AUTHOR - s.email = EMAIL - s.homepage = HOMEPATH - s.executables = BIN_FILES - s.rubyforge_project = RUBYFORGE_PROJECT - s.bindir = "bin" - s.require_path = "ext" - s.autorequire = "" - s.test_files = Dir["test/test_*.rb"] - - #s.add_dependency('activesupport', '>=1.3.1') - #s.required_ruby_version = '>= 1.8.2' - - s.files = %w(README ChangeLog Rakefile) + - Dir.glob("{bin,doc,test,lib,templates,generator,extras,website,script}/**/*") + - Dir.glob("ext/**/*.{h,c,rb}") + - Dir.glob("examples/**/*.rb") + - Dir.glob("tools/*.rb") + - Dir.glob("msgpack/*.h") - - s.extensions = FileList["ext/**/extconf.rb"].to_a -end - -Rake::GemPackageTask.new(spec) do |p| - p.need_tar = true - p.gem_spec = spec -end - -task :install do - name = "#{NAME}-#{VERS}.gem" - sh %{rake package} - sh %{sudo gem install pkg/#{name}} -end - -task :uninstall => [:clean] do - sh %{sudo gem uninstall #{NAME}} -end - - -#Rake::RDocTask.new do |rdoc| -# rdoc.rdoc_dir = 'html' -# rdoc.options += RDOC_OPTS -# rdoc.template = "resh" -# #rdoc.template = "#{ENV['template']}.rb" if ENV['template'] -# if ENV['DOC_FILES'] -# rdoc.rdoc_files.include(ENV['DOC_FILES'].split(/,\s*/)) -# else -# rdoc.rdoc_files.include('README', 'ChangeLog') -# rdoc.rdoc_files.include('lib/**/*.rb') -# rdoc.rdoc_files.include('ext/**/*.c') -# end -#end - -desc "Publish to RubyForge" -task :rubyforge => [:rdoc, :package] do - require 'rubyforge' - Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, 'frsyuki').upload -end - -desc 'Package and upload the release to rubyforge.' -task :release => [:clean, :package] do |t| - v = ENV["VERSION"] or abort "Must supply VERSION=x.y.z" - abort "Versions don't match #{v} vs #{VERS}" unless v == VERS - pkg = "pkg/#{NAME}-#{VERS}" - - rf = RubyForge.new - puts "Logging in" - rf.login - - c = rf.userconfig -# c["release_notes"] = description if description -# c["release_changes"] = changes if changes - c["preformatted"] = true - - files = [ - "#{pkg}.tgz", - "#{pkg}.gem" - ].compact - - puts "Releasing #{NAME} v. #{VERS}" - rf.add_release RUBYFORGE_PROJECT, NAME, VERS, *files -end diff --git a/ruby/gengem.sh b/ruby/gengem.sh deleted file mode 100755 index 359debf..0000000 --- a/ruby/gengem.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -mkdir -p gem/ext -mkdir -p gem/msgpack -cp extconf.rb gem/ext/ -cp pack.c gem/ext/ -cp pack.h gem/ext/ -cp rbinit.c gem/ext/ -cp unpack.c gem/ext/ -cp unpack.h gem/ext/ -cat test_case.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > gem/test/msgpack_test.rb -cp ../AUTHORS gem/AUTHORS -cp ../ChangeLog gem/ChangeLog -cp ../msgpack/pack_define.h gem/msgpack/ -cp ../msgpack/pack_template.h gem/msgpack/ -cp ../msgpack/unpack_define.h gem/msgpack/ -cp ../msgpack/unpack_template.h gem/msgpack/ -cp ../msgpack/sysdep.h gem/msgpack/ - -cd gem && rake --trace package - diff --git a/ruby/makegem.sh b/ruby/makegem.sh new file mode 100755 index 0000000..fd0db79 --- /dev/null +++ b/ruby/makegem.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +mkdir -p ext +mkdir -p msgpack +cp extconf.rb ext/ +cp pack.c ext/ +cp pack.h ext/ +cp rbinit.c ext/ +cp unpack.c ext/ +cp unpack.h ext/ +cp ../AUTHORS ./ +cp ../ChangeLog ./ +cp ../msgpack/pack_define.h msgpack/ +cp ../msgpack/pack_template.h msgpack/ +cp ../msgpack/unpack_define.h msgpack/ +cp ../msgpack/unpack_template.h msgpack/ +cp ../msgpack/sysdep.h msgpack/ +cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > test/msgpack_test.rb + +gem build msgpack.gemspec + diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec old mode 100755 new mode 100644 index 59186a4..7273881 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,12 +1,16 @@ -Gem::Specification.new do |s| - s.platform = Gem::Platform::CURRENT - s.name = "msgpack" - s.version = "0.3.2" - s.summary = "MessagePack" - s.author = "FURUHASHI Sadayuki" - s.email = "frsyuki@users.sourceforge.jp" - s.homepage = "http://msgpack.sourceforge.jp/" - s.rubyforge_project = "msgpack" - s.require_paths = ["lib", "ext"] - s.files = ["lib/**/*", "ext/**/*"].map {|g| Dir.glob(g) }.flatten +Gem::Specification.new do |s| + s.platform = Gem::Platform::RUBY + s.name = "msgpack" + s.version = "0.3.3" + s.summary = "MessagePack, a binary-based efficient data interchange format." + s.author = "FURUHASHI Sadayuki" + s.email = "frsyuki@users.sourceforge.jp" + s.homepage = "http://msgpack.sourceforge.jp/" + s.rubyforge_project = "msgpack" + s.has_rdoc = false + s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] + s.require_paths = ["lib", "ext"] + s.files = Dir["lib/**/*", "ext/**/*", "msgpack/**/*", "test/**/*"] + s.test_files = Dir["test/test_*.rb"] + s.extensions = Dir["ext/**/extconf.rb"] end diff --git a/ruby/test_case.rb b/ruby/msgpack_test.rb similarity index 100% rename from ruby/test_case.rb rename to ruby/msgpack_test.rb diff --git a/ruby/gem/test/test_helper.rb b/ruby/test/test_helper.rb similarity index 100% rename from ruby/gem/test/test_helper.rb rename to ruby/test/test_helper.rb diff --git a/ruby/test_format.rb b/ruby/test_format.rb deleted file mode 100644 index 99a27d1..0000000 --- a/ruby/test_format.rb +++ /dev/null @@ -1,122 +0,0 @@ -require 'msgpack' - -@up = MessagePack::Unpacker.new - -def check(bytes, should) - puts "----" - @up.reset - src = bytes.pack('C*') - ret = @up.execute(src, 0) - if ret != src.length - puts "** EXTRA BYTES **" - end - puts bytes.map{|x|"%x"%x}.join(' ') - data = @up.data - p data - if data != should - puts "** TEST FAILED **" - p should - end -end - -# SimpleValue -check([ - 0x93, 0xc0, 0xc2, 0xc3, -], [nil,false,true]) - -# Fixnum -check([ - 0x92, - 0x93, 0x00, 0x40, 0x7f, - 0x93, 0xe0, 0xf0, 0xff, -], [[0,64,127], [-32,-16,-1]]) - -# FixArray -check([ - 0x92, - 0x90, - 0x91, - 0x91, 0xc0, -], [[],[[nil]]]) - - -# FixRaw -check([ - 0x94, - 0xa0, - 0xa1, ?a, - 0xa2, ?b, ?c, - 0xa3, ?d, ?e, ?f, -], ["","a","bc","def"]) - -# FixMap -check([ - 0x82, - 0xc2, 0x81, - 0xc0, 0xc0, - 0xc3, 0x81, - 0xc0, 0x80, -], {false=>{nil=>nil}, true=>{nil=>{}}}) - -# unsigned int -check([ - 0x99, - 0xcc, 0, - 0xcc, 128, - 0xcc, 255, - 0xcd, 0x00, 0x00, - 0xcd, 0x80, 0x00, - 0xcd, 0xff, 0xff, - 0xce, 0x00, 0x00, 0x00, 0x00, - 0xce, 0x80, 0x00, 0x00, 0x00, - 0xce, 0xff, 0xff, 0xff, 0xff, -], [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295]) - -# signed int -check([ - 0x99, - 0xd0, 0, - 0xd0, 128, - 0xd0, 255, - 0xd1, 0x00, 0x00, - 0xd1, 0x80, 0x00, - 0xd1, 0xff, 0xff, - 0xd2, 0x00, 0x00, 0x00, 0x00, - 0xd2, 0x80, 0x00, 0x00, 0x00, - 0xd2, 0xff, 0xff, 0xff, 0xff, -], [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1]) - -# raw -check([ - 0x96, - 0xda, 0x00, 0x00, - 0xda, 0x00, 0x01, ?a, - 0xda, 0x00, 0x02, ?a, ?b, - 0xdb, 0x00, 0x00, 0x00, 0x00, - 0xdb, 0x00, 0x00, 0x00, 0x01, ?a, - 0xdb, 0x00, 0x00, 0x00, 0x02, ?a, ?b, -], ["", "a", "ab", "", "a", "ab"]) - -# array -check([ - 0x96, - 0xdc, 0x00, 0x00, - 0xdc, 0x00, 0x01, 0xc0, - 0xdc, 0x00, 0x02, 0xc2, 0xc3, - 0xdd, 0x00, 0x00, 0x00, 0x00, - 0xdd, 0x00, 0x00, 0x00, 0x01, 0xc0, - 0xdd, 0x00, 0x00, 0x00, 0x02, 0xc2, 0xc3 -], [[], [nil], [false,true], [], [nil], [false,true]]) - -# map -check([ - 0x96, - 0xde, 0x00, 0x00, - 0xde, 0x00, 0x01, 0xc0, 0xc2, - 0xde, 0x00, 0x02, 0xc0, 0xc2, 0xc3, 0xc2, - 0xdf, 0x00, 0x00, 0x00, 0x00, - 0xdf, 0x00, 0x00, 0x00, 0x01, 0xc0, 0xc2, - 0xdf, 0x00, 0x00, 0x00, 0x02, 0xc0, 0xc2, 0xc3, 0xc2, -], [{}, {nil=>false}, {true=>false, nil=>false}, {}, {nil=>false}, {true=>false, nil=>false}]) - - diff --git a/ruby/test_pack.rb b/ruby/test_pack.rb deleted file mode 100644 index 6873d57..0000000 --- a/ruby/test_pack.rb +++ /dev/null @@ -1,58 +0,0 @@ -require 'msgpack' - -def check(data) - puts "---" - pack = data.to_msgpack - p data - puts pack.unpack('C*').map{|x|"%02x"%x}.join(' ') - re = MessagePack::unpack(pack) - if re != data - p re - puts "** TEST FAILED **" - end -end - -check 0 -check 1 -check 127 -check 128 -check 255 -check 256 -check 65535 -check 65536 -check -1 -check -32 -check -33 -check -128 -check -129 -check -32768 -check -32769 - -check 1.0 - -check "" -check "a" -check "a"*31 -check "a"*32 - -check nil -check true -check false - -check [] -check [[]] -check [[], nil] - -check( {nil=>0} ) - -check (1<<23) -__END__ - -ary = [] -i = 0 -while i < (1<<16) - ary << i - i += 1 -end -check ary - From 7df178085341a5a4845d391ef7ec8317da3cc0ec Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 6 Feb 2010 21:31:16 +0900 Subject: [PATCH 0369/1648] ruby: makegem.sh --- README | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README b/README index 66ca276..6ca10d6 100644 --- a/README +++ b/README @@ -19,12 +19,12 @@ Binary-based efficient data interchange format. $ make $ sudo make install - To install Ruby binding, run ./gengem.sh script on ruby/ directory and install + To install Ruby binding, run ./makegem.sh script on ruby/ directory and install generated gem package. $ cd ruby - $ ./gengem.sh - $ gem install gem/pkg/msgpack-*.gem + $ ./makegem.sh + $ gem install msgpack-*.gem *Usage @@ -54,7 +54,7 @@ Binary-based efficient data interchange format. API Document is available at http://msgpack.sourceforge.jp/. -Copyright (C) 2008-2009 FURUHASHI Sadayuki +Copyright (C) 2008-2010 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 27787505e0897eae05ee939ad846427cf1894c99 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 27 Feb 2010 02:37:01 +0900 Subject: [PATCH 0370/1648] strict-aliasing rule 2 --- COPYING | 2 +- msgpack/pack_define.h | 3 +- msgpack/pack_template.h | 66 +++++++++++++++++++-------------------- msgpack/sysdep.h | 15 ++++++++- msgpack/unpack_define.h | 2 +- msgpack/unpack_template.h | 51 +++++++++++++----------------- 6 files changed, 72 insertions(+), 67 deletions(-) diff --git a/COPYING b/COPYING index 6f5f220..4388e8f 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,4 @@ -Copyright (C) 2008-2009 FURUHASHI Sadayuki +Copyright (C) 2008-2010 FURUHASHI Sadayuki Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h index 692ef7d..4845d52 100644 --- a/msgpack/pack_define.h +++ b/msgpack/pack_define.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ #include "msgpack/sysdep.h" #include +#include #endif /* msgpack/pack_define.h */ diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index b314d6d..493634d 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -1,7 +1,7 @@ /* * MessagePack packing routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -69,7 +69,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } while(0) @@ -89,12 +89,12 @@ do { \ if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -115,17 +115,17 @@ do { \ if(d < (1ULL<<16)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* signed 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ + buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -149,7 +149,7 @@ do { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -167,7 +167,7 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } \ @@ -179,12 +179,12 @@ do { \ if(d < -(1<<15)) { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xd2; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -202,12 +202,12 @@ do { \ } else if(d < (1<<16)) { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } \ @@ -220,19 +220,19 @@ do { \ if(d < -(1LL<<31)) { \ /* signed 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ + buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 9); \ } else { \ /* signed 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xd2; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } \ } else { \ if(d < -(1<<7)) { \ /* signed 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else { \ /* signed 8 */ \ @@ -252,19 +252,19 @@ do { \ } else { \ /* unsigned 16 */ \ unsigned char buf[3]; \ - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \ + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } \ } else { \ if(d < (1LL<<32)) { \ /* unsigned 32 */ \ unsigned char buf[5]; \ - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \ + buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ /* unsigned 64 */ \ unsigned char buf[9]; \ - buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \ + buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 9); \ } \ } \ @@ -283,21 +283,21 @@ msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) { unsigned char buf[3]; - buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); + buf[0] = 0xcd; _msgpack_store16(&buf[1], d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) { unsigned char buf[5]; - buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); + buf[0] = 0xce; _msgpack_store32(&buf[1], d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) { unsigned char buf[9]; - buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); + buf[0] = 0xcf; _msgpack_store64(&buf[1], d); msgpack_pack_append_buffer(x, buf, 9); } @@ -310,21 +310,21 @@ msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) { unsigned char buf[3]; - buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); + buf[0] = 0xd1; _msgpack_store16(&buf[1], d); msgpack_pack_append_buffer(x, buf, 3); } msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) { unsigned char buf[5]; - buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); + buf[0] = 0xd2; _msgpack_store32(&buf[1], d); msgpack_pack_append_buffer(x, buf, 5); } msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) { unsigned char buf[9]; - buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); + buf[0] = 0xd3; _msgpack_store64(&buf[1], d); msgpack_pack_append_buffer(x, buf, 9); } @@ -557,7 +557,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) union { float f; uint32_t i; } mem; mem.f = d; unsigned char buf[5]; - buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(mem.i); + buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i); msgpack_pack_append_buffer(x, buf, 5); } @@ -566,7 +566,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) union { double f; uint64_t i; } mem; mem.f = d; unsigned char buf[9]; - buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(mem.i); + buf[0] = 0xcb; _msgpack_store64(&buf[1], mem.i); msgpack_pack_append_buffer(x, buf, 9); } @@ -610,11 +610,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &d, 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n); + buf[0] = 0xdc; _msgpack_store16(&buf[1], n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n); + buf[0] = 0xdd; _msgpack_store32(&buf[1], n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -631,11 +631,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(n < 65536) { unsigned char buf[3]; - buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n); + buf[0] = 0xde; _msgpack_store16(&buf[1], n); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n); + buf[0] = 0xdf; _msgpack_store32(&buf[1], n); msgpack_pack_append_buffer(x, buf, 5); } } @@ -652,11 +652,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); } else if(l < 65536) { unsigned char buf[3]; - buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l); + buf[0] = 0xda; _msgpack_store16(&buf[1], l); msgpack_pack_append_buffer(x, buf, 3); } else { unsigned char buf[5]; - buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l); + buf[0] = 0xdb; _msgpack_store32(&buf[1], l); msgpack_pack_append_buffer(x, buf, 5); } } diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h index c69c69a..2bc01c9 100644 --- a/msgpack/sysdep.h +++ b/msgpack/sysdep.h @@ -1,7 +1,7 @@ /* * MessagePack system dependencies * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -101,5 +101,18 @@ typedef unsigned int _msgpack_atomic_counter_t; #endif +#define _msgpack_store16(to, num) \ + do { uint16_t val = _msgpack_be16(num); memcpy(to, &val, 2); } while(0); +#define _msgpack_store32(to, num) \ + do { uint32_t val = _msgpack_be32(num); memcpy(to, &val, 4); } while(0); +#define _msgpack_store64(to, num) \ + do { uint64_t val = _msgpack_be64(num); memcpy(to, &val, 8); } while(0); + + +#define _msgpack_load16(cast, from) ((cast)_msgpack_be16(*(uint16_t*)from)) +#define _msgpack_load32(cast, from) ((cast)_msgpack_be32(*(uint32_t*)from)) +#define _msgpack_load64(cast, from) ((cast)_msgpack_be64(*(uint64_t*)from)) + + #endif /* msgpack/sysdep.h */ diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index c36aa51..e61f73d 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index cf2e74c..3f871cd 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -1,7 +1,7 @@ /* * MessagePack unpacking routine template * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -130,11 +130,6 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c #define NEXT_CS(p) \ ((unsigned int)*p & 0x1f) -#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) -#define PTR_CAST_16(ptr) _msgpack_be16(*(uint16_t*)ptr) -#define PTR_CAST_32(ptr) _msgpack_be32(*(uint32_t*)ptr) -#define PTR_CAST_64(ptr) _msgpack_be64(*(uint64_t*)ptr) - #ifdef USE_CASE_RANGE #define SWITCH_RANGE_BEGIN switch(*p) { #define SWITCH_RANGE(FROM, TO) case FROM ... TO: @@ -223,69 +218,69 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c //case CS_ case CS_FLOAT: { union { uint32_t i; float f; } mem; - mem.i = PTR_CAST_32(n); + mem.i = _msgpack_load32(uint32_t,n); push_fixed_value(_float, mem.f); } case CS_DOUBLE: { union { uint64_t i; double f; } mem; - mem.i = PTR_CAST_64(n); + mem.i = _msgpack_load64(uint64_t,n); push_fixed_value(_double, mem.f); } case CS_UINT_8: - push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); + push_fixed_value(_uint8, *(uint8_t*)n); case CS_UINT_16: - push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); + push_fixed_value(_uint16, _msgpack_load16(uint16_t,n)); case CS_UINT_32: - push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); + push_fixed_value(_uint32, _msgpack_load32(uint32_t,n)); case CS_UINT_64: - push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); + push_fixed_value(_uint64, _msgpack_load64(uint64_t,n)); case CS_INT_8: - push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); + push_fixed_value(_int8, *(int8_t*)n); case CS_INT_16: - push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); + push_fixed_value(_int16, _msgpack_load16(int16_t,n)); case CS_INT_32: - push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); + push_fixed_value(_int32, _msgpack_load32(int32_t,n)); case CS_INT_64: - push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); + push_fixed_value(_int64, _msgpack_load64(int64_t,n)); //case CS_ //case CS_ //case CS_BIG_INT_16: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero); //case CS_BIG_INT_32: - // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero); //case ACS_BIG_INT_VALUE: //_big_int_zero: // // FIXME // push_variable_value(_big_int, data, n, trail); //case CS_BIG_FLOAT_16: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero); //case CS_BIG_FLOAT_32: - // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero); //case ACS_BIG_FLOAT_VALUE: //_big_float_zero: // // FIXME // push_variable_value(_big_float, data, n, trail); case CS_RAW_16: - again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero); case CS_RAW_32: - again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero); case ACS_RAW_VALUE: _raw_zero: push_variable_value(_raw, data, n, trail); case CS_ARRAY_16: - start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM); case CS_ARRAY_32: /* FIXME security guard */ - start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM); case CS_MAP_16: - start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY); case CS_MAP_32: /* FIXME security guard */ - start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY); default: goto _failed; @@ -371,8 +366,4 @@ _end: #undef start_container #undef NEXT_CS -#undef PTR_CAST_8 -#undef PTR_CAST_16 -#undef PTR_CAST_32 -#undef PTR_CAST_64 From 05a7e4eb64f2fa616c96878d586322d869c7ff9b Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 2 Mar 2010 17:09:59 +0900 Subject: [PATCH 0371/1648] ruby: add msgpack.mingw.{gemspec,sh} --- c/unpack.c | 2 +- configure.in | 2 +- ruby/makegem.sh | 5 +++++ ruby/msgpack.gemspec | 10 +++++----- ruby/msgpack.mingw.gemspec | 16 ++++++++++++++++ ruby/msgpack.mingw.sh | 21 +++++++++++++++++++++ 6 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 ruby/msgpack.mingw.gemspec create mode 100755 ruby/msgpack.mingw.sh diff --git a/c/unpack.c b/c/unpack.c index d5bcb2d..140ad1f 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -52,7 +52,7 @@ static int template_execute(template_context* ctx, static inline msgpack_object template_callback_root(unpack_user* u) -{ msgpack_object o; return o; } +{ msgpack_object o = {0}; return o; } static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_object* o) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } diff --git a/configure.in b/configure.in index 259ed37..c75957b 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.4.1) +AM_INIT_AUTOMAKE(msgpack, 0.4.2) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/ruby/makegem.sh b/ruby/makegem.sh index fd0db79..622264d 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -19,3 +19,8 @@ cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FI gem build msgpack.gemspec +# ./msgpack.mingw.sh # on msys +# gem push msgpack-$version.gem +# gem push msgpack-$version-x86-mingw32.gem +# gem push msgpack-$version-mswin32.gem + diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 7273881..4575c70 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,5 +1,5 @@ Gem::Specification.new do |s| - s.platform = Gem::Platform::RUBY + s.platform = Gem::Platform::RUBY s.name = "msgpack" s.version = "0.3.3" s.summary = "MessagePack, a binary-based efficient data interchange format." @@ -7,10 +7,10 @@ Gem::Specification.new do |s| s.email = "frsyuki@users.sourceforge.jp" s.homepage = "http://msgpack.sourceforge.jp/" s.rubyforge_project = "msgpack" - s.has_rdoc = false - s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] + s.has_rdoc = false + s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] s.require_paths = ["lib", "ext"] s.files = Dir["lib/**/*", "ext/**/*", "msgpack/**/*", "test/**/*"] - s.test_files = Dir["test/test_*.rb"] - s.extensions = Dir["ext/**/extconf.rb"] + s.test_files = Dir["test/test_*.rb"] + s.extensions = Dir["ext/**/extconf.rb"] end diff --git a/ruby/msgpack.mingw.gemspec b/ruby/msgpack.mingw.gemspec new file mode 100644 index 0000000..3d2e7dc --- /dev/null +++ b/ruby/msgpack.mingw.gemspec @@ -0,0 +1,16 @@ +Gem::Specification.new do |s| + s.platform = Gem::Platform::CURRENT + s.name = "msgpack" + s.version = "0.3.3" + s.summary = "MessagePack, a binary-based efficient data interchange format." + s.author = "FURUHASHI Sadayuki" + s.email = "frsyuki@users.sourceforge.jp" + s.homepage = "http://msgpack.sourceforge.jp/" + s.rubyforge_project = "msgpack" + s.has_rdoc = false + s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] + s.require_paths = ["lib", "ext"] + s.files = Dir["lib/**/*", "ext/**/*", "msgpack/**/*", "test/**/*"] + s.test_files = Dir["test/test_*.rb"] + s.extensions = Dir["ext/**/extconf.rb"] +end diff --git a/ruby/msgpack.mingw.sh b/ruby/msgpack.mingw.sh new file mode 100755 index 0000000..ec75234 --- /dev/null +++ b/ruby/msgpack.mingw.sh @@ -0,0 +1,21 @@ +#!/bin/sh +if [ -z "$1" ];then + echo "usage: $0 " + exit 1 +fi + +version=$1 +build=msgpack-mingw-build + +./makegem.sh +gem build msgpack.mingw.gemspec +rm -rf $build +mkdir $build +cd $build +tar xvf ../msgpack-$version-x86-mingw32.gem +gunzip metadata.gz +sed s/x86-mingw32/mswin32/ metadata > metadata.tmp +mv metadata.tmp metadata +gzip metadata +tar cvf msgpack-$version-mswin32.gem metadata.gz data.tar.gz + From 568704ed2256047fbf69657b5cc1e02f76e71855 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 19 Mar 2010 13:34:53 +0900 Subject: [PATCH 0372/1648] fixes Visual C++ compatibility --- Makefile.am | 5 + c/msgpack/zone.h | 2 - c/unpack.c | 2 +- c/vrefbuffer.c | 1 - configure.in | 2 +- msgpack_vc8.postbuild.bat | 41 +++ msgpack_vc8.sln | 20 ++ msgpack_vc8.vcproj | 558 +++++++++++++++++++------------------- 8 files changed, 351 insertions(+), 280 deletions(-) create mode 100644 msgpack_vc8.postbuild.bat create mode 100644 msgpack_vc8.sln mode change 100644 => 100755 msgpack_vc8.vcproj diff --git a/Makefile.am b/Makefile.am index 42fb233..04f62d2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -11,3 +11,8 @@ nobase_include_HEADERS = \ msgpack/unpack_template.h \ msgpack/sysdep.h +EXTRA_DIST = \ + msgpack_vc8.vcproj \ + msgpack_vc8.sln \ + msgpack_vc8.postbuild.bat + diff --git a/c/msgpack/zone.h b/c/msgpack/zone.h index 475ae51..ce5be6d 100644 --- a/c/msgpack/zone.h +++ b/c/msgpack/zone.h @@ -19,8 +19,6 @@ #define MSGPACK_ZONE_H__ #include "msgpack/sysdep.h" -#include -#include #ifdef __cplusplus extern "C" { diff --git a/c/unpack.c b/c/unpack.c index 140ad1f..8be0b92 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -52,7 +52,7 @@ static int template_execute(template_context* ctx, static inline msgpack_object template_callback_root(unpack_user* u) -{ msgpack_object o = {0}; return o; } +{ msgpack_object o = {}; return o; } static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_object* o) { o->type = MSGPACK_OBJECT_POSITIVE_INTEGER; o->via.u64 = d; return 0; } diff --git a/c/vrefbuffer.c b/c/vrefbuffer.c index b528874..136372f 100644 --- a/c/vrefbuffer.c +++ b/c/vrefbuffer.c @@ -30,7 +30,6 @@ bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf, vbuf->chunk_size = chunk_size; vbuf->ref_size = ref_size; - // glibcã¯72ãƒã‚¤ãƒˆä»¥ä¸‹ã®mallocãŒé«˜é€Ÿ size_t nfirst = (sizeof(struct iovec) < 72/2) ? 72 / sizeof(struct iovec) : 8; diff --git a/configure.in b/configure.in index c75957b..0f2c5d0 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(msgpack/unpack_template.h) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.4.2) +AM_INIT_AUTOMAKE(msgpack, 0.4.3) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) diff --git a/msgpack_vc8.postbuild.bat b/msgpack_vc8.postbuild.bat new file mode 100644 index 0000000..7ee2586 --- /dev/null +++ b/msgpack_vc8.postbuild.bat @@ -0,0 +1,41 @@ +IF NOT EXIST include MKDIR include +IF NOT EXIST include\msgpack MKDIR include\msgpack +IF NOT EXIST include\msgpack\type MKDIR include\msgpack\type +IF NOT EXIST include\msgpack\type\tr1 MKDIR include\msgpack\type\tr1 +copy msgpack\pack_define.h include\msgpack\ +copy msgpack\pack_template.h include\msgpack\ +copy msgpack\unpack_define.h include\msgpack\ +copy msgpack\unpack_template.h include\msgpack\ +copy msgpack\sysdep.h include\msgpack\ +copy c\msgpack.h include\ +copy c\msgpack\sbuffer.h include\msgpack\ +copy c\msgpack\vrefbuffer.h include\msgpack\ +copy c\msgpack\pack.h include\msgpack\ +copy c\msgpack\unpack.h include\msgpack\ +copy c\msgpack\object.h include\msgpack\ +copy c\msgpack\zone.h include\msgpack\ +copy cpp\msgpack.hpp include\ +copy cpp\msgpack\sbuffer.hpp include\msgpack\ +copy cpp\msgpack\vrefbuffer.hpp include\msgpack\ +copy cpp\msgpack\pack.hpp include\msgpack\ +copy cpp\msgpack\unpack.hpp include\msgpack\ +copy cpp\msgpack\object.hpp include\msgpack\ +copy cpp\msgpack\zone.hpp include\msgpack\ +copy cpp\msgpack\type.hpp include\msgpack\type\ +copy cpp\msgpack\type\bool.hpp include\msgpack\type\ +copy cpp\msgpack\type\float.hpp include\msgpack\type\ +copy cpp\msgpack\type\int.hpp include\msgpack\type\ +copy cpp\msgpack\type\list.hpp include\msgpack\type\ +copy cpp\msgpack\type\deque.hpp include\msgpack\type\ +copy cpp\msgpack\type\map.hpp include\msgpack\type\ +copy cpp\msgpack\type\nil.hpp include\msgpack\type\ +copy cpp\msgpack\type\pair.hpp include\msgpack\type\ +copy cpp\msgpack\type\raw.hpp include\msgpack\type\ +copy cpp\msgpack\type\set.hpp include\msgpack\type\ +copy cpp\msgpack\type\string.hpp include\msgpack\type\ +copy cpp\msgpack\type\vector.hpp include\msgpack\type\ +copy cpp\msgpack\type\tuple.hpp include\msgpack\type\ +copy cpp\msgpack\type\define.hpp include\msgpack\type\ +copy cpp\msgpack\type\tr1\unordered_map.hpp include\msgpack\type\ +copy cpp\msgpack\type\tr1\unordered_set.hpp include\msgpack\type\ + diff --git a/msgpack_vc8.sln b/msgpack_vc8.sln new file mode 100644 index 0000000..84718af --- /dev/null +++ b/msgpack_vc8.sln @@ -0,0 +1,20 @@ + +Microsoft Visual Studio Solution File, Format Version 9.00 +# Visual C++ Express 2008 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "MessagePack", "msgpack_vc8.vcproj", "{122A2EA4-B283-4241-9655-786DE78283B2}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {122A2EA4-B283-4241-9655-786DE78283B2}.Debug|Win32.ActiveCfg = Debug|Win32 + {122A2EA4-B283-4241-9655-786DE78283B2}.Debug|Win32.Build.0 = Debug|Win32 + {122A2EA4-B283-4241-9655-786DE78283B2}.Release|Win32.ActiveCfg = Release|Win32 + {122A2EA4-B283-4241-9655-786DE78283B2}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/msgpack_vc8.vcproj b/msgpack_vc8.vcproj old mode 100644 new mode 100755 index 5daec01..a3fa28f --- a/msgpack_vc8.vcproj +++ b/msgpack_vc8.vcproj @@ -1,275 +1,283 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 1b1433a6642036d51e49e35b64b3534a38d95916 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 26 Mar 2010 14:29:49 +0900 Subject: [PATCH 0373/1648] ruby: copy the deserialized string if length <= RSTRING_EMBED_LEN_MAX --- ruby/unpack.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index 0f1af38..1748329 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -110,8 +110,14 @@ static inline int template_callback_map(unpack_user* u, unsigned int n, VALUE* o static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, VALUE v) { rb_hash_aset(*c, k, v); return 0; } +#ifdef RSTRING_EMBED_LEN_MAX +#define COW_MIN_SIZE RSTRING_EMBED_LEN_MAX +#else +#define COW_MIN_SIZE ((sizeof(VALUE)*3)/sizeof(char)-1) +#endif + static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) -{ *o = (l == 0) ? rb_str_new(0,0) : rb_str_substr(u->source, p - b, l); return 0; } +{ *o = (l <= COW_MIN_SIZE) ? rb_str_new(p, l) : rb_str_substr(u->source, p - b, l); return 0; } #include "msgpack/unpack_template.h" From 72e3f9821300f53d988e546a8dc7e001d9acb717 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 26 Mar 2010 14:30:50 +0900 Subject: [PATCH 0374/1648] cpp: alias pack(Stream* s, const T& v) -> pack(Stream& const T& v) --- c/msgpack/sbuffer.h | 3 ++- c/msgpack/vrefbuffer.h | 12 ++++++------ cpp/msgpack/pack.hpp | 10 ++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/c/msgpack/sbuffer.h b/c/msgpack/sbuffer.h index b0fab46..bc0a8fd 100644 --- a/c/msgpack/sbuffer.h +++ b/c/msgpack/sbuffer.h @@ -22,13 +22,14 @@ #include #ifndef MSGPACK_SBUFFER_INIT_SIZE -#define MSGPACK_SBUFFER_INIT_SIZE 2048 +#define MSGPACK_SBUFFER_INIT_SIZE 8192 #endif #ifdef __cplusplus extern "C" { #endif + typedef struct msgpack_sbuffer { size_t size; char* data; diff --git a/c/msgpack/vrefbuffer.h b/c/msgpack/vrefbuffer.h index ffc52bf..38ead67 100644 --- a/c/msgpack/vrefbuffer.h +++ b/c/msgpack/vrefbuffer.h @@ -29,12 +29,6 @@ struct iovec { }; #endif - -#ifdef __cplusplus -extern "C" { -#endif - - #ifndef MSGPACK_VREFBUFFER_REF_SIZE #define MSGPACK_VREFBUFFER_REF_SIZE 32 #endif @@ -43,6 +37,11 @@ extern "C" { #define MSGPACK_VREFBUFFER_CHUNK_SIZE 8192 #endif +#ifdef __cplusplus +extern "C" { +#endif + + struct msgpack_vrefbuffer_chunk; typedef struct msgpack_vrefbuffer_chunk msgpack_vrefbuffer_chunk; @@ -102,6 +101,7 @@ size_t msgpack_vrefbuffer_veclen(const msgpack_vrefbuffer* vref) return vref->tail - vref->array; } + #ifdef __cplusplus } #endif diff --git a/cpp/msgpack/pack.hpp b/cpp/msgpack/pack.hpp index 9c291c1..b7b3ca1 100644 --- a/cpp/msgpack/pack.hpp +++ b/cpp/msgpack/pack.hpp @@ -28,6 +28,7 @@ namespace msgpack { template class packer { public: + packer(Stream* s); packer(Stream& s); ~packer(); @@ -111,6 +112,12 @@ private: }; +template +inline void pack(Stream* s, const T& v) +{ + packer(s).pack(v); +} + template inline void pack(Stream& s, const T& v) { @@ -133,6 +140,9 @@ inline void pack(Stream& s, const T& v) #include "msgpack/pack_template.h" +template +packer::packer(Stream* s) : m_stream(*s) { } + template packer::packer(Stream& s) : m_stream(s) { } From 5782ab7ccce243b1eac0fd6e10928d1aedcb552c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 26 Mar 2010 14:33:49 +0900 Subject: [PATCH 0375/1648] c,cpp: add msgpack_zbuffer and msgpack::zbuffer --- c/Makefile.am | 1 + c/msgpack/zbuffer.h | 175 ++++++++++++++++++++++++++++++++++++++++ cpp/Makefile.am | 1 + cpp/msgpack/zbuffer.hpp | 100 +++++++++++++++++++++++ 4 files changed, 277 insertions(+) create mode 100644 c/msgpack/zbuffer.h create mode 100644 cpp/msgpack/zbuffer.hpp diff --git a/c/Makefile.am b/c/Makefile.am index f84759a..bbe547c 100644 --- a/c/Makefile.am +++ b/c/Makefile.am @@ -10,6 +10,7 @@ nobase_include_HEADERS = \ msgpack.h \ msgpack/sbuffer.h \ msgpack/vrefbuffer.h \ + msgpack/zbuffer.h \ msgpack/pack.h \ msgpack/unpack.h \ msgpack/object.h \ diff --git a/c/msgpack/zbuffer.h b/c/msgpack/zbuffer.h new file mode 100644 index 0000000..ad3d11a --- /dev/null +++ b/c/msgpack/zbuffer.h @@ -0,0 +1,175 @@ +/* + * MessagePack for C deflate buffer implementation + * + * Copyright (C) 2010 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_ZBUFFER_H__ +#define MSGPACK_ZBUFFER_H__ + +#include "msgpack/sysdep.h" +#include +#include +#include + +#ifndef MSGPACK_ZBUFFER_INIT_SIZE +#define MSGPACK_ZBUFFER_INIT_SIZE 8192 +#endif + +#ifndef MSGPACK_ZBUFFER_RESERVE_SIZE +#define MSGPACK_ZBUFFER_RESERVE_SIZE 512 +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + +typedef struct msgpack_zbuffer { + z_stream stream; + char* data; + size_t init_size; +} msgpack_zbuffer; + + +static inline bool msgpack_zbuffer_init(msgpack_zbuffer* zbuf, + int level, size_t init_size); +static inline void msgpack_zbuffer_destroy(msgpack_zbuffer* zbuf); + +static inline char* msgpack_zbuffer_flush(msgpack_zbuffer* zbuf); + +static inline const char* msgpack_zbuffer_data(const msgpack_zbuffer* zbuf); +static inline size_t msgpack_zbuffer_size(const msgpack_zbuffer* zbuf); + +static inline bool msgpack_zbuffer_reset(msgpack_zbuffer* zbuf); +static inline void msgpack_zbuffer_reset_buffer(msgpack_zbuffer* zbuf); +static inline char* msgpack_zbuffer_release_buffer(msgpack_zbuffer* zbuf); + + +static inline int msgpack_zbuffer_write(void* data, const char* buf, unsigned int len); + +static inline bool msgpack_zbuffer_expand(msgpack_zbuffer* zbuf); + + +bool msgpack_zbuffer_init(msgpack_zbuffer* zbuf, + int level, size_t init_size) +{ + memset(zbuf, 0, sizeof(msgpack_zbuffer)); + zbuf->init_size = init_size; + if(deflateInit(&zbuf->stream, level) != Z_OK) { + free(zbuf->data); + return false; + } + return true; +} + +void msgpack_zbuffer_destroy(msgpack_zbuffer* zbuf) +{ + deflateEnd(&zbuf->stream); + free(zbuf->data); +} + +bool msgpack_zbuffer_expand(msgpack_zbuffer* zbuf) +{ + size_t used = (char*)zbuf->stream.next_out - zbuf->data; + size_t csize = used + zbuf->stream.avail_out; + size_t nsize = (csize == 0) ? zbuf->init_size : csize * 2; + + char* tmp = (char*)realloc(zbuf->data, nsize); + if(tmp == NULL) { + return false; + } + + zbuf->data = tmp; + zbuf->stream.next_out = (Bytef*)(tmp + used); + zbuf->stream.avail_out = nsize - used; + + return true; +} + +int msgpack_zbuffer_write(void* data, const char* buf, unsigned int len) +{ + msgpack_zbuffer* zbuf = (msgpack_zbuffer*)data; + + zbuf->stream.next_in = (Bytef*)buf; + zbuf->stream.avail_in = len; + + do { + if(zbuf->stream.avail_out < MSGPACK_ZBUFFER_RESERVE_SIZE) { + msgpack_zbuffer_expand(zbuf); + } + + if(deflate(&zbuf->stream, Z_NO_FLUSH) != Z_OK) { + return -1; + } + } while(zbuf->stream.avail_in > 0); + + return 0; +} + +char* msgpack_zbuffer_flush(msgpack_zbuffer* zbuf) +{ + while(true) { + switch(deflate(&zbuf->stream, Z_FINISH)) { + case Z_STREAM_END: + return zbuf->data; + case Z_OK: + msgpack_zbuffer_expand(zbuf); + default: + return NULL; + } + } +} + +const char* msgpack_zbuffer_data(const msgpack_zbuffer* zbuf) +{ + return zbuf->data; +} + +size_t msgpack_zbuffer_size(const msgpack_zbuffer* zbuf) +{ + return (char*)zbuf->stream.next_out - zbuf->data; +} + +void msgpack_zbuffer_reset_buffer(msgpack_zbuffer* zbuf) +{ + zbuf->stream.avail_out += (char*)zbuf->stream.next_out - zbuf->data; + zbuf->stream.next_out = (Bytef*)zbuf->data; +} + +bool msgpack_zbuffer_reset(msgpack_zbuffer* zbuf) +{ + if(deflateReset(&zbuf->stream) != Z_OK) { + return false; + } + msgpack_zbuffer_reset_buffer(zbuf); + return true; +} + +char* msgpack_zbuffer_release_buffer(msgpack_zbuffer* zbuf) +{ + char* tmp = zbuf->data; + zbuf->data = NULL; + zbuf->stream.next_out = NULL; + zbuf->stream.avail_out = 0; + return tmp; +} + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/zbuffer.h */ + diff --git a/cpp/Makefile.am b/cpp/Makefile.am index d4de161..c1b4981 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -7,6 +7,7 @@ nobase_include_HEADERS = \ msgpack.hpp \ msgpack/sbuffer.hpp \ msgpack/vrefbuffer.hpp \ + msgpack/zbuffer.hpp \ msgpack/pack.hpp \ msgpack/unpack.hpp \ msgpack/object.hpp \ diff --git a/cpp/msgpack/zbuffer.hpp b/cpp/msgpack/zbuffer.hpp new file mode 100644 index 0000000..278f076 --- /dev/null +++ b/cpp/msgpack/zbuffer.hpp @@ -0,0 +1,100 @@ +// +// MessagePack for C++ deflate buffer implementation +// +// Copyright (C) 2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef MSGPACK_ZBUFFER_HPP__ +#define MSGPACK_ZBUFFER_HPP__ + +#include "msgpack/zbuffer.h" +#include + +namespace msgpack { + + +class zbuffer : public msgpack_zbuffer { +public: + zbuffer(int level = Z_DEFAULT_COMPRESSION, + size_t init_size = MSGPACK_ZBUFFER_INIT_SIZE) + { + msgpack_zbuffer_init(this, level, init_size); + } + + ~zbuffer() + { + msgpack_zbuffer_destroy(this); + } + +public: + void write(const char* buf, unsigned int len) + { + if(msgpack_zbuffer_write(this, buf, len) < 0) { + throw std::bad_alloc(); + } + } + + char* flush() + { + char* buf = msgpack_zbuffer_flush(this); + if(!buf) { + throw std::bad_alloc(); + } + return buf; + } + + char* data() + { + return base::data; + } + + const char* data() const + { + return base::data; + } + + size_t size() const + { + return msgpack_zbuffer_size(this); + } + + void reset() + { + if(!msgpack_zbuffer_reset(this)) { + throw std::bad_alloc(); + } + } + + void reset_buffer() + { + msgpack_zbuffer_reset_buffer(this); + } + + char* release_buffer() + { + return msgpack_zbuffer_release_buffer(this); + } + +private: + typedef msgpack_zbuffer base; + +private: + zbuffer(const zbuffer&); +}; + + +} // namespace msgpack + +#endif /* msgpack/zbuffer.hpp */ + From df5a60fd5b0372a636f1e104d6fd5fbb2e36c306 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 26 Mar 2010 15:16:13 +0900 Subject: [PATCH 0376/1648] ruby: append_buffer calls "<<" method if the buffer object.class != String --- ruby/pack.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/ruby/pack.c b/ruby/pack.c index 0a3711f..8e88115 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -18,6 +18,9 @@ #include "ruby.h" #include "msgpack/pack_define.h" +static ID s_to_msgpack; +static ID s_append; + #define msgpack_pack_inline_func(name) \ static inline void msgpack_pack ## name @@ -27,7 +30,9 @@ #define msgpack_pack_user VALUE #define msgpack_pack_append_buffer(user, buf, len) \ - rb_str_buf_cat(user, (const void*)buf, len) + ((TYPE(user) == T_STRING) ? \ + rb_str_buf_cat(user, (const void*)buf, len) : \ + rb_funcall(user, s_append, 1, rb_str_new((const void*)buf,len))) #include "msgpack/pack_template.h" @@ -36,8 +41,6 @@ #include "st.h" // ruby hash #endif -static ID s_to_msgpack; - #define ARG_BUFFER(name, argc, argv) \ VALUE name; \ if(argc == 1) { \ @@ -142,15 +145,24 @@ static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self) } -static VALUE MessagePack_pack(VALUE self, VALUE data) +static VALUE MessagePack_pack(int argc, VALUE* argv, VALUE self) { - return rb_funcall(data, s_to_msgpack, 0); + VALUE out; + if(argc == 1) { + out = rb_str_buf_new(0); + } else if(argc == 2) { + out = argv[1]; + } else { + rb_raise(rb_eArgError, "wrong number of arguments (%d for 1)", argc); + } + return rb_funcall(argv[0], s_to_msgpack, 1, out); } void Init_msgpack_pack(VALUE mMessagePack) { s_to_msgpack = rb_intern("to_msgpack"); + s_append = rb_intern("<<"); rb_define_method_id(rb_cNilClass, s_to_msgpack, MessagePack_NilClass_to_msgpack, -1); rb_define_method_id(rb_cTrueClass, s_to_msgpack, MessagePack_TrueClass_to_msgpack, -1); rb_define_method_id(rb_cFalseClass, s_to_msgpack, MessagePack_FalseClass_to_msgpack, -1); @@ -160,6 +172,6 @@ void Init_msgpack_pack(VALUE mMessagePack) rb_define_method_id(rb_cString, s_to_msgpack, MessagePack_String_to_msgpack, -1); rb_define_method_id(rb_cArray, s_to_msgpack, MessagePack_Array_to_msgpack, -1); rb_define_method_id(rb_cHash, s_to_msgpack, MessagePack_Hash_to_msgpack, -1); - rb_define_module_function(mMessagePack, "pack", MessagePack_pack, 1); + rb_define_module_function(mMessagePack, "pack", MessagePack_pack, -1); } From f7bdda88281812d90293901454e425d3ab282773 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 29 Mar 2010 21:18:01 +0900 Subject: [PATCH 0377/1648] cpp: fixes pack_short(int) -> pack_short(short) --- cpp/msgpack/pack.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/msgpack/pack.hpp b/cpp/msgpack/pack.hpp index b7b3ca1..6a91971 100644 --- a/cpp/msgpack/pack.hpp +++ b/cpp/msgpack/pack.hpp @@ -45,7 +45,7 @@ public: packer& pack_int32(uint32_t d); packer& pack_int64(uint64_t d); - packer& pack_short(int d); + packer& pack_short(short d); packer& pack_int(int d); packer& pack_long(long d); packer& pack_long_long(long long d); @@ -183,7 +183,7 @@ inline packer& packer::pack_int64(uint64_t d) template -inline packer& packer::pack_short(int d) +inline packer& packer::pack_short(short d) { _pack_short(m_stream, d); return *this; } template From fcce8f6d5144f7e1a3cdfb3e236cb532e5565da4 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 31 Mar 2010 12:00:26 +0900 Subject: [PATCH 0378/1648] ruby: use 'readpartial' instead of 'sysread' if !io.respond_to?(:sysread) --- ruby/pack.c | 2 +- ruby/unpack.c | 25 +++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/ruby/pack.c b/ruby/pack.c index 8e88115..1d0068c 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -1,7 +1,7 @@ /* * MessagePack for Ruby packing routine * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ruby/unpack.c b/ruby/unpack.c index 1748329..df54e75 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -1,7 +1,7 @@ /* * MessagePack for Ruby unpacking routine * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ #include "msgpack/unpack_define.h" static ID s_sysread; +static ID s_readpartial; typedef struct { int finished; @@ -28,6 +29,7 @@ typedef struct { VALUE buffer; VALUE stream; VALUE streambuf; + ID stream_append_method; } unpack_user; @@ -183,6 +185,15 @@ static VALUE MessagePack_Unpacker_reset(VALUE self) return self; } +static ID append_method_of(VALUE stream) +{ + if(rb_respond_to(stream, s_sysread)) { + return s_sysread; + } else { + return s_readpartial; + } +} + static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) { VALUE stream; @@ -204,6 +215,7 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) mp->user.buffer = rb_str_new("",0); mp->user.stream = stream; mp->user.streambuf = rb_str_new("",0); + mp->user.stream_append_method = append_method_of(stream); return self; } @@ -299,7 +311,9 @@ static VALUE MessagePack_Unpacker_stream_get(VALUE self) static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) { UNPACKER(self, mp); - return mp->user.stream = val; + mp->user.stream = val; + mp->user.stream_append_method = append_method_of(val); + return val; } static VALUE MessagePack_Unpacker_fill(VALUE self) @@ -312,10 +326,12 @@ static VALUE MessagePack_Unpacker_fill(VALUE self) size_t len; if(RSTRING_LEN(mp->user.buffer) == 0) { - rb_funcall(mp->user.stream, s_sysread, 2, LONG2FIX(64*1024), mp->user.buffer); + rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, + LONG2FIX(64*1024), mp->user.buffer); len = RSTRING_LEN(mp->user.buffer); } else { - rb_funcall(mp->user.stream, s_sysread, 2, LONG2FIX(64*1024), mp->user.streambuf); + rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, + LONG2FIX(64*1024), mp->user.streambuf); len = RSTRING_LEN(mp->user.streambuf); rb_str_cat(mp->user.buffer, RSTRING_PTR(mp->user.streambuf), RSTRING_LEN(mp->user.streambuf)); } @@ -428,6 +444,7 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) void Init_msgpack_unpack(VALUE mMessagePack) { s_sysread = rb_intern("sysread"); + s_readpartial = rb_intern("readpartial"); eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); From 58201b95f2626543150c0e8738dda4babe401a64 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 31 Mar 2010 13:46:28 +0900 Subject: [PATCH 0379/1648] ruby: version 0.3.4 --- ruby/msgpack.gemspec | 4 ++-- ruby/msgpack.mingw.gemspec | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 4575c70..fd6c6a8 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,11 +1,11 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.3" + s.version = "0.3.4" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" - s.homepage = "http://msgpack.sourceforge.jp/" + s.homepage = "http://msgpack.sourceforge.net/" s.rubyforge_project = "msgpack" s.has_rdoc = false s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] diff --git a/ruby/msgpack.mingw.gemspec b/ruby/msgpack.mingw.gemspec index 3d2e7dc..e7bfecb 100644 --- a/ruby/msgpack.mingw.gemspec +++ b/ruby/msgpack.mingw.gemspec @@ -1,11 +1,11 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::CURRENT s.name = "msgpack" - s.version = "0.3.3" + s.version = "0.3.4" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" - s.homepage = "http://msgpack.sourceforge.jp/" + s.homepage = "http://msgpack.sourceforge.net/" s.rubyforge_project = "msgpack" s.has_rdoc = false s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] From f91e1c17c0c22194b7a36aaa8290ec6f5e903e79 Mon Sep 17 00:00:00 2001 From: Keiji Muraishi Date: Wed, 31 Mar 2010 17:09:00 +0900 Subject: [PATCH 0380/1648] fix typo in Makefile --- python/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/Makefile b/python/Makefile index 267415c..e06794d 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,5 +1,5 @@ all: - python setup_dev.py build_ext -i -f + python setup.py build_ext -i -f python setup.py build sdist .PHONY: test From 0159084ce958aeb1ba7ffb74ded51a9e4c5bb457 Mon Sep 17 00:00:00 2001 From: Keiji Muraishi Date: Wed, 31 Mar 2010 17:09:00 +0900 Subject: [PATCH 0381/1648] fix typo in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 267415c..e06794d 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ all: - python setup_dev.py build_ext -i -f + python setup.py build_ext -i -f python setup.py build sdist .PHONY: test From 6c6df1adaf4f2b202a6241a408538d05fb2ae430 Mon Sep 17 00:00:00 2001 From: Keiji Muraishi Date: Wed, 31 Mar 2010 17:29:07 +0900 Subject: [PATCH 0382/1648] should raise TypeError on find unsupported value --- python/msgpack/_msgpack.pyx | 2 +- python/test/test_except.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 python/test/test_except.py diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx index eb83c85..61ae36b 100644 --- a/python/msgpack/_msgpack.pyx +++ b/python/msgpack/_msgpack.pyx @@ -71,7 +71,7 @@ cdef class Packer(object): def __dealloc__(self): free(self.pk.buf); - cdef int __pack(self, object o): + cdef int __pack(self, object o) except -1: cdef long long llval cdef unsigned long long ullval cdef long longval diff --git a/python/test/test_except.py b/python/test/test_except.py new file mode 100644 index 0000000..574728f --- /dev/null +++ b/python/test/test_except.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose.tools import * +from msgpack import packs, unpacks + +import datetime + +def test_raise_on_find_unsupported_value(): + assert_raises(TypeError, packs, datetime.datetime.now()) + +if __name__ == '__main__': + from nose import main + main() From a50a83f073df4b3784d468d36d58111404505ffe Mon Sep 17 00:00:00 2001 From: Keiji Muraishi Date: Wed, 31 Mar 2010 17:29:07 +0900 Subject: [PATCH 0383/1648] should raise TypeError on find unsupported value --- msgpack/_msgpack.pyx | 2 +- test/test_except.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 test/test_except.py diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx index eb83c85..61ae36b 100644 --- a/msgpack/_msgpack.pyx +++ b/msgpack/_msgpack.pyx @@ -71,7 +71,7 @@ cdef class Packer(object): def __dealloc__(self): free(self.pk.buf); - cdef int __pack(self, object o): + cdef int __pack(self, object o) except -1: cdef long long llval cdef unsigned long long ullval cdef long longval diff --git a/test/test_except.py b/test/test_except.py new file mode 100644 index 0000000..574728f --- /dev/null +++ b/test/test_except.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# coding: utf-8 + +from nose.tools import * +from msgpack import packs, unpacks + +import datetime + +def test_raise_on_find_unsupported_value(): + assert_raises(TypeError, packs, datetime.datetime.now()) + +if __name__ == '__main__': + from nose import main + main() From 1784746e7e964a4ee4a22dfcd8dfc04ce743e404 Mon Sep 17 00:00:00 2001 From: moaikids Date: Thu, 1 Apr 2010 23:13:56 +0900 Subject: [PATCH 0384/1648] fix: CS_MAP_16 deserialize bug(line.388) --- java/src/org/msgpack/impl/UnpackerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/org/msgpack/impl/UnpackerImpl.java b/java/src/org/msgpack/impl/UnpackerImpl.java index ddf9940..adc62b0 100644 --- a/java/src/org/msgpack/impl/UnpackerImpl.java +++ b/java/src/org/msgpack/impl/UnpackerImpl.java @@ -385,11 +385,11 @@ public class UnpackerImpl { obj = new Object[count*2]; if(count == 0) { break _push; } // FIXME check IMapSchema //System.out.println("fixmap count:"+count); + ++top; stack_obj[top] = top_obj; stack_ct[top] = top_ct; stack_count[top] = top_count; stack_schema[top] = top_schema; - ++top; top_obj = obj; top_ct = CT_MAP_KEY; top_count = count; From 3416cf984e31dfd9706166d6113c48318ef4b08d Mon Sep 17 00:00:00 2001 From: moaikids Date: Thu, 1 Apr 2010 23:15:36 +0900 Subject: [PATCH 0385/1648] fix: org.msgpack.impl.UnpackerImpl.java CS_MAP_16 deserialize bug(line.388) --- java/build.xml | 2 +- java/src/org/msgpack/Packer.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/java/build.xml b/java/build.xml index 40d7d61..b98f7c3 100644 --- a/java/build.xml +++ b/java/build.xml @@ -13,6 +13,6 @@ - + diff --git a/java/src/org/msgpack/Packer.java b/java/src/org/msgpack/Packer.java index ebd8402..935728d 100644 --- a/java/src/org/msgpack/Packer.java +++ b/java/src/org/msgpack/Packer.java @@ -20,7 +20,6 @@ package org.msgpack; import java.io.OutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.Charset; import java.util.List; import java.util.Map; From 7c863c341ed17ff737e58a3ed958a6e1314fb4de Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 2 Apr 2010 02:19:41 +0900 Subject: [PATCH 0386/1648] ruby: use gem-compile gem instead of some scripts to create binary gems --- ruby/makegem.sh | 4 +++- ruby/msgpack.gemspec | 2 +- ruby/msgpack.mingw.gemspec | 16 ---------------- ruby/msgpack.mingw.sh | 21 --------------------- 4 files changed, 4 insertions(+), 39 deletions(-) delete mode 100644 ruby/msgpack.mingw.gemspec delete mode 100755 ruby/msgpack.mingw.sh diff --git a/ruby/makegem.sh b/ruby/makegem.sh index 622264d..c78769b 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -19,7 +19,9 @@ cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FI gem build msgpack.gemspec -# ./msgpack.mingw.sh # on msys +# gem install gem-compile # on msys +# gem compile msgpack-$version.gem # on msys +# gem compile msgpack-$version.gem -p mswin32 # on msys # gem push msgpack-$version.gem # gem push msgpack-$version-x86-mingw32.gem # gem push msgpack-$version-mswin32.gem diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index fd6c6a8..5ff27f2 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.4" + s.version = "0.3.5" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" diff --git a/ruby/msgpack.mingw.gemspec b/ruby/msgpack.mingw.gemspec deleted file mode 100644 index e7bfecb..0000000 --- a/ruby/msgpack.mingw.gemspec +++ /dev/null @@ -1,16 +0,0 @@ -Gem::Specification.new do |s| - s.platform = Gem::Platform::CURRENT - s.name = "msgpack" - s.version = "0.3.4" - s.summary = "MessagePack, a binary-based efficient data interchange format." - s.author = "FURUHASHI Sadayuki" - s.email = "frsyuki@users.sourceforge.jp" - s.homepage = "http://msgpack.sourceforge.net/" - s.rubyforge_project = "msgpack" - s.has_rdoc = false - s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] - s.require_paths = ["lib", "ext"] - s.files = Dir["lib/**/*", "ext/**/*", "msgpack/**/*", "test/**/*"] - s.test_files = Dir["test/test_*.rb"] - s.extensions = Dir["ext/**/extconf.rb"] -end diff --git a/ruby/msgpack.mingw.sh b/ruby/msgpack.mingw.sh deleted file mode 100755 index ec75234..0000000 --- a/ruby/msgpack.mingw.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh -if [ -z "$1" ];then - echo "usage: $0 " - exit 1 -fi - -version=$1 -build=msgpack-mingw-build - -./makegem.sh -gem build msgpack.mingw.gemspec -rm -rf $build -mkdir $build -cd $build -tar xvf ../msgpack-$version-x86-mingw32.gem -gunzip metadata.gz -sed s/x86-mingw32/mswin32/ metadata > metadata.tmp -mv metadata.tmp metadata -gzip metadata -tar cvf msgpack-$version-mswin32.gem metadata.gz data.tar.gz - From 11286524a5872f88a0988118faaf9c37f5d0c2d7 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 2 Apr 2010 03:22:29 +0900 Subject: [PATCH 0387/1648] ruby: fixes 'File not found: lib' message on gem installation --- ruby/msgpack.gemspec | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 5ff27f2..117f058 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.5" + s.version = "0.3.6" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" @@ -9,8 +9,8 @@ Gem::Specification.new do |s| s.rubyforge_project = "msgpack" s.has_rdoc = false s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] - s.require_paths = ["lib", "ext"] - s.files = Dir["lib/**/*", "ext/**/*", "msgpack/**/*", "test/**/*"] + s.require_paths = ["ext"] + s.files = Dir["ext/**/*", "msgpack/**/*", "test/**/*"] s.test_files = Dir["test/test_*.rb"] s.extensions = Dir["ext/**/extconf.rb"] end From 93c3cbeaeff73686048c687fa1a4cfcfd7fc2129 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 4 Apr 2010 21:45:56 +0900 Subject: [PATCH 0388/1648] ruby fixes gemspec: require_paths = ["lib"] --- ruby/msgpack.gemspec | 2 +- ruby/test/test_helper.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 117f058..4f7fcec 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -9,7 +9,7 @@ Gem::Specification.new do |s| s.rubyforge_project = "msgpack" s.has_rdoc = false s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] - s.require_paths = ["ext"] + s.require_paths = ["lib"] s.files = Dir["ext/**/*", "msgpack/**/*", "test/**/*"] s.test_files = Dir["test/test_*.rb"] s.extensions = Dir["ext/**/extconf.rb"] diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb index 3088faa..6a63489 100644 --- a/ruby/test/test_helper.rb +++ b/ruby/test/test_helper.rb @@ -1,3 +1,3 @@ require 'test/unit' -require File.dirname(__FILE__) + '/../ext/msgpack' +require File.dirname(__FILE__) + '/../lib/msgpack' From f88c029a4c43e8d8c8bd37c34a390ffb987788ed Mon Sep 17 00:00:00 2001 From: Masahiro Nakagawa Date: Sun, 4 Apr 2010 21:55:00 +0900 Subject: [PATCH 0389/1648] c: fixes msgpack_zbuffer_flush --- c/msgpack/zbuffer.h | 1 + 1 file changed, 1 insertion(+) diff --git a/c/msgpack/zbuffer.h b/c/msgpack/zbuffer.h index ad3d11a..cfcba29 100644 --- a/c/msgpack/zbuffer.h +++ b/c/msgpack/zbuffer.h @@ -126,6 +126,7 @@ char* msgpack_zbuffer_flush(msgpack_zbuffer* zbuf) return zbuf->data; case Z_OK: msgpack_zbuffer_expand(zbuf); + break; default: return NULL; } From e43f57fe1aa97081e1794219375b4b5a42951dfc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 4 Apr 2010 22:06:27 +0900 Subject: [PATCH 0390/1648] c: fixes msgpack_zbuffer_flush: error checking --- c/msgpack/zbuffer.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/c/msgpack/zbuffer.h b/c/msgpack/zbuffer.h index cfcba29..26eb29f 100644 --- a/c/msgpack/zbuffer.h +++ b/c/msgpack/zbuffer.h @@ -125,7 +125,9 @@ char* msgpack_zbuffer_flush(msgpack_zbuffer* zbuf) case Z_STREAM_END: return zbuf->data; case Z_OK: - msgpack_zbuffer_expand(zbuf); + if(!msgpack_zbuffer_expand(zbuf)) { + return NULL; + } break; default: return NULL; From 254ee80c16b3b0ce12b461d189aa1e6302debea0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 4 Apr 2010 22:11:16 +0900 Subject: [PATCH 0391/1648] c: fixes msgpack_zbuffer_write: error checking --- c/msgpack/zbuffer.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/c/msgpack/zbuffer.h b/c/msgpack/zbuffer.h index 26eb29f..2a32206 100644 --- a/c/msgpack/zbuffer.h +++ b/c/msgpack/zbuffer.h @@ -107,7 +107,9 @@ int msgpack_zbuffer_write(void* data, const char* buf, unsigned int len) do { if(zbuf->stream.avail_out < MSGPACK_ZBUFFER_RESERVE_SIZE) { - msgpack_zbuffer_expand(zbuf); + if(!msgpack_zbuffer_expand(zbuf)) { + return -1; + } } if(deflate(&zbuf->stream, Z_NO_FLUSH) != Z_OK) { From 99a2d2859269ddf803a802c043babcd977260faf Mon Sep 17 00:00:00 2001 From: Hideyuki TAKEI Date: Mon, 5 Apr 2010 00:10:28 +0900 Subject: [PATCH 0392/1648] import MessagePack for PHP --- php/CREDITS | 1 + php/EXPERIMENTAL | 0 php/README | 0 php/bench/bench.php | 51 +++ php/config.m4 | 14 + php/config.w32 | 13 + php/msgpack.c | 656 ++++++++++++++++++++++++++++++ php/msgpack.php | 21 + php/msgpack/pack_define.h | 26 ++ php/msgpack/pack_template.h | 741 ++++++++++++++++++++++++++++++++++ php/msgpack/unpack_define.h | 129 ++++++ php/msgpack/unpack_template.h | 361 +++++++++++++++++ php/php_msgpack.h | 86 ++++ php/test_normal.php | 19 + php/test_streaming.php | 34 ++ php/tests/test_pack.phpt | 169 ++++++++ 16 files changed, 2321 insertions(+) create mode 100644 php/CREDITS create mode 100644 php/EXPERIMENTAL create mode 100644 php/README create mode 100644 php/bench/bench.php create mode 100644 php/config.m4 create mode 100644 php/config.w32 create mode 100644 php/msgpack.c create mode 100644 php/msgpack.php create mode 100644 php/msgpack/pack_define.h create mode 100644 php/msgpack/pack_template.h create mode 100644 php/msgpack/unpack_define.h create mode 100644 php/msgpack/unpack_template.h create mode 100644 php/php_msgpack.h create mode 100755 php/test_normal.php create mode 100755 php/test_streaming.php create mode 100644 php/tests/test_pack.phpt diff --git a/php/CREDITS b/php/CREDITS new file mode 100644 index 0000000..3aae276 --- /dev/null +++ b/php/CREDITS @@ -0,0 +1 @@ +msgpack diff --git a/php/EXPERIMENTAL b/php/EXPERIMENTAL new file mode 100644 index 0000000..e69de29 diff --git a/php/README b/php/README new file mode 100644 index 0000000..e69de29 diff --git a/php/bench/bench.php b/php/bench/bench.php new file mode 100644 index 0000000..95a7ca4 --- /dev/null +++ b/php/bench/bench.php @@ -0,0 +1,51 @@ + diff --git a/php/config.m4 b/php/config.m4 new file mode 100644 index 0000000..751b471 --- /dev/null +++ b/php/config.m4 @@ -0,0 +1,14 @@ +dnl $Id$ +dnl config.m4 for extension msgpack + +PHP_ARG_ENABLE(msgpack, whether to enable MessagePack support, +Make sure that the comment is aligned: +[ --enable-msgpack Enable MessagePack support]) + +if test "$PHP_MSGPACK" != "no"; then + dnl AC_DEFINE([HAVE_MSGPACK],1 ,[whether to enable MessagePack support]) + dnl AC_HEADER_STDC + + PHP_NEW_EXTENSION(msgpack, msgpack.c, $ext_shared) + dnl PHP_SUBST(MSGPACK_SHARED_LIBADD) +fi diff --git a/php/config.w32 b/php/config.w32 new file mode 100644 index 0000000..50e9bec --- /dev/null +++ b/php/config.w32 @@ -0,0 +1,13 @@ +// $Id$ +// vim:ft=javascript + +// If your extension references something external, use ARG_WITH +// ARG_WITH("msgpack", "for msgpack support", "no"); + +// Otherwise, use ARG_ENABLE +// ARG_ENABLE("msgpack", "enable msgpack support", "no"); + +if (PHP_MSGPACK != "no") { + EXTENSION("msgpack", "msgpack.c"); +} + diff --git a/php/msgpack.c b/php/msgpack.c new file mode 100644 index 0000000..b38eb18 --- /dev/null +++ b/php/msgpack.c @@ -0,0 +1,656 @@ +/* + +----------------------------------------------------------------------+ + | PHP Version 5 | + +----------------------------------------------------------------------+ + | Copyright (c) 1997-2007 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Author: Hideyuki TAKEI | + +----------------------------------------------------------------------+ +*/ + +/* $Id: header 226204 2007-01-01 19:32:10Z iliaa $ */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "php.h" +#include "php_ini.h" +#include "ext/standard/info.h" +#include "ext/standard/php_smart_str.h" +#include "php_msgpack.h" + +#define PHP_EXT_VERSION "0.01" + +#ifndef TRUE +# define TRUE 1 +# define FALSE 0 +#endif + + +/* pack */ +#include "msgpack/pack_define.h" + +#define msgpack_pack_inline_func(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_inline_func_cint(name) \ + static inline void msgpack_pack ## name + +#define msgpack_pack_user smart_str* + +#define msgpack_pack_append_buffer(user, buf, len) \ + smart_str_appendl(user, (const void*)buf, len) + +#include "msgpack/pack_template.h" + + +/* unpack */ +#include "msgpack/unpack_define.h" + +typedef struct { + int finished; + char* source; +} unpack_user; + +#define msgpack_unpack_struct(name) \ + struct template ## name + +#define msgpack_unpack_func(ret, name) \ + ret template ## name + +#define msgpack_unpack_callback(name) \ + template_callback ## name + +#define msgpack_unpack_object zval* + +#define msgpack_unpack_user unpack_user + +struct template_context; +typedef struct template_context msgpack_unpack_t; + +static void template_init(msgpack_unpack_t* u); +static msgpack_unpack_object template_data(msgpack_unpack_t* u); +static int template_execute(msgpack_unpack_t* u, + const char* data, size_t len, size_t* off); + +ZEND_BEGIN_MODULE_GLOBALS(msgpack) + msgpack_unpack_t *global_mp; +ZEND_END_MODULE_GLOBALS(msgpack) + +#ifdef ZTS +#define MSGPACK_G(v) TSRMG(msgpack_globals_id, zend_msgpack_globals *, v) +#else +#define MSGPACK_G(v) (msgpack_globals.v) +#endif + +static inline msgpack_unpack_object template_callback_root(unpack_user* u) +{ + msgpack_unpack_object data; + ALLOC_INIT_ZVAL(data); + ZVAL_NULL(data); + return data; +} + +static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; } + +static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; } + +static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; } + +static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; } + +static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, (long)d); return 0; } + +static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, (long)d); return 0; } + +static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, (long)d); return 0; } + +static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; } + +static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_DOUBLE(*o, d); return 0; } + +static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_DOUBLE(*o, d); return 0; } + +static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_NULL(*o); return 0; } + +static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_BOOL(*o, 1); return 0; } + +static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); ZVAL_BOOL(*o, 0); return 0;} + +static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); array_init(*o); return 0; } + +static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o) +{ add_next_index_zval(*c, o); return 0; } + +static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o) +{ ALLOC_INIT_ZVAL(*o); array_init(*o); return 0; } + +static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v) +{ + switch(k->type) { + case IS_LONG: + add_index_zval(*c, Z_LVAL(*k), v); + break; + case IS_STRING: + add_assoc_zval_ex(*c, Z_STRVAL(*k), Z_STRLEN(*k)+1, v); + break; + default: + zend_error(E_WARNING, "[msgpack] (php_msgpack_decode) illegal offset type, skip this decoding"); + break; + } + return 0; +} + + +static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o) +{ + ALLOC_INIT_ZVAL(*o); + if (l == 0) { + ZVAL_STRINGL(*o, "", 0, 1); + } else { + ZVAL_STRINGL(*o, p, l, 1); + } + return 0; +} + +#include "msgpack/unpack_template.h" + +static PHP_GINIT_FUNCTION(msgpack); + +ZEND_DECLARE_MODULE_GLOBALS(msgpack) + +/* True global resources - no need for thread safety here */ +static int le_msgpack; + +/* {{{ msgpack_functions[] + * + * Every user visible function must have an entry in msgpack_functions[]. + */ +zend_function_entry msgpack_functions[] = { + PHP_FE(msgpack_pack, NULL) + PHP_FE(msgpack_unpack, NULL) + PHP_FE(msgpack_unpack_limit, NULL) + PHP_ME(msgpack, initialize, NULL, 0) + PHP_ME(msgpack, execute, NULL, 0) + PHP_ME(msgpack, execute_limit, NULL, 0) + PHP_ME(msgpack, finished, NULL, 0) + PHP_ME(msgpack, data, NULL, 0) + {NULL, NULL, NULL} /* Must be the last line in msgpack_functions[] */ +}; +/* }}} */ + +/* {{{ msgpack_module_entry + */ +zend_module_entry msgpack_module_entry = { +#if ZEND_MODULE_API_NO >= 20010901 + STANDARD_MODULE_HEADER, +#endif + "msgpack", + msgpack_functions, + PHP_MINIT(msgpack), + PHP_MSHUTDOWN(msgpack), + PHP_RINIT(msgpack), /* Replace with NULL if there's nothing to do at request start */ + PHP_RSHUTDOWN(msgpack), /* Replace with NULL if there's nothing to do at request end */ + PHP_MINFO(msgpack), +#if ZEND_MODULE_API_NO >= 20010901 + "0.1", /* Replace with version number for your extension */ +#endif + PHP_MODULE_GLOBALS(msgpack), + PHP_GINIT(msgpack), + NULL, + NULL, + STANDARD_MODULE_PROPERTIES_EX +}; +/* }}} */ + +#ifdef COMPILE_DL_MSGPACK +ZEND_GET_MODULE(msgpack) +#endif + +/* {{{ PHP_GINIT_FUNCTION */ +static PHP_GINIT_FUNCTION(msgpack) +{ + msgpack_globals->global_mp = NULL; +} +/* }}} */ + +/* {{{ PHP_MINIT_FUNCTION + */ +PHP_MINIT_FUNCTION(msgpack) +{ + zend_class_entry ce; + INIT_CLASS_ENTRY(ce, "MessagePack", msgpack_functions); + msgpack_ce = zend_register_internal_class(&ce TSRMLS_CC); + + return SUCCESS; +} +/* }}} */ + +/* {{{ PHP_MSHUTDOWN_FUNCTION + */ +PHP_MSHUTDOWN_FUNCTION(msgpack) +{ + /* uncomment this line if you have INI entries + UNREGISTER_INI_ENTRIES(); + */ + if (MSGPACK_G(global_mp)) { + efree(MSGPACK_G(global_mp)); + MSGPACK_G(global_mp) = NULL; + } + + return SUCCESS; +} +/* }}} */ + +/* Remove if there's nothing to do at request start */ +/* {{{ PHP_RINIT_FUNCTION + */ +PHP_RINIT_FUNCTION(msgpack) +{ + return SUCCESS; +} +/* }}} */ + +/* Remove if there's nothing to do at request end */ +/* {{{ PHP_RSHUTDOWN_FUNCTION + */ +PHP_RSHUTDOWN_FUNCTION(msgpack) +{ + return SUCCESS; +} +/* }}} */ + +/* {{{ PHP_MINFO_FUNCTION + */ +PHP_MINFO_FUNCTION(msgpack) +{ + php_info_print_table_start(); + php_info_print_table_header(2, "msgpack support", "enabled"); + php_info_print_table_row(2, "php extension version", PHP_EXT_VERSION); + php_info_print_table_row(2, "author", "Hideyuki TAKEI"); + php_info_print_table_row(2, "homepage", "http://msgpack.sourceforge.net"); + php_info_print_table_row(2, "open sourced by", "KLab inc."); + php_info_print_table_end(); +} +/* }}} */ + +PHP_MSGPACK_API int msgpack_determine_array_type(zval **val TSRMLS_DC) /* {{{ */ +{ + int i; + HashTable *myht = HASH_OF(*val); + + i = myht ? zend_hash_num_elements(myht) : 0; + if (i > 0) { + char *key; + ulong index, idx; + uint key_len; + HashPosition pos; + + zend_hash_internal_pointer_reset_ex(myht, &pos); + idx = 0; + for (;; zend_hash_move_forward_ex(myht, &pos)) { + i = zend_hash_get_current_key_ex(myht, &key, &key_len, &index, 0, &pos); + if (i == HASH_KEY_NON_EXISTANT) + break; + if (i == HASH_KEY_IS_STRING) { + return 1; + } else { + if (index != idx) { + return 1; + } + } + idx++; + } + } + return 0; +} +/* }}} */ + +PHP_MSGPACK_API void msgpack_pack_array_hash(smart_str *pk, zval **val TSRMLS_DC) /* {{{ */ +{ + int i, r; + HashTable *myht; + + if(Z_TYPE_PP(val) == IS_ARRAY){ + myht = HASH_OF(*val); + r = msgpack_determine_array_type(val TSRMLS_CC); + } + else{ + myht = Z_OBJPROP_PP(val); + r = 1; + } + + i = myht ? zend_hash_num_elements(myht) : 0; + + if(r == 0){ + msgpack_pack_array(pk, i); + } + else{ + msgpack_pack_map(pk, i); + } + + if(i>0){ + char *key; + zval **data; + ulong index; + uint key_len; + HashPosition pos; + HashTable *tmp_ht; + int need_comma = 0; + + zend_hash_internal_pointer_reset_ex(myht, &pos); + for(;; zend_hash_move_forward_ex(myht, &pos)){ + i = zend_hash_get_current_key_ex(myht, &key, &key_len, &index, 0, &pos); + if(i==HASH_KEY_NON_EXISTANT) + break; + if(zend_hash_get_current_data_ex(myht, (void **) &data, &pos) == SUCCESS){ + tmp_ht = HASH_OF(*data); + if (tmp_ht) + tmp_ht->nApplyCount++; + + if(r==0) + php_msgpack_pack(pk, *data TSRMLS_CC); + else if(r==1){ + if(i==HASH_KEY_IS_STRING){ + if(key[0]=='\0' && Z_TYPE_PP(val)==IS_OBJECT){ + // Skip protected and private members. + if(tmp_ht) + tmp_ht->nApplyCount--; + continue; + } + msgpack_pack_raw(pk, key_len-1); + msgpack_pack_raw_body(pk, key, key_len-1); + php_msgpack_pack(pk, *data TSRMLS_CC); + } + else{ + msgpack_pack_long(pk, index); + php_msgpack_pack(pk, *data TSRMLS_CC); + } + } + + if(tmp_ht){ + tmp_ht->nApplyCount--; + } + } + } + + } +} +/* }}} */ + +PHP_MSGPACK_API void php_msgpack_pack(smart_str *pk, zval *val TSRMLS_DC) /* {{{ */ +{ + switch(Z_TYPE_P(val)){ + case IS_NULL: + msgpack_pack_nil(pk); + break; + case IS_BOOL: + if (Z_BVAL_P(val)) + msgpack_pack_true(pk); + else + msgpack_pack_false(pk); + break; + case IS_LONG: + msgpack_pack_long(pk, Z_LVAL_P(val)); + break; + case IS_DOUBLE: + { + double dbl = Z_DVAL_P(val); + if (zend_isinf(dbl) || zend_isnan(dbl)) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_pack) double %.9g does not conform to the MSGPACK spec, encoded as 0", dbl); + ZVAL_LONG(val, 0); + } + msgpack_pack_double(pk, Z_DVAL_P(val)); + } + break; + case IS_STRING: + msgpack_pack_raw(pk, Z_STRLEN_P(val)); + msgpack_pack_raw_body(pk, Z_STRVAL_P(val), Z_STRLEN_P(val)); + break; + case IS_ARRAY: + case IS_OBJECT: + msgpack_pack_array_hash(pk, &val TSRMLS_CC); + break; + defalut: + zend_error(E_WARNING, "[msgpack] (php_msgpack_pack) type is unsupported, encoded as null"); + msgpack_pack_nil(pk); + break; + } + + return; +} +/* }}} */ + +PHP_MSGPACK_API void php_msgpack_unpack_limit(zval *return_value, const char *buf, int len, zend_bool assoc TSRMLS_DC) /* {{{ */ +{ + if (len<=0) { + RETURN_NUL(); + } + + msgpack_unpack_t mp; + template_init(&mp); + unpack_user u = {0, ""}; + + size_t from = 0; + char* dptr = (char*)buf; + long dlen = len; + int ret; + + (&mp)->user.source = (char*)buf; + ret = template_execute(&mp, dptr, (size_t)dlen, &from); + (&mp)->user.source = ""; + + if(ret < 0) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) parse error"); + } else if(ret == 0) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) insufficient bytes"); + } else { + if(from < dlen) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) extra bytes"); + } + + *return_value = *template_data(&mp); + FREE_ZVAL(template_data(&mp)); + } +} +/* }}} */ + + +PHP_FUNCTION(msgpack_pack) +{ + zval *parameter; + smart_str buf = {0}; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "z", ¶meter) == FAILURE) { + return; + } + + php_msgpack_pack(&buf, parameter TSRMLS_CC); + + ZVAL_STRINGL(return_value, buf.c, buf.len, 1); + + smart_str_free(&buf); +} + +PHP_FUNCTION(msgpack_unpack) +{ + char *parameter; + int parameter_len; + zend_bool assoc = 0; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|b", + ¶meter, ¶meter_len, &assoc) == FAILURE) { + return; + } + + if (!parameter_len) { + RETURN_NULL(); + } + + php_msgpack_unpack_limit(return_value, parameter, parameter_len, assoc TSRMLS_CC); +} + +PHP_FUNCTION(msgpack_unpack_limit) +{ + char *parameter; + int parameter_len; + int limit; + zend_bool assoc = 0; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sl|b", + ¶meter, ¶meter_len, &limit, &assoc) == FAILURE) { + return; + } + + if (!parameter_len) { + RETURN_NULL(); + } + else if (parameter_len < limit) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) limit greater than data_len"); + limit = parameter_len; + } + + php_msgpack_unpack_limit(return_value, parameter, limit, assoc TSRMLS_CC); +} + + +PHP_MSGPACK_API void php_msgpack_unpacker_execute_limit(zval *return_value, const char *buf, int off, int len, zend_bool assoc TSRMLS_DC) /* {{{ */ +{ + if (len<=0) { + RETURN_NUL(); + } + + size_t from = off; + char* dptr = (char*)buf; + long dlen = len; + int ret; + + if(from >= dlen) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpacker_execute_limit) offset is bigger than data buffer size"); + } + + MSGPACK_G(global_mp)->user.source = (char*)buf; + ret = template_execute(MSGPACK_G(global_mp), dptr, (size_t)dlen, &from); + MSGPACK_G(global_mp)->user.source = ""; + + if(ret < 0) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpacker_execute_limit) parse error"); + } else if(ret > 0) { + MSGPACK_G(global_mp)->user.finished = 1; + RETVAL_LONG(from); + } else { + MSGPACK_G(global_mp)->user.finished = 0; + RETVAL_LONG(from); + } +} +/* }}} */ + +PHP_MSGPACK_API void php_msgpack_unpacker_reset(TSRMLS_D) /* {{{ */ +{ + if(MSGPACK_G(global_mp)) { + efree(MSGPACK_G(global_mp)); + MSGPACK_G(global_mp) = NULL; + } + MSGPACK_G(global_mp) = safe_emalloc(sizeof(msgpack_unpack_t), 1, 0); + + template_init(MSGPACK_G(global_mp)); + unpack_user u = {0, ""}; + MSGPACK_G(global_mp)->user = u; + return; +} +/* }}} */ + +PHP_METHOD(msgpack, initialize) +{ + php_msgpack_unpacker_reset(TSRMLS_C); + return; +} + +PHP_METHOD(msgpack, execute) +{ + char *data; + int off; + int data_len; + zend_bool assoc = 0; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sl|b", + &data, &data_len, &off, &assoc) == FAILURE) { + return; + } + + if (!data_len) { + RETURN_NULL(); + } + + php_msgpack_unpacker_execute_limit(return_value, data, off, data_len, assoc TSRMLS_CC); +} + +PHP_METHOD(msgpack, execute_limit) +{ + char *data; + int off; + int data_len; + int limit; + zend_bool assoc = 0; + + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sll|b", + &data, &data_len, &off, &limit, &assoc) == FAILURE) { + return; + } + + if (!data_len) { + RETURN_NULL(); + } + else if (data_len < limit) { + zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) limit greater than (data+off)_len"); + limit = data_len; + } + + php_msgpack_unpacker_execute_limit(return_value, data, off, limit, assoc TSRMLS_CC); +} + +PHP_METHOD(msgpack, finished) +{ + if(MSGPACK_G(global_mp)->user.finished == 1) { + RETURN_TRUE; + } + RETURN_FALSE; +} + +PHP_METHOD(msgpack, data) +{ + *return_value = *template_data(MSGPACK_G(global_mp)); + return; +} + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * End: + * vim600: noet sw=4 ts=4 fdm=marker + * vim<600: noet sw=4 ts=4 + */ diff --git a/php/msgpack.php b/php/msgpack.php new file mode 100644 index 0000000..c2498dd --- /dev/null +++ b/php/msgpack.php @@ -0,0 +1,21 @@ +"; + +if(!extension_loaded('msgpack')) { + dl('msgpack.' . PHP_SHLIB_SUFFIX); +} +$module = 'msgpack'; +$functions = get_extension_funcs($module); +echo "Functions available in the test extension:$br\n"; +foreach($functions as $func) { + echo $func."$br\n"; +} +echo "$br\n"; +$function = 'confirm_' . $module . '_compiled'; +if (extension_loaded($module)) { + $str = $function($module); +} else { + $str = "Module $module is not compiled into PHP"; +} +echo "$str\n"; +?> diff --git a/php/msgpack/pack_define.h b/php/msgpack/pack_define.h new file mode 100644 index 0000000..33408e5 --- /dev/null +++ b/php/msgpack/pack_define.h @@ -0,0 +1,26 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_PACK_DEFINE_H__ +#define MSGPACK_PACK_DEFINE_H__ + +#include +#include +#include + +#endif /* msgpack/pack_define.h */ + diff --git a/php/msgpack/pack_template.h b/php/msgpack/pack_template.h new file mode 100644 index 0000000..aa620f5 --- /dev/null +++ b/php/msgpack/pack_template.h @@ -0,0 +1,741 @@ +/* + * MessagePack packing routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + + +#ifdef __LITTLE_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[0] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \ + ((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0] + + +#elif __BIG_ENDIAN__ + +#define STORE8_BE8(d) \ + ((uint8_t*)&d)[0] + + +#define STORE16_BE8(d) \ + ((uint8_t*)&d)[1] + +#define STORE16_BE16(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1] + + +#define STORE32_BE8(d) \ + ((uint8_t*)&d)[3] + +#define STORE32_BE16(d) \ + ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + +#define STORE32_BE32(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3] + + +#define STORE64_BE8(d) \ + ((uint8_t*)&d)[7] + +#define STORE64_BE16(d) \ + ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE32(d) \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#define STORE64_BE64(d) \ + ((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \ + ((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7] + +#endif + +#ifndef msgpack_pack_inline_func +#error msgpack_pack_inline_func template is not defined +#endif + +#ifndef msgpack_pack_user +#error msgpack_pack_user type is not defined +#endif + +#ifndef msgpack_pack_append_buffer +#error msgpack_pack_append_buffer callback is not defined +#endif + + +/* + * Integer + */ + +#define msgpack_pack_real_uint8(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ +} while(0) + +#define msgpack_pack_real_uint16(x, d) \ +do { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ +} while(0) + +#define msgpack_pack_real_uint32(x, d) \ +do { \ + if(d < (1<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_uint64(x, d) \ +do { \ + if(d < (1ULL<<8)) { \ + if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else { \ + if(d < (1ULL<<16)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else if(d < (1ULL<<32)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int8(x, d) \ +do { \ + if(d < -(1<<5)) { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \ + } \ +} while(0) + +#define msgpack_pack_real_int16(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int32(x, d) \ +do { \ + if(d < -(1<<5)) { \ + if(d < -(1<<15)) { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \ + } else { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else if(d < (1<<16)) { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } \ +} while(0) + +#define msgpack_pack_real_int64(x, d) \ +do { \ + if(d < -(1LL<<5)) { \ + if(d < -(1LL<<15)) { \ + if(d < -(1LL<<31)) { \ + /* signed 64 */ \ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } else { \ + /* signed 32 */ \ + const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } \ + } else { \ + if(d < -(1<<7)) { \ + /* signed 16 */ \ + const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } else { \ + /* signed 8 */ \ + const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } \ + } \ + } else if(d < (1<<7)) { \ + /* fixnum */ \ + msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \ + } else { \ + if(d < (1LL<<16)) { \ + if(d < (1<<8)) { \ + /* unsigned 8 */ \ + const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \ + msgpack_pack_append_buffer(x, buf, 2); \ + } else { \ + /* unsigned 16 */ \ + const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \ + msgpack_pack_append_buffer(x, buf, 3); \ + } \ + } else { \ + if(d < (1LL<<32)) { \ + /* unsigned 32 */ \ + const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \ + msgpack_pack_append_buffer(x, buf, 5); \ + } else { \ + /* unsigned 64 */ \ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \ + msgpack_pack_append_buffer(x, buf, 9); \ + } \ + } \ + } \ +} while(0) + + +#ifdef msgpack_pack_inline_func_fastint + +msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d) +{ + const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d) +{ + const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d) +{ + const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d) +{ + const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d) +{ + const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; + msgpack_pack_append_buffer(x, buf, 2); +} + +msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d) +{ + const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); +} + +msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d) +{ + const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d) +{ + const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; + msgpack_pack_append_buffer(x, buf, 9); +} + +#undef msgpack_pack_inline_func_fastint +#endif + + +msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d) +{ + msgpack_pack_real_uint8(x, d); +} + +msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d) +{ + msgpack_pack_real_uint16(x, d); +} + +msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d) +{ + msgpack_pack_real_uint32(x, d); +} + +msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d) +{ + msgpack_pack_real_uint64(x, d); +} + +msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d) +{ + msgpack_pack_real_int8(x, d); +} + +msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d) +{ + msgpack_pack_real_int16(x, d); +} + +msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d) +{ + msgpack_pack_real_int32(x, d); +} + +msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) +{ + msgpack_pack_real_int64(x, d); +} + + +#ifdef msgpack_pack_inline_func_cint + +msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) +{ +#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) +#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(short) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(short) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) +{ +#if defined(SIZEOF_INT) || defined(INT_MAX) +#if SIZEOF_INT == 2 || INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(int) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(int) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) +{ +#if defined(SIZEOF_LONG) || defined(LONG_MAX) +#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif +#else +if(sizeof(long long) == 2) { + msgpack_pack_real_int16(x, d); +} else if(sizeof(long long) == 4) { + msgpack_pack_real_int32(x, d); +} else { + msgpack_pack_real_int64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) +{ +#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) +#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned short) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned short) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) +{ +#if defined(SIZEOF_INT) || defined(UINT_MAX) +#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) +{ +#if defined(SIZEOF_LONG) || defined(ULONG_MAX) +#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned int) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned int) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) +{ +#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) +#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif +#else +if(sizeof(unsigned long long) == 2) { + msgpack_pack_real_uint16(x, d); +} else if(sizeof(unsigned long long) == 4) { + msgpack_pack_real_uint32(x, d); +} else { + msgpack_pack_real_uint64(x, d); +} +#endif +} + +#undef msgpack_pack_inline_func_cint +#endif + + + +/* + * Float + */ + +msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d) +{ + union { char buf[4]; uint32_t num; } f; + *((float*)&f.buf) = d; // FIXME + const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)}; + msgpack_pack_append_buffer(x, buf, 5); +} + +msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d) +{ + union { char buf[8]; uint64_t num; } f; + *((double*)&f.buf) = d; // FIXME + const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)}; + msgpack_pack_append_buffer(x, buf, 9); +} + + +/* + * Nil + */ + +msgpack_pack_inline_func(_nil)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc0; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Boolean + */ + +msgpack_pack_inline_func(_true)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc3; + msgpack_pack_append_buffer(x, &d, 1); +} + +msgpack_pack_inline_func(_false)(msgpack_pack_user x) +{ + static const unsigned char d = 0xc2; + msgpack_pack_append_buffer(x, &d, 1); +} + + +/* + * Array + */ + +msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x90 | n; + msgpack_pack_append_buffer(x, &d, 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xdc, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdd, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Map + */ + +msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n) +{ + if(n < 16) { + unsigned char d = 0x80 | n; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(n < 65536) { + uint16_t d = (uint16_t)n; + unsigned char buf[3] = {0xde, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)n; + unsigned char buf[5] = {0xdf, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + + +/* + * Raw + */ + +msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l) +{ + if(l < 32) { + unsigned char d = 0xa0 | l; + msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); + } else if(l < 65536) { + uint16_t d = (uint16_t)l; + unsigned char buf[3] = {0xda, STORE16_BE16(d)}; + msgpack_pack_append_buffer(x, buf, 3); + } else { + uint32_t d = (uint32_t)l; + unsigned char buf[5] = {0xdb, STORE32_BE32(d)}; + msgpack_pack_append_buffer(x, buf, 5); + } +} + +msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l) +{ + msgpack_pack_append_buffer(x, (const unsigned char*)b, l); +} + +#undef msgpack_pack_inline_func +#undef msgpack_pack_user +#undef msgpack_pack_append_buffer + +#undef STORE8_BE8 + +#undef STORE16_BE8 +#undef STORE16_BE16 + +#undef STORE32_BE8 +#undef STORE32_BE16 +#undef STORE32_BE32 + +#undef STORE64_BE8 +#undef STORE64_BE16 +#undef STORE64_BE32 +#undef STORE64_BE64 + +#undef msgpack_pack_real_uint8 +#undef msgpack_pack_real_uint16 +#undef msgpack_pack_real_uint32 +#undef msgpack_pack_real_uint64 +#undef msgpack_pack_real_int8 +#undef msgpack_pack_real_int16 +#undef msgpack_pack_real_int32 +#undef msgpack_pack_real_int64 + diff --git a/php/msgpack/unpack_define.h b/php/msgpack/unpack_define.h new file mode 100644 index 0000000..63668c2 --- /dev/null +++ b/php/msgpack/unpack_define.h @@ -0,0 +1,129 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MSGPACK_UNPACK_DEFINE_H__ +#define MSGPACK_UNPACK_DEFINE_H__ + +#include +#include +#include +#include +#include +#ifndef __WIN32__ +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifndef MSGPACK_MAX_STACK_SIZE +#define MSGPACK_MAX_STACK_SIZE 16 +#endif + + +#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__) +#if __BYTE_ORDER == __LITTLE_ENDIAN +#define __LITTLE_ENDIAN__ +#elif __BYTE_ORDER == __BIG_ENDIAN +#define __BIG_ENDIAN__ +#endif +#endif + +#define msgpack_betoh16(x) ntohs(x) +#define msgpack_betoh32(x) ntohl(x) + +#ifdef __LITTLE_ENDIAN__ +#if defined(__bswap_64) +# define msgpack_betoh64(x) __bswap_64(x) +#elif defined(__DARWIN_OSSwapInt64) +# define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x) +#else +static inline uint64_t msgpack_betoh64(uint64_t x) { + return ((x << 56) & 0xff00000000000000ULL ) | + ((x << 40) & 0x00ff000000000000ULL ) | + ((x << 24) & 0x0000ff0000000000ULL ) | + ((x << 8) & 0x000000ff00000000ULL ) | + ((x >> 8) & 0x00000000ff000000ULL ) | + ((x >> 24) & 0x0000000000ff0000ULL ) | + ((x >> 40) & 0x000000000000ff00ULL ) | + ((x >> 56) & 0x00000000000000ffULL ) ; +} +#endif +#else +#define msgpack_betoh64(x) (x) +#endif + + +typedef enum { + CS_HEADER = 0x00, // nil + + //CS_ = 0x01, + //CS_ = 0x02, // false + //CS_ = 0x03, // true + + //CS_ = 0x04, + //CS_ = 0x05, + //CS_ = 0x06, + //CS_ = 0x07, + + //CS_ = 0x08, + //CS_ = 0x09, + CS_FLOAT = 0x0a, + CS_DOUBLE = 0x0b, + CS_UINT_8 = 0x0c, + CS_UINT_16 = 0x0d, + CS_UINT_32 = 0x0e, + CS_UINT_64 = 0x0f, + CS_INT_8 = 0x10, + CS_INT_16 = 0x11, + CS_INT_32 = 0x12, + CS_INT_64 = 0x13, + + //CS_ = 0x14, + //CS_ = 0x15, + //CS_BIG_INT_16 = 0x16, + //CS_BIG_INT_32 = 0x17, + //CS_BIG_FLOAT_16 = 0x18, + //CS_BIG_FLOAT_32 = 0x19, + CS_RAW_16 = 0x1a, + CS_RAW_32 = 0x1b, + CS_ARRAY_16 = 0x1c, + CS_ARRAY_32 = 0x1d, + CS_MAP_16 = 0x1e, + CS_MAP_32 = 0x1f, + + //ACS_BIG_INT_VALUE, + //ACS_BIG_FLOAT_VALUE, + ACS_RAW_VALUE, +} msgpack_unpack_state; + + +typedef enum { + CT_ARRAY_ITEM, + CT_MAP_KEY, + CT_MAP_VALUE, +} msgpack_container_type; + + +#ifdef __cplusplus +} +#endif + +#endif /* msgpack/unpack_define.h */ + diff --git a/php/msgpack/unpack_template.h b/php/msgpack/unpack_template.h new file mode 100644 index 0000000..d67fd1e --- /dev/null +++ b/php/msgpack/unpack_template.h @@ -0,0 +1,361 @@ +/* + * MessagePack unpacking routine template + * + * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef msgpack_unpack_func +#error msgpack_unpack_func template is not defined +#endif + +#ifndef msgpack_unpack_callback +#error msgpack_unpack_callback template is not defined +#endif + +#ifndef msgpack_unpack_struct +#error msgpack_unpack_struct template is not defined +#endif + +#ifndef msgpack_unpack_struct_decl +#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name) +#endif + +#ifndef msgpack_unpack_object +#error msgpack_unpack_object type is not defined +#endif + +#ifndef msgpack_unpack_user +#error msgpack_unpack_user type is not defined +#endif + + +msgpack_unpack_struct_decl(_stack) { + msgpack_unpack_object obj; + size_t count; + unsigned int ct; + msgpack_unpack_object map_key; +}; + +msgpack_unpack_struct_decl(_context) { + msgpack_unpack_user user; + unsigned int cs; + unsigned int trail; + unsigned int top; + msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; +}; + + +msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) +{ + ctx->cs = CS_HEADER; + ctx->trail = 0; + ctx->top = 0; + ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); +} + +msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) +{ + return (ctx)->stack[0].obj; +} + + +msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off) +{ + assert(len >= *off); + + const unsigned char* p = (unsigned char*)data + *off; + const unsigned char* const pe = (unsigned char*)data + len; + const void* n = NULL; + + unsigned int trail = ctx->trail; + unsigned int cs = ctx->cs; + unsigned int top = ctx->top; + msgpack_unpack_struct(_stack)* stack = ctx->stack; + msgpack_unpack_user* user = &ctx->user; + + msgpack_unpack_object obj; + msgpack_unpack_struct(_stack)* c = NULL; + + int ret; + +#define push_simple_value(func) \ + if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \ + goto _push +#define push_fixed_value(func, arg) \ + if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \ + goto _push +#define push_variable_value(func, base, pos, len) \ + if(msgpack_unpack_callback(func)(user, \ + (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \ + goto _push + +#define again_fixed_trail(_cs, trail_len) \ + trail = trail_len; \ + cs = _cs; \ + goto _fixed_trail_again +#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \ + trail = trail_len; \ + if(trail == 0) { goto ifzero; } \ + cs = _cs; \ + goto _fixed_trail_again + +#define start_container(func, count_, ct_) \ + if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ + if((count_) == 0) { obj = stack[top].obj; goto _push; } \ + if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ + stack[top].ct = ct_; \ + stack[top].count = count_; \ + /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ + /*printf("stack push %d\n", top);*/ \ + ++top; \ + goto _header_again + +#define NEXT_CS(p) \ + ((unsigned int)*p & 0x1f) + +#define PTR_CAST_8(ptr) (*(uint8_t*)ptr) +#define PTR_CAST_16(ptr) msgpack_betoh16(*(uint16_t*)ptr) +#define PTR_CAST_32(ptr) msgpack_betoh32(*(uint32_t*)ptr) +#define PTR_CAST_64(ptr) msgpack_betoh64(*(uint64_t*)ptr) + + if(p == pe) { goto _out; } + do { + switch(cs) { + case CS_HEADER: + switch(*p) { + case 0x00 ... 0x7f: // Positive Fixnum + push_fixed_value(_uint8, *(uint8_t*)p); + case 0xe0 ... 0xff: // Negative Fixnum + push_fixed_value(_int8, *(int8_t*)p); + case 0xc0 ... 0xdf: // Variable + switch(*p) { + case 0xc0: // nil + push_simple_value(_nil); + //case 0xc1: // string + // again_terminal_trail(NEXT_CS(p), p+1); + case 0xc2: // false + push_simple_value(_false); + case 0xc3: // true + push_simple_value(_true); + //case 0xc4: + //case 0xc5: + //case 0xc6: + //case 0xc7: + //case 0xc8: + //case 0xc9: + case 0xca: // float + case 0xcb: // double + case 0xcc: // unsigned int 8 + case 0xcd: // unsigned int 16 + case 0xce: // unsigned int 32 + case 0xcf: // unsigned int 64 + case 0xd0: // signed int 8 + case 0xd1: // signed int 16 + case 0xd2: // signed int 32 + case 0xd3: // signed int 64 + again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03)); + //case 0xd4: + //case 0xd5: + //case 0xd6: // big integer 16 + //case 0xd7: // big integer 32 + //case 0xd8: // big float 16 + //case 0xd9: // big float 32 + case 0xda: // raw 16 + case 0xdb: // raw 32 + case 0xdc: // array 16 + case 0xdd: // array 32 + case 0xde: // map 16 + case 0xdf: // map 32 + again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01)); + default: + goto _failed; + } + case 0xa0 ... 0xbf: // FixRaw + again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero); + case 0x90 ... 0x9f: // FixArray + start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM); + case 0x80 ... 0x8f: // FixMap + start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY); + + default: + goto _failed; + } + // end CS_HEADER + + + _fixed_trail_again: + ++p; + + default: + if((size_t)(pe - p) < trail) { goto _out; } + n = p; p += trail - 1; + switch(cs) { + //case CS_ + //case CS_ + case CS_FLOAT: { + union { uint32_t num; char buf[4]; } f; + f.num = PTR_CAST_32(n); // FIXME + push_fixed_value(_float, *((float*)f.buf)); } + case CS_DOUBLE: { + union { uint64_t num; char buf[8]; } f; + f.num = PTR_CAST_64(n); // FIXME + push_fixed_value(_double, *((double*)f.buf)); } + case CS_UINT_8: + push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n)); + case CS_UINT_16: + push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n)); + case CS_UINT_32: + push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n)); + case CS_UINT_64: + push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n)); + + case CS_INT_8: + push_fixed_value(_int8, (int8_t)PTR_CAST_8(n)); + case CS_INT_16: + push_fixed_value(_int16, (int16_t)PTR_CAST_16(n)); + case CS_INT_32: + push_fixed_value(_int32, (int32_t)PTR_CAST_32(n)); + case CS_INT_64: + push_fixed_value(_int64, (int64_t)PTR_CAST_64(n)); + + //case CS_ + //case CS_ + //case CS_BIG_INT_16: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero); + //case CS_BIG_INT_32: + // again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero); + //case ACS_BIG_INT_VALUE: + //_big_int_zero: + // // FIXME + // push_variable_value(_big_int, data, n, trail); + + //case CS_BIG_FLOAT_16: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero); + //case CS_BIG_FLOAT_32: + // again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero); + //case ACS_BIG_FLOAT_VALUE: + //_big_float_zero: + // // FIXME + // push_variable_value(_big_float, data, n, trail); + + case CS_RAW_16: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero); + case CS_RAW_32: + again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero); + case ACS_RAW_VALUE: + _raw_zero: + push_variable_value(_raw, data, n, trail); + + case CS_ARRAY_16: + start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM); + case CS_ARRAY_32: + /* FIXME security guard */ + start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM); + + case CS_MAP_16: + start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY); + case CS_MAP_32: + /* FIXME security guard */ + start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY); + + default: + goto _failed; + } + } + +_push: + if(top == 0) { goto _finish; } + c = &stack[top-1]; + switch(c->ct) { + case CT_ARRAY_ITEM: + if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; } + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + goto _header_again; + case CT_MAP_KEY: + c->map_key = obj; + c->ct = CT_MAP_VALUE; + goto _header_again; + case CT_MAP_VALUE: + if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; } + if(--c->count == 0) { + obj = c->obj; + --top; + /*printf("stack pop %d\n", top);*/ + goto _push; + } + c->ct = CT_MAP_KEY; + goto _header_again; + + default: + goto _failed; + } + +_header_again: + cs = CS_HEADER; + ++p; + } while(p != pe); + goto _out; + + +_finish: + stack[0].obj = obj; + ++p; + ret = 1; + /*printf("-- finish --\n"); */ + goto _end; + +_failed: + /*printf("** FAILED **\n"); */ + ret = -1; + goto _end; + +_out: + ret = 0; + goto _end; + +_end: + ctx->cs = cs; + ctx->trail = trail; + ctx->top = top; + *off = p - (const unsigned char*)data; + + return ret; +} + + +#undef msgpack_unpack_func +#undef msgpack_unpack_callback +#undef msgpack_unpack_struct +#undef msgpack_unpack_object +#undef msgpack_unpack_user + +#undef push_simple_value +#undef push_fixed_value +#undef push_variable_value +#undef again_fixed_trail +#undef again_fixed_trail_if_zero +#undef start_container + +#undef NEXT_CS +#undef PTR_CAST_8 +#undef PTR_CAST_16 +#undef PTR_CAST_32 +#undef PTR_CAST_64 + diff --git a/php/php_msgpack.h b/php/php_msgpack.h new file mode 100644 index 0000000..a1f434c --- /dev/null +++ b/php/php_msgpack.h @@ -0,0 +1,86 @@ +/* + +----------------------------------------------------------------------+ + | PHP Version 5 | + +----------------------------------------------------------------------+ + | Copyright (c) 1997-2007 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Author: | + +----------------------------------------------------------------------+ +*/ + +/* $Id: header 226204 2007-01-01 19:32:10Z iliaa $ */ + +#ifndef PHP_MSGPACK_H +#define PHP_MSGPACK_H + +extern zend_module_entry msgpack_module_entry; +#define phpext_msgpack_ptr &msgpack_module_entry + +#ifdef PHP_WIN32 +#define PHP_MSGPACK_API __declspec(dllexport) +#else +#define PHP_MSGPACK_API +#endif + +#ifdef ZTS +#include "TSRM.h" +#endif + +PHP_MINIT_FUNCTION(msgpack); +PHP_MSHUTDOWN_FUNCTION(msgpack); +PHP_RINIT_FUNCTION(msgpack); +PHP_RSHUTDOWN_FUNCTION(msgpack); +PHP_MINFO_FUNCTION(msgpack); + +PHP_FUNCTION(msgpack_pack); +PHP_FUNCTION(msgpack_unpack); +PHP_FUNCTION(msgpack_unpack_limit); + +PHP_METHOD(msgpack, initialize); +PHP_METHOD(msgpack, execute); +PHP_METHOD(msgpack, execute_limit); +PHP_METHOD(msgpack, finished); +PHP_METHOD(msgpack, data); + +static zend_class_entry *msgpack_ce; + +/* + Declare any global variables you may need between the BEGIN + and END macros here: + +ZEND_BEGIN_MODULE_GLOBALS(msgpack) + long global_value; + char *global_string; +ZEND_END_MODULE_GLOBALS(msgpack) +*/ + +/* In every utility function you add that needs to use variables + in php_msgpack_globals, call TSRMLS_FETCH(); after declaring other + variables used by that function, or better yet, pass in TSRMLS_CC + after the last function argument and declare your utility function + with TSRMLS_DC after the last declared argument. Always refer to + the globals in your function as MSGPACK_G(variable). You are + encouraged to rename these macros something shorter, see + examples in any other php module directory. +*/ + + +#endif /* PHP_MSGPACK_H */ + + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * End: + * vim600: noet sw=4 ts=4 fdm=marker + * vim<600: noet sw=4 ts=4 + */ diff --git a/php/test_normal.php b/php/test_normal.php new file mode 100755 index 0000000..ee348c2 --- /dev/null +++ b/php/test_normal.php @@ -0,0 +1,19 @@ +1), array("takei"=>"hide"), 3); + //$data = array("more"=>10, "test", null); + //$data = array(); + $data = array(0=>1,1=>2,2=>3); + var_dump($data); + + // serialize + $msg = msgpack_pack($data); + + // hexadecimal + $str = unpack('H*', $msg); + var_dump("0x".$str[1]); + + // deserialize + $ret = msgpack_unpack($msg); + var_dump($ret); +?> + diff --git a/php/test_streaming.php b/php/test_streaming.php new file mode 100755 index 0000000..8f89f4c --- /dev/null +++ b/php/test_streaming.php @@ -0,0 +1,34 @@ +initialize(); + $buffer = ""; + $nread = 0; + + foreach($msgs as $msg){ + $buffer = $buffer . $msg; + + while(true){ + $nread = $unpacker->execute($buffer, $nread); + + if($unpacker->finished()){ + $msg = $unpacker->data(); + var_dump($msg); + + $unpacker->initialize(); + $buffer = substr($buffer, $nread); + $nread = 0; + + if(!empty($buffer)){ + continue; + } + } + break; + } + } +?> + diff --git a/php/tests/test_pack.phpt b/php/tests/test_pack.phpt new file mode 100644 index 0000000..595e4d0 --- /dev/null +++ b/php/tests/test_pack.phpt @@ -0,0 +1,169 @@ +--TEST-- +Test msgpack_pack() function : basic functionality +--SKIPIF-- + +--FILE-- + +===DONE=== +--EXPECT-- +*** Testing msgpack_pack() : basic functionality *** +-- Iteration 1 -- +string(4) "0xc0" +-- Iteration 2 -- +string(4) "0xc0" +-- Iteration 3 -- +string(4) "0xc2" +-- Iteration 4 -- +string(4) "0xc2" +-- Iteration 5 -- +string(4) "0xc3" +-- Iteration 6 -- +string(4) "0xc3" +-- Iteration 7 -- +string(4) "0x00" +-- Iteration 8 -- +string(4) "0x7f" +-- Iteration 9 -- +string(6) "0xcc80" +-- Iteration 10 -- +string(8) "0xcd0100" +-- Iteration 11 -- +string(4) "0xff" +-- Iteration 12 -- +string(6) "0xd0df" +-- Iteration 13 -- +string(8) "0xd1ff7f" +-- Iteration 14 -- +string(8) "0x810101" +-- Iteration 15 -- +string(20) "0xcb3ff0000000000000" +-- Iteration 16 -- +string(4) "0x90" +-- Iteration 17 -- +string(34) "0x9f000102030405060708090a0b0c0d0e" +-- Iteration 18 -- +string(40) "0xdc0010000102030405060708090a0b0c0d0e0f" +-- Iteration 19 -- +string(64) "0x8f0100020103020403050406050706080709080a090b0a0c0b0d0c0e0d0f0e" +-- Iteration 20 -- +string(72) "0xde00100100020103020403050406050706080709080a090b0a0c0b0d0c0e0d0f0e100f" +===DONE=== From d639f5747008a6246a81d648c5f3d1cb0e90c162 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Apr 2010 17:46:38 +0900 Subject: [PATCH 0393/1648] ruby: fixes Segmentation fault on MessagePack.unpack(nil) --- ruby/unpack.c | 70 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index df54e75..88a8239 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -220,15 +220,18 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) } -static VALUE MessagePack_Unpacker_execute_impl(VALUE args) +static VALUE MessagePack_Unpacker_execute_do(VALUE argv) { - VALUE self = ((VALUE*)args)[0]; - VALUE data = ((VALUE*)args)[1]; + VALUE* args = (VALUE*)argv; + VALUE self = args[0]; UNPACKER(self, mp); - size_t from = NUM2UINT(((VALUE*)args)[2]); + + VALUE data = args[1]; + + size_t from = (unsigned long)args[2]; char* dptr = RSTRING_PTR(data); - long dlen = FIX2LONG(((VALUE*)args)[3]); + size_t dlen = (unsigned long)args[3]; int ret; if(from >= dlen) { @@ -236,7 +239,7 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE args) } mp->user.source = data; - ret = template_execute(mp, dptr, (size_t)dlen, &from); + ret = template_execute(mp, dptr, dlen, &from); mp->user.source = Qnil; if(ret < 0) { @@ -260,22 +263,32 @@ static VALUE MessagePack_Unpacker_execute_rescue(VALUE nouse) #endif } -static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, - VALUE off, VALUE limit) +static inline VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data, + unsigned long off, unsigned long dlen) { // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ rb_gc_disable(); - VALUE args[4] = {self, data, off, limit}; - VALUE ret = rb_rescue(MessagePack_Unpacker_execute_impl, (VALUE)args, + VALUE args[4] = {self, data, (VALUE)off, (VALUE)dlen}; + VALUE ret = rb_rescue(MessagePack_Unpacker_execute_do, (VALUE)args, MessagePack_Unpacker_execute_rescue, Qnil); rb_gc_enable(); + return ret; } +static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, + VALUE off, VALUE limit) +{ + CHECK_STRING_TYPE(data); + return MessagePack_Unpacker_execute_impl(self, data, + NUM2ULONG(off), NUM2ULONG(limit)); +} + static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) { - return MessagePack_Unpacker_execute_limit(self, data, off, - LONG2FIX(RSTRING_LEN(data))); + CHECK_STRING_TYPE(data); + return MessagePack_Unpacker_execute_impl(self, data, + NUM2ULONG(off), RSTRING_LEN(data)); } static VALUE MessagePack_Unpacker_finished_p(VALUE self) @@ -379,18 +392,20 @@ static VALUE MessagePack_Unpacker_each(VALUE self) } -static VALUE MessagePack_unpack_impl(VALUE args) +static VALUE MessagePack_unpack_do(VALUE argv) { - msgpack_unpack_t* mp = (msgpack_unpack_t*)((VALUE*)args)[0]; - VALUE data = ((VALUE*)args)[1]; + VALUE* args = (VALUE*)argv; + + msgpack_unpack_t* mp = (msgpack_unpack_t*)args[0]; + VALUE data = args[1]; size_t from = 0; char* dptr = RSTRING_PTR(data); - long dlen = FIX2LONG(((VALUE*)args)[2]); + size_t dlen = (unsigned long)args[2]; int ret; mp->user.source = data; - ret = template_execute(mp, dptr, (size_t)dlen, &from); + ret = template_execute(mp, dptr, dlen, &from); mp->user.source = Qnil; if(ret < 0) { @@ -405,7 +420,7 @@ static VALUE MessagePack_unpack_impl(VALUE args) } } -static VALUE MessagePack_unpack_rescue(VALUE args) +static VALUE MessagePack_unpack_rescue(VALUE nouse) { rb_gc_enable(); #ifdef RUBY_VM @@ -415,29 +430,34 @@ static VALUE MessagePack_unpack_rescue(VALUE args) #endif } -static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) +static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned long dlen) { - CHECK_STRING_TYPE(data); - msgpack_unpack_t mp; template_init(&mp); init_stack(&mp); unpack_user u = {0, Qnil, 0, 0, Qnil, Qnil, Qnil}; mp.user = u; + // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ rb_gc_disable(); - VALUE args[3] = {(VALUE)&mp, data, limit}; - VALUE ret = rb_rescue(MessagePack_unpack_impl, (VALUE)args, + VALUE args[3] = {(VALUE)&mp, data, (VALUE)dlen}; + VALUE ret = rb_rescue(MessagePack_unpack_do, (VALUE)args, MessagePack_unpack_rescue, Qnil); rb_gc_enable(); return ret; } +static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) +{ + CHECK_STRING_TYPE(data); + return MessagePack_unpack_impl(self, data, NUM2ULONG(limit)); +} + static VALUE MessagePack_unpack(VALUE self, VALUE data) { - return MessagePack_unpack_limit(self, data, - LONG2FIX(RSTRING_LEN(data))); + CHECK_STRING_TYPE(data); + return MessagePack_unpack_impl(self, data, RSTRING_LEN(data)); } From c6186f2c01e159644b7ed19e865c8ac20d169453 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 6 Apr 2010 17:59:34 +0900 Subject: [PATCH 0394/1648] ruby: version 0.3.7 --- ruby/msgpack.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 4f7fcec..3a261f2 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.6" + s.version = "0.3.7" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From bd36ac2c0c99fa8e95b50ba12c2cc8ccb8b413f4 Mon Sep 17 00:00:00 2001 From: Masahiro Nakagawa Date: Wed, 7 Apr 2010 19:54:14 +0900 Subject: [PATCH 0395/1648] cpp: fixes argument type of pack_int* methods --- cpp/msgpack/pack.hpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/msgpack/pack.hpp b/cpp/msgpack/pack.hpp index 6a91971..3a68aa6 100644 --- a/cpp/msgpack/pack.hpp +++ b/cpp/msgpack/pack.hpp @@ -1,4 +1,4 @@ -// +1// // MessagePack for C++ serializing routine // // Copyright (C) 2008-2009 FURUHASHI Sadayuki @@ -40,10 +40,10 @@ public: packer& pack_uint16(uint16_t d); packer& pack_uint32(uint32_t d); packer& pack_uint64(uint64_t d); - packer& pack_int8(uint8_t d); - packer& pack_int16(uint16_t d); - packer& pack_int32(uint32_t d); - packer& pack_int64(uint64_t d); + packer& pack_int8(int8_t d); + packer& pack_int16(int16_t d); + packer& pack_int32(int32_t d); + packer& pack_int64(int64_t d); packer& pack_short(short d); packer& pack_int(int d); @@ -166,19 +166,19 @@ inline packer& packer::pack_uint64(uint64_t d) { _pack_uint64(m_stream, d); return *this; } template -inline packer& packer::pack_int8(uint8_t d) +inline packer& packer::pack_int8(int8_t d) { _pack_int8(m_stream, d); return *this; } template -inline packer& packer::pack_int16(uint16_t d) +inline packer& packer::pack_int16(int16_t d) { _pack_int16(m_stream, d); return *this; } template -inline packer& packer::pack_int32(uint32_t d) +inline packer& packer::pack_int32(int32_t d) { _pack_int32(m_stream, d); return *this; } template -inline packer& packer::pack_int64(uint64_t d) +inline packer& packer::pack_int64(int64_t d) { _pack_int64(m_stream, d); return *this;} From e79747a60015e11ab49e10d829f8fdab8ccf8945 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 7 Apr 2010 20:02:00 +0900 Subject: [PATCH 0396/1648] cpp: msgpack/pack.hpp: fixes header --- cpp/msgpack/pack.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/msgpack/pack.hpp b/cpp/msgpack/pack.hpp index 3a68aa6..ee90690 100644 --- a/cpp/msgpack/pack.hpp +++ b/cpp/msgpack/pack.hpp @@ -1,7 +1,7 @@ -1// +// // MessagePack for C++ serializing routine // -// Copyright (C) 2008-2009 FURUHASHI Sadayuki +// Copyright (C) 2008-2010 FURUHASHI Sadayuki // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 1f18af43955b22eed97b6fcb7b53c0389e4bc0f9 Mon Sep 17 00:00:00 2001 From: Masahiro Nakagawa Date: Thu, 8 Apr 2010 23:01:19 +0900 Subject: [PATCH 0397/1648] c: fixes comment in pack_template.h --- msgpack/pack_template.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index 493634d..f8f522f 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -113,17 +113,17 @@ do { \ } \ } else { \ if(d < (1ULL<<16)) { \ - /* signed 16 */ \ + /* unsigned 16 */ \ unsigned char buf[3]; \ buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 3); \ } else if(d < (1ULL<<32)) { \ - /* signed 32 */ \ + /* unsigned 32 */ \ unsigned char buf[5]; \ buf[0] = 0xce; _msgpack_store32(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 5); \ } else { \ - /* signed 64 */ \ + /* unsigned 64 */ \ unsigned char buf[9]; \ buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \ msgpack_pack_append_buffer(x, buf, 9); \ From 87835a4e60e8f7ba5c2d4283ed44f57b725d9628 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 14 Apr 2010 21:08:06 +0900 Subject: [PATCH 0398/1648] ruby: remove init_stack, adopt rb_gc_mark_maybe --- ruby/makegem.sh | 4 ++++ ruby/msgpack_test.rb | 15 +++++++++++++++ ruby/unpack.c | 14 +------------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/ruby/makegem.sh b/ruby/makegem.sh index c78769b..8737d29 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -19,6 +19,10 @@ cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FI gem build msgpack.gemspec +if [ $? -eq 0 ]; then + rm -rf ext msgpack AUTHORS ChangeLog test/msgpack_test.rb +fi + # gem install gem-compile # on msys # gem compile msgpack-$version.gem # on msys # gem compile msgpack-$version.gem -p mswin32 # on msys diff --git a/ruby/msgpack_test.rb b/ruby/msgpack_test.rb index 4fbcea3..37db6a0 100644 --- a/ruby/msgpack_test.rb +++ b/ruby/msgpack_test.rb @@ -202,6 +202,21 @@ class MessagePackTestFormat < Test::Unit::TestCase # #check_map 5, (1<<32)-1 # memory error # end + it "gc mark" do + obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + pac = MessagePack::Unpacker.new + parsed = 0 + obj.to_msgpack.split(//).each do |b| + pac.feed(b) + pac.each {|o| + assert_equal(obj, o) + parsed += 1 + } + GC.start + end + assert_equal(parsed, 1) + end + private def check(len, obj) v = obj.to_msgpack diff --git a/ruby/unpack.c b/ruby/unpack.c index 88a8239..5f5b64b 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -141,15 +141,6 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha static VALUE cUnpacker; static VALUE eUnpackError; -// FIXME slow operation -static void init_stack(msgpack_unpack_t* mp) -{ - size_t i; - for(i=0; i < MSGPACK_MAX_STACK_SIZE; ++i) { - mp->stack[i].map_key = Qnil; /* GC */ - } -} - static void MessagePack_Unpacker_free(void* data) { if(data) { free(data); } @@ -163,7 +154,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) rb_gc_mark(mp->user.streambuf); for(i=0; i < mp->top; ++i) { rb_gc_mark(mp->stack[i].obj); - rb_gc_mark(mp->stack[i].map_key); /* maybe map_key is not initialized */ + rb_gc_mark_maybe(mp->stack[i].map_key); } } @@ -180,7 +171,6 @@ static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); template_init(mp); - init_stack(mp); mp->user.finished = 0; return self; } @@ -381,7 +371,6 @@ static VALUE MessagePack_Unpacker_each(VALUE self) } else if(ret > 0) { VALUE data = template_data(mp); template_init(mp); - init_stack(mp); rb_yield(data); } else { goto do_fill; @@ -434,7 +423,6 @@ static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned lon { msgpack_unpack_t mp; template_init(&mp); - init_stack(&mp); unpack_user u = {0, Qnil, 0, 0, Qnil, Qnil, Qnil}; mp.user = u; From a6ec726ed7ca72662f0abbc8e1d8e361201483ae Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 14 Apr 2010 21:11:31 +0900 Subject: [PATCH 0399/1648] malloc/realloc the stack when its length becomes > MSGPACK_EMBED_STACK_SIZE --- c/unpack.c | 22 +++++++++++------ cpp/msgpack/object.hpp | 52 ++++++++++++++++++++++++++++++++++++--- msgpack/unpack_define.h | 5 ++-- msgpack/unpack_template.h | 33 ++++++++++++++++++++++--- perl/unpack.c | 8 +++++- 5 files changed, 103 insertions(+), 17 deletions(-) diff --git a/c/unpack.c b/c/unpack.c index 8be0b92..4334974 100644 --- a/c/unpack.c +++ b/c/unpack.c @@ -44,6 +44,7 @@ struct template_context; typedef struct template_context template_context; static void template_init(template_context* ctx); +static void template_destroy(template_context* ctx); static msgpack_object template_data(template_context* ctx); @@ -215,6 +216,7 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) void msgpack_unpacker_destroy(msgpack_unpacker* mpac) { msgpack_zone_free(mpac->z); + template_destroy(mpac->ctx); free(mpac->ctx); decl_count(mpac->buffer); } @@ -368,6 +370,7 @@ msgpack_unpack_return msgpack_unpack(const char* data, size_t len, size_t* off, msgpack_zone* z, msgpack_object* result) { + msgpack_unpack_return ret = MSGPACK_UNPACK_SUCCESS; template_context ctx; template_init(&ctx); @@ -377,23 +380,28 @@ msgpack_unpack(const char* data, size_t len, size_t* off, size_t noff = 0; if(off != NULL) { noff = *off; } - int ret = template_execute(&ctx, data, len, &noff); - if(ret < 0) { - return MSGPACK_UNPACK_PARSE_ERROR; + int e = template_execute(&ctx, data, len, &noff); + if(e < 0) { + ret = MSGPACK_UNPACK_PARSE_ERROR; + goto out; } if(off != NULL) { *off = noff; } - if(ret == 0) { - return MSGPACK_UNPACK_CONTINUE; + if(e == 0) { + ret = MSGPACK_UNPACK_CONTINUE; + goto out; } *result = template_data(&ctx); if(noff < len) { - return MSGPACK_UNPACK_EXTRA_BYTES; + ret = MSGPACK_UNPACK_EXTRA_BYTES; + goto out; } - return MSGPACK_UNPACK_SUCCESS; +out: + template_destroy(&ctx); + return ret; } diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index ed2e290..3b42a8e 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -87,9 +87,15 @@ struct object { template void convert(T* v) const; - object(); - object(msgpack_object obj); operator msgpack_object(); + object(msgpack_object obj); + + object(); + object(bool v); + object(uint64_t v); + object(int64_t v); + object(double v); + object(const char* ptr, size_t size); private: struct implicit_type; @@ -184,7 +190,46 @@ inline bool operator!=(const object x, const object y) { return !(x == y); } -inline object::object() { } +inline object::object() +{ + type = type::NIL; +} + +inline object::object(bool v) +{ + type = type::BOOLEAN; + via.boolean = v; +} + +inline object::object(uint64_t v) +{ + type = type::POSITIVE_INTEGER; + via.u64 = v; +} + +inline object::object(int64_t v) +{ + if(v >= 0) { + type = type::POSITIVE_INTEGER; + via.u64 = v; + } else { + type = type::NEGATIVE_INTEGER; + via.i64 = v; + } +} + +inline object::object(double v) +{ + type = type::DOUBLE; + via.dec = v; +} + +inline object::object(const char* ptr, size_t size) +{ + type = type::RAW; + via.raw.size = size; + via.raw.ptr = ptr; +} inline object::object(msgpack_object obj) { @@ -200,7 +245,6 @@ inline object::operator msgpack_object() return obj; } - inline object::implicit_type object::convert() const { return implicit_type(*this); diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index e61f73d..cb02a3b 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -19,6 +19,7 @@ #define MSGPACK_UNPACK_DEFINE_H__ #include "msgpack/sysdep.h" +#include #include #include #include @@ -28,8 +29,8 @@ extern "C" { #endif -#ifndef MSGPACK_MAX_STACK_SIZE -#define MSGPACK_MAX_STACK_SIZE 16 +#ifndef MSGPACK_EMBED_STACK_SIZE +#define MSGPACK_EMBED_STACK_SIZE 1 #endif diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 3f871cd..72f8e59 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -58,7 +58,9 @@ msgpack_unpack_struct_decl(_context) { unsigned int cs; unsigned int trail; unsigned int top; - msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE]; + msgpack_unpack_struct(_stack)* stack; + unsigned int stack_size; + msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; }; @@ -67,9 +69,18 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; + ctx->stack = ctx->embed_stack; + ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } +msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) +{ + if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { + free(ctx->stack); + } +} + msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { return (ctx)->stack[0].obj; @@ -119,12 +130,28 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c #define start_container(func, count_, ct_) \ if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; goto _push; } \ - if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \ stack[top].ct = ct_; \ stack[top].count = count_; \ + ++top; \ /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ /*printf("stack push %d\n", top);*/ \ - ++top; \ + if(top >= ctx->stack_size) { \ + if(ctx->stack_size == MSGPACK_EMBED_STACK_SIZE) { \ + size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ + size_t nsize = csize * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ + if(tmp == NULL) { goto _failed; } \ + memcpy(tmp, ctx->stack, csize); \ + ctx->stack = tmp; \ + ctx->stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ + } else { \ + size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ + msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ + if(tmp == NULL) { goto _failed; } \ + ctx->stack = tmp; \ + ctx->stack_size *= 2; \ + } \ + } \ goto _header_again #define NEXT_CS(p) \ diff --git a/perl/unpack.c b/perl/unpack.c index 7a4f422..c520e02 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -53,6 +53,7 @@ struct template_context; typedef struct template_context msgpack_unpack_t; static void template_init(msgpack_unpack_t* u); +static void template_destroy(msgpack_unpack_t* u); static SV* template_data(msgpack_unpack_t* u); @@ -132,6 +133,7 @@ SV* _msgpack_unpack(SV* data, int limit) { size_t from = 0; STRLEN dlen; const char * dptr = SvPV_const(data, dlen); + SV* obj; template_init(&mp); mp.user = u; @@ -140,6 +142,9 @@ SV* _msgpack_unpack(SV* data, int limit) { ret = template_execute(&mp, dptr, (size_t)dlen, &from); mp.user.source = &PL_sv_undef; + obj = template_data(&mp); + template_destroy(&mp); + if(ret < 0) { Perl_croak(aTHX_ "parse error."); } else if(ret == 0) { @@ -148,7 +153,7 @@ SV* _msgpack_unpack(SV* data, int limit) { if(from < dlen) { Perl_croak(aTHX_ "extra bytes."); } - return template_data(&mp); + return obj; } } @@ -311,6 +316,7 @@ XS(xs_unpacker_destroy) { } UNPACKER(ST(0), mp); + template_destroy(mp); Safefree(mp); XSRETURN(0); From a55affe4d5bce92ee692fd80b10365b6ec49804c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 17 Apr 2010 16:16:56 +0900 Subject: [PATCH 0400/1648] ruby: add Symbol#to_msgpack --- ruby/pack.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ruby/pack.c b/ruby/pack.c index 1d0068c..387bab6 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -112,6 +112,16 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } +static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) +{ + ARG_BUFFER(out, argc, argv); + const char* name = rb_id2name(SYM2ID(self)); + size_t len = strlen(name); + msgpack_pack_raw(out, len); + msgpack_pack_raw_body(out, name, len); + return out; +} + static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -172,6 +182,7 @@ void Init_msgpack_pack(VALUE mMessagePack) rb_define_method_id(rb_cString, s_to_msgpack, MessagePack_String_to_msgpack, -1); rb_define_method_id(rb_cArray, s_to_msgpack, MessagePack_Array_to_msgpack, -1); rb_define_method_id(rb_cHash, s_to_msgpack, MessagePack_Hash_to_msgpack, -1); + rb_define_method_id(rb_cSymbol, s_to_msgpack, MessagePack_Symbol_to_msgpack, -1); rb_define_module_function(mMessagePack, "pack", MessagePack_pack, -1); } From 228f742b2f4bfba8bc97f16b3fb85ca616b07768 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 17 Apr 2010 20:02:47 +0900 Subject: [PATCH 0401/1648] ruby: set encoding to 'ASCII-8BIT' before deserializing on ruby-1.9 --- ruby/unpack.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ruby/unpack.c b/ruby/unpack.c index 5f5b64b..11572c3 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -16,11 +16,17 @@ * limitations under the License. */ #include "ruby.h" + #include "msgpack/unpack_define.h" static ID s_sysread; static ID s_readpartial; +#ifdef HAVE_RUBY_ENCODING_H +#include "ruby/encoding.h" +int s_ascii_8bit; +#endif + typedef struct { int finished; VALUE source; @@ -426,6 +432,12 @@ static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned lon unpack_user u = {0, Qnil, 0, 0, Qnil, Qnil, Qnil}; mp.user = u; +#ifdef HAVE_RUBY_ENCODING_H + // FIXME encodingã‚’ASCII-8BITã«ã™ã‚‹ + int enc_orig = rb_enc_get_index(data); + rb_enc_set_index(data, s_ascii_8bit); +#endif + // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ rb_gc_disable(); VALUE args[3] = {(VALUE)&mp, data, (VALUE)dlen}; @@ -433,6 +445,10 @@ static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned lon MessagePack_unpack_rescue, Qnil); rb_gc_enable(); +#ifdef HAVE_RUBY_ENCODING_H + rb_enc_set_index(data, enc_orig); +#endif + return ret; } @@ -453,6 +469,11 @@ void Init_msgpack_unpack(VALUE mMessagePack) { s_sysread = rb_intern("sysread"); s_readpartial = rb_intern("readpartial"); + +#ifdef HAVE_RUBY_ENCODING_H + s_ascii_8bit = rb_enc_find_index("ASCII-8BIT"); +#endif + eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); From b4b1f0a2c961529cdabd705b2581350323952f74 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 17 Apr 2010 20:16:48 +0900 Subject: [PATCH 0402/1648] introduce JUnit framework for testing. --- java/build.xml | 207 +++++++++++++++++++++++--- java/ivy.xml | 19 +++ java/test/org/msgpack/TestSample.java | 11 ++ 3 files changed, 220 insertions(+), 17 deletions(-) create mode 100644 java/ivy.xml create mode 100644 java/test/org/msgpack/TestSample.java diff --git a/java/build.xml b/java/build.xml index b98f7c3..35ee2fe 100644 --- a/java/build.xml +++ b/java/build.xml @@ -1,18 +1,191 @@ - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tests Failed! + + + + + + + + + + diff --git a/java/ivy.xml b/java/ivy.xml new file mode 100644 index 0000000..6d98756 --- /dev/null +++ b/java/ivy.xml @@ -0,0 +1,19 @@ + + + + + MessagePack + + + + + + + + + + + + diff --git a/java/test/org/msgpack/TestSample.java b/java/test/org/msgpack/TestSample.java new file mode 100644 index 0000000..d0e3bae --- /dev/null +++ b/java/test/org/msgpack/TestSample.java @@ -0,0 +1,11 @@ +package org.msgpack; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +public class TestSample { + @Test + public void testNull() throws Exception { + assertEquals("aiueo", 0, 0); + } +}; From 08b716c96d02e281cc096783cbe64932f70919ef Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 17 Apr 2010 21:16:32 +0900 Subject: [PATCH 0403/1648] java: add TestPackUnpack for int --- java/build.xml | 8 +++- java/test/org/msgpack/TestPackUnpack.java | 52 +++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 java/test/org/msgpack/TestPackUnpack.java diff --git a/java/build.xml b/java/build.xml index 35ee2fe..baecd01 100644 --- a/java/build.xml +++ b/java/build.xml @@ -14,6 +14,7 @@ + @@ -64,6 +65,9 @@ + + + @@ -146,7 +150,7 @@ - + @@ -178,7 +182,7 @@ Tests Failed! - + diff --git a/java/test/org/msgpack/TestPackUnpack.java b/java/test/org/msgpack/TestPackUnpack.java new file mode 100644 index 0000000..f8eeae7 --- /dev/null +++ b/java/test/org/msgpack/TestPackUnpack.java @@ -0,0 +1,52 @@ +package org.msgpack; + +import org.msgpack.*; +import java.io.*; +import java.util.*; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class TestPackUnpack { + public Object unpackOne(ByteArrayOutputStream out) { + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker upk = new Unpacker(in); + Iterator it = upk.iterator(); + assertEquals(true, it.hasNext()); + Object obj = it.next(); + assertEquals(false, it.hasNext()); + return obj; + } + + @Test + public void testInt() throws Exception { + testInt(0); + testInt(-1); + testInt(1); + testInt(Integer.MIN_VALUE); + testInt(Integer.MAX_VALUE); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testInt(rand.nextInt()); + } + public void testInt(int val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Packer pk = new Packer(out); + pk.pack(val); + Object obj = unpackOne(out); + int val2 = -1; + if (obj instanceof Byte) + val2 = ((Byte)obj).intValue(); + else if (obj instanceof Integer) + val2 = ((Integer)obj).intValue(); + else if (obj instanceof Short) + val2 = ((Short)obj).intValue(); + else if (obj instanceof Long) + val2 = ((Long)obj).intValue(); + else { + System.out.println("obj = " + obj.getClass()); + assertTrue(false); + } + assertEquals(val, val2); + } +}; From 2807504a814fd54f91b32e58476a80b5ca40c2fc Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 17 Apr 2010 22:10:41 +0900 Subject: [PATCH 0404/1648] java: add tests for float, double, nil, boolean, string --- java/test/org/msgpack/TestPackUnpack.java | 140 ++++++++++++++++++++-- 1 file changed, 130 insertions(+), 10 deletions(-) diff --git a/java/test/org/msgpack/TestPackUnpack.java b/java/test/org/msgpack/TestPackUnpack.java index f8eeae7..1a02cc5 100644 --- a/java/test/org/msgpack/TestPackUnpack.java +++ b/java/test/org/msgpack/TestPackUnpack.java @@ -8,7 +8,7 @@ import org.junit.Test; import static org.junit.Assert.*; public class TestPackUnpack { - public Object unpackOne(ByteArrayOutputStream out) { + protected Object unpackOne(ByteArrayOutputStream out) { ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); Unpacker upk = new Unpacker(in); Iterator it = upk.iterator(); @@ -31,22 +31,142 @@ public class TestPackUnpack { } public void testInt(int val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); - Packer pk = new Packer(out); - pk.pack(val); + new Packer(out).pack(val); Object obj = unpackOne(out); - int val2 = -1; if (obj instanceof Byte) - val2 = ((Byte)obj).intValue(); + assertEquals(val, ((Byte)obj).intValue()); else if (obj instanceof Integer) - val2 = ((Integer)obj).intValue(); + assertEquals(val, ((Integer)obj).intValue()); else if (obj instanceof Short) - val2 = ((Short)obj).intValue(); + assertEquals(val, ((Short)obj).intValue()); else if (obj instanceof Long) - val2 = ((Long)obj).intValue(); + assertEquals(val, ((Long)obj).intValue()); else { - System.out.println("obj = " + obj.getClass()); + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } + + @Test + public void testFloat() throws Exception { + testFloat((float)0.0); + testFloat((float)-0.0); + testFloat((float)1.0); + testFloat((float)-1.0); + testFloat((float)Float.MAX_VALUE); + testFloat((float)Float.MIN_VALUE); + testFloat((float)Float.NaN); + testFloat((float)Float.NEGATIVE_INFINITY); + testFloat((float)Float.POSITIVE_INFINITY); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testFloat(rand.nextFloat()); + } + public void testFloat(float val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out); + if (obj instanceof Float) + assertEquals(val, ((Float)obj).floatValue(), 10e-10); + else { + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } + + @Test + public void testDouble() throws Exception { + testDouble((double)0.0); + testDouble((double)-0.0); + testDouble((double)1.0); + testDouble((double)-1.0); + testDouble((double)Double.MAX_VALUE); + testDouble((double)Double.MIN_VALUE); + testDouble((double)Double.NaN); + testDouble((double)Double.NEGATIVE_INFINITY); + testDouble((double)Double.POSITIVE_INFINITY); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testDouble(rand.nextDouble()); + } + public void testDouble(double val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out); + if (obj instanceof Double) + assertEquals(val, ((Double)obj).doubleValue(), 10e-10); + else { + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } + + @Test + public void testNil() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).packNil(); + Object obj = unpackOne(out); + assertEquals(null, obj); + } + + @Test + public void testBoolean() throws Exception { + testBoolean(false); + testBoolean(true); + } + public void testBoolean(boolean val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out); + if (obj instanceof Boolean) + assertEquals(val, ((Boolean)obj).booleanValue()); + else { + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } + + @Test + public void testString() throws Exception { + testString(""); + testString("a"); + testString("ab"); + testString("abc"); + // small size string + for (int i = 0; i < 100; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 31 + 1; + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + // medium size string + for (int i = 0; i < 100; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 100 + (1 << 15); + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + // large size string + for (int i = 0; i < 10; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 100 + (1 << 31); + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + } + public void testString(String val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out); + if (obj instanceof byte[]) + assertEquals(val, new String((byte[])obj)); + else { + System.out.println("obj=" + obj); + System.out.println("Got unexpected class: " + obj.getClass()); assertTrue(false); } - assertEquals(val, val2); } }; From c3f43fb0cf14f596a9ebb1c30b0417b3a8ba9958 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 17 Apr 2010 22:43:11 +0900 Subject: [PATCH 0405/1648] template_execute: fixes embed stack --- msgpack/unpack_define.h | 2 +- msgpack/unpack_template.h | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index cb02a3b..71412ee 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -30,7 +30,7 @@ extern "C" { #ifndef MSGPACK_EMBED_STACK_SIZE -#define MSGPACK_EMBED_STACK_SIZE 1 +#define MSGPACK_EMBED_STACK_SIZE 16 #endif diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 72f8e59..4b8cd14 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -99,6 +99,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c unsigned int cs = ctx->cs; unsigned int top = ctx->top; msgpack_unpack_struct(_stack)* stack = ctx->stack; + unsigned int stack_size = ctx->stack_size; msgpack_unpack_user* user = &ctx->user; msgpack_unpack_object obj; @@ -135,21 +136,21 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ++top; \ /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ /*printf("stack push %d\n", top);*/ \ - if(top >= ctx->stack_size) { \ - if(ctx->stack_size == MSGPACK_EMBED_STACK_SIZE) { \ + if(top >= stack_size) { \ + if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ size_t nsize = csize * 2; \ msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \ if(tmp == NULL) { goto _failed; } \ memcpy(tmp, ctx->stack, csize); \ - ctx->stack = tmp; \ - ctx->stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \ } else { \ size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \ msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \ if(tmp == NULL) { goto _failed; } \ - ctx->stack = tmp; \ - ctx->stack_size *= 2; \ + ctx->stack = stack = tmp; \ + ctx->stack_size = stack_size = stack_size * 2; \ } \ } \ goto _header_again From a65438c6feaf82aa137b84432bd66722e944e5bd Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 17 Apr 2010 22:53:56 +0900 Subject: [PATCH 0406/1648] java: skip building jar file for the faster testing --- java/build.xml | 27 +++++++-------------------- java/ivy.xml | 2 +- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/java/build.xml b/java/build.xml index baecd01..52c5218 100644 --- a/java/build.xml +++ b/java/build.xml @@ -5,25 +5,18 @@ - - - - - + - - - @@ -65,13 +58,9 @@ - - - - @@ -80,10 +69,10 @@ + - @@ -145,12 +134,12 @@ - - - + + + - + @@ -166,8 +155,6 @@ errorProperty="tests.failed" failureProperty="tests.failed"> - - @@ -182,7 +169,7 @@ Tests Failed! - + diff --git a/java/ivy.xml b/java/ivy.xml index 6d98756..a694214 100644 --- a/java/ivy.xml +++ b/java/ivy.xml @@ -1,7 +1,7 @@ - MessagePack From abeed3be84b7b12b71598c3535c4c9a6e70c534f Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sat, 17 Apr 2010 23:25:42 +0900 Subject: [PATCH 0407/1648] java: add tests for array and map --- java/test/org/msgpack/TestPackUnpack.java | 69 +++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/java/test/org/msgpack/TestPackUnpack.java b/java/test/org/msgpack/TestPackUnpack.java index 1a02cc5..6877853 100644 --- a/java/test/org/msgpack/TestPackUnpack.java +++ b/java/test/org/msgpack/TestPackUnpack.java @@ -9,8 +9,13 @@ import static org.junit.Assert.*; public class TestPackUnpack { protected Object unpackOne(ByteArrayOutputStream out) { + return unpackOne(out, null); + } + protected Object unpackOne(ByteArrayOutputStream out, Schema schema) { ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); Unpacker upk = new Unpacker(in); + if (schema != null) + upk = upk.useSchema(schema); Iterator it = upk.iterator(); assertEquals(true, it.hasNext()); Object obj = it.next(); @@ -169,4 +174,68 @@ public class TestPackUnpack { assertTrue(false); } } + + @Test + public void testArray() throws Exception { + for (int i = 0; i < 1000; i++) { + Schema schema = Schema.parse("(array int)"); + List l = new ArrayList(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + l.add(j); + testArray(l, schema); + } + for (int i = 0; i < 1000; i++) { + Schema schema = Schema.parse("(array string)"); + List l = new ArrayList(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + l.add(Integer.toString(j)); + testArray(l, schema); + } + } + public void testArray(List val, Schema schema) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out, schema); + if (obj instanceof List) + assertTrue(val.equals(obj)); + else { + System.out.println("obj=" + obj); + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } + + @Test + public void testMap() throws Exception { + for (int i = 0; i < 1000; i++) { + Schema schema = Schema.parse("(map int int)"); + Map m = new HashMap(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + m.put(j, j); + testMap(m, schema); + } + for (int i = 0; i < 1000; i++) { + Schema schema = Schema.parse("(map string int)"); + Map m = new HashMap(); + int len = (int)Math.random() % 1000 + 1; + for (int j = 0; j < len; j++) + m.put(Integer.toString(j), j); + testMap(m, schema); + } + } + public void testMap(Map val, Schema schema) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + Object obj = unpackOne(out, schema); + if (obj instanceof Map) + assertTrue(val.equals(obj)); + else { + System.out.println("obj=" + obj); + System.out.println("Got unexpected class: " + obj.getClass()); + assertTrue(false); + } + } }; From ab8e0c9e31585b24f1d4cbfb1e73b0ce249e408a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 18 Apr 2010 00:08:03 +0900 Subject: [PATCH 0408/1648] c,cpp: reforms source tree --- Makefile.am | 18 -- NEWS | 0 README | 70 ---- README.md | 37 +++ c/Makefile.am | 29 -- c/bench.c | 323 ------------------- c/bench.mk | 9 - AUTHORS => cpp/AUTHORS | 0 COPYING => cpp/COPYING | 0 ChangeLog => cpp/ChangeLog | 0 LICENSE => cpp/LICENSE | 0 cpp/Makefile.am | 56 +++- NOTICE => cpp/NOTICE | 0 cpp/bench.cpp | 188 ----------- cpp/bench.mk | 9 - bootstrap => cpp/bootstrap | 11 +- configure.in => cpp/configure.in | 36 +-- {c => cpp}/msgpack.h | 0 {c => cpp}/msgpack/object.h | 0 {c => cpp}/msgpack/pack.h | 0 {c => cpp}/msgpack/sbuffer.h | 0 {c => cpp}/msgpack/unpack.h | 0 {c => cpp}/msgpack/vrefbuffer.h | 0 {c => cpp}/msgpack/zbuffer.h | 0 {c => cpp}/msgpack/zone.h | 0 cpp/{test.cpp => msgpack_test.cpp} | 0 cpp/msgpack_vc8.postbuild.bat | 49 +++ msgpack_vc8.sln => cpp/msgpack_vc8.sln | 0 msgpack_vc8.vcproj => cpp/msgpack_vc8.vcproj | 0 c/test.cpp => cpp/msgpackc_test.cpp | 0 {c => cpp}/object.c | 0 cpp/{preprocess.sh => preprocess} | 4 + {c => cpp}/unpack.c | 0 {c => cpp}/vrefbuffer.c | 0 {c => cpp}/zone.c | 0 msgpack_vc8.postbuild.bat | 41 --- 36 files changed, 159 insertions(+), 721 deletions(-) delete mode 100644 Makefile.am delete mode 100644 NEWS delete mode 100644 README create mode 100644 README.md delete mode 100644 c/Makefile.am delete mode 100644 c/bench.c delete mode 100644 c/bench.mk rename AUTHORS => cpp/AUTHORS (100%) rename COPYING => cpp/COPYING (100%) rename ChangeLog => cpp/ChangeLog (100%) rename LICENSE => cpp/LICENSE (100%) rename NOTICE => cpp/NOTICE (100%) delete mode 100644 cpp/bench.cpp delete mode 100644 cpp/bench.mk rename bootstrap => cpp/bootstrap (93%) rename configure.in => cpp/configure.in (61%) rename {c => cpp}/msgpack.h (100%) rename {c => cpp}/msgpack/object.h (100%) rename {c => cpp}/msgpack/pack.h (100%) rename {c => cpp}/msgpack/sbuffer.h (100%) rename {c => cpp}/msgpack/unpack.h (100%) rename {c => cpp}/msgpack/vrefbuffer.h (100%) rename {c => cpp}/msgpack/zbuffer.h (100%) rename {c => cpp}/msgpack/zone.h (100%) rename cpp/{test.cpp => msgpack_test.cpp} (100%) create mode 100644 cpp/msgpack_vc8.postbuild.bat rename msgpack_vc8.sln => cpp/msgpack_vc8.sln (100%) rename msgpack_vc8.vcproj => cpp/msgpack_vc8.vcproj (100%) mode change 100755 => 100644 rename c/test.cpp => cpp/msgpackc_test.cpp (100%) rename {c => cpp}/object.c (100%) rename cpp/{preprocess.sh => preprocess} (62%) rename {c => cpp}/unpack.c (100%) rename {c => cpp}/vrefbuffer.c (100%) rename {c => cpp}/zone.c (100%) delete mode 100644 msgpack_vc8.postbuild.bat diff --git a/Makefile.am b/Makefile.am deleted file mode 100644 index 04f62d2..0000000 --- a/Makefile.am +++ /dev/null @@ -1,18 +0,0 @@ -if ENABLE_CXX -SUBDIRS = c cpp -else -SUBDIRS = c -endif - -nobase_include_HEADERS = \ - msgpack/pack_define.h \ - msgpack/pack_template.h \ - msgpack/unpack_define.h \ - msgpack/unpack_template.h \ - msgpack/sysdep.h - -EXTRA_DIST = \ - msgpack_vc8.vcproj \ - msgpack_vc8.sln \ - msgpack_vc8.postbuild.bat - diff --git a/NEWS b/NEWS deleted file mode 100644 index e69de29..0000000 diff --git a/README b/README deleted file mode 100644 index 6ca10d6..0000000 --- a/README +++ /dev/null @@ -1,70 +0,0 @@ -MessagePack ------------ -Binary-based efficient data interchange format. - - -*Requirements - - MessagePack is only tested on Linux and Mac OS X, but it may run on other - UNIX-like platforms. - - gcc >= 4.1 is required to build. - - -*Installation - - Simply run ./configure && make && make install to install C and C++ binding. - - $ ./configure - $ make - $ sudo make install - - To install Ruby binding, run ./makegem.sh script on ruby/ directory and install - generated gem package. - - $ cd ruby - $ ./makegem.sh - $ gem install msgpack-*.gem - - -*Usage - - C++: - include msgpack.hpp header and link libmsgpack library. - see example/simple.cc for example. - - g++ simple.cc -lmsgpack - g++ stream.cc -lmsgpack -lpthread - - - C: - include msgpack.h header and link libmsgpackc library. - see example/simple.c for example. - - gcc simple.c -lmsgpackc - - - Ruby: - require msgpack library. - see example/simple.rb for example. - - ruby -rubygems simple.rb - - - API Document is available at http://msgpack.sourceforge.jp/. - - -Copyright (C) 2008-2010 FURUHASHI Sadayuki - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/README.md b/README.md new file mode 100644 index 0000000..066c45e --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +MessagePack +=========== +Extremely efficient object serialization library. It's like JSON, but very fast and small. + + +## What's MessagePack? + +MessagePack is a binary-based efficient object serialization library. It enables to exchange structured objects between many languages like JSON. But unlike JSON, it is very fast and small. + +Typical small integer (like flags or error code) is saved only in 1 byte, and typical short string only needs 1 byte except the length of the string itself. \[1,2,3\] (3 elements array) is serialized in 4 bytes using MessagePack as follows: + + require 'msgpack' + msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03" + MessagePack.unpack(msg) #=> [1,2,3] + + +## Performance + +![Serialization + Deserialization Speed Test](http://msgpack.sourceforge.net/index/speedtest.png) + +In this test, it measured the elapsed time of serializing and deserializing 200,000 target objects. The target object consists of the three integers and 512 bytes string. +The source code of this test is available from [frsyuki' serializer-speed-test repository.](http://github.com/frsyuki/serializer-speed-test) + + +## Getting Started + +Usage and other documents about implementations in each language are found at [the web site.](http://msgpack.sourceforge.net/) + + +## Learn More + + - [Project Web Site](http://msgpack.sourceforge.net/) + - [MessagePack format specification](http://msgpack.sourceforge.net/spec) + - [Repository at github](http://github.com/msgpack/msgpack) + - [Wiki](http://msgpack.sourceforge.net/start) + - [MessagePack-RPC](http://github.com/msgpack/msgpack-rpc) + diff --git a/c/Makefile.am b/c/Makefile.am deleted file mode 100644 index bbe547c..0000000 --- a/c/Makefile.am +++ /dev/null @@ -1,29 +0,0 @@ -lib_LTLIBRARIES = libmsgpackc.la - -libmsgpackc_la_SOURCES = \ - unpack.c \ - object.c \ - vrefbuffer.c \ - zone.c - -nobase_include_HEADERS = \ - msgpack.h \ - msgpack/sbuffer.h \ - msgpack/vrefbuffer.h \ - msgpack/zbuffer.h \ - msgpack/pack.h \ - msgpack/unpack.h \ - msgpack/object.h \ - msgpack/zone.h - -# -version-info CURRENT:REVISION:AGE -libmsgpackc_la_LDFLAGS = -version-info 2:0:0 - -check_PROGRAMS = \ - msgpackc_test - -msgpackc_test_SOURCES = test.cpp -msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -msgpackc_test_LDADD = libmsgpackc.la -lgtest_main - -TESTS = $(check_PROGRAMS) diff --git a/c/bench.c b/c/bench.c deleted file mode 100644 index d72a10d..0000000 --- a/c/bench.c +++ /dev/null @@ -1,323 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include -#include - - -static struct timeval g_timer; - -void reset_timer() -{ - gettimeofday(&g_timer, NULL); -} - -void show_timer(size_t bufsz) -{ - struct timeval endtime; - gettimeofday(&endtime, NULL); - double sec = (endtime.tv_sec - g_timer.tv_sec) - + (double)(endtime.tv_usec - g_timer.tv_usec) / 1000 / 1000; - printf("%f sec\n", sec); - printf("%f MB\n", ((double)bufsz)/1024/1024); - printf("%f Mbps\n", ((double)bufsz)*8/sec/1000/1000); -} - - -static int reformat_null(void * ctx) { return 1; } -static int reformat_boolean(void * ctx, int boolean) { return 1; } -static int reformat_number(void * ctx, const char * s, unsigned int l) { return 1; } -static int reformat_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } -static int reformat_map_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) { return 1; } -static int reformat_start_map(void * ctx) { return 1; } -static int reformat_end_map(void * ctx) { return 1; } -static int reformat_start_array(void * ctx) { return 1; } -static int reformat_end_array(void * ctx) { return 1; } - - -static void* unpack_uint8(void* data, uint8_t d) { return NULL; } -static void* unpack_uint16(void* data, uint16_t d) { return NULL; } -static void* unpack_uint32(void* data, uint32_t d) { return NULL; } -static void* unpack_uint64(void* data, uint64_t d) { return NULL; } -static void* unpack_int8(void* data, int8_t d) { return NULL; } -static void* unpack_int16(void* data, int16_t d) { return NULL; } -static void* unpack_int32(void* data, int32_t d) { return NULL; } -static void* unpack_int64(void* data, int64_t d) { return NULL; } -static void* unpack_float(void* data, float d) { return NULL; } -static void* unpack_double(void* data, double d) { return NULL; } -static void* unpack_nil(void* data) { return NULL; } -static void* unpack_true(void* data) { return NULL; } -static void* unpack_false(void* data) { return NULL; } -static void* unpack_array(void* data, unsigned int n) { return NULL; } -static void unpack_array_item(void* data, void* c, void* o) { } -static void* unpack_map(void* data, unsigned int n) { return NULL; } -static void unpack_map_item(void* data, void* c, void* k, void* v) { } -static void* unpack_raw(void* data, const char* b, const char* p, unsigned int l) { /*printf("unpack raw %p %lu\n",p,l);*/ return NULL; } - - -typedef struct { - size_t allocated; - size_t length; - char* buffer; -} pack_buffer; - -static const size_t PACK_INITIAL_BUFFER_SIZE = 32*1024; - -static void pack_buffer_init(pack_buffer* data) -{ - data->buffer = malloc(PACK_INITIAL_BUFFER_SIZE); - data->length = 0; - data->allocated = PACK_INITIAL_BUFFER_SIZE; -} - -static void pack_buffer_reset(pack_buffer* data) -{ - data->buffer = realloc(data->buffer, PACK_INITIAL_BUFFER_SIZE); - data->allocated = PACK_INITIAL_BUFFER_SIZE; - data->length = 0; -} - -static void pack_buffer_free(pack_buffer* data) -{ - free(data->buffer); -} - -static void pack_append_buffer(void* user, const char* b, unsigned int l) -{ - pack_buffer* data = (pack_buffer*)user; - if(data->allocated - data->length < l) { - data->buffer = realloc(data->buffer, data->allocated*2); - data->allocated *= 2; - } - memcpy(data->buffer + data->length, b, l); - data->length += l; -} - - -static const unsigned int TASK_INT_NUM = 1<<24; -static const unsigned int TASK_STR_LEN = 1<<15; -//static const unsigned int TASK_INT_NUM = 1<<20; -//static const unsigned int TASK_STR_LEN = 1<<12; -static const char* TASK_STR_PTR; - - -void bench_json(void) -{ - puts("== JSON =="); - - - yajl_gen_config gcfg = {0, NULL}; - yajl_gen g = yajl_gen_alloc(&gcfg); - - yajl_parser_config hcfg = { 0, 0 }; - yajl_callbacks callbacks = { - reformat_null, - reformat_boolean, - NULL, - NULL, - reformat_number, - reformat_string, - reformat_start_map, - reformat_map_key, - reformat_end_map, - reformat_start_array, - reformat_end_array - }; - yajl_handle h = yajl_alloc(&callbacks, &hcfg, NULL); - - - const unsigned char * buf; - unsigned int len; - - - puts("generate integer"); - reset_timer(); - { - unsigned int i; - yajl_gen_array_open(g); - for(i=0; i < TASK_INT_NUM; ++i) { - yajl_gen_integer(g, i); - } - yajl_gen_array_close(g); - } - show_timer(len); - - yajl_gen_get_buf(g, &buf, &len); - - puts("----"); - puts("parse integer"); - reset_timer(); - { - yajl_status stat = yajl_parse(h, buf, len); - if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { - unsigned char * str = yajl_get_error(h, 1, buf, len); - fprintf(stderr, (const char *) str); - } - } - show_timer(len); - - - //yajl_gen_clear(g); - yajl_gen_free(g); - g = yajl_gen_alloc(&gcfg); - yajl_free(h); - h = yajl_alloc(&callbacks, &hcfg, NULL); - - - puts("----"); - puts("generate string"); - reset_timer(); - { - unsigned int i; - yajl_gen_array_open(g); - for(i=0; i < TASK_STR_LEN; ++i) { - yajl_gen_string(g, (const unsigned char*)TASK_STR_PTR, i); - } - yajl_gen_array_close(g); - } - show_timer(len); - - yajl_gen_get_buf(g, &buf, &len); - - puts("----"); - puts("parse string"); - reset_timer(); - { - yajl_status stat = yajl_parse(h, buf, len); - if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { - unsigned char * str = yajl_get_error(h, 1, buf, len); - fprintf(stderr, (const char *) str); - } - } - show_timer(len); - - - yajl_gen_free(g); - yajl_free(h); -} - - -void bench_msgpack(void) -{ - puts("== MessagePack =="); - - - pack_buffer mpkbuf; - pack_buffer_init(&mpkbuf); - - msgpack_pack_t* mpk = msgpack_pack_new( - &mpkbuf, pack_append_buffer); - - msgpack_unpack_callback cb = { - unpack_uint8, - unpack_uint16, - unpack_uint32, - unpack_uint64, - unpack_int8, - unpack_int16, - unpack_int32, - unpack_int64, - unpack_float, - unpack_double, - unpack_nil, - unpack_true, - unpack_false, - unpack_array, - unpack_array_item, - unpack_map, - unpack_map_item, - unpack_raw, - }; - msgpack_unpack_t* mupk = msgpack_unpack_new(NULL, &cb); - - - size_t len; - const char* buf; - - - puts("pack integer"); - reset_timer(); - { - unsigned int i; - msgpack_pack_array(mpk, TASK_INT_NUM); - for(i=0; i < TASK_INT_NUM; ++i) { - msgpack_pack_unsigned_int(mpk, i); - } - } - show_timer(mpkbuf.length); - - len = mpkbuf.length; - buf = mpkbuf.buffer; - - puts("----"); - puts("unpack integer"); - reset_timer(); - { - size_t off = 0; - int ret = msgpack_unpack_execute(mupk, buf, len, &off); - if(ret < 0) { - fprintf(stderr, "Parse error.\n"); - } else if(ret == 0) { - fprintf(stderr, "Not finished.\n"); - } - } - show_timer(mpkbuf.length); - - - pack_buffer_reset(&mpkbuf); - msgpack_unpack_reset(mupk); - - - puts("----"); - puts("pack string"); - reset_timer(); - { - unsigned int i; - msgpack_pack_array(mpk, TASK_STR_LEN); - for(i=0; i < TASK_STR_LEN; ++i) { - msgpack_pack_raw(mpk, i); - msgpack_pack_raw_body(mpk, TASK_STR_PTR, i); - } - } - show_timer(mpkbuf.length); - - len = mpkbuf.length; - buf = mpkbuf.buffer; - - puts("----"); - puts("unpack string"); - reset_timer(); - { - size_t off = 0; - int ret = msgpack_unpack_execute(mupk, buf, len, &off); - if(ret < 0) { - fprintf(stderr, "Parse error.\n"); - } else if(ret == 0) { - fprintf(stderr, "Not finished.\n"); - } - } - show_timer(mpkbuf.length); - - - msgpack_unpack_free(mupk); - msgpack_pack_free(mpk); - pack_buffer_free(&mpkbuf); -} - -int main(int argc, char* argv[]) -{ - char* str = malloc(TASK_STR_LEN); - memset(str, 'a', TASK_STR_LEN); - TASK_STR_PTR = str; - - bench_msgpack(); - bench_json(); - - return 0; -} - - diff --git a/c/bench.mk b/c/bench.mk deleted file mode 100644 index c765e31..0000000 --- a/c/bench.mk +++ /dev/null @@ -1,9 +0,0 @@ - -CFLAGS += -Wall -g -I. -I.. -O4 -LDFLAGS += -lyajl - -all: bench - -bench: bench.o pack.o unpack.o pack.h unpack.h - $(CC) bench.o pack.o unpack.o $(CFLAGS) $(LDFLAGS) -o $@ - diff --git a/AUTHORS b/cpp/AUTHORS similarity index 100% rename from AUTHORS rename to cpp/AUTHORS diff --git a/COPYING b/cpp/COPYING similarity index 100% rename from COPYING rename to cpp/COPYING diff --git a/ChangeLog b/cpp/ChangeLog similarity index 100% rename from ChangeLog rename to cpp/ChangeLog diff --git a/LICENSE b/cpp/LICENSE similarity index 100% rename from LICENSE rename to cpp/LICENSE diff --git a/cpp/Makefile.am b/cpp/Makefile.am index c1b4981..aba7e28 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -1,9 +1,38 @@ -lib_LTLIBRARIES = libmsgpack.la +lib_LTLIBRARIES = libmsgpackc.la libmsgpack.la + +libmsgpackc_la_SOURCES = \ + unpack.c \ + object.c \ + vrefbuffer.c \ + zone.c + +# -version-info CURRENT:REVISION:AGE +libmsgpackc_la_LDFLAGS = -version-info 2:0:0 + libmsgpack_la_SOURCES = \ object.cpp +libmsgpack_la_LIBADD = -lmsgpackc + +# -version-info CURRENT:REVISION:AGE +libmsgpack_la_LDFLAGS = -version-info 2:0:0 + + nobase_include_HEADERS = \ + msgpack/pack_define.h \ + msgpack/pack_template.h \ + msgpack/unpack_define.h \ + msgpack/unpack_template.h \ + msgpack/sysdep.h \ + msgpack.h \ + msgpack/sbuffer.h \ + msgpack/vrefbuffer.h \ + msgpack/zbuffer.h \ + msgpack/pack.h \ + msgpack/unpack.h \ + msgpack/object.h \ + msgpack/zone.h \ msgpack.hpp \ msgpack/sbuffer.hpp \ msgpack/vrefbuffer.hpp \ @@ -30,16 +59,31 @@ nobase_include_HEADERS = \ msgpack/type/tr1/unordered_map.hpp \ msgpack/type/tr1/unordered_set.hpp -libmsgpack_la_LIBADD = -L../c -lmsgpackc -# -version-info CURRENT:REVISION:AGE -libmsgpack_la_LDFLAGS = -version-info 2:0:0 +# work around for duplicated object file name +libmsgpackc_la_CFLAGS = $(AM_CFLAGS) +libmsgpackc_la_CXXFLAGS = $(AM_CXXFLAGS) +libmsgpack_la_CFLAGS = $(AM_CFLAGS) +libmsgpack_la_CXXFLAGS = $(AM_CXXFLAGS) + + +EXTRA_DIST = \ + msgpack_vc8.vcproj \ + msgpack_vc8.sln \ + msgpack_vc8.postbuild.bat + check_PROGRAMS = \ - msgpack_test + msgpackc_test \ + msgpack_test -msgpack_test_SOURCES = test.cpp +msgpackc_test_SOURCES = msgpackc_test.cpp +msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c +msgpackc_test_LDADD = libmsgpackc.la -lgtest_main + +msgpack_test_SOURCES = msgpack_test.cpp msgpack_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp msgpack_test_LDADD = libmsgpack.la -lgtest_main TESTS = $(check_PROGRAMS) + diff --git a/NOTICE b/cpp/NOTICE similarity index 100% rename from NOTICE rename to cpp/NOTICE diff --git a/cpp/bench.cpp b/cpp/bench.cpp deleted file mode 100644 index aa303fa..0000000 --- a/cpp/bench.cpp +++ /dev/null @@ -1,188 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -static const unsigned int TASK_INT_NUM = 1<<24; -static const unsigned int TASK_STR_LEN = 1<<15; -//static const unsigned int TASK_INT_NUM = 1<<22; -//static const unsigned int TASK_STR_LEN = 1<<13; -static const char* TASK_STR_PTR; - - -class simple_timer { -public: - void reset() { gettimeofday(&m_timeval, NULL); } - void show_stat(size_t bufsz) - { - struct timeval endtime; - gettimeofday(&endtime, NULL); - double sec = (endtime.tv_sec - m_timeval.tv_sec) - + (double)(endtime.tv_usec - m_timeval.tv_usec) / 1000 / 1000; - std::cout << sec << " sec" << std::endl; - std::cout << (double(bufsz)/1024/1024) << " MB" << std::endl; - std::cout << (bufsz/sec/1000/1000*8) << " Mbps" << std::endl; - } -private: - timeval m_timeval; -}; - - -class simple_buffer { -public: - static const size_t DEFAULT_INITIAL_SIZE = 32*1024;//512*1024*1024*2; - - simple_buffer(size_t initial_size = DEFAULT_INITIAL_SIZE) : - m_storage((char*)malloc(initial_size)), - m_allocated(initial_size), - m_used(0) - { - if(!m_storage) { throw std::bad_alloc(); } - } - - ~simple_buffer() - { - free(m_storage); - } - -public: - inline void write(const char* buf, size_t len) - { - if(m_allocated - m_used < len) { - expand_buffer(len); - } - memcpy(m_storage + m_used, buf, len); - m_used += len; - } - - void clear() - { - m_used = 0; - } - -private: - void expand_buffer(size_t req) - { - size_t nsize = m_allocated * 2; - size_t at_least = m_used + req; - while(nsize < at_least) { nsize *= 2; } - char* tmp = (char*)realloc(m_storage, nsize); - if(!tmp) { throw std::bad_alloc(); } - m_storage = tmp; - m_allocated = nsize; - } - -public: - size_t size() const { return m_used; } - const char* data() const { return m_storage; } - -private: - char* m_storage; - size_t m_allocated; - size_t m_used; -}; - - -void bench_msgpack_int() -{ - simple_buffer buf; - simple_timer timer; - - std::cout << "----" << std::endl; - std::cout << "pack integer" << std::endl; - - timer.reset(); - { - msgpack::packer pk(buf); - pk.pack_array(TASK_INT_NUM); - for(unsigned int i=0; i < TASK_INT_NUM; ++i) { - pk.pack_unsigned_int(i); - } - } - timer.show_stat(buf.size()); - - - std::cout << "----" << std::endl; - std::cout << "unpack integer" << std::endl; - - msgpack::zone z; - msgpack::object obj; - - timer.reset(); - { - obj = msgpack::unpack(buf.data(), buf.size(), z); - } - timer.show_stat(buf.size()); - - /* - std::cout << "----" << std::endl; - std::cout << "dynamic pack integer" << std::endl; - - buf.clear(); - - timer.reset(); - msgpack::pack(buf, obj); - timer.show_stat(buf.size()); - */ -} - -void bench_msgpack_str() -{ - simple_buffer buf; - simple_timer timer; - - std::cout << "----" << std::endl; - std::cout << "pack string" << std::endl; - - timer.reset(); - { - msgpack::packer pk(buf); - pk.pack_array(TASK_STR_LEN); - for(unsigned int i=0; i < TASK_STR_LEN; ++i) { - pk.pack_raw(i); - pk.pack_raw_body(TASK_STR_PTR, i); - } - } - timer.show_stat(buf.size()); - - - std::cout << "----" << std::endl; - std::cout << "unpack string" << std::endl; - - msgpack::zone z; - msgpack::object obj; - - timer.reset(); - { - obj = msgpack::unpack(buf.data(), buf.size(), z); - } - timer.show_stat(buf.size()); - - - /* - std::cout << "----" << std::endl; - std::cout << "dynamic pack string" << std::endl; - - buf.clear(); - - timer.reset(); - msgpack::pack(buf, obj); - timer.show_stat(buf.size()); - */ -} - -int main(void) -{ - char* str = (char*)malloc(TASK_STR_LEN); - memset(str, 'a', TASK_STR_LEN); - TASK_STR_PTR = str; - - bench_msgpack_int(); - bench_msgpack_str(); - - return 0; -} - diff --git a/cpp/bench.mk b/cpp/bench.mk deleted file mode 100644 index 8da2b7f..0000000 --- a/cpp/bench.mk +++ /dev/null @@ -1,9 +0,0 @@ - -CXXFLAGS += -Wall -g -I. -I.. -O4 -LDFLAGS += - -all: bench - -bench: bench.o unpack.o zone.o object.o pack.hpp unpack.hpp zone.hpp object.hpp - $(CXX) bench.o unpack.o zone.o object.o $(CXXFLAGS) $(LDFLAGS) -o $@ - diff --git a/bootstrap b/cpp/bootstrap similarity index 93% rename from bootstrap rename to cpp/bootstrap index 8ac504b..4a04e0a 100755 --- a/bootstrap +++ b/cpp/bootstrap @@ -32,7 +32,16 @@ fi mkdir -p ac -(cd cpp && ./preprocess.sh $@; cd ..) +test -f AUTHORS || touch AUTHORS +test -f COPYING || touch COPYING +test -f ChangeLog || touch ChangeLog +test -f NEWS || touch NEWS +test -f README || touch README + +if ! ./preprocess; then + exit 1 +fi + ACLOCAL="aclocal" diff --git a/configure.in b/cpp/configure.in similarity index 61% rename from configure.in rename to cpp/configure.in index 0f2c5d0..a4cb4d8 100644 --- a/configure.in +++ b/cpp/configure.in @@ -1,35 +1,21 @@ -AC_INIT(msgpack/unpack_template.h) +AC_INIT(object.cpp) AC_CONFIG_AUX_DIR(ac) AM_INIT_AUTOMAKE(msgpack, 0.4.3) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) -if test "" = "$CFLAGS"; then - CFLAGS="-g -O4" -fi +CFLAGS="-O4 -Wall $CFLAGS" + +AC_SUBST(CXXFLAGS) +CXXFLAGS="-O4 -Wall $CXXFLAGS" AC_PROG_CC - -CFLAGS="-O4 -Wall $CFLAGS -I.." - -AC_MSG_CHECKING([if c++ api is enabled]) -AC_ARG_ENABLE(cxx, - AS_HELP_STRING([--disable-cxx], - [don't build c++ api.]) ) -AC_MSG_RESULT($enable_cxx) -if test "$enable_cxx" != "no"; then - AC_SUBST(CXXFLAGS) - if test "" = "$CXXFLAGS"; then - CXXFLAGS="-g -O4" - fi -fi - -# FIXME enable_cxx AC_PROG_CXX -CXXFLAGS="-O4 -Wall $CXXFLAGS -I.. -I../c" +AC_PROG_LIBTOOL +AM_PROG_AS +AM_PROG_CC_C_O -# FIXME enable_cxx AC_LANG_PUSH([C++]) AC_CHECK_HEADERS(tr1/unordered_map) AC_CHECK_HEADERS(tr1/unordered_set) @@ -53,9 +39,5 @@ add CFLAGS="--march=i686" and CXXFLAGS="-march=i668" options to ./configure as f ]) fi -AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no") - -AC_PROG_LIBTOOL - -AC_OUTPUT([Makefile c/Makefile cpp/Makefile]) +AC_OUTPUT([Makefile]) diff --git a/c/msgpack.h b/cpp/msgpack.h similarity index 100% rename from c/msgpack.h rename to cpp/msgpack.h diff --git a/c/msgpack/object.h b/cpp/msgpack/object.h similarity index 100% rename from c/msgpack/object.h rename to cpp/msgpack/object.h diff --git a/c/msgpack/pack.h b/cpp/msgpack/pack.h similarity index 100% rename from c/msgpack/pack.h rename to cpp/msgpack/pack.h diff --git a/c/msgpack/sbuffer.h b/cpp/msgpack/sbuffer.h similarity index 100% rename from c/msgpack/sbuffer.h rename to cpp/msgpack/sbuffer.h diff --git a/c/msgpack/unpack.h b/cpp/msgpack/unpack.h similarity index 100% rename from c/msgpack/unpack.h rename to cpp/msgpack/unpack.h diff --git a/c/msgpack/vrefbuffer.h b/cpp/msgpack/vrefbuffer.h similarity index 100% rename from c/msgpack/vrefbuffer.h rename to cpp/msgpack/vrefbuffer.h diff --git a/c/msgpack/zbuffer.h b/cpp/msgpack/zbuffer.h similarity index 100% rename from c/msgpack/zbuffer.h rename to cpp/msgpack/zbuffer.h diff --git a/c/msgpack/zone.h b/cpp/msgpack/zone.h similarity index 100% rename from c/msgpack/zone.h rename to cpp/msgpack/zone.h diff --git a/cpp/test.cpp b/cpp/msgpack_test.cpp similarity index 100% rename from cpp/test.cpp rename to cpp/msgpack_test.cpp diff --git a/cpp/msgpack_vc8.postbuild.bat b/cpp/msgpack_vc8.postbuild.bat new file mode 100644 index 0000000..1bdfabe --- /dev/null +++ b/cpp/msgpack_vc8.postbuild.bat @@ -0,0 +1,49 @@ +IF NOT EXIST include MKDIR include +IF NOT EXIST include\msgpack MKDIR include\msgpack +IF NOT EXIST include\msgpack\type MKDIR include\msgpack\type +IF NOT EXIST include\msgpack\type\tr1 MKDIR include\msgpack\type\tr1 +IF EXIST bootstrap ( + copy ..\msgpack\pack_define.h include\msgpack\ + copy ..\msgpack\pack_template.h include\msgpack\ + copy ..\msgpack\unpack_define.h include\msgpack\ + copy ..\msgpack\unpack_template.h include\msgpack\ + copy ..\msgpack\sysdep.h include\msgpack\ +) ELSE ( + copy msgpack\pack_define.h include\msgpack\ + copy msgpack\pack_template.h include\msgpack\ + copy msgpack\unpack_define.h include\msgpack\ + copy msgpack\unpack_template.h include\msgpack\ + copy msgpack\sysdep.h include\msgpack\ +) +copy msgpack.h include\ +copy msgpack\sbuffer.h include\msgpack\ +copy msgpack\vrefbuffer.h include\msgpack\ +copy msgpack\pack.h include\msgpack\ +copy msgpack\unpack.h include\msgpack\ +copy msgpack\object.h include\msgpack\ +copy msgpack\zone.h include\msgpack\ +copy msgpack.hpp include\ +copy msgpack\sbuffer.hpp include\msgpack\ +copy msgpack\vrefbuffer.hpp include\msgpack\ +copy msgpack\pack.hpp include\msgpack\ +copy msgpack\unpack.hpp include\msgpack\ +copy msgpack\object.hpp include\msgpack\ +copy msgpack\zone.hpp include\msgpack\ +copy msgpack\type.hpp include\msgpack\type\ +copy msgpack\type\bool.hpp include\msgpack\type\ +copy msgpack\type\float.hpp include\msgpack\type\ +copy msgpack\type\int.hpp include\msgpack\type\ +copy msgpack\type\list.hpp include\msgpack\type\ +copy msgpack\type\deque.hpp include\msgpack\type\ +copy msgpack\type\map.hpp include\msgpack\type\ +copy msgpack\type\nil.hpp include\msgpack\type\ +copy msgpack\type\pair.hpp include\msgpack\type\ +copy msgpack\type\raw.hpp include\msgpack\type\ +copy msgpack\type\set.hpp include\msgpack\type\ +copy msgpack\type\string.hpp include\msgpack\type\ +copy msgpack\type\vector.hpp include\msgpack\type\ +copy msgpack\type\tuple.hpp include\msgpack\type\ +copy msgpack\type\define.hpp include\msgpack\type\ +copy msgpack\type\tr1\unordered_map.hpp include\msgpack\type\ +copy msgpack\type\tr1\unordered_set.hpp include\msgpack\type\ + diff --git a/msgpack_vc8.sln b/cpp/msgpack_vc8.sln similarity index 100% rename from msgpack_vc8.sln rename to cpp/msgpack_vc8.sln diff --git a/msgpack_vc8.vcproj b/cpp/msgpack_vc8.vcproj old mode 100755 new mode 100644 similarity index 100% rename from msgpack_vc8.vcproj rename to cpp/msgpack_vc8.vcproj diff --git a/c/test.cpp b/cpp/msgpackc_test.cpp similarity index 100% rename from c/test.cpp rename to cpp/msgpackc_test.cpp diff --git a/c/object.c b/cpp/object.c similarity index 100% rename from c/object.c rename to cpp/object.c diff --git a/cpp/preprocess.sh b/cpp/preprocess similarity index 62% rename from cpp/preprocess.sh rename to cpp/preprocess index 2e06c10..63af4c6 100755 --- a/cpp/preprocess.sh +++ b/cpp/preprocess @@ -14,4 +14,8 @@ preprocess() { preprocess msgpack/type/tuple.hpp preprocess msgpack/type/define.hpp preprocess msgpack/zone.hpp +cp -f ../msgpack/pack_define.h msgpack/ +cp -f ../msgpack/pack_template.h msgpack/ +cp -f ../msgpack/unpack_define.h msgpack/ +cp -f ../msgpack/unpack_template.h msgpack/ diff --git a/c/unpack.c b/cpp/unpack.c similarity index 100% rename from c/unpack.c rename to cpp/unpack.c diff --git a/c/vrefbuffer.c b/cpp/vrefbuffer.c similarity index 100% rename from c/vrefbuffer.c rename to cpp/vrefbuffer.c diff --git a/c/zone.c b/cpp/zone.c similarity index 100% rename from c/zone.c rename to cpp/zone.c diff --git a/msgpack_vc8.postbuild.bat b/msgpack_vc8.postbuild.bat deleted file mode 100644 index 7ee2586..0000000 --- a/msgpack_vc8.postbuild.bat +++ /dev/null @@ -1,41 +0,0 @@ -IF NOT EXIST include MKDIR include -IF NOT EXIST include\msgpack MKDIR include\msgpack -IF NOT EXIST include\msgpack\type MKDIR include\msgpack\type -IF NOT EXIST include\msgpack\type\tr1 MKDIR include\msgpack\type\tr1 -copy msgpack\pack_define.h include\msgpack\ -copy msgpack\pack_template.h include\msgpack\ -copy msgpack\unpack_define.h include\msgpack\ -copy msgpack\unpack_template.h include\msgpack\ -copy msgpack\sysdep.h include\msgpack\ -copy c\msgpack.h include\ -copy c\msgpack\sbuffer.h include\msgpack\ -copy c\msgpack\vrefbuffer.h include\msgpack\ -copy c\msgpack\pack.h include\msgpack\ -copy c\msgpack\unpack.h include\msgpack\ -copy c\msgpack\object.h include\msgpack\ -copy c\msgpack\zone.h include\msgpack\ -copy cpp\msgpack.hpp include\ -copy cpp\msgpack\sbuffer.hpp include\msgpack\ -copy cpp\msgpack\vrefbuffer.hpp include\msgpack\ -copy cpp\msgpack\pack.hpp include\msgpack\ -copy cpp\msgpack\unpack.hpp include\msgpack\ -copy cpp\msgpack\object.hpp include\msgpack\ -copy cpp\msgpack\zone.hpp include\msgpack\ -copy cpp\msgpack\type.hpp include\msgpack\type\ -copy cpp\msgpack\type\bool.hpp include\msgpack\type\ -copy cpp\msgpack\type\float.hpp include\msgpack\type\ -copy cpp\msgpack\type\int.hpp include\msgpack\type\ -copy cpp\msgpack\type\list.hpp include\msgpack\type\ -copy cpp\msgpack\type\deque.hpp include\msgpack\type\ -copy cpp\msgpack\type\map.hpp include\msgpack\type\ -copy cpp\msgpack\type\nil.hpp include\msgpack\type\ -copy cpp\msgpack\type\pair.hpp include\msgpack\type\ -copy cpp\msgpack\type\raw.hpp include\msgpack\type\ -copy cpp\msgpack\type\set.hpp include\msgpack\type\ -copy cpp\msgpack\type\string.hpp include\msgpack\type\ -copy cpp\msgpack\type\vector.hpp include\msgpack\type\ -copy cpp\msgpack\type\tuple.hpp include\msgpack\type\ -copy cpp\msgpack\type\define.hpp include\msgpack\type\ -copy cpp\msgpack\type\tr1\unordered_map.hpp include\msgpack\type\ -copy cpp\msgpack\type\tr1\unordered_set.hpp include\msgpack\type\ - From 58854fdae90e428d9ab18bc3f457f1eb0b447446 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 18 Apr 2010 00:39:23 +0900 Subject: [PATCH 0409/1648] cpp: add cpp/README.md --- cpp/README.md | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 cpp/README.md diff --git a/cpp/README.md b/cpp/README.md new file mode 100644 index 0000000..cc1ba38 --- /dev/null +++ b/cpp/README.md @@ -0,0 +1,78 @@ +MessagePack for C/C++ +===================== +Binary-based efficient object serialization library. + + +## Installation + +Download latest package from [releases of MessagePack](http://sourceforge.net/projects/msgpack/files/) and extract it. + +On UNIX-like platform, run ./configure && make && sudo make install: + + $ ./configure + $ make + $ sudo make install + +On Windows, open msgpack_vc8.vcproj file and build it using batch build. DLLs are built on lib folder, and the headers are built on include folder. + +To use the library in your program, include msgpack.hpp header and link msgpack and msgpackc library. + + +## Example + + #include + #include + + int main(void) { + // This is target object. + std::vector target; + target.push_back("Hello,"); + target.push_back("World!"); + + // Serialize it. + msgpack::sbuffer buffer; // simple buffer + msgpack::pack(&buffer, target); + + // Deserialize the serialized data. + msgpack::zone mempool; // this manages the life of deserialized object + msgpack::object obj; + msgpack::unpack_return ret = + msgpack::unpack(buffer.data, buffer.size, NULL, &mempool, &obj); + + if(ret != msgapck::UNPACK_SUCCESS) { + // error check + exit(1); + } + + // Print the deserialized object to stdout. + std::cout << obj << std::endl; // ["Hello," "World!"] + + // Convert the deserialized object to staticaly typed object. + std::vector result; + obj.convert(&result); + + // If the type is mismatched, it throws msgpack::type_error. + obj.as(); // type is mismatched, msgpack::type_error is thrown + } + +API document and other example codes are available at the [wiki.](http://msgpack.sourceforge.net/start) + + +## License + +Copyright (C) 2008-2010 FURUHASHI Sadayuki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +See also NOTICE file. + From 05b8c00ee78b5806814f133f6ccd33148d1e13a0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 18 Apr 2010 00:39:45 +0900 Subject: [PATCH 0410/1648] cpp: fixes windows compatibility --- cpp/Makefile.am | 6 +++--- cpp/bootstrap | 2 +- cpp/msgpack_test.cpp | 4 ++-- cpp/msgpack_vc8.postbuild.bat | 20 +++++++------------- cpp/msgpack_vc8.vcproj | 18 +++++++++--------- cpp/preprocess | 1 + cpp/unpack.c | 2 +- 7 files changed, 24 insertions(+), 29 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index aba7e28..a297ba7 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -68,21 +68,21 @@ libmsgpack_la_CXXFLAGS = $(AM_CXXFLAGS) EXTRA_DIST = \ + README.md \ + LICENSE \ + NOTICE \ msgpack_vc8.vcproj \ msgpack_vc8.sln \ msgpack_vc8.postbuild.bat - check_PROGRAMS = \ msgpackc_test \ msgpack_test msgpackc_test_SOURCES = msgpackc_test.cpp -msgpackc_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c msgpackc_test_LDADD = libmsgpackc.la -lgtest_main msgpack_test_SOURCES = msgpack_test.cpp -msgpack_test_CXXFLAGS = -I$(top_srcdir) -I$(top_srcdir)/c -I$(top_srcdir)/cpp msgpack_test_LDADD = libmsgpack.la -lgtest_main TESTS = $(check_PROGRAMS) diff --git a/cpp/bootstrap b/cpp/bootstrap index 4a04e0a..a95c304 100755 --- a/cpp/bootstrap +++ b/cpp/bootstrap @@ -36,7 +36,7 @@ test -f AUTHORS || touch AUTHORS test -f COPYING || touch COPYING test -f ChangeLog || touch ChangeLog test -f NEWS || touch NEWS -test -f README || touch README +test -f README || cp -f README.md README if ! ./preprocess; then exit 1 diff --git a/cpp/msgpack_test.cpp b/cpp/msgpack_test.cpp index 113914a..0dd0ffc 100644 --- a/cpp/msgpack_test.cpp +++ b/cpp/msgpack_test.cpp @@ -438,7 +438,7 @@ TEST(MSGPACK_STL, simple_buffer_multiset) #ifdef HAVE_TR1_UNORDERED_MAP #include -#include "cpp/msgpack/type/tr1/unordered_map.hpp" +#include "msgpack/type/tr1/unordered_map.hpp" TEST(MSGPACK_TR1, simple_buffer_unordered_map) { for (unsigned int k = 0; k < kLoop; k++) { @@ -499,7 +499,7 @@ TEST(MSGPACK_TR1, simple_buffer_unordered_multimap) #ifdef HAVE_TR1_UNORDERED_SET #include -#include "cpp/msgpack/type/tr1/unordered_set.hpp" +#include "msgpack/type/tr1/unordered_set.hpp" TEST(MSGPACK_TR1, simple_buffer_unordered_set) { for (unsigned int k = 0; k < kLoop; k++) { diff --git a/cpp/msgpack_vc8.postbuild.bat b/cpp/msgpack_vc8.postbuild.bat index 1bdfabe..bae13f3 100644 --- a/cpp/msgpack_vc8.postbuild.bat +++ b/cpp/msgpack_vc8.postbuild.bat @@ -2,22 +2,15 @@ IF NOT EXIST include MKDIR include IF NOT EXIST include\msgpack MKDIR include\msgpack IF NOT EXIST include\msgpack\type MKDIR include\msgpack\type IF NOT EXIST include\msgpack\type\tr1 MKDIR include\msgpack\type\tr1 -IF EXIST bootstrap ( - copy ..\msgpack\pack_define.h include\msgpack\ - copy ..\msgpack\pack_template.h include\msgpack\ - copy ..\msgpack\unpack_define.h include\msgpack\ - copy ..\msgpack\unpack_template.h include\msgpack\ - copy ..\msgpack\sysdep.h include\msgpack\ -) ELSE ( - copy msgpack\pack_define.h include\msgpack\ - copy msgpack\pack_template.h include\msgpack\ - copy msgpack\unpack_define.h include\msgpack\ - copy msgpack\unpack_template.h include\msgpack\ - copy msgpack\sysdep.h include\msgpack\ -) +copy msgpack\pack_define.h include\msgpack\ +copy msgpack\pack_template.h include\msgpack\ +copy msgpack\unpack_define.h include\msgpack\ +copy msgpack\unpack_template.h include\msgpack\ +copy msgpack\sysdep.h include\msgpack\ copy msgpack.h include\ copy msgpack\sbuffer.h include\msgpack\ copy msgpack\vrefbuffer.h include\msgpack\ +copy msgpack\zbuffer.h include\msgpack\ copy msgpack\pack.h include\msgpack\ copy msgpack\unpack.h include\msgpack\ copy msgpack\object.h include\msgpack\ @@ -25,6 +18,7 @@ copy msgpack\zone.h include\msgpack\ copy msgpack.hpp include\ copy msgpack\sbuffer.hpp include\msgpack\ copy msgpack\vrefbuffer.hpp include\msgpack\ +copy msgpack\zbuffer.hpp include\msgpack\ copy msgpack\pack.hpp include\msgpack\ copy msgpack\unpack.hpp include\msgpack\ copy msgpack\object.hpp include\msgpack\ diff --git a/cpp/msgpack_vc8.vcproj b/cpp/msgpack_vc8.vcproj index a3fa28f..2a0eebe 100644 --- a/cpp/msgpack_vc8.vcproj +++ b/cpp/msgpack_vc8.vcproj @@ -43,7 +43,7 @@ z); - template_destroy(mpac->ctx); + template_destroy(CTX_CAST(mpac->ctx)); free(mpac->ctx); decl_count(mpac->buffer); } From fb96617377ed7330edcf239d807d2ae378e336e9 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sun, 18 Apr 2010 01:18:40 +0900 Subject: [PATCH 0411/1648] java: add javadoc,javadoc-jar,pom,dist,mvn-install,mvn-deploy ANT tasks. Now maven2 repository for msgpack is created at the following URL. - http://msgpack.sourceforge.net/maven2/ --- java/build.xml | 127 +++++++++++++++++++++++++++++++++++++++++++++++-- java/ivy.xml | 2 +- 2 files changed, 125 insertions(+), 4 deletions(-) diff --git a/java/build.xml b/java/build.xml index 52c5218..f6e93ae 100644 --- a/java/build.xml +++ b/java/build.xml @@ -2,6 +2,12 @@ xmlns:ivy="antlib:org.apache.ivy.ant" xmlns:mvn="urn:maven-artifact-ant"> + + + + + + @@ -10,7 +16,11 @@ + + + + @@ -27,6 +37,10 @@ + + + - - - @@ -179,4 +190,114 @@ + + + + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + Javadoc warnings! + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/ivy.xml b/java/ivy.xml index a694214..0e07cca 100644 --- a/java/ivy.xml +++ b/java/ivy.xml @@ -1,7 +1,7 @@ - MessagePack From f53c351fd28b3bea6a03416a54aff631499af65a Mon Sep 17 00:00:00 2001 From: Hideyuki Tanaka Date: Sun, 18 Apr 2010 02:17:49 +0900 Subject: [PATCH 0412/1648] haskell binding --- haskell/LICENSE | 24 + haskell/README | 0 haskell/Setup.lhs | 3 + haskell/cbits/msgpack.c | 137 ++++++ haskell/msgpack.cabal | 32 ++ haskell/src/Data/MessagePack.hs | 63 +++ haskell/src/Data/MessagePack/Base.hsc | 581 +++++++++++++++++++++++++ haskell/src/Data/MessagePack/Class.hs | 97 +++++ haskell/src/Data/MessagePack/Feed.hs | 59 +++ haskell/src/Data/MessagePack/Monad.hs | 153 +++++++ haskell/src/Data/MessagePack/Stream.hs | 84 ++++ haskell/test/Monad.hs | 16 + haskell/test/Stream.hs | 14 + haskell/test/Test.hs | 36 ++ 14 files changed, 1299 insertions(+) create mode 100644 haskell/LICENSE create mode 100644 haskell/README create mode 100644 haskell/Setup.lhs create mode 100644 haskell/cbits/msgpack.c create mode 100644 haskell/msgpack.cabal create mode 100644 haskell/src/Data/MessagePack.hs create mode 100644 haskell/src/Data/MessagePack/Base.hsc create mode 100644 haskell/src/Data/MessagePack/Class.hs create mode 100644 haskell/src/Data/MessagePack/Feed.hs create mode 100644 haskell/src/Data/MessagePack/Monad.hs create mode 100644 haskell/src/Data/MessagePack/Stream.hs create mode 100644 haskell/test/Monad.hs create mode 100644 haskell/test/Stream.hs create mode 100644 haskell/test/Test.hs diff --git a/haskell/LICENSE b/haskell/LICENSE new file mode 100644 index 0000000..2de30f6 --- /dev/null +++ b/haskell/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2009, Hideyuki Tanaka +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the Hideyuki Tanaka nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY Hideyuki Tanaka ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/haskell/README b/haskell/README new file mode 100644 index 0000000..e69de29 diff --git a/haskell/Setup.lhs b/haskell/Setup.lhs new file mode 100644 index 0000000..5bde0de --- /dev/null +++ b/haskell/Setup.lhs @@ -0,0 +1,3 @@ +#!/usr/bin/env runhaskell +> import Distribution.Simple +> main = defaultMain diff --git a/haskell/cbits/msgpack.c b/haskell/cbits/msgpack.c new file mode 100644 index 0000000..be44592 --- /dev/null +++ b/haskell/cbits/msgpack.c @@ -0,0 +1,137 @@ +#include + +void msgpack_sbuffer_init_wrap(msgpack_sbuffer* sbuf) +{ + msgpack_sbuffer_init(sbuf); +} + +void msgpack_sbuffer_destroy_wrap(msgpack_sbuffer* sbuf) +{ + msgpack_sbuffer_destroy(sbuf); +} + +int msgpack_sbuffer_write_wrap(void* data, const char* buf, unsigned int len) +{ + return msgpack_sbuffer_write(data, buf, len); +} + +msgpack_packer* msgpack_packer_new_wrap(void *data, msgpack_packer_write callback) +{ + return msgpack_packer_new(data, callback); +} + +void msgpack_packer_free_wrap(msgpack_packer* pk) +{ + msgpack_packer_free(pk); +} + +int msgpack_pack_uint8_wrap(msgpack_packer* pk, uint8_t d) +{ + return msgpack_pack_uint8(pk, d); +} + +int msgpack_pack_uint16_wrap(msgpack_packer* pk, uint16_t d) +{ + return msgpack_pack_uint16(pk, d); +} + +int msgpack_pack_uint32_wrap(msgpack_packer* pk, uint32_t d) +{ + return msgpack_pack_uint32(pk, d); +} + +int msgpack_pack_uint64_wrap(msgpack_packer* pk, uint64_t d) +{ + return msgpack_pack_uint64(pk, d); +} + +int msgpack_pack_int8_wrap(msgpack_packer* pk, int8_t d) +{ + return msgpack_pack_int8(pk, d); +} + +int msgpack_pack_int16_wrap(msgpack_packer* pk, int16_t d) +{ + return msgpack_pack_int16(pk, d); +} + +int msgpack_pack_int32_wrap(msgpack_packer* pk, int32_t d) +{ + return msgpack_pack_int32(pk, d); +} + +int msgpack_pack_int64_wrap(msgpack_packer* pk, int64_t d) +{ + return msgpack_pack_int64(pk, d); +} + +int msgpack_pack_double_wrap(msgpack_packer* pk, double d) +{ + return msgpack_pack_double(pk, d); +} + +int msgpack_pack_nil_wrap(msgpack_packer* pk) +{ + return msgpack_pack_nil(pk); +} + +int msgpack_pack_true_wrap(msgpack_packer* pk) +{ + return msgpack_pack_true(pk); +} + +int msgpack_pack_false_wrap(msgpack_packer* pk) +{ + return msgpack_pack_false(pk); +} + +int msgpack_pack_array_wrap(msgpack_packer* pk, unsigned int n) +{ + return msgpack_pack_array(pk, n); +} + +int msgpack_pack_map_wrap(msgpack_packer* pk, unsigned int n) +{ + return msgpack_pack_map(pk, n); +} + +int msgpack_pack_raw_wrap(msgpack_packer* pk, size_t l) +{ + return msgpack_pack_raw(pk, l); +} + +int msgpack_pack_raw_body_wrap(msgpack_packer* pk, const void *b, size_t l) +{ + return msgpack_pack_raw_body(pk, b, l); +} + +bool msgpack_unpacker_reserve_buffer_wrap(msgpack_unpacker *mpac, size_t size) +{ + return msgpack_unpacker_reserve_buffer(mpac, size); +} + +char *msgpack_unpacker_buffer_wrap(msgpack_unpacker *mpac) +{ + return msgpack_unpacker_buffer(mpac); +} + +size_t msgpack_unpacker_buffer_capacity_wrap(const msgpack_unpacker *mpac) +{ + return msgpack_unpacker_buffer_capacity(mpac); +} + +void msgpack_unpacker_buffer_consumed_wrap(msgpack_unpacker *mpac, size_t size) +{ + msgpack_unpacker_buffer_consumed(mpac, size); +} + +void msgpack_unpacker_data_wrap(msgpack_unpacker *mpac, msgpack_object *obj) +{ + *obj=msgpack_unpacker_data(mpac); +} + +size_t msgpack_unpacker_message_size_wrap(const msgpack_unpacker *mpac) +{ + return msgpack_unpacker_message_size(mpac); +} + diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal new file mode 100644 index 0000000..505a2b9 --- /dev/null +++ b/haskell/msgpack.cabal @@ -0,0 +1,32 @@ +Name: msgpack +Version: 0.2.0 +License: BSD3 +License-File: LICENSE +Author: Hideyuki Tanaka +Maintainer: Hideyuki Tanaka +Category: Data +Synopsis: A Haskell binding to MessagePack +Description: + A Haskell binding to MessagePack +Homepage: http://github.com/tanakh/hsmsgpack +Stability: Experimental +Tested-with: GHC==6.10.4 +Cabal-Version: >=1.2 +Build-Type: Simple + +library + build-depends: base>=4 && <5, mtl, bytestring + ghc-options: -O2 -Wall + hs-source-dirs: src + extra-libraries: msgpackc + + Exposed-modules: + Data.MessagePack + Data.MessagePack.Base + Data.MessagePack.Class + Data.MessagePack.Feed + Data.MessagePack.Monad + Data.MessagePack.Stream + + C-Sources: + cbits/msgpack.c diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs new file mode 100644 index 0000000..2949e60 --- /dev/null +++ b/haskell/src/Data/MessagePack.hs @@ -0,0 +1,63 @@ +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Simple interface to pack and unpack MessagePack data. +-- +-------------------------------------------------------------------- + +module Data.MessagePack( + module Data.MessagePack.Base, + module Data.MessagePack.Class, + module Data.MessagePack.Feed, + module Data.MessagePack.Monad, + module Data.MessagePack.Stream, + + -- * Pack and Unpack + packb, + unpackb, + + -- * Pure version of Pack and Unpack + packb', + unpackb', + ) where + +import Data.ByteString (ByteString) +import System.IO.Unsafe + +import Data.MessagePack.Base +import Data.MessagePack.Class +import Data.MessagePack.Feed +import Data.MessagePack.Monad +import Data.MessagePack.Stream + +-- | Pack Haskell data to MessagePack string. +packb :: OBJECT a => a -> IO ByteString +packb dat = do + sb <- newSimpleBuffer + pc <- newPacker sb + pack pc dat + simpleBufferData sb + +-- | Unpack MessagePack string to Haskell data. +unpackb :: OBJECT a => ByteString -> IO (Result a) +unpackb bs = do + withZone $ \z -> do + r <- unpackObject z bs + return $ case r of + Left err -> Left (show err) + Right (_, dat) -> fromObject dat + +-- | Pure version of 'packb'. +packb' :: OBJECT a => a -> ByteString +packb' dat = unsafePerformIO $ packb dat + +-- | Pure version of 'unpackb'. +unpackb' :: OBJECT a => ByteString -> Result a +unpackb' bs = unsafePerformIO $ unpackb bs diff --git a/haskell/src/Data/MessagePack/Base.hsc b/haskell/src/Data/MessagePack/Base.hsc new file mode 100644 index 0000000..ad71712 --- /dev/null +++ b/haskell/src/Data/MessagePack/Base.hsc @@ -0,0 +1,581 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE ForeignFunctionInterface #-} + +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Base +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Low Level Interface to MessagePack C API +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Base( + -- * Simple Buffer + SimpleBuffer, + newSimpleBuffer, + simpleBufferData, + + -- * Serializer + Packer, + newPacker, + + packU8, + packU16, + packU32, + packU64, + packS8, + packS16, + packS32, + packS64, + + packTrue, + packFalse, + + packInt, + packDouble, + packNil, + packBool, + + packArray, + packMap, + packRAW, + packRAWBody, + packRAW', + + -- * Stream Deserializer + Unpacker, + defaultInitialBufferSize, + newUnpacker, + unpackerReserveBuffer, + unpackerBuffer, + unpackerBufferCapacity, + unpackerBufferConsumed, + unpackerFeed, + unpackerExecute, + unpackerData, + unpackerReleaseZone, + unpackerResetZone, + unpackerReset, + unpackerMessageSize, + + -- * MessagePack Object + Object(..), + packObject, + + UnpackReturn(..), + unpackObject, + + -- * Memory Zone + Zone, + newZone, + freeZone, + withZone, + ) where + +import Control.Exception +import Control.Monad +import Data.ByteString (ByteString) +import qualified Data.ByteString as BS hiding (pack, unpack) +import Data.Int +import Data.Word +import Foreign.C +import Foreign.Concurrent +import Foreign.ForeignPtr hiding (newForeignPtr) +import Foreign.Marshal.Alloc +import Foreign.Marshal.Array +import Foreign.Ptr +import Foreign.Storable + +#include + +type SimpleBuffer = ForeignPtr () + +type WriteCallback = Ptr () -> CString -> CUInt -> IO CInt + +-- | Create a new Simple Buffer. It will be deleted automatically. +newSimpleBuffer :: IO SimpleBuffer +newSimpleBuffer = do + ptr <- mallocBytes (#size msgpack_sbuffer) + fptr <- newForeignPtr ptr $ do + msgpack_sbuffer_destroy ptr + free ptr + withForeignPtr fptr $ \p -> + msgpack_sbuffer_init p + return fptr + +-- | Get data of Simple Buffer. +simpleBufferData :: SimpleBuffer -> IO ByteString +simpleBufferData sb = + withForeignPtr sb $ \ptr -> do + size <- (#peek msgpack_sbuffer, size) ptr + dat <- (#peek msgpack_sbuffer, data) ptr + BS.packCStringLen (dat, fromIntegral (size :: CSize)) + +foreign import ccall "msgpack_sbuffer_init_wrap" msgpack_sbuffer_init :: + Ptr () -> IO () + +foreign import ccall "msgpack_sbuffer_destroy_wrap" msgpack_sbuffer_destroy :: + Ptr () -> IO () + +foreign import ccall "msgpack_sbuffer_write_wrap" msgpack_sbuffer_write :: + WriteCallback + +type Packer = ForeignPtr () + +-- | Create new Packer. It will be deleted automatically. +newPacker :: SimpleBuffer -> IO Packer +newPacker sbuf = do + cb <- wrap_callback msgpack_sbuffer_write + ptr <- withForeignPtr sbuf $ \ptr -> + msgpack_packer_new ptr cb + fptr <- newForeignPtr ptr $ do + msgpack_packer_free ptr + return fptr + +foreign import ccall "msgpack_packer_new_wrap" msgpack_packer_new :: + Ptr () -> FunPtr WriteCallback -> IO (Ptr ()) + +foreign import ccall "msgpack_packer_free_wrap" msgpack_packer_free :: + Ptr () -> IO () + +foreign import ccall "wrapper" wrap_callback :: + WriteCallback -> IO (FunPtr WriteCallback) + +packU8 :: Packer -> Word8 -> IO Int +packU8 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_uint8 ptr n + +foreign import ccall "msgpack_pack_uint8_wrap" msgpack_pack_uint8 :: + Ptr () -> Word8 -> IO CInt + +packU16 :: Packer -> Word16 -> IO Int +packU16 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_uint16 ptr n + +foreign import ccall "msgpack_pack_uint16_wrap" msgpack_pack_uint16 :: + Ptr () -> Word16 -> IO CInt + +packU32 :: Packer -> Word32 -> IO Int +packU32 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_uint32 ptr n + +foreign import ccall "msgpack_pack_uint32_wrap" msgpack_pack_uint32 :: + Ptr () -> Word32 -> IO CInt + +packU64 :: Packer -> Word64 -> IO Int +packU64 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_uint64 ptr n + +foreign import ccall "msgpack_pack_uint64_wrap" msgpack_pack_uint64 :: + Ptr () -> Word64 -> IO CInt + +packS8 :: Packer -> Int8 -> IO Int +packS8 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_int8 ptr n + +foreign import ccall "msgpack_pack_int8_wrap" msgpack_pack_int8 :: + Ptr () -> Int8 -> IO CInt + +packS16 :: Packer -> Int16 -> IO Int +packS16 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_int16 ptr n + +foreign import ccall "msgpack_pack_int16_wrap" msgpack_pack_int16 :: + Ptr () -> Int16 -> IO CInt + +packS32 :: Packer -> Int32 -> IO Int +packS32 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_int32 ptr n + +foreign import ccall "msgpack_pack_int32_wrap" msgpack_pack_int32 :: + Ptr () -> Int32 -> IO CInt + +packS64 :: Packer -> Int64 -> IO Int +packS64 pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_int64 ptr n + +foreign import ccall "msgpack_pack_int64_wrap" msgpack_pack_int64 :: + Ptr () -> Int64 -> IO CInt + +-- | Pack an integral data. +packInt :: Integral a => Packer -> a -> IO Int +packInt pc n = packS64 pc $ fromIntegral n + +-- | Pack a double data. +packDouble :: Packer -> Double -> IO Int +packDouble pc d = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_double ptr (realToFrac d) + +foreign import ccall "msgpack_pack_double_wrap" msgpack_pack_double :: + Ptr () -> CDouble -> IO CInt + +-- | Pack a nil. +packNil :: Packer -> IO Int +packNil pc = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_nil ptr + +foreign import ccall "msgpack_pack_nil_wrap" msgpack_pack_nil :: + Ptr () -> IO CInt + +packTrue :: Packer -> IO Int +packTrue pc = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_true ptr + +foreign import ccall "msgpack_pack_true_wrap" msgpack_pack_true :: + Ptr () -> IO CInt + +packFalse :: Packer -> IO Int +packFalse pc = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_false ptr + +foreign import ccall "msgpack_pack_false_wrap" msgpack_pack_false :: + Ptr () -> IO CInt + +-- | Pack a bool data. +packBool :: Packer -> Bool -> IO Int +packBool pc True = packTrue pc +packBool pc False = packFalse pc + +-- | 'packArray' @p n@ starts packing an array. +-- Next @n@ data will consist this array. +packArray :: Packer -> Int -> IO Int +packArray pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_array ptr (fromIntegral n) + +foreign import ccall "msgpack_pack_array_wrap" msgpack_pack_array :: + Ptr () -> CUInt -> IO CInt + +-- | 'packMap' @p n@ starts packing a map. +-- Next @n@ pairs of data (2*n data) will consist this map. +packMap :: Packer -> Int -> IO Int +packMap pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_map ptr (fromIntegral n) + +foreign import ccall "msgpack_pack_map_wrap" msgpack_pack_map :: + Ptr () -> CUInt -> IO CInt + +-- | 'packRAW' @p n@ starts packing a byte sequence. +-- Next total @n@ bytes of 'packRAWBody' call will consist this sequence. +packRAW :: Packer -> Int -> IO Int +packRAW pc n = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + msgpack_pack_raw ptr (fromIntegral n) + +foreign import ccall "msgpack_pack_raw_wrap" msgpack_pack_raw :: + Ptr () -> CSize -> IO CInt + +-- | Pack a byte sequence. +packRAWBody :: Packer -> ByteString -> IO Int +packRAWBody pc bs = + liftM fromIntegral $ withForeignPtr pc $ \ptr -> + BS.useAsCStringLen bs $ \(str, len) -> + msgpack_pack_raw_body ptr (castPtr str) (fromIntegral len) + +foreign import ccall "msgpack_pack_raw_body_wrap" msgpack_pack_raw_body :: + Ptr () -> Ptr () -> CSize -> IO CInt + +-- | Pack a single byte stream. It calls 'packRAW' and 'packRAWBody'. +packRAW' :: Packer -> ByteString -> IO Int +packRAW' pc bs = do + packRAW pc (BS.length bs) + packRAWBody pc bs + +type Unpacker = ForeignPtr () + +defaultInitialBufferSize :: Int +defaultInitialBufferSize = 32 * 1024 -- #const MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE + +-- | 'newUnpacker' @initialBufferSize@ creates a new Unpacker. It will be deleted automatically. +newUnpacker :: Int -> IO Unpacker +newUnpacker initialBufferSize = do + ptr <- msgpack_unpacker_new (fromIntegral initialBufferSize) + fptr <- newForeignPtr ptr $ do + msgpack_unpacker_free ptr + return fptr + +foreign import ccall "msgpack_unpacker_new" msgpack_unpacker_new :: + CSize -> IO (Ptr ()) + +foreign import ccall "msgpack_unpacker_free" msgpack_unpacker_free :: + Ptr() -> IO () + +-- | 'unpackerReserveBuffer' @up size@ reserves at least @size@ bytes of buffer. +unpackerReserveBuffer :: Unpacker -> Int -> IO Bool +unpackerReserveBuffer up size = + withForeignPtr up $ \ptr -> + liftM (/=0) $ msgpack_unpacker_reserve_buffer ptr (fromIntegral size) + +foreign import ccall "msgpack_unpacker_reserve_buffer_wrap" msgpack_unpacker_reserve_buffer :: + Ptr () -> CSize -> IO CChar + +-- | Get a pointer of unpacker buffer. +unpackerBuffer :: Unpacker -> IO (Ptr CChar) +unpackerBuffer up = + withForeignPtr up $ \ptr -> + msgpack_unpacker_buffer ptr + +foreign import ccall "msgpack_unpacker_buffer_wrap" msgpack_unpacker_buffer :: + Ptr () -> IO (Ptr CChar) + +-- | Get size of allocated buffer. +unpackerBufferCapacity :: Unpacker -> IO Int +unpackerBufferCapacity up = + withForeignPtr up $ \ptr -> + liftM fromIntegral $ msgpack_unpacker_buffer_capacity ptr + +foreign import ccall "msgpack_unpacker_buffer_capacity_wrap" msgpack_unpacker_buffer_capacity :: + Ptr () -> IO CSize + +-- | 'unpackerBufferConsumed' @up size@ notices that writed @size@ bytes to buffer. +unpackerBufferConsumed :: Unpacker -> Int -> IO () +unpackerBufferConsumed up size = + withForeignPtr up $ \ptr -> + msgpack_unpacker_buffer_consumed ptr (fromIntegral size) + +foreign import ccall "msgpack_unpacker_buffer_consumed_wrap" msgpack_unpacker_buffer_consumed :: + Ptr () -> CSize -> IO () + +-- | Write byte sequence to Unpacker. It is utility funciton, calls 'unpackerReserveBuffer', 'unpackerBuffer' and 'unpackerBufferConsumed'. +unpackerFeed :: Unpacker -> ByteString -> IO () +unpackerFeed up bs = + BS.useAsCStringLen bs $ \(str, len) -> do + True <- unpackerReserveBuffer up len + ptr <- unpackerBuffer up + copyArray ptr str len + unpackerBufferConsumed up len + +-- | Execute deserializing. It returns 0 when buffer contains not enough bytes, returns 1 when succeeded, returns negative value when it failed. +unpackerExecute :: Unpacker -> IO Int +unpackerExecute up = + withForeignPtr up $ \ptr -> + liftM fromIntegral $ msgpack_unpacker_execute ptr + +foreign import ccall "msgpack_unpacker_execute" msgpack_unpacker_execute :: + Ptr () -> IO CInt + +-- | Returns a deserialized object when 'unpackerExecute' returned 1. +unpackerData :: Unpacker -> IO Object +unpackerData up = + withForeignPtr up $ \ptr -> + allocaBytes (#size msgpack_object) $ \pobj -> do + msgpack_unpacker_data ptr pobj + peekObject pobj + +foreign import ccall "msgpack_unpacker_data_wrap" msgpack_unpacker_data :: + Ptr () -> Ptr () -> IO () + +-- | Release memory zone. The returned zone must be freed by calling 'freeZone'. +unpackerReleaseZone :: Unpacker -> IO Zone +unpackerReleaseZone up = + withForeignPtr up $ \ptr -> + msgpack_unpacker_release_zone ptr + +foreign import ccall "msgpack_unpacker_release_zone" msgpack_unpacker_release_zone :: + Ptr () -> IO (Ptr ()) + +-- | Free memory zone used by Unapcker. +unpackerResetZone :: Unpacker -> IO () +unpackerResetZone up = + withForeignPtr up $ \ptr -> + msgpack_unpacker_reset_zone ptr + +foreign import ccall "msgpack_unpacker_reset_zone" msgpack_unpacker_reset_zone :: + Ptr () -> IO () + +-- | Reset Unpacker state except memory zone. +unpackerReset :: Unpacker -> IO () +unpackerReset up = + withForeignPtr up $ \ptr -> + msgpack_unpacker_reset ptr + +foreign import ccall "msgpack_unpacker_reset" msgpack_unpacker_reset :: + Ptr () -> IO () + +-- | Returns number of bytes of sequence of deserializing object. +unpackerMessageSize :: Unpacker -> IO Int +unpackerMessageSize up = + withForeignPtr up $ \ptr -> + liftM fromIntegral $ msgpack_unpacker_message_size ptr + +foreign import ccall "msgpack_unpacker_message_size_wrap" msgpack_unpacker_message_size :: + Ptr () -> IO CSize + +type Zone = Ptr () + +-- | Create a new memory zone. It must be freed manually. +newZone :: IO Zone +newZone = + msgpack_zone_new (#const MSGPACK_ZONE_CHUNK_SIZE) + +-- | Free a memory zone. +freeZone :: Zone -> IO () +freeZone z = + msgpack_zone_free z + +-- | Create a memory zone, then execute argument, then free memory zone. +withZone :: (Zone -> IO a) -> IO a +withZone z = + bracket newZone freeZone z + +foreign import ccall "msgpack_zone_new" msgpack_zone_new :: + CSize -> IO Zone + +foreign import ccall "msgpack_zone_free" msgpack_zone_free :: + Zone -> IO () + +-- | Object Representation of MessagePack data. +data Object = + ObjectNil + | ObjectBool Bool + | ObjectInteger Int + | ObjectDouble Double + | ObjectRAW ByteString + | ObjectArray [Object] + | ObjectMap [(Object, Object)] + deriving (Show) + +peekObject :: Ptr a -> IO Object +peekObject ptr = do + typ <- (#peek msgpack_object, type) ptr + case (typ :: CInt) of + (#const MSGPACK_OBJECT_NIL) -> + return ObjectNil + (#const MSGPACK_OBJECT_BOOLEAN) -> + peekObjectBool ptr + (#const MSGPACK_OBJECT_POSITIVE_INTEGER) -> + peekObjectPositiveInteger ptr + (#const MSGPACK_OBJECT_NEGATIVE_INTEGER) -> + peekObjectNegativeInteger ptr + (#const MSGPACK_OBJECT_DOUBLE) -> + peekObjectDouble ptr + (#const MSGPACK_OBJECT_RAW) -> + peekObjectRAW ptr + (#const MSGPACK_OBJECT_ARRAY) -> + peekObjectArray ptr + (#const MSGPACK_OBJECT_MAP) -> + peekObjectMap ptr + _ -> + fail "peekObject: unknown object type" + +peekObjectBool :: Ptr a -> IO Object +peekObjectBool ptr = do + b <- (#peek msgpack_object, via.boolean) ptr + return $ ObjectBool $ (b :: CUChar) /= 0 + +peekObjectPositiveInteger :: Ptr a -> IO Object +peekObjectPositiveInteger ptr = do + n <- (#peek msgpack_object, via.u64) ptr + return $ ObjectInteger $ fromIntegral (n :: Word64) + +peekObjectNegativeInteger :: Ptr a -> IO Object +peekObjectNegativeInteger ptr = do + n <- (#peek msgpack_object, via.i64) ptr + return $ ObjectInteger $ fromIntegral (n :: Int64) + +peekObjectDouble :: Ptr a -> IO Object +peekObjectDouble ptr = do + d <- (#peek msgpack_object, via.dec) ptr + return $ ObjectDouble $ realToFrac (d :: CDouble) + +peekObjectRAW :: Ptr a -> IO Object +peekObjectRAW ptr = do + size <- (#peek msgpack_object, via.raw.size) ptr + p <- (#peek msgpack_object, via.raw.ptr) ptr + bs <- BS.packCStringLen (p, fromIntegral (size :: Word32)) + return $ ObjectRAW bs + +peekObjectArray :: Ptr a -> IO Object +peekObjectArray ptr = do + size <- (#peek msgpack_object, via.array.size) ptr + p <- (#peek msgpack_object, via.array.ptr) ptr + objs <- mapM (\i -> peekObject $ p `plusPtr` + ((#size msgpack_object) * i)) + [0..size-1] + return $ ObjectArray objs + +peekObjectMap :: Ptr a -> IO Object +peekObjectMap ptr = do + size <- (#peek msgpack_object, via.map.size) ptr + p <- (#peek msgpack_object, via.map.ptr) ptr + dat <- mapM (\i -> peekObjectKV $ p `plusPtr` + ((#size msgpack_object_kv) * i)) + [0..size-1] + return $ ObjectMap dat + +peekObjectKV :: Ptr a -> IO (Object, Object) +peekObjectKV ptr = do + k <- peekObject $ ptr `plusPtr` (#offset msgpack_object_kv, key) + v <- peekObject $ ptr `plusPtr` (#offset msgpack_object_kv, val) + return (k, v) + +-- | Pack a Object. +packObject :: Packer -> Object -> IO () +packObject pc ObjectNil = packNil pc >> return () + +packObject pc (ObjectBool b) = packBool pc b >> return () + +packObject pc (ObjectInteger n) = packInt pc n >> return () + +packObject pc (ObjectDouble d) = packDouble pc d >> return () + +packObject pc (ObjectRAW bs) = packRAW' pc bs >> return () + +packObject pc (ObjectArray ls) = do + packArray pc (length ls) + mapM_ (packObject pc) ls + +packObject pc (ObjectMap ls) = do + packMap pc (length ls) + mapM_ (\(a, b) -> packObject pc a >> packObject pc b) ls + +data UnpackReturn = + UnpackContinue -- ^ not enough bytes to unpack object + | UnpackParseError -- ^ got invalid bytes + | UnpackError -- ^ other error + deriving (Eq, Show) + +-- | Unpack a single MessagePack object from byte sequence. +unpackObject :: Zone -> ByteString -> IO (Either UnpackReturn (Int, Object)) +unpackObject z dat = + allocaBytes (#size msgpack_object) $ \ptr -> + BS.useAsCStringLen dat $ \(str, len) -> + alloca $ \poff -> do + ret <- msgpack_unpack str (fromIntegral len) poff z ptr + case ret of + (#const MSGPACK_UNPACK_SUCCESS) -> do + off <- peek poff + obj <- peekObject ptr + return $ Right (fromIntegral off, obj) + (#const MSGPACK_UNPACK_EXTRA_BYTES) -> do + off <- peek poff + obj <- peekObject ptr + return $ Right (fromIntegral off, obj) + (#const MSGPACK_UNPACK_CONTINUE) -> + return $ Left UnpackContinue + (#const MSGPACK_UNPACK_PARSE_ERROR) -> + return $ Left UnpackParseError + _ -> + return $ Left UnpackError + +foreign import ccall "msgpack_unpack" msgpack_unpack :: + Ptr CChar -> CSize -> Ptr CSize -> Zone -> Ptr () -> IO CInt diff --git a/haskell/src/Data/MessagePack/Class.hs b/haskell/src/Data/MessagePack/Class.hs new file mode 100644 index 0000000..f50a4d8 --- /dev/null +++ b/haskell/src/Data/MessagePack/Class.hs @@ -0,0 +1,97 @@ +{-# LANGUAGE TypeSynonymInstances #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE OverlappingInstances #-} +{-# LANGUAGE IncoherentInstances #-} + +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Class +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Serializing Haskell values to and from MessagePack Objects. +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Class( + -- * Serialization to and from Object + OBJECT(..), + Result, + pack, + ) where + +import Control.Monad.Error +import Data.ByteString.Char8 (ByteString) +import qualified Data.ByteString.Char8 as C8 +import Data.Either + +import Data.MessagePack.Base + +-- | The class of types serializable to and from MessagePack object +class OBJECT a where + toObject :: a -> Object + fromObject :: Object -> Result a + +-- | A type for parser results +type Result a = Either String a + +instance OBJECT Object where + toObject = id + fromObject = Right + +fromObjectError :: String +fromObjectError = "fromObject: cannot cast" + +instance OBJECT Int where + toObject = ObjectInteger + fromObject (ObjectInteger n) = Right n + fromObject _ = Left fromObjectError + +instance OBJECT Bool where + toObject = ObjectBool + fromObject (ObjectBool b) = Right b + fromObject _ = Left fromObjectError + +instance OBJECT Double where + toObject = ObjectDouble + fromObject (ObjectDouble d) = Right d + fromObject _ = Left fromObjectError + +instance OBJECT ByteString where + toObject = ObjectRAW + fromObject (ObjectRAW bs) = Right bs + fromObject _ = Left fromObjectError + +instance OBJECT String where + toObject = toObject . C8.pack + fromObject obj = liftM C8.unpack $ fromObject obj + +instance OBJECT a => OBJECT [a] where + toObject = ObjectArray . map toObject + fromObject (ObjectArray arr) = + mapM fromObject arr + fromObject _ = + Left fromObjectError + +instance (OBJECT a, OBJECT b) => OBJECT [(a, b)] where + toObject = + ObjectMap . map (\(a, b) -> (toObject a, toObject b)) + fromObject (ObjectMap mem) = do + mapM (\(a, b) -> liftM2 (,) (fromObject a) (fromObject b)) mem + fromObject _ = + Left fromObjectError + +instance OBJECT a => OBJECT (Maybe a) where + toObject (Just a) = toObject a + toObject Nothing = ObjectNil + + fromObject ObjectNil = return Nothing + fromObject obj = liftM Just $ fromObject obj + +-- | Pack a serializable Haskell value. +pack :: OBJECT a => Packer -> a -> IO () +pack pc = packObject pc . toObject diff --git a/haskell/src/Data/MessagePack/Feed.hs b/haskell/src/Data/MessagePack/Feed.hs new file mode 100644 index 0000000..afd3f6c --- /dev/null +++ b/haskell/src/Data/MessagePack/Feed.hs @@ -0,0 +1,59 @@ +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Feed +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Feeders for Stream Deserializers +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Feed( + -- * Feeder type + Feeder, + -- * Feeders + feederFromHandle, + feederFromFile, + feederFromString, + ) where + +import Control.Monad +import Data.ByteString (ByteString) +import qualified Data.ByteString as BS +import Data.IORef +import System.IO + +-- | Feeder returns Just ByteString when bytes remains, otherwise Nothing. +type Feeder = IO (Maybe ByteString) + +-- | Feeder from Handle +feederFromHandle :: Handle -> IO Feeder +feederFromHandle h = return $ do + bs <- BS.hGet h bufSize + if BS.length bs > 0 + then return $ Just bs + else do + hClose h + return Nothing + where + bufSize = 4096 + +-- | Feeder from File +feederFromFile :: FilePath -> IO Feeder +feederFromFile path = + openFile path ReadMode >>= feederFromHandle + +-- | Feeder from ByteString +feederFromString :: ByteString -> IO Feeder +feederFromString bs = do + r <- newIORef (Just bs) + return $ f r + where + f r = do + mb <- readIORef r + writeIORef r Nothing + return mb diff --git a/haskell/src/Data/MessagePack/Monad.hs b/haskell/src/Data/MessagePack/Monad.hs new file mode 100644 index 0000000..bf1514f --- /dev/null +++ b/haskell/src/Data/MessagePack/Monad.hs @@ -0,0 +1,153 @@ +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Monad +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Monadic Stream Serializers and Deserializers +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Monad( + -- * Classes + MonadPacker(..), + MonadUnpacker(..), + + -- * Packer and Unpacker type + PackerT(..), + UnpackerT(..), + + -- * Packers + packToString, + packToHandle, + packToFile, + + -- * Unpackers + unpackFrom, + unpackFromString, + unpackFromHandle, + unpackFromFile, + ) where + +import Control.Monad +import Control.Monad.Trans +import Data.ByteString (ByteString) +import qualified Data.ByteString as BS +import System.IO + +import Data.MessagePack.Base hiding (Unpacker) +import qualified Data.MessagePack.Base as Base +import Data.MessagePack.Class +import Data.MessagePack.Feed + +class Monad m => MonadPacker m where + -- | Serialize a object + put :: OBJECT a => a -> m () + +class Monad m => MonadUnpacker m where + -- | Deserialize a object + get :: OBJECT a => m a + +-- | Serializer Type +newtype PackerT m r = PackerT { runPackerT :: Base.Packer -> m r } + +instance Monad m => Monad (PackerT m) where + a >>= b = + PackerT $ \pc -> do + r <- runPackerT a pc + runPackerT (b r) pc + + return r = + PackerT $ \_ -> return r + +instance MonadTrans PackerT where + lift m = PackerT $ \_ -> m + +instance MonadIO m => MonadIO (PackerT m) where + liftIO = lift . liftIO + +instance MonadIO m => MonadPacker (PackerT m) where + put v = PackerT $ \pc -> liftIO $ do + pack pc v + +-- | Execute given serializer and returns byte sequence. +packToString :: MonadIO m => PackerT m r -> m ByteString +packToString m = do + sb <- liftIO $ newSimpleBuffer + pc <- liftIO $ newPacker sb + runPackerT m pc + liftIO $ simpleBufferData sb + +-- | Execcute given serializer and write byte sequence to Handle. +packToHandle :: MonadIO m => Handle -> PackerT m r -> m () +packToHandle h m = do + sb <- packToString m + liftIO $ BS.hPut h sb + liftIO $ hFlush h + +-- | Execute given serializer and write byte sequence to file. +packToFile :: MonadIO m => FilePath -> PackerT m r -> m () +packToFile p m = do + sb <- packToString m + liftIO $ BS.writeFile p sb + +-- | Deserializer type +newtype UnpackerT m r = UnpackerT { runUnpackerT :: Base.Unpacker -> Feeder -> m r } + +instance Monad m => Monad (UnpackerT m) where + a >>= b = + UnpackerT $ \up feed -> do + r <- runUnpackerT a up feed + runUnpackerT (b r) up feed + + return r = + UnpackerT $ \_ _ -> return r + +instance MonadTrans UnpackerT where + lift m = UnpackerT $ \_ _ -> m + +instance MonadIO m => MonadIO (UnpackerT m) where + liftIO = lift . liftIO + +instance MonadIO m => MonadUnpacker (UnpackerT m) where + get = UnpackerT $ \up feed -> liftIO $ do + resp <- unpackerExecute up + guard $ resp>=0 + when (resp==0) $ do + Just bs <- feed + unpackerFeed up bs + resp2 <- unpackerExecute up + guard $ resp2==1 + obj <- unpackerData up + freeZone =<< unpackerReleaseZone up + unpackerReset up + let Right r = fromObject obj + return r + +-- | Execute deserializer using given feeder. +unpackFrom :: MonadIO m => Feeder -> UnpackerT m r -> m r +unpackFrom f m = do + up <- liftIO $ newUnpacker defaultInitialBufferSize + runUnpackerT m up f + +-- | Execute deserializer using given handle. +unpackFromHandle :: MonadIO m => Handle -> UnpackerT m r -> m r +unpackFromHandle h m = + flip unpackFrom m =<< liftIO (feederFromHandle h) + +-- | Execute deserializer using given file content. +unpackFromFile :: MonadIO m => FilePath -> UnpackerT m r -> m r +unpackFromFile p m = do + h <- liftIO $ openFile p ReadMode + r <- flip unpackFrom m =<< liftIO (feederFromHandle h) + liftIO $ hClose h + return r + +-- | Execute deserializer from given byte sequence. +unpackFromString :: MonadIO m => ByteString -> UnpackerT m r -> m r +unpackFromString bs m = do + flip unpackFrom m =<< liftIO (feederFromString bs) diff --git a/haskell/src/Data/MessagePack/Stream.hs b/haskell/src/Data/MessagePack/Stream.hs new file mode 100644 index 0000000..bd17f46 --- /dev/null +++ b/haskell/src/Data/MessagePack/Stream.hs @@ -0,0 +1,84 @@ +-------------------------------------------------------------------- +-- | +-- Module : Data.MessagePack.Stream +-- Copyright : (c) Hideyuki Tanaka, 2009 +-- License : BSD3 +-- +-- Maintainer: tanaka.hideyuki@gmail.com +-- Stability : experimental +-- Portability: portable +-- +-- Lazy Stream Serializers and Deserializers +-- +-------------------------------------------------------------------- + +module Data.MessagePack.Stream( + unpackObjects, + unpackObjectsFromFile, + unpackObjectsFromHandle, + unpackObjectsFromString, + ) where + +import Control.Monad +import Data.ByteString (ByteString) +import qualified Data.ByteString as BS +import System.IO +import System.IO.Unsafe + +import Data.MessagePack.Base +import Data.MessagePack.Feed + +-- | Unpack objects using given feeder. +unpackObjects :: Feeder -> IO [Object] +unpackObjects feeder = do + up <- newUnpacker defaultInitialBufferSize + f up + where + f up = unsafeInterleaveIO $ do + mbo <- unpackOnce up + case mbo of + Just o -> do + os <- f up + return $ o:os + Nothing -> + return [] + + unpackOnce up = do + resp <- unpackerExecute up + case resp of + 0 -> do + r <- feedOnce up + if r + then unpackOnce up + else return Nothing + 1 -> do + obj <- unpackerData up + freeZone =<< unpackerReleaseZone up + unpackerReset up + return $ Just obj + _ -> + error $ "unpackerExecute fails: " ++ show resp + + feedOnce up = do + dat <- feeder + case dat of + Nothing -> + return False + Just bs -> do + unpackerFeed up bs + return True + +-- | Unpack objects from file. +unpackObjectsFromFile :: FilePath -> IO [Object] +unpackObjectsFromFile fname = + unpackObjects =<< feederFromFile fname + +-- | Unpack objects from handle. +unpackObjectsFromHandle :: Handle -> IO [Object] +unpackObjectsFromHandle h = + unpackObjects =<< feederFromHandle h + +-- | Unpack oobjects from given byte sequence. +unpackObjectsFromString :: ByteString -> IO [Object] +unpackObjectsFromString bs = + unpackObjects =<< feederFromString bs diff --git a/haskell/test/Monad.hs b/haskell/test/Monad.hs new file mode 100644 index 0000000..4bee5c5 --- /dev/null +++ b/haskell/test/Monad.hs @@ -0,0 +1,16 @@ +import Control.Monad.Trans +import Data.MessagePack + +main = do + sb <- packToString $ do + put [1,2,3::Int] + put (3.14 :: Double) + put "Hoge" + + print sb + + unpackFromString sb $ do + arr <- get + dbl <- get + str <- get + liftIO $ print (arr :: [Int], dbl :: Double, str :: String) diff --git a/haskell/test/Stream.hs b/haskell/test/Stream.hs new file mode 100644 index 0000000..ce060de --- /dev/null +++ b/haskell/test/Stream.hs @@ -0,0 +1,14 @@ +import Control.Applicative +import qualified Data.ByteString as BS +import Data.MessagePack + +main = do + sb <- newSimpleBuffer + pc <- newPacker sb + pack pc [1,2,3::Int] + pack pc True + pack pc "hoge" + bs <- simpleBufferData sb + + os <- unpackObjectsFromString bs + mapM_ print os diff --git a/haskell/test/Test.hs b/haskell/test/Test.hs new file mode 100644 index 0000000..4e713ba --- /dev/null +++ b/haskell/test/Test.hs @@ -0,0 +1,36 @@ +import Control.Monad +import Data.MessagePack + +{- +main = do + sb <- newSimpleBuffer + pc <- newPacker sb + + pack pc [(1,2),(2,3),(3::Int,4::Int)] + pack pc [4,5,6::Int] + pack pc "hoge" + + bs <- simpleBufferData sb + print bs + + up <- newUnpacker defaultInitialBufferSize + + unpackerFeed up bs + + let f = do + res <- unpackerExecute up + when (res==1) $ do + obj <- unpackerData up + print obj + f + + f + + return () +-} + +main = do + bs <- packb [(1,2),(2,3),(3::Int,4::Int)] + print bs + dat <- unpackb bs + print (dat :: Result [(Int, Int)]) From 11f7aa4212a2539b008f7e357474ebc412647c71 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sun, 18 Apr 2010 15:11:50 +0900 Subject: [PATCH 0413/1648] cpp: fixed small typo in configure.in --- cpp/configure.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/configure.in b/cpp/configure.in index a4cb4d8..8c59dda 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -33,7 +33,7 @@ if test "$msgpack_cv_atomic_ops" != "yes"; then Note that gcc < 4.1 is not supported. If you are using gcc >= 4.1 and the default target CPU architecture is "i386", try to -add CFLAGS="--march=i686" and CXXFLAGS="-march=i668" options to ./configure as follows: +add CFLAGS="--march=i686" and CXXFLAGS="-march=i686" options to ./configure as follows: $ ./configure CFLAGS="-march=i686" CXXFLAGS="-march=i686" ]) From 20fe9b6dde977cb7d923167292bbe89027a01795 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Mon, 19 Apr 2010 22:39:53 +0900 Subject: [PATCH 0414/1648] added pom.xml --- java/.gitignore | 4 + java/.settings/org.eclipse.jdt.core.prefs | 5 + java/Makefile | 16 +++ java/pom.xml | 125 ++++++++++++++++++++++ 4 files changed, 150 insertions(+) create mode 100755 java/.gitignore create mode 100755 java/.settings/org.eclipse.jdt.core.prefs create mode 100755 java/Makefile create mode 100755 java/pom.xml diff --git a/java/.gitignore b/java/.gitignore new file mode 100755 index 0000000..156d227 --- /dev/null +++ b/java/.gitignore @@ -0,0 +1,4 @@ +target +.project +.classpath +*~ diff --git a/java/.settings/org.eclipse.jdt.core.prefs b/java/.settings/org.eclipse.jdt.core.prefs new file mode 100755 index 0000000..c75ad47 --- /dev/null +++ b/java/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +#Mon Apr 19 22:18:48 JST 2010 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.source=1.6 +org.eclipse.jdt.core.compiler.compliance=1.6 diff --git a/java/Makefile b/java/Makefile new file mode 100755 index 0000000..2b6b07d --- /dev/null +++ b/java/Makefile @@ -0,0 +1,16 @@ + +.PHONY: compile test eclipse clean + + +compile: + mvn compile + +test: + mvn test + +# generate .project and .classpath file for Eclipse +eclipse: + mvn eclipse:eclipse + +clean: + mvn clean diff --git a/java/pom.xml b/java/pom.xml new file mode 100755 index 0000000..5a7b4a5 --- /dev/null +++ b/java/pom.xml @@ -0,0 +1,125 @@ + + + 4.0.0 + org.msgpack + msgpack + MessagePack for Java + 0.0.1-SNAPSHOT + MessagePack for Java + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + scm:git://github.com/msgpack/msgpack.git + + + + src + test + + + + src/main/resources + + + + + src/test/resources + + + + + + maven-compiler-plugin + + 1.6 + 1.6 + + + + + maven-eclipse-plugin + 2.5.1 + + + + maven-release-plugin + + + deploy + scm:git://github.com/msgpack/msgpack.git + + + + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + ${project.name} ${project.version} API + true + en_US + UTF-8 + + + + + org.apache.maven.plugins + maven-jxr-plugin + + true + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + + true + + + + + + + + + sourceforge + + + sourceforge.net + Repository at sourceforge.net + scpexe://shell.sourceforge.net/home/groups/m/ms/msgpack/htdocs/maven2/ + + + + + + + + junit + junit + 4.8.1 + test + + + + org.apache.activemq.protobuf + activemq-protobuf + 1.0 + test + + + + From f2622e54e350bcec9233084a0d52eb61dcdeddb1 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Mon, 19 Apr 2010 23:05:00 +0900 Subject: [PATCH 0415/1648] moved src and test codes to src/{main,test}/java --- java/src/{ => main/java}/org/msgpack/MessageMergeable.java | 0 java/src/{ => main/java}/org/msgpack/MessagePackable.java | 0 java/src/{ => main/java}/org/msgpack/MessageTypeException.java | 0 java/src/{ => main/java}/org/msgpack/Packer.java | 0 java/src/{ => main/java}/org/msgpack/Schema.java | 0 java/src/{ => main/java}/org/msgpack/UnbufferedUnpacker.java | 0 java/src/{ => main/java}/org/msgpack/UnpackException.java | 0 java/src/{ => main/java}/org/msgpack/UnpackIterator.java | 0 java/src/{ => main/java}/org/msgpack/Unpacker.java | 0 java/src/{ => main/java}/org/msgpack/impl/UnpackerImpl.java | 0 java/src/{ => main/java}/org/msgpack/schema/ArraySchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/ByteSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/ClassGenerator.java | 0 java/src/{ => main/java}/org/msgpack/schema/ClassSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/DoubleSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/FieldSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/FloatSchema.java | 0 .../{ => main/java}/org/msgpack/schema/GenericClassSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/GenericSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/IArraySchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/IMapSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/IntSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/LongSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/MapSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/RawSchema.java | 0 .../{ => main/java}/org/msgpack/schema/ReflectionClassSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/SSchemaParser.java | 0 java/src/{ => main/java}/org/msgpack/schema/ShortSchema.java | 0 .../{ => main/java}/org/msgpack/schema/SpecificClassSchema.java | 0 java/src/{ => main/java}/org/msgpack/schema/StringSchema.java | 0 java/{test => src/test/java}/org/msgpack/TestPackUnpack.java | 0 java/{test => src/test/java}/org/msgpack/TestSample.java | 0 32 files changed, 0 insertions(+), 0 deletions(-) rename java/src/{ => main/java}/org/msgpack/MessageMergeable.java (100%) rename java/src/{ => main/java}/org/msgpack/MessagePackable.java (100%) rename java/src/{ => main/java}/org/msgpack/MessageTypeException.java (100%) rename java/src/{ => main/java}/org/msgpack/Packer.java (100%) rename java/src/{ => main/java}/org/msgpack/Schema.java (100%) rename java/src/{ => main/java}/org/msgpack/UnbufferedUnpacker.java (100%) rename java/src/{ => main/java}/org/msgpack/UnpackException.java (100%) rename java/src/{ => main/java}/org/msgpack/UnpackIterator.java (100%) rename java/src/{ => main/java}/org/msgpack/Unpacker.java (100%) rename java/src/{ => main/java}/org/msgpack/impl/UnpackerImpl.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ArraySchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ByteSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ClassGenerator.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ClassSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/DoubleSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/FieldSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/FloatSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/GenericClassSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/GenericSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/IArraySchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/IMapSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/IntSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/LongSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/MapSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/RawSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ReflectionClassSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/SSchemaParser.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/ShortSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/SpecificClassSchema.java (100%) rename java/src/{ => main/java}/org/msgpack/schema/StringSchema.java (100%) rename java/{test => src/test/java}/org/msgpack/TestPackUnpack.java (100%) rename java/{test => src/test/java}/org/msgpack/TestSample.java (100%) diff --git a/java/src/org/msgpack/MessageMergeable.java b/java/src/main/java/org/msgpack/MessageMergeable.java similarity index 100% rename from java/src/org/msgpack/MessageMergeable.java rename to java/src/main/java/org/msgpack/MessageMergeable.java diff --git a/java/src/org/msgpack/MessagePackable.java b/java/src/main/java/org/msgpack/MessagePackable.java similarity index 100% rename from java/src/org/msgpack/MessagePackable.java rename to java/src/main/java/org/msgpack/MessagePackable.java diff --git a/java/src/org/msgpack/MessageTypeException.java b/java/src/main/java/org/msgpack/MessageTypeException.java similarity index 100% rename from java/src/org/msgpack/MessageTypeException.java rename to java/src/main/java/org/msgpack/MessageTypeException.java diff --git a/java/src/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java similarity index 100% rename from java/src/org/msgpack/Packer.java rename to java/src/main/java/org/msgpack/Packer.java diff --git a/java/src/org/msgpack/Schema.java b/java/src/main/java/org/msgpack/Schema.java similarity index 100% rename from java/src/org/msgpack/Schema.java rename to java/src/main/java/org/msgpack/Schema.java diff --git a/java/src/org/msgpack/UnbufferedUnpacker.java b/java/src/main/java/org/msgpack/UnbufferedUnpacker.java similarity index 100% rename from java/src/org/msgpack/UnbufferedUnpacker.java rename to java/src/main/java/org/msgpack/UnbufferedUnpacker.java diff --git a/java/src/org/msgpack/UnpackException.java b/java/src/main/java/org/msgpack/UnpackException.java similarity index 100% rename from java/src/org/msgpack/UnpackException.java rename to java/src/main/java/org/msgpack/UnpackException.java diff --git a/java/src/org/msgpack/UnpackIterator.java b/java/src/main/java/org/msgpack/UnpackIterator.java similarity index 100% rename from java/src/org/msgpack/UnpackIterator.java rename to java/src/main/java/org/msgpack/UnpackIterator.java diff --git a/java/src/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java similarity index 100% rename from java/src/org/msgpack/Unpacker.java rename to java/src/main/java/org/msgpack/Unpacker.java diff --git a/java/src/org/msgpack/impl/UnpackerImpl.java b/java/src/main/java/org/msgpack/impl/UnpackerImpl.java similarity index 100% rename from java/src/org/msgpack/impl/UnpackerImpl.java rename to java/src/main/java/org/msgpack/impl/UnpackerImpl.java diff --git a/java/src/org/msgpack/schema/ArraySchema.java b/java/src/main/java/org/msgpack/schema/ArraySchema.java similarity index 100% rename from java/src/org/msgpack/schema/ArraySchema.java rename to java/src/main/java/org/msgpack/schema/ArraySchema.java diff --git a/java/src/org/msgpack/schema/ByteSchema.java b/java/src/main/java/org/msgpack/schema/ByteSchema.java similarity index 100% rename from java/src/org/msgpack/schema/ByteSchema.java rename to java/src/main/java/org/msgpack/schema/ByteSchema.java diff --git a/java/src/org/msgpack/schema/ClassGenerator.java b/java/src/main/java/org/msgpack/schema/ClassGenerator.java similarity index 100% rename from java/src/org/msgpack/schema/ClassGenerator.java rename to java/src/main/java/org/msgpack/schema/ClassGenerator.java diff --git a/java/src/org/msgpack/schema/ClassSchema.java b/java/src/main/java/org/msgpack/schema/ClassSchema.java similarity index 100% rename from java/src/org/msgpack/schema/ClassSchema.java rename to java/src/main/java/org/msgpack/schema/ClassSchema.java diff --git a/java/src/org/msgpack/schema/DoubleSchema.java b/java/src/main/java/org/msgpack/schema/DoubleSchema.java similarity index 100% rename from java/src/org/msgpack/schema/DoubleSchema.java rename to java/src/main/java/org/msgpack/schema/DoubleSchema.java diff --git a/java/src/org/msgpack/schema/FieldSchema.java b/java/src/main/java/org/msgpack/schema/FieldSchema.java similarity index 100% rename from java/src/org/msgpack/schema/FieldSchema.java rename to java/src/main/java/org/msgpack/schema/FieldSchema.java diff --git a/java/src/org/msgpack/schema/FloatSchema.java b/java/src/main/java/org/msgpack/schema/FloatSchema.java similarity index 100% rename from java/src/org/msgpack/schema/FloatSchema.java rename to java/src/main/java/org/msgpack/schema/FloatSchema.java diff --git a/java/src/org/msgpack/schema/GenericClassSchema.java b/java/src/main/java/org/msgpack/schema/GenericClassSchema.java similarity index 100% rename from java/src/org/msgpack/schema/GenericClassSchema.java rename to java/src/main/java/org/msgpack/schema/GenericClassSchema.java diff --git a/java/src/org/msgpack/schema/GenericSchema.java b/java/src/main/java/org/msgpack/schema/GenericSchema.java similarity index 100% rename from java/src/org/msgpack/schema/GenericSchema.java rename to java/src/main/java/org/msgpack/schema/GenericSchema.java diff --git a/java/src/org/msgpack/schema/IArraySchema.java b/java/src/main/java/org/msgpack/schema/IArraySchema.java similarity index 100% rename from java/src/org/msgpack/schema/IArraySchema.java rename to java/src/main/java/org/msgpack/schema/IArraySchema.java diff --git a/java/src/org/msgpack/schema/IMapSchema.java b/java/src/main/java/org/msgpack/schema/IMapSchema.java similarity index 100% rename from java/src/org/msgpack/schema/IMapSchema.java rename to java/src/main/java/org/msgpack/schema/IMapSchema.java diff --git a/java/src/org/msgpack/schema/IntSchema.java b/java/src/main/java/org/msgpack/schema/IntSchema.java similarity index 100% rename from java/src/org/msgpack/schema/IntSchema.java rename to java/src/main/java/org/msgpack/schema/IntSchema.java diff --git a/java/src/org/msgpack/schema/LongSchema.java b/java/src/main/java/org/msgpack/schema/LongSchema.java similarity index 100% rename from java/src/org/msgpack/schema/LongSchema.java rename to java/src/main/java/org/msgpack/schema/LongSchema.java diff --git a/java/src/org/msgpack/schema/MapSchema.java b/java/src/main/java/org/msgpack/schema/MapSchema.java similarity index 100% rename from java/src/org/msgpack/schema/MapSchema.java rename to java/src/main/java/org/msgpack/schema/MapSchema.java diff --git a/java/src/org/msgpack/schema/RawSchema.java b/java/src/main/java/org/msgpack/schema/RawSchema.java similarity index 100% rename from java/src/org/msgpack/schema/RawSchema.java rename to java/src/main/java/org/msgpack/schema/RawSchema.java diff --git a/java/src/org/msgpack/schema/ReflectionClassSchema.java b/java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java similarity index 100% rename from java/src/org/msgpack/schema/ReflectionClassSchema.java rename to java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java diff --git a/java/src/org/msgpack/schema/SSchemaParser.java b/java/src/main/java/org/msgpack/schema/SSchemaParser.java similarity index 100% rename from java/src/org/msgpack/schema/SSchemaParser.java rename to java/src/main/java/org/msgpack/schema/SSchemaParser.java diff --git a/java/src/org/msgpack/schema/ShortSchema.java b/java/src/main/java/org/msgpack/schema/ShortSchema.java similarity index 100% rename from java/src/org/msgpack/schema/ShortSchema.java rename to java/src/main/java/org/msgpack/schema/ShortSchema.java diff --git a/java/src/org/msgpack/schema/SpecificClassSchema.java b/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java similarity index 100% rename from java/src/org/msgpack/schema/SpecificClassSchema.java rename to java/src/main/java/org/msgpack/schema/SpecificClassSchema.java diff --git a/java/src/org/msgpack/schema/StringSchema.java b/java/src/main/java/org/msgpack/schema/StringSchema.java similarity index 100% rename from java/src/org/msgpack/schema/StringSchema.java rename to java/src/main/java/org/msgpack/schema/StringSchema.java diff --git a/java/test/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java similarity index 100% rename from java/test/org/msgpack/TestPackUnpack.java rename to java/src/test/java/org/msgpack/TestPackUnpack.java diff --git a/java/test/org/msgpack/TestSample.java b/java/src/test/java/org/msgpack/TestSample.java similarity index 100% rename from java/test/org/msgpack/TestSample.java rename to java/src/test/java/org/msgpack/TestSample.java From d693d9270225eb6e6ab133b124f1adc626ed319c Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Mon, 19 Apr 2010 23:34:18 +0900 Subject: [PATCH 0416/1648] removed unnecessary settings --- java/Makefile | 10 +++++++++- java/pom.xml | 18 +----------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/java/Makefile b/java/Makefile index 2b6b07d..7885a13 100755 --- a/java/Makefile +++ b/java/Makefile @@ -1,6 +1,14 @@ -.PHONY: compile test eclipse clean +.PHONY: compile test eclipse clean package +all: + compile + +package: + mvn package + +install: + mvn install compile: mvn compile diff --git a/java/pom.xml b/java/pom.xml index 5a7b4a5..321e8d5 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ org.msgpack msgpack MessagePack for Java - 0.0.1-SNAPSHOT + 1.0-SNAPSHOT MessagePack for Java @@ -20,9 +20,6 @@ - src - test - src/main/resources @@ -77,17 +74,11 @@ org.apache.maven.plugins maven-jxr-plugin - - true - org.apache.maven.plugins maven-surefire-report-plugin - - true - @@ -114,12 +105,5 @@ test - - org.apache.activemq.protobuf - activemq-protobuf - 1.0 - test - - From 5aa620966482ead63198ff15f59aad482cacb180 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Mon, 19 Apr 2010 23:52:40 +0900 Subject: [PATCH 0417/1648] add README --- java/README | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 java/README diff --git a/java/README b/java/README new file mode 100755 index 0000000..33e6376 --- /dev/null +++ b/java/README @@ -0,0 +1,28 @@ + +To build the JAR file of Message Pack, you need to install Maven (http://maven.apache.org), then type the following command: + +$ mvn package + +To locally install the project, type +$ mvn install + +To generate project files (.project, .classpath) for Eclipse, do + +$ mvn eclipse:eclipse + +then import the folder from your Eclipse. + +Next, open the preference page in Eclipse and add the CLASSPATH variable: + +M2_REPO = $HOME/.m2/repository + +where $HOME is your home directory. In Windows XP, $HOME is: +C:/Documents and Settings/(user name)/.m2/repository + + +# How to release the project (compile, test, tagging, deploy) + +$ mvn release:prepare +$ mvn release:perform + + From bccac610a462f6f989f10d58b9fc55522624d76a Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Tue, 20 Apr 2010 00:05:26 +0900 Subject: [PATCH 0418/1648] changed the src/test folder --- java/build.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 java/build.xml diff --git a/java/build.xml b/java/build.xml old mode 100644 new mode 100755 index f6e93ae..bae8923 --- a/java/build.xml +++ b/java/build.xml @@ -11,7 +11,7 @@ - + @@ -24,11 +24,11 @@ - + - + From 354af69f6204ba4523cf7ca3e50f2af1ad8d609e Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 22 Apr 2010 14:38:10 +0900 Subject: [PATCH 0419/1648] ruby: fixes SEGV on MessagePack_Unpacker_each --- cpp/README.md | 4 +- ruby/msgpack_test.rb | 1 + ruby/unpack.c | 318 ++++++++++++++++++++----------------------- 3 files changed, 149 insertions(+), 174 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index cc1ba38..454ce1a 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -28,7 +28,7 @@ To use the library in your program, include msgpack.hpp header and link msgpack std::vector target; target.push_back("Hello,"); target.push_back("World!"); - + // Serialize it. msgpack::sbuffer buffer; // simple buffer msgpack::pack(&buffer, target); @@ -38,7 +38,7 @@ To use the library in your program, include msgpack.hpp header and link msgpack msgpack::object obj; msgpack::unpack_return ret = msgpack::unpack(buffer.data, buffer.size, NULL, &mempool, &obj); - + if(ret != msgapck::UNPACK_SUCCESS) { // error check exit(1); diff --git a/ruby/msgpack_test.rb b/ruby/msgpack_test.rb index 37db6a0..455aa40 100644 --- a/ruby/msgpack_test.rb +++ b/ruby/msgpack_test.rb @@ -209,6 +209,7 @@ class MessagePackTestFormat < Test::Unit::TestCase obj.to_msgpack.split(//).each do |b| pac.feed(b) pac.each {|o| + GC.start assert_equal(obj, o) parsed += 1 } diff --git a/ruby/unpack.c b/ruby/unpack.c index 11572c3..73aeeeb 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -31,7 +31,6 @@ typedef struct { int finished; VALUE source; size_t offset; - size_t parsed; VALUE buffer; VALUE stream; VALUE streambuf; @@ -144,9 +143,71 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha rb_raise(rb_eTypeError, "instance of String needed"); \ } + +static VALUE template_execute_rescue(VALUE nouse) +{ + rb_gc_enable(); +#ifdef RUBY_VM + rb_exc_raise(rb_errinfo()); +#else + rb_exc_raise(ruby_errinfo); +#endif +} + +static VALUE template_execute_do(VALUE argv) +{ + VALUE* args = (VALUE*)argv; + + msgpack_unpack_t* mp = (msgpack_unpack_t*)args[0]; + char* dptr = (char*)args[1]; + size_t dlen = (size_t)args[2]; + size_t* from = (size_t*)args[3]; + + int ret = template_execute(mp, dptr, dlen, from); + + return (VALUE)ret; +} + +static int template_execute_wrap(msgpack_unpack_t* mp, + VALUE str, size_t dlen, size_t* from) +{ + VALUE args[4] = { + (VALUE)mp, + (VALUE)RSTRING_PTR(str), + (VALUE)dlen, + (VALUE)from, + }; + +#ifdef HAVE_RUBY_ENCODING_H + // FIXME encodingã‚’ASCII-8BITã«ã™ã‚‹ + int enc_orig = rb_enc_get_index(str); + rb_enc_set_index(str, s_ascii_8bit); +#endif + + // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ + rb_gc_disable(); + + mp->user.source = str; + + int ret = (int)rb_rescue(template_execute_do, (VALUE)args, + template_execute_rescue, Qnil); + + mp->user.source = Qnil; + + rb_gc_enable(); + +#ifdef HAVE_RUBY_ENCODING_H + rb_enc_set_index(str, enc_orig); +#endif + + return ret; +} + + static VALUE cUnpacker; static VALUE eUnpackError; + static void MessagePack_Unpacker_free(void* data) { if(data) { free(data); } @@ -173,14 +234,6 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass) return obj; } -static VALUE MessagePack_Unpacker_reset(VALUE self) -{ - UNPACKER(self, mp); - template_init(mp); - mp->user.finished = 0; - return self; -} - static ID append_method_of(VALUE stream) { if(rb_respond_to(stream, s_sysread)) { @@ -204,10 +257,10 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) rb_raise(rb_eArgError, "wrong number of arguments (%d for 0)", argc); } - MessagePack_Unpacker_reset(self); UNPACKER(self, mp); + template_init(mp); + mp->user.finished = 0; mp->user.offset = 0; - mp->user.parsed = 0; mp->user.buffer = rb_str_new("",0); mp->user.stream = stream; mp->user.streambuf = rb_str_new("",0); @@ -215,102 +268,6 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) return self; } - -static VALUE MessagePack_Unpacker_execute_do(VALUE argv) -{ - VALUE* args = (VALUE*)argv; - - VALUE self = args[0]; - UNPACKER(self, mp); - - VALUE data = args[1]; - - size_t from = (unsigned long)args[2]; - char* dptr = RSTRING_PTR(data); - size_t dlen = (unsigned long)args[3]; - int ret; - - if(from >= dlen) { - rb_raise(eUnpackError, "offset is bigger than data buffer size."); - } - - mp->user.source = data; - ret = template_execute(mp, dptr, dlen, &from); - mp->user.source = Qnil; - - if(ret < 0) { - rb_raise(eUnpackError, "parse error."); - } else if(ret > 0) { - mp->user.finished = 1; - return ULONG2NUM(from); - } else { - mp->user.finished = 0; - return ULONG2NUM(from); - } -} - -static VALUE MessagePack_Unpacker_execute_rescue(VALUE nouse) -{ - rb_gc_enable(); -#ifdef RUBY_VM - rb_exc_raise(rb_errinfo()); -#else - rb_exc_raise(ruby_errinfo); -#endif -} - -static inline VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data, - unsigned long off, unsigned long dlen) -{ - // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ - rb_gc_disable(); - VALUE args[4] = {self, data, (VALUE)off, (VALUE)dlen}; - VALUE ret = rb_rescue(MessagePack_Unpacker_execute_do, (VALUE)args, - MessagePack_Unpacker_execute_rescue, Qnil); - rb_gc_enable(); - - return ret; -} - -static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, - VALUE off, VALUE limit) -{ - CHECK_STRING_TYPE(data); - return MessagePack_Unpacker_execute_impl(self, data, - NUM2ULONG(off), NUM2ULONG(limit)); -} - -static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) -{ - CHECK_STRING_TYPE(data); - return MessagePack_Unpacker_execute_impl(self, data, - NUM2ULONG(off), RSTRING_LEN(data)); -} - -static VALUE MessagePack_Unpacker_finished_p(VALUE self) -{ - UNPACKER(self, mp); - if(mp->user.finished) { - return Qtrue; - } - return Qfalse; -} - -static VALUE MessagePack_Unpacker_data(VALUE self) -{ - UNPACKER(self, mp); - return template_data(mp); -} - - -static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) -{ - UNPACKER(self, mp); - StringValue(data); - rb_str_cat(mp->user.buffer, RSTRING_PTR(data), RSTRING_LEN(data)); - return Qnil; -} - static VALUE MessagePack_Unpacker_stream_get(VALUE self) { UNPACKER(self, mp); @@ -325,6 +282,14 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) return val; } +static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) +{ + UNPACKER(self, mp); + StringValue(data); + rb_str_cat(mp->user.buffer, RSTRING_PTR(data), RSTRING_LEN(data)); + return Qnil; +} + static VALUE MessagePack_Unpacker_fill(VALUE self) { UNPACKER(self, mp); @@ -333,7 +298,7 @@ static VALUE MessagePack_Unpacker_fill(VALUE self) return Qnil; } - size_t len; + long len; if(RSTRING_LEN(mp->user.buffer) == 0) { rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, LONG2FIX(64*1024), mp->user.buffer); @@ -368,16 +333,17 @@ static VALUE MessagePack_Unpacker_each(VALUE self) } } - mp->user.source = mp->user.buffer; - ret = template_execute(mp, RSTRING_PTR(mp->user.buffer), RSTRING_LEN(mp->user.buffer), &mp->user.offset); - mp->user.source = Qnil; + ret = template_execute_wrap(mp, mp->user.buffer, + RSTRING_LEN(mp->user.buffer), &mp->user.offset); if(ret < 0) { rb_raise(eUnpackError, "parse error."); + } else if(ret > 0) { VALUE data = template_data(mp); template_init(mp); rb_yield(data); + } else { goto do_fill; } @@ -386,70 +352,28 @@ static VALUE MessagePack_Unpacker_each(VALUE self) return Qnil; } - -static VALUE MessagePack_unpack_do(VALUE argv) -{ - VALUE* args = (VALUE*)argv; - - msgpack_unpack_t* mp = (msgpack_unpack_t*)args[0]; - VALUE data = args[1]; - - size_t from = 0; - char* dptr = RSTRING_PTR(data); - size_t dlen = (unsigned long)args[2]; - int ret; - - mp->user.source = data; - ret = template_execute(mp, dptr, dlen, &from); - mp->user.source = Qnil; - - if(ret < 0) { - rb_raise(eUnpackError, "parse error."); - } else if(ret == 0) { - rb_raise(eUnpackError, "insufficient bytes."); - } else { - if(from < dlen) { - rb_raise(eUnpackError, "extra bytes."); - } - return template_data(mp); - } -} - -static VALUE MessagePack_unpack_rescue(VALUE nouse) -{ - rb_gc_enable(); -#ifdef RUBY_VM - rb_exc_raise(rb_errinfo()); -#else - rb_exc_raise(ruby_errinfo); -#endif -} - static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned long dlen) { msgpack_unpack_t mp; template_init(&mp); - unpack_user u = {0, Qnil, 0, 0, Qnil, Qnil, Qnil}; - mp.user = u; -#ifdef HAVE_RUBY_ENCODING_H - // FIXME encodingã‚’ASCII-8BITã«ã™ã‚‹ - int enc_orig = rb_enc_get_index(data); - rb_enc_set_index(data, s_ascii_8bit); -#endif + mp.user.finished = 0; - // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ - rb_gc_disable(); - VALUE args[3] = {(VALUE)&mp, data, (VALUE)dlen}; - VALUE ret = rb_rescue(MessagePack_unpack_do, (VALUE)args, - MessagePack_unpack_rescue, Qnil); - rb_gc_enable(); + size_t from = 0; + int ret = template_execute_wrap(&mp, data, dlen, &from); -#ifdef HAVE_RUBY_ENCODING_H - rb_enc_set_index(data, enc_orig); -#endif + if(ret < 0) { + rb_raise(eUnpackError, "parse error."); - return ret; + } else if(ret == 0) { + rb_raise(eUnpackError, "insufficient bytes."); + + } else { + if(from < dlen) { + rb_raise(eUnpackError, "extra bytes."); + } + return template_data(&mp); + } } static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) @@ -465,6 +389,54 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) } +/* compat */ +static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, + VALUE off, VALUE limit) +{ + CHECK_STRING_TYPE(data); + UNPACKER(self, mp); + size_t from = (size_t)NUM2ULONG(off); + int ret = template_execute_wrap(mp, data, NUM2ULONG(limit), &from); + return INT2FIX(ret); +} + +/* compat */ +static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) +{ + CHECK_STRING_TYPE(data); + UNPACKER(self, mp); + size_t from = (size_t)NUM2ULONG(off); + int ret = template_execute_wrap(mp, data, RSTRING_LEN(data), &from); + return INT2FIX(ret); +} + +/* compat */ +static VALUE MessagePack_Unpacker_finished_p(VALUE self) +{ + UNPACKER(self, mp); + if(mp->user.finished) { + return Qtrue; + } + return Qfalse; +} + +/* compat */ +static VALUE MessagePack_Unpacker_data(VALUE self) +{ + UNPACKER(self, mp); + return template_data(mp); +} + +/* compat */ +static VALUE MessagePack_Unpacker_reset(VALUE self) +{ + UNPACKER(self, mp); + template_init(mp); + mp->user.finished = 0; + return self; +} + + void Init_msgpack_unpack(VALUE mMessagePack) { s_sysread = rb_intern("sysread"); @@ -478,11 +450,6 @@ void Init_msgpack_unpack(VALUE mMessagePack) cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, -1); - rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); - rb_define_method(cUnpacker, "execute_limit", MessagePack_Unpacker_execute_limit, 3); - rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); - rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); - rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); rb_define_method(cUnpacker, "feed", MessagePack_Unpacker_feed, 1); rb_define_method(cUnpacker, "fill", MessagePack_Unpacker_fill, 0); rb_define_method(cUnpacker, "each", MessagePack_Unpacker_each, 0); @@ -490,5 +457,12 @@ void Init_msgpack_unpack(VALUE mMessagePack) rb_define_method(cUnpacker, "stream=", MessagePack_Unpacker_stream_set, 1); rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2); + + /* backward compatibility */ + rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); + rb_define_method(cUnpacker, "execute_limit", MessagePack_Unpacker_execute_limit, 3); + rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); + rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); + rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); } From d24193630ee5f3b5fcf5fd00bcaf95106b7a9c6d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 22 Apr 2010 14:46:54 +0900 Subject: [PATCH 0420/1648] reverts variable-length stack: avoids memory leak --- cpp/unpack.c | 16 ++++------------ msgpack/unpack_define.h | 2 +- msgpack/unpack_template.h | 12 ++++++++++++ perl/unpack.c | 3 --- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/cpp/unpack.c b/cpp/unpack.c index 98c8653..34016fd 100644 --- a/cpp/unpack.c +++ b/cpp/unpack.c @@ -44,7 +44,6 @@ struct template_context; typedef struct template_context template_context; static void template_init(template_context* ctx); -static void template_destroy(template_context* ctx); static msgpack_object template_data(template_context* ctx); @@ -216,7 +215,6 @@ bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size) void msgpack_unpacker_destroy(msgpack_unpacker* mpac) { msgpack_zone_free(mpac->z); - template_destroy(CTX_CAST(mpac->ctx)); free(mpac->ctx); decl_count(mpac->buffer); } @@ -370,7 +368,6 @@ msgpack_unpack_return msgpack_unpack(const char* data, size_t len, size_t* off, msgpack_zone* z, msgpack_object* result) { - msgpack_unpack_return ret = MSGPACK_UNPACK_SUCCESS; template_context ctx; template_init(&ctx); @@ -382,26 +379,21 @@ msgpack_unpack(const char* data, size_t len, size_t* off, int e = template_execute(&ctx, data, len, &noff); if(e < 0) { - ret = MSGPACK_UNPACK_PARSE_ERROR; - goto out; + return MSGPACK_UNPACK_PARSE_ERROR; } if(off != NULL) { *off = noff; } if(e == 0) { - ret = MSGPACK_UNPACK_CONTINUE; - goto out; + return MSGPACK_UNPACK_CONTINUE; } *result = template_data(&ctx); if(noff < len) { - ret = MSGPACK_UNPACK_EXTRA_BYTES; - goto out; + return MSGPACK_UNPACK_EXTRA_BYTES; } -out: - template_destroy(&ctx); - return ret; + return MSGPACK_UNPACK_SUCCESS; } diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h index 71412ee..959d351 100644 --- a/msgpack/unpack_define.h +++ b/msgpack/unpack_define.h @@ -30,7 +30,7 @@ extern "C" { #ifndef MSGPACK_EMBED_STACK_SIZE -#define MSGPACK_EMBED_STACK_SIZE 16 +#define MSGPACK_EMBED_STACK_SIZE 32 #endif diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 4b8cd14..0fbfbb7 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -58,9 +58,12 @@ msgpack_unpack_struct_decl(_context) { unsigned int cs; unsigned int trail; unsigned int top; + /* msgpack_unpack_struct(_stack)* stack; unsigned int stack_size; msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE]; + */ + msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE]; }; @@ -69,17 +72,21 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx) ctx->cs = CS_HEADER; ctx->trail = 0; ctx->top = 0; + /* ctx->stack = ctx->embed_stack; ctx->stack_size = MSGPACK_EMBED_STACK_SIZE; + */ ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user); } +/* msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx) { if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) { free(ctx->stack); } } +*/ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx) { @@ -99,7 +106,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c unsigned int cs = ctx->cs; unsigned int top = ctx->top; msgpack_unpack_struct(_stack)* stack = ctx->stack; + /* unsigned int stack_size = ctx->stack_size; + */ msgpack_unpack_user* user = &ctx->user; msgpack_unpack_object obj; @@ -129,6 +138,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c goto _fixed_trail_again #define start_container(func, count_, ct_) \ + if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \ if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \ if((count_) == 0) { obj = stack[top].obj; goto _push; } \ stack[top].ct = ct_; \ @@ -136,6 +146,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ++top; \ /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \ /*printf("stack push %d\n", top);*/ \ + /* FIXME \ if(top >= stack_size) { \ if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \ size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \ @@ -153,6 +164,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c ctx->stack_size = stack_size = stack_size * 2; \ } \ } \ + */ \ goto _header_again #define NEXT_CS(p) \ diff --git a/perl/unpack.c b/perl/unpack.c index c520e02..69017f1 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -53,7 +53,6 @@ struct template_context; typedef struct template_context msgpack_unpack_t; static void template_init(msgpack_unpack_t* u); -static void template_destroy(msgpack_unpack_t* u); static SV* template_data(msgpack_unpack_t* u); @@ -143,7 +142,6 @@ SV* _msgpack_unpack(SV* data, int limit) { mp.user.source = &PL_sv_undef; obj = template_data(&mp); - template_destroy(&mp); if(ret < 0) { Perl_croak(aTHX_ "parse error."); @@ -316,7 +314,6 @@ XS(xs_unpacker_destroy) { } UNPACKER(ST(0), mp); - template_destroy(mp); Safefree(mp); XSRETURN(0); From 60fbaf7612a5340de16ba98277ef01ffb80d1da0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 22 Apr 2010 14:56:25 +0900 Subject: [PATCH 0421/1648] ruby: 0.3.8 --- ruby/AUTHORS | 1 + ruby/ChangeLog | 0 ruby/makegem.sh | 4 +--- ruby/msgpack.gemspec | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 ruby/AUTHORS create mode 100644 ruby/ChangeLog diff --git a/ruby/AUTHORS b/ruby/AUTHORS new file mode 100644 index 0000000..ababacb --- /dev/null +++ b/ruby/AUTHORS @@ -0,0 +1 @@ +FURUHASHI Sadayuki diff --git a/ruby/ChangeLog b/ruby/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/ruby/makegem.sh b/ruby/makegem.sh index 8737d29..5ea66f1 100755 --- a/ruby/makegem.sh +++ b/ruby/makegem.sh @@ -8,8 +8,6 @@ cp pack.h ext/ cp rbinit.c ext/ cp unpack.c ext/ cp unpack.h ext/ -cp ../AUTHORS ./ -cp ../ChangeLog ./ cp ../msgpack/pack_define.h msgpack/ cp ../msgpack/pack_template.h msgpack/ cp ../msgpack/unpack_define.h msgpack/ @@ -20,7 +18,7 @@ cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FI gem build msgpack.gemspec if [ $? -eq 0 ]; then - rm -rf ext msgpack AUTHORS ChangeLog test/msgpack_test.rb + rm -rf ext msgpack test/msgpack_test.rb fi # gem install gem-compile # on msys diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 3a261f2..b8e88d5 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.7" + s.version = "0.3.8" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From b10a7367440a33c9675be8fa435c9b202ff0c4f8 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 23 Apr 2010 18:13:36 +0900 Subject: [PATCH 0422/1648] ruby: fixese backward compatibility of streaming deserializer --- ruby/msgpack_test.rb | 22 ++++++++++++++++++++++ ruby/unpack.c | 34 ++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/ruby/msgpack_test.rb b/ruby/msgpack_test.rb index 455aa40..45adf0b 100644 --- a/ruby/msgpack_test.rb +++ b/ruby/msgpack_test.rb @@ -218,6 +218,28 @@ class MessagePackTestFormat < Test::Unit::TestCase assert_equal(parsed, 1) end + it "streaming backward compatibility" do + obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + pac = MessagePack::Unpacker.new + buffer = "" + nread = 0 + parsed = 0 + obj.to_msgpack.split(//).each do |b| + buffer << b + nread = pac.execute(buffer, nread) + if pac.finished? + o = pac.data + assert_equal(obj, o) + parsed += 1 + pac.reset + buffer.slice!(0, nread) + nread = 0 + next unless buffer.empty? + end + end + assert_equal(parsed, 1) + end + private def check(len, obj) v = obj.to_msgpack diff --git a/ruby/unpack.c b/ruby/unpack.c index 73aeeeb..3a95e5a 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -389,25 +389,43 @@ static VALUE MessagePack_unpack(VALUE self, VALUE data) } +static VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data, + size_t from, size_t limit) +{ + UNPACKER(self, mp); + + if(from >= limit) { + rb_raise(eUnpackError, "offset is bigger than data buffer size."); + } + + int ret = template_execute_wrap(mp, data, limit, &from); + + if(ret < 0) { + rb_raise(eUnpackError, "parse error."); + } else if(ret > 0) { + mp->user.finished = 1; + return ULONG2NUM(from); + } else { + mp->user.finished = 0; + return ULONG2NUM(from); + } +} + /* compat */ static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, VALUE off, VALUE limit) { CHECK_STRING_TYPE(data); - UNPACKER(self, mp); - size_t from = (size_t)NUM2ULONG(off); - int ret = template_execute_wrap(mp, data, NUM2ULONG(limit), &from); - return INT2FIX(ret); + return MessagePack_Unpacker_execute_impl(self, data, + (size_t)NUM2ULONG(off), (size_t)NUM2ULONG(limit)); } /* compat */ static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) { CHECK_STRING_TYPE(data); - UNPACKER(self, mp); - size_t from = (size_t)NUM2ULONG(off); - int ret = template_execute_wrap(mp, data, RSTRING_LEN(data), &from); - return INT2FIX(ret); + return MessagePack_Unpacker_execute_impl(self, data, + (size_t)NUM2ULONG(off), (size_t)RSTRING_LEN(data)); } /* compat */ From c9fcf4020f309722e9f42a65bf7fe3b9c66a3a59 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 23 Apr 2010 20:18:48 +0900 Subject: [PATCH 0423/1648] ruby: streaming deserializer test --- ruby/msgpack_test.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ruby/msgpack_test.rb b/ruby/msgpack_test.rb index 45adf0b..8cbb586 100644 --- a/ruby/msgpack_test.rb +++ b/ruby/msgpack_test.rb @@ -204,9 +204,11 @@ class MessagePackTestFormat < Test::Unit::TestCase it "gc mark" do obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + num = 4 + raw = obj.to_msgpack * num pac = MessagePack::Unpacker.new parsed = 0 - obj.to_msgpack.split(//).each do |b| + raw.split(//).each do |b| pac.feed(b) pac.each {|o| GC.start @@ -215,16 +217,18 @@ class MessagePackTestFormat < Test::Unit::TestCase } GC.start end - assert_equal(parsed, 1) + assert_equal(parsed, num) end it "streaming backward compatibility" do obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"] + num = 4 + raw = obj.to_msgpack * num pac = MessagePack::Unpacker.new buffer = "" nread = 0 parsed = 0 - obj.to_msgpack.split(//).each do |b| + raw.split(//).each do |b| buffer << b nread = pac.execute(buffer, nread) if pac.finished? @@ -237,7 +241,7 @@ class MessagePackTestFormat < Test::Unit::TestCase next unless buffer.empty? end end - assert_equal(parsed, 1) + assert_equal(parsed, num) end private From 8335823748973b07dd628d3a94d49bdb615600ee Mon Sep 17 00:00:00 2001 From: frsyuki Date: Fri, 23 Apr 2010 20:24:36 +0900 Subject: [PATCH 0424/1648] ruby-0.3.9 --- ruby/msgpack.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index b8e88d5..c5e8c8c 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.8" + s.version = "0.3.9" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" From 120e8bffd7917e9529229e796b21ececc51df016 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 00:03:09 +0900 Subject: [PATCH 0425/1648] cpp: object::object(const T& v) and object::operator=(const T& v) --- cpp/msgpack/object.hpp | 107 +++++++++++++++---------------------- cpp/msgpack/type/bool.hpp | 6 +++ cpp/msgpack/type/float.hpp | 13 +++++ cpp/msgpack/type/int.hpp | 34 +++++++++++- cpp/msgpack/type/nil.hpp | 5 ++ 5 files changed, 101 insertions(+), 64 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 3b42a8e..0125d0a 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -87,15 +87,15 @@ struct object { template void convert(T* v) const; - operator msgpack_object(); - object(msgpack_object obj); - object(); - object(bool v); - object(uint64_t v); - object(int64_t v); - object(double v); - object(const char* ptr, size_t size); + + template + object(const T& v); + + template + object& operator=(const T& v); + + operator msgpack_object(); private: struct implicit_type; @@ -115,9 +115,11 @@ bool operator!=(const object x, const object y); std::ostream& operator<< (std::ostream& s, const object o); +// serialize operator template packer& operator<< (packer& o, const T& v); +// convert operator template T& operator>> (object o, T& v); @@ -190,61 +192,6 @@ inline bool operator!=(const object x, const object y) { return !(x == y); } -inline object::object() -{ - type = type::NIL; -} - -inline object::object(bool v) -{ - type = type::BOOLEAN; - via.boolean = v; -} - -inline object::object(uint64_t v) -{ - type = type::POSITIVE_INTEGER; - via.u64 = v; -} - -inline object::object(int64_t v) -{ - if(v >= 0) { - type = type::POSITIVE_INTEGER; - via.u64 = v; - } else { - type = type::NEGATIVE_INTEGER; - via.i64 = v; - } -} - -inline object::object(double v) -{ - type = type::DOUBLE; - via.dec = v; -} - -inline object::object(const char* ptr, size_t size) -{ - type = type::RAW; - via.raw.size = size; - via.raw.ptr = ptr; -} - -inline object::object(msgpack_object obj) -{ - // FIXME beter way? - ::memcpy(this, &obj, sizeof(obj)); -} - -inline object::operator msgpack_object() -{ - // FIXME beter way? - msgpack_object obj; - ::memcpy(&obj, this, sizeof(obj)); - return obj; -} - inline object::implicit_type object::convert() const { return implicit_type(*this); @@ -265,6 +212,40 @@ inline T object::as() const } +inline object::object() +{ + type = type::NIL; +} + +template +inline object::object(const T& v) +{ + *this << v; +} + +template +inline object& object::operator=(const T& v) +{ + *this << v; + return *this; +} + + +inline object::operator msgpack_object() +{ + // FIXME beter way? + msgpack_object obj; + ::memcpy(&obj, this, sizeof(obj)); + return obj; +} + +inline void operator<< (object& o, msgpack_object v) +{ + // FIXME beter way? + ::memcpy(&o, &v, sizeof(v)); +} + + // obsolete template inline void convert(T& v, object o) diff --git a/cpp/msgpack/type/bool.hpp b/cpp/msgpack/type/bool.hpp index b945d85..9d63435 100644 --- a/cpp/msgpack/type/bool.hpp +++ b/cpp/msgpack/type/bool.hpp @@ -39,6 +39,12 @@ inline packer& operator<< (packer& o, const bool& v) return o; } +inline void operator<< (object& o, bool v) +{ + o.type = type::BOOLEAN; + o.via.boolean = v; +} + } // namespace msgpack diff --git a/cpp/msgpack/type/float.hpp b/cpp/msgpack/type/float.hpp index 390e340..28131d1 100644 --- a/cpp/msgpack/type/float.hpp +++ b/cpp/msgpack/type/float.hpp @@ -57,6 +57,19 @@ inline packer& operator<< (packer& o, const double& v) } +inline void operator<< (object& o, float v) +{ + o.type = type::DOUBLE; + o.via.dec = v; +} + +inline void operator<< (object& o, double v) +{ + o.type = type::DOUBLE; + o.via.dec = v; +} + + } // namespace msgpack #endif /* msgpack/type/float.hpp */ diff --git a/cpp/msgpack/type/int.hpp b/cpp/msgpack/type/int.hpp index 03500f7..2dab78b 100644 --- a/cpp/msgpack/type/int.hpp +++ b/cpp/msgpack/type/int.hpp @@ -56,7 +56,7 @@ namespace detail { throw type_error(); } }; - + template static inline T convert_integer(object o) { @@ -141,6 +141,38 @@ inline packer& operator<< (packer& o, const unsigned long long& { o.pack_unsigned_long_long(v); return o; } +inline void operator<< (object& o, signed char v) + { v < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v : o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, signed short v) + { v < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v : o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, signed int v) + { v < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v : o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, signed long v) + { v < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v : o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, signed long long v) + { v < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v : o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + + +inline void operator<< (object& o, unsigned char v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, unsigned short v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, unsigned int v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, unsigned long v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + +inline void operator<< (object& o, unsigned long long v) + { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } + + } // namespace msgpack #endif /* msgpack/type/int.hpp */ diff --git a/cpp/msgpack/type/nil.hpp b/cpp/msgpack/type/nil.hpp index 93e66ff..ed58384 100644 --- a/cpp/msgpack/type/nil.hpp +++ b/cpp/msgpack/type/nil.hpp @@ -42,6 +42,11 @@ inline packer& operator<< (packer& o, const type::nil& v) return o; } +inline void operator<< (object& o, type::nil v) +{ + o.type = type::NIL; +} + } // namespace msgpack From 4e85ebbf9863201107a8f5c68fdabc92749e04fc Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 01:12:25 +0900 Subject: [PATCH 0426/1648] cpp: object::object(const T& v, zone* z) --- cpp/msgpack/object.hpp | 25 +++++++++++++++++++++++++ cpp/msgpack/type/deque.hpp | 16 ++++++++++++++++ cpp/msgpack/type/list.hpp | 16 ++++++++++++++++ cpp/msgpack/type/pair.hpp | 11 +++++++++++ cpp/msgpack/type/raw.hpp | 7 +++++++ cpp/msgpack/type/set.hpp | 32 ++++++++++++++++++++++++++++++++ cpp/msgpack/type/string.hpp | 9 +++++++++ cpp/msgpack/type/tuple.hpp.erb | 20 ++++++++++++++++++++ cpp/msgpack/type/vector.hpp | 16 ++++++++++++++++ cpp/msgpack/zone.hpp.erb | 1 - 10 files changed, 152 insertions(+), 1 deletion(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 0125d0a..a094c56 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -20,6 +20,7 @@ #include "msgpack/object.h" #include "msgpack/pack.hpp" +#include "msgpack/zone.hpp" #include #include #include @@ -92,6 +93,9 @@ struct object { template object(const T& v); + template + object(const T& v, zone* z); + template object& operator=(const T& v); @@ -100,6 +104,10 @@ struct object { private: struct implicit_type; +public: + // FIXME private? + struct object_zone; + public: implicit_type convert() const; }; @@ -109,6 +117,14 @@ struct object_kv { object val; }; +struct object::object_zone : object { + object_zone(msgpack::zone* zone) : zone(zone) { } + msgpack::zone* zone; +private: + object_zone(); +}; + + bool operator==(const object x, const object y); bool operator!=(const object x, const object y); @@ -230,6 +246,15 @@ inline object& object::operator=(const T& v) return *this; } +template +object::object(const T& v, zone* z) +{ + object_zone oz(z); + oz << v; + type = oz.type; + via = oz.via; +} + inline object::operator msgpack_object() { diff --git a/cpp/msgpack/type/deque.hpp b/cpp/msgpack/type/deque.hpp index d34d243..5e919a8 100644 --- a/cpp/msgpack/type/deque.hpp +++ b/cpp/msgpack/type/deque.hpp @@ -49,6 +49,22 @@ inline packer& operator<< (packer& o, const std::deque& v) return o; } +template +inline void operator<< (object::object_zone& o, const std::deque& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::deque::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); +} + } // namespace msgpack diff --git a/cpp/msgpack/type/list.hpp b/cpp/msgpack/type/list.hpp index 6ecc02f..5318639 100644 --- a/cpp/msgpack/type/list.hpp +++ b/cpp/msgpack/type/list.hpp @@ -49,6 +49,22 @@ inline packer& operator<< (packer& o, const std::list& v) return o; } +template +inline void operator<< (object::object_zone& o, const std::list& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::list::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); +} + } // namespace msgpack diff --git a/cpp/msgpack/type/pair.hpp b/cpp/msgpack/type/pair.hpp index ba72c1f..6c68288 100644 --- a/cpp/msgpack/type/pair.hpp +++ b/cpp/msgpack/type/pair.hpp @@ -43,6 +43,17 @@ inline packer& operator<< (packer& o, const std::pair& v return o; } +template +inline void operator<< (object::object_zone& o, const std::pair& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*2); + o.via.array.ptr = p; + o.via.array.size = 2; + p[0] = object(v.first, o.zone); + p[1] = object(v.second, o.zone); +} + } // namespace msgpack diff --git a/cpp/msgpack/type/raw.hpp b/cpp/msgpack/type/raw.hpp index 8c68aba..d854d28 100644 --- a/cpp/msgpack/type/raw.hpp +++ b/cpp/msgpack/type/raw.hpp @@ -77,6 +77,13 @@ inline packer& operator<< (packer& o, const type::raw_ref& v) return o; } +inline void operator<< (object& o, const type::raw_ref& v) +{ + o.type = type::RAW; + o.via.raw.ptr = v.ptr; + o.via.raw.size = v.size; +} + } // namespace msgpack diff --git a/cpp/msgpack/type/set.hpp b/cpp/msgpack/type/set.hpp index f2c5bfb..6f5fb18 100644 --- a/cpp/msgpack/type/set.hpp +++ b/cpp/msgpack/type/set.hpp @@ -48,6 +48,22 @@ inline packer& operator<< (packer& o, const std::set& v) return o; } +template +inline void operator<< (object::object_zone& o, const std::set& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::set::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); +} + template inline std::multiset& operator>> (object o, std::multiset& v) @@ -73,6 +89,22 @@ inline packer& operator<< (packer& o, const std::multiset& v) return o; } +template +inline void operator<< (object::object_zone& o, const std::multiset& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::multiset::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); +} + } // namespace msgpack diff --git a/cpp/msgpack/type/string.hpp b/cpp/msgpack/type/string.hpp index a085d53..796603f 100644 --- a/cpp/msgpack/type/string.hpp +++ b/cpp/msgpack/type/string.hpp @@ -39,6 +39,15 @@ inline packer& operator<< (packer& o, const std::string& v) return o; } +inline void operator<< (object::object_zone& o, const std::string& v) +{ + o.type = type::RAW; + char* ptr = (char*)o.zone->malloc(v.size()); + o.via.raw.ptr = ptr; + o.via.raw.size = v.size(); + memcpy(ptr, v.data(), v.size()); +} + } // namespace msgpack diff --git a/cpp/msgpack/type/tuple.hpp.erb b/cpp/msgpack/type/tuple.hpp.erb index 2930ae0..e57fe98 100644 --- a/cpp/msgpack/type/tuple.hpp.erb +++ b/cpp/msgpack/type/tuple.hpp.erb @@ -165,6 +165,26 @@ const packer& operator<< ( } <%}%> +inline void operator<< ( + object::object_zone& o, + const type::tuple<>& v) { + o.type = type::ARRAY; + o.via.array.ptr = NULL; + o.via.array.size = 0; +} +<%0.upto(GENERATION_LIMIT) {|i|%> +template , typename A<%=j%><%}%>> +inline void operator<< ( + object::object_zone& o, + const type::tuple, A<%=j%><%}%>>& v) { + o.type = type::ARRAY; + o.via.array.ptr = (object*)o.zone->malloc(sizeof(object)*<%=i+1%>); + o.via.array.size = <%=i+1%>; + <%0.upto(i) {|j|%> + o.via.array.ptr[<%=j%>] = object(v.template get<<%=j%>>(), o.zone);<%}%> +} +<%}%> + } // namespace msgpack #endif /* msgpack/type/tuple.hpp */ diff --git a/cpp/msgpack/type/vector.hpp b/cpp/msgpack/type/vector.hpp index 385d070..f9e709b 100644 --- a/cpp/msgpack/type/vector.hpp +++ b/cpp/msgpack/type/vector.hpp @@ -53,6 +53,22 @@ inline packer& operator<< (packer& o, const std::vector& v) return o; } +template +inline void operator<< (object::object_zone& o, const std::vector& v) +{ + o.type = type::ARRAY; + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::vector::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); +} + } // namespace msgpack diff --git a/cpp/msgpack/zone.hpp.erb b/cpp/msgpack/zone.hpp.erb index 48988ab..8e69aa4 100644 --- a/cpp/msgpack/zone.hpp.erb +++ b/cpp/msgpack/zone.hpp.erb @@ -18,7 +18,6 @@ #ifndef MSGPACK_ZONE_HPP__ #define MSGPACK_ZONE_HPP__ -#include "msgpack/object.hpp" #include "msgpack/zone.h" #include #include From 01b6673528127538457fdf445531f3574dc0c20a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 01:24:24 +0900 Subject: [PATCH 0427/1648] cpp: bool operator==(object& x, const T& y) --- cpp/msgpack/object.hpp | 11 +++++++++++ cpp/msgpack/type/string.hpp | 7 +++++++ cpp/object.cpp | 3 +++ 3 files changed, 21 insertions(+) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index a094c56..df1079b 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -128,6 +128,9 @@ private: bool operator==(const object x, const object y); bool operator!=(const object x, const object y); +template +bool operator==(const object x, const T& y); + std::ostream& operator<< (std::ostream& s, const object o); @@ -207,6 +210,14 @@ inline packer& operator<< (packer& o, const T& v) inline bool operator!=(const object x, const object y) { return !(x == y); } +template +inline bool operator==(const object x, const T& y) +try { + return x == object(y); +} catch (msgpack::type_error&) { + return false; +} + inline object::implicit_type object::convert() const { diff --git a/cpp/msgpack/type/string.hpp b/cpp/msgpack/type/string.hpp index 796603f..8ca2724 100644 --- a/cpp/msgpack/type/string.hpp +++ b/cpp/msgpack/type/string.hpp @@ -48,6 +48,13 @@ inline void operator<< (object::object_zone& o, const std::string& v) memcpy(ptr, v.data(), v.size()); } +inline void operator<< (object& o, const std::string& v) +{ + o.type = type::RAW; + o.via.raw.ptr = v.data(); + o.via.raw.size = v.size(); +} + } // namespace msgpack diff --git a/cpp/object.cpp b/cpp/object.cpp index a7adb71..626d186 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -103,6 +103,9 @@ bool operator==(const object x, const object y) case type::NEGATIVE_INTEGER: return x.via.i64 == y.via.i64; + case type::DOUBLE: + return x.via.dec == y.via.dec; + case type::RAW: return x.via.raw.size == y.via.raw.size && memcmp(x.via.raw.ptr, y.via.raw.ptr, x.via.raw.size) == 0; From 05e28752f111f6faa8b61432a7063a30b9f51ebf Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 01:57:05 +0900 Subject: [PATCH 0428/1648] cpp: MSGPACK_DEFINE defines T::msgpack_object(object*, zone*) --- cpp/msgpack/object.hpp | 22 ++++++++++++++++------ cpp/msgpack/type/bool.hpp | 3 +++ cpp/msgpack/type/define.hpp.erb | 18 ++++++++++++++++++ cpp/msgpack/type/deque.hpp | 2 +- cpp/msgpack/type/float.hpp | 6 ++++++ cpp/msgpack/type/int.hpp | 32 ++++++++++++++++++++++++++++++++ cpp/msgpack/type/list.hpp | 2 +- cpp/msgpack/type/nil.hpp | 3 +++ cpp/msgpack/type/pair.hpp | 2 +- cpp/msgpack/type/raw.hpp | 3 +++ cpp/msgpack/type/set.hpp | 4 ++-- cpp/msgpack/type/string.hpp | 2 +- cpp/msgpack/type/tuple.hpp.erb | 4 ++-- cpp/msgpack/type/vector.hpp | 2 +- 14 files changed, 90 insertions(+), 15 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index df1079b..bfbaeb1 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -106,7 +106,7 @@ private: public: // FIXME private? - struct object_zone; + struct with_zone; public: implicit_type convert() const; @@ -117,11 +117,11 @@ struct object_kv { object val; }; -struct object::object_zone : object { - object_zone(msgpack::zone* zone) : zone(zone) { } +struct object::with_zone : object { + with_zone(msgpack::zone* zone) : zone(zone) { } msgpack::zone* zone; private: - object_zone(); + with_zone(); }; @@ -142,6 +142,10 @@ packer& operator<< (packer& o, const T& v); template T& operator>> (object o, T& v); +// deconvert operator +template +void operator<< (object::with_zone& o, const T& v); + struct object::implicit_type { implicit_type(object o) : obj(o) { } @@ -206,6 +210,12 @@ inline packer& operator<< (packer& o, const T& v) return o; } +template +void operator<< (object::with_zone& o, const T& v) +{ + v.msgpack_object(static_cast(&o), o.zone); +} + inline bool operator!=(const object x, const object y) { return !(x == y); } @@ -253,14 +263,14 @@ inline object::object(const T& v) template inline object& object::operator=(const T& v) { - *this << v; + *this = object(v); return *this; } template object::object(const T& v, zone* z) { - object_zone oz(z); + with_zone oz(z); oz << v; type = oz.type; via = oz.via; diff --git a/cpp/msgpack/type/bool.hpp b/cpp/msgpack/type/bool.hpp index 9d63435..9433a98 100644 --- a/cpp/msgpack/type/bool.hpp +++ b/cpp/msgpack/type/bool.hpp @@ -45,6 +45,9 @@ inline void operator<< (object& o, bool v) o.via.boolean = v; } +inline void operator<< (object::with_zone& o, bool v) + { static_cast(o) << v; } + } // namespace msgpack diff --git a/cpp/msgpack/type/define.hpp.erb b/cpp/msgpack/type/define.hpp.erb index 2eac4f4..369e12f 100644 --- a/cpp/msgpack/type/define.hpp.erb +++ b/cpp/msgpack/type/define.hpp.erb @@ -27,6 +27,10 @@ void msgpack_unpack(msgpack::object o) \ { \ msgpack::type::make_define(__VA_ARGS__).msgpack_unpack(o); \ + }\ + void msgpack_object(msgpack::object* o, msgpack::zone* z) const \ + { \ + msgpack::type::make_define(__VA_ARGS__).msgpack_object(o, z); \ } namespace msgpack { @@ -51,6 +55,12 @@ struct define<> { { if(o.type != type::ARRAY) { throw type_error(); } } + void msgpack_object(msgpack::object* o, msgpack::zone* z) const + { + o->type = type::ARRAY; + o->via.array.ptr = NULL; + o->via.array.size = 0; + } }; <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> @@ -73,6 +83,14 @@ struct define, A<%=j%><%}%>> { <%0.upto(i) {|j|%> if(size <= <%=j%>) { return; } o.via.array.ptr[<%=j%>].convert(&a<%=j%>);<%}%> } + void msgpack_object(msgpack::object* o, msgpack::zone* z) const + { + o->type = type::ARRAY; + o->via.array.ptr = (object*)z->malloc(sizeof(object)*<%=i+1%>); + o->via.array.size = <%=i+1%>; + <%0.upto(i) {|j|%> + o->via.array.ptr[<%=j%>] = object(a<%=j%>, z);<%}%> + } <%0.upto(i) {|j|%> A<%=j%>& a<%=j%>;<%}%> }; diff --git a/cpp/msgpack/type/deque.hpp b/cpp/msgpack/type/deque.hpp index 5e919a8..6ae00a8 100644 --- a/cpp/msgpack/type/deque.hpp +++ b/cpp/msgpack/type/deque.hpp @@ -50,7 +50,7 @@ inline packer& operator<< (packer& o, const std::deque& v) } template -inline void operator<< (object::object_zone& o, const std::deque& v) +inline void operator<< (object::with_zone& o, const std::deque& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); diff --git a/cpp/msgpack/type/float.hpp b/cpp/msgpack/type/float.hpp index 28131d1..a60ef0b 100644 --- a/cpp/msgpack/type/float.hpp +++ b/cpp/msgpack/type/float.hpp @@ -69,6 +69,12 @@ inline void operator<< (object& o, double v) o.via.dec = v; } +inline void operator<< (object::with_zone& o, float v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, double v) + { static_cast(o) << v; } + } // namespace msgpack diff --git a/cpp/msgpack/type/int.hpp b/cpp/msgpack/type/int.hpp index 2dab78b..e2d1820 100644 --- a/cpp/msgpack/type/int.hpp +++ b/cpp/msgpack/type/int.hpp @@ -173,6 +173,38 @@ inline void operator<< (object& o, unsigned long long v) { o.type = type::POSITIVE_INTEGER, o.via.u64 = v; } +inline void operator<< (object::with_zone& o, signed char v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, signed short v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, signed int v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, signed long v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, signed long long v) + { static_cast(o) << v; } + + +inline void operator<< (object::with_zone& o, unsigned char v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, unsigned short v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, unsigned int v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, unsigned long v) + { static_cast(o) << v; } + +inline void operator<< (object::with_zone& o, unsigned long long v) + { static_cast(o) << v; } + + } // namespace msgpack #endif /* msgpack/type/int.hpp */ diff --git a/cpp/msgpack/type/list.hpp b/cpp/msgpack/type/list.hpp index 5318639..6807345 100644 --- a/cpp/msgpack/type/list.hpp +++ b/cpp/msgpack/type/list.hpp @@ -50,7 +50,7 @@ inline packer& operator<< (packer& o, const std::list& v) } template -inline void operator<< (object::object_zone& o, const std::list& v) +inline void operator<< (object::with_zone& o, const std::list& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); diff --git a/cpp/msgpack/type/nil.hpp b/cpp/msgpack/type/nil.hpp index ed58384..e58bc9d 100644 --- a/cpp/msgpack/type/nil.hpp +++ b/cpp/msgpack/type/nil.hpp @@ -47,6 +47,9 @@ inline void operator<< (object& o, type::nil v) o.type = type::NIL; } +inline void operator<< (object::with_zone& o, type::nil v) + { static_cast(o) << v; } + } // namespace msgpack diff --git a/cpp/msgpack/type/pair.hpp b/cpp/msgpack/type/pair.hpp index 6c68288..296a8b6 100644 --- a/cpp/msgpack/type/pair.hpp +++ b/cpp/msgpack/type/pair.hpp @@ -44,7 +44,7 @@ inline packer& operator<< (packer& o, const std::pair& v } template -inline void operator<< (object::object_zone& o, const std::pair& v) +inline void operator<< (object::with_zone& o, const std::pair& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*2); diff --git a/cpp/msgpack/type/raw.hpp b/cpp/msgpack/type/raw.hpp index d854d28..21d9a0d 100644 --- a/cpp/msgpack/type/raw.hpp +++ b/cpp/msgpack/type/raw.hpp @@ -84,6 +84,9 @@ inline void operator<< (object& o, const type::raw_ref& v) o.via.raw.size = v.size; } +inline void operator<< (object::with_zone& o, const type::raw_ref& v) + { static_cast(o) << v; } + } // namespace msgpack diff --git a/cpp/msgpack/type/set.hpp b/cpp/msgpack/type/set.hpp index 6f5fb18..2a6e7a4 100644 --- a/cpp/msgpack/type/set.hpp +++ b/cpp/msgpack/type/set.hpp @@ -49,7 +49,7 @@ inline packer& operator<< (packer& o, const std::set& v) } template -inline void operator<< (object::object_zone& o, const std::set& v) +inline void operator<< (object::with_zone& o, const std::set& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); @@ -90,7 +90,7 @@ inline packer& operator<< (packer& o, const std::multiset& v) } template -inline void operator<< (object::object_zone& o, const std::multiset& v) +inline void operator<< (object::with_zone& o, const std::multiset& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); diff --git a/cpp/msgpack/type/string.hpp b/cpp/msgpack/type/string.hpp index 8ca2724..f11a5e6 100644 --- a/cpp/msgpack/type/string.hpp +++ b/cpp/msgpack/type/string.hpp @@ -39,7 +39,7 @@ inline packer& operator<< (packer& o, const std::string& v) return o; } -inline void operator<< (object::object_zone& o, const std::string& v) +inline void operator<< (object::with_zone& o, const std::string& v) { o.type = type::RAW; char* ptr = (char*)o.zone->malloc(v.size()); diff --git a/cpp/msgpack/type/tuple.hpp.erb b/cpp/msgpack/type/tuple.hpp.erb index e57fe98..1b0c172 100644 --- a/cpp/msgpack/type/tuple.hpp.erb +++ b/cpp/msgpack/type/tuple.hpp.erb @@ -166,7 +166,7 @@ const packer& operator<< ( <%}%> inline void operator<< ( - object::object_zone& o, + object::with_zone& o, const type::tuple<>& v) { o.type = type::ARRAY; o.via.array.ptr = NULL; @@ -175,7 +175,7 @@ inline void operator<< ( <%0.upto(GENERATION_LIMIT) {|i|%> template , typename A<%=j%><%}%>> inline void operator<< ( - object::object_zone& o, + object::with_zone& o, const type::tuple, A<%=j%><%}%>>& v) { o.type = type::ARRAY; o.via.array.ptr = (object*)o.zone->malloc(sizeof(object)*<%=i+1%>); diff --git a/cpp/msgpack/type/vector.hpp b/cpp/msgpack/type/vector.hpp index f9e709b..382f501 100644 --- a/cpp/msgpack/type/vector.hpp +++ b/cpp/msgpack/type/vector.hpp @@ -54,7 +54,7 @@ inline packer& operator<< (packer& o, const std::vector& v) } template -inline void operator<< (object::object_zone& o, const std::vector& v) +inline void operator<< (object::with_zone& o, const std::vector& v) { o.type = type::ARRAY; object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); From 7d945d3c8e24de21ba5862d699521502bce17fe5 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 02:30:53 +0900 Subject: [PATCH 0429/1648] cpp: explicit object(const T& v) --- cpp/msgpack/object.hpp | 37 +++++++++++++++++++++++++++++---- cpp/msgpack/type/define.hpp.erb | 3 ++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index bfbaeb1..3567ac8 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -90,8 +90,10 @@ struct object { object(); + object(msgpack_object o); + template - object(const T& v); + explicit object(const T& v); template object(const T& v, zone* z); @@ -131,6 +133,15 @@ bool operator!=(const object x, const object y); template bool operator==(const object x, const T& y); +template +bool operator==(const T& y, const object x); + +template +bool operator!=(const object x, const T& y); + +template +bool operator!=(const T& y, const object x); + std::ostream& operator<< (std::ostream& s, const object o); @@ -217,9 +228,6 @@ void operator<< (object::with_zone& o, const T& v) } -inline bool operator!=(const object x, const object y) -{ return !(x == y); } - template inline bool operator==(const object x, const T& y) try { @@ -228,6 +236,21 @@ try { return false; } +inline bool operator!=(const object x, const object y) +{ return !(x == y); } + +template +inline bool operator==(const T& y, const object x) +{ return x == y; } + +template +inline bool operator!=(const object x, const T& y) +{ return !(x == y); } + +template +inline bool operator!=(const T& y, const object x) +{ return x != y; } + inline object::implicit_type object::convert() const { @@ -277,6 +300,12 @@ object::object(const T& v, zone* z) } +inline object::object(msgpack_object o) +{ + // FIXME beter way? + ::memcpy(this, &o, sizeof(o)); +} + inline object::operator msgpack_object() { // FIXME beter way? diff --git a/cpp/msgpack/type/define.hpp.erb b/cpp/msgpack/type/define.hpp.erb index 369e12f..9db6f08 100644 --- a/cpp/msgpack/type/define.hpp.erb +++ b/cpp/msgpack/type/define.hpp.erb @@ -28,7 +28,8 @@ { \ msgpack::type::make_define(__VA_ARGS__).msgpack_unpack(o); \ }\ - void msgpack_object(msgpack::object* o, msgpack::zone* z) const \ + template \ + void msgpack_object(MSGPACK_OBJECT* o, msgpack::zone* z) const \ { \ msgpack::type::make_define(__VA_ARGS__).msgpack_object(o, z); \ } From 9bfa2354ff9e667436fc4e6ced2274a5c0e55aab Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 02:37:04 +0900 Subject: [PATCH 0430/1648] cpp: fixes serialization of object::type == DOUBLE --- cpp/msgpack/object.hpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 3567ac8..37c1926 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -306,6 +306,12 @@ inline object::object(msgpack_object o) ::memcpy(this, &o, sizeof(o)); } +inline void operator<< (object& o, msgpack_object v) +{ + // FIXME beter way? + ::memcpy(&o, &v, sizeof(v)); +} + inline object::operator msgpack_object() { // FIXME beter way? @@ -314,12 +320,6 @@ inline object::operator msgpack_object() return obj; } -inline void operator<< (object& o, msgpack_object v) -{ - // FIXME beter way? - ::memcpy(&o, &v, sizeof(v)); -} - // obsolete template @@ -391,6 +391,10 @@ packer& operator<< (packer& o, const object& v) } return o; + case type::DOUBLE: + o.pack_double(v.via.dec); + return o; + case type::RAW: o.pack_raw(v.via.raw.size); o.pack_raw_body(v.via.raw.ptr, v.via.raw.size); From 9df6916029e2eba4ca0f93578f631d34208e6415 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 06:39:12 +0900 Subject: [PATCH 0431/1648] cpp: object::object(const T& v, zone* z) 2 --- cpp/msgpack/object.hpp | 6 +-- cpp/msgpack/type/deque.hpp | 25 ++++++---- cpp/msgpack/type/list.hpp | 25 ++++++---- cpp/msgpack/type/map.hpp | 66 ++++++++++++++++++++++++++ cpp/msgpack/type/set.hpp | 50 +++++++++++-------- cpp/msgpack/type/tr1/unordered_map.hpp | 44 +++++++++++++++++ cpp/msgpack/type/tr1/unordered_set.hpp | 42 ++++++++++++++++ cpp/msgpack/type/vector.hpp | 25 ++++++---- 8 files changed, 229 insertions(+), 54 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 37c1926..a72913a 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -103,13 +103,11 @@ struct object { operator msgpack_object(); + struct with_zone; + private: struct implicit_type; -public: - // FIXME private? - struct with_zone; - public: implicit_type convert() const; }; diff --git a/cpp/msgpack/type/deque.hpp b/cpp/msgpack/type/deque.hpp index 6ae00a8..d21ceea 100644 --- a/cpp/msgpack/type/deque.hpp +++ b/cpp/msgpack/type/deque.hpp @@ -53,16 +53,21 @@ template inline void operator<< (object::with_zone& o, const std::deque& v) { o.type = type::ARRAY; - object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); - object* const pend = p + v.size(); - o.via.array.ptr = p; - o.via.array.size = v.size(); - typename std::deque::const_iterator it(v.begin()); - do { - *p = object(*it, o.zone); - ++p; - ++it; - } while(p < pend); + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::deque::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } } diff --git a/cpp/msgpack/type/list.hpp b/cpp/msgpack/type/list.hpp index 6807345..c0f8ce6 100644 --- a/cpp/msgpack/type/list.hpp +++ b/cpp/msgpack/type/list.hpp @@ -53,16 +53,21 @@ template inline void operator<< (object::with_zone& o, const std::list& v) { o.type = type::ARRAY; - object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); - object* const pend = p + v.size(); - o.via.array.ptr = p; - o.via.array.size = v.size(); - typename std::list::const_iterator it(v.begin()); - do { - *p = object(*it, o.zone); - ++p; - ++it; - } while(p < pend); + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::list::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } } diff --git a/cpp/msgpack/type/map.hpp b/cpp/msgpack/type/map.hpp index 552de57..958447d 100644 --- a/cpp/msgpack/type/map.hpp +++ b/cpp/msgpack/type/map.hpp @@ -70,6 +70,28 @@ inline packer& operator<< (packer& o, const type::assoc_vector +inline void operator<< (object::with_zone& o, const type::assoc_vector& v) +{ + o.type = type::MAP; + if(v.empty()) { + o.via.map.ptr = NULL; + o.via.map.size = 0; + } else { + object_kv* p = (object_kv*)o.zone->malloc(sizeof(object_kv)*v.size()); + object_kv* const pend = p + v.size(); + o.via.map.ptr = p; + o.via.map.size = v.size(); + typename type::assoc_vector::const_iterator it(v.begin()); + do { + p->key = object(it->first, o.zone); + p->val = object(it->second, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + template inline std::map operator>> (object o, std::map& v) @@ -104,6 +126,28 @@ inline packer& operator<< (packer& o, const std::map& v) return o; } +template +inline void operator<< (object::with_zone& o, const std::map& v) +{ + o.type = type::MAP; + if(v.empty()) { + o.via.map.ptr = NULL; + o.via.map.size = 0; + } else { + object_kv* p = (object_kv*)o.zone->malloc(sizeof(object_kv)*v.size()); + object_kv* const pend = p + v.size(); + o.via.map.ptr = p; + o.via.map.size = v.size(); + typename std::map::const_iterator it(v.begin()); + do { + p->key = object(it->first, o.zone); + p->val = object(it->second, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + template inline std::multimap operator>> (object o, std::multimap& v) @@ -132,6 +176,28 @@ inline packer& operator<< (packer& o, const std::multimap& return o; } +template +inline void operator<< (object::with_zone& o, const std::multimap& v) +{ + o.type = type::MAP; + if(v.empty()) { + o.via.map.ptr = NULL; + o.via.map.size = 0; + } else { + object_kv* p = (object_kv*)o.zone->malloc(sizeof(object_kv)*v.size()); + object_kv* const pend = p + v.size(); + o.via.map.ptr = p; + o.via.map.size = v.size(); + typename std::multimap::const_iterator it(v.begin()); + do { + p->key = object(it->first, o.zone); + p->val = object(it->second, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + } // namespace msgpack diff --git a/cpp/msgpack/type/set.hpp b/cpp/msgpack/type/set.hpp index 2a6e7a4..bcf1030 100644 --- a/cpp/msgpack/type/set.hpp +++ b/cpp/msgpack/type/set.hpp @@ -52,16 +52,21 @@ template inline void operator<< (object::with_zone& o, const std::set& v) { o.type = type::ARRAY; - object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); - object* const pend = p + v.size(); - o.via.array.ptr = p; - o.via.array.size = v.size(); - typename std::set::const_iterator it(v.begin()); - do { - *p = object(*it, o.zone); - ++p; - ++it; - } while(p < pend); + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::set::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } } @@ -93,16 +98,21 @@ template inline void operator<< (object::with_zone& o, const std::multiset& v) { o.type = type::ARRAY; - object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); - object* const pend = p + v.size(); - o.via.array.ptr = p; - o.via.array.size = v.size(); - typename std::multiset::const_iterator it(v.begin()); - do { - *p = object(*it, o.zone); - ++p; - ++it; - } while(p < pend); + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::multiset::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } } diff --git a/cpp/msgpack/type/tr1/unordered_map.hpp b/cpp/msgpack/type/tr1/unordered_map.hpp index 1996cfd..4b29f0c 100644 --- a/cpp/msgpack/type/tr1/unordered_map.hpp +++ b/cpp/msgpack/type/tr1/unordered_map.hpp @@ -50,6 +50,28 @@ inline packer& operator<< (packer& o, const std::tr1::unordered_ return o; } +template +inline void operator<< (object::with_zone& o, const std::tr1::unordered_map& v) +{ + o.type = type::MAP; + if(v.empty()) { + o.via.map.ptr = NULL; + o.via.map.size = 0; + } else { + object_kv* p = (object_kv*)o.zone->malloc(sizeof(object_kv)*v.size()); + object_kv* const pend = p + v.size(); + o.via.map.ptr = p; + o.via.map.size = v.size(); + typename std::tr1::unordered_map::const_iterator it(v.begin()); + do { + p->key = object(it->first, o.zone); + p->val = object(it->second, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + template inline std::tr1::unordered_multimap operator>> (object o, std::tr1::unordered_multimap& v) @@ -78,6 +100,28 @@ inline packer& operator<< (packer& o, const std::tr1::unordered_ return o; } +template +inline void operator<< (object::with_zone& o, const std::tr1::unordered_multimap& v) +{ + o.type = type::MAP; + if(v.empty()) { + o.via.map.ptr = NULL; + o.via.map.size = 0; + } else { + object_kv* p = (object_kv*)o.zone->malloc(sizeof(object_kv)*v.size()); + object_kv* const pend = p + v.size(); + o.via.map.ptr = p; + o.via.map.size = v.size(); + typename std::tr1::unordered_multimap::const_iterator it(v.begin()); + do { + p->key = object(it->first, o.zone); + p->val = object(it->second, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + } // namespace msgpack diff --git a/cpp/msgpack/type/tr1/unordered_set.hpp b/cpp/msgpack/type/tr1/unordered_set.hpp index eb127b5..4af6801 100644 --- a/cpp/msgpack/type/tr1/unordered_set.hpp +++ b/cpp/msgpack/type/tr1/unordered_set.hpp @@ -48,6 +48,27 @@ inline packer& operator<< (packer& o, const std::tr1::unordered_ return o; } +template +inline void operator<< (object::with_zone& o, const std::tr1::unordered_set& v) +{ + o.type = type::ARRAY; + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::tr1::unordered_set::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + template inline std::tr1::unordered_multiset& operator>> (object o, std::tr1::unordered_multiset& v) @@ -73,6 +94,27 @@ inline packer& operator<< (packer& o, const std::tr1::unordered_ return o; } +template +inline void operator<< (object::with_zone& o, const std::tr1::unordered_multiset& v) +{ + o.type = type::ARRAY; + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::tr1::unordered_multiset::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } +} + } // namespace msgpack diff --git a/cpp/msgpack/type/vector.hpp b/cpp/msgpack/type/vector.hpp index 382f501..bd073ef 100644 --- a/cpp/msgpack/type/vector.hpp +++ b/cpp/msgpack/type/vector.hpp @@ -57,16 +57,21 @@ template inline void operator<< (object::with_zone& o, const std::vector& v) { o.type = type::ARRAY; - object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); - object* const pend = p + v.size(); - o.via.array.ptr = p; - o.via.array.size = v.size(); - typename std::vector::const_iterator it(v.begin()); - do { - *p = object(*it, o.zone); - ++p; - ++it; - } while(p < pend); + if(v.empty()) { + o.via.array.ptr = NULL; + o.via.array.size = 0; + } else { + object* p = (object*)o.zone->malloc(sizeof(object)*v.size()); + object* const pend = p + v.size(); + o.via.array.ptr = p; + o.via.array.size = v.size(); + typename std::vector::const_iterator it(v.begin()); + do { + *p = object(*it, o.zone); + ++p; + ++it; + } while(p < pend); + } } From 72160aac9a1536234700539eacc9bdc7bf6f9f61 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 07:56:19 +0900 Subject: [PATCH 0432/1648] cpp: combines libmsgpackc and libmsgpack into libmsgpack --- cpp/Makefile.am | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index a297ba7..f1a1d6c 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -1,22 +1,15 @@ -lib_LTLIBRARIES = libmsgpackc.la libmsgpack.la - -libmsgpackc_la_SOURCES = \ - unpack.c \ - object.c \ - vrefbuffer.c \ - zone.c - -# -version-info CURRENT:REVISION:AGE -libmsgpackc_la_LDFLAGS = -version-info 2:0:0 +lib_LTLIBRARIES = libmsgpack.la libmsgpack_la_SOURCES = \ + unpack.c \ + objectc.c \ + vrefbuffer.c \ + zone.c \ object.cpp -libmsgpack_la_LIBADD = -lmsgpackc - # -version-info CURRENT:REVISION:AGE -libmsgpack_la_LDFLAGS = -version-info 2:0:0 +libmsgpack_la_LDFLAGS = -version-info 3:0:0 nobase_include_HEADERS = \ @@ -60,13 +53,6 @@ nobase_include_HEADERS = \ msgpack/type/tr1/unordered_set.hpp -# work around for duplicated object file name -libmsgpackc_la_CFLAGS = $(AM_CFLAGS) -libmsgpackc_la_CXXFLAGS = $(AM_CXXFLAGS) -libmsgpack_la_CFLAGS = $(AM_CFLAGS) -libmsgpack_la_CXXFLAGS = $(AM_CXXFLAGS) - - EXTRA_DIST = \ README.md \ LICENSE \ From 35802ba94948463db19308c8ec385243c51d1485 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 08:00:04 +0900 Subject: [PATCH 0433/1648] cpp: msgpack_object_equal --- cpp/Makefile.am | 2 +- cpp/msgpack/object.h | 2 ++ cpp/msgpack/object.hpp | 9 +++-- cpp/object.cpp | 54 ---------------------------- cpp/{object.c => objectc.c} | 70 +++++++++++++++++++++++++++++++++++-- 5 files changed, 78 insertions(+), 59 deletions(-) rename cpp/{object.c => objectc.c} (71%) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index f1a1d6c..fa8d6cf 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -66,7 +66,7 @@ check_PROGRAMS = \ msgpack_test msgpackc_test_SOURCES = msgpackc_test.cpp -msgpackc_test_LDADD = libmsgpackc.la -lgtest_main +msgpackc_test_LDADD = libmsgpack.la -lgtest_main msgpack_test_SOURCES = msgpack_test.cpp msgpack_test_LDADD = libmsgpack.la -lgtest_main diff --git a/cpp/msgpack/object.h b/cpp/msgpack/object.h index 9a014be..bbfac19 100644 --- a/cpp/msgpack/object.h +++ b/cpp/msgpack/object.h @@ -79,6 +79,8 @@ typedef struct msgpack_object_kv { void msgpack_object_print(FILE* out, msgpack_object o); +bool msgpack_object_equal(const msgpack_object x, const msgpack_object y); + #ifdef __cplusplus } diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index a72913a..1362818 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -101,7 +101,7 @@ struct object { template object& operator=(const T& v); - operator msgpack_object(); + operator msgpack_object() const; struct with_zone; @@ -226,6 +226,11 @@ void operator<< (object::with_zone& o, const T& v) } +inline bool operator==(const object x, const object y) +{ + return msgpack_object_equal(x, y); +} + template inline bool operator==(const object x, const T& y) try { @@ -310,7 +315,7 @@ inline void operator<< (object& o, msgpack_object v) ::memcpy(&o, &v, sizeof(v)); } -inline object::operator msgpack_object() +inline object::operator msgpack_object() const { // FIXME beter way? msgpack_object obj; diff --git a/cpp/object.cpp b/cpp/object.cpp index 626d186..dfe32bb 100644 --- a/cpp/object.cpp +++ b/cpp/object.cpp @@ -16,7 +16,6 @@ // limitations under the License. // #include "msgpack/object.hpp" -#include namespace msgpack { @@ -61,7 +60,6 @@ std::ostream& operator<< (std::ostream& s, const object o) } s << "]"; break; - // FIXME loop optimiziation case type::MAP: s << "{"; @@ -76,7 +74,6 @@ std::ostream& operator<< (std::ostream& s, const object o) } s << "}"; break; - // FIXME loop optimiziation default: // FIXME @@ -86,56 +83,5 @@ std::ostream& operator<< (std::ostream& s, const object o) } -bool operator==(const object x, const object y) -{ - if(x.type != y.type) { return false; } - - switch(x.type) { - case type::NIL: - return true; - - case type::BOOLEAN: - return x.via.boolean == y.via.boolean; - - case type::POSITIVE_INTEGER: - return x.via.u64 == y.via.u64; - - case type::NEGATIVE_INTEGER: - return x.via.i64 == y.via.i64; - - case type::DOUBLE: - return x.via.dec == y.via.dec; - - case type::RAW: - return x.via.raw.size == y.via.raw.size && - memcmp(x.via.raw.ptr, y.via.raw.ptr, x.via.raw.size) == 0; - - case type::ARRAY: - if(x.via.array.size != y.via.array.size) { return false; } - for(object* px(x.via.array.ptr), - * const pxend(x.via.array.ptr + x.via.array.size), - * py(y.via.array.ptr); - px < pxend; ++px, ++py) { - if(*px != *py) { return false; } - } - return true; - // FIXME loop optimiziation - - case type::MAP: - if(x.via.map.size != y.via.map.size) { return false; } - for(object_kv* px(x.via.map.ptr), - * const pxend(x.via.map.ptr + x.via.map.size), - * py(y.via.map.ptr); - px < pxend; ++px, ++py) { - if(px->key != py->key || px->val != py->val) { return false; } - } - return true; - - default: - return false; - } -} - - } // namespace msgpack diff --git a/cpp/object.c b/cpp/objectc.c similarity index 71% rename from cpp/object.c rename to cpp/objectc.c index a22ce21..d4f1c8a 100644 --- a/cpp/object.c +++ b/cpp/objectc.c @@ -18,6 +18,7 @@ #include "msgpack/object.h" #include "msgpack/pack.h" #include +#include #ifndef _MSC_VER #include @@ -141,7 +142,6 @@ void msgpack_object_print(FILE* out, msgpack_object o) } fprintf(out, "]"); break; - // FIXME loop optimiziation case MSGPACK_OBJECT_MAP: fprintf(out, "{"); @@ -161,7 +161,6 @@ void msgpack_object_print(FILE* out, msgpack_object o) } fprintf(out, "}"); break; - // FIXME loop optimiziation default: // FIXME @@ -169,3 +168,70 @@ void msgpack_object_print(FILE* out, msgpack_object o) } } +bool msgpack_object_equal(const msgpack_object x, const msgpack_object y) +{ + if(x.type != y.type) { return false; } + + switch(x.type) { + case MSGPACK_OBJECT_NIL: + return true; + + case MSGPACK_OBJECT_BOOLEAN: + return x.via.boolean == y.via.boolean; + + case MSGPACK_OBJECT_POSITIVE_INTEGER: + return x.via.u64 == y.via.u64; + + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + return x.via.i64 == y.via.i64; + + case MSGPACK_OBJECT_DOUBLE: + return x.via.dec == y.via.dec; + + case MSGPACK_OBJECT_RAW: + return x.via.raw.size == y.via.raw.size && + memcmp(x.via.raw.ptr, y.via.raw.ptr, x.via.raw.size) == 0; + + case MSGPACK_OBJECT_ARRAY: + if(x.via.array.size != y.via.array.size) { + return false; + } else if(x.via.array.size == 0) { + return true; + } else { + msgpack_object* px = x.via.array.ptr; + msgpack_object* const pxend = x.via.array.ptr + x.via.array.size; + msgpack_object* py = y.via.array.ptr; + do { + if(!msgpack_object_equal(*px, *py)) { + return false; + } + ++px; + ++py; + } while(px < pxend); + return true; + } + + case MSGPACK_OBJECT_MAP: + if(x.via.map.size != y.via.map.size) { + return false; + } else if(x.via.map.size == 0) { + return true; + } else { + msgpack_object_kv* px = x.via.map.ptr; + msgpack_object_kv* const pxend = x.via.map.ptr + x.via.map.size; + msgpack_object_kv* py = y.via.map.ptr; + do { + if(!msgpack_object_equal(px->key, py->key) || !msgpack_object_equal(px->val, py->val)) { + return false; + } + ++px; + ++py; + } while(px < pxend); + return true; + } + + default: + return false; + } +} + From 0a5c2e7ab9ade5b9789782f10cc51eff9cacc0a9 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 08:16:12 +0900 Subject: [PATCH 0434/1648] c,cpp: MSGPACK_OBJECT_NIL = 0x00 --- cpp/msgpack/object.h | 16 ++++++++-------- cpp/msgpack/object.hpp | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cpp/msgpack/object.h b/cpp/msgpack/object.h index bbfac19..71a27bb 100644 --- a/cpp/msgpack/object.h +++ b/cpp/msgpack/object.h @@ -27,14 +27,14 @@ extern "C" { typedef enum { - MSGPACK_OBJECT_NIL = 0x01, - MSGPACK_OBJECT_BOOLEAN = 0x02, - MSGPACK_OBJECT_POSITIVE_INTEGER = 0x03, - MSGPACK_OBJECT_NEGATIVE_INTEGER = 0x04, - MSGPACK_OBJECT_DOUBLE = 0x05, - MSGPACK_OBJECT_RAW = 0x06, - MSGPACK_OBJECT_ARRAY = 0x07, - MSGPACK_OBJECT_MAP = 0x08, + MSGPACK_OBJECT_NIL = 0x00, + MSGPACK_OBJECT_BOOLEAN = 0x01, + MSGPACK_OBJECT_POSITIVE_INTEGER = 0x02, + MSGPACK_OBJECT_NEGATIVE_INTEGER = 0x03, + MSGPACK_OBJECT_DOUBLE = 0x04, + MSGPACK_OBJECT_RAW = 0x05, + MSGPACK_OBJECT_ARRAY = 0x06, + MSGPACK_OBJECT_MAP = 0x07, } msgpack_object_type; diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 1362818..4df4437 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -35,14 +35,14 @@ class type_error : public std::bad_cast { }; namespace type { enum object_type { - NIL = 0x01, - BOOLEAN = 0x02, - POSITIVE_INTEGER = 0x03, - NEGATIVE_INTEGER = 0x04, - DOUBLE = 0x05, - RAW = 0x06, - ARRAY = 0x07, - MAP = 0x08, + NIL = MSGPACK_OBJECT_NIL, + BOOLEAN = MSGPACK_OBJECT_BOOLEAN, + POSITIVE_INTEGER = MSGPACK_OBJECT_POSITIVE_INTEGER, + NEGATIVE_INTEGER = MSGPACK_OBJECT_NEGATIVE_INTEGER, + DOUBLE = MSGPACK_OBJECT_DOUBLE, + RAW = MSGPACK_OBJECT_RAW, + ARRAY = MSGPACK_OBJECT_ARRAY, + MAP = MSGPACK_OBJECT_MAP, }; } From 53d5ddb3455dceb98c97b18026ad436d180a8b19 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 08:26:42 +0900 Subject: [PATCH 0435/1648] cpp: fixes operator<<(packer&, const object&) --- cpp/msgpack/object.hpp | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/cpp/msgpack/object.hpp b/cpp/msgpack/object.hpp index 4df4437..f80a390 100644 --- a/cpp/msgpack/object.hpp +++ b/cpp/msgpack/object.hpp @@ -363,35 +363,11 @@ packer& operator<< (packer& o, const object& v) return o; case type::POSITIVE_INTEGER: - if(v.via.u64 <= (uint64_t)std::numeric_limits::max()) { - if(v.via.u64 <= (uint16_t)std::numeric_limits::max()) { - o.pack_uint8(v.via.u64); - } else { - o.pack_uint16(v.via.u64); - } - } else { - if(v.via.u64 <= (uint64_t)std::numeric_limits::max()) { - o.pack_uint32(v.via.u64); - } else { - o.pack_uint64(v.via.u64); - } - } + o.pack_uint64(v.via.u64); return o; case type::NEGATIVE_INTEGER: - if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { - if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { - o.pack_int8(v.via.i64); - } else { - o.pack_int16(v.via.i64); - } - } else { - if(v.via.i64 >= (int64_t)std::numeric_limits::min()) { - o.pack_int64(v.via.i64); - } else { - o.pack_int64(v.via.i64); - } - } + o.pack_int64(v.via.i64); return o; case type::DOUBLE: From d19bfaa2cb16eaabee79922f7d8b59ef9bccb592 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 25 Apr 2010 09:09:06 +0900 Subject: [PATCH 0436/1648] cpp: fixes msgpack_vc8.vcproj --- cpp/msgpack_vc8.vcproj | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cpp/msgpack_vc8.vcproj b/cpp/msgpack_vc8.vcproj index 2a0eebe..5804790 100644 --- a/cpp/msgpack_vc8.vcproj +++ b/cpp/msgpack_vc8.vcproj @@ -157,15 +157,13 @@ UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}" > @@ -174,16 +172,10 @@ > - - @@ -244,6 +236,10 @@ /> + + Date: Sun, 25 Apr 2010 18:08:14 +0900 Subject: [PATCH 0437/1648] cpp: add test/{zone,pack_unpack,streaming,object,convert,buffer}.cc --- cpp/Makefile.am | 2 + cpp/configure.in | 2 +- cpp/msgpack/unpack.hpp | 2 +- cpp/test/Makefile.am | 26 ++++++++ cpp/test/buffer.cc | 50 +++++++++++++++ cpp/test/convert.cc | 74 ++++++++++++++++++++++ cpp/test/object.cc | 134 ++++++++++++++++++++++++++++++++++++++++ cpp/test/pack_unpack.cc | 101 ++++++++++++++++++++++++++++++ cpp/test/streaming.cc | 109 ++++++++++++++++++++++++++++++++ cpp/test/zone.cc | 78 +++++++++++++++++++++++ 10 files changed, 576 insertions(+), 2 deletions(-) create mode 100644 cpp/test/Makefile.am create mode 100644 cpp/test/buffer.cc create mode 100644 cpp/test/convert.cc create mode 100644 cpp/test/object.cc create mode 100644 cpp/test/pack_unpack.cc create mode 100644 cpp/test/streaming.cc create mode 100644 cpp/test/zone.cc diff --git a/cpp/Makefile.am b/cpp/Makefile.am index fa8d6cf..3cbb400 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -61,6 +61,8 @@ EXTRA_DIST = \ msgpack_vc8.sln \ msgpack_vc8.postbuild.bat +SUBDIRS = test + check_PROGRAMS = \ msgpackc_test \ msgpack_test diff --git a/cpp/configure.in b/cpp/configure.in index 8c59dda..596b28d 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -39,5 +39,5 @@ add CFLAGS="--march=i686" and CXXFLAGS="-march=i686" options to ./configure as f ]) fi -AC_OUTPUT([Makefile]) +AC_OUTPUT([Makefile test/Makefile]) diff --git a/cpp/msgpack/unpack.hpp b/cpp/msgpack/unpack.hpp index 324111a..d39b5df 100644 --- a/cpp/msgpack/unpack.hpp +++ b/cpp/msgpack/unpack.hpp @@ -81,7 +81,7 @@ public: // while( /* readable */ ) { // // // 1. - // pac.reserve(1024); + // pac.reserve_buffer(1024); // // // 2. // ssize_t bytes = diff --git a/cpp/test/Makefile.am b/cpp/test/Makefile.am new file mode 100644 index 0000000..a80f319 --- /dev/null +++ b/cpp/test/Makefile.am @@ -0,0 +1,26 @@ + +AM_LDFLAGS = ../libmsgpack.la -lgtest_main + +check_PROGRAMS = \ + zone \ + pack_unpack \ + streaming \ + object \ + convert \ + buffer + +TESTS = $(check_PROGRAMS) + +zone_SOURCES = zone.cc + +pack_unpack_SOURCES = pack_unpack.cc + +streaming_SOURCES = streaming.cc + +object_SOURCES = object.cc + +convert_SOURCES = convert.cc + +buffer_SOURCES = buffer.cc +buffer_LDADD = -lz + diff --git a/cpp/test/buffer.cc b/cpp/test/buffer.cc new file mode 100644 index 0000000..a2e9037 --- /dev/null +++ b/cpp/test/buffer.cc @@ -0,0 +1,50 @@ +#include +#include +#include +#include + +TEST(buffer, sbuffer) +{ + msgpack::sbuffer sbuf; + sbuf.write("a", 1); + sbuf.write("a", 1); + sbuf.write("a", 1); + + EXPECT_EQ(3, sbuf.size()); + EXPECT_TRUE( memcmp(sbuf.data(), "aaa", 3) == 0 ); +} + + +TEST(buffer, vrefbuffer) +{ + msgpack::vrefbuffer vbuf; + vbuf.write("a", 1); + vbuf.write("a", 1); + vbuf.write("a", 1); + + const struct iovec* vec = vbuf.vector(); + size_t veclen = vbuf.vector_size(); + + msgpack::sbuffer sbuf; + for(size_t i=0; i < veclen; ++i) { + sbuf.write((const char*)vec[i].iov_base, vec[i].iov_len); + } + + EXPECT_EQ(3, sbuf.size()); + EXPECT_TRUE( memcmp(sbuf.data(), "aaa", 3) == 0 ); +} + + +TEST(buffer, zbuffer) +{ + msgpack::zbuffer zbuf; + zbuf.write("a", 1); + zbuf.write("a", 1); + zbuf.write("a", 1); + + zbuf.flush(); + + char* data = zbuf.data(); + size_t size = zbuf.size(); +} + diff --git a/cpp/test/convert.cc b/cpp/test/convert.cc new file mode 100644 index 0000000..f2a8523 --- /dev/null +++ b/cpp/test/convert.cc @@ -0,0 +1,74 @@ +#include +#include + +class compatibility { +public: + compatibility() : str1("default"), str2("default") { } + + std::string str1; + std::string str2; + + MSGPACK_DEFINE(str1, str2); +}; + +TEST(convert, compatibility_less) +{ + std::vector src(1); + src[0] = "kumofs"; + + msgpack::zone z; + msgpack::object obj(src, &z); + + compatibility c; + EXPECT_NO_THROW( obj.convert(&c) ); + + EXPECT_EQ("kumofs", c.str1); + EXPECT_EQ("default", c.str2); +} + +TEST(convert, compatibility_more) +{ + std::vector src(3); + src[0] = "kumofs"; + src[1] = "mpio"; + src[2] = "cloudy"; + + msgpack::zone z; + msgpack::object obj(src, &z); + + compatibility to; + EXPECT_NO_THROW( obj.convert(&to) ); + + EXPECT_EQ("kumofs", to.str1); + EXPECT_EQ("mpio", to.str2); +} + + +class enum_member { +public: + enum_member() : flag(A) { } + + enum flags_t { + A = 0, + B = 1, + }; + + flags_t flag; + + MSGPACK_DEFINE((int&)flag); +}; + +TEST(convert, enum_member) +{ + enum_member src; + src.flag = enum_member::B; + + msgpack::zone z; + msgpack::object obj(src, &z); + + enum_member to; + EXPECT_NO_THROW( obj.convert(&to) ); + + EXPECT_EQ(enum_member::B, to.flag); +} + diff --git a/cpp/test/object.cc b/cpp/test/object.cc new file mode 100644 index 0000000..5390c4a --- /dev/null +++ b/cpp/test/object.cc @@ -0,0 +1,134 @@ +#include +#include + +struct myclass { + myclass() : num(0), str("default") { } + + myclass(int num, const std::string& str) : + num(0), str("default") { } + + ~myclass() { } + + int num; + std::string str; + + MSGPACK_DEFINE(num, str); + + bool operator==(const myclass& o) const + { + return num == o.num && str == o.str; + } +}; + +std::ostream& operator<<(std::ostream& o, const myclass& m) +{ + return o << "myclass("<()); +} + + +TEST(object, print) +{ + msgpack::object obj; + std::cout << obj << std::endl; +} + + +TEST(object, is_nil) +{ + msgpack::object obj; + EXPECT_TRUE(obj.is_nil()); +} + + +TEST(object, type_error) +{ + msgpack::object obj(1); + EXPECT_THROW(obj.as(), msgpack::type_error); + EXPECT_THROW(obj.as >(), msgpack::type_error); + EXPECT_EQ(1, obj.as()); + EXPECT_EQ(1, obj.as()); + EXPECT_EQ(1u, obj.as()); + EXPECT_EQ(1u, obj.as()); +} + + +TEST(object, equal_primitive) +{ + msgpack::object obj_nil; + EXPECT_EQ(obj_nil, msgpack::object()); + + msgpack::object obj_int(1); + EXPECT_EQ(obj_int, msgpack::object(1)); + EXPECT_EQ(obj_int, 1); + + msgpack::object obj_double(1.2); + EXPECT_EQ(obj_double, msgpack::object(1.2)); + EXPECT_EQ(obj_double, 1.2); + + msgpack::object obj_bool(true); + EXPECT_EQ(obj_bool, msgpack::object(true)); + EXPECT_EQ(obj_bool, true); +} + + +TEST(object, construct_primitive) +{ + msgpack::object obj_nil; + EXPECT_EQ(msgpack::type::NIL, obj_nil.type); + + msgpack::object obj_uint(1); + EXPECT_EQ(msgpack::type::POSITIVE_INTEGER, obj_uint.type); + EXPECT_EQ(1u, obj_uint.via.u64); + + msgpack::object obj_int(-1); + EXPECT_EQ(msgpack::type::NEGATIVE_INTEGER, obj_int.type); + EXPECT_EQ(-1, obj_int.via.i64); + + msgpack::object obj_double(1.2); + EXPECT_EQ(msgpack::type::DOUBLE, obj_double.type); + EXPECT_EQ(1.2, obj_double.via.dec); + + msgpack::object obj_bool(true); + EXPECT_EQ(msgpack::type::BOOLEAN, obj_bool.type); + EXPECT_EQ(true, obj_bool.via.boolean); +} + diff --git a/cpp/test/pack_unpack.cc b/cpp/test/pack_unpack.cc new file mode 100644 index 0000000..ecf52c5 --- /dev/null +++ b/cpp/test/pack_unpack.cc @@ -0,0 +1,101 @@ +#include +#include +#include + +TEST(pack, num) +{ + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, 1); +} + + +TEST(pack, vector) +{ + msgpack::sbuffer sbuf; + std::vector vec; + vec.push_back(1); + vec.push_back(2); + vec.push_back(3); + msgpack::pack(sbuf, vec); +} + + +TEST(pack, to_ostream) +{ + std::ostringstream stream; + msgpack::pack(stream, 1); +} + + +struct myclass { + myclass() : num(0), str("default") { } + + myclass(int num, const std::string& str) : + num(0), str("default") { } + + ~myclass() { } + + int num; + std::string str; + + MSGPACK_DEFINE(num, str); +}; + + +TEST(pack, myclass) +{ + msgpack::sbuffer sbuf; + myclass m(1, "msgpack"); + msgpack::pack(sbuf, m); +} + + +TEST(unpack, myclass) +{ + msgpack::sbuffer sbuf; + myclass m1(1, "phraser"); + msgpack::pack(sbuf, m1); + + msgpack::zone z; + msgpack::object obj; + + msgpack::unpack_return ret = + msgpack::unpack(sbuf.data(), sbuf.size(), NULL, &z, &obj); + + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + + myclass m2 = obj.as(); + EXPECT_EQ(m1.num, m2.num); + EXPECT_EQ(m1.str, m2.str); +} + + +TEST(unpack, sequence) +{ + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, 1); + msgpack::pack(sbuf, 2); + msgpack::pack(sbuf, 3); + + size_t offset = 0; + + msgpack::zone z; + msgpack::object obj; + msgpack::unpack_return ret; + + ret = msgpack::unpack(sbuf.data(), sbuf.size(), &offset, &z, &obj); + EXPECT_TRUE(ret >= 0); + EXPECT_EQ(ret, msgpack::UNPACK_EXTRA_BYTES); + EXPECT_EQ(1, obj.as()); + + ret = msgpack::unpack(sbuf.data(), sbuf.size(), &offset, &z, &obj); + EXPECT_TRUE(ret >= 0); + EXPECT_EQ(ret, msgpack::UNPACK_EXTRA_BYTES); + EXPECT_EQ(2, obj.as()); + + ret = msgpack::unpack(sbuf.data(), sbuf.size(), &offset, &z, &obj); + EXPECT_TRUE(ret >= 0); + EXPECT_EQ(ret, msgpack::UNPACK_SUCCESS); + EXPECT_EQ(3, obj.as()); +} + diff --git a/cpp/test/streaming.cc b/cpp/test/streaming.cc new file mode 100644 index 0000000..2d03976 --- /dev/null +++ b/cpp/test/streaming.cc @@ -0,0 +1,109 @@ +#include +#include +#include + +TEST(streaming, basic) +{ + std::ostringstream stream; + msgpack::packer pk(&stream); + + pk.pack(1); + pk.pack(2); + pk.pack(3); + + std::istringstream input(stream.str()); + + msgpack::unpacker pac; + + int count = 0; + while(count < 3) { + pac.reserve_buffer(32*1024); + + size_t len = input.readsome(pac.buffer(), pac.buffer_capacity()); + pac.buffer_consumed(len); + + while(pac.execute()) { + std::auto_ptr z(pac.release_zone()); + msgpack::object obj = pac.data(); + pac.reset(); + + switch(count++) { + case 0: + EXPECT_EQ(1, obj.as()); + break; + case 1: + EXPECT_EQ(2, obj.as()); + break; + case 2: + EXPECT_EQ(3, obj.as()); + return; + } + + } + } +} + + +class event_handler { +public: + event_handler(std::istream& input) : input(input) { } + ~event_handler() { } + + void on_read() + { + while(true) { + pac.reserve_buffer(32*1024); + + size_t len = input.readsome(pac.buffer(), pac.buffer_capacity()); + + if(len == 0) { + return; + } + + pac.buffer_consumed(len); + + while(pac.execute()) { + std::auto_ptr z(pac.release_zone()); + msgpack::object obj = pac.data(); + pac.reset(); + on_message(obj, z); + } + + if(pac.message_size() > 10*1024*1024) { + throw std::runtime_error("message is too large"); + } + } + } + + void on_message(msgpack::object obj, std::auto_ptr z) + { + EXPECT_EQ(expect, obj.as()); + } + + int expect; + +private: + std::istream& input; + msgpack::unpacker pac; +}; + +TEST(streaming, event) +{ + std::stringstream stream; + msgpack::packer pk(&stream); + + event_handler handler(stream); + + pk.pack(1); + handler.expect = 1; + handler.on_read(); + + pk.pack(2); + handler.expect = 2; + handler.on_read(); + + pk.pack(3); + handler.expect = 3; + handler.on_read(); +} + diff --git a/cpp/test/zone.cc b/cpp/test/zone.cc new file mode 100644 index 0000000..5274e9f --- /dev/null +++ b/cpp/test/zone.cc @@ -0,0 +1,78 @@ +#include +#include + +TEST(zone, malloc) +{ + msgpack::zone z; + char* buf1 = (char*)z.malloc(4); + memcpy(buf1, "test", 4); + char* buf2 = (char*)z.malloc(4); + memcpy(buf2, "test", 4); +} + + +class myclass { +public: + myclass() : num(0), str("default") { } + + myclass(int num, const std::string& str) : + num(num), str(str) { } + + ~myclass() { } + + int num; + std::string str; + +private: + myclass(const myclass&); +}; + + +TEST(zone, allocate) +{ + msgpack::zone z; + myclass* m = z.allocate(); + EXPECT_EQ(m->num, 0); + EXPECT_EQ(m->str, "default"); +} + + +TEST(zone, allocate_constructor) +{ + msgpack::zone z; + myclass* m = z.allocate(7, "msgpack"); + EXPECT_EQ(m->num, 7); + EXPECT_EQ(m->str, "msgpack"); +} + + +static void custom_finalizer_func(void* user) +{ + myclass* m = (myclass*)user; + delete m; +} + +TEST(zone, push_finalizer) +{ + msgpack::zone z; + myclass* m = new myclass(); + z.push_finalizer(custom_finalizer_func, (void*)m); +} + + +TEST(zone, push_finalizer_auto_ptr) +{ + msgpack::zone z; + std::auto_ptr am(new myclass()); + z.push_finalizer(am); +} + + +TEST(zone, malloc_no_align) +{ + msgpack::zone z; + char* buf1 = (char*)z.malloc_no_align(4); + char* buf2 = (char*)z.malloc_no_align(4); + EXPECT_EQ(buf1+4, buf2); +} + From 68f60568acdafb2205c1436b61bfcaf0087b1572 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Mon, 26 Apr 2010 21:52:19 +0900 Subject: [PATCH 0438/1648] cpp: build libmsgpackc.so for backward compatibility. --- cpp/Makefile.am | 16 ++++++++++++++++ cpp/test/Makefile.am | 14 ++++++++------ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/cpp/Makefile.am b/cpp/Makefile.am index 3cbb400..08eb7a5 100644 --- a/cpp/Makefile.am +++ b/cpp/Makefile.am @@ -12,6 +12,22 @@ libmsgpack_la_SOURCES = \ libmsgpack_la_LDFLAGS = -version-info 3:0:0 +# backward compatibility +lib_LTLIBRARIES += libmsgpackc.la + +libmsgpackc_la_SOURCES = \ + unpack.c \ + objectc.c \ + vrefbuffer.c \ + zone.c + +libmsgpackc_la_LDFLAGS = -version-info 2:0:0 + +# work around for duplicated file name +kumo_manager_CFLAGS = $(AM_CFLAGS) +kumo_manager_CXXFLAGS = $(AM_CXXFLAGS) + + nobase_include_HEADERS = \ msgpack/pack_define.h \ msgpack/pack_template.h \ diff --git a/cpp/test/Makefile.am b/cpp/test/Makefile.am index a80f319..2b96669 100644 --- a/cpp/test/Makefile.am +++ b/cpp/test/Makefile.am @@ -1,13 +1,15 @@ +AM_CPPFLAGS = -I.. +AM_C_CPPFLAGS = -I.. AM_LDFLAGS = ../libmsgpack.la -lgtest_main check_PROGRAMS = \ - zone \ - pack_unpack \ - streaming \ - object \ - convert \ - buffer + zone \ + pack_unpack \ + streaming \ + object \ + convert \ + buffer TESTS = $(check_PROGRAMS) From b10cb658caa245d9c21abbe94f32fde02e8cba43 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 29 Apr 2010 00:39:45 +0900 Subject: [PATCH 0439/1648] pack_template.h: template_unsigned_long: wrong size checking on !defined(SIZEOF_SHORT) && !defined(SHRT_MAX)" --- msgpack/pack_template.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index f8f522f..c0a1073 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -512,9 +512,9 @@ msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long msgpack_pack_real_uint64(x, d); #endif #else -if(sizeof(unsigned int) == 2) { +if(sizeof(unsigned long) == 2) { msgpack_pack_real_uint16(x, d); -} else if(sizeof(unsigned int) == 4) { +} else if(sizeof(unsigned long) == 4) { msgpack_pack_real_uint32(x, d); } else { msgpack_pack_real_uint64(x, d); From c51fabf6ed70180dce2557dc2987c261e9449c03 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 29 Apr 2010 02:39:53 +0900 Subject: [PATCH 0440/1648] msgpack/pack_template.h: don't evaluate undefined macro --- msgpack/pack_template.h | 132 ++++++++++++++++++++++++++++++++-------- 1 file changed, 106 insertions(+), 26 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index c0a1073..daa8f81 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -16,12 +16,12 @@ * limitations under the License. */ -#ifdef __LITTLE_ENDIAN__ +#if defined(__LITTLE_ENDIAN__) #define TAKE8_8(d) ((uint8_t*)&d)[0] #define TAKE8_16(d) ((uint8_t*)&d)[0] #define TAKE8_32(d) ((uint8_t*)&d)[0] #define TAKE8_64(d) ((uint8_t*)&d)[0] -#elif __BIG_ENDIAN__ +#elif defined(__BIG_ENDIAN__) #define TAKE8_8(d) ((uint8_t*)&d)[0] #define TAKE8_16(d) ((uint8_t*)&d)[1] #define TAKE8_32(d) ((uint8_t*)&d)[3] @@ -377,14 +377,24 @@ msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d) msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d) { -#if defined(SIZEOF_SHORT) || defined(SHRT_MAX) -#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff +#if defined(SIZEOF_SHORT) +#if SIZEOF_SHORT == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff +#elif SIZEOF_SHORT == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(SHRT_MAX) +#if SHRT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif SHRT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(short) == 2) { msgpack_pack_real_int16(x, d); @@ -398,14 +408,24 @@ if(sizeof(short) == 2) { msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d) { -#if defined(SIZEOF_INT) || defined(INT_MAX) -#if SIZEOF_INT == 2 || INT_MAX == 0x7fff +#if defined(SIZEOF_INT) +#if SIZEOF_INT == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff +#elif SIZEOF_INT == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(INT_MAX) +#if INT_MAX == 0x7fff + msgpack_pack_real_int16(x, d); +#elif INT_MAX == 0x7fffffff + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(int) == 2) { msgpack_pack_real_int16(x, d); @@ -419,14 +439,24 @@ if(sizeof(int) == 2) { msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d) { -#if defined(SIZEOF_LONG) || defined(LONG_MAX) -#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL +#if defined(SIZEOF_LONG) +#if SIZEOF_LONG == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL +#elif SIZEOF_LONG == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(LONG_MAX) +#if LONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif LONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(long) == 2) { msgpack_pack_real_int16(x, d); @@ -440,14 +470,24 @@ if(sizeof(long) == 2) { msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d) { -#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX) -#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL +#if defined(SIZEOF_LONG_LONG) +#if SIZEOF_LONG_LONG == 2 msgpack_pack_real_int16(x, d); -#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL +#elif SIZEOF_LONG_LONG == 4 msgpack_pack_real_int32(x, d); #else msgpack_pack_real_int64(x, d); #endif + +#elif defined(LLONG_MAX) +#if LLONG_MAX == 0x7fffL + msgpack_pack_real_int16(x, d); +#elif LLONG_MAX == 0x7fffffffL + msgpack_pack_real_int32(x, d); +#else + msgpack_pack_real_int64(x, d); +#endif + #else if(sizeof(long long) == 2) { msgpack_pack_real_int16(x, d); @@ -461,14 +501,24 @@ if(sizeof(long long) == 2) { msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d) { -#if defined(SIZEOF_SHORT) || defined(USHRT_MAX) -#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU +#if defined(SIZEOF_SHORT) +#if SIZEOF_SHORT == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU +#elif SIZEOF_SHORT == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif + +#elif defined(USHRT_MAX) +#if USHRT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif USHRT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + #else if(sizeof(unsigned short) == 2) { msgpack_pack_real_uint16(x, d); @@ -482,14 +532,24 @@ if(sizeof(unsigned short) == 2) { msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d) { -#if defined(SIZEOF_INT) || defined(UINT_MAX) -#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU +#if defined(SIZEOF_INT) +#if SIZEOF_INT == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU +#elif SIZEOF_INT == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif + +#elif defined(UINT_MAX) +#if UINT_MAX == 0xffffU + msgpack_pack_real_uint16(x, d); +#elif UINT_MAX == 0xffffffffU + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + #else if(sizeof(unsigned int) == 2) { msgpack_pack_real_uint16(x, d); @@ -503,14 +563,24 @@ if(sizeof(unsigned int) == 2) { msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d) { -#if defined(SIZEOF_LONG) || defined(ULONG_MAX) -#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL +#if defined(SIZEOF_LONG) +#if SIZEOF_LONG == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL +#elif SIZEOF_LONG == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif + +#elif defined(ULONG_MAX) +#if ULONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif ULONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + #else if(sizeof(unsigned long) == 2) { msgpack_pack_real_uint16(x, d); @@ -524,14 +594,24 @@ if(sizeof(unsigned long) == 2) { msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d) { -#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX) -#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL +#if defined(SIZEOF_LONG_LONG) +#if SIZEOF_LONG_LONG == 2 msgpack_pack_real_uint16(x, d); -#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL +#elif SIZEOF_LONG_LONG == 4 msgpack_pack_real_uint32(x, d); #else msgpack_pack_real_uint64(x, d); #endif + +#elif defined(ULLONG_MAX) +#if ULLONG_MAX == 0xffffUL + msgpack_pack_real_uint16(x, d); +#elif ULLONG_MAX == 0xffffffffUL + msgpack_pack_real_uint32(x, d); +#else + msgpack_pack_real_uint64(x, d); +#endif + #else if(sizeof(unsigned long long) == 2) { msgpack_pack_real_uint16(x, d); From f77d76a320547dc8ae2dde78b37f4119c9a2caf0 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:01:16 +0900 Subject: [PATCH 0441/1648] Add COPYING file to python package. --- python/COPYING | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 python/COPYING diff --git a/python/COPYING b/python/COPYING new file mode 100644 index 0000000..3cb1f1e --- /dev/null +++ b/python/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008-2010 KLab Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + From 42cb39a9775c2fb67071505091c9ef7afcdbf806 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:01:16 +0900 Subject: [PATCH 0442/1648] Add COPYING file to python package. --- COPYING | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 COPYING diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..3cb1f1e --- /dev/null +++ b/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008-2010 KLab Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + From dda3d24bca8c00f58598523c60902c89581c8da2 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:08:41 +0900 Subject: [PATCH 0443/1648] Add download url. --- python/MANIFEST.in | 1 + python/setup.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 6841ffe..cbc9ada 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -1,2 +1,3 @@ include setup.py +include COPYING recursive-include msgpack *.h *.c *.pyx diff --git a/python/setup.py b/python/setup.py index 66cf27e..2feaff9 100755 --- a/python/setup.py +++ b/python/setup.py @@ -29,6 +29,10 @@ if have_cython: else: sources = ['msgpack/_msgpack.c'] + for f in sources: + if not os.path.exists(f): + raise ImportError("Building msgpack from VCS needs Cython. Install Cython or use sdist package.") + Sdist = sdist msgpack_mod = Extension('msgpack._msgpack', @@ -60,6 +64,7 @@ setup(name='msgpack', description=desc, long_description=long_desc, url="http://msgpack.sourceforge.jp/", + download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', From 316a0fa7b921dad15fe9b47a31e367b0218c0f38 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:08:41 +0900 Subject: [PATCH 0444/1648] Add download url. --- MANIFEST.in | 1 + setup.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 6841ffe..cbc9ada 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include setup.py +include COPYING recursive-include msgpack *.h *.c *.pyx diff --git a/setup.py b/setup.py index 66cf27e..2feaff9 100755 --- a/setup.py +++ b/setup.py @@ -29,6 +29,10 @@ if have_cython: else: sources = ['msgpack/_msgpack.c'] + for f in sources: + if not os.path.exists(f): + raise ImportError("Building msgpack from VCS needs Cython. Install Cython or use sdist package.") + Sdist = sdist msgpack_mod = Extension('msgpack._msgpack', @@ -60,6 +64,7 @@ setup(name='msgpack', description=desc, long_description=long_desc, url="http://msgpack.sourceforge.jp/", + download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', From 833ee6484c731d0419a9cbd122a499f2ae77ae6b Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:52:32 +0900 Subject: [PATCH 0445/1648] Release msgpack-python 0.1.3 --- python/setup.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/python/setup.py b/python/setup.py index 2feaff9..27dbc48 100755 --- a/python/setup.py +++ b/python/setup.py @@ -14,7 +14,7 @@ except ImportError: from distutils.command.build_ext import build_ext have_cython = False -version = '0.2.0dev' +version = '0.1.3' # take care of extension modules. if have_cython: @@ -42,11 +42,7 @@ del sources desc = 'MessagePack (de)serializer.' -long_desc = desc + """ - -MessagePack_ (de)serializer for Python. - -.. _MessagePack: http://msgpack.sourceforge.jp/ +long_desc = """MessagePack (de)serializer for Python. What's MessagePack? (from http://msgpack.sourceforge.jp/) @@ -54,7 +50,7 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) focused on high performance. It is like JSON, but very fast and small. """ -setup(name='msgpack', +setup(name='msgpack-python', author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, @@ -63,7 +59,7 @@ setup(name='msgpack', packages=['msgpack'], description=desc, long_description=long_desc, - url="http://msgpack.sourceforge.jp/", + url='http://msgpack.sourceforge.jp/', download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Development Status :: 4 - Beta', From 4f41080b942657b8bca4a3befe00eba0c80c6ce7 Mon Sep 17 00:00:00 2001 From: Naoki INADA Date: Thu, 29 Apr 2010 07:52:32 +0900 Subject: [PATCH 0446/1648] Release msgpack-python 0.1.3 --- setup.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 2feaff9..27dbc48 100755 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ except ImportError: from distutils.command.build_ext import build_ext have_cython = False -version = '0.2.0dev' +version = '0.1.3' # take care of extension modules. if have_cython: @@ -42,11 +42,7 @@ del sources desc = 'MessagePack (de)serializer.' -long_desc = desc + """ - -MessagePack_ (de)serializer for Python. - -.. _MessagePack: http://msgpack.sourceforge.jp/ +long_desc = """MessagePack (de)serializer for Python. What's MessagePack? (from http://msgpack.sourceforge.jp/) @@ -54,7 +50,7 @@ What's MessagePack? (from http://msgpack.sourceforge.jp/) focused on high performance. It is like JSON, but very fast and small. """ -setup(name='msgpack', +setup(name='msgpack-python', author='INADA Naoki', author_email='songofacandy@gmail.com', version=version, @@ -63,7 +59,7 @@ setup(name='msgpack', packages=['msgpack'], description=desc, long_description=long_desc, - url="http://msgpack.sourceforge.jp/", + url='http://msgpack.sourceforge.jp/', download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Development Status :: 4 - Beta', From 6352472c5fbe1d293dfff2d48dca81ea91bba29f Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 29 Apr 2010 09:06:46 +0900 Subject: [PATCH 0447/1648] python: sourceforge.jp -> sourceforge.net --- python/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index 27dbc48..8d8a6f4 100755 --- a/python/setup.py +++ b/python/setup.py @@ -44,7 +44,7 @@ del sources desc = 'MessagePack (de)serializer.' long_desc = """MessagePack (de)serializer for Python. -What's MessagePack? (from http://msgpack.sourceforge.jp/) +What's MessagePack? (from http://msgpack.sourceforge.net/) MessagePack is a binary-based efficient data interchange format that is focused on high performance. It is like JSON, but very fast and small. @@ -59,7 +59,7 @@ setup(name='msgpack-python', packages=['msgpack'], description=desc, long_description=long_desc, - url='http://msgpack.sourceforge.jp/', + url='http://msgpack.sourceforge.net/', download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Development Status :: 4 - Beta', From 76cc80c25d10d1224f0f16c6a74b531bf452aede Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 29 Apr 2010 09:06:46 +0900 Subject: [PATCH 0448/1648] python: sourceforge.jp -> sourceforge.net --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 27dbc48..8d8a6f4 100755 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ del sources desc = 'MessagePack (de)serializer.' long_desc = """MessagePack (de)serializer for Python. -What's MessagePack? (from http://msgpack.sourceforge.jp/) +What's MessagePack? (from http://msgpack.sourceforge.net/) MessagePack is a binary-based efficient data interchange format that is focused on high performance. It is like JSON, but very fast and small. @@ -59,7 +59,7 @@ setup(name='msgpack-python', packages=['msgpack'], description=desc, long_description=long_desc, - url='http://msgpack.sourceforge.jp/', + url='http://msgpack.sourceforge.net/', download_url='http://pypi.python.org/pypi/msgpack/', classifiers=[ 'Development Status :: 4 - Beta', From fc5e8ddca151e60e02ef8b3be68f1c2590b56db8 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Thu, 29 Apr 2010 21:27:41 +0900 Subject: [PATCH 0449/1648] java: add Unpacker.feed() function for java.nio.ByteBuffer --- java/src/main/java/org/msgpack/Unpacker.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java index 458ffee..c8a8823 100644 --- a/java/src/main/java/org/msgpack/Unpacker.java +++ b/java/src/main/java/org/msgpack/Unpacker.java @@ -21,6 +21,7 @@ import java.lang.Iterable; import java.io.InputStream; import java.io.IOException; import java.util.Iterator; +import java.nio.ByteBuffer; import org.msgpack.impl.UnpackerImpl; public class Unpacker extends UnpackerImpl implements Iterable { @@ -103,6 +104,14 @@ public class Unpacker extends UnpackerImpl implements Iterable { used += size; } + public void feed(ByteBuffer buffer) { + int length = buffer.remaining(); + if (length == 0) return; + reserveBuffer(length); + buffer.get(this.buffer, this.offset, length); + bufferConsumed(length); + } + public void feed(byte[] buffer) { feed(buffer, 0, buffer.length); } From 91a1f8d9e121c1b1379476aa61426dd8a3366987 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 29 Apr 2010 22:15:03 +0900 Subject: [PATCH 0450/1648] cpp: new streaming deserialier API. --- cpp/msgpack/unpack.hpp | 149 ++++++++++++++++++++++++++++++---------- cpp/test/pack_unpack.cc | 26 +++++++ cpp/test/streaming.cc | 114 ++++++++++++++++++++++++++++-- 3 files changed, 247 insertions(+), 42 deletions(-) diff --git a/cpp/msgpack/unpack.hpp b/cpp/msgpack/unpack.hpp index d39b5df..dbe7777 100644 --- a/cpp/msgpack/unpack.hpp +++ b/cpp/msgpack/unpack.hpp @@ -37,6 +37,31 @@ struct unpack_error : public std::runtime_error { }; +class unpacked { +public: + unpacked() { } + + unpacked(object obj, std::auto_ptr z) : + m_obj(obj), m_zone(z) { } + + object& get() + { return m_obj; } + + const object& get() const + { return m_obj; } + + std::auto_ptr& zone() + { return m_zone; } + + const std::auto_ptr& zone() const + { return m_zone; } + +private: + object m_obj; + std::auto_ptr m_zone; +}; + + class unpacker : public msgpack_unpacker { public: unpacker(size_t init_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE); @@ -53,39 +78,22 @@ public: /*! 3. specify the number of bytes actually copied */ void buffer_consumed(size_t size); - /*! 4. repeat execute() until it retunrs false */ - bool execute(); + /*! 4. repeat next() until it retunrs false */ + bool next(unpacked* result); - /*! 5.1. if execute() returns true, take out the parsed object */ - object data(); - - /*! 5.2. the object is valid until the zone is deleted */ - // Note that once release_zone() from unpacker, you must delete it - // otherwise the memrory will leak. - zone* release_zone(); - - /*! 5.2. this method is equivalence to `delete release_zone()` */ - void reset_zone(); - - /*! 5.3. after release_zone(), re-initialize unpacker */ - void reset(); - - /*! 6. check if the size of message doesn't exceed assumption. */ + /*! 5. check if the size of message doesn't exceed assumption. */ size_t message_size() const; - // Basic usage of the unpacker is as following: // // msgpack::unpacker pac; - // - // while( /* readable */ ) { + // while( /* input is readable */ ) { // // // 1. - // pac.reserve_buffer(1024); + // pac.reserve_buffer(32*1024); // // // 2. - // ssize_t bytes = - // read(the_source, pac.buffer(), pac.buffer_capacity()); + // size_t bytes = input.readsome(pac.buffer(), pac.buffer_capacity()); // // // error handling ... // @@ -93,25 +101,40 @@ public: // pac.buffer_consumed(bytes); // // // 4. - // while(pac.execute()) { - // // 5.1 - // object o = pac.data(); + // msgpack::unpacked result; + // while(pac.next(&result)) { + // // do some with the object with the zone. + // msgpack::object obj = result.get(); + // std::auto_ptr z = result.zone(); + // on_message(obj, z); // - // // 5.2 - // std::auto_ptr olife( pac.release_zone() ); + // //// boost::shared_ptr is also usable: + // // boost::shared_ptr life(z.release()); + // // on_message(result.get(), life); + // } // - // // boost::shared_ptr is also usable: - // // boost::shared_ptr olife( pac.release_zone() ); - // - // // 5.3 - // pac.reset(); - // - // // do some with the object with the old zone. - // do_something(o, olife); + // // 5. + // if(pac.message_size() > 10*1024*1024) { + // throw std::runtime_error("message is too large"); // } // } // + /*! for backward compatibility */ + bool execute(); + + /*! for backward compatibility */ + object data(); + + /*! for backward compatibility */ + zone* release_zone(); + + /*! for backward compatibility */ + void reset_zone(); + + /*! for backward compatibility */ + void reset(); + public: // These functions are usable when non-MessagePack message follows after // MessagePack message. @@ -137,6 +160,11 @@ private: }; +static bool unpack(unpacked* result, + const char* data, size_t len, size_t* offset = NULL); + + +// obsolete typedef enum { UNPACK_SUCCESS = 2, UNPACK_EXTRA_BYTES = 1, @@ -144,6 +172,7 @@ typedef enum { UNPACK_PARSE_ERROR = -1, } unpack_return; +// obsolete static unpack_return unpack(const char* data, size_t len, size_t* off, zone* z, object* result); @@ -187,6 +216,20 @@ inline void unpacker::buffer_consumed(size_t size) return msgpack_unpacker_buffer_consumed(this, size); } +inline bool unpacker::next(unpacked* result) +{ + int ret = msgpack_unpacker_execute(this); + if(ret < 0) { + throw unpack_error("parse error"); + } + + result->zone().reset( release_zone() ); + result->get() = data(); + reset(); + + return ret > 0; +} + inline bool unpacker::execute() { @@ -230,12 +273,12 @@ inline void unpacker::reset() msgpack_unpacker_reset(this); } + inline size_t unpacker::message_size() const { return msgpack_unpacker_message_size(this); } - inline size_t unpacker::parsed_size() const { return msgpack_unpacker_parsed_size(this); @@ -262,6 +305,38 @@ inline void unpacker::remove_nonparsed_buffer() } +inline bool unpack(unpacked* result, + const char* data, size_t len, size_t* offset) +{ + msgpack::object obj; + std::auto_ptr z(new zone()); + + unpack_return ret = (unpack_return)msgpack_unpack( + data, len, offset, z.get(), + reinterpret_cast(&obj)); + + switch(ret) { + case UNPACK_SUCCESS: + result->get() = obj; + result->zone() = z; + return false; + + case UNPACK_EXTRA_BYTES: + result->get() = obj; + result->zone() = z; + return true; + + case UNPACK_CONTINUE: + throw unpack_error("insufficient bytes"); + + case UNPACK_PARSE_ERROR: + default: + throw unpack_error("parse error"); + } +} + + +// obsolete inline unpack_return unpack(const char* data, size_t len, size_t* off, zone* z, object* result) { diff --git a/cpp/test/pack_unpack.cc b/cpp/test/pack_unpack.cc index ecf52c5..ca9b7d5 100644 --- a/cpp/test/pack_unpack.cc +++ b/cpp/test/pack_unpack.cc @@ -77,6 +77,32 @@ TEST(unpack, sequence) msgpack::pack(sbuf, 2); msgpack::pack(sbuf, 3); + bool cont; + size_t offset = 0; + + msgpack::unpacked msg; + + cont = msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset); + EXPECT_TRUE(cont); + EXPECT_EQ(1, msg.get().as()); + + cont = msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset); + EXPECT_TRUE(cont); + EXPECT_EQ(2, msg.get().as()); + + cont = msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset); + EXPECT_FALSE(cont); + EXPECT_EQ(3, msg.get().as()); +} + + +TEST(unpack, sequence_compat) +{ + msgpack::sbuffer sbuf; + msgpack::pack(sbuf, 1); + msgpack::pack(sbuf, 2); + msgpack::pack(sbuf, 3); + size_t offset = 0; msgpack::zone z; diff --git a/cpp/test/streaming.cc b/cpp/test/streaming.cc index 2d03976..c01b8be 100644 --- a/cpp/test/streaming.cc +++ b/cpp/test/streaming.cc @@ -2,6 +2,7 @@ #include #include + TEST(streaming, basic) { std::ostringstream stream; @@ -15,6 +16,108 @@ TEST(streaming, basic) msgpack::unpacker pac; + int count = 0; + while(count < 3) { + pac.reserve_buffer(32*1024); + + size_t len = input.readsome(pac.buffer(), pac.buffer_capacity()); + pac.buffer_consumed(len); + + msgpack::unpacked result; + while(pac.next(&result)) { + msgpack::object obj = result.get(); + switch(count++) { + case 0: + EXPECT_EQ(1, obj.as()); + break; + case 1: + EXPECT_EQ(2, obj.as()); + break; + case 2: + EXPECT_EQ(3, obj.as()); + return; + } + } + } +} + + +class event_handler { +public: + event_handler(std::istream& input) : input(input) { } + ~event_handler() { } + + void on_read() + { + while(true) { + pac.reserve_buffer(32*1024); + + size_t len = input.readsome(pac.buffer(), pac.buffer_capacity()); + + if(len == 0) { + return; + } + + pac.buffer_consumed(len); + + msgpack::unpacked result; + while(pac.next(&result)) { + on_message(result.get(), result.zone()); + } + + if(pac.message_size() > 10*1024*1024) { + throw std::runtime_error("message is too large"); + } + } + } + + void on_message(msgpack::object obj, std::auto_ptr z) + { + EXPECT_EQ(expect, obj.as()); + } + + int expect; + +private: + std::istream& input; + msgpack::unpacker pac; +}; + +TEST(streaming, event) +{ + std::stringstream stream; + msgpack::packer pk(&stream); + + event_handler handler(stream); + + pk.pack(1); + handler.expect = 1; + handler.on_read(); + + pk.pack(2); + handler.expect = 2; + handler.on_read(); + + pk.pack(3); + handler.expect = 3; + handler.on_read(); +} + + +// backward compatibility +TEST(streaming, basic_compat) +{ + std::ostringstream stream; + msgpack::packer pk(&stream); + + pk.pack(1); + pk.pack(2); + pk.pack(3); + + std::istringstream input(stream.str()); + + msgpack::unpacker pac; + int count = 0; while(count < 3) { pac.reserve_buffer(32*1024); @@ -44,10 +147,11 @@ TEST(streaming, basic) } -class event_handler { +// backward compatibility +class event_handler_compat { public: - event_handler(std::istream& input) : input(input) { } - ~event_handler() { } + event_handler_compat(std::istream& input) : input(input) { } + ~event_handler_compat() { } void on_read() { @@ -87,12 +191,12 @@ private: msgpack::unpacker pac; }; -TEST(streaming, event) +TEST(streaming, event_compat) { std::stringstream stream; msgpack::packer pk(&stream); - event_handler handler(stream); + event_handler_compat handler(stream); pk.pack(1); handler.expect = 1; From 8783cf8ec3abca1675a47940c1f4e162a66a562d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 29 Apr 2010 22:32:43 +0900 Subject: [PATCH 0451/1648] cpp: fixes unpacker::next --- cpp/msgpack/unpack.hpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cpp/msgpack/unpack.hpp b/cpp/msgpack/unpack.hpp index dbe7777..56ce0f6 100644 --- a/cpp/msgpack/unpack.hpp +++ b/cpp/msgpack/unpack.hpp @@ -219,15 +219,22 @@ inline void unpacker::buffer_consumed(size_t size) inline bool unpacker::next(unpacked* result) { int ret = msgpack_unpacker_execute(this); + if(ret < 0) { throw unpack_error("parse error"); } - result->zone().reset( release_zone() ); - result->get() = data(); - reset(); + if(ret == 0) { + result->zone().reset(); + result->get() = object(); + return false; - return ret > 0; + } else { + result->zone().reset( release_zone() ); + result->get() = data(); + reset(); + return true; + } } From e57084f6dfa99e63542952a3fa39b0e75a1e66de Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 29 Apr 2010 23:45:48 +0900 Subject: [PATCH 0452/1648] cpp: update ChangeLog --- cpp/ChangeLog | 10 ++++++++++ cpp/configure.in | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cpp/ChangeLog b/cpp/ChangeLog index e69de29..53a9991 100644 --- a/cpp/ChangeLog +++ b/cpp/ChangeLog @@ -0,0 +1,10 @@ + +2010-04-29 version 0.5.0: + + * msgpack_object_type is changed. MSGPACK_OBJECT_NIL is now 0x00. + * New safe streaming deserializer API. + * Add object::object(const T&) and object::operator=(const T&) + * Add operator==(object, const T&) + * MSGPACK_DEFINE macro defines msgpack_object(object* obj, zone* z) + * C++ programs doesn't need to link "msgpackc" library. + diff --git a/cpp/configure.in b/cpp/configure.in index 596b28d..61fde4f 100644 --- a/cpp/configure.in +++ b/cpp/configure.in @@ -1,6 +1,6 @@ AC_INIT(object.cpp) AC_CONFIG_AUX_DIR(ac) -AM_INIT_AUTOMAKE(msgpack, 0.4.3) +AM_INIT_AUTOMAKE(msgpack, 0.5.0) AC_CONFIG_HEADER(config.h) AC_SUBST(CFLAGS) From 70d2c47367ca56be99f0b7e7b815171162ff211e Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sun, 2 May 2010 22:09:18 +0900 Subject: [PATCH 0453/1648] perl: added more test case for streaming deserializer. --- perl/t/03_stream_unpack.t | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/perl/t/03_stream_unpack.t b/perl/t/03_stream_unpack.t index 0a8d7d9..a4ab4eb 100644 --- a/perl/t/03_stream_unpack.t +++ b/perl/t/03_stream_unpack.t @@ -23,10 +23,21 @@ sub pis ($$) { my @dat = do 't/data.pl'; -plan tests => 1*(scalar(@dat)/2) + 1; +plan tests => 1*(scalar(@dat)/2) + 3; isa_ok $up, 'Data::MessagePack::Unpacker'; for (my $i=0; $inew(); + $up->execute("\x95", 0); # array marker + for (1..5) { + $up->execute("\xc0", 0); # nil + } + ok $up->is_finished; + is_deeply $up->data, [undef, undef, undef, undef, undef]; +} + From 517ced2a54870e1c5aa9339d2483787477e529bd Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 3 May 2010 00:08:02 +0900 Subject: [PATCH 0454/1648] Perl: added more test case for streaming unpacker --- perl/t/06_stream_unpack2.t | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 perl/t/06_stream_unpack2.t diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t new file mode 100644 index 0000000..dc82c41 --- /dev/null +++ b/perl/t/06_stream_unpack2.t @@ -0,0 +1,26 @@ +use strict; +use warnings; +use Data::MessagePack; +use Test::More; + +my $input = [(undef)x16]; +my $packed = Data::MessagePack->pack($input); +is_deeply(Data::MessagePack->unpack($packed), $input); + +{ + my $up = Data::MessagePack::Unpacker->new(); + $up->execute($packed, 0); + ok $up->is_finished; + is_deeply $up->data, $input; +} + +{ + my $up = Data::MessagePack::Unpacker->new(); + is $up->execute(substr($packed, 0, 3), 0), 3; + $up->execute($packed, 3); + ok $up->is_finished; + is_deeply $up->data, $input; +} + +done_testing; + From c77eac325e0097987f7bfcc2f1d62913c7326f21 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 3 May 2010 00:22:16 +0900 Subject: [PATCH 0455/1648] Perl: added failing test case for memory leaks --- perl/Makefile.PL | 4 +- perl/xt/leaks/normal.t | 93 +++++++++++++++++++++++++++++++++++++ perl/xt/leaks/stream.t | 101 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 perl/xt/leaks/normal.t create mode 100644 perl/xt/leaks/stream.t diff --git a/perl/Makefile.PL b/perl/Makefile.PL index ac83f72..27db363 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -8,7 +8,7 @@ license 'perl'; can_cc or die "This module requires a C compiler"; tests 't/*.t'; -author_tests('xt'); +recursive_author_tests('xt'); use_ppport 3.19; clean_files qw{ @@ -32,6 +32,8 @@ if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) { } } +requires 'Test::More' => 0.95; + auto_set_repository; build_requires 'Test::More'; use_test_base; diff --git a/perl/xt/leaks/normal.t b/perl/xt/leaks/normal.t new file mode 100644 index 0000000..370b23e --- /dev/null +++ b/perl/xt/leaks/normal.t @@ -0,0 +1,93 @@ +use strict; +use warnings; +use Test::More; +use Data::MessagePack; +use Devel::Peek; + +plan skip_all => '$ENV{LEAK_TEST} is required' unless $ENV{LEAK_TEST}; + +my $input = [ + { + "ZCPGBENCH-1276933268" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "VDORBENCH-5637665303" => + { "1271859210" => [ "\x00\x01\x00\x01\x00", 1 ] }, + "ZVTHBENCH-7648578738" => { + "1271859210" => [ + "\x0a\x02\x04\x00\x00", "2600", + "\x0a\x05\x04\x00\x00", "4600" + ] + }, + "VMVTBENCH-5237337637" => + { "1271859210" => [ "\x00\x01\x00\x01\x00", 1 ] }, + "ZPLSBENCH-1823993880" => + { "1271859210" => [ "\x01\x07\x07\x03\x06", "10001" ] }, + "ZCPGBENCH-1995524375" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "ZCPGBENCH-2330423245" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "ZCPGBENCH-2963065090" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "MINT0" => { "1271859210" => [ "\x00\x01\x00\x01\x00", "D" ] } + } +]; +my $r = Data::MessagePack->pack($input); +my $n1 = trace(10); +my $n2 = trace(10000); +diag("$n1, $n2"); + +cmp_ok abs($n2-$n1), '<', 100; + +done_testing; + +sub trace { + my $n = shift; + my $before = memoryusage(); + for ( 1 .. $n ) { + my $x = Data::MessagePack->unpack($r); + # is_deeply($x, $input); + } + my $after = memoryusage(); + diag("$n\t: $after - $before"); + return $after - $before; +} + +sub memoryusage { + my $status = `cat /proc/$$/status`; + my @lines = split( "\n", $status ); + foreach my $line (@lines) { + if ( $line =~ /^VmRSS:/ ) { + $line =~ s/.*:\s*(\d+).*/$1/; + return int($line); + } + } + return -1; +} + +__END__ + [ + { + "ZCPGBENCH-1276933268" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "VDORBENCH-5637665303" => + { "1271859210" => [ "\x00\x01\x00\x01\x00", 1 ] }, + "ZVTHBENCH-7648578738" => { + "1271859210" => [ + "\x0a\x02\x04\x00\x00", "2600", + "\x0a\x05\x04\x00\x00", "4600" + ] + }, + "VMVTBENCH-5237337637" => + { "1271859210" => [ "\x00\x01\x00\x01\x00", 1 ] }, + "ZPLSBENCH-1823993880" => + { "1271859210" => [ "\x01\x07\x07\x03\x06", "10001" ] }, + "ZCPGBENCH-1995524375" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "ZCPGBENCH-2330423245" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "ZCPGBENCH-2963065090" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "MINT0" => { "1271859210" => [ "\x00\x01\x00\x01\x00", "D" ] } + } + ] + diff --git a/perl/xt/leaks/stream.t b/perl/xt/leaks/stream.t new file mode 100644 index 0000000..c196d4d --- /dev/null +++ b/perl/xt/leaks/stream.t @@ -0,0 +1,101 @@ +use strict; +use warnings; +use Test::More; +use Data::MessagePack; +use Test::Requires 'Test::LeakTrace'; +use Devel::Peek; + +plan skip_all => '$ENV{LEAK_TEST} is required' unless $ENV{LEAK_TEST}; + +my $input = [ + { + "ZCPGBENCH-1276933268" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "VDORBENCH-5637665303" => + { "1271859210" => [ "\x00\x01\x00\x01\x00", 1 ] }, + "ZVTHBENCH-7648578738" => { + "1271859210" => [ + "\x0a\x02\x04\x00\x00", "2600", + "\x0a\x05\x04\x00\x00", "4600" + ] + }, + "VMVTBENCH-5237337637" => + { "1271859210" => [ "\x00\x01\x00\x01\x00", 1 ] }, + "ZPLSBENCH-1823993880" => + { "1271859210" => [ "\x01\x07\x07\x03\x06", "10001" ] }, + "ZCPGBENCH-1995524375" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "ZCPGBENCH-2330423245" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "ZCPGBENCH-2963065090" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "MINT0" => { "1271859210" => [ "\x00\x01\x00\x01\x00", "D" ] } + } +]; +$input = [(undef)x10]; +my $r = Data::MessagePack->pack($input); + +my $n1 = trace(10); +my $n2 = trace(10000); +diag("$n1, $n2"); + +cmp_ok abs($n2-$n1), '<', 100; + +done_testing; + +sub trace { + my $n = shift; + my $before = memoryusage(); + for ( 1 .. $n ) { + my $unpacker = Data::MessagePack::Unpacker->new(); + $unpacker->execute($r, 0); + # ok $unpacker->is_finished if $i % 100 == 0; + if ($unpacker->is_finished) { + my $x = $unpacker->data; + # is_deeply($x, $input) if $i % 100 == 0; + } + } + my $after = memoryusage(); + diag("$n\t: $after - $before"); + return $after - $before; +} + +sub memoryusage { + my $status = `cat /proc/$$/status`; + my @lines = split( "\n", $status ); + foreach my $line (@lines) { + if ( $line =~ /^VmRSS:/ ) { + $line =~ s/.*:\s*(\d+).*/$1/; + return int($line); + } + } + return -1; +} + +__END__ + [ + { + "ZCPGBENCH-1276933268" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "VDORBENCH-5637665303" => + { "1271859210" => [ "\x00\x01\x00\x01\x00", 1 ] }, + "ZVTHBENCH-7648578738" => { + "1271859210" => [ + "\x0a\x02\x04\x00\x00", "2600", + "\x0a\x05\x04\x00\x00", "4600" + ] + }, + "VMVTBENCH-5237337637" => + { "1271859210" => [ "\x00\x01\x00\x01\x00", 1 ] }, + "ZPLSBENCH-1823993880" => + { "1271859210" => [ "\x01\x07\x07\x03\x06", "10001" ] }, + "ZCPGBENCH-1995524375" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "ZCPGBENCH-2330423245" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "ZCPGBENCH-2963065090" => + { "1271859210" => [ "\x14\x02\x07\x00\x00", 1 ] }, + "MINT0" => { "1271859210" => [ "\x00\x01\x00\x01\x00", "D" ] } + } + ] + From 77d48f9ceec8f4f3348eac77b1a6ddd8e7f203cf Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 3 May 2010 00:46:15 +0900 Subject: [PATCH 0456/1648] Perl: fixed memory leak issue --- perl/Makefile.PL | 2 +- perl/unpack.c | 59 +++++++++++++++++++++++++++++------------- perl/xt/leaks/stream.t | 8 ++++++ 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 27db363..61afc3d 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -32,7 +32,7 @@ if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) { } } -requires 'Test::More' => 0.95; +requires 'Test::More' => 0.95; # done_testing auto_set_repository; build_requires 'Test::More'; diff --git a/perl/unpack.c b/perl/unpack.c index 69017f1..1bb1f47 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -17,6 +17,7 @@ extern "C" { typedef struct { int finished; SV* source; + int incremented; } unpack_user; #include "msgpack/unpack_define.h" @@ -63,34 +64,34 @@ static INLINE SV* template_callback_root(unpack_user* u) { return &PL_sv_undef; } static INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o) -{ *o = newSVuv(d); return 0; } +{ *o = sv_2mortal(newSVuv(d)); return 0; } static INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o) -{ *o = newSVuv(d); return 0; } +{ *o = sv_2mortal(newSVuv(d)); return 0; } static INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) -{ *o = newSVuv(d); return 0; } +{ *o = sv_2mortal(newSVuv(d)); return 0; } static INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) -{ *o = newSVuv(d); return 0; } +{ *o = sv_2mortal(newSVuv(d)); return 0; } static INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o) -{ *o = newSViv((long)d); return 0; } +{ *o = sv_2mortal(newSViv((long)d)); return 0; } static INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o) -{ *o = newSViv((long)d); return 0; } +{ *o = sv_2mortal(newSViv((long)d)); return 0; } static INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o) -{ *o = newSViv((long)d); return 0; } +{ *o = sv_2mortal(newSViv((long)d)); return 0; } static INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o) -{ *o = newSViv(d); return 0; } +{ *o = sv_2mortal(newSViv(d)); return 0; } static INLINE int template_callback_float(unpack_user* u, float d, SV** o) -{ *o = newSVnv(d); return 0; } +{ *o = sv_2mortal(newSVnv(d)); return 0; } static INLINE int template_callback_double(unpack_user* u, double d, SV** o) -{ *o = newSVnv(d); return 0; } +{ *o = sv_2mortal(newSVnv(d)); return 0; } static INLINE int template_callback_nil(unpack_user* u, SV** o) { *o = &PL_sv_undef; return 0; } @@ -102,19 +103,19 @@ static INLINE int template_callback_false(unpack_user* u, SV** o) { *o = get_bool("Data::MessagePack::false") ; return 0; } static INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o) -{ AV* a = newAV(); *o = (SV*)newRV_noinc((SV*)a); av_extend(a, n); return 0; } +{ AV* a = (AV*)sv_2mortal((SV*)newAV()); *o = sv_2mortal((SV*)newRV_inc((SV*)a)); av_extend(a, n); return 0; } static INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o) { av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; } /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */ static INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o) -{ HV * h = newHV(); *o = newRV_noinc((SV*)h); return 0; } +{ HV * h = (HV*)sv_2mortal((SV*)newHV()); *o = sv_2mortal(newRV_inc((SV*)h)); return 0; } static INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v) { hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } static INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) -{ *o = (l == 0) ? newSVpv("", 0) : newSVpv(p, l); return 0; } +{ *o = sv_2mortal((l == 0) ? newSVpv("", 0) : newSVpv(p, l)); return 0; } #define UNPACKER(from, name) \ msgpack_unpack_t *name; \ @@ -186,11 +187,13 @@ XS(xs_unpack) { } /* ------------------------------ stream -- */ +/* http://twitter.com/frsyuki/status/13249304748 */ static void _reset(SV* self) { + unpack_user u = {0, &PL_sv_undef, 0}; + UNPACKER(self, mp); template_init(mp); - unpack_user u = {0, &PL_sv_undef}; mp->user = u; } @@ -232,10 +235,10 @@ static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) { Perl_croak(aTHX_ "parse error."); } else if(ret > 0) { mp->user.finished = 1; - return newSVuv(from); + return sv_2mortal(newSVuv(from)); } else { mp->user.finished = 0; - return newSVuv(from); + return sv_2mortal(newSVuv(from)); } } @@ -245,12 +248,21 @@ XS(xs_unpacker_execute) { Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)"); } + UNPACKER(ST(0), mp); { SV* self = ST(0); SV* data = ST(1); - IV off = SvIV(ST(2)); + IV off = SvIV(ST(2)); /* offset of $data. normaly, 0. */ ST(0) = _execute_impl(self, data, off, sv_len(data)); + + { + SV * d2 = template_data(mp); + if (!mp->user.incremented && d2) { + SvREFCNT_inc(d2); + mp->user.incremented = 1; + } + } } XSRETURN(1); @@ -291,7 +303,7 @@ XS(xs_unpacker_data) { } UNPACKER(ST(0), mp); - ST(0) = template_data(mp); + ST(0) = sv_2mortal(newSVsv(template_data(mp))); XSRETURN(1); } @@ -302,6 +314,13 @@ XS(xs_unpacker_reset) { Perl_croak(aTHX_ "Usage: $unpacker->reset()"); } + UNPACKER(ST(0), mp); + { + SV * data = template_data(mp); + if (data) { + SvREFCNT_dec(data); + } + } _reset(ST(0)); XSRETURN(0); @@ -314,6 +333,10 @@ XS(xs_unpacker_destroy) { } UNPACKER(ST(0), mp); + SV * data = template_data(mp); + if (SvOK(data)) { + SvREFCNT_dec(data); + } Safefree(mp); XSRETURN(0); diff --git a/perl/xt/leaks/stream.t b/perl/xt/leaks/stream.t index c196d4d..09ec984 100644 --- a/perl/xt/leaks/stream.t +++ b/perl/xt/leaks/stream.t @@ -54,6 +54,14 @@ sub trace { my $x = $unpacker->data; # is_deeply($x, $input) if $i % 100 == 0; } + $unpacker->reset(); + $unpacker->execute($r, 0); + $unpacker->reset(); + $unpacker->execute(substr($r, 0, 1), 0); + $unpacker->execute(substr($r, 0, 2), 1); + $unpacker->execute($r, 2); + $unpacker->reset(); + $r or die; } my $after = memoryusage(); diag("$n\t: $after - $before"); From 77f5cb1f1ffda47625a4ac3aafb28b2142cbb4f9 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 3 May 2010 01:09:21 +0900 Subject: [PATCH 0457/1648] Perl: updated docs. --- perl/lib/Data/MessagePack.pm | 14 ++++++++++++++ perl/lib/Data/MessagePack/Unpacker.pod | 17 +++++++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 4a95e45..b6300c9 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -30,6 +30,20 @@ Data::MessagePack - messagepack Data::MessagePack is a binary packer for perl. +=head1 METHODS + +=over 4 + +=item my $packed = Data::MessagePack->pack($data); + +pack the $data to messagepack format string. + +=item my $unpacked = Data::MessagePack->unpack($msgpackstr); + +unpack the $msgpackstr to messagepack format string. + +=back + =head1 Configuration Variables =over 4 diff --git a/perl/lib/Data/MessagePack/Unpacker.pod b/perl/lib/Data/MessagePack/Unpacker.pod index 61cbd21..c24eaf1 100644 --- a/perl/lib/Data/MessagePack/Unpacker.pod +++ b/perl/lib/Data/MessagePack/Unpacker.pod @@ -22,21 +22,26 @@ This is an streaming deserializer for messagepack. =item my $up = Data::MessagePack::Unpacker->new() -create new stream deserializer +create new instance of stream deserializer. -=item $up->execute() +=item my $ret = $up->execute($data, $offset); -=item $up->execute_limit() +=item my $ret = $up->execute_limit($data, $offset, $limit) -=item $up->is_finished() + $up->execute(substr($data, 0, 3), 0); + $up->execute($data, 3); + +$offset is the offset of $data. + +=item my $bool = $up->is_finished(); is this deserializer finished? -=item $up->data() +=item my $data = $up->data(); returns deserialized object. -=item $up->reset() +=item $up->reset(); reset the stream deserializer, without memory zone. From 7b68b04efdd400b9d851201ae4d7ba47f30e74e2 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 3 May 2010 01:36:48 +0900 Subject: [PATCH 0458/1648] Perl: change for release Data-MessagePack-0.09_01 --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 4ff69c2..47323a5 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.09_01 + + - fixed memory leak issue(reported by Maxime Soulé) + 0.09 - support NVTYPE=="long double" or IVTYPE=="long long" environment diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index b6300c9..52f89bb 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.09'; +our $VERSION = '0.09_01'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From 674c26d9c7213744193d8dd9b5269be66a80c4d5 Mon Sep 17 00:00:00 2001 From: Hideyuki Tanaka Date: Tue, 4 May 2010 16:22:04 +0900 Subject: [PATCH 0459/1648] fix feed function from Handle --- haskell/msgpack.cabal | 2 +- haskell/src/Data/MessagePack/Feed.hs | 12 ++++++++---- haskell/src/Data/MessagePack/Monad.hs | 17 ++++++++++------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal index 505a2b9..31cad3b 100644 --- a/haskell/msgpack.cabal +++ b/haskell/msgpack.cabal @@ -1,5 +1,5 @@ Name: msgpack -Version: 0.2.0 +Version: 0.2.1 License: BSD3 License-File: LICENSE Author: Hideyuki Tanaka diff --git a/haskell/src/Data/MessagePack/Feed.hs b/haskell/src/Data/MessagePack/Feed.hs index afd3f6c..93bdd9b 100644 --- a/haskell/src/Data/MessagePack/Feed.hs +++ b/haskell/src/Data/MessagePack/Feed.hs @@ -33,12 +33,16 @@ type Feeder = IO (Maybe ByteString) -- | Feeder from Handle feederFromHandle :: Handle -> IO Feeder feederFromHandle h = return $ do - bs <- BS.hGet h bufSize + bs <- BS.hGetNonBlocking h bufSize if BS.length bs > 0 - then return $ Just bs + then do return $ Just bs else do - hClose h - return Nothing + bs <- BS.hGet h 1 + if BS.length bs > 0 + then do return $ Just bs + else do + hClose h + return Nothing where bufSize = 4096 diff --git a/haskell/src/Data/MessagePack/Monad.hs b/haskell/src/Data/MessagePack/Monad.hs index bf1514f..cf3a0fd 100644 --- a/haskell/src/Data/MessagePack/Monad.hs +++ b/haskell/src/Data/MessagePack/Monad.hs @@ -115,18 +115,21 @@ instance MonadIO m => MonadIO (UnpackerT m) where instance MonadIO m => MonadUnpacker (UnpackerT m) where get = UnpackerT $ \up feed -> liftIO $ do - resp <- unpackerExecute up - guard $ resp>=0 - when (resp==0) $ do - Just bs <- feed - unpackerFeed up bs - resp2 <- unpackerExecute up - guard $ resp2==1 + executeOne up feed obj <- unpackerData up freeZone =<< unpackerReleaseZone up unpackerReset up let Right r = fromObject obj return r + + where + executeOne up feed = do + resp <- unpackerExecute up + guard $ resp>=0 + when (resp==0) $ do + Just bs <- feed + unpackerFeed up bs + executeOne up feed -- | Execute deserializer using given feeder. unpackFrom :: MonadIO m => Feeder -> UnpackerT m r -> m r From dbe760d6e2b0a9d800efa742e4fe264e898e8f3b Mon Sep 17 00:00:00 2001 From: Hideyuki Tanaka Date: Tue, 4 May 2010 16:24:45 +0900 Subject: [PATCH 0460/1648] make () to OBJECT instance (Nil) --- haskell/src/Data/MessagePack/Class.hs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/haskell/src/Data/MessagePack/Class.hs b/haskell/src/Data/MessagePack/Class.hs index f50a4d8..34a7772 100644 --- a/haskell/src/Data/MessagePack/Class.hs +++ b/haskell/src/Data/MessagePack/Class.hs @@ -46,6 +46,11 @@ instance OBJECT Object where fromObjectError :: String fromObjectError = "fromObject: cannot cast" +instance OBJECT () where + toObject = const ObjectNil + fromObject ObjectNil = Right () + fromObject _ = Left fromObjectError + instance OBJECT Int where toObject = ObjectInteger fromObject (ObjectInteger n) = Right n From 8ce23f8e3ecab832f3c263acafa81e32a58c78b2 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Tue, 4 May 2010 18:33:29 +0900 Subject: [PATCH 0461/1648] java: fixed pom.xml to work "mvn deploy" command --- java/pom.xml | 79 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 321e8d5..d1f6c34 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -3,10 +3,12 @@ 4.0.0 org.msgpack msgpack - MessagePack for Java - 1.0-SNAPSHOT + 0.2 MessagePack for Java + MessagePack for Java + http://msgpack.sourceforge.net/ + The Apache Software License, Version 2.0 @@ -16,9 +18,19 @@ - scm:git://github.com/msgpack/msgpack.git + scm:git:git://github.com/msgpack/msgpack.git + scm:git:git://github.com/msgpack/msgpack.git + + + junit + junit + 4.8.1 + test + + + @@ -83,27 +95,50 @@ + + + msgpack.sourceforge.net + MessagePack Maven2 Repository + http://msgpack.sourceforge.net/maven2 + + + msgpack.sourceforge.net + MessagePack Maven2 Snapshot Repository + http://msgpack.sourceforge.net/maven2-snapshot + + + + + + false + shell.sourceforge.net + Repository at sourceforge.net + scp://shell.sourceforge.net/home/groups/m/ms/msgpack/htdocs/maven2/ + + + true + shell.sourceforge.net + Repository Name + scp://shell.sourceforge.net/home/groups/m/ms/msgpack/htdocs/maven2-snapshot/ + + + - - sourceforge - - - sourceforge.net - Repository at sourceforge.net - scpexe://shell.sourceforge.net/home/groups/m/ms/msgpack/htdocs/maven2/ - - + release + + + + true + org.apache.maven.plugins + maven-deploy-plugin + 2.4 + + true + + + + - - - - junit - junit - 4.8.1 - test - - - From 2f12e6c3d03f4b22be2c7f36b1b1bf7908adb462 Mon Sep 17 00:00:00 2001 From: Hideyuki Tanaka Date: Wed, 5 May 2010 04:28:04 +0900 Subject: [PATCH 0462/1648] remove compiler warnings --- haskell/src/Data/MessagePack/Base.hsc | 8 ++++---- haskell/src/Data/MessagePack/Class.hs | 1 - haskell/src/Data/MessagePack/Feed.hs | 7 +++---- haskell/src/Data/MessagePack/Monad.hs | 2 +- haskell/src/Data/MessagePack/Stream.hs | 2 -- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/haskell/src/Data/MessagePack/Base.hsc b/haskell/src/Data/MessagePack/Base.hsc index ad71712..72c421c 100644 --- a/haskell/src/Data/MessagePack/Base.hsc +++ b/haskell/src/Data/MessagePack/Base.hsc @@ -297,7 +297,7 @@ foreign import ccall "msgpack_pack_raw_body_wrap" msgpack_pack_raw_body :: -- | Pack a single byte stream. It calls 'packRAW' and 'packRAWBody'. packRAW' :: Packer -> ByteString -> IO Int packRAW' pc bs = do - packRAW pc (BS.length bs) + _ <- packRAW pc (BS.length bs) packRAWBody pc bs type Unpacker = ForeignPtr () @@ -475,7 +475,7 @@ peekObject ptr = do (#const MSGPACK_OBJECT_MAP) -> peekObjectMap ptr _ -> - fail "peekObject: unknown object type" + fail $ "peekObject: unknown object type (" ++ show typ ++ ")" peekObjectBool :: Ptr a -> IO Object peekObjectBool ptr = do @@ -541,11 +541,11 @@ packObject pc (ObjectDouble d) = packDouble pc d >> return () packObject pc (ObjectRAW bs) = packRAW' pc bs >> return () packObject pc (ObjectArray ls) = do - packArray pc (length ls) + _ <- packArray pc (length ls) mapM_ (packObject pc) ls packObject pc (ObjectMap ls) = do - packMap pc (length ls) + _ <- packMap pc (length ls) mapM_ (\(a, b) -> packObject pc a >> packObject pc b) ls data UnpackReturn = diff --git a/haskell/src/Data/MessagePack/Class.hs b/haskell/src/Data/MessagePack/Class.hs index 34a7772..365acc5 100644 --- a/haskell/src/Data/MessagePack/Class.hs +++ b/haskell/src/Data/MessagePack/Class.hs @@ -27,7 +27,6 @@ module Data.MessagePack.Class( import Control.Monad.Error import Data.ByteString.Char8 (ByteString) import qualified Data.ByteString.Char8 as C8 -import Data.Either import Data.MessagePack.Base diff --git a/haskell/src/Data/MessagePack/Feed.hs b/haskell/src/Data/MessagePack/Feed.hs index 93bdd9b..4b48639 100644 --- a/haskell/src/Data/MessagePack/Feed.hs +++ b/haskell/src/Data/MessagePack/Feed.hs @@ -21,7 +21,6 @@ module Data.MessagePack.Feed( feederFromString, ) where -import Control.Monad import Data.ByteString (ByteString) import qualified Data.ByteString as BS import Data.IORef @@ -37,9 +36,9 @@ feederFromHandle h = return $ do if BS.length bs > 0 then do return $ Just bs else do - bs <- BS.hGet h 1 - if BS.length bs > 0 - then do return $ Just bs + c <- BS.hGet h 1 + if BS.length c > 0 + then do return $ Just c else do hClose h return Nothing diff --git a/haskell/src/Data/MessagePack/Monad.hs b/haskell/src/Data/MessagePack/Monad.hs index cf3a0fd..c718b8a 100644 --- a/haskell/src/Data/MessagePack/Monad.hs +++ b/haskell/src/Data/MessagePack/Monad.hs @@ -79,7 +79,7 @@ packToString :: MonadIO m => PackerT m r -> m ByteString packToString m = do sb <- liftIO $ newSimpleBuffer pc <- liftIO $ newPacker sb - runPackerT m pc + _ <- runPackerT m pc liftIO $ simpleBufferData sb -- | Execcute given serializer and write byte sequence to Handle. diff --git a/haskell/src/Data/MessagePack/Stream.hs b/haskell/src/Data/MessagePack/Stream.hs index bd17f46..c56fe8d 100644 --- a/haskell/src/Data/MessagePack/Stream.hs +++ b/haskell/src/Data/MessagePack/Stream.hs @@ -19,9 +19,7 @@ module Data.MessagePack.Stream( unpackObjectsFromString, ) where -import Control.Monad import Data.ByteString (ByteString) -import qualified Data.ByteString as BS import System.IO import System.IO.Unsafe From 2c2bf60d0c86fdcccc68077904ef115017e03194 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 5 May 2010 15:50:07 +0900 Subject: [PATCH 0463/1648] Perl: added README file. --- perl/README | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 perl/README diff --git a/perl/README b/perl/README new file mode 100644 index 0000000..31aae99 --- /dev/null +++ b/perl/README @@ -0,0 +1,34 @@ +NAME + Data::MessagePack - messagepack + +SYNOPSIS + my $packed = Data::MessagePack->pack($dat); + my $unpacked = Data::MessagePack->unpack($dat); + +DESCRIPTION + Data::MessagePack is a binary packer for perl. + +METHODS + my $packed = Data::MessagePack->pack($data); + pack the $data to messagepack format string. + + my $unpacked = Data::MessagePack->unpack($msgpackstr); + unpack the $msgpackstr to messagepack format string. + +Configuration Variables + $Data::MessagePack::PreferInteger + Pack the string as int when the value looks like int(EXPERIMENTAL). + +AUTHORS + Tokuhiro Matsuno + +THANKS TO + Jun Kuriyama + +LICENSE + This library is free software; you can redistribute it and/or modify it + under the same terms as Perl itself. + +SEE ALSO + + From 2b8f853b96fc4e2e43e0547dde091609d757386e Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 5 May 2010 16:17:57 +0900 Subject: [PATCH 0464/1648] Perl: fixed some issues. thanks to gfx++ http://gist.github.com/387743 --- perl/unpack.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/perl/unpack.c b/perl/unpack.c index 1bb1f47..70dc3cc 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -2,6 +2,7 @@ extern "C" { #endif +#define PERL_NO_GET_CONTEXT #include "EXTERN.h" #include "perl.h" #include "XSUB.h" @@ -40,7 +41,7 @@ typedef struct { static INLINE SV * get_bool (const char *name) { - SV * sv = get_sv( name, 1 ); + SV * sv = sv_mortalcopy(get_sv( name, 1 )); SvREADONLY_on(sv); SvREADONLY_on( SvRV(sv) ); @@ -73,7 +74,14 @@ static INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o) { *o = sv_2mortal(newSVuv(d)); return 0; } static INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o) -{ *o = sv_2mortal(newSVuv(d)); return 0; } +{ +#if IVSIZE==4 + *o = sv_2mortal(newSVnv(d)); +#else + *o = sv_2mortal(newSVuv(d)); +#endif + return 0; +} static INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o) { *o = sv_2mortal(newSViv((long)d)); return 0; } @@ -93,8 +101,9 @@ static INLINE int template_callback_float(unpack_user* u, float d, SV** o) static INLINE int template_callback_double(unpack_user* u, double d, SV** o) { *o = sv_2mortal(newSVnv(d)); return 0; } +/* &PL_sv_undef is not so good. see http://gist.github.com/387743 */ static INLINE int template_callback_nil(unpack_user* u, SV** o) -{ *o = &PL_sv_undef; return 0; } +{ *o = sv_newmortal(); return 0; } static INLINE int template_callback_true(unpack_user* u, SV** o) { *o = get_bool("Data::MessagePack::true") ; return 0; } @@ -115,7 +124,8 @@ static INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* { hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; } static INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o) -{ *o = sv_2mortal((l == 0) ? newSVpv("", 0) : newSVpv(p, l)); return 0; } +{ *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l)); return 0; } +/* { *o = newSVpvn_flags(p, l, SVs_TEMP); return 0; } <= this does not works. */ #define UNPACKER(from, name) \ msgpack_unpack_t *name; \ From 9420436c09e4e85383d119f45c18d16287661a7f Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 5 May 2010 16:22:40 +0900 Subject: [PATCH 0465/1648] Perl: added test case --- perl/t/07_break.t | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 perl/t/07_break.t diff --git a/perl/t/07_break.t b/perl/t/07_break.t new file mode 100644 index 0000000..dd27ad2 --- /dev/null +++ b/perl/t/07_break.t @@ -0,0 +1,12 @@ +use Test::More; +use Data::MessagePack; +use t::Util; +no warnings 'uninitialized'; # i need this. i need this. + +plan tests => 1; + +my $d = Data::MessagePack->unpack(Data::MessagePack->pack([{x => undef}])); +$d->[0]->{x} = 1; +ok delete $d->[0]->{x}; +$d->[0] = 4; + From f0f574a15b1545de7eb4f2b64d6776b71d7c01cb Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 5 May 2010 16:55:06 +0900 Subject: [PATCH 0466/1648] Perl: releng for 0.10 --- perl/Changes | 6 ++++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 47323a5..5cde8c2 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,9 @@ +0.10 + + - added more test cases. + - fixed portability issue + - (reviewed by gfx++) + 0.09_01 - fixed memory leak issue(reported by Maxime Soulé) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index 52f89bb..d44f0aa 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.09_01'; +our $VERSION = '0.10'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From ebe41a24f127407a269f2417af7dff0470b42f2d Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 5 May 2010 16:59:31 +0900 Subject: [PATCH 0467/1648] perl: releng for data-messagepack 0.11 --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index 5cde8c2..dae261f 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.11 + + - oops(no feature changes) + 0.10 - added more test cases. diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index d44f0aa..ebc7eb7 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.10'; +our $VERSION = '0.11'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From 09bae0a9e82b0c5697846e3e3e667a8679faf93d Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 5 May 2010 17:25:08 +0900 Subject: [PATCH 0468/1648] Perl: Test::Requires is not needed for this test. --- perl/xt/leaks/stream.t | 1 - 1 file changed, 1 deletion(-) diff --git a/perl/xt/leaks/stream.t b/perl/xt/leaks/stream.t index 09ec984..7765d73 100644 --- a/perl/xt/leaks/stream.t +++ b/perl/xt/leaks/stream.t @@ -2,7 +2,6 @@ use strict; use warnings; use Test::More; use Data::MessagePack; -use Test::Requires 'Test::LeakTrace'; use Devel::Peek; plan skip_all => '$ENV{LEAK_TEST} is required' unless $ENV{LEAK_TEST}; From 1864df5ed0d9e268640bf362284517a7857954c9 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 5 May 2010 17:25:25 +0900 Subject: [PATCH 0469/1648] Perl: cleanup Makefile.PL --- perl/Makefile.PL | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 61afc3d..29710c2 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -1,7 +1,7 @@ use inc::Module::Install; name 'Data-MessagePack'; all_from 'lib/Data/MessagePack.pm'; -readme_from 'lib/Data/MessagePack.pm'; +readme_from('lib/Data/MessagePack.pm'); perl_version '5.008005'; license 'perl'; @@ -33,10 +33,10 @@ if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) { } requires 'Test::More' => 0.95; # done_testing +test_requires('Test::Requires'); -auto_set_repository; +auto_set_repository(); build_requires 'Test::More'; -use_test_base; auto_include; WriteAll; From 262fe96c291de26424763016c0b42454229d9108 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 5 May 2010 17:25:45 +0900 Subject: [PATCH 0470/1648] Perl: PERL_NO_GET_CONTEXT makes horrible dTHXs. remove it. --- perl/unpack.c | 1 - 1 file changed, 1 deletion(-) diff --git a/perl/unpack.c b/perl/unpack.c index 70dc3cc..eb6e0dd 100644 --- a/perl/unpack.c +++ b/perl/unpack.c @@ -2,7 +2,6 @@ extern "C" { #endif -#define PERL_NO_GET_CONTEXT #include "EXTERN.h" #include "perl.h" #include "XSUB.h" From 120a85a3e52086c9a746f7bf5f268432e741d8af Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Wed, 5 May 2010 17:28:38 +0900 Subject: [PATCH 0471/1648] Perl: releng for 0.12 --- perl/Changes | 4 ++++ perl/lib/Data/MessagePack.pm | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/perl/Changes b/perl/Changes index dae261f..b03a9d0 100644 --- a/perl/Changes +++ b/perl/Changes @@ -1,3 +1,7 @@ +0.12 + + - PERL_NO_GET_CONTEXT makes horrible dTHXs. remove it. + 0.11 - oops(no feature changes) diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm index ebc7eb7..dcc713d 100644 --- a/perl/lib/Data/MessagePack.pm +++ b/perl/lib/Data/MessagePack.pm @@ -4,7 +4,7 @@ use warnings; use XSLoader; use 5.008001; -our $VERSION = '0.11'; +our $VERSION = '0.12'; our $PreferInteger = 0; our $true = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" }; From be6376ee2d58602f2eac6ee6537cbdc8fbc526fe Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Sat, 8 May 2010 12:02:31 +0900 Subject: [PATCH 0472/1648] Perl: build_requires and requires are duped. --- perl/Makefile.PL | 1 - 1 file changed, 1 deletion(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 29710c2..5bf43f3 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -36,7 +36,6 @@ requires 'Test::More' => 0.95; # done_testing test_requires('Test::Requires'); auto_set_repository(); -build_requires 'Test::More'; auto_include; WriteAll; From f51123d009ebe285b91442d03e220560ee75ef21 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 17 May 2010 05:49:39 +0900 Subject: [PATCH 0473/1648] oops --- perl/Makefile.PL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/Makefile.PL b/perl/Makefile.PL index 5bf43f3..58ab7c7 100644 --- a/perl/Makefile.PL +++ b/perl/Makefile.PL @@ -32,7 +32,7 @@ if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) { } } -requires 'Test::More' => 0.95; # done_testing +requires 'Test::More' => 0.94; # done_testing test_requires('Test::Requires'); auto_set_repository(); From 6ea75f3a9f1888d70bfaf97468922e30a7e7e661 Mon Sep 17 00:00:00 2001 From: tokuhirom Date: Mon, 17 May 2010 05:52:32 +0900 Subject: [PATCH 0474/1648] Perl: do not use done_testing --- perl/t/06_stream_unpack2.t | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t index dc82c41..eaf2cb4 100644 --- a/perl/t/06_stream_unpack2.t +++ b/perl/t/06_stream_unpack2.t @@ -1,7 +1,7 @@ use strict; use warnings; use Data::MessagePack; -use Test::More; +use Test::More tests => 6; my $input = [(undef)x16]; my $packed = Data::MessagePack->pack($input); @@ -22,5 +22,4 @@ is_deeply(Data::MessagePack->unpack($packed), $input); is_deeply $up->data, $input; } -done_testing; From 18967162cfc9737764b0e4862936d32364f2fc84 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 18 May 2010 14:48:23 +0900 Subject: [PATCH 0475/1648] cpp: fix return type mismatch in unpack.c --- cpp/unpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/unpack.c b/cpp/unpack.c index 34016fd..4a42526 100644 --- a/cpp/unpack.c +++ b/cpp/unpack.c @@ -323,7 +323,7 @@ msgpack_object msgpack_unpacker_data(msgpack_unpacker* mpac) msgpack_zone* msgpack_unpacker_release_zone(msgpack_unpacker* mpac) { if(!msgpack_unpacker_flush_zone(mpac)) { - return false; + return NULL; } msgpack_zone* r = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); From 5cad81bf4ce2bc46f5b3ec4663c0eda8d3dff469 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 18 May 2010 14:48:36 +0900 Subject: [PATCH 0476/1648] cpp: fix return type mismatch in zone.c --- cpp/zone.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/zone.c b/cpp/zone.c index 3d0634e..85de765 100644 --- a/cpp/zone.c +++ b/cpp/zone.c @@ -204,7 +204,7 @@ msgpack_zone* msgpack_zone_new(size_t chunk_size) if(!init_chunk_list(&zone->chunk_list, chunk_size)) { free(zone); - return false; + return NULL; } init_finalizer_array(&zone->finalizer_array); From 979ff809827ab25005364dad41d2fd043b8eaa4d Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 20 May 2010 03:49:26 +0900 Subject: [PATCH 0477/1648] java: redesign --- cpp/msgpack/type/nil.hpp | 8 + .../org/msgpack/BufferedUnpackerImpl.java | 384 ++++++++++++++++++ ...Mergeable.java => MessageConvertable.java} | 4 +- .../org/msgpack/MessageTypeException.java | 4 +- .../java/org/msgpack/MessageUnpackable.java | 25 ++ java/src/main/java/org/msgpack/Packer.java | 2 +- .../java/org/msgpack/UnbufferedUnpacker.java | 82 ---- .../main/java/org/msgpack/UnpackCursor.java | 96 +++++ .../main/java/org/msgpack/UnpackIterator.java | 24 +- .../main/java/org/msgpack/UnpackResult.java | 42 ++ java/src/main/java/org/msgpack/Unpacker.java | 269 ++++++------ .../org/msgpack/{impl => }/UnpackerImpl.java | 26 +- .../org/msgpack/TestDirectConversion.java | 154 +++++++ .../test/java/org/msgpack/TestPackUnpack.java | 2 +- 14 files changed, 864 insertions(+), 258 deletions(-) create mode 100644 java/src/main/java/org/msgpack/BufferedUnpackerImpl.java rename java/src/main/java/org/msgpack/{MessageMergeable.java => MessageConvertable.java} (86%) create mode 100644 java/src/main/java/org/msgpack/MessageUnpackable.java delete mode 100644 java/src/main/java/org/msgpack/UnbufferedUnpacker.java create mode 100644 java/src/main/java/org/msgpack/UnpackCursor.java create mode 100644 java/src/main/java/org/msgpack/UnpackResult.java rename java/src/main/java/org/msgpack/{impl => }/UnpackerImpl.java (97%) create mode 100644 java/src/test/java/org/msgpack/TestDirectConversion.java diff --git a/cpp/msgpack/type/nil.hpp b/cpp/msgpack/type/nil.hpp index e58bc9d..f44e45e 100644 --- a/cpp/msgpack/type/nil.hpp +++ b/cpp/msgpack/type/nil.hpp @@ -51,6 +51,14 @@ inline void operator<< (object::with_zone& o, type::nil v) { static_cast(o) << v; } +template <> +inline void object::as() const +{ + msgpack::type::nil v; + convert(&v); +} + + } // namespace msgpack #endif /* msgpack/type/nil.hpp */ diff --git a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java new file mode 100644 index 0000000..0ef9c12 --- /dev/null +++ b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java @@ -0,0 +1,384 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; +import java.nio.ByteBuffer; +//import java.math.BigInteger; + +abstract class BufferedUnpackerImpl extends UnpackerImpl { + int filled = 0; + byte[] buffer = null; + private ByteBuffer castBuffer = ByteBuffer.allocate(8); + + abstract boolean fill() throws IOException; + + final int next(int offset, UnpackResult result) throws IOException, UnpackException { + if(filled == 0) { + if(!fill()) { + return offset; + } + } + + do { + int noffset = super.execute(buffer, offset, filled); + if(noffset <= offset) { + if(!fill()) { + return offset; + } + } + offset = noffset; + } while(!super.isFinished()); + + Object obj = super.getData(); + super.reset(); + result.done(obj); + + return offset; + } + + private final void more(int offset, int require) throws IOException, UnpackException { + while(filled - offset < require) { + if(!fill()) { + // FIXME + throw new UnpackException("insufficient buffer"); + } + } + } + + final byte unpackByte(UnpackCursor c, int offset) throws IOException, MessageTypeException { + int o = unpackInt(c, offset); + if(0x7f < o || o < -0x80) { + throw new MessageTypeException(); + } + return (byte)o; + } + + final short unpackShort(UnpackCursor c, int offset) throws IOException, MessageTypeException { + int o = unpackInt(c, offset); + if(0x7fff < o || o < -0x8000) { + throw new MessageTypeException(); + } + return (short)o; + } + + final int unpackInt(UnpackCursor c, int offset) throws IOException, MessageTypeException { + more(offset, 1); + int b = buffer[offset]; + if((b & 0x80) == 0 || (b & 0xe0) == 0xe0) { // Fixnum + return (int)b; + } + switch(b & 0xff) { + case 0xcc: // unsigned int 8 + more(offset, 2); + c.advance(2); + return (int)((short)buffer[offset+1] & 0xff); + case 0xcd: // unsigned int 16 + more(offset, 3); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 2); + c.advance(3); + return (int)((int)castBuffer.getShort(0) & 0xffff); + case 0xce: // unsigned int 32 + more(offset, 5); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 4); + { + int o = castBuffer.getInt(0); + if(o < 0) { + throw new MessageTypeException(); + } + c.advance(5); + return o; + } + case 0xcf: // unsigned int 64 + more(offset, 9); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 8); + { + long o = castBuffer.getLong(0); + if(o < 0 || o > 0x7fffffffL) { + throw new MessageTypeException(); + } + c.advance(9); + return (int)o; + } + case 0xd0: // signed int 8 + more(offset, 2); + c.advance(2); + return (int)buffer[offset+1]; + case 0xd1: // signed int 16 + more(offset, 3); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 2); + c.advance(3); + return (int)castBuffer.getShort(0); + case 0xd2: // signed int 32 + more(offset, 4); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 4); + c.advance(4); + return (int)castBuffer.getInt(0); + case 0xd3: // signed int 64 + more(offset, 9); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 8); + { + long o = castBuffer.getLong(0); + if(0x7fffffffL < o || o < -0x80000000L) { + throw new MessageTypeException(); + } + c.advance(9); + return (int)o; + } + default: + throw new MessageTypeException(); + } + } + + final long unpackLong(UnpackCursor c, int offset) throws IOException, MessageTypeException { + more(offset, 1); + int b = buffer[offset]; + if((b & 0x80) == 0 || (b & 0xe0) == 0xe0) { // Fixnum + return (long)b; + } + switch(b & 0xff) { + case 0xcc: // unsigned int 8 + more(offset, 2); + c.advance(2); + return (long)((short)buffer[offset+1] & 0xff); + case 0xcd: // unsigned int 16 + more(offset, 3); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 2); + c.advance(3); + return (long)((int)castBuffer.getShort(0) & 0xffff); + case 0xce: // unsigned int 32 + more(offset, 5); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 4); + c.advance(5); + return ((long)castBuffer.getInt(0) & 0xffffffffL); + case 0xcf: // unsigned int 64 + more(offset, 9); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 8); + { + long o = castBuffer.getLong(0); + if(o < 0) { + // FIXME + throw new MessageTypeException("uint 64 bigger than 0x7fffffff is not supported"); + } + c.advance(9); + return o; + } + case 0xd0: // signed int 8 + more(offset, 2); + c.advance(2); + return (long)buffer[offset+1]; + case 0xd1: // signed int 16 + more(offset, 3); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 2); + c.advance(3); + return (long)castBuffer.getShort(0); + case 0xd2: // signed int 32 + more(offset, 4); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 4); + c.advance(4); + return (long)castBuffer.getInt(0); + case 0xd3: // signed int 64 + more(offset, 9); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 8); + c.advance(9); + return (long)castBuffer.getLong(0); + default: + throw new MessageTypeException(); + } + } + + final float unpackFloat(UnpackCursor c, int offset) throws IOException, MessageTypeException { + more(offset, 1); + int b = buffer[offset]; + switch(b & 0xff) { + case 0xca: // float + more(offset, 5); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 4); + c.advance(5); + return castBuffer.getFloat(0); + case 0xcb: // double + more(offset, 9); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 8); + c.advance(9); + // FIXME overflow check + return (float)castBuffer.getDouble(0); + default: + throw new MessageTypeException(); + } + } + + final double unpackDouble(UnpackCursor c, int offset) throws IOException, MessageTypeException { + more(offset, 1); + int b = buffer[offset]; + switch(b & 0xff) { + case 0xca: // float + more(offset, 5); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 4); + c.advance(5); + return (double)castBuffer.getFloat(0); + case 0xcb: // double + more(offset, 9); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 8); + c.advance(9); + return castBuffer.getDouble(0); + default: + throw new MessageTypeException(); + } + } + + final Object unpackNull(UnpackCursor c, int offset) throws IOException, MessageTypeException { + more(offset, 1); + int b = buffer[offset] & 0xff; + if(b != 0xc0) { // nil + throw new MessageTypeException(); + } + c.advance(1); + return null; + } + + final boolean unpackBoolean(UnpackCursor c, int offset) throws IOException, MessageTypeException { + more(offset, 1); + int b = buffer[offset] & 0xff; + if(b == 0xc2) { // false + c.advance(1); + return false; + } else if(b == 0xc3) { // true + c.advance(1); + return true; + } else { + throw new MessageTypeException(); + } + } + + final int unpackArray(UnpackCursor c, int offset) throws IOException, MessageTypeException { + more(offset, 1); + int b = buffer[offset]; + if((b & 0xf0) == 0x90) { // FixArray + c.advance(1); + return (int)(b & 0x0f); + } + switch(b & 0xff) { + case 0xdc: // array 16 + more(offset, 3); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 2); + c.advance(3); + return (int)castBuffer.getShort(0) & 0xffff; + case 0xdd: // array 32 + more(offset, 5); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 4); + c.advance(5); + // FIXME overflow check + return castBuffer.getInt(0) & 0x7fffffff; + default: + throw new MessageTypeException(); + } + } + + final int unpackMap(UnpackCursor c, int offset) throws IOException, MessageTypeException { + more(offset, 1); + int b = buffer[offset]; + if((b & 0xf0) == 0x80) { // FixMap + c.advance(1); + return (int)(b & 0x0f); + } + switch(b & 0xff) { + case 0xde: // map 16 + more(offset, 3); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 2); + c.advance(3); + return (int)castBuffer.getShort(0) & 0xffff; + case 0xdf: // map 32 + more(offset, 5); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 4); + c.advance(5); + // FIXME overflow check + return castBuffer.getInt(0) & 0x7fffffff; + default: + throw new MessageTypeException(); + } + } + + final int unpackRaw(UnpackCursor c, int offset) throws IOException, MessageTypeException { + more(offset, 1); + int b = buffer[offset]; + if((b & 0xe0) == 0xa0) { // FixRaw + c.advance(1); + return (int)(b & 0x0f); + } + switch(b & 0xff) { + case 0xda: // raw 16 + more(offset, 3); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 2); + c.advance(3); + return (int)castBuffer.getShort(0) & 0xffff; + case 0xdb: // raw 32 + more(offset, 5); + castBuffer.rewind(); + castBuffer.put(buffer, offset+1, 4); + c.advance(5); + // FIXME overflow check + return castBuffer.getInt(0) & 0x7fffffff; + default: + throw new MessageTypeException(); + } + } + + final byte[] unpackRawBody(UnpackCursor c, int offset, int length) throws IOException, MessageTypeException { + more(offset, length); + byte[] bytes = new byte[length]; + System.arraycopy(buffer, offset, bytes, 0, length); + c.advance(length); + return bytes; + } + + final String unpackString(UnpackCursor c, int offset) throws IOException, MessageTypeException { + int length = unpackRaw(c, offset); + offset = c.getOffset(); + more(offset, length); + String s; + try { + s = new String(buffer, offset, length, "UTF-8"); + } catch (Exception e) { + throw new MessageTypeException(); + } + c.advance(length); + return s; + } +} + diff --git a/java/src/main/java/org/msgpack/MessageMergeable.java b/java/src/main/java/org/msgpack/MessageConvertable.java similarity index 86% rename from java/src/main/java/org/msgpack/MessageMergeable.java rename to java/src/main/java/org/msgpack/MessageConvertable.java index e5a5b45..da251dc 100644 --- a/java/src/main/java/org/msgpack/MessageMergeable.java +++ b/java/src/main/java/org/msgpack/MessageConvertable.java @@ -17,7 +17,7 @@ // package org.msgpack; -public interface MessageMergeable { - public void messageMerge(Object obj) throws MessageTypeException; +public interface MessageConvertable { + public void messageConvert(Object obj) throws MessageTypeException; } diff --git a/java/src/main/java/org/msgpack/MessageTypeException.java b/java/src/main/java/org/msgpack/MessageTypeException.java index feb6c08..0fa37b7 100644 --- a/java/src/main/java/org/msgpack/MessageTypeException.java +++ b/java/src/main/java/org/msgpack/MessageTypeException.java @@ -17,9 +17,7 @@ // package org.msgpack; -import java.io.IOException; - -public class MessageTypeException extends IOException { +public class MessageTypeException extends RuntimeException { public MessageTypeException() { } public MessageTypeException(String s) { diff --git a/java/src/main/java/org/msgpack/MessageUnpackable.java b/java/src/main/java/org/msgpack/MessageUnpackable.java new file mode 100644 index 0000000..20e4d56 --- /dev/null +++ b/java/src/main/java/org/msgpack/MessageUnpackable.java @@ -0,0 +1,25 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; + +public interface MessageUnpackable { + public void messageUnpack(Unpacker pk) throws IOException, MessageTypeException; +} + diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java index 935728d..6d79414 100644 --- a/java/src/main/java/org/msgpack/Packer.java +++ b/java/src/main/java/org/msgpack/Packer.java @@ -401,7 +401,7 @@ public class Packer { } else if(o instanceof Double) { return packDouble((Double)o); } else { - throw new IOException("unknown object "+o+" ("+o.getClass()+")"); + throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")"); } } } diff --git a/java/src/main/java/org/msgpack/UnbufferedUnpacker.java b/java/src/main/java/org/msgpack/UnbufferedUnpacker.java deleted file mode 100644 index b427973..0000000 --- a/java/src/main/java/org/msgpack/UnbufferedUnpacker.java +++ /dev/null @@ -1,82 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack; - -import java.lang.Iterable; -import java.io.InputStream; -import java.io.IOException; -import java.util.Iterator; -import org.msgpack.impl.UnpackerImpl; - -public class UnbufferedUnpacker extends UnpackerImpl { - private int offset; - private boolean finished; - private Object data; - - public UnbufferedUnpacker() { - super(); - this.offset = 0; - this.finished = false; - } - - public UnbufferedUnpacker useSchema(Schema s) { - super.setSchema(s); - return this; - } - - public Object getData() { - return data; - } - - public boolean isFinished() { - return finished; - } - - public void reset() { - super.reset(); - this.offset = 0; - } - - int getOffset() { - return offset; - } - - void setOffset(int offset) { - this.offset = offset; - } - - public int execute(byte[] buffer) throws UnpackException { - return execute(buffer, 0, buffer.length); - } - - // FIXME - public int execute(byte[] buffer, int offset, int length) throws UnpackException - { - int noffset = super.execute(buffer, offset + this.offset, length); - this.offset = noffset - offset; - if(super.isFinished()) { - this.data = super.getData(); - this.finished = true; - super.reset(); - } else { - this.finished = false; - } - return noffset; - } -} - diff --git a/java/src/main/java/org/msgpack/UnpackCursor.java b/java/src/main/java/org/msgpack/UnpackCursor.java new file mode 100644 index 0000000..33f4258 --- /dev/null +++ b/java/src/main/java/org/msgpack/UnpackCursor.java @@ -0,0 +1,96 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +import java.io.IOException; + +public class UnpackCursor { + private Unpacker pac; + private int offset; + + UnpackCursor(Unpacker pac, int offset) + { + this.pac = pac; + this.offset = offset; + } + + final void advance(int length) { + offset += length; + } + + final int getOffset() { + return offset; + } + + public byte unpackByte() throws IOException, MessageTypeException { + return pac.impl.unpackByte(this, offset); + } + + public short unpackShort() throws IOException, MessageTypeException { + return pac.impl.unpackShort(this, offset); + } + + public int unpackInt() throws IOException, MessageTypeException { + return pac.impl.unpackInt(this, offset); + } + + public long unpackLong() throws IOException, MessageTypeException { + return pac.impl.unpackLong(this, offset); + } + + public float unpackFloat() throws IOException, MessageTypeException { + return pac.impl.unpackFloat(this, offset); + } + + public double unpackDouble() throws IOException, MessageTypeException { + return pac.impl.unpackDouble(this, offset); + } + + public Object unpackNull() throws IOException, MessageTypeException { + return pac.impl.unpackNull(this, offset); + } + + public boolean unpackBoolean() throws IOException, MessageTypeException { + return pac.impl.unpackBoolean(this, offset); + } + + public int unpackArray() throws IOException, MessageTypeException { + return pac.impl.unpackArray(this, offset); + } + + public int unpackMap() throws IOException, MessageTypeException { + return pac.impl.unpackMap(this, offset); + } + + public int unpackRaw() throws IOException, MessageTypeException { + return pac.impl.unpackRaw(this, offset); + } + + public byte[] unpackRawBody(int length) throws IOException, MessageTypeException { + return pac.impl.unpackRawBody(this, offset, length); + } + + public String unpackString() throws IOException, MessageTypeException { + return pac.impl.unpackString(this, offset); + } + + public void commit() { + pac.setOffset(offset); + } +} + diff --git a/java/src/main/java/org/msgpack/UnpackIterator.java b/java/src/main/java/org/msgpack/UnpackIterator.java index 0a78e83..5cc994d 100644 --- a/java/src/main/java/org/msgpack/UnpackIterator.java +++ b/java/src/main/java/org/msgpack/UnpackIterator.java @@ -21,41 +21,27 @@ import java.io.IOException; import java.util.Iterator; import java.util.NoSuchElementException; -public class UnpackIterator implements Iterator { +public class UnpackIterator extends UnpackResult implements Iterator { private Unpacker pac; - private boolean have; - private Object data; UnpackIterator(Unpacker pac) { + super(); this.pac = pac; - this.have = false; } public boolean hasNext() { - if(have) { return true; } try { - while(true) { - if(pac.execute()) { - data = pac.getData(); - pac.reset(); - have = true; - return true; - } - - if(!pac.fill()) { - return false; - } - } + return pac.next(this); } catch (IOException e) { return false; } } public Object next() { - if(!have && !hasNext()) { + if(!finished && !hasNext()) { throw new NoSuchElementException(); } - have = false; + finished = false; return data; } diff --git a/java/src/main/java/org/msgpack/UnpackResult.java b/java/src/main/java/org/msgpack/UnpackResult.java new file mode 100644 index 0000000..cec18a1 --- /dev/null +++ b/java/src/main/java/org/msgpack/UnpackResult.java @@ -0,0 +1,42 @@ +// +// MessagePack for Java +// +// Copyright (C) 2009-2010 FURUHASHI Sadayuki +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +package org.msgpack; + +public class UnpackResult { + protected boolean finished = false; + protected Object data = null; + + public boolean isFinished() { + return finished; + } + + public Object getData() { + return data; + } + + public void reset() { + finished = false; + data = null; + } + + void done(Object obj) { + finished = true; + data = obj; + } +} + diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java index c8a8823..20a8c4a 100644 --- a/java/src/main/java/org/msgpack/Unpacker.java +++ b/java/src/main/java/org/msgpack/Unpacker.java @@ -22,18 +22,43 @@ import java.io.InputStream; import java.io.IOException; import java.util.Iterator; import java.nio.ByteBuffer; -import org.msgpack.impl.UnpackerImpl; -public class Unpacker extends UnpackerImpl implements Iterable { +public class Unpacker implements Iterable { - public static final int DEFAULT_BUFFER_SIZE = 32*1024; + // buffer: + // +---------------------------------------------+ + // | [object] | [obje| unparsed ... | unused ...| + // +---------------------------------------------+ + // ^ parsed + // ^ offset + // ^ filled + // ^ buffer.length + + private static final int DEFAULT_BUFFER_SIZE = 32*1024; + + protected int offset; + protected int parsed; + protected int bufferReserveSize; + protected InputStream stream; + + class BufferedUnpackerMixin extends BufferedUnpackerImpl { + boolean fill() throws IOException { + if(stream == null) { + return false; + } + reserveBuffer(bufferReserveSize); + int rl = stream.read(buffer, filled, buffer.length - filled); + // equals: stream.read(getBuffer(), getBufferOffset(), getBufferCapacity()); + if(rl <= 0) { + return false; + } + bufferConsumed(rl); + return true; + } + }; + + final BufferedUnpackerMixin impl = new BufferedUnpackerMixin(); - private int used; - private int offset; - private int parsed; - private byte[] buffer; - private int bufferReserveSize; - private InputStream stream; public Unpacker() { this(DEFAULT_BUFFER_SIZE); @@ -48,67 +73,31 @@ public class Unpacker extends UnpackerImpl implements Iterable { } public Unpacker(InputStream stream, int bufferReserveSize) { - super(); - this.used = 0; this.offset = 0; this.parsed = 0; - this.buffer = new byte[bufferReserveSize]; this.bufferReserveSize = bufferReserveSize/2; this.stream = stream; } public Unpacker useSchema(Schema s) { - super.setSchema(s); + impl.setSchema(s); return this; } - public void reserveBuffer(int size) { - if(buffer.length - used >= size) { - return; - } - /* - if(used == parsed && buffer.length >= size) { - // rewind buffer - used = 0; - offset = 0; - return; - } - */ - int nextSize = buffer.length * 2; - while(nextSize < size + used) { - nextSize *= 2; - } - - byte[] tmp = new byte[nextSize]; - System.arraycopy(buffer, offset, tmp, 0, used - offset); - - buffer = tmp; - used -= offset; - offset = 0; + public InputStream getStream() { + return this.stream; } - public byte[] getBuffer() { - return buffer; - } - - public int getBufferOffset() { - return used; - } - - public int getBufferCapacity() { - return buffer.length - used; - } - - public void bufferConsumed(int size) { - used += size; + public void setStream(InputStream stream) { + this.stream = stream; } public void feed(ByteBuffer buffer) { int length = buffer.remaining(); if (length == 0) return; reserveBuffer(length); - buffer.get(this.buffer, this.offset, length); + buffer.get(impl.buffer, this.offset, length); bufferConsumed(length); } @@ -118,48 +107,116 @@ public class Unpacker extends UnpackerImpl implements Iterable { public void feed(byte[] buffer, int offset, int length) { reserveBuffer(length); - System.arraycopy(buffer, offset, this.buffer, this.offset, length); + System.arraycopy(buffer, offset, impl.buffer, this.offset, length); bufferConsumed(length); } public boolean fill() throws IOException { - if(stream == null) { - return false; - } - reserveBuffer(bufferReserveSize); - int rl = stream.read(getBuffer(), getBufferOffset(), getBufferCapacity()); - if(rl <= 0) { - return false; - } - bufferConsumed(rl); - return true; + return impl.fill(); } public Iterator iterator() { return new UnpackIterator(this); } + public UnpackResult next() throws IOException, UnpackException { + UnpackResult result = new UnpackResult(); + this.offset = impl.next(this.offset, result); + return result; + } + + public boolean next(UnpackResult result) throws IOException, UnpackException { + this.offset = impl.next(this.offset, result); + return result.isFinished(); + } + + + public void reserveBuffer(int require) { + if(impl.buffer == null) { + int nextSize = (bufferReserveSize < require) ? require : bufferReserveSize; + impl.buffer = new byte[nextSize]; + return; + } + + if(impl.buffer.length - impl.filled >= require) { + return; + } + + int nextSize = impl.buffer.length * 2; + int notParsed = impl.filled - this.offset; + while(nextSize < require + notParsed) { + nextSize *= 2; + } + + byte[] tmp = new byte[nextSize]; + System.arraycopy(impl.buffer, this.offset, tmp, 0, impl.filled - this.offset); + + impl.buffer = tmp; + impl.filled = notParsed; + this.offset = 0; + } + + public byte[] getBuffer() { + return impl.buffer; + } + + public int getBufferOffset() { + return impl.filled; + } + + public int getBufferCapacity() { + return impl.buffer.length - impl.filled; + } + + public void bufferConsumed(int size) { + impl.filled += size; + } + public boolean execute() throws UnpackException { - int noffset = super.execute(buffer, offset, used); + int noffset = impl.execute(impl.buffer, offset, impl.filled); if(noffset <= offset) { return false; } parsed += noffset - offset; offset = noffset; - return super.isFinished(); + return impl.isFinished(); + } + + + public int execute(byte[] buffer) throws UnpackException { + return execute(buffer, 0, buffer.length); + } + + public int execute(byte[] buffer, int offset, int length) throws UnpackException { + int noffset = impl.execute(buffer, offset + this.offset, length); + this.offset = noffset - offset; + if(impl.isFinished()) { + impl.resetState(); + } + return noffset; + } + + public boolean isFinished() { + return impl.isFinished(); } public Object getData() { - return super.getData(); + return impl.getData(); } public void reset() { - super.reset(); - parsed = 0; + impl.reset(); } + + public UnpackCursor begin() + { + return new UnpackCursor(this, offset); + } + + public int getMessageSize() { - return parsed - offset + used; + return parsed - offset + impl.filled; } public int getParsedSize() { @@ -167,7 +224,7 @@ public class Unpacker extends UnpackerImpl implements Iterable { } public int getNonParsedSize() { - return used - offset; + return impl.filled - offset; } public void skipNonparsedBuffer(int size) { @@ -175,80 +232,14 @@ public class Unpacker extends UnpackerImpl implements Iterable { } public void removeNonparsedBuffer() { - used = offset; + impl.filled = offset; } - /* - public static class Context { - private boolean finished; - private Object data; - private int offset; - private UnpackerImpl impl; - public Context() - { - this.finished = false; - this.impl = new UnpackerImpl(); - } - - public boolean isFinished() - { - return finished; - } - - public Object getData() - { - return data; - } - - int getOffset() - { - return offset; - } - - void setFinished(boolean finished) - { - this.finished = finished; - } - - void setData(Object data) - { - this.data = data; - } - - void setOffset(int offset) - { - this.offset = offset; - } - - UnpackerImpl getImpl() - { - return impl; - } - } - - public static int unpack(Context ctx, byte[] buffer) throws UnpackException + void setOffset(int offset) { - return unpack(ctx, buffer, 0, buffer.length); + parsed += offset - this.offset; + this.offset = offset; } - - public static int unpack(Context ctx, byte[] buffer, int offset, int length) throws UnpackException - { - UnpackerImpl impl = ctx.getImpl(); - int noffset = impl.execute(buffer, offset + ctx.getOffset(), length); - ctx.setOffset(noffset - offset); - if(impl.isFinished()) { - ctx.setData(impl.getData()); - ctx.setFinished(false); - impl.reset(); - } else { - ctx.setData(null); - ctx.setFinished(true); - } - int parsed = noffset - offset; - ctx.setOffset(parsed); - return noffset; - } - */ } diff --git a/java/src/main/java/org/msgpack/impl/UnpackerImpl.java b/java/src/main/java/org/msgpack/UnpackerImpl.java similarity index 97% rename from java/src/main/java/org/msgpack/impl/UnpackerImpl.java rename to java/src/main/java/org/msgpack/UnpackerImpl.java index adc62b0..ae01289 100644 --- a/java/src/main/java/org/msgpack/impl/UnpackerImpl.java +++ b/java/src/main/java/org/msgpack/UnpackerImpl.java @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -package org.msgpack.impl; +package org.msgpack; import java.nio.ByteBuffer; //import java.math.BigInteger; @@ -47,7 +47,7 @@ public class UnpackerImpl { static final int CT_MAP_KEY = 0x01; static final int CT_MAP_VALUE = 0x02; - static final int MAX_STACK_SIZE = 16; + static final int MAX_STACK_SIZE = 32; private int cs; private int trail; @@ -67,41 +67,45 @@ public class UnpackerImpl { private static final Schema GENERIC_SCHEMA = new GenericSchema(); private Schema rootSchema; - protected UnpackerImpl() + public UnpackerImpl() { setSchema(GENERIC_SCHEMA); } - protected void setSchema(Schema schema) + public void setSchema(Schema schema) { this.rootSchema = schema; reset(); } - protected Object getData() + public final Object getData() { return data; } - protected boolean isFinished() + public final boolean isFinished() { return finished; } - protected void reset() - { + public final void resetState() { cs = CS_HEADER; top = -1; - finished = false; - data = null; top_ct = 0; top_count = 0; top_obj = null; top_schema = rootSchema; } + public final void reset() + { + resetState(); + finished = false; + data = null; + } + @SuppressWarnings("unchecked") - protected int execute(byte[] src, int off, int length) throws UnpackException + public final int execute(byte[] src, int off, int length) throws UnpackException { if(off >= length) { return off; } diff --git a/java/src/test/java/org/msgpack/TestDirectConversion.java b/java/src/test/java/org/msgpack/TestDirectConversion.java new file mode 100644 index 0000000..d77fe13 --- /dev/null +++ b/java/src/test/java/org/msgpack/TestDirectConversion.java @@ -0,0 +1,154 @@ +package org.msgpack; + +import org.msgpack.*; +import java.io.*; +import java.util.*; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class TestDirectConversion { + private UnpackCursor prepareCursor(ByteArrayOutputStream out) { + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker upk = new Unpacker(in); + return upk.begin(); + } + + @Test + public void testInt() throws Exception { + testInt(0); + testInt(-1); + testInt(1); + testInt(Integer.MIN_VALUE); + testInt(Integer.MAX_VALUE); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testInt(rand.nextInt()); + } + public void testInt(int val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + UnpackCursor c = prepareCursor(out); + assertEquals(val, c.unpackInt()); + c.commit(); + } + + @Test + public void testFloat() throws Exception { + testFloat((float)0.0); + testFloat((float)-0.0); + testFloat((float)1.0); + testFloat((float)-1.0); + testFloat((float)Float.MAX_VALUE); + testFloat((float)Float.MIN_VALUE); + testFloat((float)Float.NaN); + testFloat((float)Float.NEGATIVE_INFINITY); + testFloat((float)Float.POSITIVE_INFINITY); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testFloat(rand.nextFloat()); + } + public void testFloat(float val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + UnpackCursor c = prepareCursor(out); + float f = c.unpackFloat(); + if(Float.isNaN(val)) { + assertTrue(Float.isNaN(f)); + } else { + assertEquals(val, f, 10e-10); + } + c.commit(); + } + + @Test + public void testDouble() throws Exception { + testDouble((double)0.0); + testDouble((double)-0.0); + testDouble((double)1.0); + testDouble((double)-1.0); + testDouble((double)Double.MAX_VALUE); + testDouble((double)Double.MIN_VALUE); + testDouble((double)Double.NaN); + testDouble((double)Double.NEGATIVE_INFINITY); + testDouble((double)Double.POSITIVE_INFINITY); + Random rand = new Random(); + for (int i = 0; i < 1000; i++) + testDouble(rand.nextDouble()); + } + public void testDouble(double val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + UnpackCursor c = prepareCursor(out); + double f = c.unpackDouble(); + if(Double.isNaN(val)) { + assertTrue(Double.isNaN(f)); + } else { + assertEquals(val, f, 10e-10); + } + c.commit(); + } + + @Test + public void testNil() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).packNil(); + UnpackCursor c = prepareCursor(out); + assertEquals(null, c.unpackNull()); + c.commit(); + } + + @Test + public void testBoolean() throws Exception { + testBoolean(false); + testBoolean(true); + } + public void testBoolean(boolean val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + UnpackCursor c = prepareCursor(out); + assertEquals(val, c.unpackBoolean()); + c.commit(); + } + + @Test + public void testString() throws Exception { + testString(""); + testString("a"); + testString("ab"); + testString("abc"); + // small size string + for (int i = 0; i < 100; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 31 + 1; + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + // medium size string + for (int i = 0; i < 100; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 100 + (1 << 15); + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + // large size string + for (int i = 0; i < 10; i++) { + StringBuilder sb = new StringBuilder(); + int len = (int)Math.random() % 100 + (1 << 31); + for (int j = 0; j < len; j++) + sb.append('a' + ((int)Math.random()) & 26); + testString(sb.toString()); + } + } + public void testString(String val) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + new Packer(out).pack(val); + UnpackCursor c = prepareCursor(out); + assertEquals(val, c.unpackString()); + c.commit(); + } + + // FIXME container types +}; diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java index 6877853..a16b5b1 100644 --- a/java/src/test/java/org/msgpack/TestPackUnpack.java +++ b/java/src/test/java/org/msgpack/TestPackUnpack.java @@ -237,5 +237,5 @@ public class TestPackUnpack { System.out.println("Got unexpected class: " + obj.getClass()); assertTrue(false); } - } + } }; From 135a9f558600ddbd4cd0d07a57ae1f7fb5b8634a Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 20 May 2010 05:44:44 +0900 Subject: [PATCH 0478/1648] java: fix direct conversion API --- .../org/msgpack/BufferedUnpackerImpl.java | 205 ++++++++++-------- .../java/org/msgpack/MessageUnpackable.java | 2 +- .../main/java/org/msgpack/UnpackCursor.java | 96 -------- .../main/java/org/msgpack/UnpackIterator.java | 1 + java/src/main/java/org/msgpack/Unpacker.java | 102 ++++++--- .../org/msgpack/schema/ClassGenerator.java | 10 +- .../msgpack/schema/SpecificClassSchema.java | 4 +- .../org/msgpack/TestDirectConversion.java | 42 ++-- java/test/README | 2 +- .../tpc/src/serializers/BenchmarkRunner.java | 2 + .../src/serializers/msgpack/MediaContent.java | 78 ++++++- .../msgpack/MessagePackDirectSerializer.java | 68 ++++++ .../msgpack/MessagePackDynamicSerializer.java | 2 +- .../msgpack/MessagePackGenericSerializer.java | 2 +- .../MessagePackIndirectSerializer.java | 2 +- .../MessagePackSpecificSerializer.java | 2 +- 16 files changed, 356 insertions(+), 264 deletions(-) delete mode 100644 java/src/main/java/org/msgpack/UnpackCursor.java create mode 100644 java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDirectSerializer.java diff --git a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java index 0ef9c12..5fde4e1 100644 --- a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java +++ b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java @@ -22,16 +22,17 @@ import java.nio.ByteBuffer; //import java.math.BigInteger; abstract class BufferedUnpackerImpl extends UnpackerImpl { + int offset = 0; int filled = 0; byte[] buffer = null; private ByteBuffer castBuffer = ByteBuffer.allocate(8); abstract boolean fill() throws IOException; - final int next(int offset, UnpackResult result) throws IOException, UnpackException { + final boolean next(UnpackResult result) throws IOException, UnpackException { if(filled == 0) { if(!fill()) { - return offset; + return false; } } @@ -39,8 +40,9 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { int noffset = super.execute(buffer, offset, filled); if(noffset <= offset) { if(!fill()) { - return offset; + return false; } + continue; } offset = noffset; } while(!super.isFinished()); @@ -49,10 +51,10 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { super.reset(); result.done(obj); - return offset; + return true; } - private final void more(int offset, int require) throws IOException, UnpackException { + private final void more(int require) throws IOException, UnpackException { while(filled - offset < require) { if(!fill()) { // FIXME @@ -61,41 +63,46 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { } } - final byte unpackByte(UnpackCursor c, int offset) throws IOException, MessageTypeException { - int o = unpackInt(c, offset); + private final void advance(int length) { + offset += length; + } + + final byte unpackByte() throws IOException, MessageTypeException { + int o = unpackInt(); if(0x7f < o || o < -0x80) { throw new MessageTypeException(); } return (byte)o; } - final short unpackShort(UnpackCursor c, int offset) throws IOException, MessageTypeException { - int o = unpackInt(c, offset); + final short unpackShort() throws IOException, MessageTypeException { + int o = unpackInt(); if(0x7fff < o || o < -0x8000) { throw new MessageTypeException(); } return (short)o; } - final int unpackInt(UnpackCursor c, int offset) throws IOException, MessageTypeException { - more(offset, 1); + final int unpackInt() throws IOException, MessageTypeException { + more(1); int b = buffer[offset]; if((b & 0x80) == 0 || (b & 0xe0) == 0xe0) { // Fixnum + advance(1); return (int)b; } switch(b & 0xff) { case 0xcc: // unsigned int 8 - more(offset, 2); - c.advance(2); + more(2); + advance(2); return (int)((short)buffer[offset+1] & 0xff); case 0xcd: // unsigned int 16 - more(offset, 3); + more(3); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 2); - c.advance(3); + advance(3); return (int)((int)castBuffer.getShort(0) & 0xffff); case 0xce: // unsigned int 32 - more(offset, 5); + more(5); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 4); { @@ -103,11 +110,11 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { if(o < 0) { throw new MessageTypeException(); } - c.advance(5); + advance(5); return o; } case 0xcf: // unsigned int 64 - more(offset, 9); + more(9); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 8); { @@ -115,27 +122,27 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { if(o < 0 || o > 0x7fffffffL) { throw new MessageTypeException(); } - c.advance(9); + advance(9); return (int)o; } case 0xd0: // signed int 8 - more(offset, 2); - c.advance(2); + more(2); + advance(2); return (int)buffer[offset+1]; case 0xd1: // signed int 16 - more(offset, 3); + more(3); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 2); - c.advance(3); + advance(3); return (int)castBuffer.getShort(0); case 0xd2: // signed int 32 - more(offset, 4); + more(4); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 4); - c.advance(4); + advance(4); return (int)castBuffer.getInt(0); case 0xd3: // signed int 64 - more(offset, 9); + more(9); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 8); { @@ -143,7 +150,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { if(0x7fffffffL < o || o < -0x80000000L) { throw new MessageTypeException(); } - c.advance(9); + advance(9); return (int)o; } default: @@ -151,31 +158,32 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { } } - final long unpackLong(UnpackCursor c, int offset) throws IOException, MessageTypeException { - more(offset, 1); + final long unpackLong() throws IOException, MessageTypeException { + more(1); int b = buffer[offset]; if((b & 0x80) == 0 || (b & 0xe0) == 0xe0) { // Fixnum + advance(1); return (long)b; } switch(b & 0xff) { case 0xcc: // unsigned int 8 - more(offset, 2); - c.advance(2); + more(2); + advance(2); return (long)((short)buffer[offset+1] & 0xff); case 0xcd: // unsigned int 16 - more(offset, 3); + more(3); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 2); - c.advance(3); + advance(3); return (long)((int)castBuffer.getShort(0) & 0xffff); case 0xce: // unsigned int 32 - more(offset, 5); + more(5); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 4); - c.advance(5); + advance(5); return ((long)castBuffer.getInt(0) & 0xffffffffL); case 0xcf: // unsigned int 64 - more(offset, 9); + more(9); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 8); { @@ -184,51 +192,51 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { // FIXME throw new MessageTypeException("uint 64 bigger than 0x7fffffff is not supported"); } - c.advance(9); + advance(9); return o; } case 0xd0: // signed int 8 - more(offset, 2); - c.advance(2); + more(2); + advance(2); return (long)buffer[offset+1]; case 0xd1: // signed int 16 - more(offset, 3); + more(3); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 2); - c.advance(3); + advance(3); return (long)castBuffer.getShort(0); case 0xd2: // signed int 32 - more(offset, 4); + more(4); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 4); - c.advance(4); + advance(4); return (long)castBuffer.getInt(0); case 0xd3: // signed int 64 - more(offset, 9); + more(9); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 8); - c.advance(9); + advance(9); return (long)castBuffer.getLong(0); default: throw new MessageTypeException(); } } - final float unpackFloat(UnpackCursor c, int offset) throws IOException, MessageTypeException { - more(offset, 1); + final float unpackFloat() throws IOException, MessageTypeException { + more(1); int b = buffer[offset]; switch(b & 0xff) { case 0xca: // float - more(offset, 5); + more(5); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 4); - c.advance(5); + advance(5); return castBuffer.getFloat(0); case 0xcb: // double - more(offset, 9); + more(9); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 8); - c.advance(9); + advance(9); // FIXME overflow check return (float)castBuffer.getDouble(0); default: @@ -236,70 +244,70 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { } } - final double unpackDouble(UnpackCursor c, int offset) throws IOException, MessageTypeException { - more(offset, 1); + final double unpackDouble() throws IOException, MessageTypeException { + more(1); int b = buffer[offset]; switch(b & 0xff) { case 0xca: // float - more(offset, 5); + more(5); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 4); - c.advance(5); + advance(5); return (double)castBuffer.getFloat(0); case 0xcb: // double - more(offset, 9); + more(9); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 8); - c.advance(9); + advance(9); return castBuffer.getDouble(0); default: throw new MessageTypeException(); } } - final Object unpackNull(UnpackCursor c, int offset) throws IOException, MessageTypeException { - more(offset, 1); + final Object unpackNull() throws IOException, MessageTypeException { + more(1); int b = buffer[offset] & 0xff; if(b != 0xc0) { // nil throw new MessageTypeException(); } - c.advance(1); + advance(1); return null; } - final boolean unpackBoolean(UnpackCursor c, int offset) throws IOException, MessageTypeException { - more(offset, 1); + final boolean unpackBoolean() throws IOException, MessageTypeException { + more(1); int b = buffer[offset] & 0xff; if(b == 0xc2) { // false - c.advance(1); + advance(1); return false; } else if(b == 0xc3) { // true - c.advance(1); + advance(1); return true; } else { throw new MessageTypeException(); } } - final int unpackArray(UnpackCursor c, int offset) throws IOException, MessageTypeException { - more(offset, 1); + final int unpackArray() throws IOException, MessageTypeException { + more(1); int b = buffer[offset]; if((b & 0xf0) == 0x90) { // FixArray - c.advance(1); + advance(1); return (int)(b & 0x0f); } switch(b & 0xff) { case 0xdc: // array 16 - more(offset, 3); + more(3); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 2); - c.advance(3); + advance(3); return (int)castBuffer.getShort(0) & 0xffff; case 0xdd: // array 32 - more(offset, 5); + more(5); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 4); - c.advance(5); + advance(5); // FIXME overflow check return castBuffer.getInt(0) & 0x7fffffff; default: @@ -307,25 +315,25 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { } } - final int unpackMap(UnpackCursor c, int offset) throws IOException, MessageTypeException { - more(offset, 1); + final int unpackMap() throws IOException, MessageTypeException { + more(1); int b = buffer[offset]; if((b & 0xf0) == 0x80) { // FixMap - c.advance(1); + advance(1); return (int)(b & 0x0f); } switch(b & 0xff) { case 0xde: // map 16 - more(offset, 3); + more(3); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 2); - c.advance(3); + advance(3); return (int)castBuffer.getShort(0) & 0xffff; case 0xdf: // map 32 - more(offset, 5); + more(5); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 4); - c.advance(5); + advance(5); // FIXME overflow check return castBuffer.getInt(0) & 0x7fffffff; default: @@ -333,25 +341,25 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { } } - final int unpackRaw(UnpackCursor c, int offset) throws IOException, MessageTypeException { - more(offset, 1); + final int unpackRaw() throws IOException, MessageTypeException { + more(1); int b = buffer[offset]; if((b & 0xe0) == 0xa0) { // FixRaw - c.advance(1); - return (int)(b & 0x0f); + advance(1); + return (int)(b & 0x1f); } switch(b & 0xff) { case 0xda: // raw 16 - more(offset, 3); + more(3); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 2); - c.advance(3); + advance(3); return (int)castBuffer.getShort(0) & 0xffff; case 0xdb: // raw 32 - more(offset, 5); + more(5); castBuffer.rewind(); castBuffer.put(buffer, offset+1, 4); - c.advance(5); + advance(5); // FIXME overflow check return castBuffer.getInt(0) & 0x7fffffff; default: @@ -359,26 +367,35 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { } } - final byte[] unpackRawBody(UnpackCursor c, int offset, int length) throws IOException, MessageTypeException { - more(offset, length); + final byte[] unpackRawBody(int length) throws IOException, MessageTypeException { + more(length); byte[] bytes = new byte[length]; System.arraycopy(buffer, offset, bytes, 0, length); - c.advance(length); + advance(length); return bytes; } - final String unpackString(UnpackCursor c, int offset) throws IOException, MessageTypeException { - int length = unpackRaw(c, offset); - offset = c.getOffset(); - more(offset, length); + final String unpackString() throws IOException, MessageTypeException { + int length = unpackRaw(); + more(length); String s; try { s = new String(buffer, offset, length, "UTF-8"); } catch (Exception e) { throw new MessageTypeException(); } - c.advance(length); + advance(length); return s; } + + final Object unpackObject() throws IOException, MessageTypeException { + // FIXME save state, restore state + UnpackResult result = new UnpackResult(); + if(!next(result)) { + super.reset(); + throw new MessageTypeException(); + } + return result.getData(); + } } diff --git a/java/src/main/java/org/msgpack/MessageUnpackable.java b/java/src/main/java/org/msgpack/MessageUnpackable.java index 20e4d56..cc206e7 100644 --- a/java/src/main/java/org/msgpack/MessageUnpackable.java +++ b/java/src/main/java/org/msgpack/MessageUnpackable.java @@ -20,6 +20,6 @@ package org.msgpack; import java.io.IOException; public interface MessageUnpackable { - public void messageUnpack(Unpacker pk) throws IOException, MessageTypeException; + public void messageUnpack(Unpacker pac) throws IOException, MessageTypeException; } diff --git a/java/src/main/java/org/msgpack/UnpackCursor.java b/java/src/main/java/org/msgpack/UnpackCursor.java deleted file mode 100644 index 33f4258..0000000 --- a/java/src/main/java/org/msgpack/UnpackCursor.java +++ /dev/null @@ -1,96 +0,0 @@ -// -// MessagePack for Java -// -// Copyright (C) 2009-2010 FURUHASHI Sadayuki -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -package org.msgpack; - -import java.io.IOException; - -public class UnpackCursor { - private Unpacker pac; - private int offset; - - UnpackCursor(Unpacker pac, int offset) - { - this.pac = pac; - this.offset = offset; - } - - final void advance(int length) { - offset += length; - } - - final int getOffset() { - return offset; - } - - public byte unpackByte() throws IOException, MessageTypeException { - return pac.impl.unpackByte(this, offset); - } - - public short unpackShort() throws IOException, MessageTypeException { - return pac.impl.unpackShort(this, offset); - } - - public int unpackInt() throws IOException, MessageTypeException { - return pac.impl.unpackInt(this, offset); - } - - public long unpackLong() throws IOException, MessageTypeException { - return pac.impl.unpackLong(this, offset); - } - - public float unpackFloat() throws IOException, MessageTypeException { - return pac.impl.unpackFloat(this, offset); - } - - public double unpackDouble() throws IOException, MessageTypeException { - return pac.impl.unpackDouble(this, offset); - } - - public Object unpackNull() throws IOException, MessageTypeException { - return pac.impl.unpackNull(this, offset); - } - - public boolean unpackBoolean() throws IOException, MessageTypeException { - return pac.impl.unpackBoolean(this, offset); - } - - public int unpackArray() throws IOException, MessageTypeException { - return pac.impl.unpackArray(this, offset); - } - - public int unpackMap() throws IOException, MessageTypeException { - return pac.impl.unpackMap(this, offset); - } - - public int unpackRaw() throws IOException, MessageTypeException { - return pac.impl.unpackRaw(this, offset); - } - - public byte[] unpackRawBody(int length) throws IOException, MessageTypeException { - return pac.impl.unpackRawBody(this, offset, length); - } - - public String unpackString() throws IOException, MessageTypeException { - return pac.impl.unpackString(this, offset); - } - - public void commit() { - pac.setOffset(offset); - } -} - diff --git a/java/src/main/java/org/msgpack/UnpackIterator.java b/java/src/main/java/org/msgpack/UnpackIterator.java index 5cc994d..f17e229 100644 --- a/java/src/main/java/org/msgpack/UnpackIterator.java +++ b/java/src/main/java/org/msgpack/UnpackIterator.java @@ -30,6 +30,7 @@ public class UnpackIterator extends UnpackResult implements Iterator { } public boolean hasNext() { + if(finished) { return true; } try { return pac.next(this); } catch (IOException e) { diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java index 20a8c4a..4d8da7b 100644 --- a/java/src/main/java/org/msgpack/Unpacker.java +++ b/java/src/main/java/org/msgpack/Unpacker.java @@ -36,7 +36,6 @@ public class Unpacker implements Iterable { private static final int DEFAULT_BUFFER_SIZE = 32*1024; - protected int offset; protected int parsed; protected int bufferReserveSize; protected InputStream stream; @@ -73,7 +72,6 @@ public class Unpacker implements Iterable { } public Unpacker(InputStream stream, int bufferReserveSize) { - this.offset = 0; this.parsed = 0; this.bufferReserveSize = bufferReserveSize/2; this.stream = stream; @@ -97,7 +95,7 @@ public class Unpacker implements Iterable { int length = buffer.remaining(); if (length == 0) return; reserveBuffer(length); - buffer.get(impl.buffer, this.offset, length); + buffer.get(impl.buffer, impl.offset, length); bufferConsumed(length); } @@ -107,7 +105,7 @@ public class Unpacker implements Iterable { public void feed(byte[] buffer, int offset, int length) { reserveBuffer(length); - System.arraycopy(buffer, offset, impl.buffer, this.offset, length); + System.arraycopy(buffer, offset, impl.buffer, impl.offset, length); bufferConsumed(length); } @@ -121,13 +119,12 @@ public class Unpacker implements Iterable { public UnpackResult next() throws IOException, UnpackException { UnpackResult result = new UnpackResult(); - this.offset = impl.next(this.offset, result); + impl.next(result); return result; } public boolean next(UnpackResult result) throws IOException, UnpackException { - this.offset = impl.next(this.offset, result); - return result.isFinished(); + return impl.next(result); } @@ -143,17 +140,17 @@ public class Unpacker implements Iterable { } int nextSize = impl.buffer.length * 2; - int notParsed = impl.filled - this.offset; + int notParsed = impl.filled - impl.offset; while(nextSize < require + notParsed) { nextSize *= 2; } byte[] tmp = new byte[nextSize]; - System.arraycopy(impl.buffer, this.offset, tmp, 0, impl.filled - this.offset); + System.arraycopy(impl.buffer, impl.offset, tmp, 0, impl.filled - impl.offset); impl.buffer = tmp; impl.filled = notParsed; - this.offset = 0; + impl.offset = 0; } public byte[] getBuffer() { @@ -173,12 +170,12 @@ public class Unpacker implements Iterable { } public boolean execute() throws UnpackException { - int noffset = impl.execute(impl.buffer, offset, impl.filled); - if(noffset <= offset) { + int noffset = impl.execute(impl.buffer, impl.offset, impl.filled); + if(noffset <= impl.offset) { return false; } - parsed += noffset - offset; - offset = noffset; + parsed += noffset - impl.offset; + impl.offset = noffset; return impl.isFinished(); } @@ -188,8 +185,8 @@ public class Unpacker implements Iterable { } public int execute(byte[] buffer, int offset, int length) throws UnpackException { - int noffset = impl.execute(buffer, offset + this.offset, length); - this.offset = noffset - offset; + int noffset = impl.execute(buffer, offset + impl.offset, length); + impl.offset = noffset - offset; if(impl.isFinished()) { impl.resetState(); } @@ -208,15 +205,8 @@ public class Unpacker implements Iterable { impl.reset(); } - - public UnpackCursor begin() - { - return new UnpackCursor(this, offset); - } - - public int getMessageSize() { - return parsed - offset + impl.filled; + return parsed - impl.offset + impl.filled; } public int getParsedSize() { @@ -224,22 +214,72 @@ public class Unpacker implements Iterable { } public int getNonParsedSize() { - return impl.filled - offset; + return impl.filled - impl.offset; } public void skipNonparsedBuffer(int size) { - offset += size; + impl.offset += size; } public void removeNonparsedBuffer() { - impl.filled = offset; + impl.filled = impl.offset; } - void setOffset(int offset) - { - parsed += offset - this.offset; - this.offset = offset; + final public byte unpackByte() throws IOException, MessageTypeException { + return impl.unpackByte(); + } + + final public short unpackShort() throws IOException, MessageTypeException { + return impl.unpackShort(); + } + + final public int unpackInt() throws IOException, MessageTypeException { + return impl.unpackInt(); + } + + final public long unpackLong() throws IOException, MessageTypeException { + return impl.unpackLong(); + } + + final public float unpackFloat() throws IOException, MessageTypeException { + return impl.unpackFloat(); + } + + final public double unpackDouble() throws IOException, MessageTypeException { + return impl.unpackDouble(); + } + + final public Object unpackNull() throws IOException, MessageTypeException { + return impl.unpackNull(); + } + + final public boolean unpackBoolean() throws IOException, MessageTypeException { + return impl.unpackBoolean(); + } + + final public int unpackArray() throws IOException, MessageTypeException { + return impl.unpackArray(); + } + + final public int unpackMap() throws IOException, MessageTypeException { + return impl.unpackMap(); + } + + final public int unpackRaw() throws IOException, MessageTypeException { + return impl.unpackRaw(); + } + + final public byte[] unpackRawBody(int length) throws IOException, MessageTypeException { + return impl.unpackRawBody(length); + } + + final public String unpackString() throws IOException, MessageTypeException { + return impl.unpackString(); + } + + final public Object unpackObject() throws IOException, MessageTypeException { + return impl.unpackObject(); } } diff --git a/java/src/main/java/org/msgpack/schema/ClassGenerator.java b/java/src/main/java/org/msgpack/schema/ClassGenerator.java index 061dcbb..f8a13fa 100644 --- a/java/src/main/java/org/msgpack/schema/ClassGenerator.java +++ b/java/src/main/java/org/msgpack/schema/ClassGenerator.java @@ -105,7 +105,7 @@ public class ClassGenerator { private void writeClass() throws IOException { line(); - line("public final class "+schema.getName()+" implements MessagePackable, MessageMergeable"); + line("public final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); line("{"); pushIndent(); writeSchema(); @@ -117,7 +117,7 @@ public class ClassGenerator { private void writeSubclass() throws IOException { line(); - line("final class "+schema.getName()+" implements MessagePackable, MessageMergeable"); + line("final class "+schema.getName()+" implements MessagePackable, MessageConvertable"); line("{"); pushIndent(); writeSchema(); @@ -150,7 +150,7 @@ public class ClassGenerator { writeConstructors(); writeAccessors(); writePackFunction(); - writeMergeFunction(); + writeConvertFunction(); writeFactoryFunction(); } @@ -184,11 +184,11 @@ public class ClassGenerator { line("}"); } - private void writeMergeFunction() throws IOException { + private void writeConvertFunction() throws IOException { line(); line("@Override"); line("@SuppressWarnings(\"unchecked\")"); - line("public void messageMerge(Object obj) throws MessageTypeException"); + line("public void messageConvert(Object obj) throws MessageTypeException"); line("{"); pushIndent(); line("Object[] _source = ((List)obj).toArray();"); diff --git a/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java b/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java index 30bd9e1..850f621 100644 --- a/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java +++ b/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java @@ -59,8 +59,8 @@ public class SpecificClassSchema extends ClassSchema { cacheConstructor(); } try { - MessageMergeable o = (MessageMergeable)constructorCache.newInstance((Object[])null); - o.messageMerge(obj); + MessageConvertable o = (MessageConvertable)constructorCache.newInstance((Object[])null); + o.messageConvert(obj); return o; } catch (InvocationTargetException e) { throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage()); diff --git a/java/src/test/java/org/msgpack/TestDirectConversion.java b/java/src/test/java/org/msgpack/TestDirectConversion.java index d77fe13..77bbc58 100644 --- a/java/src/test/java/org/msgpack/TestDirectConversion.java +++ b/java/src/test/java/org/msgpack/TestDirectConversion.java @@ -8,12 +8,6 @@ import org.junit.Test; import static org.junit.Assert.*; public class TestDirectConversion { - private UnpackCursor prepareCursor(ByteArrayOutputStream out) { - ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); - Unpacker upk = new Unpacker(in); - return upk.begin(); - } - @Test public void testInt() throws Exception { testInt(0); @@ -28,9 +22,9 @@ public class TestDirectConversion { public void testInt(int val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(val); - UnpackCursor c = prepareCursor(out); - assertEquals(val, c.unpackInt()); - c.commit(); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker upk = new Unpacker(in); + assertEquals(val, upk.unpackInt()); } @Test @@ -51,14 +45,14 @@ public class TestDirectConversion { public void testFloat(float val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(val); - UnpackCursor c = prepareCursor(out); - float f = c.unpackFloat(); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker upk = new Unpacker(in); + float f = upk.unpackFloat(); if(Float.isNaN(val)) { assertTrue(Float.isNaN(f)); } else { assertEquals(val, f, 10e-10); } - c.commit(); } @Test @@ -79,23 +73,23 @@ public class TestDirectConversion { public void testDouble(double val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(val); - UnpackCursor c = prepareCursor(out); - double f = c.unpackDouble(); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker upk = new Unpacker(in); + double f = upk.unpackDouble(); if(Double.isNaN(val)) { assertTrue(Double.isNaN(f)); } else { assertEquals(val, f, 10e-10); } - c.commit(); } @Test public void testNil() throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).packNil(); - UnpackCursor c = prepareCursor(out); - assertEquals(null, c.unpackNull()); - c.commit(); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker upk = new Unpacker(in); + assertEquals(null, upk.unpackNull()); } @Test @@ -106,9 +100,9 @@ public class TestDirectConversion { public void testBoolean(boolean val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(val); - UnpackCursor c = prepareCursor(out); - assertEquals(val, c.unpackBoolean()); - c.commit(); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker upk = new Unpacker(in); + assertEquals(val, upk.unpackBoolean()); } @Test @@ -145,9 +139,9 @@ public class TestDirectConversion { public void testString(String val) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); new Packer(out).pack(val); - UnpackCursor c = prepareCursor(out); - assertEquals(val, c.unpackString()); - c.commit(); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + Unpacker upk = new Unpacker(in); + assertEquals(val, upk.unpackString()); } // FIXME container types diff --git a/java/test/README b/java/test/README index 4e16454..9b98d91 100644 --- a/java/test/README +++ b/java/test/README @@ -1,7 +1,7 @@ #!/bin/sh svn checkout -r114 http://thrift-protobuf-compare.googlecode.com/svn/trunk/ thrift-protobuf-compare-base cp -rf thrift-protobuf-compare/tpc thrift-protobuf-compare-base -cp ../dist/msgpack.jar thrift-protobuf-compare-base/tpc/lib/ +cp ../target/msgpack*.jar thrift-protobuf-compare-base/tpc/lib/msgpack.jar cd thrift-protobuf-compare-base/tpc/ ant compile ./run-benchmark.sh diff --git a/java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java index b17dfb2..fa88b6b 100644 --- a/java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java +++ b/java/test/thrift-protobuf-compare/tpc/src/serializers/BenchmarkRunner.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Set; import java.util.Map.Entry; +import serializers.msgpack.MessagePackDirectSerializer; import serializers.msgpack.MessagePackSpecificSerializer; import serializers.msgpack.MessagePackIndirectSerializer; import serializers.msgpack.MessagePackDynamicSerializer; @@ -39,6 +40,7 @@ public class BenchmarkRunner BenchmarkRunner runner = new BenchmarkRunner(); // binary codecs first + runner.addObjectSerializer(new MessagePackDirectSerializer()); runner.addObjectSerializer(new MessagePackSpecificSerializer()); runner.addObjectSerializer(new MessagePackIndirectSerializer()); runner.addObjectSerializer(new MessagePackDynamicSerializer()); diff --git a/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java index 5dfbc8d..e750b5a 100644 --- a/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java +++ b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MediaContent.java @@ -6,7 +6,7 @@ import org.msgpack.*; import org.msgpack.schema.ClassSchema; import org.msgpack.schema.FieldSchema; -public final class MediaContent implements MessagePackable, MessageMergeable +public final class MediaContent implements MessagePackable, MessageConvertable, MessageUnpackable { private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class MediaContent (package serializers.msgpack) (field image (array (class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int)))) (field media (class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))))"); public static ClassSchema getSchema() { return _SCHEMA; } @@ -27,7 +27,7 @@ public final class MediaContent implements MessagePackable, MessageMergeable @Override @SuppressWarnings("unchecked") - public void messageMerge(Object obj) throws MessageTypeException + public void messageConvert(Object obj) throws MessageTypeException { Object[] _source = ((List)obj).toArray(); FieldSchema[] _fields = _SCHEMA.getFields(); @@ -35,6 +35,23 @@ public final class MediaContent implements MessagePackable, MessageMergeable if(_source.length <= 1) { return; } this.media = (Media)_fields[1].getSchema().convert(_source[1]); } + @Override + public void messageUnpack(Unpacker _pac) throws IOException, MessageTypeException { + int _length = _pac.unpackArray(); + if(_length <= 0) { return; } + int _image_length = _pac.unpackArray(); + this.image = new ArrayList(_image_length); + for(int _i=0; _i < _image_length; ++_i) { + Image _image_i = new Image(); + _image_i.messageUnpack(_pac); + this.image.add(_image_i); + } + if(_length <= 1) { return; } + this.media = new Media(); + this.media.messageUnpack(_pac); + for(int _i=2; _i < _length; ++_i) { _pac.unpackObject(); } + } + @SuppressWarnings("unchecked") public static MediaContent createFromMessage(Object[] _message) { @@ -45,7 +62,7 @@ public final class MediaContent implements MessagePackable, MessageMergeable } } -final class Image implements MessagePackable, MessageMergeable +final class Image implements MessagePackable, MessageConvertable, MessageUnpackable { private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Image (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field size int))"); public static ClassSchema getSchema() { return _SCHEMA; } @@ -72,7 +89,7 @@ final class Image implements MessagePackable, MessageMergeable @Override @SuppressWarnings("unchecked") - public void messageMerge(Object obj) throws MessageTypeException + public void messageConvert(Object obj) throws MessageTypeException { Object[] _source = ((List)obj).toArray(); FieldSchema[] _fields = _SCHEMA.getFields(); @@ -83,6 +100,22 @@ final class Image implements MessagePackable, MessageMergeable if(_source.length <= 4) { return; } this.size = (Integer)_fields[4].getSchema().convert(_source[4]); } + @Override + public void messageUnpack(Unpacker _pac) throws IOException, MessageTypeException { + int _length = _pac.unpackArray(); + if(_length <= 0) { return; } + this.uri = _pac.unpackString(); + if(_length <= 1) { return; } + this.title = _pac.unpackString(); + if(_length <= 2) { return; } + this.width = _pac.unpackInt(); + if(_length <= 3) { return; } + this.height = _pac.unpackInt(); + if(_length <= 4) { return; } + this.size = _pac.unpackInt(); + for(int _i=5; _i < _length; ++_i) { _pac.unpackObject(); } + } + @SuppressWarnings("unchecked") public static Image createFromMessage(Object[] _message) { @@ -96,7 +129,7 @@ final class Image implements MessagePackable, MessageMergeable } } -final class Media implements MessagePackable, MessageMergeable +final class Media implements MessagePackable, MessageConvertable, MessageUnpackable { private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load("(class Media (package serializers.msgpack) (field uri string) (field title string) (field width int) (field height int) (field format string) (field duration long) (field size long) (field bitrate int) (field person (array string)) (field player int) (field copyright string))"); public static ClassSchema getSchema() { return _SCHEMA; } @@ -135,7 +168,7 @@ final class Media implements MessagePackable, MessageMergeable @Override @SuppressWarnings("unchecked") - public void messageMerge(Object obj) throws MessageTypeException + public void messageConvert(Object obj) throws MessageTypeException { Object[] _source = ((List)obj).toArray(); FieldSchema[] _fields = _SCHEMA.getFields(); @@ -152,6 +185,39 @@ final class Media implements MessagePackable, MessageMergeable if(_source.length <= 10) { return; } this.copyright = (String)_fields[10].getSchema().convert(_source[10]); } + @Override + public void messageUnpack(Unpacker _pac) throws IOException, MessageTypeException { + int _length = _pac.unpackArray(); + if(_length <= 0) { return; } + this.uri = _pac.unpackString(); + if(_length <= 1) { return; } + this.title = _pac.unpackString(); + if(_length <= 2) { return; } + this.width = _pac.unpackInt(); + if(_length <= 3) { return; } + this.height = _pac.unpackInt(); + if(_length <= 4) { return; } + this.format = _pac.unpackString(); + if(_length <= 5) { return; } + this.duration = _pac.unpackLong(); + if(_length <= 6) { return; } + this.size = _pac.unpackLong(); + if(_length <= 7) { return; } + this.bitrate = _pac.unpackInt(); + if(_length <= 8) { return; } + int _person_length = _pac.unpackArray(); + this.person = new ArrayList(_person_length); + for(int _i=0; _i < _person_length; ++_i) { + String _person_i = _pac.unpackString(); + this.person.add(_person_i); + } + if(_length <= 9) { return; } + this.player = _pac.unpackInt(); + if(_length <= 10) { return; } + this.copyright = _pac.unpackString(); + for(int _i=11; _i < _length; ++_i) { _pac.unpackObject(); } + } + @SuppressWarnings("unchecked") public static Media createFromMessage(Object[] _message) { diff --git a/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDirectSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDirectSerializer.java new file mode 100644 index 0000000..721e95b --- /dev/null +++ b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDirectSerializer.java @@ -0,0 +1,68 @@ +package serializers.msgpack; + +import java.io.*; +import java.util.*; +import org.msgpack.*; +import serializers.ObjectSerializer; + +public class MessagePackDirectSerializer implements ObjectSerializer +{ + public String getName() { + return "msgpack-direct"; + } + + public MediaContent create() throws Exception { + Media media = new Media(); + media.uri = "http://javaone.com/keynote.mpg"; + media.format = "video/mpg4"; + media.title = "Javaone Keynote"; + media.duration = 1234567L; + media.bitrate = 0; + media.person = new ArrayList(2); + media.person.add("Bill Gates"); + media.person.add("Steve Jobs"); + media.player = 0; + media.height = 0; + media.width = 0; + media.size = 123L; + media.copyright = ""; + + Image image1 = new Image(); + image1.uri = "http://javaone.com/keynote_large.jpg"; + image1.width = 0; + image1.height = 0; + image1.size = 2; + image1.title = "Javaone Keynote"; + + Image image2 = new Image(); + image2.uri = "http://javaone.com/keynote_thumbnail.jpg"; + image2.width = 0; + image2.height = 0; + image2.size = 1; + image2.title = "Javaone Keynote"; + + MediaContent content = new MediaContent(); + content.media = media; + content.image = new ArrayList(2); + content.image.add(image1); + content.image.add(image2); + + return content; + } + + public byte[] serialize(MediaContent content) throws Exception { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + Packer pk = new Packer(os); + pk.pack(content); + return os.toByteArray(); + } + + public MediaContent deserialize(byte[] array) throws Exception { + Unpacker pac = new Unpacker(); + pac.feed(array); + MediaContent obj = new MediaContent(); + obj.messageUnpack(pac); + return obj; + } +} + diff --git a/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java index c8a88ac..9c8ccbe 100644 --- a/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java +++ b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDynamicSerializer.java @@ -60,7 +60,7 @@ public class MessagePackDynamicSerializer implements ObjectSerializer } public Object deserialize(byte[] array) throws Exception { - UnbufferedUnpacker pac = new UnbufferedUnpacker(); + Unpacker pac = new Unpacker(); pac.execute(array); return (Object)pac.getData(); } diff --git a/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java index 4935899..316389a 100644 --- a/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java +++ b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackGenericSerializer.java @@ -62,7 +62,7 @@ public class MessagePackGenericSerializer implements ObjectSerializer } public Object deserialize(byte[] array) throws Exception { - UnbufferedUnpacker pac = new UnbufferedUnpacker().useSchema(MEDIA_CONTENT_SCHEMA); + Unpacker pac = new Unpacker().useSchema(MEDIA_CONTENT_SCHEMA); pac.execute(array); return (Object)pac.getData(); } diff --git a/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java index 2767474..e24472b 100644 --- a/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java +++ b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackIndirectSerializer.java @@ -58,7 +58,7 @@ public class MessagePackIndirectSerializer implements ObjectSerializer Date: Thu, 20 May 2010 06:18:32 +0900 Subject: [PATCH 0479/1648] java: add Unpacker.wrap method --- java/src/main/java/org/msgpack/Unpacker.java | 36 +++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java index 4d8da7b..7fd6fcd 100644 --- a/java/src/main/java/org/msgpack/Unpacker.java +++ b/java/src/main/java/org/msgpack/Unpacker.java @@ -40,7 +40,7 @@ public class Unpacker implements Iterable { protected int bufferReserveSize; protected InputStream stream; - class BufferedUnpackerMixin extends BufferedUnpackerImpl { + final class BufferedUnpackerMixin extends BufferedUnpackerImpl { boolean fill() throws IOException { if(stream == null) { return false; @@ -109,6 +109,16 @@ public class Unpacker implements Iterable { bufferConsumed(length); } + public void wrap(byte[] buffer) { + wrap(buffer, 0, buffer.length); + } + + public void wrap(byte[] buffer, int offset, int length) { + impl.buffer = buffer; + impl.offset = offset; + impl.filled = length; + } + public boolean fill() throws IOException { return impl.fill(); } @@ -226,51 +236,51 @@ public class Unpacker implements Iterable { } - final public byte unpackByte() throws IOException, MessageTypeException { + public byte unpackByte() throws IOException, MessageTypeException { return impl.unpackByte(); } - final public short unpackShort() throws IOException, MessageTypeException { + public short unpackShort() throws IOException, MessageTypeException { return impl.unpackShort(); } - final public int unpackInt() throws IOException, MessageTypeException { + public int unpackInt() throws IOException, MessageTypeException { return impl.unpackInt(); } - final public long unpackLong() throws IOException, MessageTypeException { + public long unpackLong() throws IOException, MessageTypeException { return impl.unpackLong(); } - final public float unpackFloat() throws IOException, MessageTypeException { + public float unpackFloat() throws IOException, MessageTypeException { return impl.unpackFloat(); } - final public double unpackDouble() throws IOException, MessageTypeException { + public double unpackDouble() throws IOException, MessageTypeException { return impl.unpackDouble(); } - final public Object unpackNull() throws IOException, MessageTypeException { + public Object unpackNull() throws IOException, MessageTypeException { return impl.unpackNull(); } - final public boolean unpackBoolean() throws IOException, MessageTypeException { + public boolean unpackBoolean() throws IOException, MessageTypeException { return impl.unpackBoolean(); } - final public int unpackArray() throws IOException, MessageTypeException { + public int unpackArray() throws IOException, MessageTypeException { return impl.unpackArray(); } - final public int unpackMap() throws IOException, MessageTypeException { + public int unpackMap() throws IOException, MessageTypeException { return impl.unpackMap(); } - final public int unpackRaw() throws IOException, MessageTypeException { + public int unpackRaw() throws IOException, MessageTypeException { return impl.unpackRaw(); } - final public byte[] unpackRawBody(int length) throws IOException, MessageTypeException { + public byte[] unpackRawBody(int length) throws IOException, MessageTypeException { return impl.unpackRawBody(length); } From c2525bcc05477600f8c093483df179d89e8f0707 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Thu, 20 May 2010 06:19:26 +0900 Subject: [PATCH 0480/1648] java: add Unpacker.wrap method --- .../src/serializers/msgpack/MessagePackDirectSerializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDirectSerializer.java b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDirectSerializer.java index 721e95b..25f932b 100644 --- a/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDirectSerializer.java +++ b/java/test/thrift-protobuf-compare/tpc/src/serializers/msgpack/MessagePackDirectSerializer.java @@ -59,7 +59,7 @@ public class MessagePackDirectSerializer implements ObjectSerializer Date: Thu, 20 May 2010 17:32:15 +0900 Subject: [PATCH 0481/1648] java: javadoc --- .../org/msgpack/BufferedUnpackerImpl.java | 6 +- java/src/main/java/org/msgpack/Unpacker.java | 293 ++++++++++++++++-- 2 files changed, 276 insertions(+), 23 deletions(-) diff --git a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java index 5fde4e1..0aba62b 100644 --- a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java +++ b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java @@ -367,7 +367,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { } } - final byte[] unpackRawBody(int length) throws IOException, MessageTypeException { + final byte[] unpackRawBody(int length) throws IOException { more(length); byte[] bytes = new byte[length]; System.arraycopy(buffer, offset, bytes, 0, length); @@ -388,12 +388,12 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { return s; } - final Object unpackObject() throws IOException, MessageTypeException { + final Object unpackObject() throws IOException { // FIXME save state, restore state UnpackResult result = new UnpackResult(); if(!next(result)) { super.reset(); - throw new MessageTypeException(); + throw new UnpackException("insufficient buffer"); } return result.getData(); } diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java index 7fd6fcd..7563b39 100644 --- a/java/src/main/java/org/msgpack/Unpacker.java +++ b/java/src/main/java/org/msgpack/Unpacker.java @@ -23,6 +23,82 @@ import java.io.IOException; import java.util.Iterator; import java.nio.ByteBuffer; +/** + * Deserializer class that includes Buffered API, Unbuffered API, + * Schema API and Direct Conversion API. + * + * Buffered API uses the internal buffer of the Unpacker. + * Following code uses Buffered API with an input stream: + * 
+ * // create an unpacker with input stream
+ * Unpacker pac = new Unpacker(System.in);
+ *
+ * // take a object out using next() method, or ...
+ * UnpackResult result = pac.next();
+ *
+ * // use an iterator.
+ * for(Object obj : pac) {
+ *   // use MessageConvertable interface to convert the
+ *   // the generic object to the specific type.
+ * }
+ *
+ * + * Following code doesn't use the input stream and feeds buffer + * using {@link feed(byte[])} method. This is useful to use + * special stream like zlib or event-driven I/O library. + * 
+ * // create an unpacker without input stream
+ * Unpacker pac = new Unpacker();
+ *
+ * // feed buffer to the internal buffer.
+ * pac.feed(input_bytes);
+ *
+ * // use next() method or iterators.
+ * for(Object obj : pac) {
+ *   // ...
+ * }
+ *
+ * + * The combination of {@link reserveBuffer()}, {@link getBuffer()}, + * {@link getBufferOffset()}, {@link getBufferCapacity()} and + * {@link bufferConsumed()} is useful to omit copying. + * 
+ * // create an unpacker without input stream
+ * Unpacker pac = new Unpacker();
+ *
+ * // reserve internal buffer at least 1024 bytes.
+ * pac.reserveBuffer(1024);
+ *
+ * // feed buffer to the internal buffer upto pac.getBufferCapacity() bytes.
+ * System.in.read(pac.getBuffer(), pac.getBufferOffset(), pac.getBufferCapacity());
+ *
+ * // use next() method or iterators.
+ * for(Object obj : pac) {
+ *     // ...
+ * }
+ *
+ * + * Unbuffered API doesn't initialize the internal buffer. + * You can manage the buffer manually. + * 
+ * // create an unpacker with input stream
+ * Unpacker pac = new Unpacker(System.in);
+ *
+ * // manage the buffer manually.
+ * byte[] buffer = new byte[1024];
+ * int filled = System.in.read(buffer);
+ * int offset = 0;
+ *
+ * // deserialize objects using execute() method.
+ * int nextOffset = pac.execute(buffer, offset, filled);
+ *
+ * // take out object if deserialized object is ready.
+ * if(pac.isFinished()) {
+ *     Object obj = pac.getData();
+ *     // ...
+ * }
+ *
+ */ public class Unpacker implements Iterable { // buffer: @@ -59,38 +135,86 @@ public class Unpacker implements Iterable { final BufferedUnpackerMixin impl = new BufferedUnpackerMixin(); + /** + * Calls {@link Unpacker(DEFAULT_BUFFER_SIZE)} + */ public Unpacker() { this(DEFAULT_BUFFER_SIZE); } + /** + * Calls {@link Unpacker(null, bufferReserveSize)} + */ public Unpacker(int bufferReserveSize) { this(null, bufferReserveSize); } + /** + * Calls {@link Unpacker(stream, DEFAULT_BUFFER_SIZE)} + */ public Unpacker(InputStream stream) { this(stream, DEFAULT_BUFFER_SIZE); } + /** + * Constructs the unpacker. + * The stream is used to fill the buffer when more buffer is required by {@link next()} or {@link UnpackIterator#hasNext()} method. + * @param stream input stream to fill the buffer + * @param bufferReserveSize threshold size to expand the size of buffer + */ public Unpacker(InputStream stream, int bufferReserveSize) { this.parsed = 0; this.bufferReserveSize = bufferReserveSize/2; this.stream = stream; } + /** + * Sets schema to convert deserialized object into specific type. + * Default schema is {@link GenericSchema} that leaves objects for generic type. Use {@link MessageConvertable#messageConvert(Object)} method to convert the generic object. + * @param s schem to use + */ public Unpacker useSchema(Schema s) { impl.setSchema(s); return this; } + /** + * Gets the input stream. + * @return the input stream. it may be null. + */ public InputStream getStream() { return this.stream; } + /** + * Sets the input stream. + * @param stream the input stream to set. + */ public void setStream(InputStream stream) { this.stream = stream; } + + /** + * Fills the buffer with the specified buffer. + */ + public void feed(byte[] buffer) { + feed(buffer, 0, buffer.length); + } + + /** + * Fills the buffer with the specified buffer. + */ + public void feed(byte[] buffer, int offset, int length) { + reserveBuffer(length); + System.arraycopy(buffer, offset, impl.buffer, impl.offset, length); + bufferConsumed(length); + } + + /** + * Fills the buffer with the specified buffer. + */ public void feed(ByteBuffer buffer) { int length = buffer.remaining(); if (length == 0) return; @@ -99,45 +223,62 @@ public class Unpacker implements Iterable { bufferConsumed(length); } - public void feed(byte[] buffer) { - feed(buffer, 0, buffer.length); - } - - public void feed(byte[] buffer, int offset, int length) { - reserveBuffer(length); - System.arraycopy(buffer, offset, impl.buffer, impl.offset, length); - bufferConsumed(length); - } - + /** + * Swaps the internal buffer with the specified buffer. + * This method doesn't copy the buffer and the its contents will be rewritten by {@link fill()} or {@link feed(byte[])} method. + */ public void wrap(byte[] buffer) { wrap(buffer, 0, buffer.length); } + /** + * Swaps the internal buffer with the specified buffer. + * This method doesn't copy the buffer and the its contents will be rewritten by {@link fill()} or {@link feed(byte[])} method. + */ public void wrap(byte[] buffer, int offset, int length) { impl.buffer = buffer; impl.offset = offset; impl.filled = length; } + /** + * Fills the internal using the input stream. + * @return false if the stream is null or stream.read returns <= 0. + */ public boolean fill() throws IOException { return impl.fill(); } + + /** + * Returns the iterator that calls {@link next()} method repeatedly. + */ public Iterator iterator() { return new UnpackIterator(this); } + /** + * Deserializes one object and returns it. + * @return {@link UnpackResult#isFinished()} returns false if the buffer is insufficient to deserialize one object. + */ public UnpackResult next() throws IOException, UnpackException { UnpackResult result = new UnpackResult(); impl.next(result); return result; } + /** + * Deserializes one object and returns it. + * @return false if the buffer is insufficient to deserialize one object. + */ public boolean next(UnpackResult result) throws IOException, UnpackException { return impl.next(result); } + /** + * Reserve free space of the internal buffer at least specified size and expands {@link getBufferCapacity()}. + */ public void reserveBuffer(int require) { if(impl.buffer == null) { int nextSize = (bufferReserveSize < require) ? require : bufferReserveSize; @@ -163,22 +304,40 @@ public class Unpacker implements Iterable { impl.offset = 0; } + /** + * Returns the internal buffer. + */ public byte[] getBuffer() { return impl.buffer; } - public int getBufferOffset() { - return impl.filled; - } - + /** + * Returns the size of free space of the internal buffer. + */ public int getBufferCapacity() { return impl.buffer.length - impl.filled; } + /** + * Returns the offset of free space in the internal buffer. + */ + public int getBufferOffset() { + return impl.filled; + } + + /** + * Moves front the offset of the free space in the internal buffer. + * Call this method after fill the buffer manually using {@link reserveBuffer()}, {@link getBuffer()}, {@link getBufferOffset()} and {@link getBufferCapacity()} methods. + */ public void bufferConsumed(int size) { impl.filled += size; } + /** + * Deserializes one object upto the offset of the internal buffer. + * Call {@link reset()} method before calling this method again. + * @return true if one object is deserialized. Use {@link getData()} to get the deserialized object. + */ public boolean execute() throws UnpackException { int noffset = impl.execute(impl.buffer, impl.offset, impl.filled); if(noffset <= impl.offset) { @@ -190,10 +349,24 @@ public class Unpacker implements Iterable { } + /** + * Deserializes one object over the specified buffer. + * This method doesn't use the internal buffer. + * Use {@link isFinished()} method to known a object is ready to get. + * Call {@link reset()} method before calling this method again. + * @return offset position that is parsed. + */ public int execute(byte[] buffer) throws UnpackException { return execute(buffer, 0, buffer.length); } + /** + * Deserializes one object over the specified buffer. + * This method doesn't use the internal buffer. + * Use {@link isFinished()} method to known a object is ready to get. + * Call {@link reset()} method before calling this method again. + * @return offset position that is parsed. + */ public int execute(byte[] buffer, int offset, int length) throws UnpackException { int noffset = impl.execute(buffer, offset + impl.offset, length); impl.offset = noffset - offset; @@ -203,14 +376,23 @@ public class Unpacker implements Iterable { return noffset; } - public boolean isFinished() { - return impl.isFinished(); - } - + /** + * Gets the object deserialized by {@link execute(byte[])} method. + */ public Object getData() { return impl.getData(); } + /** + * Returns true if an object is ready to get with {@link getData()} method. + */ + public boolean isFinished() { + return impl.isFinished(); + } + + /** + * Resets the internal state of the unpacker. + */ public void reset() { impl.reset(); } @@ -236,59 +418,130 @@ public class Unpacker implements Iterable { } + /** + * Gets one {@code byte} value from the buffer. + * This method calls {@link fill()} method if needed. + * @throws MessageTypeException the first value of the buffer is not a {@code byte}. + */ public byte unpackByte() throws IOException, MessageTypeException { return impl.unpackByte(); } + /** + * Gets one {@code short} value from the buffer. + * This method calls {@link fill()} method if needed. + * @throws MessageTypeException the first value of the buffer is not a {@code short}. + */ public short unpackShort() throws IOException, MessageTypeException { return impl.unpackShort(); } + /** + * Gets one {@code int} value from the buffer. + * This method calls {@link fill()} method if needed. + * @throws MessageTypeException the first value of the buffer is not a {@code int}. + */ public int unpackInt() throws IOException, MessageTypeException { return impl.unpackInt(); } + /** + * Gets one {@code long} value from the buffer. + * This method calls {@link fill()} method if needed. + * @throws MessageTypeException the first value of the buffer is not a {@code long}. + */ public long unpackLong() throws IOException, MessageTypeException { return impl.unpackLong(); } + /** + * Gets one {@code float} value from the buffer. + * This method calls {@link fill()} method if needed. + * @throws MessageTypeException the first value of the buffer is not a {@code float}. + */ public float unpackFloat() throws IOException, MessageTypeException { return impl.unpackFloat(); } + /** + * Gets one {@code double} value from the buffer. + * This method calls {@link fill()} method if needed. + * @throws MessageTypeException the first value of the buffer is not a {@code double}. + */ public double unpackDouble() throws IOException, MessageTypeException { return impl.unpackDouble(); } + /** + * Gets one {@code null} value from the buffer. + * This method calls {@link fill()} method if needed. + * @throws MessageTypeException the first value of the buffer is not a {@code null}. + */ public Object unpackNull() throws IOException, MessageTypeException { return impl.unpackNull(); } + /** + * Gets one {@code boolean} value from the buffer. + * This method calls {@link fill()} method if needed. + * @throws MessageTypeException the first value of the buffer is not a {@code boolean}. + */ public boolean unpackBoolean() throws IOException, MessageTypeException { return impl.unpackBoolean(); } + /** + * Gets one array header from the buffer. + * This method calls {@link fill()} method if needed. + * @return the length of the map. There are {@code retval} objects to get. + * @throws MessageTypeException the first value of the buffer is not a array. + */ public int unpackArray() throws IOException, MessageTypeException { return impl.unpackArray(); } + /** + * Gets one map header from the buffer. + * This method calls {@link fill()} method if needed. + * @return the length of the map. There are {@code retval * 2} objects to get. + * @throws MessageTypeException the first value of the buffer is not a map. + */ public int unpackMap() throws IOException, MessageTypeException { return impl.unpackMap(); } + /** + * Gets one raw header from the buffer. + * This method calls {@link fill()} method if needed. + * @return the length of the raw bytes. There are {@code retval} bytes to get. + * @throws MessageTypeException the first value of the buffer is not a raw bytes. + */ public int unpackRaw() throws IOException, MessageTypeException { return impl.unpackRaw(); } - public byte[] unpackRawBody(int length) throws IOException, MessageTypeException { + /** + * Gets one raw header from the buffer. + * This method calls {@link fill()} method if needed. + */ + public byte[] unpackRawBody(int length) throws IOException { return impl.unpackRawBody(length); } + /** + * Gets one {@code String} value from the buffer. + * This method calls {@link fill()} method if needed. + * @throws MessageTypeException the first value of the buffer is not a {@code String}. + */ final public String unpackString() throws IOException, MessageTypeException { return impl.unpackString(); } - final public Object unpackObject() throws IOException, MessageTypeException { + /** + * Gets one {@code Object} value from the buffer. + * This method calls {@link fill()} method if needed. + */ + final public Object unpackObject() throws IOException { return impl.unpackObject(); } } From 1fe35d7efe4d5a4993d2392cc259dec17fc744e4 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 22 May 2010 03:34:17 +0900 Subject: [PATCH 0482/1648] java: fix Packer.packByte --- java/src/main/java/org/msgpack/Packer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java index 6d79414..3fa421f 100644 --- a/java/src/main/java/org/msgpack/Packer.java +++ b/java/src/main/java/org/msgpack/Packer.java @@ -34,7 +34,7 @@ public class Packer { public Packer packByte(byte d) throws IOException { if(d < -(1<<5)) { - castBytes[0] = (byte)0xd1; + castBytes[0] = (byte)0xd0; castBytes[1] = d; out.write(castBytes, 0, 2); } else { From b9cb270b8f771e5504e36469112daa6a32b24d63 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 22 May 2010 03:34:43 +0900 Subject: [PATCH 0483/1648] java: add Unpacker.unpack(MessageUnpackable) and Unpacker.tryUnpackNil() --- .../org/msgpack/BufferedUnpackerImpl.java | 22 ++++++++++++++++++- java/src/main/java/org/msgpack/Unpacker.java | 8 +++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java index 0aba62b..4b2f302 100644 --- a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java +++ b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java @@ -63,6 +63,15 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { } } + private final boolean tryMore(int require) throws IOException, UnpackException { + while(filled - offset < require) { + if(!fill()) { + return false; + } + } + return true; + } + private final void advance(int length) { offset += length; } @@ -275,6 +284,18 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { return null; } + final boolean tryUnpackNull() throws IOException { + if(!tryMore(1)) { + return false; + } + int b = buffer[offset] & 0xff; + if(b != 0xc0) { // nil + return false; + } + advance(1); + return 1; + } + final boolean unpackBoolean() throws IOException, MessageTypeException { more(1); int b = buffer[offset] & 0xff; @@ -389,7 +410,6 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { } final Object unpackObject() throws IOException { - // FIXME save state, restore state UnpackResult result = new UnpackResult(); if(!next(result)) { super.reset(); diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java index 7563b39..1917b9f 100644 --- a/java/src/main/java/org/msgpack/Unpacker.java +++ b/java/src/main/java/org/msgpack/Unpacker.java @@ -544,5 +544,13 @@ public class Unpacker implements Iterable { final public Object unpackObject() throws IOException { return impl.unpackObject(); } + + final void unpack(MessageUnpackable obj) throws IOException, MessageTypeException { + obj.unpackMessage(this); + } + + final boolean tryUnpackNull() throws IOException { + return impl.tryUnpackNull(); + } } From b4fc79c38ee44a1da5c2973fa213753a2d309666 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 22 May 2010 17:05:17 +0900 Subject: [PATCH 0484/1648] java: fixes compile error --- .../main/java/org/msgpack/BufferedUnpackerImpl.java | 7 ++++++- java/src/main/java/org/msgpack/Unpacker.java | 12 ++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java index 4b2f302..cc6604d 100644 --- a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java +++ b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java @@ -293,7 +293,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { return false; } advance(1); - return 1; + return true; } final boolean unpackBoolean() throws IOException, MessageTypeException { @@ -396,6 +396,11 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl { return bytes; } + final byte[] unpackByteArray() throws IOException, MessageTypeException { + int length = unpackRaw(); + return unpackRawBody(length); + } + final String unpackString() throws IOException, MessageTypeException { int length = unpackRaw(); more(length); diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java index 1917b9f..e84aff9 100644 --- a/java/src/main/java/org/msgpack/Unpacker.java +++ b/java/src/main/java/org/msgpack/Unpacker.java @@ -521,13 +521,21 @@ public class Unpacker implements Iterable { } /** - * Gets one raw header from the buffer. + * Gets one raw body from the buffer. * This method calls {@link fill()} method if needed. */ public byte[] unpackRawBody(int length) throws IOException { return impl.unpackRawBody(length); } + /** + * Gets one raw bytes from the buffer. + * This method calls {@link fill()} method if needed. + */ + public byte[] unpackByteArray() throws IOException { + return impl.unpackByteArray(); + } + /** * Gets one {@code String} value from the buffer. * This method calls {@link fill()} method if needed. @@ -546,7 +554,7 @@ public class Unpacker implements Iterable { } final void unpack(MessageUnpackable obj) throws IOException, MessageTypeException { - obj.unpackMessage(this); + obj.messageUnpack(this); } final boolean tryUnpackNull() throws IOException { From c43e5e0c95105c0dbf17e41d15068bfdb08450ce Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sun, 23 May 2010 01:31:15 +0900 Subject: [PATCH 0485/1648] java: added testcases for empty array and empty map --- java/src/test/java/org/msgpack/TestPackUnpack.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java index a16b5b1..b02bbb4 100644 --- a/java/src/test/java/org/msgpack/TestPackUnpack.java +++ b/java/src/test/java/org/msgpack/TestPackUnpack.java @@ -177,6 +177,9 @@ public class TestPackUnpack { @Test public void testArray() throws Exception { + List emptyList = new ArrayList(); + testArray(emptyList, Schema.parse("(array int)")); + for (int i = 0; i < 1000; i++) { Schema schema = Schema.parse("(array int)"); List l = new ArrayList(); @@ -209,6 +212,9 @@ public class TestPackUnpack { @Test public void testMap() throws Exception { + Map emptyMap = new HashMap(); + testMap(emptyMap, Schema.parse("(map int int)")); + for (int i = 0; i < 1000; i++) { Schema schema = Schema.parse("(map int int)"); Map m = new HashMap(); From 5982970e21d9bab7ea2bd507b360317f40628260 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 23 May 2010 01:34:45 +0900 Subject: [PATCH 0486/1648] java: fixed problem that empty array and empty map don't check Schema --- java/src/main/java/org/msgpack/UnpackerImpl.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/java/src/main/java/org/msgpack/UnpackerImpl.java b/java/src/main/java/org/msgpack/UnpackerImpl.java index ae01289..9b885a0 100644 --- a/java/src/main/java/org/msgpack/UnpackerImpl.java +++ b/java/src/main/java/org/msgpack/UnpackerImpl.java @@ -157,7 +157,10 @@ public class UnpackerImpl { count = b & 0x0f; //System.out.println("fixarray count:"+count); obj = new Object[count]; - if(count == 0) { break _push; } // FIXME check IArraySchema + if(count == 0) { + obj = ((IArraySchema)top_schema).createFromArray((Object[])obj); + break _push; + } ++top; stack_obj[top] = top_obj; stack_ct[top] = top_ct; @@ -179,7 +182,10 @@ public class UnpackerImpl { } count = b & 0x0f; obj = new Object[count*2]; - if(count == 0) { break _push; } // FIXME check IMapSchema + if(count == 0) { + obj = ((IMapSchema)top_schema).createFromMap((Object[])obj); + break _push; + } //System.out.println("fixmap count:"+count); ++top; stack_obj[top] = top_obj; From fa6ea6848f3c639d64e223afe31d0fa2ba13d333 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 23 May 2010 01:38:01 +0900 Subject: [PATCH 0487/1648] java: fixed problem that empty array and empty map don't check Schema --- .../main/java/org/msgpack/UnpackerImpl.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/java/src/main/java/org/msgpack/UnpackerImpl.java b/java/src/main/java/org/msgpack/UnpackerImpl.java index 9b885a0..10cf5f0 100644 --- a/java/src/main/java/org/msgpack/UnpackerImpl.java +++ b/java/src/main/java/org/msgpack/UnpackerImpl.java @@ -348,7 +348,10 @@ public class UnpackerImpl { castBuffer.put(src, n, 2); count = ((int)castBuffer.getShort(0)) & 0xffff; obj = new Object[count]; - if(count == 0) { break _push; } // FIXME check IArraySchema + if(count == 0) { + obj = ((IArraySchema)top_schema).createFromArray((Object[])obj); + break _push; + } ++top; stack_obj[top] = top_obj; stack_ct[top] = top_ct; @@ -371,7 +374,10 @@ public class UnpackerImpl { // FIXME overflow check count = castBuffer.getInt(0) & 0x7fffffff; obj = new Object[count]; - if(count == 0) { break _push; } // FIXME check IArraySchema + if(count == 0) { + obj = ((IArraySchema)top_schema).createFromArray((Object[])obj); + break _push; + } ++top; stack_obj[top] = top_obj; stack_ct[top] = top_ct; @@ -393,7 +399,10 @@ public class UnpackerImpl { castBuffer.put(src, n, 2); count = ((int)castBuffer.getShort(0)) & 0xffff; obj = new Object[count*2]; - if(count == 0) { break _push; } // FIXME check IMapSchema + if(count == 0) { + obj = ((IMapSchema)top_schema).createFromMap((Object[])obj); + break _push; + } //System.out.println("fixmap count:"+count); ++top; stack_obj[top] = top_obj; @@ -417,7 +426,10 @@ public class UnpackerImpl { // FIXME overflow check count = castBuffer.getInt(0) & 0x7fffffff; obj = new Object[count*2]; - if(count == 0) { break _push; } // FIXME check IMapSchema + if(count == 0) { + obj = ((IMapSchema)top_schema).createFromMap((Object[])obj); + break _push; + } //System.out.println("fixmap count:"+count); ++top; stack_obj[top] = top_obj; From f8173e93f5df4c6c0289d58b92a4acde816f1cf3 Mon Sep 17 00:00:00 2001 From: Kazuki Ohta Date: Sun, 23 May 2010 01:48:20 +0900 Subject: [PATCH 0488/1648] java: version 0.3 (added CHANGES.txt and LICENSE.txt) --- java/CHANGES.txt | 9 +++ java/LICENSE.txt | 202 +++++++++++++++++++++++++++++++++++++++++++++++ java/pom.xml | 2 +- 3 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 java/CHANGES.txt create mode 100644 java/LICENSE.txt diff --git a/java/CHANGES.txt b/java/CHANGES.txt new file mode 100644 index 0000000..77e9318 --- /dev/null +++ b/java/CHANGES.txt @@ -0,0 +1,9 @@ +Release 0.3 - 2010/05/23 + NEW FEATURES + Added Unbuffered API + Direct Conversion API to the Unpacker. + + BUG FIXES + Zero-length Array and Map is deserialized as List and Map, instead of the + array of the Object. + + fixed the bug around Packer.packByte(). diff --git a/java/LICENSE.txt b/java/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/java/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/java/pom.xml b/java/pom.xml index d1f6c34..9b74b4c 100755 --- a/java/pom.xml +++ b/java/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.msgpack msgpack - 0.2 + 0.3 MessagePack for Java MessagePack for Java From d0af8aa9f11bfd9c6ded625f5591db25d42153ad Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sun, 23 May 2010 21:10:49 +0900 Subject: [PATCH 0489/1648] ruby: rdoc --- ruby/pack.c | 132 ++++++++++++++++++++++++--- ruby/rbinit.c | 12 +++ ruby/unpack.c | 244 ++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 370 insertions(+), 18 deletions(-) diff --git a/ruby/pack.c b/ruby/pack.c index 387bab6..7f56923 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -51,6 +51,16 @@ static ID s_append; rb_raise(rb_eArgError, "wrong number of arguments (%d for 0)", argc); \ } + +/* + * Document-method: NilClass#to_msgpack + * + * call-seq: + * nil.to_msgpack(out = '') -> String + * + * Serializes the nil into raw bytes. + * This calls to_msgpack reflectively for internal elements. + */ static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -58,6 +68,16 @@ static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: TrueClass#to_msgpack + * + * call-seq: + * true.to_msgpack(out = '') -> String + * + * Serializes the true into raw bytes. + * This calls to_msgpack reflectively for internal elements. + */ static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -65,6 +85,16 @@ static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: FalseClass#to_msgpack + * + * call-seq: + * false.to_msgpack(out = '') -> String + * + * Serializes false into raw bytes. + * This calls to_msgpack reflectively for internal elements. + */ static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -73,6 +103,15 @@ static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self } +/* + * Document-method: Fixnum#to_msgpack + * + * call-seq: + * fixnum.to_msgpack(out = '') -> String + * + * Serializes the Fixnum into raw bytes. + * This calls to_msgpack reflectively for internal elements. + */ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -85,6 +124,15 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) #define RBIGNUM_SIGN(b) (RBIGNUM(b)->sign) #endif +/* + * Document-method: Bignum#to_msgpack + * + * call-seq: + * bignum.to_msgpack(out = '') -> String + * + * Serializes the Bignum into raw bytes. + * This calls to_msgpack reflectively for internal elements. + */ static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -97,6 +145,15 @@ static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: Float#to_msgpack + * + * call-seq: + * float.to_msgpack(out = '') -> String + * + * Serializes the Float into raw bytes. + */ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -104,6 +161,15 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: String#to_msgpack + * + * call-seq: + * string.to_msgpack(out = '') -> String + * + * Serializes the String into raw bytes. + */ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -112,6 +178,15 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: Symbol#to_msgpack + * + * call-seq: + * symbol.to_msgpack(out = '') -> String + * + * Serializes the Symbol into raw bytes. + */ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -122,6 +197,16 @@ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + +/* + * Document-method: Array#to_msgpack + * + * call-seq: + * array.to_msgpack(out = '') -> String + * + * Serializes the Array into raw bytes. + * This calls to_msgpack method reflectively for internal elements. + */ static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -134,6 +219,7 @@ static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self) return out; } + #ifndef RHASH_SIZE // Ruby 1.8 #define RHASH_SIZE(h) (RHASH(h)->tbl ? RHASH(h)->tbl->num_entries : 0) #endif @@ -146,6 +232,15 @@ static int MessagePack_Hash_to_msgpack_foreach(VALUE key, VALUE value, VALUE out return ST_CONTINUE; } +/* + * Document-method: Hash#to_msgpack + * + * call-seq: + * hash.to_msgpack(out = '') -> String + * + * Serializes the Hash into raw bytes. + * This calls to_msgpack method reflectively for internal keys and values. + */ static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self) { ARG_BUFFER(out, argc, argv); @@ -155,6 +250,17 @@ static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self) } +/** + * Document-method: MessagePack.pack + * + * call-seq: + * MessagePack.pack(object, out = '') -> String + * + * Serializes the object into raw bytes. The encoding of the string is ASCII-8BIT on Ruby 1.9. + * This method is same as object.to_msgpack(out = ''). + * + * _out_ is an object that implements *<<* method like String or IO. + */ static VALUE MessagePack_pack(int argc, VALUE* argv, VALUE self) { VALUE out; @@ -173,16 +279,22 @@ void Init_msgpack_pack(VALUE mMessagePack) { s_to_msgpack = rb_intern("to_msgpack"); s_append = rb_intern("<<"); - rb_define_method_id(rb_cNilClass, s_to_msgpack, MessagePack_NilClass_to_msgpack, -1); - rb_define_method_id(rb_cTrueClass, s_to_msgpack, MessagePack_TrueClass_to_msgpack, -1); - rb_define_method_id(rb_cFalseClass, s_to_msgpack, MessagePack_FalseClass_to_msgpack, -1); - rb_define_method_id(rb_cFixnum, s_to_msgpack, MessagePack_Fixnum_to_msgpack, -1); - rb_define_method_id(rb_cBignum, s_to_msgpack, MessagePack_Bignum_to_msgpack, -1); - rb_define_method_id(rb_cFloat, s_to_msgpack, MessagePack_Float_to_msgpack, -1); - rb_define_method_id(rb_cString, s_to_msgpack, MessagePack_String_to_msgpack, -1); - rb_define_method_id(rb_cArray, s_to_msgpack, MessagePack_Array_to_msgpack, -1); - rb_define_method_id(rb_cHash, s_to_msgpack, MessagePack_Hash_to_msgpack, -1); - rb_define_method_id(rb_cSymbol, s_to_msgpack, MessagePack_Symbol_to_msgpack, -1); + + rb_define_method(rb_cNilClass, "to_msgpack", MessagePack_NilClass_to_msgpack, -1); + rb_define_method(rb_cTrueClass, "to_msgpack", MessagePack_TrueClass_to_msgpack, -1); + rb_define_method(rb_cFalseClass, "to_msgpack", MessagePack_FalseClass_to_msgpack, -1); + rb_define_method(rb_cFixnum, "to_msgpack", MessagePack_Fixnum_to_msgpack, -1); + rb_define_method(rb_cBignum, "to_msgpack", MessagePack_Bignum_to_msgpack, -1); + rb_define_method(rb_cFloat, "to_msgpack", MessagePack_Float_to_msgpack, -1); + rb_define_method(rb_cString, "to_msgpack", MessagePack_String_to_msgpack, -1); + rb_define_method(rb_cArray, "to_msgpack", MessagePack_Array_to_msgpack, -1); + rb_define_method(rb_cHash, "to_msgpack", MessagePack_Hash_to_msgpack, -1); + rb_define_method(rb_cSymbol, "to_msgpack", MessagePack_Symbol_to_msgpack, -1); + + /** + * MessagePack module is defined in rbinit.c file. + * mMessagePack = rb_define_module("MessagePack"); + */ rb_define_module_function(mMessagePack, "pack", MessagePack_pack, -1); } diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 80d1d8c..050abde 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -20,6 +20,18 @@ static VALUE mMessagePack; +/** + * Document-module: MessagePack + * + * MessagePack is a binary-based efficient object serialization library. + * It enables to exchange structured objects between many languages like JSON. + * But unlike JSON, it is very fast and small. + * + * require 'msgpack' + * msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03" + * MessagePack.unpack(msg) #=> [1,2,3] + * + */ void Init_msgpack(void) { mMessagePack = rb_define_module("MessagePack"); diff --git a/ruby/unpack.c b/ruby/unpack.c index 3a95e5a..9a2c457 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -205,6 +205,12 @@ static int template_execute_wrap(msgpack_unpack_t* mp, static VALUE cUnpacker; + + +/** + * Document-module: MessagePack::UnpackerError + * + */ static VALUE eUnpackError; @@ -243,6 +249,22 @@ static ID append_method_of(VALUE stream) } } +/** + * Document-method: MessagePack::Unpacker#initialize + * + * call-seq: + * MessagePack::Unpacker.new(stream = nil) + * + * Creates instance of MessagePack::Unpacker. + * + * You can specify a _stream_ for input stream. + * It is required to implement *sysread* or *readpartial* method. + * + * With the input stream, buffers will be feeded into the deserializer automatically. + * + * Without the input stream, use *feed* method manually. Or you can manage the buffer manually + * with *execute*, *finished?*, *data* and *reset* methods. + */ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) { VALUE stream; @@ -268,12 +290,29 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) return self; } + +/** + * Document-method: MessagePack::Unpacker#stream + * + * call-seq: + * unpacker.stream + * + * Gets the input stream. + */ static VALUE MessagePack_Unpacker_stream_get(VALUE self) { UNPACKER(self, mp); return mp->user.stream; } +/** + * Document-method: MessagePack::Unpacker#stream= + * + * call-seq: + * unpacker.stream = stream + * + * Resets the input stream. You can set nil not to use input stream. + */ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) { UNPACKER(self, mp); @@ -282,6 +321,15 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) return val; } + +/** + * Document-method: MessagePack::Unpacker#feed + * + * call-seq: + * unpacker.feed(data) + * + * Fills the internal buffer with the specified buffer. + */ static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) { UNPACKER(self, mp); @@ -290,6 +338,20 @@ static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) return Qnil; } +/** + * Document-method: MessagePack::Unpacker#fill + * + * call-seq: + * unpacker.fill -> length of read data + * + * Fills the internal buffer using the input stream. + * + * If the input stream is not specified, it returns nil. + * You can set it on *initialize* or *stream=* methods. + * + * This methods raises exceptions that _stream.sysread_ or + * _stream.readpartial_ method raises. + */ static VALUE MessagePack_Unpacker_fill(VALUE self) { UNPACKER(self, mp); @@ -313,6 +375,18 @@ static VALUE MessagePack_Unpacker_fill(VALUE self) return LONG2FIX(len); } + +/** + * Document-method: MessagePack::Unpacker#each + * + * call-seq: + * unpacker.each {|object| } + * + * Deserializes objects repeatedly. This calls *fill* method automatically. + * + * UnpackError is throw when parse error is occured. + * This method raises exceptions that *fill* method raises. + */ static VALUE MessagePack_Unpacker_each(VALUE self) { UNPACKER(self, mp); @@ -352,6 +426,7 @@ static VALUE MessagePack_Unpacker_each(VALUE self) return Qnil; } + static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned long dlen) { msgpack_unpack_t mp; @@ -376,12 +451,34 @@ static inline VALUE MessagePack_unpack_impl(VALUE self, VALUE data, unsigned lon } } +/** + * Document-method: MessagePack::Unpacker.unpack_limit + * + * call-seq: + * MessagePack::Unpacker.unpack_limit(data, limit) -> object + * + * Deserializes one object over the specified buffer upto _limit_ bytes. + * + * UnpackError is throw when parse error is occured, the buffer is insufficient + * to deserialize one object or there are extra bytes. + */ static VALUE MessagePack_unpack_limit(VALUE self, VALUE data, VALUE limit) { CHECK_STRING_TYPE(data); return MessagePack_unpack_impl(self, data, NUM2ULONG(limit)); } +/** + * Document-method: MessagePack::Unpacker.unpack + * + * call-seq: + * MessagePack::Unpacker.unpack(data) -> object + * + * Deserializes one object over the specified buffer. + * + * UnpackError is throw when parse error is occured, the buffer is insufficient + * to deserialize one object or there are extra bytes. + */ static VALUE MessagePack_unpack(VALUE self, VALUE data) { CHECK_STRING_TYPE(data); @@ -411,7 +508,20 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data, } } -/* compat */ +/** + * Document-method: MessagePack::Unpacker#execute_limit + * + * call-seq: + * unpacker.unpack_limit(data, offset, limit) -> next offset + * + * Deserializes one object over the specified buffer from _offset_ bytes upto _limit_ bytes. + * + * This method doesn't use the internal buffer. + * + * Call *reset()* method before calling this method again. + * + * UnpackError is throw when parse error is occured. + */ static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, VALUE off, VALUE limit) { @@ -420,7 +530,24 @@ static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data, (size_t)NUM2ULONG(off), (size_t)NUM2ULONG(limit)); } -/* compat */ +/** + * Document-method: MessagePack::Unpacker#execute + * + * call-seq: + * unpacker.unpack(data, offset) -> next offset + * + * Deserializes one object over the specified buffer from _offset_ bytes. + * + * This method doesn't use the internal buffer. + * + * Call *reset()* method before calling this method again. + * + * This returns offset that was parsed to. + * Use *finished?* method to check an object is deserialized and call *data* + * method if it returns true. + * + * UnpackError is throw when parse error is occured. + */ static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) { CHECK_STRING_TYPE(data); @@ -428,7 +555,16 @@ static VALUE MessagePack_Unpacker_execute(VALUE self, VALUE data, VALUE off) (size_t)NUM2ULONG(off), (size_t)RSTRING_LEN(data)); } -/* compat */ +/** + * Document-method: MessagePack::Unpacker#finished? + * + * call-seq: + * unpacker.finished? + * + * Returns true if an object is ready to get with data method. + * + * Use this method with execute method. + */ static VALUE MessagePack_Unpacker_finished_p(VALUE self) { UNPACKER(self, mp); @@ -438,14 +574,30 @@ static VALUE MessagePack_Unpacker_finished_p(VALUE self) return Qfalse; } -/* compat */ +/** + * Document-method: MessagePack::Unpacker#data + * + * call-seq: + * unpacker.data + * + * Gets the object deserialized by execute method. + * + * Use this method with execute method. + */ static VALUE MessagePack_Unpacker_data(VALUE self) { UNPACKER(self, mp); return template_data(mp); } -/* compat */ +/** + * Document-method: MessagePack::Unpacker#reset + * + * call-seq: + * unpacker.reset + * + * Resets the internal state of the unpacker. + */ static VALUE MessagePack_Unpacker_reset(VALUE self) { UNPACKER(self, mp); @@ -467,20 +619,96 @@ void Init_msgpack_unpack(VALUE mMessagePack) eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); + rb_define_method(cUnpacker, "initialize", MessagePack_Unpacker_initialize, -1); + + /* Buffered API */ rb_define_method(cUnpacker, "feed", MessagePack_Unpacker_feed, 1); rb_define_method(cUnpacker, "fill", MessagePack_Unpacker_fill, 0); rb_define_method(cUnpacker, "each", MessagePack_Unpacker_each, 0); rb_define_method(cUnpacker, "stream", MessagePack_Unpacker_stream_get, 0); rb_define_method(cUnpacker, "stream=", MessagePack_Unpacker_stream_set, 1); - rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); - rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2); - /* backward compatibility */ + /* Unbuffered API */ rb_define_method(cUnpacker, "execute", MessagePack_Unpacker_execute, 2); rb_define_method(cUnpacker, "execute_limit", MessagePack_Unpacker_execute_limit, 3); rb_define_method(cUnpacker, "finished?", MessagePack_Unpacker_finished_p, 0); rb_define_method(cUnpacker, "data", MessagePack_Unpacker_data, 0); rb_define_method(cUnpacker, "reset", MessagePack_Unpacker_reset, 0); + + /** + * MessagePack module is defined in rbinit.c file. + * mMessagePack = rb_define_module("MessagePack"); + */ + rb_define_module_function(mMessagePack, "unpack", MessagePack_unpack, 1); + rb_define_module_function(mMessagePack, "unpack_limit", MessagePack_unpack_limit, 2); } +/** + * Document-module: MessagePack::Unpacker + * + * Deserializer class that includes Buffered API and Unbuffered API. + * + * + * Buffered API uses the internal buffer of the Unpacker. + * Following code uses Buffered API with an input stream: + * + * # create an unpacker with input stream. + * pac = MessagePack::Unpacker.new(stdin) + * + * # deserialize object one after another. + * pac.each {|obj| + * # ... + * } + * + * + * Following code doesn't use the input stream and feeds buffer + * using *fill* method. This is useful to use special stream + * or with event-driven I/O library. + * + * # create an unpacker without input stream. + * pac = MessagePack::Unpacker.new() + * + * # feed buffer to the internal buffer. + * pac.feed(input_bytes) + * + * # deserialize object one after another. + * pac.each {|obj| + * # ... + * } + * + * You can manage the buffer manually with the combination of + * *execute*, *finished?*, *data* and *reset* method. + * + * # create an unpacker. + * pac = MessagePack::Unpacker.new() + * + * # manage buffer and offset manually. + * offset = 0 + * buffer = '' + * + * # read some data into the buffer. + * buffer << [1,2,3].to_msgpack + * buffer << [4,5,6].to_msgpack + * + * while true + * offset = pac.execute(buffer, offset) + * + * if pac.finished? + * obj = pac.data + * + * buffer.slice!(0, offset) + * offset = 0 + * pac.reset + * + * # do something with the object + * # ... + * + * # repeat execution if there are more data. + * next unless buffer.empty? + * end + * + * break + * end + */ + From dbebe9771b276bd286b2ccdb8ac88dd17a83524c Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 25 May 2010 02:55:58 +0900 Subject: [PATCH 0490/1648] ruby: update rdoc --- ruby/pack.c | 5 ----- ruby/rbinit.c | 8 ++++++++ ruby/unpack.c | 7 ++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ruby/pack.c b/ruby/pack.c index 7f56923..bbeac4a 100644 --- a/ruby/pack.c +++ b/ruby/pack.c @@ -59,7 +59,6 @@ static ID s_append; * nil.to_msgpack(out = '') -> String * * Serializes the nil into raw bytes. - * This calls to_msgpack reflectively for internal elements. */ static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self) { @@ -76,7 +75,6 @@ static VALUE MessagePack_NilClass_to_msgpack(int argc, VALUE *argv, VALUE self) * true.to_msgpack(out = '') -> String * * Serializes the true into raw bytes. - * This calls to_msgpack reflectively for internal elements. */ static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self) { @@ -93,7 +91,6 @@ static VALUE MessagePack_TrueClass_to_msgpack(int argc, VALUE *argv, VALUE self) * false.to_msgpack(out = '') -> String * * Serializes false into raw bytes. - * This calls to_msgpack reflectively for internal elements. */ static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self) { @@ -110,7 +107,6 @@ static VALUE MessagePack_FalseClass_to_msgpack(int argc, VALUE *argv, VALUE self * fixnum.to_msgpack(out = '') -> String * * Serializes the Fixnum into raw bytes. - * This calls to_msgpack reflectively for internal elements. */ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) { @@ -131,7 +127,6 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self) * bignum.to_msgpack(out = '') -> String * * Serializes the Bignum into raw bytes. - * This calls to_msgpack reflectively for internal elements. */ static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self) { diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 050abde..4ad6beb 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -27,10 +27,18 @@ static VALUE mMessagePack; * It enables to exchange structured objects between many languages like JSON. * But unlike JSON, it is very fast and small. * + * You can install MessagePack with rubygems. + * + * gem install msgpack + * + * Simple usage is as follows. + * * require 'msgpack' * msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03" * MessagePack.unpack(msg) #=> [1,2,3] * + * Use Unpacker class for streaming deserialization. + * */ void Init_msgpack(void) { diff --git a/ruby/unpack.c b/ruby/unpack.c index 9a2c457..dec40c6 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -654,7 +654,7 @@ void Init_msgpack_unpack(VALUE mMessagePack) * Following code uses Buffered API with an input stream: * * # create an unpacker with input stream. - * pac = MessagePack::Unpacker.new(stdin) + * pac = MessagePack::Unpacker.new(STDIN) * * # deserialize object one after another. * pac.each {|obj| @@ -663,8 +663,8 @@ void Init_msgpack_unpack(VALUE mMessagePack) * * * Following code doesn't use the input stream and feeds buffer - * using *fill* method. This is useful to use special stream - * or with event-driven I/O library. + * manually. This is useful to use special stream or with + * event-driven I/O library. * * # create an unpacker without input stream. * pac = MessagePack::Unpacker.new() @@ -677,6 +677,7 @@ void Init_msgpack_unpack(VALUE mMessagePack) * # ... * } * + * * You can manage the buffer manually with the combination of * *execute*, *finished?*, *data* and *reset* method. * From fc7da17fa2dbdc6385ad95f6848ee59598164440 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Tue, 25 May 2010 02:57:37 +0900 Subject: [PATCH 0491/1648] cpp: add sbuffer::clear() and vrefbuffer::clear() --- cpp/msgpack/sbuffer.h | 4 ++++ cpp/msgpack/sbuffer.hpp | 5 +++++ cpp/msgpack/vrefbuffer.h | 3 +++ cpp/msgpack/vrefbuffer.hpp | 5 +++++ cpp/test/buffer.cc | 25 +++++++++++++++++++++++++ cpp/vrefbuffer.c | 19 +++++++++++++++++++ 6 files changed, 61 insertions(+) diff --git a/cpp/msgpack/sbuffer.h b/cpp/msgpack/sbuffer.h index bc0a8fd..57f424a 100644 --- a/cpp/msgpack/sbuffer.h +++ b/cpp/msgpack/sbuffer.h @@ -77,6 +77,10 @@ static inline char* msgpack_sbuffer_release(msgpack_sbuffer* sbuf) return tmp; } +static inline void msgpack_sbuffer_clear(msgpack_sbuffer* sbuf) +{ + sbuf->size = 0; +} #ifdef __cplusplus } diff --git a/cpp/msgpack/sbuffer.hpp b/cpp/msgpack/sbuffer.hpp index ca06884e..e4a3f96 100644 --- a/cpp/msgpack/sbuffer.hpp +++ b/cpp/msgpack/sbuffer.hpp @@ -72,6 +72,11 @@ public: return msgpack_sbuffer_release(this); } + void clear() + { + msgpack_sbuffer_clear(this); + } + private: void expand_buffer(size_t len) { diff --git a/cpp/msgpack/vrefbuffer.h b/cpp/msgpack/vrefbuffer.h index 38ead67..a08e0d0 100644 --- a/cpp/msgpack/vrefbuffer.h +++ b/cpp/msgpack/vrefbuffer.h @@ -80,6 +80,9 @@ int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to); +void msgpack_vrefbuffer_clear(msgpack_vrefbuffer* vref); + + int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len) { msgpack_vrefbuffer* vbuf = (msgpack_vrefbuffer*)data; diff --git a/cpp/msgpack/vrefbuffer.hpp b/cpp/msgpack/vrefbuffer.hpp index c8eca7b..7e0ffb2 100644 --- a/cpp/msgpack/vrefbuffer.hpp +++ b/cpp/msgpack/vrefbuffer.hpp @@ -78,6 +78,11 @@ public: } } + void clear() + { + msgpack_vrefbuffer_clear(this); + } + private: typedef msgpack_vrefbuffer base; diff --git a/cpp/test/buffer.cc b/cpp/test/buffer.cc index a2e9037..aff0699 100644 --- a/cpp/test/buffer.cc +++ b/cpp/test/buffer.cc @@ -12,6 +12,14 @@ TEST(buffer, sbuffer) EXPECT_EQ(3, sbuf.size()); EXPECT_TRUE( memcmp(sbuf.data(), "aaa", 3) == 0 ); + + sbuf.clear(); + sbuf.write("a", 1); + sbuf.write("a", 1); + sbuf.write("a", 1); + + EXPECT_EQ(3, sbuf.size()); + EXPECT_TRUE( memcmp(sbuf.data(), "aaa", 3) == 0 ); } @@ -32,6 +40,23 @@ TEST(buffer, vrefbuffer) EXPECT_EQ(3, sbuf.size()); EXPECT_TRUE( memcmp(sbuf.data(), "aaa", 3) == 0 ); + + + vbuf.clear(); + vbuf.write("a", 1); + vbuf.write("a", 1); + vbuf.write("a", 1); + + vec = vbuf.vector(); + veclen = vbuf.vector_size(); + + sbuf.clear(); + for(size_t i=0; i < veclen; ++i) { + sbuf.write((const char*)vec[i].iov_base, vec[i].iov_len); + } + + EXPECT_EQ(3, sbuf.size()); + EXPECT_TRUE( memcmp(sbuf.data(), "aaa", 3) == 0 ); } diff --git a/cpp/vrefbuffer.c b/cpp/vrefbuffer.c index 136372f..a27b138 100644 --- a/cpp/vrefbuffer.c +++ b/cpp/vrefbuffer.c @@ -75,6 +75,25 @@ void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf) free(vbuf->array); } +void msgpack_vrefbuffer_clear(msgpack_vrefbuffer* vbuf) +{ + msgpack_vrefbuffer_chunk* c = vbuf->inner_buffer.head->next; + msgpack_vrefbuffer_chunk* n; + while(c != NULL) { + n = c->next; + free(c); + c = n; + } + + msgpack_vrefbuffer_inner_buffer* const ib = &vbuf->inner_buffer; + msgpack_vrefbuffer_chunk* chunk = ib->head; + chunk->next = NULL; + ib->free = vbuf->chunk_size; + ib->ptr = ((char*)chunk) + sizeof(msgpack_vrefbuffer_chunk); + + vbuf->tail = vbuf->array; +} + int msgpack_vrefbuffer_append_ref(msgpack_vrefbuffer* vbuf, const char* buf, unsigned int len) { From 26bc835c7e702769375201391bf4965b08c52516 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 04:30:49 +0900 Subject: [PATCH 0492/1648] ruby: buffer rewinding --- ruby/unpack.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index dec40c6..ba20ae6 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -27,6 +27,11 @@ static ID s_readpartial; int s_ascii_8bit; #endif +static ID s_slice_bang; +#ifdef RUBY_VM +static ID s_clear; +#endif + typedef struct { int finished; VALUE source; @@ -125,6 +130,7 @@ static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) { *o = (l <= COW_MIN_SIZE) ? rb_str_new(p, l) : rb_str_substr(u->source, p - b, l); return 0; } +//{ *o = rb_str_new(p, l); return 0; } #include "msgpack/unpack_template.h" @@ -249,6 +255,14 @@ static ID append_method_of(VALUE stream) } } +#ifndef MSGPACK_UNPACKER_BUFFER_INIT_SIZE +#define MSGPACK_UNPACKER_BUFFER_INIT_SIZE (32*1024) +#endif + +#ifndef MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE +#define MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE (8*1024) +#endif + /** * Document-method: MessagePack::Unpacker#initialize * @@ -283,9 +297,9 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) template_init(mp); mp->user.finished = 0; mp->user.offset = 0; - mp->user.buffer = rb_str_new("",0); + mp->user.buffer = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_INIT_SIZE); mp->user.stream = stream; - mp->user.streambuf = rb_str_new("",0); + mp->user.streambuf = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE); mp->user.stream_append_method = append_method_of(stream); return self; } @@ -322,6 +336,63 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) } +#ifdef RUBY_VM +# ifndef STR_SHARED +# define STR_SHARED FL_USER2 +# endif +# ifndef STR_NOEMBED +# define STR_NOEMBED FL_USER1 +# endif +# ifndef STR_ASSOC +# define STR_ASSOC FL_USER3 +# endif +# ifndef STR_NOCAPA_P +# define STR_NOCAPA_P(s) (FL_TEST(s,STR_NOEMBED) && FL_ANY(s,STR_SHARED|STR_ASSOC)) +# endif +# define NEED_MORE_CAPA(s,size) (!STR_NOCAPA_P(s) && RSTRING(s)->as.heap.aux.capa < size) +#else +# ifndef STR_NOCAPA +# ifndef STR_ASSOC +# define STR_ASSOC FL_USER3 +# endif +# ifndef ELTS_SHARED +# define ELTS_SHARED FL_USER2 +# endif +# define STR_NOCAPA (ELTS_SHARED|STR_ASSOC) +# endif +# define NEED_MORE_CAPA(s,size) (!FL_TEST(s,STR_NOCAPA) && RSTRING(s)->aux.capa < size) +#endif + +static void try_rewind_buffer(msgpack_unpack_t* mp, size_t required) +{ + VALUE buffer = mp->user.buffer; + + size_t need_capa = RSTRING_LEN(buffer) + required; + + if(NEED_MORE_CAPA(buffer, need_capa)) { + /* FIXME +#ifdef RUBY_VM + if(RSTRING_LEN(buffer) <= mp->user.offset) { + rb_funcall(buffer, s_clear, 0); + mp->user.offset = 0; + return; + } +#endif + rb_funcall(buffer, s_slice_bang, 2, LONG2FIX(0), LONG2FIX(mp->user.offset)); + mp->user.offset = 0; + */ + size_t not_parsed = RSTRING_LEN(buffer) - mp->user.offset; + size_t nsize = MSGPACK_UNPACKER_BUFFER_INIT_SIZE * 2; + while(nsize < not_parsed + required) { + nsize *= 2; + } + VALUE nbuffer = rb_str_buf_new(nsize); + rb_str_buf_cat(nbuffer, RSTRING_PTR(buffer)+mp->user.offset, not_parsed); + mp->user.buffer = nbuffer; + mp->user.offset = 0; + } +} + /** * Document-method: MessagePack::Unpacker#feed * @@ -334,6 +405,7 @@ static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) { UNPACKER(self, mp); StringValue(data); + try_rewind_buffer(mp, RSTRING_LEN(data)); rb_str_cat(mp->user.buffer, RSTRING_PTR(data), RSTRING_LEN(data)); return Qnil; } @@ -363,12 +435,13 @@ static VALUE MessagePack_Unpacker_fill(VALUE self) long len; if(RSTRING_LEN(mp->user.buffer) == 0) { rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, - LONG2FIX(64*1024), mp->user.buffer); + LONG2FIX(MSGPACK_UNPACKER_BUFFER_INIT_SIZE), mp->user.buffer); len = RSTRING_LEN(mp->user.buffer); } else { rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, - LONG2FIX(64*1024), mp->user.streambuf); + LONG2FIX(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE), mp->user.streambuf); len = RSTRING_LEN(mp->user.streambuf); + try_rewind_buffer(mp, len); rb_str_cat(mp->user.buffer, RSTRING_PTR(mp->user.streambuf), RSTRING_LEN(mp->user.streambuf)); } @@ -616,6 +689,11 @@ void Init_msgpack_unpack(VALUE mMessagePack) s_ascii_8bit = rb_enc_find_index("ASCII-8BIT"); #endif + s_slice_bang = rb_intern("slice!"); +#ifdef RUBY_VM + s_clear = rb_intern("clear"); +#endif + eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); From 5fa589691c780735fae153e7e6907906663349a1 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 07:01:28 +0900 Subject: [PATCH 0493/1648] ruby: use malloc/realloc for stream buffer --- ruby/unpack.c | 180 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 118 insertions(+), 62 deletions(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index ba20ae6..c4b5e29 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -27,16 +27,17 @@ static ID s_readpartial; int s_ascii_8bit; #endif -static ID s_slice_bang; -#ifdef RUBY_VM -static ID s_clear; -#endif +struct unpack_buffer { + size_t size; + size_t free; + char* ptr; +}; typedef struct { int finished; VALUE source; size_t offset; - VALUE buffer; + struct unpack_buffer buffer; VALUE stream; VALUE streambuf; ID stream_append_method; @@ -129,8 +130,16 @@ static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, #endif static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) -{ *o = (l <= COW_MIN_SIZE) ? rb_str_new(p, l) : rb_str_substr(u->source, p - b, l); return 0; } +//{ *o = (l <= COW_MIN_SIZE) ? rb_str_new(p, l) : rb_str_substr(u->source, p - b, l); return 0; } //{ *o = rb_str_new(p, l); return 0; } +{ + if(u->source == Qnil || l <= COW_MIN_SIZE) { + *o = rb_str_new(p, l); + } else { + *o = rb_str_substr(u->source, p - b, l); + } + return 0; +} #include "msgpack/unpack_template.h" @@ -209,6 +218,27 @@ static int template_execute_wrap(msgpack_unpack_t* mp, return ret; } +static int template_execute_wrap_each(msgpack_unpack_t* mp, + const char* ptr, size_t dlen, size_t* from) +{ + VALUE args[4] = { + (VALUE)mp, + (VALUE)ptr, + (VALUE)dlen, + (VALUE)from, + }; + + // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ + rb_gc_disable(); + + int ret = (int)rb_rescue(template_execute_do, (VALUE)args, + template_execute_rescue, Qnil); + + rb_gc_enable(); + + return ret; +} + static VALUE cUnpacker; @@ -220,15 +250,26 @@ static VALUE cUnpacker; static VALUE eUnpackError; +#ifndef MSGPACK_UNPACKER_BUFFER_INIT_SIZE +#define MSGPACK_UNPACKER_BUFFER_INIT_SIZE (32*1024) +#endif + +#ifndef MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE +#define MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE (8*1024) +#endif + static void MessagePack_Unpacker_free(void* data) { - if(data) { free(data); } + if(data) { + msgpack_unpack_t* mp = (msgpack_unpack_t*)data; + free(mp->user.buffer.ptr); + free(mp); + } } static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp) { unsigned int i; - rb_gc_mark(mp->user.buffer); rb_gc_mark(mp->user.stream); rb_gc_mark(mp->user.streambuf); for(i=0; i < mp->top; ++i) { @@ -255,14 +296,6 @@ static ID append_method_of(VALUE stream) } } -#ifndef MSGPACK_UNPACKER_BUFFER_INIT_SIZE -#define MSGPACK_UNPACKER_BUFFER_INIT_SIZE (32*1024) -#endif - -#ifndef MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE -#define MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE (8*1024) -#endif - /** * Document-method: MessagePack::Unpacker#initialize * @@ -297,7 +330,9 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self) template_init(mp); mp->user.finished = 0; mp->user.offset = 0; - mp->user.buffer = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_INIT_SIZE); + mp->user.buffer.size = 0; + mp->user.buffer.free = 0; + mp->user.buffer.ptr = NULL; mp->user.stream = stream; mp->user.streambuf = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE); mp->user.stream_append_method = append_method_of(stream); @@ -363,34 +398,66 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val) # define NEED_MORE_CAPA(s,size) (!FL_TEST(s,STR_NOCAPA) && RSTRING(s)->aux.capa < size) #endif -static void try_rewind_buffer(msgpack_unpack_t* mp, size_t required) +static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len) { - VALUE buffer = mp->user.buffer; + struct unpack_buffer* buffer = &mp->user.buffer; - size_t need_capa = RSTRING_LEN(buffer) + required; + if(buffer->size == 0) { + char* tmp = (char*)malloc(MSGPACK_UNPACKER_BUFFER_INIT_SIZE); + // FIXME check tmp == NULL + buffer->ptr = tmp; + buffer->free = MSGPACK_UNPACKER_BUFFER_INIT_SIZE; + buffer->size = 0; - if(NEED_MORE_CAPA(buffer, need_capa)) { - /* FIXME -#ifdef RUBY_VM - if(RSTRING_LEN(buffer) <= mp->user.offset) { - rb_funcall(buffer, s_clear, 0); - mp->user.offset = 0; - return; - } -#endif - rb_funcall(buffer, s_slice_bang, 2, LONG2FIX(0), LONG2FIX(mp->user.offset)); - mp->user.offset = 0; - */ - size_t not_parsed = RSTRING_LEN(buffer) - mp->user.offset; - size_t nsize = MSGPACK_UNPACKER_BUFFER_INIT_SIZE * 2; - while(nsize < not_parsed + required) { - nsize *= 2; - } - VALUE nbuffer = rb_str_buf_new(nsize); - rb_str_buf_cat(nbuffer, RSTRING_PTR(buffer)+mp->user.offset, not_parsed); - mp->user.buffer = nbuffer; + } else if(buffer->size <= mp->user.offset) { + /* clear buffer and rewind offset */ + buffer->free += buffer->size; + buffer->size = 0; mp->user.offset = 0; } + + if(len <= buffer->free) { + /* enough free space: just copy */ + memcpy(buffer->ptr+buffer->size, ptr, len); + buffer->size += len; + buffer->free -= len; + return; + } + + size_t csize = buffer->size + buffer->free; + + if(mp->user.offset <= buffer->size / 2) { + /* parsed less than half: realloc and copy */ + csize *= 2; + while(csize < buffer->size + len) { + csize *= 2; + } + char* tmp = (char*)realloc(buffer->ptr, csize); + // FIXME check tmp == NULL + memcpy(tmp + buffer->size, ptr, len); + buffer->ptr = tmp; + buffer->free = csize - buffer->size; + return; + } + + size_t not_parsed = buffer->size - mp->user.offset; + + if(csize < not_parsed + len) { + /* more buffer size */ + csize *= 2; + while(csize < not_parsed + len) { + csize *= 2; + } + char* tmp = (char*)realloc(buffer->ptr, csize); + // FIXME check tmp == NULL + buffer->ptr = tmp; + } + + memcpy(buffer->ptr+not_parsed, ptr, not_parsed); + buffer->size = not_parsed; + buffer->free = csize - buffer->size; + buffer->ptr = buffer->ptr; + mp->user.offset = 0; } /** @@ -405,8 +472,7 @@ static VALUE MessagePack_Unpacker_feed(VALUE self, VALUE data) { UNPACKER(self, mp); StringValue(data); - try_rewind_buffer(mp, RSTRING_LEN(data)); - rb_str_cat(mp->user.buffer, RSTRING_PTR(data), RSTRING_LEN(data)); + feed_buffer(mp, RSTRING_PTR(data), RSTRING_LEN(data)); return Qnil; } @@ -432,18 +498,12 @@ static VALUE MessagePack_Unpacker_fill(VALUE self) return Qnil; } - long len; - if(RSTRING_LEN(mp->user.buffer) == 0) { - rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, - LONG2FIX(MSGPACK_UNPACKER_BUFFER_INIT_SIZE), mp->user.buffer); - len = RSTRING_LEN(mp->user.buffer); - } else { - rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, - LONG2FIX(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE), mp->user.streambuf); - len = RSTRING_LEN(mp->user.streambuf); - try_rewind_buffer(mp, len); - rb_str_cat(mp->user.buffer, RSTRING_PTR(mp->user.streambuf), RSTRING_LEN(mp->user.streambuf)); - } + rb_funcall(mp->user.stream, mp->user.stream_append_method, 2, + LONG2FIX(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE), + mp->user.streambuf); + + size_t len = RSTRING_LEN(mp->user.streambuf); + feed_buffer(mp, RSTRING_PTR(mp->user.streambuf), len); return LONG2FIX(len); } @@ -470,7 +530,7 @@ static VALUE MessagePack_Unpacker_each(VALUE self) #endif while(1) { - if(RSTRING_LEN(mp->user.buffer) <= mp->user.offset) { + if(mp->user.buffer.size <= mp->user.offset) { do_fill: { VALUE len = MessagePack_Unpacker_fill(self); @@ -480,8 +540,9 @@ static VALUE MessagePack_Unpacker_each(VALUE self) } } - ret = template_execute_wrap(mp, mp->user.buffer, - RSTRING_LEN(mp->user.buffer), &mp->user.offset); + ret = template_execute_wrap_each(mp, + mp->user.buffer.ptr, mp->user.buffer.size, + &mp->user.offset); if(ret < 0) { rb_raise(eUnpackError, "parse error."); @@ -689,11 +750,6 @@ void Init_msgpack_unpack(VALUE mMessagePack) s_ascii_8bit = rb_enc_find_index("ASCII-8BIT"); #endif - s_slice_bang = rb_intern("slice!"); -#ifdef RUBY_VM - s_clear = rb_intern("clear"); -#endif - eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError); cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject); rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc); From 94c39985079cff23d85d333ea69d554216ac79e0 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 07:43:05 +0900 Subject: [PATCH 0494/1648] ruby: update gemspec --- ruby/README | 28 ++++++++++++++++++---------- ruby/msgpack.gemspec | 6 +++--- ruby/pack.h | 2 +- ruby/rbinit.c | 4 ++-- ruby/unpack.h | 2 +- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/ruby/README b/ruby/README index 859ae2b..051a769 100644 --- a/ruby/README +++ b/ruby/README @@ -1,29 +1,37 @@ = MessagePack - == Description +MessagePack is a binary-based efficient object serialization library. +It enables to exchange structured objects between many languages like JSON. +But unlike JSON, it is very fast and small. + +Simple usage is as follows: + + require 'msgpack' + msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03" + MessagePack.unpack(msg) #=> [1,2,3] + +Use MessagePack::Unpacker for streaming deserialization. + == Installation === Archive Installation - rake install + ruby extconf.rb + make + make install === Gem Installation - gem install msgpack - - -== Features/Problems - - -== Synopsis + gem install msgpack == Copyright Author:: frsyuki -Copyright:: Copyright (c) 2008-2009 frsyuki +Copyright:: Copyright (c) 2008-2010 FURUHASHI Sadayuki License:: Apache License, Version 2.0 + diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index c5e8c8c..0f08628 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,14 +1,14 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.3.9" + s.version = "0.4.0" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" s.homepage = "http://msgpack.sourceforge.net/" s.rubyforge_project = "msgpack" - s.has_rdoc = false - s.extra_rdoc_files = ["README", "ChangeLog", "AUTHORS"] + s.has_rdoc = true + s.rdoc_options = ["ext"] s.require_paths = ["lib"] s.files = Dir["ext/**/*", "msgpack/**/*", "test/**/*"] s.test_files = Dir["test/test_*.rb"] diff --git a/ruby/pack.h b/ruby/pack.h index c9b08a4..f162a86 100644 --- a/ruby/pack.h +++ b/ruby/pack.h @@ -1,7 +1,7 @@ /* * MessagePack for Ruby packing routine * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ruby/rbinit.c b/ruby/rbinit.c index 4ad6beb..ad51f6b 100644 --- a/ruby/rbinit.c +++ b/ruby/rbinit.c @@ -1,7 +1,7 @@ /* * MessagePack for Ruby * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ static VALUE mMessagePack; * * gem install msgpack * - * Simple usage is as follows. + * Simple usage is as follows: * * require 'msgpack' * msg = [1,2,3].to_msgpack #=> "\x93\x01\x02\x03" diff --git a/ruby/unpack.h b/ruby/unpack.h index ce2a8de..91d3eb7 100644 --- a/ruby/unpack.h +++ b/ruby/unpack.h @@ -1,7 +1,7 @@ /* * MessagePack for Ruby unpacking routine * - * Copyright (C) 2008-2009 FURUHASHI Sadayuki + * Copyright (C) 2008-2010 FURUHASHI Sadayuki * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 47185d757ebee52cc06775a843b3bf06292d8bf1 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 07:55:02 +0900 Subject: [PATCH 0495/1648] ruby: version 0.4.0 --- ruby/unpack.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index c4b5e29..c93af35 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -130,8 +130,6 @@ static inline int template_callback_map_item(unpack_user* u, VALUE* c, VALUE k, #endif static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, VALUE* o) -//{ *o = (l <= COW_MIN_SIZE) ? rb_str_new(p, l) : rb_str_substr(u->source, p - b, l); return 0; } -//{ *o = rb_str_new(p, l); return 0; } { if(u->source == Qnil || l <= COW_MIN_SIZE) { *o = rb_str_new(p, l); From 293293c23cf6672827248910f133b22d0d58fcf3 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 18:01:27 +0900 Subject: [PATCH 0496/1648] ruby: set mp->user.source = Qnil before tempalte_execute_do on Unpacker#each --- ruby/unpack.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/ruby/unpack.c b/ruby/unpack.c index c93af35..65852b9 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -156,15 +156,25 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha rb_raise(rb_eTypeError, "instance of String needed"); \ } +#ifdef RUBY_VM +#define RERAISE rb_exc_raise(rb_errinfo()) +#else +#define RERAISE rb_exc_raise(ruby_errinfo) +#endif -static VALUE template_execute_rescue(VALUE nouse) + +static VALUE template_execute_rescue(VALUE data) { rb_gc_enable(); -#ifdef RUBY_VM - rb_exc_raise(rb_errinfo()); -#else - rb_exc_raise(ruby_errinfo); -#endif + VALUE* resc = (VALUE*)data; + rb_enc_set_index(resc[0], (int)resc[1]); + RERAISE; +} + +static VALUE template_execute_rescue_each(VALUE nouse) +{ + rb_gc_enable(); + RERAISE; } static VALUE template_execute_do(VALUE argv) @@ -192,7 +202,6 @@ static int template_execute_wrap(msgpack_unpack_t* mp, }; #ifdef HAVE_RUBY_ENCODING_H - // FIXME encodingã‚’ASCII-8BITã«ã™ã‚‹ int enc_orig = rb_enc_get_index(str); rb_enc_set_index(str, s_ascii_8bit); #endif @@ -202,10 +211,10 @@ static int template_execute_wrap(msgpack_unpack_t* mp, mp->user.source = str; - int ret = (int)rb_rescue(template_execute_do, (VALUE)args, - template_execute_rescue, Qnil); + VALUE resc[2] = {str, enc_orig}; - mp->user.source = Qnil; + int ret = (int)rb_rescue(template_execute_do, (VALUE)args, + template_execute_rescue, (VALUE)resc); rb_gc_enable(); @@ -229,8 +238,10 @@ static int template_execute_wrap_each(msgpack_unpack_t* mp, // FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„ rb_gc_disable(); + mp->user.source = Qnil; + int ret = (int)rb_rescue(template_execute_do, (VALUE)args, - template_execute_rescue, Qnil); + template_execute_rescue_each, Qnil); rb_gc_enable(); @@ -401,8 +412,7 @@ static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len) struct unpack_buffer* buffer = &mp->user.buffer; if(buffer->size == 0) { - char* tmp = (char*)malloc(MSGPACK_UNPACKER_BUFFER_INIT_SIZE); - // FIXME check tmp == NULL + char* tmp = ALLOC_N(char, MSGPACK_UNPACKER_BUFFER_INIT_SIZE); buffer->ptr = tmp; buffer->free = MSGPACK_UNPACKER_BUFFER_INIT_SIZE; buffer->size = 0; @@ -430,8 +440,7 @@ static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len) while(csize < buffer->size + len) { csize *= 2; } - char* tmp = (char*)realloc(buffer->ptr, csize); - // FIXME check tmp == NULL + char* tmp = REALLOC_N(buffer->ptr, char, csize); memcpy(tmp + buffer->size, ptr, len); buffer->ptr = tmp; buffer->free = csize - buffer->size; @@ -446,8 +455,7 @@ static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len) while(csize < not_parsed + len) { csize *= 2; } - char* tmp = (char*)realloc(buffer->ptr, csize); - // FIXME check tmp == NULL + char* tmp = REALLOC_N(buffer->ptr, char, csize); buffer->ptr = tmp; } From 3fbcde4bd74e00e208b10aa00c389256de0ba317 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Wed, 26 May 2010 18:11:09 +0900 Subject: [PATCH 0497/1648] ruby: don't use rb_enc_set/get on ruby 1.8 --- ruby/msgpack.gemspec | 2 +- ruby/unpack.c | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec index 0f08628..e10622f 100644 --- a/ruby/msgpack.gemspec +++ b/ruby/msgpack.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.name = "msgpack" - s.version = "0.4.0" + s.version = "0.4.1" s.summary = "MessagePack, a binary-based efficient data interchange format." s.author = "FURUHASHI Sadayuki" s.email = "frsyuki@users.sourceforge.jp" diff --git a/ruby/unpack.c b/ruby/unpack.c index 65852b9..0f7b9f0 100644 --- a/ruby/unpack.c +++ b/ruby/unpack.c @@ -163,15 +163,17 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha #endif -static VALUE template_execute_rescue(VALUE data) +#ifdef HAVE_RUBY_ENCODING_H +static VALUE template_execute_rescue_enc(VALUE data) { rb_gc_enable(); VALUE* resc = (VALUE*)data; rb_enc_set_index(resc[0], (int)resc[1]); RERAISE; } +#endif -static VALUE template_execute_rescue_each(VALUE nouse) +static VALUE template_execute_rescue(VALUE nouse) { rb_gc_enable(); RERAISE; @@ -211,10 +213,14 @@ static int template_execute_wrap(msgpack_unpack_t* mp, mp->user.source = str; +#ifdef HAVE_RUBY_ENCODING_H VALUE resc[2] = {str, enc_orig}; - int ret = (int)rb_rescue(template_execute_do, (VALUE)args, - template_execute_rescue, (VALUE)resc); + template_execute_rescue_enc, (VALUE)resc); +#else + int ret = (int)rb_rescue(template_execute_do, (VALUE)args, + template_execute_rescue, Qnil); +#endif rb_gc_enable(); @@ -241,7 +247,7 @@ static int template_execute_wrap_each(msgpack_unpack_t* mp, mp->user.source = Qnil; int ret = (int)rb_rescue(template_execute_do, (VALUE)args, - template_execute_rescue_each, Qnil); + template_execute_rescue, Qnil); rb_gc_enable(); From 6df86384ca2e20c34b78aeb5d9f72a885bd50a16 Mon Sep 17 00:00:00 2001 From: frsyuki Date: Sat, 29 May 2010 07:54:49 +0900 Subject: [PATCH 0498/1648] java: update javadoc --- java/src/main/java/org/msgpack/Packer.java | 16 ++++++++++++++++ java/src/main/java/org/msgpack/Unpacker.java | 8 +++++--- java/src/main/java/org/msgpack/package-info.java | 8 ++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 java/src/main/java/org/msgpack/package-info.java diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java index 3fa421f..a92d2b6 100644 --- a/java/src/main/java/org/msgpack/Packer.java +++ b/java/src/main/java/org/msgpack/Packer.java @@ -23,6 +23,22 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.Map; +/** + * Packer enables you to serialize objects into OutputStream. + * + * 
+ * // create a packer with output stream
+ * Packer pk = new Packer(System.out);
+ *
+ * // store an object with pack() method.
+ * pk.pack(1);
+ *
+ * // you can store String, List, Map, byte[] and primitive types.
+ * pk.pack(new ArrayList());
+ *
+ * + * You can serialize objects that implements {@link MessagePackable} interface. + */ public class Packer { protected byte[] castBytes = new byte[9]; protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes); diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java index e84aff9..39bd8fa 100644 --- a/java/src/main/java/org/msgpack/Unpacker.java +++ b/java/src/main/java/org/msgpack/Unpacker.java @@ -24,11 +24,13 @@ import java.util.Iterator; import java.nio.ByteBuffer; /** - * Deserializer class that includes Buffered API, Unbuffered API, - * Schema API and Direct Conversion API. + * Unpacker enables you to deserialize objects from stream. + * + * Unpacker provides Buffered API, Unbuffered API, Schema API + * and Direct Conversion API. * * Buffered API uses the internal buffer of the Unpacker. - * Following code uses Buffered API with an input stream: + * Following code uses Buffered API with an InputStream: * 
  * // create an unpacker with input stream
  * Unpacker pac = new Unpacker(System.in);
diff --git a/java/src/main/java/org/msgpack/package-info.java b/java/src/main/java/org/msgpack/package-info.java
new file mode 100644
index 0000000..7e9b8a2
--- /dev/null
+++ b/java/src/main/java/org/msgpack/package-info.java
@@ -0,0 +1,8 @@
+/**
+ * MessagePack is a binary-based efficient object serialization library.
+ * It enables to exchange structured objects between many languages like JSON.
+ * But unlike JSON, it is very fast and small.
+ *
+ * Use {@link Packer} to serialize and {@link Unpacker} to deserialize.
+ */
+package org.msgpack;

From 81b0c316cda14629821005cba5ce34c80ccd61d2 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 30 May 2010 01:39:48 +0900
Subject: [PATCH 0499/1648] java: Unpacker: rewind internal buffer on filled <=
 offset

---
 java/src/main/java/org/msgpack/Unpacker.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index 39bd8fa..f22c58b 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -288,6 +288,12 @@ public class Unpacker implements Iterable {
 			return;
 		}
 
+		if(impl.filled <= impl.offset) {
+			// rewind the buffer
+			impl.filled = 0;
+			impl.offset = 0;
+		}
+
 		if(impl.buffer.length - impl.filled >= require) {
 			return;
 		}

From 2f5d83f07d7a50bfb7287e1fa28bc7a7fd7e7d49 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 30 May 2010 01:45:07 +0900
Subject: [PATCH 0500/1648] cpp: type::tuple& operator>>: fix conversion type

---
 cpp/msgpack/type/tuple.hpp.erb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cpp/msgpack/type/tuple.hpp.erb b/cpp/msgpack/type/tuple.hpp.erb
index 1b0c172..0d9ae91 100644
--- a/cpp/msgpack/type/tuple.hpp.erb
+++ b/cpp/msgpack/type/tuple.hpp.erb
@@ -141,7 +141,7 @@ type::tuple, A<%=j%><%}%>>& operator>> (
 	if(o.type != type::ARRAY) { throw type_error(); }
 	if(o.via.array.size < <%=i+1%>) { throw type_error(); }
 	<%0.upto(i) {|j|%>
-	o.via.array.ptr[<%=j%>].convert>(&v.template get<<%=j%>>());<%}%>
+	o.via.array.ptr[<%=j%>].convert>::type>(&v.template get<<%=j%>>());<%}%>
 	return v;
 }
 <%}%>

From 602971408ba8c2c1490bd87d2987ab65900a5297 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 30 May 2010 03:02:40 +0900
Subject: [PATCH 0501/1648] cpp: move source files into src/ directory

---
 cpp/Makefile.am                               | 88 +------------------
 cpp/configure.in                              | 19 +++-
 cpp/preprocess                                | 16 ++--
 cpp/src/Makefile.am                           | 75 ++++++++++++++++
 cpp/{ => src}/msgpack.h                       |  0
 cpp/{ => src}/msgpack.hpp                     |  0
 cpp/{ => src}/msgpack/object.h                |  0
 cpp/{ => src}/msgpack/object.hpp              |  0
 cpp/{ => src}/msgpack/pack.h                  |  0
 cpp/{ => src}/msgpack/pack.hpp                |  0
 cpp/{ => src}/msgpack/sbuffer.h               |  0
 cpp/{ => src}/msgpack/sbuffer.hpp             |  0
 cpp/{ => src}/msgpack/type.hpp                |  0
 cpp/{ => src}/msgpack/type/bool.hpp           |  0
 cpp/{ => src}/msgpack/type/define.hpp.erb     |  0
 cpp/{ => src}/msgpack/type/deque.hpp          |  0
 cpp/{ => src}/msgpack/type/float.hpp          |  0
 cpp/{ => src}/msgpack/type/int.hpp            |  0
 cpp/{ => src}/msgpack/type/list.hpp           |  0
 cpp/{ => src}/msgpack/type/map.hpp            |  0
 cpp/{ => src}/msgpack/type/nil.hpp            |  0
 cpp/{ => src}/msgpack/type/pair.hpp           |  0
 cpp/{ => src}/msgpack/type/raw.hpp            |  0
 cpp/{ => src}/msgpack/type/set.hpp            |  0
 cpp/{ => src}/msgpack/type/string.hpp         |  0
 .../msgpack/type/tr1/unordered_map.hpp        |  0
 .../msgpack/type/tr1/unordered_set.hpp        |  0
 cpp/{ => src}/msgpack/type/tuple.hpp.erb      |  0
 cpp/{ => src}/msgpack/type/vector.hpp         |  0
 cpp/{ => src}/msgpack/unpack.h                |  0
 cpp/{ => src}/msgpack/unpack.hpp              |  0
 cpp/{ => src}/msgpack/vrefbuffer.h            |  0
 cpp/{ => src}/msgpack/vrefbuffer.hpp          |  0
 cpp/{ => src}/msgpack/zbuffer.h               |  0
 cpp/{ => src}/msgpack/zbuffer.hpp             |  0
 cpp/{ => src}/msgpack/zone.h                  |  0
 cpp/{ => src}/msgpack/zone.hpp.erb            |  0
 cpp/{ => src}/object.cpp                      |  0
 cpp/{ => src}/objectc.c                       |  0
 cpp/{ => src}/unpack.c                        |  0
 cpp/{ => src}/vrefbuffer.c                    |  0
 cpp/{ => src}/zone.c                          |  0
 cpp/test.mk                                   |  9 --
 cpp/test/Makefile.am                          | 14 ++-
 cpp/{ => test}/msgpack_test.cpp               |  0
 cpp/{ => test}/msgpackc_test.cpp              |  0
 46 files changed, 114 insertions(+), 107 deletions(-)
 create mode 100644 cpp/src/Makefile.am
 rename cpp/{ => src}/msgpack.h (100%)
 rename cpp/{ => src}/msgpack.hpp (100%)
 rename cpp/{ => src}/msgpack/object.h (100%)
 rename cpp/{ => src}/msgpack/object.hpp (100%)
 rename cpp/{ => src}/msgpack/pack.h (100%)
 rename cpp/{ => src}/msgpack/pack.hpp (100%)
 rename cpp/{ => src}/msgpack/sbuffer.h (100%)
 rename cpp/{ => src}/msgpack/sbuffer.hpp (100%)
 rename cpp/{ => src}/msgpack/type.hpp (100%)
 rename cpp/{ => src}/msgpack/type/bool.hpp (100%)
 rename cpp/{ => src}/msgpack/type/define.hpp.erb (100%)
 rename cpp/{ => src}/msgpack/type/deque.hpp (100%)
 rename cpp/{ => src}/msgpack/type/float.hpp (100%)
 rename cpp/{ => src}/msgpack/type/int.hpp (100%)
 rename cpp/{ => src}/msgpack/type/list.hpp (100%)
 rename cpp/{ => src}/msgpack/type/map.hpp (100%)
 rename cpp/{ => src}/msgpack/type/nil.hpp (100%)
 rename cpp/{ => src}/msgpack/type/pair.hpp (100%)
 rename cpp/{ => src}/msgpack/type/raw.hpp (100%)
 rename cpp/{ => src}/msgpack/type/set.hpp (100%)
 rename cpp/{ => src}/msgpack/type/string.hpp (100%)
 rename cpp/{ => src}/msgpack/type/tr1/unordered_map.hpp (100%)
 rename cpp/{ => src}/msgpack/type/tr1/unordered_set.hpp (100%)
 rename cpp/{ => src}/msgpack/type/tuple.hpp.erb (100%)
 rename cpp/{ => src}/msgpack/type/vector.hpp (100%)
 rename cpp/{ => src}/msgpack/unpack.h (100%)
 rename cpp/{ => src}/msgpack/unpack.hpp (100%)
 rename cpp/{ => src}/msgpack/vrefbuffer.h (100%)
 rename cpp/{ => src}/msgpack/vrefbuffer.hpp (100%)
 rename cpp/{ => src}/msgpack/zbuffer.h (100%)
 rename cpp/{ => src}/msgpack/zbuffer.hpp (100%)
 rename cpp/{ => src}/msgpack/zone.h (100%)
 rename cpp/{ => src}/msgpack/zone.hpp.erb (100%)
 rename cpp/{ => src}/object.cpp (100%)
 rename cpp/{ => src}/objectc.c (100%)
 rename cpp/{ => src}/unpack.c (100%)
 rename cpp/{ => src}/vrefbuffer.c (100%)
 rename cpp/{ => src}/zone.c (100%)
 delete mode 100644 cpp/test.mk
 rename cpp/{ => test}/msgpack_test.cpp (100%)
 rename cpp/{ => test}/msgpackc_test.cpp (100%)

diff --git a/cpp/Makefile.am b/cpp/Makefile.am
index 08eb7a5..6b37803 100644
--- a/cpp/Makefile.am
+++ b/cpp/Makefile.am
@@ -1,75 +1,6 @@
+SUBDIRS = src test
 
-lib_LTLIBRARIES = libmsgpack.la
-
-libmsgpack_la_SOURCES = \
-		unpack.c \
-		objectc.c \
-		vrefbuffer.c \
-		zone.c \
-		object.cpp
-
-# -version-info CURRENT:REVISION:AGE
-libmsgpack_la_LDFLAGS = -version-info 3:0:0
-
-
-# backward compatibility
-lib_LTLIBRARIES += libmsgpackc.la
-
-libmsgpackc_la_SOURCES = \
-		unpack.c \
-		objectc.c \
-		vrefbuffer.c \
-		zone.c
-
-libmsgpackc_la_LDFLAGS = -version-info 2:0:0
-
-# work around for duplicated file name
-kumo_manager_CFLAGS = $(AM_CFLAGS)
-kumo_manager_CXXFLAGS = $(AM_CXXFLAGS)
-
-
-nobase_include_HEADERS = \
-		msgpack/pack_define.h \
-		msgpack/pack_template.h \
-		msgpack/unpack_define.h \
-		msgpack/unpack_template.h \
-		msgpack/sysdep.h \
-		msgpack.h \
-		msgpack/sbuffer.h \
-		msgpack/vrefbuffer.h \
-		msgpack/zbuffer.h \
-		msgpack/pack.h \
-		msgpack/unpack.h \
-		msgpack/object.h \
-		msgpack/zone.h \
-		msgpack.hpp \
-		msgpack/sbuffer.hpp \
-		msgpack/vrefbuffer.hpp \
-		msgpack/zbuffer.hpp \
-		msgpack/pack.hpp \
-		msgpack/unpack.hpp \
-		msgpack/object.hpp \
-		msgpack/zone.hpp \
-		msgpack/type.hpp \
-		msgpack/type/bool.hpp \
-		msgpack/type/float.hpp \
-		msgpack/type/int.hpp \
-		msgpack/type/list.hpp \
-		msgpack/type/deque.hpp \
-		msgpack/type/map.hpp \
-		msgpack/type/nil.hpp \
-		msgpack/type/pair.hpp \
-		msgpack/type/raw.hpp \
-		msgpack/type/set.hpp \
-		msgpack/type/string.hpp \
-		msgpack/type/vector.hpp \
-		msgpack/type/tuple.hpp \
-		msgpack/type/define.hpp \
-		msgpack/type/tr1/unordered_map.hpp \
-		msgpack/type/tr1/unordered_set.hpp
-
-
-EXTRA_DIST = \
+DOC_FILES = \
 		README.md \
 		LICENSE \
 		NOTICE \
@@ -77,17 +8,6 @@ EXTRA_DIST = \
 		msgpack_vc8.sln \
 		msgpack_vc8.postbuild.bat
 
-SUBDIRS = test
-
-check_PROGRAMS = \
-		msgpackc_test \
-		msgpack_test
-
-msgpackc_test_SOURCES = msgpackc_test.cpp
-msgpackc_test_LDADD = libmsgpack.la -lgtest_main
-
-msgpack_test_SOURCES = msgpack_test.cpp
-msgpack_test_LDADD = libmsgpack.la -lgtest_main
-
-TESTS = $(check_PROGRAMS)
+EXTRA_DIST = \
+		$(DOC_FILES)
 
diff --git a/cpp/configure.in b/cpp/configure.in
index 61fde4f..0895be4 100644
--- a/cpp/configure.in
+++ b/cpp/configure.in
@@ -1,4 +1,4 @@
-AC_INIT(object.cpp)
+AC_INIT(src/object.cpp)
 AC_CONFIG_AUX_DIR(ac)
 AM_INIT_AUTOMAKE(msgpack, 0.5.0)
 AC_CONFIG_HEADER(config.h)
@@ -21,6 +21,21 @@ AC_CHECK_HEADERS(tr1/unordered_map)
 AC_CHECK_HEADERS(tr1/unordered_set)
 AC_LANG_POP([C++])
 
+
+AC_MSG_CHECKING([if debug option is enabled])
+AC_ARG_ENABLE(debug,
+	AS_HELP_STRING([--disable-debug],
+				   [disable assert macros and omit -g option.]) )
+if test "$enable_debug" != "no"; then
+	CXXFLAGS="$CXXFLAGS -g"
+	CFLAGS="$CFLAGS -g"
+else
+	CXXFLAGS="$CXXFLAGS -DNDEBUG"
+	CFLAGS="$CFLAGS -DNDEBUG"
+fi
+AC_MSG_RESULT($enable_debug)
+
+
 AC_CACHE_CHECK([for __sync_* atomic operations], msgpack_cv_atomic_ops, [
 	AC_TRY_LINK([
 		int atomic_sub(int i) { return __sync_sub_and_fetch(&i, 1); }
@@ -39,5 +54,5 @@ add CFLAGS="--march=i686" and CXXFLAGS="-march=i686" options to ./configure as f
 ])
 fi
 
-AC_OUTPUT([Makefile test/Makefile])
+AC_OUTPUT([Makefile src/Makefile test/Makefile])
 
diff --git a/cpp/preprocess b/cpp/preprocess
index 80a8357..452006a 100755
--- a/cpp/preprocess
+++ b/cpp/preprocess
@@ -11,12 +11,12 @@ preprocess() {
 	fi
 }
 
-preprocess msgpack/type/tuple.hpp
-preprocess msgpack/type/define.hpp
-preprocess msgpack/zone.hpp
-cp -f ../msgpack/sysdep.h          msgpack/
-cp -f ../msgpack/pack_define.h     msgpack/
-cp -f ../msgpack/pack_template.h   msgpack/
-cp -f ../msgpack/unpack_define.h   msgpack/
-cp -f ../msgpack/unpack_template.h msgpack/
+preprocess src/msgpack/type/tuple.hpp
+preprocess src/msgpack/type/define.hpp
+preprocess src/msgpack/zone.hpp
+cp -f ../msgpack/sysdep.h          src/msgpack/
+cp -f ../msgpack/pack_define.h     src/msgpack/
+cp -f ../msgpack/pack_template.h   src/msgpack/
+cp -f ../msgpack/unpack_define.h   src/msgpack/
+cp -f ../msgpack/unpack_template.h src/msgpack/
 
diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am
new file mode 100644
index 0000000..4cfa87e
--- /dev/null
+++ b/cpp/src/Makefile.am
@@ -0,0 +1,75 @@
+
+lib_LTLIBRARIES = libmsgpack.la
+
+libmsgpack_la_SOURCES = \
+		unpack.c \
+		objectc.c \
+		vrefbuffer.c \
+		zone.c \
+		object.cpp
+
+# -version-info CURRENT:REVISION:AGE
+libmsgpack_la_LDFLAGS = -version-info 3:0:0
+
+
+# backward compatibility
+lib_LTLIBRARIES += libmsgpackc.la
+
+libmsgpackc_la_SOURCES = \
+		unpack.c \
+		objectc.c \
+		vrefbuffer.c \
+		zone.c
+
+libmsgpackc_la_LDFLAGS = -version-info 2:0:0
+
+# work around for duplicated file name
+kumo_manager_CFLAGS = $(AM_CFLAGS)
+kumo_manager_CXXFLAGS = $(AM_CXXFLAGS)
+
+
+nobase_include_HEADERS = \
+		msgpack/pack_define.h \
+		msgpack/pack_template.h \
+		msgpack/unpack_define.h \
+		msgpack/unpack_template.h \
+		msgpack/sysdep.h \
+		msgpack.h \
+		msgpack/sbuffer.h \
+		msgpack/vrefbuffer.h \
+		msgpack/zbuffer.h \
+		msgpack/pack.h \
+		msgpack/unpack.h \
+		msgpack/object.h \
+		msgpack/zone.h \
+		msgpack.hpp \
+		msgpack/sbuffer.hpp \
+		msgpack/vrefbuffer.hpp \
+		msgpack/zbuffer.hpp \
+		msgpack/pack.hpp \
+		msgpack/unpack.hpp \
+		msgpack/object.hpp \
+		msgpack/zone.hpp \
+		msgpack/type.hpp \
+		msgpack/type/bool.hpp \
+		msgpack/type/float.hpp \
+		msgpack/type/int.hpp \
+		msgpack/type/list.hpp \
+		msgpack/type/deque.hpp \
+		msgpack/type/map.hpp \
+		msgpack/type/nil.hpp \
+		msgpack/type/pair.hpp \
+		msgpack/type/raw.hpp \
+		msgpack/type/set.hpp \
+		msgpack/type/string.hpp \
+		msgpack/type/vector.hpp \
+		msgpack/type/tuple.hpp \
+		msgpack/type/define.hpp \
+		msgpack/type/tr1/unordered_map.hpp \
+		msgpack/type/tr1/unordered_set.hpp
+
+EXTRA_DIST = \
+		msgpack/zone.hpp.erb \
+		msgpack/type/define.hpp.erb \
+		msgpack/type/tuple.hpp.erb
+
diff --git a/cpp/msgpack.h b/cpp/src/msgpack.h
similarity index 100%
rename from cpp/msgpack.h
rename to cpp/src/msgpack.h
diff --git a/cpp/msgpack.hpp b/cpp/src/msgpack.hpp
similarity index 100%
rename from cpp/msgpack.hpp
rename to cpp/src/msgpack.hpp
diff --git a/cpp/msgpack/object.h b/cpp/src/msgpack/object.h
similarity index 100%
rename from cpp/msgpack/object.h
rename to cpp/src/msgpack/object.h
diff --git a/cpp/msgpack/object.hpp b/cpp/src/msgpack/object.hpp
similarity index 100%
rename from cpp/msgpack/object.hpp
rename to cpp/src/msgpack/object.hpp
diff --git a/cpp/msgpack/pack.h b/cpp/src/msgpack/pack.h
similarity index 100%
rename from cpp/msgpack/pack.h
rename to cpp/src/msgpack/pack.h
diff --git a/cpp/msgpack/pack.hpp b/cpp/src/msgpack/pack.hpp
similarity index 100%
rename from cpp/msgpack/pack.hpp
rename to cpp/src/msgpack/pack.hpp
diff --git a/cpp/msgpack/sbuffer.h b/cpp/src/msgpack/sbuffer.h
similarity index 100%
rename from cpp/msgpack/sbuffer.h
rename to cpp/src/msgpack/sbuffer.h
diff --git a/cpp/msgpack/sbuffer.hpp b/cpp/src/msgpack/sbuffer.hpp
similarity index 100%
rename from cpp/msgpack/sbuffer.hpp
rename to cpp/src/msgpack/sbuffer.hpp
diff --git a/cpp/msgpack/type.hpp b/cpp/src/msgpack/type.hpp
similarity index 100%
rename from cpp/msgpack/type.hpp
rename to cpp/src/msgpack/type.hpp
diff --git a/cpp/msgpack/type/bool.hpp b/cpp/src/msgpack/type/bool.hpp
similarity index 100%
rename from cpp/msgpack/type/bool.hpp
rename to cpp/src/msgpack/type/bool.hpp
diff --git a/cpp/msgpack/type/define.hpp.erb b/cpp/src/msgpack/type/define.hpp.erb
similarity index 100%
rename from cpp/msgpack/type/define.hpp.erb
rename to cpp/src/msgpack/type/define.hpp.erb
diff --git a/cpp/msgpack/type/deque.hpp b/cpp/src/msgpack/type/deque.hpp
similarity index 100%
rename from cpp/msgpack/type/deque.hpp
rename to cpp/src/msgpack/type/deque.hpp
diff --git a/cpp/msgpack/type/float.hpp b/cpp/src/msgpack/type/float.hpp
similarity index 100%
rename from cpp/msgpack/type/float.hpp
rename to cpp/src/msgpack/type/float.hpp
diff --git a/cpp/msgpack/type/int.hpp b/cpp/src/msgpack/type/int.hpp
similarity index 100%
rename from cpp/msgpack/type/int.hpp
rename to cpp/src/msgpack/type/int.hpp
diff --git a/cpp/msgpack/type/list.hpp b/cpp/src/msgpack/type/list.hpp
similarity index 100%
rename from cpp/msgpack/type/list.hpp
rename to cpp/src/msgpack/type/list.hpp
diff --git a/cpp/msgpack/type/map.hpp b/cpp/src/msgpack/type/map.hpp
similarity index 100%
rename from cpp/msgpack/type/map.hpp
rename to cpp/src/msgpack/type/map.hpp
diff --git a/cpp/msgpack/type/nil.hpp b/cpp/src/msgpack/type/nil.hpp
similarity index 100%
rename from cpp/msgpack/type/nil.hpp
rename to cpp/src/msgpack/type/nil.hpp
diff --git a/cpp/msgpack/type/pair.hpp b/cpp/src/msgpack/type/pair.hpp
similarity index 100%
rename from cpp/msgpack/type/pair.hpp
rename to cpp/src/msgpack/type/pair.hpp
diff --git a/cpp/msgpack/type/raw.hpp b/cpp/src/msgpack/type/raw.hpp
similarity index 100%
rename from cpp/msgpack/type/raw.hpp
rename to cpp/src/msgpack/type/raw.hpp
diff --git a/cpp/msgpack/type/set.hpp b/cpp/src/msgpack/type/set.hpp
similarity index 100%
rename from cpp/msgpack/type/set.hpp
rename to cpp/src/msgpack/type/set.hpp
diff --git a/cpp/msgpack/type/string.hpp b/cpp/src/msgpack/type/string.hpp
similarity index 100%
rename from cpp/msgpack/type/string.hpp
rename to cpp/src/msgpack/type/string.hpp
diff --git a/cpp/msgpack/type/tr1/unordered_map.hpp b/cpp/src/msgpack/type/tr1/unordered_map.hpp
similarity index 100%
rename from cpp/msgpack/type/tr1/unordered_map.hpp
rename to cpp/src/msgpack/type/tr1/unordered_map.hpp
diff --git a/cpp/msgpack/type/tr1/unordered_set.hpp b/cpp/src/msgpack/type/tr1/unordered_set.hpp
similarity index 100%
rename from cpp/msgpack/type/tr1/unordered_set.hpp
rename to cpp/src/msgpack/type/tr1/unordered_set.hpp
diff --git a/cpp/msgpack/type/tuple.hpp.erb b/cpp/src/msgpack/type/tuple.hpp.erb
similarity index 100%
rename from cpp/msgpack/type/tuple.hpp.erb
rename to cpp/src/msgpack/type/tuple.hpp.erb
diff --git a/cpp/msgpack/type/vector.hpp b/cpp/src/msgpack/type/vector.hpp
similarity index 100%
rename from cpp/msgpack/type/vector.hpp
rename to cpp/src/msgpack/type/vector.hpp
diff --git a/cpp/msgpack/unpack.h b/cpp/src/msgpack/unpack.h
similarity index 100%
rename from cpp/msgpack/unpack.h
rename to cpp/src/msgpack/unpack.h
diff --git a/cpp/msgpack/unpack.hpp b/cpp/src/msgpack/unpack.hpp
similarity index 100%
rename from cpp/msgpack/unpack.hpp
rename to cpp/src/msgpack/unpack.hpp
diff --git a/cpp/msgpack/vrefbuffer.h b/cpp/src/msgpack/vrefbuffer.h
similarity index 100%
rename from cpp/msgpack/vrefbuffer.h
rename to cpp/src/msgpack/vrefbuffer.h
diff --git a/cpp/msgpack/vrefbuffer.hpp b/cpp/src/msgpack/vrefbuffer.hpp
similarity index 100%
rename from cpp/msgpack/vrefbuffer.hpp
rename to cpp/src/msgpack/vrefbuffer.hpp
diff --git a/cpp/msgpack/zbuffer.h b/cpp/src/msgpack/zbuffer.h
similarity index 100%
rename from cpp/msgpack/zbuffer.h
rename to cpp/src/msgpack/zbuffer.h
diff --git a/cpp/msgpack/zbuffer.hpp b/cpp/src/msgpack/zbuffer.hpp
similarity index 100%
rename from cpp/msgpack/zbuffer.hpp
rename to cpp/src/msgpack/zbuffer.hpp
diff --git a/cpp/msgpack/zone.h b/cpp/src/msgpack/zone.h
similarity index 100%
rename from cpp/msgpack/zone.h
rename to cpp/src/msgpack/zone.h
diff --git a/cpp/msgpack/zone.hpp.erb b/cpp/src/msgpack/zone.hpp.erb
similarity index 100%
rename from cpp/msgpack/zone.hpp.erb
rename to cpp/src/msgpack/zone.hpp.erb
diff --git a/cpp/object.cpp b/cpp/src/object.cpp
similarity index 100%
rename from cpp/object.cpp
rename to cpp/src/object.cpp
diff --git a/cpp/objectc.c b/cpp/src/objectc.c
similarity index 100%
rename from cpp/objectc.c
rename to cpp/src/objectc.c
diff --git a/cpp/unpack.c b/cpp/src/unpack.c
similarity index 100%
rename from cpp/unpack.c
rename to cpp/src/unpack.c
diff --git a/cpp/vrefbuffer.c b/cpp/src/vrefbuffer.c
similarity index 100%
rename from cpp/vrefbuffer.c
rename to cpp/src/vrefbuffer.c
diff --git a/cpp/zone.c b/cpp/src/zone.c
similarity index 100%
rename from cpp/zone.c
rename to cpp/src/zone.c
diff --git a/cpp/test.mk b/cpp/test.mk
deleted file mode 100644
index f1beac5..0000000
--- a/cpp/test.mk
+++ /dev/null
@@ -1,9 +0,0 @@
-
-CXXFLAGS  += -Wall -g -I. -I.. -O4
-LDFLAGS +=
-
-all: test
-
-test: test.o unpack.o zone.o object.o pack.hpp unpack.hpp zone.hpp object.hpp
-	$(CXX) test.o unpack.o zone.o object.o $(CXXFLAGS) $(LDFLAGS) -o $@
-
diff --git a/cpp/test/Makefile.am b/cpp/test/Makefile.am
index 2b96669..3670d7f 100644
--- a/cpp/test/Makefile.am
+++ b/cpp/test/Makefile.am
@@ -1,7 +1,7 @@
 
-AM_CPPFLAGS   = -I..
-AM_C_CPPFLAGS = -I..
-AM_LDFLAGS = ../libmsgpack.la -lgtest_main
+AM_CPPFLAGS   = -I../src
+AM_C_CPPFLAGS = -I../src
+AM_LDFLAGS = ../src/libmsgpack.la -lgtest_main
 
 check_PROGRAMS = \
 		zone \
@@ -9,7 +9,9 @@ check_PROGRAMS = \
 		streaming \
 		object \
 		convert \
-		buffer
+		buffer \
+		msgpackc_test \
+		msgpack_test
 
 TESTS = $(check_PROGRAMS)
 
@@ -26,3 +28,7 @@ convert_SOURCES = convert.cc
 buffer_SOURCES = buffer.cc
 buffer_LDADD = -lz
 
+msgpackc_test_SOURCES = msgpackc_test.cpp
+
+msgpack_test_SOURCES = msgpack_test.cpp
+
diff --git a/cpp/msgpack_test.cpp b/cpp/test/msgpack_test.cpp
similarity index 100%
rename from cpp/msgpack_test.cpp
rename to cpp/test/msgpack_test.cpp
diff --git a/cpp/msgpackc_test.cpp b/cpp/test/msgpackc_test.cpp
similarity index 100%
rename from cpp/msgpackc_test.cpp
rename to cpp/test/msgpackc_test.cpp

From 6b5b76b0c908c338dbc4aaa86aa42722da4adca3 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Sun, 30 May 2010 15:01:10 +0900
Subject: [PATCH 0502/1648] initial import from
 http://bitbucket.org/kuenishi/messagepack-for-erlang

---
 erlang/.gitignore  |   3 +
 erlang/msgpack.erl | 377 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 380 insertions(+)
 create mode 100644 erlang/.gitignore
 create mode 100644 erlang/msgpack.erl

diff --git a/erlang/.gitignore b/erlang/.gitignore
new file mode 100644
index 0000000..911d6e3
--- /dev/null
+++ b/erlang/.gitignore
@@ -0,0 +1,3 @@
+MANIFEST
+*.beam
+
diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
new file mode 100644
index 0000000..4b8da47
--- /dev/null
+++ b/erlang/msgpack.erl
@@ -0,0 +1,377 @@
+%%
+%% MessagePack for Erlang
+%%
+%% Copyright (C) 2009 UENISHI Kota
+%%
+%%    Licensed under the Apache License, Version 2.0 (the "License");
+%%    you may not use this file except in compliance with the License.
+%%    You may obtain a copy of the License at
+%%
+%%        http://www.apache.org/licenses/LICENSE-2.0
+%%
+%%    Unless required by applicable law or agreed to in writing, software
+%%    distributed under the License is distributed on an "AS IS" BASIS,
+%%    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%%    See the License for the specific language governing permissions and
+%%    limitations under the License.
+%
+% Thanks to id:frsyuki for his sophiscated binary format specification.
+%
+
+-module(msgpack).
+-author('kuenishi+msgpack@gmail.com').
+
+-export([pack/1, unpack/1, unpack_all/1, test/0]).
+
+% compile:
+% erl> c(msgpack).
+% erl> S = .
+% erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
+
+%% tuples, atoms are not supported.
+%% lists, integers, double, and so on.
+%% see http://msgpack.sourceforge.jp/spec for
+%% supported formats. APIs are almost compatible
+%% for C API (http://msgpack.sourceforge.jp/c:doc)
+%% except buffering functions (both copying and zero-copying).
+-export([
+	 pack_fixnum/1,
+	 pack_nfixnum/1,
+	 pack_uint8/1,
+	 pack_uint16/1,
+	 pack_uint32/1,
+	 pack_uint64/1,
+	 pack_short/1,
+	 pack_int8/1,
+	 pack_int/1,
+	 pack_long/1,
+	 pack_long_long/1,
+	 pack_unsigned_short/1,
+	 pack_unsigned_int/1,
+	 pack_unsigned_long/1,
+	 pack_unsigned_long_long/1,
+	 pack_nil/0,
+	 pack_bool/1,
+	 pack_float/1,
+	 pack_double/1,
+	 pack_raw/1,
+	 pack_array/1,
+	 pack_map/1,
+	 pack_object/1	]).
+
+
+%  packing functions
+
+pack_short(N) when is_integer(N)->
+    pack_int16(N).
+pack_int(N) when is_integer(N)->
+    pack_int32(N).
+pack_long(N) when is_integer(N)->
+    pack_int32(N).
+pack_long_long(N) when is_integer(N)->
+    pack_int64(N).
+pack_unsigned_short(N) when is_integer(N)->
+    pack_uint16(N).
+pack_unsigned_int(N) when is_integer(N)->
+    pack_uint32(N).
+pack_unsigned_long(N) when is_integer(N)->
+    pack_uint32(N).
+pack_unsigned_long_long(N) when is_integer(N)->
+    pack_uint64(N).
+
+% positive fixnum
+pack_fixnum( N ) when is_integer( N ) and  N >= 0 ,  N < 128 ->  
+    << 2#0:1, N:7 >>.
+% negative fixnum
+pack_nfixnum( N ) when is_integer( N ) and N >= -32 , N < 0 ->
+    << 2#111:3, N:5 >>.
+% uint 8
+pack_uint8( N ) when is_integer( N )->
+    << 16#CC:8, N:8 >>.
+% uint 16
+pack_uint16( N ) when is_integer( N )->
+    << 16#CD:8, N:16/big-unsigned-integer-unit:1 >>.
+% uint 32
+pack_uint32( N ) when is_integer( N )->
+    << 16#CE:8, N:32/big-unsigned-integer-unit:1 >>.
+% uint 64
+pack_uint64( N ) when is_integer( N )->
+    << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>.
+% int 8
+pack_int8( N ) when is_integer( N )->
+    << 16#D0:8, N:8 >>.
+% int 16
+pack_int16( N ) when is_integer( N )->
+    << 16#D1:8, N:16/big-signed-integer-unit:1 >>.
+% int 32
+pack_int32( N ) when is_integer( N )->
+    << 16#D2:8, N:32/big-signed-integer-unit:1 >>.
+% int 64
+pack_int64( N ) when is_integer( N )->
+    << 16#D3:8, N:64/big-signed-integer-unit:1 >>.
+
+% nil
+pack_nil()->
+    << 16#C0:8 >>.
+% pack_true / pack_false
+pack_bool(true)->    << 16#C3:8 >>;
+pack_bool(false)->   << 16#C2:8 >>.
+
+% float : erlang's float is always IEEE 754 64bit format.
+pack_float(F) when is_float(F)->
+%    << 16#CA:8, F:32/big-float-unit:1 >>.
+    pack_double(F).
+% double
+pack_double(F) when is_float(F)->
+    << 16#CB:8, F:64/big-float-unit:1 >>.
+
+power(N,0) when is_integer(N) -> 1;
+power(N,D) when is_integer(N) and is_integer(D) -> N * power(N, D-1).
+
+% raw bytes
+pack_raw(Bin) when is_binary(Bin)->
+    MaxLen = power(2,16),
+    case byte_size(Bin) of
+	Len when Len < 6->
+	    << 2#101:3, Len:5, Bin/binary >>;
+	Len when Len < MaxLen ->
+	    << 16#DA:8, Len:16/big-unsigned-integer-unit:1, Bin/binary >>;
+	Len ->
+	    << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >>
+    end.
+
+% list / tuple
+pack_array(L) when is_list(L)->
+    MaxLen = power(2,16),
+    case length(L) of
+ 	Len when Len < 16 ->
+ 	    << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L))/binary >>;
+	Len when Len < MaxLen ->
+	    << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L))/binary >>;
+	Len ->
+	    << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L))/binary >>
+    end.
+pack_array_([])-> <<>>;
+pack_array_([Head|Tail])->
+    << (pack_object(Head))/binary, (pack_array_(Tail))/binary >>.
+
+unpack_array_(<<>>, 0)-> [];
+unpack_array_(Remain, 0) when is_binary(Remain)-> [Remain];
+unpack_array_(Bin, RestLen) when is_binary(Bin)->
+    {Term, Rest} = unpack(Bin),
+    [Term|unpack_array_(Rest, RestLen-1)].
+    
+pack_map({dict,M})->
+    MaxLen = power(2,16),
+    case dict:size(M) of
+	Len when Len < 16 ->
+ 	    << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>;
+	Len when Len < MaxLen ->
+	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M))) >>;
+	Len ->
+	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M))) >>
+    end.
+
+pack_map_([])-> <<>>;
+pack_map_([{Key,Value}|Tail]) ->
+    << (pack_object(Key)),(pack_object(Value)),(pack_map_(Tail)) >>.
+
+unpack_map_(<<>>, 0)-> [];
+unpack_map_(Bin,  0) when is_binary(Bin)-> [Bin];
+unpack_map_(Bin, Len) when is_binary(Bin) and is_integer(Len) ->
+    { Key, Rest } = unpack(Bin),
+    { Value, Rest2 } = unpack(Rest),
+    [{Key,Value}|unpack_map_(Rest2,Len-1)].
+
+pack_object(O) when is_integer(O)->
+    pack_long_long(O);
+pack_object(O) when is_float(O)->
+    pack_double(O);
+pack_object(nil) ->
+    pack_nil();
+pack_object(Bool) when is_atom(Bool) ->
+    pack_bool(Bool);
+pack_object(Bin) when is_binary(Bin)->
+    pack_raw(Bin);
+pack_object(List)  when is_list(List)->
+    pack_array(List);
+pack_object({dict, Map})->
+    pack_map({dict, Map});
+pack_object(_) ->
+    undefined.
+
+pack(Obj)->
+    pack_object(Obj).
+
+
+% unpacking.
+% if failed in decoding and not end, get more data 
+% and feed more Bin into this function.
+% TODO: error case for imcomplete format when short for any type formats.
+-spec unpack( binary() )-> {term(), binary()}.
+unpack(Bin) when bit_size(Bin) >= 8 ->
+    << Flag:8, Payload/binary >> = Bin,
+    case Flag of 
+	16#C0 ->
+	    {nil, Payload};
+	16#C2 ->
+	    {false, Payload};
+	16#C3 ->
+	    {true, Payload};
+	16#CA -> % 32bit float
+	    << Return:32/float-unit:1, Rest/binary >> = Payload,
+	    {Return, Rest};
+	16#CB -> % 64bit float
+	    << Return:64/float-unit:1, Rest/binary >> = Payload,
+	    {Return, Rest};
+	16#CC ->
+	    << Int:8/unsigned-integer, Rest/binary >> = Payload,
+	    {Int, Rest};
+	16#CD ->
+	    << Int:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
+	    {Int, Rest};
+	16#CE ->
+	    << Int:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
+	    {Int, Rest};
+	16#CF ->
+	    << Int:64/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
+	    {Int, Rest};
+	16#D0 ->
+	    << Int:8/big-signed-integer-unit:1, Rest/binary >> = Payload,
+	    {Int, Rest};
+	16#D1 ->
+	    << Int:16/big-signed-integer-unit:1, Rest/binary >> = Payload,
+	    {Int, Rest};
+	16#D2 ->
+	    << Int:32/big-signed-integer-unit:1, Rest/binary >> = Payload,
+	    {Int, Rest};
+	16#D3 ->
+	    << Int:64/big-signed-integer-unit:1, Rest/binary >> = Payload,
+	    {Int, Rest};
+	16#DA -> % raw 16
+	    << Len:16/unsigned-integer-unit:1, Rest/binary >> = Payload,
+	    << Return:Len/binary, Remain/binary >> = Rest,
+	    {Return, Remain};
+	16#DB -> % raw 32
+	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
+	    << Return:Len/binary, Remain/binary >> = Rest,
+	    {Return, Remain};
+	16#DC -> % array 16
+	    << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
+	    Array=unpack_array_(Rest, Len),
+	    case length(Array) of
+		Len -> {Array, <<>>};
+		_ -> 
+		    {Return, RemainRest} = lists:split(Len, Array),
+		    [Remain] = RemainRest,
+		    {Return, Remain}
+	    end;
+	16#DD -> % array 32
+	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
+	    Array=unpack_array_(Rest, Len),
+	    case length(Array) of
+		Len -> {Array, <<>>};
+		_ -> 
+		    {Return, RemainRest} = lists:split(Len, Array),
+		    [Remain] = RemainRest,
+		    {Return, Remain}
+	    end;
+	16#DE -> % map 16
+	    << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
+	    Array=unpack_map_(Rest, Len),
+	    case length(Array) of
+		Len -> { dict:from_list(Array), <<>>};
+		_ -> 
+		    {Return, RemainRest} = lists:split(Len, Array),
+		    [Remain] = RemainRest,
+		    {dict:from_list(Return), Remain}
+	    end;
+	16#DF -> % map 32
+	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
+	    Array=unpack_map_(Rest, Len),
+	    case length(Array) of
+		Len -> { dict:from_list(Array), <<>>};
+		_ -> 
+		    {Return, RemainRest} = lists:split(Len, Array),
+		    [Remain] = RemainRest,
+		    {dict:from_list(Return), Remain}
+	    end;
+
+	Code when Code >= 2#10100000 , Code < 2#11000000 ->
+%	 101XXXXX for FixRaw 
+	    Len = Code rem 2#10100000,
+	    << Return:Len/binary, Remain/binary >> = Payload,
+	    {Return, Remain};
+
+	Code when Code >= 2#10010000 , Code < 2#10100000 ->
+%        1001XXXX for FixArray
+	    Len = Code rem 2#10010000,
+	    Array=unpack_array_(Payload, Len),
+	    case length(Array) of
+		Len -> { Array, <<>>};
+		_ -> 
+		    {Return, RemainRest} = lists:split(Len, Array),
+		    [Remain] = RemainRest,
+		    {Return, Remain}
+	    end;
+
+	Code when Code >= 2#10000000 , Code < 2#10010000 ->
+%        1000XXXX for FixMap
+	    Len = Code rem 2#10000000,
+	    Array=unpack_map_(Payload, Len),
+	    case length(Array) of
+		Len -> { dict:from_list(Array), <<>>};
+		_ -> 
+		    {Return, RemainRest} = lists:split(Len, Array),
+		    [Remain] = RemainRest,
+		    {dict:from_list(Return), Remain}
+	    end;
+
+	_ ->
+	    {error, no_code_matches}
+    end.
+    
+unpack_all(Data)->
+    case unpack(Data) of
+	{ Term, Binary } when bit_size(Binary) =:= 0 -> 
+	    [Term];
+	{ Term, Binary } when is_binary(Binary) ->
+	    [Term|unpack_all(Binary)]
+    end.
+
+test()->
+    Tests = [0, 1, 2, 123, 123.123, [23, 234, 0.23], "hogehoge", <<"hoasfdafdas][">>],
+    Passed = test_(Tests),
+    Passed = length(Tests).
+%%     Port = open_port({spawn, "./a.out"}, [stream]),
+%%     receive {Port, {data, Data}}->
+%%     io:format("~p~n", [unpack_all( list_to_binary(Data) )])
+%%     after 1024-> timeout end,
+%%     Passed2 = test_(Tests, Port),
+%%     Passed2 = length(Tests),
+%%     Port ! {self(), close},
+%%     receive {Port, closed}-> ok 
+%%     after 1024 -> timeout end.
+
+test_([]) -> 0;
+test_([S|Rest])->
+%    io:format("testing: ~p~n", [S]),
+    {S, <<>>} = msgpack:unpack( msgpack:pack(S) ),
+    test_(Rest) + 1.
+
+test_([], _)-> 0;
+test_([S|Rest], Port) ->
+    Bin = msgpack:pack(S),
+    io:format("sending: ~p  -  ", [S]),
+    Port ! {self(), {command, Bin}},
+    receive
+	{Port, {data, Data}}->
+	    io:format("~p~n", [Data]),
+	    test_(Rest, Port) + 1;
+	Other->
+	    io:format("fail: ~p~n", [Other])
+    after 1024->
+	    test_(Rest, Port)
+    end.
+     
+

From d43921823ea5a20f0677410e965a54dd19c2effe Mon Sep 17 00:00:00 2001
From: Hideyuki Tanaka 
Date: Sun, 30 May 2010 17:19:43 +0900
Subject: [PATCH 0503/1648] fix initialize pointer

---
 haskell/src/Data/MessagePack/Base.hsc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/haskell/src/Data/MessagePack/Base.hsc b/haskell/src/Data/MessagePack/Base.hsc
index 72c421c..8c7b9f5 100644
--- a/haskell/src/Data/MessagePack/Base.hsc
+++ b/haskell/src/Data/MessagePack/Base.hsc
@@ -560,6 +560,7 @@ unpackObject z dat =
   allocaBytes (#size msgpack_object) $ \ptr ->
   BS.useAsCStringLen dat $ \(str, len) ->
   alloca $ \poff -> do
+    poke poff 0
     ret <- msgpack_unpack str (fromIntegral len) poff z ptr
     case ret of
       (#const MSGPACK_UNPACK_SUCCESS) -> do

From 0da22193bd628690bfc58d4c36d6d48493fa7f51 Mon Sep 17 00:00:00 2001
From: Hideyuki Tanaka 
Date: Sun, 30 May 2010 17:20:49 +0900
Subject: [PATCH 0504/1648] fix typo

---
 haskell/src/Data/MessagePack/Monad.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/haskell/src/Data/MessagePack/Monad.hs b/haskell/src/Data/MessagePack/Monad.hs
index c718b8a..15f21fe 100644
--- a/haskell/src/Data/MessagePack/Monad.hs
+++ b/haskell/src/Data/MessagePack/Monad.hs
@@ -82,7 +82,7 @@ packToString m = do
   _ <- runPackerT m pc
   liftIO $ simpleBufferData sb
 
--- | Execcute given serializer and write byte sequence to Handle.
+-- | Execute given serializer and write byte sequence to Handle.
 packToHandle :: MonadIO m => Handle -> PackerT m r -> m ()
 packToHandle h m = do
   sb <- packToString m

From e61dc76ae1abb6235d09b52ad24383ccd75d5222 Mon Sep 17 00:00:00 2001
From: Hideyuki Tanaka 
Date: Sun, 30 May 2010 19:11:04 +0900
Subject: [PATCH 0505/1648] fix peek object

---
 haskell/src/Data/MessagePack/Base.hsc | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/haskell/src/Data/MessagePack/Base.hsc b/haskell/src/Data/MessagePack/Base.hsc
index 8c7b9f5..b6cdc28 100644
--- a/haskell/src/Data/MessagePack/Base.hsc
+++ b/haskell/src/Data/MessagePack/Base.hsc
@@ -506,20 +506,22 @@ peekObjectRAW ptr = do
 
 peekObjectArray :: Ptr a -> IO Object
 peekObjectArray ptr = do
-  size <- (#peek msgpack_object, via.array.size) ptr
-  p    <- (#peek msgpack_object, via.array.ptr) ptr
-  objs <- mapM (\i -> peekObject $ p `plusPtr`
+  csize <- (#peek msgpack_object, via.array.size) ptr
+  let size = fromIntegral (csize :: Word32)
+  p     <- (#peek msgpack_object, via.array.ptr) ptr
+  objs  <- mapM (\i -> peekObject $ p `plusPtr`
                       ((#size msgpack_object) * i))
-          [0..size-1]
+           [0..size-1]
   return $ ObjectArray objs
 
 peekObjectMap :: Ptr a -> IO Object
 peekObjectMap ptr = do
-  size <- (#peek msgpack_object, via.map.size) ptr
-  p    <- (#peek msgpack_object, via.map.ptr) ptr
-  dat  <- mapM (\i -> peekObjectKV $ p `plusPtr`
+  csize <- (#peek msgpack_object, via.map.size) ptr
+  let size = fromIntegral (csize :: Word32)
+  p     <- (#peek msgpack_object, via.map.ptr) ptr
+  dat   <- mapM (\i -> peekObjectKV $ p `plusPtr`
                       ((#size msgpack_object_kv) * i))
-          [0..size-1]
+           [0..size-1]
   return $ ObjectMap dat
 
 peekObjectKV :: Ptr a -> IO (Object, Object)

From 5a12d36a0a68e97dcdd2e030486f6a67138c8b44 Mon Sep 17 00:00:00 2001
From: Hideyuki Tanaka 
Date: Sun, 30 May 2010 19:45:00 +0900
Subject: [PATCH 0506/1648] incr version

---
 haskell/msgpack.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal
index 31cad3b..82cdb52 100644
--- a/haskell/msgpack.cabal
+++ b/haskell/msgpack.cabal
@@ -1,5 +1,5 @@
 Name:                msgpack
-Version:             0.2.1
+Version:             0.2.2
 License:             BSD3
 License-File:        LICENSE
 Author:              Hideyuki Tanaka

From d7d78d9a2b77df0661fb0cd6b0be7b565e6f7a25 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Mon, 31 May 2010 00:25:53 +0900
Subject: [PATCH 0507/1648] added more tests, and OMake continuous building.

---
 erlang/OMakefile   |  42 +++++++++++++
 erlang/OMakeroot   |  45 ++++++++++++++
 erlang/msgpack.erl | 152 +++++++++++++++++++--------------------------
 3 files changed, 151 insertions(+), 88 deletions(-)
 create mode 100644 erlang/OMakefile
 create mode 100644 erlang/OMakeroot

diff --git a/erlang/OMakefile b/erlang/OMakefile
new file mode 100644
index 0000000..ee72f78
--- /dev/null
+++ b/erlang/OMakefile
@@ -0,0 +1,42 @@
+########################################################################
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this file, to deal in the File without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the File, and to permit persons to whom the
+# File is furnished to do so, subject to the following condition:
+#
+# THE FILE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE FILE OR
+# THE USE OR OTHER DEALINGS IN THE FILE.
+
+########################################################################
+# The standard OMakefile.
+# You will usually need to modify this file for your project.
+
+########################################################################
+# Phony targets are scoped, so you probably want to declare them first.
+#
+
+.PHONY: all clean test #install
+
+########################################################################
+# Subdirectories.
+# You may want to include some subdirectories in this project.
+# If so, define the subdirectory targets and uncomment this section.
+#
+
+.DEFAULT: msgpack.beam
+
+msgpack.beam: msgpack.erl
+	erlc $<
+
+test: msgpack.beam
+	erl -s msgpack test -s init stop
+
+clean:
+	-rm *.beam
diff --git a/erlang/OMakeroot b/erlang/OMakeroot
new file mode 100644
index 0000000..35c219d
--- /dev/null
+++ b/erlang/OMakeroot
@@ -0,0 +1,45 @@
+########################################################################
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this file, to deal in the File without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the File, and to permit persons to whom the
+# File is furnished to do so, subject to the following condition:
+#
+# THE FILE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE FILE OR
+# THE USE OR OTHER DEALINGS IN THE FILE.
+
+########################################################################
+# The standard OMakeroot file.
+# You will not normally need to modify this file.
+# By default, your changes should be placed in the
+# OMakefile in this directory.
+#
+# If you decide to modify this file, note that it uses exactly
+# the same syntax as the OMakefile.
+#
+
+#
+# Include the standard installed configuration files.
+# Any of these can be deleted if you are not using them,
+# but you probably want to keep the Common file.
+#
+open build/C
+open build/OCaml
+open build/LaTeX
+
+#
+# The command-line variables are defined *after* the
+# standard configuration has been loaded.
+#
+DefineCommandVars()
+
+#
+# Include the OMakefile in this directory.
+#
+.SUBDIRS: .
diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 4b8da47..df7974d 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -14,42 +14,25 @@
 %%    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 %%    See the License for the specific language governing permissions and
 %%    limitations under the License.
-%
-% Thanks to id:frsyuki for his sophiscated binary format specification.
-%
 
 -module(msgpack).
 -author('kuenishi+msgpack@gmail.com').
 
 -export([pack/1, unpack/1, unpack_all/1, test/0]).
 
+-include_lib("eunit/include/eunit.hrl").
+
 % compile:
 % erl> c(msgpack).
 % erl> S = .
 % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
 
-%% tuples, atoms are not supported.
-%% lists, integers, double, and so on.
+%% tuples, atoms are not supported.  lists, integers, double, and so on.
 %% see http://msgpack.sourceforge.jp/spec for
 %% supported formats. APIs are almost compatible
 %% for C API (http://msgpack.sourceforge.jp/c:doc)
 %% except buffering functions (both copying and zero-copying).
 -export([
-	 pack_fixnum/1,
-	 pack_nfixnum/1,
-	 pack_uint8/1,
-	 pack_uint16/1,
-	 pack_uint32/1,
-	 pack_uint64/1,
-	 pack_short/1,
-	 pack_int8/1,
-	 pack_int/1,
-	 pack_long/1,
-	 pack_long_long/1,
-	 pack_unsigned_short/1,
-	 pack_unsigned_int/1,
-	 pack_unsigned_long/1,
-	 pack_unsigned_long_long/1,
 	 pack_nil/0,
 	 pack_bool/1,
 	 pack_float/1,
@@ -59,55 +42,39 @@
 	 pack_map/1,
 	 pack_object/1	]).
 
-
-%  packing functions
-
-pack_short(N) when is_integer(N)->
-    pack_int16(N).
-pack_int(N) when is_integer(N)->
-    pack_int32(N).
-pack_long(N) when is_integer(N)->
-    pack_int32(N).
-pack_long_long(N) when is_integer(N)->
-    pack_int64(N).
-pack_unsigned_short(N) when is_integer(N)->
-    pack_uint16(N).
-pack_unsigned_int(N) when is_integer(N)->
-    pack_uint32(N).
-pack_unsigned_long(N) when is_integer(N)->
-    pack_uint32(N).
-pack_unsigned_long_long(N) when is_integer(N)->
-    pack_uint64(N).
-
 % positive fixnum
-pack_fixnum( N ) when is_integer( N ) and  N >= 0 ,  N < 128 ->  
-    << 2#0:1, N:7 >>.
-% negative fixnum
-pack_nfixnum( N ) when is_integer( N ) and N >= -32 , N < 0 ->
-    << 2#111:3, N:5 >>.
+pack_uint_(N) when is_integer( N ) , N < 128 ->  
+    << 2#0:1, N:7 >>;
 % uint 8
-pack_uint8( N ) when is_integer( N )->
-    << 16#CC:8, N:8 >>.
+pack_uint_( N ) when is_integer( N ) andalso N < 256 ->
+    << 16#CC:8, N:8 >>;
+
 % uint 16
-pack_uint16( N ) when is_integer( N )->
-    << 16#CD:8, N:16/big-unsigned-integer-unit:1 >>.
+pack_uint_( N ) when is_integer( N ) andalso N < 65536 ->
+    << 16#CD:8, N:16/big-unsigned-integer-unit:1 >>;
+
 % uint 32
-pack_uint32( N ) when is_integer( N )->
-    << 16#CE:8, N:32/big-unsigned-integer-unit:1 >>.
+pack_uint_( N ) when is_integer( N ) andalso N < 16#FFFFFFFF->
+    << 16#CE:8, N:32/big-unsigned-integer-unit:1 >>;
+
 % uint 64
-pack_uint64( N ) when is_integer( N )->
+pack_uint_( N ) when is_integer( N )->
     << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>.
+
+% negative fixnum
+pack_int_( N ) when is_integer( N ) , N >= -32->
+    << 2#111:3, N:5 >>;
 % int 8
-pack_int8( N ) when is_integer( N )->
-    << 16#D0:8, N:8 >>.
+pack_int_( N ) when is_integer( N ) , N >= -256 ->
+    << 16#D0:8, N:8 >>;
 % int 16
-pack_int16( N ) when is_integer( N )->
-    << 16#D1:8, N:16/big-signed-integer-unit:1 >>.
+pack_int_( N ) when is_integer( N ), N >= -65536 ->
+    << 16#D1:8, N:16/big-signed-integer-unit:1 >>;
 % int 32
-pack_int32( N ) when is_integer( N )->
-    << 16#D2:8, N:32/big-signed-integer-unit:1 >>.
+pack_int_( N ) when is_integer( N ), N >= -16#FFFFFFFF ->
+    << 16#D2:8, N:32/big-signed-integer-unit:1 >>;
 % int 64
-pack_int64( N ) when is_integer( N )->
+pack_int_( N ) when is_integer( N )->
     << 16#D3:8, N:64/big-signed-integer-unit:1 >>.
 
 % nil
@@ -183,8 +150,10 @@ unpack_map_(Bin, Len) when is_binary(Bin) and is_integer(Len) ->
     { Value, Rest2 } = unpack(Rest),
     [{Key,Value}|unpack_map_(Rest2,Len-1)].
 
-pack_object(O) when is_integer(O)->
-    pack_long_long(O);
+pack_object(O) when is_integer(O) andalso O < 0 ->
+    pack_int_(O);
+pack_object(O) when is_integer(O) ->
+    pack_uint_(O);
 pack_object(O) when is_float(O)->
     pack_double(O);
 pack_object(nil) ->
@@ -210,7 +179,7 @@ pack(Obj)->
 % TODO: error case for imcomplete format when short for any type formats.
 -spec unpack( binary() )-> {term(), binary()}.
 unpack(Bin) when bit_size(Bin) >= 8 ->
-    << Flag:8, Payload/binary >> = Bin,
+    << Flag:8/unsigned-integer, Payload/binary >> = Bin,
     case Flag of 
 	16#C0 ->
 	    {nil, Payload};
@@ -218,16 +187,18 @@ unpack(Bin) when bit_size(Bin) >= 8 ->
 	    {false, Payload};
 	16#C3 ->
 	    {true, Payload};
+
 	16#CA -> % 32bit float
 	    << Return:32/float-unit:1, Rest/binary >> = Payload,
 	    {Return, Rest};
 	16#CB -> % 64bit float
 	    << Return:64/float-unit:1, Rest/binary >> = Payload,
 	    {Return, Rest};
-	16#CC ->
+
+	16#CC -> % uint 8
 	    << Int:8/unsigned-integer, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#CD ->
+	16#CD -> % uint 16
 	    << Int:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
 	16#CE ->
@@ -236,16 +207,17 @@ unpack(Bin) when bit_size(Bin) >= 8 ->
 	16#CF ->
 	    << Int:64/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#D0 ->
+
+	16#D0 -> % int 8
 	    << Int:8/big-signed-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#D1 ->
+	16#D1 -> % int 16
 	    << Int:16/big-signed-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#D2 ->
+	16#D2 -> % int 32
 	    << Int:32/big-signed-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#D3 ->
+	16#D3 -> % int 64
 	    << Int:64/big-signed-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
 	16#DA -> % raw 16
@@ -297,6 +269,14 @@ unpack(Bin) when bit_size(Bin) >= 8 ->
 		    {dict:from_list(Return), Remain}
 	    end;
 
+	% positive fixnum
+	Code when Code >= 2#00000000, Code < 2#10000000->
+	    {Code, Payload};
+
+	% negative fixnum
+	Code when Code >= 2#11100000 ->
+	    {(Code - 16#100), Payload};
+
 	Code when Code >= 2#10100000 , Code < 2#11000000 ->
 %	 101XXXXX for FixRaw 
 	    Len = Code rem 2#10100000,
@@ -327,7 +307,8 @@ unpack(Bin) when bit_size(Bin) >= 8 ->
 		    {dict:from_list(Return), Remain}
 	    end;
 
-	_ ->
+	_Other ->
+	    erlang:display(_Other),
 	    {error, no_code_matches}
     end.
     
@@ -339,8 +320,17 @@ unpack_all(Data)->
 	    [Term|unpack_all(Binary)]
     end.
 
+-ifdef(EUNIT).
+
 test()->
-    Tests = [0, 1, 2, 123, 123.123, [23, 234, 0.23], "hogehoge", <<"hoasfdafdas][">>],
+    Tests = [0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF,
+	     -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF,
+	     123.123, -234.4355, 1.0e-34, 1.0e64,
+	     [23, 234, 0.23],
+	     "hogehoge", "243546rf7g68h798j",
+	     <<"hoasfdafdas][">>,
+	     [0,42,"sum", [1,2]], [1,42, nil, [3]]
+	    ],
     Passed = test_(Tests),
     Passed = length(Tests).
 %%     Port = open_port({spawn, "./a.out"}, [stream]),
@@ -355,23 +345,9 @@ test()->
 
 test_([]) -> 0;
 test_([S|Rest])->
-%    io:format("testing: ~p~n", [S]),
-    {S, <<>>} = msgpack:unpack( msgpack:pack(S) ),
+    Pack = msgpack:pack(S),
+%    io:format("testing: ~p => ~p~n", [S, Pack]),
+    {S, <<>>} = msgpack:unpack( Pack ),
     test_(Rest) + 1.
 
-test_([], _)-> 0;
-test_([S|Rest], Port) ->
-    Bin = msgpack:pack(S),
-    io:format("sending: ~p  -  ", [S]),
-    Port ! {self(), {command, Bin}},
-    receive
-	{Port, {data, Data}}->
-	    io:format("~p~n", [Data]),
-	    test_(Rest, Port) + 1;
-	Other->
-	    io:format("fail: ~p~n", [Other])
-    after 1024->
-	    test_(Rest, Port)
-    end.
-     
-
+-endif.

From 98a5e438834b222f478e1af41ecd0e9590f6b08e Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Mon, 31 May 2010 17:16:40 +0900
Subject: [PATCH 0508/1648] add crosslang.cc

---
 cpp/preprocess |  12 +++--
 crosslang.cc   | 128 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 137 insertions(+), 3 deletions(-)
 create mode 100644 crosslang.cc

diff --git a/cpp/preprocess b/cpp/preprocess
index 452006a..aab89f5 100755
--- a/cpp/preprocess
+++ b/cpp/preprocess
@@ -11,9 +11,15 @@ preprocess() {
 	fi
 }
 
-preprocess src/msgpack/type/tuple.hpp
-preprocess src/msgpack/type/define.hpp
-preprocess src/msgpack/zone.hpp
+if [ "$1" == "clean" ];then
+	rm -f src/msgpack/type/tuple.hpp
+	rm -f src/msgpack/type/define.hpp
+	rm -f src/msgpack/zone.hpp
+else
+	preprocess src/msgpack/type/tuple.hpp
+	preprocess src/msgpack/type/define.hpp
+	preprocess src/msgpack/zone.hpp
+fi
 cp -f ../msgpack/sysdep.h          src/msgpack/
 cp -f ../msgpack/pack_define.h     src/msgpack/
 cp -f ../msgpack/pack_template.h   src/msgpack/
diff --git a/crosslang.cc b/crosslang.cc
new file mode 100644
index 0000000..3b22b74
--- /dev/null
+++ b/crosslang.cc
@@ -0,0 +1,128 @@
+//
+// How to compile:
+// $ g++ -Wall -lmsgpack crosslang.cc
+//
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+static int run(int infd, int outfd)
+try {
+	msgpack::unpacker pac;
+
+	while(true) {
+		pac.reserve_buffer(32*1024);
+
+		ssize_t count =
+			read(infd, pac.buffer(), pac.buffer_capacity());
+
+		if(count <= 0) {
+			if(count == 0) {
+				return 0;
+			}
+			if(errno == EAGAIN || errno == EINTR) {
+				continue;
+			}
+			return 1;
+		}
+
+		pac.buffer_consumed(count);
+
+		msgpack::unpacked result;
+		while(pac.next(&result)) {
+			msgpack::sbuffer sbuf;
+			msgpack::pack(sbuf, result.get());
+
+			const char* p = sbuf.data();
+			const char* const pend = p + sbuf.size();
+			while(p < pend) {
+				ssize_t bytes = write(outfd, p, pend-p);
+
+				if(bytes <= 0) {
+					if(count == 0) {
+						return 0;
+					}
+					if(errno == EAGAIN || errno == EINTR) {
+						continue;
+					}
+					return 1;
+				}
+
+				p += bytes;
+			}
+
+		}
+	}
+
+	return 0;
+
+} catch (std::exception& e) {
+	std::cerr << e.what() << std::endl;
+	return 1;
+}
+
+static void usage(const char* prog)
+{
+	printf(
+		"Usage: %s [in-file] [out-file]\n"
+		"\n"
+		"This tool is for testing of MessagePack implementation.\n"
+		"This does following behavior:\n"
+		"\n"
+		"  1. Reads objects serialized by MessagePack from  (default: stdin)\n"
+		"  2. Re-serializes the objects using C++ implementation of MessagePack (Note that C++ implementation is considered valid)\n"
+		"  3. Writes the re-serialized objects into  (default: stdout)\n"
+		"\n"
+		, prog);
+	exit(0);
+}
+
+int main(int argc, char* argv[])
+{
+	int infd = 0;
+	int outfd = 1;
+
+	if(argc < 1 || argc > 3) {
+		usage(argv[0]);
+	}
+
+	for(int i=1; i < argc; ++i) {
+		if(strlen(argv[i]) > 1 && argv[i][0] == '-') {
+			usage(argv[0]);
+		}
+	}
+
+	if(argc >= 2) {
+		const char* fname = argv[1];
+		if(strcmp(fname, "-") != 0) {
+			infd = open(fname, O_RDONLY);
+			if(infd < 0) {
+				perror("can't open input file");
+				exit(1);
+			}
+		}
+	}
+
+	if(argc >= 3) {
+		const char* fname = argv[2];
+		if(strcmp(fname, "-") != 0) {
+			outfd = open(fname, O_WRONLY | O_CREAT| O_TRUNC, 0666);
+			if(outfd < 0) {
+				perror("can't open output file");
+				exit(1);
+			}
+		}
+	}
+
+	int code = run(infd, outfd);
+
+	close(infd);
+	close(outfd);
+
+	return code;
+}
+

From a0071c2f9f1760b6eb1a40c1b6179f02c8ce7cb6 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Mon, 31 May 2010 17:27:51 +0900
Subject: [PATCH 0509/1648] add crosslang.rb

---
 crosslang.cc |  2 +-
 crosslang.rb | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+), 1 deletion(-)
 create mode 100644 crosslang.rb

diff --git a/crosslang.cc b/crosslang.cc
index 3b22b74..9ec7c0f 100644
--- a/crosslang.cc
+++ b/crosslang.cc
@@ -78,7 +78,7 @@ static void usage(const char* prog)
 		"  3. Writes the re-serialized objects into  (default: stdout)\n"
 		"\n"
 		, prog);
-	exit(0);
+	exit(1);
 }
 
 int main(int argc, char* argv[])
diff --git a/crosslang.rb b/crosslang.rb
new file mode 100644
index 0000000..af36dd1
--- /dev/null
+++ b/crosslang.rb
@@ -0,0 +1,81 @@
+begin
+require 'rubygems'
+rescue LoadError
+end
+require 'msgpack'
+
+def run(inio, outio)
+	pac = MessagePack::Unpacker.new(inio)
+
+	begin
+		pac.each {|obj|
+			outio.write MessagePack.pack(obj)
+			outio.flush
+		}
+	rescue EOFError
+		return 0
+	rescue
+		$stderr.puts $!
+		return 1
+	end
+
+	return 0
+end
+
+def usage
+	puts < (default: stdin)
+  2. Re-serializes the objects using Ruby implementation of MessagePack (Note that Ruby implementation is considered valid)
+  3. Writes the re-serialized objects into  (default: stdout)
+
+EOF
+	exit 1
+end
+
+inio = $stdin
+outio = $stdout
+
+if ARGV.length > 2
+	usage
+end
+
+ARGV.each {|str|
+	if str.size > 1 && str[0] == ?-
+		usage
+	end
+}
+
+if fname = ARGV[0]
+	unless fname == "-"
+		begin
+			inio = File.open(fname)
+		rescue
+			puts "can't open output file: #{$!}"
+			exit 1
+		end
+	end
+end
+
+if fname = ARGV[1]
+	unless fname == "-"
+		begin
+			outio = File.open(fname, "w")
+		rescue
+			puts "can't open output file: #{$!}"
+			exit 1
+		end
+	end
+end
+
+code = run(inio, outio)
+
+inio.close
+outio.close
+
+exit code
+

From 7d1e51437e6b9e562420c13d18b2f7aedaa26b13 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Mon, 31 May 2010 23:13:32 +0900
Subject: [PATCH 0510/1648] erlang: added usage of cross-language test.

---
 erlang/msgpack.erl | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index df7974d..90ddb76 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -1,7 +1,7 @@
 %%
 %% MessagePack for Erlang
 %%
-%% Copyright (C) 2009 UENISHI Kota
+%% Copyright (C) 2009-2010 UENISHI Kota
 %%
 %%    Licensed under the Apache License, Version 2.0 (the "License");
 %%    you may not use this file except in compliance with the License.
@@ -332,16 +332,15 @@ test()->
 	     [0,42,"sum", [1,2]], [1,42, nil, [3]]
 	    ],
     Passed = test_(Tests),
-    Passed = length(Tests).
-%%     Port = open_port({spawn, "./a.out"}, [stream]),
-%%     receive {Port, {data, Data}}->
-%%     io:format("~p~n", [unpack_all( list_to_binary(Data) )])
-%%     after 1024-> timeout end,
-%%     Passed2 = test_(Tests, Port),
-%%     Passed2 = length(Tests),
-%%     Port ! {self(), close},
-%%     receive {Port, closed}-> ok 
-%%     after 1024 -> timeout end.
+    Passed = length(Tests),
+    {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])),
+    Port = open_port({spawn, "ruby ../crosslang.rb"}, [binary]),
+    true = port_command(Port, msgpack:pack(Tests) ),
+     %Port ! {self, {command, msgpack:pack(Tests)}}, ... not owner
+    receive
+	{Port, {data, Data}}->  {Tests, <<>>}=msgpack:unpack(Data)
+    after 1024-> ?assert(false)   end,
+    port_close(Port).
 
 test_([]) -> 0;
 test_([S|Rest])->

From d9b467098a8cc5c4521717d58c2561f92222b333 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Mon, 31 May 2010 23:56:06 +0900
Subject: [PATCH 0511/1648] erlang: added more cross-language tests. better
 type specification.

---
 erlang/msgpack.erl           | 80 ++++++++++++++++++++++--------------
 erlang/testcase_generator.rb | 61 +++++++++++++++++++++++++++
 2 files changed, 111 insertions(+), 30 deletions(-)
 create mode 100644 erlang/testcase_generator.rb

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 90ddb76..f23ec09 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -18,6 +18,11 @@
 -module(msgpack).
 -author('kuenishi+msgpack@gmail.com').
 
+%% tuples, atoms are not supported.  lists, integers, double, and so on.
+%% see http://msgpack.sourceforge.jp/spec for
+%% supported formats. APIs are almost compatible
+%% for C API (http://msgpack.sourceforge.jp/c:doc)
+%% except buffering functions (both copying and zero-copying).
 -export([pack/1, unpack/1, unpack_all/1, test/0]).
 
 -include_lib("eunit/include/eunit.hrl").
@@ -27,20 +32,8 @@
 % erl> S = .
 % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
 
-%% tuples, atoms are not supported.  lists, integers, double, and so on.
-%% see http://msgpack.sourceforge.jp/spec for
-%% supported formats. APIs are almost compatible
-%% for C API (http://msgpack.sourceforge.jp/c:doc)
-%% except buffering functions (both copying and zero-copying).
--export([
-	 pack_nil/0,
-	 pack_bool/1,
-	 pack_float/1,
-	 pack_double/1,
-	 pack_raw/1,
-	 pack_array/1,
-	 pack_map/1,
-	 pack_object/1	]).
+
+-type reason() ::  enomem.
 
 % positive fixnum
 pack_uint_(N) when is_integer( N ) , N < 128 ->  
@@ -85,9 +78,9 @@ pack_bool(true)->    << 16#C3:8 >>;
 pack_bool(false)->   << 16#C2:8 >>.
 
 % float : erlang's float is always IEEE 754 64bit format.
-pack_float(F) when is_float(F)->
+%pack_float(F) when is_float(F)->
 %    << 16#CA:8, F:32/big-float-unit:1 >>.
-    pack_double(F).
+%    pack_double(F).
 % double
 pack_double(F) when is_float(F)->
     << 16#CB:8, F:64/big-float-unit:1 >>.
@@ -177,7 +170,9 @@ pack(Obj)->
 % if failed in decoding and not end, get more data 
 % and feed more Bin into this function.
 % TODO: error case for imcomplete format when short for any type formats.
--spec unpack( binary() )-> {term(), binary()}.
+-spec unpack( binary() )-> {term(), binary()} | {more, non_neg_integer()} | {error, reason()}.
+unpack(Bin) when not is_binary(Bin)->
+    {error, badard};
 unpack(Bin) when bit_size(Bin) >= 8 ->
     << Flag:8/unsigned-integer, Payload/binary >> = Bin,
     case Flag of 
@@ -310,7 +305,9 @@ unpack(Bin) when bit_size(Bin) >= 8 ->
 	_Other ->
 	    erlang:display(_Other),
 	    {error, no_code_matches}
-    end.
+    end;
+unpack(_)-> % when bit_size(Bin) < 8 ->
+    {more, 8}.
     
 unpack_all(Data)->
     case unpack(Data) of
@@ -322,31 +319,54 @@ unpack_all(Data)->
 
 -ifdef(EUNIT).
 
-test()->
-    Tests = [0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF,
-	     -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF,
-	     123.123, -234.4355, 1.0e-34, 1.0e64,
-	     [23, 234, 0.23],
-	     "hogehoge", "243546rf7g68h798j",
-	     <<"hoasfdafdas][">>,
-	     [0,42,"sum", [1,2]], [1,42, nil, [3]]
-	    ],
+test_data()->
+    [0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF,
+     -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF,
+     123.123, -234.4355, 1.0e-34, 1.0e64,
+     [23, 234, 0.23],
+     "hogehoge", "243546rf7g68h798j",
+     <<"hoasfdafdas][">>,
+     [0,42,"sum", [1,2]], [1,42, nil, [3]]
+    ].
+
+basic_test()->
+    Tests = test_data(),
     Passed = test_(Tests),
-    Passed = length(Tests),
+    Passed = length(Tests).
+
+port_test()->
+    Tests = test_data(),
     {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])),
     Port = open_port({spawn, "ruby ../crosslang.rb"}, [binary]),
     true = port_command(Port, msgpack:pack(Tests) ),
-     %Port ! {self, {command, msgpack:pack(Tests)}}, ... not owner
+    %Port ! {self, {command, msgpack:pack(Tests)}}, ... not owner
     receive
 	{Port, {data, Data}}->  {Tests, <<>>}=msgpack:unpack(Data)
     after 1024-> ?assert(false)   end,
     port_close(Port).
 
+unknown_test()->
+    Tests = [0, 1, 2, 123, 512, 1230, 678908,
+	     -1, -23, -512, -1230, -567898,
+%	     "hogehoge", "243546rf7g68h798j",
+	     123.123 %-234.4355, 1.0e-34, 1.0e64,
+%	     [23, 234, 0.23]
+%	     [0,42,"sum", [1,2]], [1,42, nil, [3]]
+	    ],
+    Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]),
+    %Port ! {self, {command, msgpack:pack(Tests)}}, ... not owner
+    receive
+	{Port, {data, Data}}->
+	    Tests=msgpack:unpack_all(Data)
+%	    io:format("~p~n", [Tests])
+    after 1024-> ?assert(false)   end,
+    port_close(Port).
+
 test_([]) -> 0;
 test_([S|Rest])->
     Pack = msgpack:pack(S),
 %    io:format("testing: ~p => ~p~n", [S, Pack]),
     {S, <<>>} = msgpack:unpack( Pack ),
-    test_(Rest) + 1.
+    1+test_(Rest).
 
 -endif.
diff --git a/erlang/testcase_generator.rb b/erlang/testcase_generator.rb
new file mode 100644
index 0000000..a173790
--- /dev/null
+++ b/erlang/testcase_generator.rb
@@ -0,0 +1,61 @@
+begin
+require 'rubygems'
+rescue LoadError
+end
+require 'msgpack'
+
+def usage
+	puts < (default: stdout)
+
+EOF
+	exit 1
+end
+
+code = 1
+outio = $stdout
+
+if ARGV.length > 2
+	usage
+end
+
+if fname = ARGV[0]
+	unless fname == "-"
+		begin
+			outio = File.open(fname, "w")
+		rescue
+			puts "can't open output file: #{$!}"
+			exit 1
+		end
+	end
+end
+
+objs = [0, 1, 2, 123, 512, 1230, 678908,
+        -1, -23, -512, -1230, -567898,
+#        "hogehoge", "243546rf7g68h798j",
+        123.123, #-234.4355, 1.0e-34, 1.0e64,
+#        [23, 234, 0.23]
+#        [0,42,"sum", [1,2]], [1,42, nil, [3]]
+       ]
+begin
+  objs.each do |obj|
+    outio.write MessagePack.pack(obj)
+    outio.flush
+  end
+rescue EOFError
+  code=0
+rescue
+  $stderr.puts $!
+  code=1
+end
+
+outio.close
+exit code
+

From 49f3872d047624b1995b8c60edec8bad35429fd3 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Tue, 1 Jun 2010 00:31:12 +0900
Subject: [PATCH 0512/1648] erlang: temporary documentation and .gitignore

---
 erlang/.gitignore |  3 ++-
 erlang/README.md  | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 erlang/README.md

diff --git a/erlang/.gitignore b/erlang/.gitignore
index 911d6e3..7d6d324 100644
--- a/erlang/.gitignore
+++ b/erlang/.gitignore
@@ -1,3 +1,4 @@
 MANIFEST
 *.beam
-
+.omakedb*
+*.omc
diff --git a/erlang/README.md b/erlang/README.md
new file mode 100644
index 0000000..50d446d
--- /dev/null
+++ b/erlang/README.md
@@ -0,0 +1,19 @@
+MessagePack for Erlang
+======================
+Binary-based efficient object serialization library.
+
+## Status 
+
+still in development.
+
+TODOs:
+ - decide string specification.
+
+## Installation
+
+## Example
+
+## License
+
+
+ - 

From 062ed8a4c46be31da26538a4144fd441e453f30c Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 1 Jun 2010 04:47:28 +0900
Subject: [PATCH 0513/1648] add test/cases.mpac and test/cases_compact.mpac

---
 test/cases.mpac                   | Bin 0 -> 213 bytes
 test/cases_compact.mpac           | Bin 0 -> 116 bytes
 test/cases_gen.rb                 |  76 ++++++++++++++++++++++++++++++
 crosslang.cc => test/crosslang.cc |   7 ++-
 crosslang.rb => test/crosslang.rb |   7 +++
 5 files changed, 89 insertions(+), 1 deletion(-)
 create mode 100644 test/cases.mpac
 create mode 100644 test/cases_compact.mpac
 create mode 100644 test/cases_gen.rb
 rename crosslang.cc => test/crosslang.cc (94%)
 rename crosslang.rb => test/crosslang.rb (91%)

diff --git a/test/cases.mpac b/test/cases.mpac
new file mode 100644
index 0000000000000000000000000000000000000000..5ec08c6a9af42d9568eb429a209a639616e94711
GIT binary patch
literal 213
zcmXYp!4<+V3`3R4n8lOSY|v~#CV>aXw2-v7Qc6c)17ihrkbe}**V_dHM&J(DgGLop
zU?R;l%8FI9$y_sy>V|HFdDW~{neAn-roN|Wd+OcH160;F91fo!97} FixMap
+de 00 01 a1 61 61           # {"a"=>97} map 16
+df 00 00 00 01 a1 61 61     # {"a"=>97} map 32
+91 90                       # [[]]
+91 91 a1 61                 # [["a"]]
+EOF
+
+source.gsub!(/\#.+$/,'')
+bytes = source.strip.split(/\s+/).map {|x| x.to_i(16) }.pack('C*')
+
+compact_bytes = ""
+pac = MessagePack::Unpacker.new
+pac.feed(bytes)
+pac.each {|obj|
+	p obj
+	compact_bytes << obj.to_msgpack
+}
+
+File.open("cases.mpac","w") {|f| f.write(bytes) }
+File.open("cases_compact.mpac","w") {|f| f.write(compact_bytes) }
+
diff --git a/crosslang.cc b/test/crosslang.cc
similarity index 94%
rename from crosslang.cc
rename to test/crosslang.cc
index 9ec7c0f..be521b3 100644
--- a/crosslang.cc
+++ b/test/crosslang.cc
@@ -1,5 +1,10 @@
 //
-// How to compile:
+// MessagePack cross-language test tool
+//
+// $ cd ../cpp && ./configure && make && make install
+// or
+// $ port install msgpack  # MacPorts
+//
 // $ g++ -Wall -lmsgpack crosslang.cc
 //
 #include 
diff --git a/crosslang.rb b/test/crosslang.rb
similarity index 91%
rename from crosslang.rb
rename to test/crosslang.rb
index af36dd1..7aa4482 100644
--- a/crosslang.rb
+++ b/test/crosslang.rb
@@ -1,3 +1,10 @@
+#
+# MessagePack cross-language test tool
+#
+# $ gem install msgpack
+# or
+# $ port install rb_msgpack   # MacPorts
+#
 begin
 require 'rubygems'
 rescue LoadError

From 6056f939103624d21092a5e4a4d8ffaf9204c191 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 1 Jun 2010 05:15:36 +0900
Subject: [PATCH 0514/1648] cpp: add cases.mpac test

---
 cpp/preprocess       |  2 ++
 cpp/test/Makefile.am |  5 +++++
 cpp/test/cases.cc    | 36 ++++++++++++++++++++++++++++++++++++
 test/cases_gen.rb    | 15 +++++++++++++++
 4 files changed, 58 insertions(+)
 create mode 100644 cpp/test/cases.cc

diff --git a/cpp/preprocess b/cpp/preprocess
index aab89f5..1a5e6a3 100755
--- a/cpp/preprocess
+++ b/cpp/preprocess
@@ -25,4 +25,6 @@ cp -f ../msgpack/pack_define.h     src/msgpack/
 cp -f ../msgpack/pack_template.h   src/msgpack/
 cp -f ../msgpack/unpack_define.h   src/msgpack/
 cp -f ../msgpack/unpack_template.h src/msgpack/
+cp -f ../test/cases.mpac           test/
+cp -f ../test/cases_compact.mpac   test/
 
diff --git a/cpp/test/Makefile.am b/cpp/test/Makefile.am
index 3670d7f..f9c5f22 100644
--- a/cpp/test/Makefile.am
+++ b/cpp/test/Makefile.am
@@ -10,6 +10,7 @@ check_PROGRAMS = \
 		object \
 		convert \
 		buffer \
+		cases \
 		msgpackc_test \
 		msgpack_test
 
@@ -28,7 +29,11 @@ convert_SOURCES = convert.cc
 buffer_SOURCES = buffer.cc
 buffer_LDADD = -lz
 
+cases_SOURCES = cases.cc
+
 msgpackc_test_SOURCES = msgpackc_test.cpp
 
 msgpack_test_SOURCES = msgpack_test.cpp
 
+EXTRA_DIST = cases.mpac cases_compact.mpac
+
diff --git a/cpp/test/cases.cc b/cpp/test/cases.cc
new file mode 100644
index 0000000..b408876
--- /dev/null
+++ b/cpp/test/cases.cc
@@ -0,0 +1,36 @@
+#include 
+#include 
+#include 
+
+static void feed_file(msgpack::unpacker& pac, const char* path)
+{
+	std::ifstream fin(path);
+	while(true) {
+		pac.reserve_buffer(32*1024);
+		fin.read(pac.buffer(), pac.buffer_capacity());
+		if(fin.bad()) {
+			throw std::runtime_error("read failed");
+		}
+		pac.buffer_consumed(fin.gcount());
+		if(fin.fail()) {
+			break;
+		}
+	}
+}
+
+TEST(cases, format)
+{
+	msgpack::unpacker pac;
+	msgpack::unpacker pac_compact;
+
+	feed_file(pac, "cases.mpac");
+	feed_file(pac_compact, "cases_compact.mpac");
+
+	msgpack::unpacked result;
+	while(pac.next(&result)) {
+		msgpack::unpacked result_compact;
+		EXPECT_TRUE( pac_compact.next(&result_compact) );
+		EXPECT_EQ(result_compact.get(), result.get());
+	}
+}
+
diff --git a/test/cases_gen.rb b/test/cases_gen.rb
index 2fb7273..7efbfe7 100644
--- a/test/cases_gen.rb
+++ b/test/cases_gen.rb
@@ -63,14 +63,29 @@ EOF
 source.gsub!(/\#.+$/,'')
 bytes = source.strip.split(/\s+/).map {|x| x.to_i(16) }.pack('C*')
 
+objs = []
 compact_bytes = ""
 pac = MessagePack::Unpacker.new
 pac.feed(bytes)
 pac.each {|obj|
 	p obj
+	objs << obj
 	compact_bytes << obj.to_msgpack
 }
 
+# self check
+cpac = MessagePack::Unpacker.new
+cpac.feed(compact_bytes)
+cpac.each {|cobj|
+	obj = objs.shift
+	if obj != cobj
+		puts "** SELF CHECK FAILED **"
+		puts "expected: #{obj.inspect}"
+		puts "actual: #{cobj.inspect}"
+		exit 1
+	end
+}
+
 File.open("cases.mpac","w") {|f| f.write(bytes) }
 File.open("cases_compact.mpac","w") {|f| f.write(compact_bytes) }
 

From e49f091b4ebc7abbad92848502a6e06d4e9d6cfa Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 1 Jun 2010 07:10:17 +0900
Subject: [PATCH 0515/1648] cpp: adds msgpack_sbuffer_new and
 msgpack_sbuffer_free

---
 cpp/src/msgpack/sbuffer.h | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/cpp/src/msgpack/sbuffer.h b/cpp/src/msgpack/sbuffer.h
index 57f424a..caff2e8 100644
--- a/cpp/src/msgpack/sbuffer.h
+++ b/cpp/src/msgpack/sbuffer.h
@@ -46,6 +46,18 @@ static inline void msgpack_sbuffer_destroy(msgpack_sbuffer* sbuf)
 	free(sbuf->data);
 }
 
+static inline msgpack_sbuffer* msgpack_sbuffer_new(void)
+{
+	return (msgpack_sbuffer*)calloc(1, sizeof(msgpack_sbuffer));
+}
+
+static inline void msgpack_sbuffer_free(msgpack_sbuffer* sbuf)
+{
+	if(sbuf == NULL) { return; }
+	msgpack_sbuffer_destroy(sbuf);
+	free(sbuf);
+}
+
 static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned int len)
 {
 	msgpack_sbuffer* sbuf = (msgpack_sbuffer*)data;

From 103b14ea3c8a3f72213295d975328dc2399725eb Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 1 Jun 2010 07:10:39 +0900
Subject: [PATCH 0516/1648] cpp: adds msgpack_zbuffer_new and
 msgpack_zbuffer_free

---
 cpp/src/msgpack/zbuffer.h | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/cpp/src/msgpack/zbuffer.h b/cpp/src/msgpack/zbuffer.h
index 2a32206..0ff9675 100644
--- a/cpp/src/msgpack/zbuffer.h
+++ b/cpp/src/msgpack/zbuffer.h
@@ -47,6 +47,9 @@ static inline bool msgpack_zbuffer_init(msgpack_zbuffer* zbuf,
 		int level, size_t init_size);
 static inline void msgpack_zbuffer_destroy(msgpack_zbuffer* zbuf);
 
+static inline msgpack_zbuffer* msgpack_zbuffer_new(int level, size_t init_size);
+static inline void msgpack_zbuffer_free(msgpack_zbuffer* zbuf);
+
 static inline char* msgpack_zbuffer_flush(msgpack_zbuffer* zbuf);
 
 static inline const char* msgpack_zbuffer_data(const msgpack_zbuffer* zbuf);
@@ -80,6 +83,23 @@ void msgpack_zbuffer_destroy(msgpack_zbuffer* zbuf)
 	free(zbuf->data);
 }
 
+msgpack_zbuffer* msgpack_zbuffer_new(int level, size_t init_size)
+{
+	msgpack_zbuffer* zbuf = (msgpack_zbuffer*)malloc(sizeof(msgpack_zbuffer));
+	if(!msgpack_zbuffer_init(zbuf, level, init_size)) {
+		free(zbuf);
+		return NULL;
+	}
+	return zbuf;
+}
+
+void msgpack_zbuffer_free(msgpack_zbuffer* zbuf)
+{
+	if(zbuf == NULL) { return; }
+	msgpack_zbuffer_destroy(zbuf);
+	free(zbuf);
+}
+
 bool msgpack_zbuffer_expand(msgpack_zbuffer* zbuf)
 {
 	size_t used = (char*)zbuf->stream.next_out - zbuf->data;

From 5a92c861e377c7db3b240dd3661feed0b58f85e0 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 1 Jun 2010 07:11:01 +0900
Subject: [PATCH 0517/1648] cpp: adds msgpack_vrefbuffer_new and
 msgpack_vrefbuffer_free

---
 cpp/src/msgpack/vrefbuffer.h | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/cpp/src/msgpack/vrefbuffer.h b/cpp/src/msgpack/vrefbuffer.h
index a08e0d0..ffb2302 100644
--- a/cpp/src/msgpack/vrefbuffer.h
+++ b/cpp/src/msgpack/vrefbuffer.h
@@ -19,6 +19,7 @@
 #define MSGPACK_VREFBUFFER_H__
 
 #include "msgpack/zone.h"
+#include 
 
 #ifndef _WIN32
 #include 
@@ -67,6 +68,9 @@ bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf,
 		size_t ref_size, size_t chunk_size);
 void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf);
 
+static inline msgpack_vrefbuffer* msgpack_vrefbuffer_new(size_t ref_size, size_t chunk_size);
+static inline void msgpack_vrefbuffer_free(msgpack_vrefbuffer* vbuf);
+
 static inline int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len);
 
 static inline const struct iovec* msgpack_vrefbuffer_vec(const msgpack_vrefbuffer* vref);
@@ -83,6 +87,23 @@ int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to)
 void msgpack_vrefbuffer_clear(msgpack_vrefbuffer* vref);
 
 
+msgpack_vrefbuffer* msgpack_vrefbuffer_new(size_t ref_size, size_t chunk_size)
+{
+	msgpack_vrefbuffer* vbuf = (msgpack_vrefbuffer*)malloc(sizeof(msgpack_vrefbuffer));
+	if(!msgpack_vrefbuffer_init(vbuf, ref_size, chunk_size)) {
+		free(vbuf);
+		return NULL;
+	}
+	return vbuf;
+}
+
+void msgpack_vrefbuffer_free(msgpack_vrefbuffer* vbuf)
+{
+	if(vbuf == NULL) { return; }
+	msgpack_vrefbuffer_destroy(vbuf);
+	free(vbuf);
+}
+
 int msgpack_vrefbuffer_write(void* data, const char* buf, unsigned int len)
 {
 	msgpack_vrefbuffer* vbuf = (msgpack_vrefbuffer*)data;

From d42ecccf6f201eb92b6c04b0dde1b86a8861e58e Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 1 Jun 2010 07:13:47 +0900
Subject: [PATCH 0518/1648] cpp: msgpack::unpack returns void

---
 cpp/src/msgpack/unpack.hpp |  8 ++++----
 cpp/test/pack_unpack.cc    | 10 +++-------
 2 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/cpp/src/msgpack/unpack.hpp b/cpp/src/msgpack/unpack.hpp
index 56ce0f6..e1617ef 100644
--- a/cpp/src/msgpack/unpack.hpp
+++ b/cpp/src/msgpack/unpack.hpp
@@ -160,7 +160,7 @@ private:
 };
 
 
-static bool unpack(unpacked* result,
+static void unpack(unpacked* result,
 		const char* data, size_t len, size_t* offset = NULL);
 
 
@@ -312,7 +312,7 @@ inline void unpacker::remove_nonparsed_buffer()
 }
 
 
-inline bool unpack(unpacked* result,
+inline void unpack(unpacked* result,
 		const char* data, size_t len, size_t* offset)
 {
 	msgpack::object obj;
@@ -326,12 +326,12 @@ inline bool unpack(unpacked* result,
 	case UNPACK_SUCCESS:
 		result->get() = obj;
 		result->zone() = z;
-		return false;
+		return;
 
 	case UNPACK_EXTRA_BYTES:
 		result->get() = obj;
 		result->zone() = z;
-		return true;
+		return;
 
 	case UNPACK_CONTINUE:
 		throw unpack_error("insufficient bytes");
diff --git a/cpp/test/pack_unpack.cc b/cpp/test/pack_unpack.cc
index ca9b7d5..fe4625a 100644
--- a/cpp/test/pack_unpack.cc
+++ b/cpp/test/pack_unpack.cc
@@ -77,21 +77,17 @@ TEST(unpack, sequence)
 	msgpack::pack(sbuf, 2);
 	msgpack::pack(sbuf, 3);
 
-	bool cont;
 	size_t offset = 0;
 
 	msgpack::unpacked msg;
 
-	cont = msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
-	EXPECT_TRUE(cont);
+	msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
 	EXPECT_EQ(1, msg.get().as());
 
-	cont = msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
-	EXPECT_TRUE(cont);
+	msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
 	EXPECT_EQ(2, msg.get().as());
 
-	cont = msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
-	EXPECT_FALSE(cont);
+	msgpack::unpack(&msg, sbuf.data(), sbuf.size(), &offset);
 	EXPECT_EQ(3, msg.get().as());
 }
 

From 684bca203ad862f30558429081d1a27681fe4559 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 1 Jun 2010 07:15:58 +0900
Subject: [PATCH 0519/1648] cpp: adds msgpack_unpacker_next and
 msgpack_unpack_next

---
 cpp/src/msgpack/unpack.h   | 50 ++++++++++++++++++++++++++-
 cpp/src/msgpack/unpack.hpp |  7 ++--
 cpp/src/unpack.c           | 70 +++++++++++++++++++++++++++++++++++---
 cpp/src/zone.c             |  1 +
 4 files changed, 119 insertions(+), 9 deletions(-)

diff --git a/cpp/src/msgpack/unpack.h b/cpp/src/msgpack/unpack.h
index e17d0d8..1f43ab7 100644
--- a/cpp/src/msgpack/unpack.h
+++ b/cpp/src/msgpack/unpack.h
@@ -20,6 +20,15 @@
 
 #include "msgpack/zone.h"
 #include "msgpack/object.h"
+#include 
+
+#ifndef MSGPACK_UNPACKER_INIT_BUFFER_SIZE
+#define MSGPACK_UNPACKER_INIT_BUFFER_SIZE (64*1024)
+#endif
+
+#ifndef MSGPACK_UNPACKER_RESERVE_SIZE
+#define MSGPACK_UNPACKER_RESERVE_SIZE (32*1024)
+#endif
 
 #ifdef __cplusplus
 extern "C" {
@@ -37,6 +46,10 @@ typedef struct msgpack_unpacker {
 	void* ctx;
 } msgpack_unpacker;
 
+typedef struct msgpack_unpacked {
+	msgpack_zone* zone;
+	msgpack_object data;
+} msgpack_unpacked;
 
 bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size);
 void msgpack_unpacker_destroy(msgpack_unpacker* mpac);
@@ -49,6 +62,13 @@ static inline char*  msgpack_unpacker_buffer(msgpack_unpacker* mpac);
 static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac);
 static inline void   msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size);
 
+bool msgpack_unpacker_next(msgpack_unpacker* mpac, msgpack_unpacked* pac);
+
+static inline void msgpack_unpacked_init(msgpack_unpacked* result);
+static inline void msgpack_unpacked_destroy(msgpack_unpacked* result);
+
+static inline msgpack_zone* msgpack_unpacked_release_zone(msgpack_unpacked* result);
+
 
 int msgpack_unpacker_execute(msgpack_unpacker* mpac);
 
@@ -63,6 +83,9 @@ void msgpack_unpacker_reset(msgpack_unpacker* mpac);
 static inline size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac);
 
 
+bool msgpack_unpack_next(msgpack_unpacked* result,
+		const char* data, size_t len, size_t* off);
+
 
 typedef enum {
 	MSGPACK_UNPACK_SUCCESS				=  2,
@@ -73,7 +96,7 @@ typedef enum {
 
 msgpack_unpack_return
 msgpack_unpack(const char* data, size_t len, size_t* off,
-		msgpack_zone* z, msgpack_object* result);
+		msgpack_zone* result_zone, msgpack_object* result);
 
 
 static inline size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac);
@@ -115,6 +138,31 @@ size_t msgpack_unpacker_parsed_size(const msgpack_unpacker* mpac)
 }
 
 
+void msgpack_unpacked_init(msgpack_unpacked* result)
+{
+	memset(result, 0, sizeof(msgpack_unpacked));
+}
+
+void msgpack_unpacked_destroy(msgpack_unpacked* result)
+{
+	if(result->zone != NULL) {
+		msgpack_zone_free(result->zone);
+		result->zone = NULL;
+		memset(&result->data, 0, sizeof(msgpack_object));
+	}
+}
+
+msgpack_zone* msgpack_unpacked_release_zone(msgpack_unpacked* result)
+{
+	if(result->zone != NULL) {
+		msgpack_zone* z = result->zone;
+		result->zone = NULL;
+		return z;
+	}
+	return NULL;
+}
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/cpp/src/msgpack/unpack.hpp b/cpp/src/msgpack/unpack.hpp
index e1617ef..7b45017 100644
--- a/cpp/src/msgpack/unpack.hpp
+++ b/cpp/src/msgpack/unpack.hpp
@@ -24,8 +24,9 @@
 #include 
 #include 
 
+// backward compatibility
 #ifndef MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE
-#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE (32*1024)
+#define MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE MSGPACK_UNPACKER_INIT_BUFFER_SIZE
 #endif
 
 namespace msgpack {
@@ -64,12 +65,12 @@ private:
 
 class unpacker : public msgpack_unpacker {
 public:
-	unpacker(size_t init_buffer_size = MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE);
+	unpacker(size_t init_buffer_size = MSGPACK_UNPACKER_INIT_BUFFER_SIZE);
 	~unpacker();
 
 public:
 	/*! 1. reserve buffer. at least `size' bytes of capacity will be ready */
-	void reserve_buffer(size_t size);
+	void reserve_buffer(size_t size = MSGPACK_UNPACKER_RESERVE_SIZE);
 
 	/*! 2. read data to the buffer() up to buffer_capacity() bytes */
 	char* buffer();
diff --git a/cpp/src/unpack.c b/cpp/src/unpack.c
index 4a42526..0c21780 100644
--- a/cpp/src/unpack.c
+++ b/cpp/src/unpack.c
@@ -363,20 +363,46 @@ void msgpack_unpacker_reset(msgpack_unpacker* mpac)
 	mpac->parsed = 0;
 }
 
+bool msgpack_unpacker_next(msgpack_unpacker* mpac, msgpack_unpacked* result)
+{
+	if(result->zone != NULL) {
+		msgpack_zone_free(result->zone);
+	}
+
+	int ret = msgpack_unpacker_execute(mpac);
+
+	if(ret <= 0) {
+		result->zone = NULL;
+		memset(&result->data, 0, sizeof(msgpack_object));
+		return false;
+	}
+
+	result->zone = msgpack_unpacker_release_zone(mpac);
+	result->data = msgpack_unpacker_data(mpac);
+	msgpack_unpacker_reset(mpac);
+
+	return true;
+}
+
 
 msgpack_unpack_return
 msgpack_unpack(const char* data, size_t len, size_t* off,
-		msgpack_zone* z, msgpack_object* result)
+		msgpack_zone* result_zone, msgpack_object* result)
 {
+	size_t noff = 0;
+	if(off != NULL) { noff = *off; }
+
+	if(len <= noff) {
+		// FIXME
+		return MSGPACK_UNPACK_CONTINUE;
+	}
+
 	template_context ctx;
 	template_init(&ctx);
 
-	ctx.user.z = z;
+	ctx.user.z = result_zone;
 	ctx.user.referenced = false;
 
-	size_t noff = 0;
-	if(off != NULL) { noff = *off; }
-
 	int e = template_execute(&ctx, data, len, &noff);
 	if(e < 0) {
 		return MSGPACK_UNPACK_PARSE_ERROR;
@@ -397,3 +423,37 @@ msgpack_unpack(const char* data, size_t len, size_t* off,
 	return MSGPACK_UNPACK_SUCCESS;
 }
 
+bool msgpack_unpack_next(msgpack_unpacked* result,
+		const char* data, size_t len, size_t* off)
+{
+	msgpack_unpacked_destroy(result);
+
+	size_t noff = 0;
+	if(off != NULL) { noff = *off; }
+
+	if(len <= noff) {
+		return false;
+	}
+
+	msgpack_zone* z = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE);
+
+	template_context ctx;
+	template_init(&ctx);
+
+	ctx.user.z = z;
+	ctx.user.referenced = false;
+
+	int e = template_execute(&ctx, data, len, &noff);
+	if(e <= 0) {
+		msgpack_zone_free(z);
+		return false;
+	}
+
+	if(off != NULL) { *off = noff; }
+
+	result->zone = z;
+	result->data = template_data(&ctx);
+
+	return true;
+}
+
diff --git a/cpp/src/zone.c b/cpp/src/zone.c
index 85de765..8cc8b0d 100644
--- a/cpp/src/zone.c
+++ b/cpp/src/zone.c
@@ -214,6 +214,7 @@ msgpack_zone* msgpack_zone_new(size_t chunk_size)
 
 void msgpack_zone_free(msgpack_zone* zone)
 {
+	if(zone == NULL) { return; }
 	msgpack_zone_destroy(zone);
 	free(zone);
 }

From eabcf15790a774ce718ae1738c6ddb407e1cd279 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 1 Jun 2010 07:16:25 +0900
Subject: [PATCH 0520/1648] cpp: update tests

---
 cpp/test/Makefile.am      |  6 ++++
 cpp/test/pack_unpack_c.cc | 70 +++++++++++++++++++++++++++++++++++++++
 cpp/test/streaming.cc     | 19 +++++++----
 cpp/test/streaming_c.cc   | 57 +++++++++++++++++++++++++++++++
 4 files changed, 146 insertions(+), 6 deletions(-)
 create mode 100644 cpp/test/pack_unpack_c.cc
 create mode 100644 cpp/test/streaming_c.cc

diff --git a/cpp/test/Makefile.am b/cpp/test/Makefile.am
index f9c5f22..1f52215 100644
--- a/cpp/test/Makefile.am
+++ b/cpp/test/Makefile.am
@@ -6,7 +6,9 @@ AM_LDFLAGS = ../src/libmsgpack.la -lgtest_main
 check_PROGRAMS = \
 		zone \
 		pack_unpack \
+		pack_unpack_c \
 		streaming \
+		streaming_c \
 		object \
 		convert \
 		buffer \
@@ -20,8 +22,12 @@ zone_SOURCES = zone.cc
 
 pack_unpack_SOURCES = pack_unpack.cc
 
+pack_unpack_c_SOURCES = pack_unpack_c.cc
+
 streaming_SOURCES = streaming.cc
 
+streaming_c_SOURCES = streaming_c.cc
+
 object_SOURCES = object.cc
 
 convert_SOURCES = convert.cc
diff --git a/cpp/test/pack_unpack_c.cc b/cpp/test/pack_unpack_c.cc
new file mode 100644
index 0000000..e9a0389
--- /dev/null
+++ b/cpp/test/pack_unpack_c.cc
@@ -0,0 +1,70 @@
+#include 
+#include 
+#include 
+
+TEST(pack, num)
+{
+	msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
+	msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
+
+	EXPECT_EQ(0, msgpack_pack_int(pk, 1));
+
+	msgpack_sbuffer_free(sbuf);
+	msgpack_packer_free(pk);
+}
+
+
+TEST(pack, array)
+{
+	msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
+	msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
+
+	EXPECT_EQ(0, msgpack_pack_array(pk, 3));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 1));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 2));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 3));
+
+	msgpack_sbuffer_free(sbuf);
+	msgpack_packer_free(pk);
+}
+
+
+TEST(unpack, sequence)
+{
+	msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
+	msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
+
+	EXPECT_EQ(0, msgpack_pack_int(pk, 1));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 2));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 3));
+
+	msgpack_packer_free(pk);
+
+	bool success;
+	size_t offset = 0;
+
+	msgpack_unpacked msg;
+	msgpack_unpacked_init(&msg);
+
+	success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
+	EXPECT_TRUE(success);
+	EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
+	EXPECT_EQ(1, msg.data.via.u64);
+
+	success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
+	EXPECT_TRUE(success);
+	EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
+	EXPECT_EQ(2, msg.data.via.u64);
+
+	success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
+	EXPECT_TRUE(success);
+	EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, msg.data.type);
+	EXPECT_EQ(3, msg.data.via.u64);
+
+	success = msgpack_unpack_next(&msg, sbuf->data, sbuf->size, &offset);
+	EXPECT_FALSE(success);
+
+	msgpack_sbuffer_free(sbuf);
+	msgpack_unpacked_destroy(&msg);
+}
+
diff --git a/cpp/test/streaming.cc b/cpp/test/streaming.cc
index c01b8be..e80c671 100644
--- a/cpp/test/streaming.cc
+++ b/cpp/test/streaming.cc
@@ -2,28 +2,33 @@
 #include 
 #include 
 
-
 TEST(streaming, basic)
 {
-	std::ostringstream stream;
-	msgpack::packer pk(&stream);
+	msgpack::sbuffer buffer;
 
+	msgpack::packer pk(&buffer);
 	pk.pack(1);
 	pk.pack(2);
 	pk.pack(3);
 
-	std::istringstream input(stream.str());
+	const char* input = buffer.data();
+	const char* const eof = input + buffer.size();
 
 	msgpack::unpacker pac;
+	msgpack::unpacked result;
 
 	int count = 0;
 	while(count < 3) {
 		pac.reserve_buffer(32*1024);
 
-		size_t len = input.readsome(pac.buffer(), pac.buffer_capacity());
+		// read buffer into pac.buffer() upto
+		// pac.buffer_capacity() bytes.
+		size_t len = 1;
+		memcpy(pac.buffer(), input, len);
+		input += len;
+
 		pac.buffer_consumed(len);
 
-		msgpack::unpacked result;
 		while(pac.next(&result)) {
 			msgpack::object obj = result.get();
 			switch(count++) {
@@ -38,6 +43,8 @@ TEST(streaming, basic)
 				return;
 			}
 		}
+
+		EXPECT_TRUE(input < eof);
 	}
 }
 
diff --git a/cpp/test/streaming_c.cc b/cpp/test/streaming_c.cc
new file mode 100644
index 0000000..6c87ac6
--- /dev/null
+++ b/cpp/test/streaming_c.cc
@@ -0,0 +1,57 @@
+#include 
+#include 
+#include 
+
+TEST(streaming, basic)
+{
+	msgpack_sbuffer* buffer = msgpack_sbuffer_new();
+
+	msgpack_packer* pk = msgpack_packer_new(buffer, msgpack_sbuffer_write);
+	EXPECT_EQ(0, msgpack_pack_int(pk, 1));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 2));
+	EXPECT_EQ(0, msgpack_pack_int(pk, 3));
+	msgpack_packer_free(pk);
+
+	const char* input = buffer->data;
+	const char* const eof = input + buffer->size;
+
+	msgpack_unpacker pac;
+	msgpack_unpacker_init(&pac, MSGPACK_UNPACKER_INIT_BUFFER_SIZE);
+
+	msgpack_unpacked result;
+	msgpack_unpacked_init(&result);
+
+	int count = 0;
+	while(count < 3) {
+		msgpack_unpacker_reserve_buffer(&pac, 32*1024);
+
+		/* read buffer into msgpack_unapcker_buffer(&pac) upto
+		 * msgpack_unpacker_buffer_capacity(&pac) bytes. */
+		size_t len = 1;
+		memcpy(msgpack_unpacker_buffer(&pac), input, len);
+		input += len;
+
+		msgpack_unpacker_buffer_consumed(&pac, len);
+
+		while(msgpack_unpacker_next(&pac, &result)) {
+			msgpack_object obj = result.data;
+			switch(count++) {
+			case 0:
+				EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, result.data.type);
+				EXPECT_EQ(1, result.data.via.u64);
+				break;
+			case 1:
+				EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, result.data.type);
+				EXPECT_EQ(2, result.data.via.u64);
+				break;
+			case 2:
+				EXPECT_EQ(MSGPACK_OBJECT_POSITIVE_INTEGER, result.data.type);
+				EXPECT_EQ(3, result.data.via.u64);
+				return;
+			}
+		}
+
+		EXPECT_TRUE(input < eof);
+	}
+}
+

From 3d3af3284e3a5fd03395b6694fd745491509aa64 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 1 Jun 2010 08:43:30 +0900
Subject: [PATCH 0521/1648] cpp: adds Doxyfile

---
 cpp/Doxyfile                 | 1552 ++++++++++++++++++++++++++++++++++
 cpp/Makefile.am              |    6 +
 cpp/preprocess               |    1 +
 cpp/src/Makefile.am          |   17 +
 cpp/src/msgpack.h            |    5 +
 cpp/src/msgpack/object.h     |    8 +
 cpp/src/msgpack/pack.h       |   15 +
 cpp/src/msgpack/sbuffer.h    |   17 +-
 cpp/src/msgpack/unpack.h     |  117 ++-
 cpp/src/msgpack/vrefbuffer.h |   24 +-
 cpp/src/msgpack/zbuffer.h    |   23 +-
 cpp/src/msgpack/zone.h       |    7 +
 12 files changed, 1758 insertions(+), 34 deletions(-)
 create mode 100644 cpp/Doxyfile

diff --git a/cpp/Doxyfile b/cpp/Doxyfile
new file mode 100644
index 0000000..ca77230
--- /dev/null
+++ b/cpp/Doxyfile
@@ -0,0 +1,1552 @@
+# Doxyfile 1.6.2
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project
+#
+# All text after a hash (#) is considered a comment and will be ignored
+# The format is:
+#       TAG = value [value, ...]
+# For lists items can also be appended using:
+#       TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (" ")
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+
+# This tag specifies the encoding used for all characters in the config file
+# that follow. The default is UTF-8 which is also the encoding used for all
+# text before the first occurrence of this tag. Doxygen uses libiconv (or the
+# iconv built into libc) for the transcoding. See
+# http://www.gnu.org/software/libiconv for the list of possible encodings.
+
+DOXYFILE_ENCODING      = UTF-8
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded
+# by quotes) that should identify the project.
+
+PROJECT_NAME           = "MessagePack"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number.
+# This could be handy for archiving the generated documentation or
+# if some version control system is used.
+
+PROJECT_NUMBER         =
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute)
+# base path where the generated documentation will be put.
+# If a relative path is entered, it will be relative to the location
+# where doxygen was started. If left blank the current directory will be used.
+
+OUTPUT_DIRECTORY       = doc
+
+# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create
+# 4096 sub-directories (in 2 levels) under the output directory of each output
+# format and will distribute the generated files over these directories.
+# Enabling this option can be useful when feeding doxygen a huge amount of
+# source files, where putting all generated files in the same directory would
+# otherwise cause performance problems for the file system.
+
+CREATE_SUBDIRS         = NO
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all constant output in the proper language.
+# The default language is English, other supported languages are:
+# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional,
+# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German,
+# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English
+# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian,
+# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak,
+# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese.
+
+OUTPUT_LANGUAGE        = English
+
+# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will
+# include brief member descriptions after the members that are listed in
+# the file and class documentation (similar to JavaDoc).
+# Set to NO to disable this.
+
+BRIEF_MEMBER_DESC      = YES
+
+# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend
+# the brief description of a member or function before the detailed description.
+# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
+# brief descriptions will be completely suppressed.
+
+REPEAT_BRIEF           = YES
+
+# This tag implements a quasi-intelligent brief description abbreviator
+# that is used to form the text in various listings. Each string
+# in this list, if found as the leading text of the brief description, will be
+# stripped from the text and the result after processing the whole list, is
+# used as the annotated text. Otherwise, the brief description is used as-is.
+# If left blank, the following values are used ("$name" is automatically
+# replaced with the name of the entity): "The $name class" "The $name widget"
+# "The $name file" "is" "provides" "specifies" "contains"
+# "represents" "a" "an" "the"
+
+ABBREVIATE_BRIEF       =
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
+# Doxygen will generate a detailed section even if there is only a brief
+# description.
+
+ALWAYS_DETAILED_SEC    = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
+# inherited members of a class in the documentation of that class as if those
+# members were ordinary class members. Constructors, destructors and assignment
+# operators of the base classes will not be shown.
+
+INLINE_INHERITED_MEMB  = NO
+
+# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full
+# path before files name in the file list and in the header files. If set
+# to NO the shortest path that makes the file name unique will be used.
+
+FULL_PATH_NAMES        = YES
+
+# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag
+# can be used to strip a user-defined part of the path. Stripping is
+# only done if one of the specified strings matches the left-hand part of
+# the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the
+# path to strip.
+
+STRIP_FROM_PATH        =
+
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of
+# the path mentioned in the documentation of a class, which tells
+# the reader which header file to include in order to use a class.
+# If left blank only the name of the header file containing the class
+# definition is used. Otherwise one should specify the include paths that
+# are normally passed to the compiler using the -I flag.
+
+STRIP_FROM_INC_PATH    =
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter
+# (but less readable) file names. This can be useful is your file systems
+# doesn't support long names like on DOS, Mac, or CD-ROM.
+
+SHORT_NAMES            = NO
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen
+# will interpret the first line (until the first dot) of a JavaDoc-style
+# comment as the brief description. If set to NO, the JavaDoc
+# comments will behave just like regular Qt-style comments
+# (thus requiring an explicit @brief command for a brief description.)
+
+JAVADOC_AUTOBRIEF      = YES
+
+# If the QT_AUTOBRIEF tag is set to YES then Doxygen will
+# interpret the first line (until the first dot) of a Qt-style
+# comment as the brief description. If set to NO, the comments
+# will behave just like regular Qt-style comments (thus requiring
+# an explicit \brief command for a brief description.)
+
+QT_AUTOBRIEF           = NO
+
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen
+# treat a multi-line C++ special comment block (i.e. a block of //! or ///
+# comments) as a brief description. This used to be the default behaviour.
+# The new default is to treat a multi-line C++ comment block as a detailed
+# description. Set this tag to YES if you prefer the old behaviour instead.
+
+MULTILINE_CPP_IS_BRIEF = NO
+
+# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented
+# member inherits the documentation from any documented member that it
+# re-implements.
+
+INHERIT_DOCS           = YES
+
+# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce
+# a new page for each member. If set to NO, the documentation of a member will
+# be part of the file/class/namespace that contains it.
+
+SEPARATE_MEMBER_PAGES  = NO
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab.
+# Doxygen uses this value to replace tabs by spaces in code fragments.
+
+TAB_SIZE               = 8
+
+# This tag can be used to specify a number of aliases that acts
+# as commands in the documentation. An alias has the form "name=value".
+# For example adding "sideeffect=\par Side Effects:\n" will allow you to
+# put the command \sideeffect (or @sideeffect) in the documentation, which
+# will result in a user-defined paragraph with heading "Side Effects:".
+# You can put \n's in the value part of an alias to insert newlines.
+
+ALIASES                =
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C
+# sources only. Doxygen will then generate output that is more tailored for C.
+# For instance, some of the names that are used will be different. The list
+# of all members will be omitted, etc.
+
+OPTIMIZE_OUTPUT_FOR_C  = NO
+
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java
+# sources only. Doxygen will then generate output that is more tailored for
+# Java. For instance, namespaces will be presented as packages, qualified
+# scopes will look different, etc.
+
+OPTIMIZE_OUTPUT_JAVA   = NO
+
+# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
+# sources only. Doxygen will then generate output that is more tailored for
+# Fortran.
+
+OPTIMIZE_FOR_FORTRAN   = NO
+
+# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
+# sources. Doxygen will then generate output that is tailored for
+# VHDL.
+
+OPTIMIZE_OUTPUT_VHDL   = NO
+
+# Doxygen selects the parser to use depending on the extension of the files it parses.
+# With this tag you can assign which parser to use for a given extension.
+# Doxygen has a built-in mapping, but you can override or extend it using this tag.
+# The format is ext=language, where ext is a file extension, and language is one of
+# the parsers supported by doxygen: IDL, Java, Javascript, C#, C, C++, D, PHP,
+# Objective-C, Python, Fortran, VHDL, C, C++. For instance to make doxygen treat
+# .inc files as Fortran files (default is PHP), and .f files as C (default is Fortran),
+# use: inc=Fortran f=C. Note that for custom extensions you also need to set FILE_PATTERNS otherwise the files are not read by doxygen.
+
+EXTENSION_MAPPING      =
+
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
+# to include (a tag file for) the STL sources as input, then you should
+# set this tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string); v.s.
+# func(std::string) {}). This also make the inheritance and collaboration
+# diagrams that involve STL classes more complete and accurate.
+
+BUILTIN_STL_SUPPORT    = NO
+
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+
+CPP_CLI_SUPPORT        = NO
+
+# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only.
+# Doxygen will parse them like normal C++ but will assume all classes use public
+# instead of private inheritance when no explicit protection keyword is present.
+
+SIP_SUPPORT            = NO
+
+# For Microsoft's IDL there are propget and propput attributes to indicate getter
+# and setter methods for a property. Setting this option to YES (the default)
+# will make doxygen to replace the get and set methods by a property in the
+# documentation. This will only work if the methods are indeed getting or
+# setting a simple type. If this is not the case, or you want to show the
+# methods anyway, you should set this option to NO.
+
+IDL_PROPERTY_SUPPORT   = YES
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
+# tag is set to YES, then doxygen will reuse the documentation of the first
+# member in the group (if any) for the other members of the group. By default
+# all members of a group must be documented explicitly.
+
+DISTRIBUTE_GROUP_DOC   = NO
+
+# Set the SUBGROUPING tag to YES (the default) to allow class member groups of
+# the same type (for instance a group of public functions) to be put as a
+# subgroup of that type (e.g. under the Public Functions section). Set it to
+# NO to prevent subgrouping. Alternatively, this can be done per class using
+# the \nosubgrouping command.
+
+SUBGROUPING            = YES
+
+# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum
+# is documented as struct, union, or enum with the name of the typedef. So
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
+# with name TypeT. When disabled the typedef will appear as a member of a file,
+# namespace, or class. And the struct will be named TypeS. This can typically
+# be useful for C code in case the coding convention dictates that all compound
+# types are typedef'ed and only the typedef is referenced, never the tag name.
+
+TYPEDEF_HIDES_STRUCT   = NO
+
+# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to
+# determine which symbols to keep in memory and which to flush to disk.
+# When the cache is full, less often used symbols will be written to disk.
+# For small to medium size projects (<1000 input files) the default value is
+# probably good enough. For larger projects a too small cache size can cause
+# doxygen to be busy swapping symbols to and from disk most of the time
+# causing a significant performance penality.
+# If the system has enough physical memory increasing the cache will improve the
+# performance by keeping more symbols in memory. Note that the value works on
+# a logarithmic scale so increasing the size by one will rougly double the
+# memory usage. The cache size is given by this formula:
+# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0,
+# corresponding to a cache size of 2^16 = 65536 symbols
+
+SYMBOL_CACHE_SIZE      = 0
+
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+
+# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in
+# documentation are documented, even if no documentation was available.
+# Private class members and static file members will be hidden unless
+# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES
+
+EXTRACT_ALL            = NO
+
+# If the EXTRACT_PRIVATE tag is set to YES all private members of a class
+# will be included in the documentation.
+
+EXTRACT_PRIVATE        = NO
+
+# If the EXTRACT_STATIC tag is set to YES all static members of a file
+# will be included in the documentation.
+
+EXTRACT_STATIC         = NO
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs)
+# defined locally in source files will be included in the documentation.
+# If set to NO only classes defined in header files are included.
+
+EXTRACT_LOCAL_CLASSES  = YES
+
+# This flag is only useful for Objective-C code. When set to YES local
+# methods, which are defined in the implementation section but not in
+# the interface are included in the documentation.
+# If set to NO (the default) only methods in the interface are included.
+
+EXTRACT_LOCAL_METHODS  = NO
+
+# If this flag is set to YES, the members of anonymous namespaces will be
+# extracted and appear in the documentation as a namespace called
+# 'anonymous_namespace{file}', where file will be replaced with the base
+# name of the file that contains the anonymous namespace. By default
+# anonymous namespace are hidden.
+
+EXTRACT_ANON_NSPACES   = NO
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all
+# undocumented members of documented classes, files or namespaces.
+# If set to NO (the default) these members will be included in the
+# various overviews, but no documentation section is generated.
+# This option has no effect if EXTRACT_ALL is enabled.
+
+HIDE_UNDOC_MEMBERS     = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy.
+# If set to NO (the default) these classes will be included in the various
+# overviews. This option has no effect if EXTRACT_ALL is enabled.
+
+HIDE_UNDOC_CLASSES     = NO
+
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all
+# friend (class|struct|union) declarations.
+# If set to NO (the default) these declarations will be included in the
+# documentation.
+
+HIDE_FRIEND_COMPOUNDS  = NO
+
+# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any
+# documentation blocks found inside the body of a function.
+# If set to NO (the default) these blocks will be appended to the
+# function's detailed documentation block.
+
+HIDE_IN_BODY_DOCS      = NO
+
+# The INTERNAL_DOCS tag determines if documentation
+# that is typed after a \internal command is included. If the tag is set
+# to NO (the default) then the documentation will be excluded.
+# Set it to YES to include the internal documentation.
+
+INTERNAL_DOCS          = NO
+
+# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate
+# file names in lower-case letters. If set to YES upper-case letters are also
+# allowed. This is useful if you have classes or files whose names only differ
+# in case and if your file system supports case sensitive file names. Windows
+# and Mac users are advised to set this option to NO.
+
+CASE_SENSE_NAMES       = NO
+
+# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen
+# will show members with their full class and namespace scopes in the
+# documentation. If set to YES the scope will be hidden.
+
+HIDE_SCOPE_NAMES       = NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen
+# will put a list of the files that are included by a file in the documentation
+# of that file.
+
+SHOW_INCLUDE_FILES     = YES
+
+# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen
+# will list include files with double quotes in the documentation
+# rather than with sharp brackets.
+
+FORCE_LOCAL_INCLUDES   = NO
+
+# If the INLINE_INFO tag is set to YES (the default) then a tag [inline]
+# is inserted in the documentation for inline members.
+
+INLINE_INFO            = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen
+# will sort the (detailed) documentation of file and class members
+# alphabetically by member name. If set to NO the members will appear in
+# declaration order.
+
+SORT_MEMBER_DOCS       = YES
+
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the
+# brief documentation of file, namespace and class members alphabetically
+# by member name. If set to NO (the default) the members will appear in
+# declaration order.
+
+SORT_BRIEF_DOCS        = NO
+
+# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the (brief and detailed) documentation of class members so that constructors and destructors are listed first. If set to NO (the default) the constructors will appear in the respective orders defined by SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO.
+
+SORT_MEMBERS_CTORS_1ST = NO
+
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the
+# hierarchy of group names into alphabetical order. If set to NO (the default)
+# the group names will appear in their defined order.
+
+SORT_GROUP_NAMES       = NO
+
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be
+# sorted by fully-qualified names, including namespaces. If set to
+# NO (the default), the class list will be sorted only by class name,
+# not including the namespace part.
+# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
+# Note: This option applies only to the class list, not to the
+# alphabetical list.
+
+SORT_BY_SCOPE_NAME     = NO
+
+# The GENERATE_TODOLIST tag can be used to enable (YES) or
+# disable (NO) the todo list. This list is created by putting \todo
+# commands in the documentation.
+
+GENERATE_TODOLIST      = YES
+
+# The GENERATE_TESTLIST tag can be used to enable (YES) or
+# disable (NO) the test list. This list is created by putting \test
+# commands in the documentation.
+
+GENERATE_TESTLIST      = YES
+
+# The GENERATE_BUGLIST tag can be used to enable (YES) or
+# disable (NO) the bug list. This list is created by putting \bug
+# commands in the documentation.
+
+GENERATE_BUGLIST       = YES
+
+# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or
+# disable (NO) the deprecated list. This list is created by putting
+# \deprecated commands in the documentation.
+
+GENERATE_DEPRECATEDLIST= YES
+
+# The ENABLED_SECTIONS tag can be used to enable conditional
+# documentation sections, marked by \if sectionname ... \endif.
+
+ENABLED_SECTIONS       =
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines
+# the initial value of a variable or define consists of for it to appear in
+# the documentation. If the initializer consists of more lines than specified
+# here it will be hidden. Use a value of 0 to hide initializers completely.
+# The appearance of the initializer of individual variables and defines in the
+# documentation can be controlled using \showinitializer or \hideinitializer
+# command in the documentation regardless of this setting.
+
+MAX_INITIALIZER_LINES  = 30
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated
+# at the bottom of the documentation of classes and structs. If set to YES the
+# list will mention the files that were used to generate the documentation.
+
+SHOW_USED_FILES        = YES
+
+# If the sources in your project are distributed over multiple directories
+# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy
+# in the documentation. The default is NO.
+
+SHOW_DIRECTORIES       = YES
+
+# Set the SHOW_FILES tag to NO to disable the generation of the Files page.
+# This will remove the Files entry from the Quick Index and from the
+# Folder Tree View (if specified). The default is YES.
+
+SHOW_FILES             = YES
+
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the
+# Namespaces page.
+# This will remove the Namespaces entry from the Quick Index
+# and from the Folder Tree View (if specified). The default is YES.
+
+SHOW_NAMESPACES        = YES
+
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that
+# doxygen should invoke to get the current version for each file (typically from
+# the version control system). Doxygen will invoke the program by executing (via
+# popen()) the command  , where  is the value of
+# the FILE_VERSION_FILTER tag, and  is the name of an input file
+# provided by doxygen. Whatever the program writes to standard output
+# is used as the file version. See the manual for examples.
+
+FILE_VERSION_FILTER    =
+
+# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed by
+# doxygen. The layout file controls the global structure of the generated output files
+# in an output format independent way. The create the layout file that represents
+# doxygen's defaults, run doxygen with the -l option. You can optionally specify a
+# file name after the option, if omitted DoxygenLayout.xml will be used as the name
+# of the layout file.
+
+LAYOUT_FILE            =
+
+#---------------------------------------------------------------------------
+# configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated
+# by doxygen. Possible values are YES and NO. If left blank NO is used.
+
+QUIET                  = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are
+# generated by doxygen. Possible values are YES and NO. If left blank
+# NO is used.
+
+WARNINGS               = YES
+
+# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings
+# for undocumented members. If EXTRACT_ALL is set to YES then this flag will
+# automatically be disabled.
+
+WARN_IF_UNDOCUMENTED   = YES
+
+# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some
+# parameters in a documented function, or documenting parameters that
+# don't exist or using markup commands wrongly.
+
+WARN_IF_DOC_ERROR      = YES
+
+# This WARN_NO_PARAMDOC option can be abled to get warnings for
+# functions that are documented, but have no documentation for their parameters
+# or return value. If set to NO (the default) doxygen will only warn about
+# wrong or incomplete parameter documentation, but not about the absence of
+# documentation.
+
+WARN_NO_PARAMDOC       = NO
+
+# The WARN_FORMAT tag determines the format of the warning messages that
+# doxygen can produce. The string should contain the $file, $line, and $text
+# tags, which will be replaced by the file and line number from which the
+# warning originated and the warning text. Optionally the format may contain
+# $version, which will be replaced by the version of the file (if it could
+# be obtained via FILE_VERSION_FILTER)
+
+WARN_FORMAT            = "$file:$line: $text"
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning
+# and error messages should be written. If left blank the output is written
+# to stderr.
+
+WARN_LOGFILE           =
+
+#---------------------------------------------------------------------------
+# configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag can be used to specify the files and/or directories that contain
+# documented source files. You may enter file names like "myfile.cpp" or
+# directories like "/usr/src/myproject". Separate the files or directories
+# with spaces.
+
+INPUT                  =
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is
+# also the default input encoding. Doxygen uses libiconv (or the iconv built
+# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for
+# the list of possible encodings.
+
+INPUT_ENCODING         = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp
+# and *.h) to filter out the source-files in the directories. If left
+# blank the following patterns are tested:
+# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx
+# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90
+
+FILE_PATTERNS          = *.hpp *.h
+
+# The RECURSIVE tag can be used to turn specify whether or not subdirectories
+# should be searched for input files as well. Possible values are YES and NO.
+# If left blank NO is used.
+
+#RECURSIVE              = NO
+RECURSIVE              = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+
+EXCLUDE                =
+
+# The EXCLUDE_SYMLINKS tag can be used select whether or not files or
+# directories that are symbolic links (a Unix filesystem feature) are excluded
+# from the input.
+
+EXCLUDE_SYMLINKS       = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories. Note that the wildcards are matched
+# against the file with absolute path, so to exclude all test directories
+# for example use the pattern */test/*
+
+EXCLUDE_PATTERNS       =
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+
+EXCLUDE_SYMBOLS        =
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or
+# directories that contain example code fragments that are included (see
+# the \include command).
+
+EXAMPLE_PATH           =
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp
+# and *.h) to filter out the source-files in the directories. If left
+# blank all files are included.
+
+EXAMPLE_PATTERNS       =
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
+# searched for input files to be used with the \include or \dontinclude
+# commands irrespective of the value of the RECURSIVE tag.
+# Possible values are YES and NO. If left blank NO is used.
+
+EXAMPLE_RECURSIVE      = NO
+
+# The IMAGE_PATH tag can be used to specify one or more files or
+# directories that contain image that are included in the documentation (see
+# the \image command).
+
+IMAGE_PATH             =
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should
+# invoke to filter for each input file. Doxygen will invoke the filter program
+# by executing (via popen()) the command  , where 
+# is the value of the INPUT_FILTER tag, and  is the name of an
+# input file. Doxygen will then use the output that the filter program writes
+# to standard output.
+# If FILTER_PATTERNS is specified, this tag will be
+# ignored.
+
+INPUT_FILTER           =
+
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
+# basis.
+# Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match.
+# The filters are a list of the form:
+# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further
+# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER
+# is applied to all files.
+
+FILTER_PATTERNS        =
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
+# INPUT_FILTER) will be used to filter the input files when producing source
+# files to browse (i.e. when SOURCE_BROWSER is set to YES).
+
+FILTER_SOURCE_FILES    = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will
+# be generated. Documented entities will be cross-referenced with these sources.
+# Note: To get rid of all source code in the generated output, make sure also
+# VERBATIM_HEADERS is set to NO.
+
+SOURCE_BROWSER         = NO
+
+# Setting the INLINE_SOURCES tag to YES will include the body
+# of functions and classes directly in the documentation.
+
+INLINE_SOURCES         = NO
+
+# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct
+# doxygen to hide any special comment blocks from generated source code
+# fragments. Normal C and C++ comments will always remain visible.
+
+STRIP_CODE_COMMENTS    = YES
+
+# If the REFERENCED_BY_RELATION tag is set to YES
+# then for each documented function all documented
+# functions referencing it will be listed.
+
+REFERENCED_BY_RELATION = YES
+
+# If the REFERENCES_RELATION tag is set to YES
+# then for each documented function all documented entities
+# called/used by that function will be listed.
+
+REFERENCES_RELATION    = YES
+
+# If the REFERENCES_LINK_SOURCE tag is set to YES (the default)
+# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from
+# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will
+# link to the source code.
+# Otherwise they will link to the documentation.
+
+REFERENCES_LINK_SOURCE = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code
+# will point to the HTML generated by the htags(1) tool instead of doxygen
+# built-in source browser. The htags tool is part of GNU's global source
+# tagging system (see http://www.gnu.org/software/global/global.html). You
+# will need version 4.8.6 or higher.
+
+USE_HTAGS              = NO
+
+# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen
+# will generate a verbatim copy of the header file for each class for
+# which an include is specified. Set to NO to disable this.
+
+VERBATIM_HEADERS       = YES
+
+#---------------------------------------------------------------------------
+# configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index
+# of all compounds will be generated. Enable this if the project
+# contains a lot of classes, structs, unions or interfaces.
+
+ALPHABETICAL_INDEX     = NO
+
+# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then
+# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns
+# in which this list will be split (can be a number in the range [1..20])
+
+COLS_IN_ALPHA_INDEX    = 5
+
+# In case all classes in a project start with a common prefix, all
+# classes will be put under the same header in the alphabetical index.
+# The IGNORE_PREFIX tag can be used to specify one or more prefixes that
+# should be ignored while generating the index headers.
+
+IGNORE_PREFIX          =
+
+#---------------------------------------------------------------------------
+# configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES (the default) Doxygen will
+# generate HTML output.
+
+GENERATE_HTML          = YES
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be
+# put in front of it. If left blank `html' will be used as the default path.
+
+HTML_OUTPUT            = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for
+# each generated HTML page (for example: .htm,.php,.asp). If it is left blank
+# doxygen will generate files with .html extension.
+
+HTML_FILE_EXTENSION    = .html
+
+# The HTML_HEADER tag can be used to specify a personal HTML header for
+# each generated HTML page. If it is left blank doxygen will generate a
+# standard header.
+
+HTML_HEADER            =
+
+# The HTML_FOOTER tag can be used to specify a personal HTML footer for
+# each generated HTML page. If it is left blank doxygen will generate a
+# standard footer.
+
+HTML_FOOTER            =
+
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading
+# style sheet that is used by each HTML page. It can be used to
+# fine-tune the look of the HTML output. If the tag is left blank doxygen
+# will generate a default style sheet. Note that doxygen will try to copy
+# the style sheet file to the HTML output directory, so don't put your own
+# stylesheet in the HTML output directory as well, or it will be erased!
+
+HTML_STYLESHEET        =
+
+# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
+# page will contain the date and time when the page was generated. Setting
+# this to NO can help when comparing the output of multiple runs.
+
+HTML_TIMESTAMP         = NO
+
+# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes,
+# files or namespaces will be aligned in HTML using tables. If set to
+# NO a bullet list will be used.
+
+HTML_ALIGN_MEMBERS     = YES
+
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded. For this to work a browser that supports
+# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox
+# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari).
+
+HTML_DYNAMIC_SECTIONS  = NO
+
+# If the GENERATE_DOCSET tag is set to YES, additional index files
+# will be generated that can be used as input for Apple's Xcode 3
+# integrated development environment, introduced with OSX 10.5 (Leopard).
+# To create a documentation set, doxygen will generate a Makefile in the
+# HTML output directory. Running make will produce the docset in that
+# directory and running "make install" will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find
+# it at startup.
+# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html for more information.
+
+GENERATE_DOCSET        = NO
+
+# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the
+# feed. A documentation feed provides an umbrella under which multiple
+# documentation sets from a single provider (such as a company or product suite)
+# can be grouped.
+
+DOCSET_FEEDNAME        = "Doxygen generated docs"
+
+# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that
+# should uniquely identify the documentation set bundle. This should be a
+# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen
+# will append .docset to the name.
+
+DOCSET_BUNDLE_ID       = org.doxygen.Project
+
+# If the GENERATE_HTMLHELP tag is set to YES, additional index files
+# will be generated that can be used as input for tools like the
+# Microsoft HTML help workshop to generate a compiled HTML help file (.chm)
+# of the generated HTML documentation.
+
+GENERATE_HTMLHELP      = NO
+
+# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can
+# be used to specify the file name of the resulting .chm file. You
+# can add a path in front of the file if the result should not be
+# written to the html output directory.
+
+CHM_FILE               =
+
+# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can
+# be used to specify the location (absolute path including file name) of
+# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run
+# the HTML help compiler on the generated index.hhp.
+
+HHC_LOCATION           =
+
+# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag
+# controls if a separate .chi index file is generated (YES) or that
+# it should be included in the master .chm file (NO).
+
+GENERATE_CHI           = NO
+
+# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING
+# is used to encode HtmlHelp index (hhk), content (hhc) and project file
+# content.
+
+CHM_INDEX_ENCODING     =
+
+# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag
+# controls whether a binary table of contents is generated (YES) or a
+# normal table of contents (NO) in the .chm file.
+
+BINARY_TOC             = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members
+# to the contents of the HTML help documentation and to the tree view.
+
+TOC_EXPAND             = NO
+
+# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and QHP_VIRTUAL_FOLDER
+# are set, an additional index file will be generated that can be used as input for
+# Qt's qhelpgenerator to generate a Qt Compressed Help (.qch) of the generated
+# HTML documentation.
+
+GENERATE_QHP           = NO
+
+# If the QHG_LOCATION tag is specified, the QCH_FILE tag can
+# be used to specify the file name of the resulting .qch file.
+# The path specified is relative to the HTML output folder.
+
+QCH_FILE               =
+
+# The QHP_NAMESPACE tag specifies the namespace to use when generating
+# Qt Help Project output. For more information please see
+# http://doc.trolltech.com/qthelpproject.html#namespace
+
+QHP_NAMESPACE          = org.doxygen.Project
+
+# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating
+# Qt Help Project output. For more information please see
+# http://doc.trolltech.com/qthelpproject.html#virtual-folders
+
+QHP_VIRTUAL_FOLDER     = doc
+
+# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to add.
+# For more information please see
+# http://doc.trolltech.com/qthelpproject.html#custom-filters
+
+QHP_CUST_FILTER_NAME   =
+
+# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the custom filter to add.For more information please see
+# Qt Help Project / Custom Filters.
+
+QHP_CUST_FILTER_ATTRS  =
+
+# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this project's
+# filter section matches.
+# Qt Help Project / Filter Attributes.
+
+QHP_SECT_FILTER_ATTRS  =
+
+# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can
+# be used to specify the location of Qt's qhelpgenerator.
+# If non-empty doxygen will try to run qhelpgenerator on the generated
+# .qhp file.
+
+QHG_LOCATION           =
+
+# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files
+#  will be generated, which together with the HTML files, form an Eclipse help
+#  plugin. To install this plugin and make it available under the help contents
+# menu in Eclipse, the contents of the directory containing the HTML and XML
+# files needs to be copied into the plugins directory of eclipse. The name of
+# the directory within the plugins directory should be the same as
+# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before the help appears.
+
+GENERATE_ECLIPSEHELP   = NO
+
+# A unique identifier for the eclipse help plugin. When installing the plugin
+# the directory name containing the HTML and XML files should also have
+# this name.
+
+ECLIPSE_DOC_ID         = org.doxygen.Project
+
+# The DISABLE_INDEX tag can be used to turn on/off the condensed index at
+# top of each HTML page. The value NO (the default) enables the index and
+# the value YES disables it.
+
+DISABLE_INDEX          = NO
+
+# This tag can be used to set the number of enum values (range [1..20])
+# that doxygen will group on one line in the generated HTML documentation.
+
+ENUM_VALUES_PER_LINE   = 4
+
+# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
+# structure should be generated to display hierarchical information.
+# If the tag value is set to YES, a side panel will be generated
+# containing a tree-like index structure (just like the one that
+# is generated for HTML Help). For this to work a browser that supports
+# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser).
+# Windows users are probably better off using the HTML help feature.
+
+GENERATE_TREEVIEW      = NO
+
+# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories,
+# and Class Hierarchy pages using a tree view instead of an ordered list.
+
+USE_INLINE_TREES       = NO
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be
+# used to set the initial width (in pixels) of the frame in which the tree
+# is shown.
+
+TREEVIEW_WIDTH         = 250
+
+# Use this tag to change the font size of Latex formulas included
+# as images in the HTML documentation. The default is 10. Note that
+# when you change the font size after a successful doxygen run you need
+# to manually remove any form_*.png images from the HTML output directory
+# to force them to be regenerated.
+
+FORMULA_FONTSIZE       = 10
+
+# When the SEARCHENGINE tag is enabled doxygen will generate a search box for the HTML output. The underlying search engine uses javascript
+# and DHTML and should work on any modern browser. Note that when using HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) there is already a search function so this one should
+# typically be disabled. For large projects the javascript based search engine
+# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution.
+
+SEARCHENGINE           = YES
+
+# When the SERVER_BASED_SEARCH tag is enabled the search engine will be implemented using a PHP enabled web server instead of at the web client using Javascript. Doxygen will generate the search PHP script and index
+# file to put on the web server. The advantage of the server based approach is that it scales better to large projects and allows full text search. The disadvances is that it is more difficult to setup
+# and does not have live searching capabilities.
+
+SERVER_BASED_SEARCH    = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will
+# generate Latex output.
+
+GENERATE_LATEX         = YES
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be
+# put in front of it. If left blank `latex' will be used as the default path.
+
+LATEX_OUTPUT           = latex
+
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
+# invoked. If left blank `latex' will be used as the default command name.
+# Note that when enabling USE_PDFLATEX this option is only used for
+# generating bitmaps for formulas in the HTML output, but not in the
+# Makefile that is written to the output directory.
+
+LATEX_CMD_NAME         = latex
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to
+# generate index for LaTeX. If left blank `makeindex' will be used as the
+# default command name.
+
+MAKEINDEX_CMD_NAME     = makeindex
+
+# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact
+# LaTeX documents. This may be useful for small projects and may help to
+# save some trees in general.
+
+COMPACT_LATEX          = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used
+# by the printer. Possible values are: a4, a4wide, letter, legal and
+# executive. If left blank a4wide will be used.
+
+PAPER_TYPE             = a4wide
+
+# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX
+# packages that should be included in the LaTeX output.
+
+EXTRA_PACKAGES         =
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for
+# the generated latex document. The header should contain everything until
+# the first chapter. If it is left blank doxygen will generate a
+# standard header. Notice: only use this tag if you know what you are doing!
+
+LATEX_HEADER           =
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated
+# is prepared for conversion to pdf (using ps2pdf). The pdf file will
+# contain links (just like the HTML output) instead of page references
+# This makes the output suitable for online browsing using a pdf viewer.
+
+PDF_HYPERLINKS         = YES
+
+# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of
+# plain latex in the generated Makefile. Set this option to YES to get a
+# higher quality PDF documentation.
+
+USE_PDFLATEX           = YES
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode.
+# command to the generated LaTeX files. This will instruct LaTeX to keep
+# running if errors occur, instead of asking the user for help.
+# This option is also used when generating formulas in HTML.
+
+LATEX_BATCHMODE        = NO
+
+# If LATEX_HIDE_INDICES is set to YES then doxygen will not
+# include the index chapters (such as File Index, Compound Index, etc.)
+# in the output.
+
+LATEX_HIDE_INDICES     = NO
+
+# If LATEX_SOURCE_CODE is set to YES then doxygen will include source code with syntax highlighting in the LaTeX output. Note that which sources are shown also depends on other settings such as SOURCE_BROWSER.
+
+LATEX_SOURCE_CODE      = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output
+# The RTF output is optimized for Word 97 and may not look very pretty with
+# other RTF readers or editors.
+
+GENERATE_RTF           = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be
+# put in front of it. If left blank `rtf' will be used as the default path.
+
+RTF_OUTPUT             = rtf
+
+# If the COMPACT_RTF tag is set to YES Doxygen generates more compact
+# RTF documents. This may be useful for small projects and may help to
+# save some trees in general.
+
+COMPACT_RTF            = NO
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated
+# will contain hyperlink fields. The RTF file will
+# contain links (just like the HTML output) instead of page references.
+# This makes the output suitable for online browsing using WORD or other
+# programs which support those fields.
+# Note: wordpad (write) and others do not support links.
+
+RTF_HYPERLINKS         = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's
+# config file, i.e. a series of assignments. You only have to provide
+# replacements, missing definitions are set to their default value.
+
+RTF_STYLESHEET_FILE    =
+
+# Set optional variables used in the generation of an rtf document.
+# Syntax is similar to doxygen's config file.
+
+RTF_EXTENSIONS_FILE    =
+
+#---------------------------------------------------------------------------
+# configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES (the default) Doxygen will
+# generate man pages
+
+GENERATE_MAN           = NO
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be
+# put in front of it. If left blank `man' will be used as the default path.
+
+MAN_OUTPUT             = man
+
+# The MAN_EXTENSION tag determines the extension that is added to
+# the generated man pages (default is the subroutine's section .3)
+
+MAN_EXTENSION          = .3
+
+# If the MAN_LINKS tag is set to YES and Doxygen generates man output,
+# then it will generate one additional man file for each entity
+# documented in the real man page(s). These additional files
+# only source the real man page, but without them the man command
+# would be unable to find the correct page. The default is NO.
+
+MAN_LINKS              = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES Doxygen will
+# generate an XML file that captures the structure of
+# the code including all documentation.
+
+GENERATE_XML           = NO
+
+# The XML_OUTPUT tag is used to specify where the XML pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be
+# put in front of it. If left blank `xml' will be used as the default path.
+
+XML_OUTPUT             = xml
+
+# The XML_SCHEMA tag can be used to specify an XML schema,
+# which can be used by a validating XML parser to check the
+# syntax of the XML files.
+
+XML_SCHEMA             =
+
+# The XML_DTD tag can be used to specify an XML DTD,
+# which can be used by a validating XML parser to check the
+# syntax of the XML files.
+
+XML_DTD                =
+
+# If the XML_PROGRAMLISTING tag is set to YES Doxygen will
+# dump the program listings (including syntax highlighting
+# and cross-referencing information) to the XML output. Note that
+# enabling this will significantly increase the size of the XML output.
+
+XML_PROGRAMLISTING     = YES
+
+#---------------------------------------------------------------------------
+# configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will
+# generate an AutoGen Definitions (see autogen.sf.net) file
+# that captures the structure of the code including all
+# documentation. Note that this feature is still experimental
+# and incomplete at the moment.
+
+GENERATE_AUTOGEN_DEF   = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_PERLMOD tag is set to YES Doxygen will
+# generate a Perl module file that captures the structure of
+# the code including all documentation. Note that this
+# feature is still experimental and incomplete at the
+# moment.
+
+GENERATE_PERLMOD       = NO
+
+# If the PERLMOD_LATEX tag is set to YES Doxygen will generate
+# the necessary Makefile rules, Perl scripts and LaTeX code to be able
+# to generate PDF and DVI output from the Perl module output.
+
+PERLMOD_LATEX          = NO
+
+# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be
+# nicely formatted so it can be parsed by a human reader.
+# This is useful
+# if you want to understand what is going on.
+# On the other hand, if this
+# tag is set to NO the size of the Perl module output will be much smaller
+# and Perl will parse it just the same.
+
+PERLMOD_PRETTY         = YES
+
+# The names of the make variables in the generated doxyrules.make file
+# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX.
+# This is useful so different doxyrules.make files included by the same
+# Makefile don't overwrite each other's variables.
+
+PERLMOD_MAKEVAR_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will
+# evaluate all C-preprocessor directives found in the sources and include
+# files.
+
+ENABLE_PREPROCESSING   = YES
+
+# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro
+# names in the source code. If set to NO (the default) only conditional
+# compilation will be performed. Macro expansion can be done in a controlled
+# way by setting EXPAND_ONLY_PREDEF to YES.
+
+MACRO_EXPANSION        = NO
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES
+# then the macro expansion is limited to the macros specified with the
+# PREDEFINED and EXPAND_AS_DEFINED tags.
+
+EXPAND_ONLY_PREDEF     = NO
+
+# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files
+# in the INCLUDE_PATH (see below) will be search if a #include is found.
+
+SEARCH_INCLUDES        = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by
+# the preprocessor.
+
+INCLUDE_PATH           =
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
+# patterns (like *.h and *.hpp) to filter out the header-files in the
+# directories. If left blank, the patterns specified with FILE_PATTERNS will
+# be used.
+
+INCLUDE_FILE_PATTERNS  =
+
+# The PREDEFINED tag can be used to specify one or more macro names that
+# are defined before the preprocessor is started (similar to the -D option of
+# gcc). The argument of the tag is a list of macros of the form: name
+# or name=definition (no spaces). If the definition and the = are
+# omitted =1 is assumed. To prevent a macro definition from being
+# undefined via #undef or recursively expanded use the := operator
+# instead of the = operator.
+
+PREDEFINED             =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then
+# this tag can be used to specify a list of macro names that should be expanded.
+# The macro definition that is found in the sources will be used.
+# Use the PREDEFINED tag if you want to use a different macro definition.
+
+EXPAND_AS_DEFINED      =
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then
+# doxygen's preprocessor will remove all function-like macros that are alone
+# on a line, have an all uppercase name, and do not end with a semicolon. Such
+# function macros are typically used for boiler-plate code, and will confuse
+# the parser if not removed.
+
+SKIP_FUNCTION_MACROS   = YES
+
+#---------------------------------------------------------------------------
+# Configuration::additions related to external references
+#---------------------------------------------------------------------------
+
+# The TAGFILES option can be used to specify one or more tagfiles.
+# Optionally an initial location of the external documentation
+# can be added for each tagfile. The format of a tag file without
+# this location is as follows:
+#
+# TAGFILES = file1 file2 ...
+# Adding location for the tag files is done as follows:
+#
+# TAGFILES = file1=loc1 "file2 = loc2" ...
+# where "loc1" and "loc2" can be relative or absolute paths or
+# URLs. If a location is present for each tag, the installdox tool
+# does not have to be run to correct the links.
+# Note that each tag file must have a unique name
+# (where the name does NOT include the path)
+# If a tag file is not located in the directory in which doxygen
+# is run, you must also specify the path to the tagfile here.
+
+TAGFILES               =
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create
+# a tag file that is based on the input files it reads.
+
+GENERATE_TAGFILE       =
+
+# If the ALLEXTERNALS tag is set to YES all external classes will be listed
+# in the class index. If set to NO only the inherited external classes
+# will be listed.
+
+ALLEXTERNALS           = NO
+
+# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed
+# in the modules index. If set to NO, only the current project's groups will
+# be listed.
+
+EXTERNAL_GROUPS        = YES
+
+# The PERL_PATH should be the absolute path and name of the perl script
+# interpreter (i.e. the result of `which perl').
+
+PERL_PATH              = /usr/bin/perl
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will
+# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base
+# or super classes. Setting the tag to NO turns the diagrams off. Note that
+# this option is superseded by the HAVE_DOT option below. This is only a
+# fallback. It is recommended to install and use dot, since it yields more
+# powerful graphs.
+
+CLASS_DIAGRAMS         = YES
+
+# You can define message sequence charts within doxygen comments using the \msc
+# command. Doxygen will then run the mscgen tool (see
+# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the
+# documentation. The MSCGEN_PATH tag allows you to specify the directory where
+# the mscgen tool resides. If left empty the tool is assumed to be found in the
+# default search path.
+
+MSCGEN_PATH            =
+
+# If set to YES, the inheritance and collaboration graphs will hide
+# inheritance and usage relations if the target is undocumented
+# or is not a class.
+
+HIDE_UNDOC_RELATIONS   = YES
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
+# available from the path. This tool is part of Graphviz, a graph visualization
+# toolkit from AT&T and Lucent Bell Labs. The other options in this section
+# have no effect if this option is set to NO (the default)
+
+HAVE_DOT               = NO
+
+# By default doxygen will write a font called FreeSans.ttf to the output
+# directory and reference it in all dot files that doxygen generates. This
+# font does not include all possible unicode characters however, so when you need
+# these (or just want a differently looking font) you can specify the font name
+# using DOT_FONTNAME. You need need to make sure dot is able to find the font,
+# which can be done by putting it in a standard location or by setting the
+# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory
+# containing the font.
+
+DOT_FONTNAME           = FreeSans
+
+# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs.
+# The default size is 10pt.
+
+DOT_FONTSIZE           = 10
+
+# By default doxygen will tell dot to use the output directory to look for the
+# FreeSans.ttf font (which doxygen will put there itself). If you specify a
+# different font using DOT_FONTNAME you can set the path where dot
+# can find it using this tag.
+
+DOT_FONTPATH           =
+
+# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen
+# will generate a graph for each documented class showing the direct and
+# indirect inheritance relations. Setting this tag to YES will force the
+# the CLASS_DIAGRAMS tag to NO.
+
+CLASS_GRAPH            = YES
+
+# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen
+# will generate a graph for each documented class showing the direct and
+# indirect implementation dependencies (inheritance, containment, and
+# class references variables) of the class with other documented classes.
+
+COLLABORATION_GRAPH    = YES
+
+# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen
+# will generate a graph for groups, showing the direct groups dependencies
+
+GROUP_GRAPHS           = YES
+
+# If the UML_LOOK tag is set to YES doxygen will generate inheritance and
+# collaboration diagrams in a style similar to the OMG's Unified Modeling
+# Language.
+
+UML_LOOK               = NO
+
+# If set to YES, the inheritance and collaboration graphs will show the
+# relations between templates and their instances.
+
+TEMPLATE_RELATIONS     = NO
+
+# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT
+# tags are set to YES then doxygen will generate a graph for each documented
+# file showing the direct and indirect include dependencies of the file with
+# other documented files.
+
+INCLUDE_GRAPH          = YES
+
+# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and
+# HAVE_DOT tags are set to YES then doxygen will generate a graph for each
+# documented header file showing the documented files that directly or
+# indirectly include this file.
+
+INCLUDED_BY_GRAPH      = YES
+
+# If the CALL_GRAPH and HAVE_DOT options are set to YES then
+# doxygen will generate a call dependency graph for every global function
+# or class method. Note that enabling this option will significantly increase
+# the time of a run. So in most cases it will be better to enable call graphs
+# for selected functions only using the \callgraph command.
+
+CALL_GRAPH             = NO
+
+# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then
+# doxygen will generate a caller dependency graph for every global function
+# or class method. Note that enabling this option will significantly increase
+# the time of a run. So in most cases it will be better to enable caller
+# graphs for selected functions only using the \callergraph command.
+
+CALLER_GRAPH           = NO
+
+# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen
+# will graphical hierarchy of all classes instead of a textual one.
+
+GRAPHICAL_HIERARCHY    = YES
+
+# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES
+# then doxygen will show the dependencies a directory has on other directories
+# in a graphical way. The dependency relations are determined by the #include
+# relations between the files in the directories.
+
+DIRECTORY_GRAPH        = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
+# generated by dot. Possible values are png, jpg, or gif
+# If left blank png will be used.
+
+DOT_IMAGE_FORMAT       = png
+
+# The tag DOT_PATH can be used to specify the path where the dot tool can be
+# found. If left blank, it is assumed the dot tool can be found in the path.
+
+DOT_PATH               =
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that
+# contain dot files that are included in the documentation (see the
+# \dotfile command).
+
+DOTFILE_DIRS           =
+
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of
+# nodes that will be shown in the graph. If the number of nodes in a graph
+# becomes larger than this value, doxygen will truncate the graph, which is
+# visualized by representing a node as a red box. Note that doxygen if the
+# number of direct children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note
+# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+
+DOT_GRAPH_MAX_NODES    = 50
+
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the
+# graphs generated by dot. A depth value of 3 means that only nodes reachable
+# from the root by following a path via at most 3 edges will be shown. Nodes
+# that lay further from the root node will be omitted. Note that setting this
+# option to 1 or 2 may greatly reduce the computation time needed for large
+# code bases. Also note that the size of a graph can be further restricted by
+# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+
+MAX_DOT_GRAPH_DEPTH    = 0
+
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
+# background. This is disabled by default, because dot on Windows does not
+# seem to support this out of the box. Warning: Depending on the platform used,
+# enabling this option may lead to badly anti-aliased labels on the edges of
+# a graph (i.e. they become hard to read).
+
+DOT_TRANSPARENT        = NO
+
+# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output
+# files in one run (i.e. multiple -o and -T options on the command line). This
+# makes dot run faster, but since only newer versions of dot (>1.8.10)
+# support this, this feature is disabled by default.
+
+DOT_MULTI_TARGETS      = NO
+
+# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will
+# generate a legend page explaining the meaning of the various boxes and
+# arrows in the dot generated graphs.
+
+GENERATE_LEGEND        = YES
+
+# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will
+# remove the intermediate dot files that are used to generate
+# the various graphs.
+
+DOT_CLEANUP            = YES
diff --git a/cpp/Makefile.am b/cpp/Makefile.am
index 6b37803..871ff8c 100644
--- a/cpp/Makefile.am
+++ b/cpp/Makefile.am
@@ -11,3 +11,9 @@ DOC_FILES = \
 EXTRA_DIST = \
 		$(DOC_FILES)
 
+doxygen:
+	./preprocess
+	./preprocess clean
+	cd src && $(MAKE) doxygen
+	./preprocess
+
diff --git a/cpp/preprocess b/cpp/preprocess
index 1a5e6a3..1c169e9 100755
--- a/cpp/preprocess
+++ b/cpp/preprocess
@@ -6,6 +6,7 @@ preprocess() {
 		echo ""
 		echo "** preprocess failed **"
 		echo ""
+		exit 1
 	else
 		mv $1.tmp $1
 	fi
diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am
index 4cfa87e..6b6457e 100644
--- a/cpp/src/Makefile.am
+++ b/cpp/src/Makefile.am
@@ -73,3 +73,20 @@ EXTRA_DIST = \
 		msgpack/type/define.hpp.erb \
 		msgpack/type/tuple.hpp.erb
 
+
+doxygen_c:
+	cat ../Doxyfile > Doxyfile_c
+	echo "FILE_PATTERNS      = *.h" >> Doxyfile_c
+	echo "OUTPUT_DIRECTORY   = doc_c" >> Doxyfile_c
+	echo "PROJECT_NAME       = \"MessagePack for C\"" >> Doxyfile_c
+	doxygen Doxyfile_c
+
+doxygen_cpp:
+	cat ../Doxyfile > Doxyfile_cpp
+	echo "FILE_PATTERNS      = *.hpp" >> Doxyfile_cpp
+	echo "OUTPUT_DIRECTORY   = doc_cpp" >> Doxyfile_cpp
+	echo "PROJECT_NAME       = \"MessagePack for C++\"" >> Doxyfile_cpp
+	doxygen Doxyfile_cpp
+
+doxygen: doxygen_c doxygen_cpp
+
diff --git a/cpp/src/msgpack.h b/cpp/src/msgpack.h
index 21729f4..0cd8a19 100644
--- a/cpp/src/msgpack.h
+++ b/cpp/src/msgpack.h
@@ -15,6 +15,11 @@
  *    See the License for the specific language governing permissions and
  *    limitations under the License.
  */
+/**
+ * @defgroup msgpack MessagePack C
+ * @{
+ * @}
+ */
 #include "msgpack/object.h"
 #include "msgpack/zone.h"
 #include "msgpack/pack.h"
diff --git a/cpp/src/msgpack/object.h b/cpp/src/msgpack/object.h
index 71a27bb..cf0b4c1 100644
--- a/cpp/src/msgpack/object.h
+++ b/cpp/src/msgpack/object.h
@@ -26,6 +26,12 @@ extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_object Dynamically typed object
+ * @ingroup msgpack
+ * @{
+ */
+
 typedef enum {
 	MSGPACK_OBJECT_NIL					= 0x00,
 	MSGPACK_OBJECT_BOOLEAN				= 0x01,
@@ -81,6 +87,8 @@ void msgpack_object_print(FILE* out, msgpack_object o);
 
 bool msgpack_object_equal(const msgpack_object x, const msgpack_object y);
 
+/** @} */
+
 
 #ifdef __cplusplus
 }
diff --git a/cpp/src/msgpack/pack.h b/cpp/src/msgpack/pack.h
index 1525e0f..1252895 100644
--- a/cpp/src/msgpack/pack.h
+++ b/cpp/src/msgpack/pack.h
@@ -27,6 +27,19 @@ extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_buffer Buffers
+ * @ingroup msgpack
+ * @{
+ * @}
+ */
+
+/**
+ * @defgroup msgpack_pack Serializer
+ * @ingroup msgpack
+ * @{
+ */
+
 typedef int (*msgpack_packer_write)(void* data, const char* buf, unsigned int len);
 
 typedef struct msgpack_packer {
@@ -74,6 +87,8 @@ static int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
 int msgpack_pack_object(msgpack_packer* pk, msgpack_object d);
 
 
+/** @} */
+
 
 #define msgpack_pack_inline_func(name) \
 	inline int msgpack_pack ## name
diff --git a/cpp/src/msgpack/sbuffer.h b/cpp/src/msgpack/sbuffer.h
index caff2e8..778dea7 100644
--- a/cpp/src/msgpack/sbuffer.h
+++ b/cpp/src/msgpack/sbuffer.h
@@ -21,15 +21,17 @@
 #include 
 #include 
 
-#ifndef MSGPACK_SBUFFER_INIT_SIZE
-#define MSGPACK_SBUFFER_INIT_SIZE 8192
-#endif
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_sbuffer Simple buffer
+ * @ingroup msgpack_buffer
+ * @{
+ */
+
 typedef struct msgpack_sbuffer {
 	size_t size;
 	char* data;
@@ -58,6 +60,10 @@ static inline void msgpack_sbuffer_free(msgpack_sbuffer* sbuf)
 	free(sbuf);
 }
 
+#ifndef MSGPACK_SBUFFER_INIT_SIZE
+#define MSGPACK_SBUFFER_INIT_SIZE 8192
+#endif
+
 static inline int msgpack_sbuffer_write(void* data, const char* buf, unsigned int len)
 {
 	msgpack_sbuffer* sbuf = (msgpack_sbuffer*)data;
@@ -94,6 +100,9 @@ static inline void msgpack_sbuffer_clear(msgpack_sbuffer* sbuf)
 	sbuf->size = 0;
 }
 
+/** @} */
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/cpp/src/msgpack/unpack.h b/cpp/src/msgpack/unpack.h
index 1f43ab7..82698fc 100644
--- a/cpp/src/msgpack/unpack.h
+++ b/cpp/src/msgpack/unpack.h
@@ -22,19 +22,34 @@
 #include "msgpack/object.h"
 #include 
 
-#ifndef MSGPACK_UNPACKER_INIT_BUFFER_SIZE
-#define MSGPACK_UNPACKER_INIT_BUFFER_SIZE (64*1024)
-#endif
-
-#ifndef MSGPACK_UNPACKER_RESERVE_SIZE
-#define MSGPACK_UNPACKER_RESERVE_SIZE (32*1024)
-#endif
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_unpack Deserializer
+ * @ingroup msgpack
+ * @{
+ */
+
+typedef struct msgpack_unpacked {
+	msgpack_zone* zone;
+	msgpack_object data;
+} msgpack_unpacked;
+
+bool msgpack_unpack_next(msgpack_unpacked* result,
+		const char* data, size_t len, size_t* off);
+
+/** @} */
+
+
+/**
+ * @defgroup msgpack_unpacker Streaming deserializer
+ * @ingroup msgpack
+ * @{
+ */
+
 typedef struct msgpack_unpacker {
 	char* buffer;
 	size_t used;
@@ -46,27 +61,100 @@ typedef struct msgpack_unpacker {
 	void* ctx;
 } msgpack_unpacker;
 
-typedef struct msgpack_unpacked {
-	msgpack_zone* zone;
-	msgpack_object data;
-} msgpack_unpacked;
 
+#ifndef MSGPACK_UNPACKER_INIT_BUFFER_SIZE
+#define MSGPACK_UNPACKER_INIT_BUFFER_SIZE (64*1024)
+#endif
+
+/**
+ * Initializes a streaming deserializer.
+ * The initialized deserializer must be destroyed by msgpack_unpacker_destroy(msgpack_unpacker*).
+ */
 bool msgpack_unpacker_init(msgpack_unpacker* mpac, size_t initial_buffer_size);
+
+/**
+ * Destroys a streaming deserializer initialized by msgpack_unpacker_init(msgpack_unpacker*, size_t).
+ */
 void msgpack_unpacker_destroy(msgpack_unpacker* mpac);
 
+
+/**
+ * Creates a streaming deserializer.
+ * The created deserializer must be destroyed by msgpack_unpacker_free(msgpack_unpacker*).
+ */
 msgpack_unpacker* msgpack_unpacker_new(size_t initial_buffer_size);
+
+/**
+ * Frees a streaming deserializer created by msgpack_unpacker_new(size_t).
+ */
 void msgpack_unpacker_free(msgpack_unpacker* mpac);
 
+
+#ifndef MSGPACK_UNPACKER_RESERVE_SIZE
+#define MSGPACK_UNPACKER_RESERVE_SIZE (32*1024)
+#endif
+
+/**
+ * Reserves free space of the internal buffer.
+ * Use this function to fill the internal buffer with
+ * msgpack_unpacker_buffer(msgpack_unpacker*),
+ * msgpack_unpacker_buffer_capacity(const msgpack_unpacker*) and
+ * msgpack_unpacker_buffer_consumed(msgpack_unpacker*).
+ */
 static inline bool   msgpack_unpacker_reserve_buffer(msgpack_unpacker* mpac, size_t size);
+
+/**
+ * Gets pointer to the free space of the internal buffer.
+ * Use this function to fill the internal buffer with
+ * msgpack_unpacker_reserve_buffer(msgpack_unpacker*, size_t),
+ * msgpack_unpacker_buffer_capacity(const msgpack_unpacker*) and
+ * msgpack_unpacker_buffer_consumed(msgpack_unpacker*).
+ */
 static inline char*  msgpack_unpacker_buffer(msgpack_unpacker* mpac);
+
+/**
+ * Gets size of the free space of the internal buffer.
+ * Use this function to fill the internal buffer with
+ * msgpack_unpacker_reserve_buffer(msgpack_unpacker*, size_t),
+ * msgpack_unpacker_buffer(const msgpack_unpacker*) and
+ * msgpack_unpacker_buffer_consumed(msgpack_unpacker*).
+ */
 static inline size_t msgpack_unpacker_buffer_capacity(const msgpack_unpacker* mpac);
+
+/**
+ * Notifies the deserializer that the internal buffer filled.
+ * Use this function to fill the internal buffer with
+ * msgpack_unpacker_reserve_buffer(msgpack_unpacker*, size_t),
+ * msgpack_unpacker_buffer(msgpack_unpacker*) and
+ * msgpack_unpacker_buffer_capacity(const msgpack_unpacker*).
+ */
 static inline void   msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, size_t size);
 
+
+/**
+ * Deserializes one object.
+ * Returns true if it successes. Otherwise false is returned.
+ * @param pac  pointer to an initialized msgpack_unpacked object.
+ */
 bool msgpack_unpacker_next(msgpack_unpacker* mpac, msgpack_unpacked* pac);
 
+/**
+ * Initializes a msgpack_unpacked object.
+ * The initialized object must be destroyed by msgpack_unpacked_destroy(msgpack_unpacker*).
+ * Use the object with msgpack_unpacker_next(msgpack_unpacker*, msgpack_unpacked*) or
+ * msgpack_unpack_next(msgpack_unpacked*, const char*, size_t, size_t*).
+ */
 static inline void msgpack_unpacked_init(msgpack_unpacked* result);
+
+/**
+ * Destroys a streaming deserializer initialized by msgpack_unpacked().
+ */
 static inline void msgpack_unpacked_destroy(msgpack_unpacked* result);
 
+/**
+ * Releases the memory zone from msgpack_unpacked object.
+ * The released zone must be freed by msgpack_zone_free(msgpack_zone*).
+ */
 static inline msgpack_zone* msgpack_unpacked_release_zone(msgpack_unpacked* result);
 
 
@@ -83,10 +171,10 @@ void msgpack_unpacker_reset(msgpack_unpacker* mpac);
 static inline size_t msgpack_unpacker_message_size(const msgpack_unpacker* mpac);
 
 
-bool msgpack_unpack_next(msgpack_unpacked* result,
-		const char* data, size_t len, size_t* off);
+/** @} */
 
 
+// obsolete
 typedef enum {
 	MSGPACK_UNPACK_SUCCESS				=  2,
 	MSGPACK_UNPACK_EXTRA_BYTES			=  1,
@@ -94,6 +182,7 @@ typedef enum {
 	MSGPACK_UNPACK_PARSE_ERROR			= -1,
 } msgpack_unpack_return;
 
+// obsolete
 msgpack_unpack_return
 msgpack_unpack(const char* data, size_t len, size_t* off,
 		msgpack_zone* result_zone, msgpack_object* result);
diff --git a/cpp/src/msgpack/vrefbuffer.h b/cpp/src/msgpack/vrefbuffer.h
index ffb2302..123499d 100644
--- a/cpp/src/msgpack/vrefbuffer.h
+++ b/cpp/src/msgpack/vrefbuffer.h
@@ -30,19 +30,17 @@ struct iovec {
 };
 #endif
 
-#ifndef MSGPACK_VREFBUFFER_REF_SIZE
-#define MSGPACK_VREFBUFFER_REF_SIZE 32
-#endif
-
-#ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE
-#define MSGPACK_VREFBUFFER_CHUNK_SIZE 8192
-#endif
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_vrefbuffer Vectored Referencing buffer
+ * @ingroup msgpack_buffer
+ * @{
+ */
+
 struct msgpack_vrefbuffer_chunk;
 typedef struct msgpack_vrefbuffer_chunk msgpack_vrefbuffer_chunk;
 
@@ -64,6 +62,14 @@ typedef struct msgpack_vrefbuffer {
 } msgpack_vrefbuffer;
 
 
+#ifndef MSGPACK_VREFBUFFER_REF_SIZE
+#define MSGPACK_VREFBUFFER_REF_SIZE 32
+#endif
+
+#ifndef MSGPACK_VREFBUFFER_CHUNK_SIZE
+#define MSGPACK_VREFBUFFER_CHUNK_SIZE 8192
+#endif
+
 bool msgpack_vrefbuffer_init(msgpack_vrefbuffer* vbuf,
 		size_t ref_size, size_t chunk_size);
 void msgpack_vrefbuffer_destroy(msgpack_vrefbuffer* vbuf);
@@ -86,6 +92,8 @@ int msgpack_vrefbuffer_migrate(msgpack_vrefbuffer* vbuf, msgpack_vrefbuffer* to)
 
 void msgpack_vrefbuffer_clear(msgpack_vrefbuffer* vref);
 
+/** @} */
+
 
 msgpack_vrefbuffer* msgpack_vrefbuffer_new(size_t ref_size, size_t chunk_size)
 {
diff --git a/cpp/src/msgpack/zbuffer.h b/cpp/src/msgpack/zbuffer.h
index 0ff9675..abb9c50 100644
--- a/cpp/src/msgpack/zbuffer.h
+++ b/cpp/src/msgpack/zbuffer.h
@@ -23,25 +23,26 @@
 #include 
 #include 
 
-#ifndef MSGPACK_ZBUFFER_INIT_SIZE
-#define MSGPACK_ZBUFFER_INIT_SIZE 8192
-#endif
-
-#ifndef MSGPACK_ZBUFFER_RESERVE_SIZE
-#define MSGPACK_ZBUFFER_RESERVE_SIZE 512
-#endif
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_zbuffer Compressed buffer
+ * @ingroup msgpack_buffer
+ * @{
+ */
+
 typedef struct msgpack_zbuffer {
 	z_stream stream;
 	char* data;
 	size_t init_size;
 } msgpack_zbuffer;
 
+#ifndef MSGPACK_ZBUFFER_INIT_SIZE
+#define MSGPACK_ZBUFFER_INIT_SIZE 8192
+#endif
 
 static inline bool msgpack_zbuffer_init(msgpack_zbuffer* zbuf,
 		int level, size_t init_size);
@@ -60,6 +61,10 @@ static inline void msgpack_zbuffer_reset_buffer(msgpack_zbuffer* zbuf);
 static inline char* msgpack_zbuffer_release_buffer(msgpack_zbuffer* zbuf);
 
 
+#ifndef MSGPACK_ZBUFFER_RESERVE_SIZE
+#define MSGPACK_ZBUFFER_RESERVE_SIZE 512
+#endif
+
 static inline int msgpack_zbuffer_write(void* data, const char* buf, unsigned int len);
 
 static inline bool msgpack_zbuffer_expand(msgpack_zbuffer* zbuf);
@@ -191,6 +196,8 @@ char* msgpack_zbuffer_release_buffer(msgpack_zbuffer* zbuf)
 	return tmp;
 }
 
+/** @} */
+
 
 #ifdef __cplusplus
 }
diff --git a/cpp/src/msgpack/zone.h b/cpp/src/msgpack/zone.h
index ce5be6d..0e811df 100644
--- a/cpp/src/msgpack/zone.h
+++ b/cpp/src/msgpack/zone.h
@@ -25,6 +25,12 @@ extern "C" {
 #endif
 
 
+/**
+ * @defgroup msgpack_zone Memory zone
+ * @ingroup msgpack
+ * @{
+ */
+
 typedef struct msgpack_zone_finalizer {
 	void (*func)(void* data);
 	void* data;
@@ -71,6 +77,7 @@ bool msgpack_zone_is_empty(msgpack_zone* zone);
 
 void msgpack_zone_clear(msgpack_zone* zone);
 
+/** @} */
 
 
 #ifndef MSGPACK_ZONE_ALIGN

From fb3e11408c4c94d3584cb68b8dcbc5a6c16db88f Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 1 Jun 2010 15:56:29 +0900
Subject: [PATCH 0522/1648] add test/cases.json

---
 test/README.md    | 51 +++++++++++++++++++++++++++++++++++++++++++++++
 test/cases.json   |  1 +
 test/cases_gen.rb |  5 +++++
 3 files changed, 57 insertions(+)
 create mode 100644 test/README.md
 create mode 100644 test/cases.json

diff --git a/test/README.md b/test/README.md
new file mode 100644
index 0000000..9139e61
--- /dev/null
+++ b/test/README.md
@@ -0,0 +1,51 @@
+MessagePack cross-language test cases
+=====================================
+
+## cases
+
+Valid serialized data are stored in "cases.mpac" and "cases_compact.mpac".
+These files describe same objects. And "cases.json" describes an array of the described objects.
+
+Thus you can verify your implementations as comparing the objects.
+
+
+## crosslang
+
+The *crosslang* tool reads serialized data from stdin and writes re-serialize data to stdout.
+
+There are C++ and Ruby implementation of crosslang tool. You can verify your implementation
+as comparing that implementations.
+
+### C++ version
+
+    $ cd ../cpp && ./configure && make && make install
+    or
+    $ port install msgpack  # MacPorts
+    
+    $ g++ -Wall -lmsgpack crosslang.cc
+
+    Usage: ./crosslang [in-file] [out-file]
+    
+    This tool is for testing of MessagePack implementation.
+    This does following behavior:
+    
+      1. Reads objects serialized by MessagePack from  (default: stdin)
+      2. Re-serializes the objects using C++ implementation of MessagePack (Note that C++ implementation is considered valid)
+      3. Writes the re-serialized objects into  (default: stdout)
+
+### Ruby version
+
+    $ gem install msgpack
+    or
+    $ port install rb_msgpack   # MacPorts
+
+    $ ruby crosslang.rb
+    Usage: crosslang.rb [in-file] [out-file]
+    
+    This tool is for testing of MessagePack implementation.
+    This does following behavior:
+    
+      1. Reads objects serialized by MessagePack from  (default: stdin)
+      2. Re-serializes the objects using Ruby implementation of MessagePack (Note that Ruby implementation is considered valid)
+      3. Writes the re-serialized objects into  (default: stdout)
+
diff --git a/test/cases.json b/test/cases.json
new file mode 100644
index 0000000..fd390d4
--- /dev/null
+++ b/test/cases.json
@@ -0,0 +1 @@
+[false,true,null,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,127,127,255,65535,4294967295,-32,-32,-128,-32768,-2147483648,0.0,-0.0,1.0,-1.0,"a","a","a","","","",[0],[0],[0],[],[],[],{},{},{},{"a":97},{"a":97},{"a":97},[[]],[["a"]]]
\ No newline at end of file
diff --git a/test/cases_gen.rb b/test/cases_gen.rb
index 7efbfe7..b349f4e 100644
--- a/test/cases_gen.rb
+++ b/test/cases_gen.rb
@@ -3,6 +3,7 @@
 #
 require 'rubygems' rescue nil
 require 'msgpack'
+require 'json'
 
 source = <
Date: Tue, 1 Jun 2010 15:58:44 +0900
Subject: [PATCH 0523/1648] update test/README.md

---
 test/README.md | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/test/README.md b/test/README.md
index 9139e61..8027002 100644
--- a/test/README.md
+++ b/test/README.md
@@ -22,16 +22,10 @@ as comparing that implementations.
     or
     $ port install msgpack  # MacPorts
     
-    $ g++ -Wall -lmsgpack crosslang.cc
+    $ g++ -Wall -lmsgpack crosslang.cc -o crosslang
 
+    $ ./crosslang
     Usage: ./crosslang [in-file] [out-file]
-    
-    This tool is for testing of MessagePack implementation.
-    This does following behavior:
-    
-      1. Reads objects serialized by MessagePack from  (default: stdin)
-      2. Re-serializes the objects using C++ implementation of MessagePack (Note that C++ implementation is considered valid)
-      3. Writes the re-serialized objects into  (default: stdout)
 
 ### Ruby version
 
@@ -41,11 +35,4 @@ as comparing that implementations.
 
     $ ruby crosslang.rb
     Usage: crosslang.rb [in-file] [out-file]
-    
-    This tool is for testing of MessagePack implementation.
-    This does following behavior:
-    
-      1. Reads objects serialized by MessagePack from  (default: stdin)
-      2. Re-serializes the objects using Ruby implementation of MessagePack (Note that Ruby implementation is considered valid)
-      3. Writes the re-serialized objects into  (default: stdout)
 

From d4049fe593ae4465e7a258d138c2166571a0f1a7 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 1 Jun 2010 16:35:21 +0900
Subject: [PATCH 0524/1648] ruby: add test/test_cases.rb

---
 ruby/makegem.sh                               |  4 +-
 ruby/test/test_cases.rb                       | 46 +++++++++++++++++++
 ruby/test/test_helper.rb                      |  4 ++
 .../test_pack_unpack.rb}                      | 25 +++++-----
 test/cases_gen.rb                             |  5 +-
 5 files changed, 70 insertions(+), 14 deletions(-)
 create mode 100644 ruby/test/test_cases.rb
 rename ruby/{msgpack_test.rb => test/test_pack_unpack.rb} (86%)

diff --git a/ruby/makegem.sh b/ruby/makegem.sh
index 5ea66f1..827f452 100755
--- a/ruby/makegem.sh
+++ b/ruby/makegem.sh
@@ -13,7 +13,9 @@ cp ../msgpack/pack_template.h   msgpack/
 cp ../msgpack/unpack_define.h   msgpack/
 cp ../msgpack/unpack_template.h msgpack/
 cp ../msgpack/sysdep.h          msgpack/
-cat msgpack_test.rb | sed "s/require ['\"]msgpack['\"]/require File.dirname(__FILE__) + '\/test_helper.rb'/" > test/msgpack_test.rb
+cp ../test/cases.mpac           test/
+cp ../test/cases_compact.mpac   test/
+cp ../test/cases.json           test/
 
 gem build msgpack.gemspec
 
diff --git a/ruby/test/test_cases.rb b/ruby/test/test_cases.rb
new file mode 100644
index 0000000..bfb752e
--- /dev/null
+++ b/ruby/test/test_cases.rb
@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+here = File.dirname(__FILE__)
+require "#{here}/test_helper"
+
+begin
+require 'json'
+rescue LoadError
+require 'rubygems'
+require 'json'
+end
+
+CASES_PATH         = "#{here}/cases.mpac"
+CASES_COMPACT_PATH = "#{here}/cases_compact.mpac"
+CASES_JSON_PATH    = "#{here}/cases.json"
+
+class MessagePackTestCases < Test::Unit::TestCase
+	def feed_file(path)
+		pac = MessagePack::Unpacker.new
+		pac.feed File.read(path)
+		pac
+	end
+
+	def test_compare_compact
+		pac  = feed_file(CASES_PATH)
+		cpac = feed_file(CASES_COMPACT_PATH)
+
+		 objs = [];  pac.each {| obj|  objs <<  obj }
+		cobjs = []; cpac.each {|cobj| cobjs << cobj }
+
+		objs.zip(cobjs).each {|obj, cobj|
+			assert_equal(obj, cobj)
+		}
+	end
+
+	def test_compare_json
+		pac  = feed_file(CASES_PATH)
+
+		objs = []; pac.each {|obj| objs <<  obj }
+		jobjs = JSON.load File.read(CASES_JSON_PATH)
+
+		objs.zip(jobjs) {|obj, jobj|
+			assert_equal(obj, jobj)
+		}
+	end
+end
+
diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb
index 6a63489..19226ef 100644
--- a/ruby/test/test_helper.rb
+++ b/ruby/test/test_helper.rb
@@ -1,3 +1,7 @@
 require 'test/unit'
+begin
+require File.dirname(__FILE__) + '/../msgpack'
+rescue LoadError
 require File.dirname(__FILE__) + '/../lib/msgpack'
+end
 
diff --git a/ruby/msgpack_test.rb b/ruby/test/test_pack_unpack.rb
similarity index 86%
rename from ruby/msgpack_test.rb
rename to ruby/test/test_pack_unpack.rb
index 8cbb586..e22bab3 100644
--- a/ruby/msgpack_test.rb
+++ b/ruby/test/test_pack_unpack.rb
@@ -1,8 +1,7 @@
 #!/usr/bin/env ruby
-require 'msgpack'
-require 'test/unit'
+require File.dirname(__FILE__)+'/test_helper'
 
-class MessagePackTestFormat < Test::Unit::TestCase
+class MessagePackTestPackUnpack < Test::Unit::TestCase
 	def self.it(name, &block)
 		define_method("test_#{name}", &block)
 	end
@@ -177,16 +176,18 @@ class MessagePackTestFormat < Test::Unit::TestCase
 		match ({}), "\x80"
 	end
 
-	it "{0=>0, 1=>1, ..., 14=>14}" do
-		a = (0..14).to_a;
-		match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"
-	end
-
-	it "{0=>0, 1=>1, ..., 15=>15}" do
-		a = (0..15).to_a;
-		match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"
-	end
+## FIXME
+#	it "{0=>0, 1=>1, ..., 14=>14}" do
+#		a = (0..14).to_a;
+#		match Hash[*a.zip(a).flatten], "\x8f\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x04\x04\x0a\x0a"
+#	end
+#
+#	it "{0=>0, 1=>1, ..., 15=>15}" do
+#		a = (0..15).to_a;
+#		match Hash[*a.zip(a).flatten], "\xde\x00\x10\x05\x05\x0b\x0b\x00\x00\x06\x06\x0c\x0c\x01\x01\x07\x07\x0d\x0d\x02\x02\x08\x08\x0e\x0e\x03\x03\x09\x09\x0f\x0f\x04\x04\x0a\x0a"
+#	end
 
+## FIXME
 #	it "fixmap" do
 #		check_map 1, 0
 #		check_map 1, (1<<4)-1
diff --git a/test/cases_gen.rb b/test/cases_gen.rb
index b349f4e..95662fb 100644
--- a/test/cases_gen.rb
+++ b/test/cases_gen.rb
@@ -1,7 +1,10 @@
 #
 # MessagePack format test case
 #
-require 'rubygems' rescue nil
+begin
+require 'rubygems'
+rescue LoadError
+end
 require 'msgpack'
 require 'json'
 

From 7cd41aeb7280e5752deb5b60a140b627bd50d61e Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Thu, 3 Jun 2010 00:17:17 +0900
Subject: [PATCH 0525/1648] erlang: tracing crosslang.rb moving to ../test

---
 erlang/msgpack.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index f23ec09..ca9769e 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -337,7 +337,7 @@ basic_test()->
 port_test()->
     Tests = test_data(),
     {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])),
-    Port = open_port({spawn, "ruby ../crosslang.rb"}, [binary]),
+    Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]),
     true = port_command(Port, msgpack:pack(Tests) ),
     %Port ! {self, {command, msgpack:pack(Tests)}}, ... not owner
     receive

From 9c3ed173b1c497cfe9bdfff2e5e695e5a3377e0a Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Thu, 3 Jun 2010 21:51:40 +0900
Subject: [PATCH 0526/1648] ruby: fixes buffering routine

---
 ruby/unpack.c | 98 ++++++++++++++++++++-------------------------------
 1 file changed, 39 insertions(+), 59 deletions(-)

diff --git a/ruby/unpack.c b/ruby/unpack.c
index 0f7b9f0..65ae476 100644
--- a/ruby/unpack.c
+++ b/ruby/unpack.c
@@ -386,90 +386,70 @@ static VALUE MessagePack_Unpacker_stream_set(VALUE self, VALUE val)
 }
 
 
-#ifdef RUBY_VM
-#  ifndef STR_SHARED
-#    define STR_SHARED  FL_USER2
-#  endif
-#  ifndef STR_NOEMBED
-#    define STR_NOEMBED FL_USER1
-#  endif
-#  ifndef STR_ASSOC
-#    define STR_ASSOC   FL_USER3
-#  endif
-#  ifndef STR_NOCAPA_P
-#    define STR_NOCAPA_P(s) (FL_TEST(s,STR_NOEMBED) && FL_ANY(s,STR_SHARED|STR_ASSOC))
-#  endif
-#  define NEED_MORE_CAPA(s,size) (!STR_NOCAPA_P(s) && RSTRING(s)->as.heap.aux.capa < size)
-#else
-#  ifndef STR_NOCAPA
-#    ifndef STR_ASSOC
-#      define STR_ASSOC   FL_USER3
-#    endif
-#    ifndef ELTS_SHARED
-#      define ELTS_SHARED FL_USER2
-#    endif
-#    define STR_NOCAPA  (ELTS_SHARED|STR_ASSOC)
-#  endif
-#  define NEED_MORE_CAPA(s,size) (!FL_TEST(s,STR_NOCAPA) && RSTRING(s)->aux.capa < size)
-#endif
-
-static void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len)
+static void reserve_buffer(msgpack_unpack_t* mp, size_t require)
 {
 	struct unpack_buffer* buffer = &mp->user.buffer;
 
 	if(buffer->size == 0) {
-		char* tmp = ALLOC_N(char, MSGPACK_UNPACKER_BUFFER_INIT_SIZE);
+		size_t nsize = MSGPACK_UNPACKER_BUFFER_INIT_SIZE;
+		while(nsize < require) {
+			nsize *= 2;
+		}
+		char* tmp = ALLOC_N(char, nsize);
 		buffer->ptr = tmp;
-		buffer->free = MSGPACK_UNPACKER_BUFFER_INIT_SIZE;
+		buffer->free = nsize;
 		buffer->size = 0;
+		return;
+	}
 
-	} else if(buffer->size <= mp->user.offset) {
+	if(buffer->size <= mp->user.offset) {
 		/* clear buffer and rewind offset */
 		buffer->free += buffer->size;
 		buffer->size = 0;
 		mp->user.offset = 0;
 	}
 
-	if(len <= buffer->free) {
-		/* enough free space: just copy */
-		memcpy(buffer->ptr+buffer->size, ptr, len);
-		buffer->size += len;
-		buffer->free -= len;
+	if(require <= buffer->free) {
+		/* enough free space */
 		return;
 	}
 
-	size_t csize = buffer->size + buffer->free;
+	size_t nsize = (buffer->size + buffer->free) * 2;
 
 	if(mp->user.offset <= buffer->size / 2) {
-		/* parsed less than half: realloc and copy */
-		csize *= 2;
-		while(csize < buffer->size + len) {
-			csize *= 2;
+		/* parsed less than half: realloc only */
+		while(nsize < buffer->size + require) {
+			nsize *= 2;
 		}
-		char* tmp = REALLOC_N(buffer->ptr, char, csize);
-		memcpy(tmp + buffer->size, ptr, len);
+		char* tmp = REALLOC_N(buffer->ptr, char, nsize);
+		buffer->free = nsize - buffer->size;
 		buffer->ptr = tmp;
-		buffer->free = csize - buffer->size;
-		return;
-	}
 
-	size_t not_parsed = buffer->size - mp->user.offset;
-
-	if(csize < not_parsed + len) {
-		/* more buffer size */
-		csize *= 2;
-		while(csize < not_parsed + len) {
-			csize *= 2;
+	} else {
+		/* parsed more than half: realloc and move */
+		size_t not_parsed = buffer->size - mp->user.offset;
+		while(nsize < not_parsed + require) {
+			nsize *= 2;
 		}
-		char* tmp = REALLOC_N(buffer->ptr, char, csize);
+		char* tmp = REALLOC_N(buffer->ptr, char, nsize);
+		memcpy(tmp, tmp + mp->user.offset, not_parsed);
+		buffer->free = nsize - buffer->size;
+		buffer->size = not_parsed;
 		buffer->ptr = tmp;
+		mp->user.offset = 0;
 	}
+}
 
-	memcpy(buffer->ptr+not_parsed, ptr, not_parsed);
-	buffer->size = not_parsed;
-	buffer->free = csize - buffer->size;
-	buffer->ptr = buffer->ptr;
-	mp->user.offset = 0;
+static inline void feed_buffer(msgpack_unpack_t* mp, const char* ptr, size_t len)
+{
+	struct unpack_buffer* buffer = &mp->user.buffer;
+
+	if(buffer->free < len) {
+		reserve_buffer(mp, len);
+	}
+	memcpy(buffer->ptr + buffer->size, ptr, len);
+	buffer->size += len;
+	buffer->free -= len;
 }
 
 /**

From 251090406a642d968e314db3aa1d5240db00598a Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Thu, 3 Jun 2010 21:52:01 +0900
Subject: [PATCH 0527/1648] ruby: adds a test case for buffering

---
 ruby/test/test_pack_unpack.rb | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/ruby/test/test_pack_unpack.rb b/ruby/test/test_pack_unpack.rb
index e22bab3..9dff44f 100644
--- a/ruby/test/test_pack_unpack.rb
+++ b/ruby/test/test_pack_unpack.rb
@@ -203,6 +203,37 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase
 #		#check_map 5, (1<<32)-1  # memory error
 #	end
 
+	it "buffer" do
+		str = "a"*32*1024*4
+		raw = str.to_msgpack
+		pac = MessagePack::Unpacker.new
+
+		len = 0
+		parsed = false
+
+		n = 655
+		time = raw.size / n
+		time += 1 unless raw.size % n == 0
+		off = 0
+
+		time.times do
+			assert(!parsed)
+
+			fe = raw[off, n]
+			assert(fe.length > 0)
+			off += fe.length
+
+			pac.feed fe
+			pac.each {|obj|
+				assert(!parsed)
+				assert_equal(obj, str)
+				parsed = true
+			}
+		end
+
+		assert(parsed)
+	end
+
 	it "gc mark" do
 		obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"]
 		num = 4

From b3e0ad13030cdaee442582a8dd16c46518e9d6c9 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Thu, 3 Jun 2010 22:00:15 +0900
Subject: [PATCH 0528/1648] ruby: 0.4.2

---
 ruby/msgpack.gemspec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec
index e10622f..fb6338a 100644
--- a/ruby/msgpack.gemspec
+++ b/ruby/msgpack.gemspec
@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.platform = Gem::Platform::RUBY
   s.name = "msgpack"
-  s.version = "0.4.1"
+  s.version = "0.4.2"
   s.summary = "MessagePack, a binary-based efficient data interchange format."
   s.author = "FURUHASHI Sadayuki"
   s.email = "frsyuki@users.sourceforge.jp"

From 82a5dd6cf9a49f6ffa444d5fc0ba5d4e3f10eb3e Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Thu, 10 Jun 2010 14:02:58 -0700
Subject: [PATCH 0529/1648] java: update

---
 java/src/main/java/org/msgpack/Packer.java    |   4 +
 java/src/main/java/org/msgpack/Schema.java    |  79 ++----------
 java/src/main/java/org/msgpack/Unpacker.java  |   4 +-
 .../org/msgpack/schema/BooleanSchema.java     |  64 ++++++++++
 .../{RawSchema.java => ByteArraySchema.java}  |  58 ++++-----
 .../java/org/msgpack/schema/ByteSchema.java   |  55 +++++----
 .../org/msgpack/schema/ClassGenerator.java    |  23 ++--
 .../java/org/msgpack/schema/ClassSchema.java  |  58 ++++-----
 .../java/org/msgpack/schema/DoubleSchema.java |  50 +++-----
 .../java/org/msgpack/schema/FloatSchema.java  |  50 +++-----
 .../msgpack/schema/GenericClassSchema.java    |   4 -
 .../org/msgpack/schema/GenericSchema.java     |  73 +----------
 .../java/org/msgpack/schema/IntSchema.java    |  55 +++++----
 .../{ArraySchema.java => ListSchema.java}     |  80 +++++-------
 .../java/org/msgpack/schema/LongSchema.java   |  39 +++---
 .../java/org/msgpack/schema/MapSchema.java    |  43 ++++---
 .../org/msgpack/schema/SSchemaParser.java     |  22 +++-
 .../java/org/msgpack/schema/SetSchema.java    | 115 ++++++++++++++++++
 .../java/org/msgpack/schema/ShortSchema.java  |  52 ++++----
 .../java/org/msgpack/schema/StringSchema.java |  65 +++++-----
 20 files changed, 512 insertions(+), 481 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/schema/BooleanSchema.java
 rename java/src/main/java/org/msgpack/schema/{RawSchema.java => ByteArraySchema.java} (70%)
 rename java/src/main/java/org/msgpack/schema/{ArraySchema.java => ListSchema.java} (60%)
 create mode 100644 java/src/main/java/org/msgpack/schema/SetSchema.java

diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index a92d2b6..dd510f3 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -223,6 +223,10 @@ public class Packer {
 		return this;
 	}
 
+	public Packer packBoolean(boolean d) throws IOException {
+		return d ? packTrue() : packFalse();
+	}
+
 	public Packer packArray(int n) throws IOException {
 		if(n < 16) {
 			final int d = 0x90 | n;
diff --git a/java/src/main/java/org/msgpack/Schema.java b/java/src/main/java/org/msgpack/Schema.java
index f191f7a..25e10f9 100644
--- a/java/src/main/java/org/msgpack/Schema.java
+++ b/java/src/main/java/org/msgpack/Schema.java
@@ -20,28 +20,13 @@ package org.msgpack;
 import java.io.Writer;
 import java.io.IOException;
 import org.msgpack.schema.SSchemaParser;
-import org.msgpack.schema.ClassGenerator;
+//import org.msgpack.schema.ClassGenerator;
 
 public abstract class Schema {
-	private String expression;
-	private String name;
+	public Schema() { }
 
-	public Schema(String name) {
-		this.expression = expression;
-		this.name = name;
-	}
-
-	public String getName() {
-		return name;
-	}
-
-	public String getFullName() {
-		return name;
-	}
-
-	public String getExpression() {
-		return name;
-	}
+	public abstract String getClassName();
+	public abstract String getExpression();
 
 	public static Schema parse(String source) {
 		return SSchemaParser.parse(source);
@@ -51,83 +36,43 @@ public abstract class Schema {
 		return SSchemaParser.load(source);
 	}
 
-	public void write(Writer output) throws IOException {
-		ClassGenerator.write(this, output);
-	}
-
 	public abstract void pack(Packer pk, Object obj) throws IOException;
-
 	public abstract Object convert(Object obj) throws MessageTypeException;
 
-
 	public Object createFromNil() {
 		return null;
 	}
 
 	public Object createFromBoolean(boolean v) {
-		throw new RuntimeException("type error");
+		throw new MessageTypeException("type error");
 	}
 
 	public Object createFromByte(byte v) {
-		throw new RuntimeException("type error");
+		throw new MessageTypeException("type error");
 	}
 
 	public Object createFromShort(short v) {
-		throw new RuntimeException("type error");
+		throw new MessageTypeException("type error");
 	}
 
 	public Object createFromInt(int v) {
-		throw new RuntimeException("type error");
+		throw new MessageTypeException("type error");
 	}
 
 	public Object createFromLong(long v) {
-		throw new RuntimeException("type error");
+		throw new MessageTypeException("type error");
 	}
 
 	public Object createFromFloat(float v) {
-		throw new RuntimeException("type error");
+		throw new MessageTypeException("type error");
 	}
 
 	public Object createFromDouble(double v) {
-		throw new RuntimeException("type error");
+		throw new MessageTypeException("type error");
 	}
 
 	public Object createFromRaw(byte[] b, int offset, int length) {
-		throw new RuntimeException("type error");
+		throw new MessageTypeException("type error");
 	}
-
-	/* FIXME
-	public Object createFromBoolean(boolean v) {
-		throw MessageTypeException.schemaMismatch(this);
-	}
-
-	public Object createFromByte(byte v) {
-		throw MessageTypeException.schemaMismatch(this);
-	}
-
-	public Object createFromShort(short v) {
-		throw MessageTypeException.schemaMismatch(this);
-	}
-
-	public Object createFromInt(int v) {
-		throw MessageTypeException.schemaMismatch(this);
-	}
-
-	public Object createFromLong(long v) {
-		throw MessageTypeException.schemaMismatch(this);
-	}
-
-	public Object createFromFloat(float v) {
-		throw MessageTypeException.schemaMismatch(this);
-	}
-
-	public Object createFromDouble(double v) {
-		throw MessageTypeException.schemaMismatch(this);
-	}
-
-	public Object createFromRaw(byte[] b, int offset, int length) {
-		throw MessageTypeException.schemaMismatch(this);
-	}
-	*/
 }
 
diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index f22c58b..32bab64 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -561,11 +561,11 @@ public class Unpacker implements Iterable {
 		return impl.unpackObject();
 	}
 
-	final void unpack(MessageUnpackable obj) throws IOException, MessageTypeException {
+	final public void unpack(MessageUnpackable obj) throws IOException, MessageTypeException {
 		obj.messageUnpack(this);
 	}
 
-	final boolean tryUnpackNull() throws IOException {
+	final public boolean tryUnpackNull() throws IOException {
 		return impl.tryUnpackNull();
 	}
 }
diff --git a/java/src/main/java/org/msgpack/schema/BooleanSchema.java b/java/src/main/java/org/msgpack/schema/BooleanSchema.java
new file mode 100644
index 0000000..2c325f1
--- /dev/null
+++ b/java/src/main/java/org/msgpack/schema/BooleanSchema.java
@@ -0,0 +1,64 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.schema;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class BooleanSchema extends Schema {
+	public BooleanSchema() { }
+
+	@Override
+	public String getClassName() {
+		return "Boolean";
+	}
+
+	@Override
+	public String getExpression() {
+		return "boolean";
+	}
+
+	@Override
+	public void pack(Packer pk, Object obj) throws IOException {
+		if(obj instanceof Boolean) {
+			pk.packBoolean((Boolean)obj);
+		} else if(obj == null) {
+			pk.packNil();
+		} else {
+			throw MessageTypeException.invalidConvert(obj, this);
+		}
+	}
+
+	public static final boolean convertBoolean(Object obj) throws MessageTypeException {
+		if(obj instanceof Boolean) {
+			return (Boolean)obj;
+		}
+		throw new MessageTypeException();
+	}
+
+	@Override
+	public Object convert(Object obj) throws MessageTypeException {
+		return convertBoolean(obj);
+	}
+
+	@Override
+	public Object createFromBoolean(boolean v) {
+		return v;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/schema/RawSchema.java b/java/src/main/java/org/msgpack/schema/ByteArraySchema.java
similarity index 70%
rename from java/src/main/java/org/msgpack/schema/RawSchema.java
rename to java/src/main/java/org/msgpack/schema/ByteArraySchema.java
index f621e4c..af9c0ed 100644
--- a/java/src/main/java/org/msgpack/schema/RawSchema.java
+++ b/java/src/main/java/org/msgpack/schema/ByteArraySchema.java
@@ -22,59 +22,48 @@ import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import org.msgpack.*;
 
-public class RawSchema extends Schema {
-	public RawSchema() {
-		super("raw");
-	}
+public class ByteArraySchema extends Schema {
+	public ByteArraySchema() { }
 
-	public String getFullName() {
+	@Override
+	public String getClassName() {
 		return "byte[]";
 	}
 
+	@Override
+	public String getExpression() {
+		return "raw";
+	}
+
 	@Override
 	public void pack(Packer pk, Object obj) throws IOException {
-		// FIXME instanceof GenericObject
 		if(obj instanceof byte[]) {
-			byte[] d = (byte[])obj;
-			pk.packRaw(d.length);
-			pk.packRawBody(d);
-
-		} else if(obj instanceof ByteBuffer) {
-			ByteBuffer d = (ByteBuffer)obj;
-			if(!d.hasArray()) {
-				throw MessageTypeException.invalidConvert(obj, this);
-			}
-			pk.packRaw(d.capacity());
-			pk.packRawBody(d.array(), d.position(), d.capacity());
-
+			byte[] b = (byte[])obj;
+			pk.packRaw(b.length);
+			pk.packRawBody(b);
 		} else if(obj instanceof String) {
 			try {
-				byte[] d = ((String)obj).getBytes("UTF-8");
-				pk.packRaw(d.length);
-				pk.packRawBody(d);
+				byte[] b = ((String)obj).getBytes("UTF-8");
+				pk.packRaw(b.length);
+				pk.packRawBody(b);
 			} catch (UnsupportedEncodingException e) {
-				throw MessageTypeException.invalidConvert(obj, this);
+				throw new MessageTypeException();
 			}
-
 		} else if(obj == null) {
 			pk.packNil();
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
 	}
 
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		// FIXME instanceof GenericObject
+	public static final byte[] convertByteArray(Object obj) throws MessageTypeException {
 		if(obj instanceof byte[]) {
 			// FIXME copy?
 			//byte[] d = (byte[])obj;
 			//byte[] v = new byte[d.length];
 			//System.arraycopy(d, 0, v, 0, d.length);
 			//return v;
-			return obj;
-
+			return (byte[])obj;
 		} else if(obj instanceof ByteBuffer) {
 			ByteBuffer d = (ByteBuffer)obj;
 			byte[] v = new byte[d.capacity()];
@@ -82,19 +71,22 @@ public class RawSchema extends Schema {
 			d.get(v);
 			d.position(pos);
 			return v;
-
 		} else if(obj instanceof String) {
 			try {
 				return ((String)obj).getBytes("UTF-8");
 			} catch (UnsupportedEncodingException e) {
-				throw MessageTypeException.invalidConvert(obj, this);
+				throw new MessageTypeException();
 			}
-
 		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
+			throw new MessageTypeException();
 		}
 	}
 
+	@Override
+	public Object convert(Object obj) throws MessageTypeException {
+		return convertByteArray(obj);
+	}
+
 	@Override
 	public Object createFromRaw(byte[] b, int offset, int length) {
 		byte[] d = new byte[length];
diff --git a/java/src/main/java/org/msgpack/schema/ByteSchema.java b/java/src/main/java/org/msgpack/schema/ByteSchema.java
index 9ee6a82..6003a0f 100644
--- a/java/src/main/java/org/msgpack/schema/ByteSchema.java
+++ b/java/src/main/java/org/msgpack/schema/ByteSchema.java
@@ -21,8 +21,11 @@ import java.io.IOException;
 import org.msgpack.*;
 
 public class ByteSchema extends Schema {
-	public ByteSchema() {
-		super("Byte");
+	public ByteSchema() { }
+
+	@Override
+	public String getClassName() {
+		return "Byte";
 	}
 
 	@Override
@@ -33,27 +36,32 @@ public class ByteSchema extends Schema {
 	@Override
 	public void pack(Packer pk, Object obj) throws IOException {
 		if(obj instanceof Number) {
-			pk.packByte( ((Number)obj).byteValue() );
-
+			short value = ((Number)obj).shortValue();
+			if(value > Byte.MAX_VALUE) {
+				throw new MessageTypeException();
+			}
+			pk.packByte((byte)value);
 		} else if(obj == null) {
 			pk.packNil();
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
 	}
 
+	public static final byte convertByte(Object obj) throws MessageTypeException {
+		if(obj instanceof Number) {
+			short value = ((Number)obj).shortValue();
+			if(value > Byte.MAX_VALUE) {
+				throw new MessageTypeException();
+			}
+			return (byte)value;
+		}
+		throw new MessageTypeException();
+	}
+
 	@Override
 	public Object convert(Object obj) throws MessageTypeException {
-		if(obj instanceof Byte) {
-			return obj;
-
-		} else if(obj instanceof Number) {
-			return ((Number)obj).byteValue();
-
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
+		return convertByte(obj);
 	}
 
 	@Override
@@ -63,26 +71,25 @@ public class ByteSchema extends Schema {
 
 	@Override
 	public Object createFromShort(short v) {
+		if(v > Byte.MAX_VALUE) {
+			throw new MessageTypeException();
+		}
 		return (byte)v;
 	}
 
 	@Override
 	public Object createFromInt(int v) {
+		if(v > Byte.MAX_VALUE) {
+			throw new MessageTypeException();
+		}
 		return (byte)v;
 	}
 
 	@Override
 	public Object createFromLong(long v) {
-		return (byte)v;
-	}
-
-	@Override
-	public Object createFromFloat(float v) {
-		return (byte)v;
-	}
-
-	@Override
-	public Object createFromDouble(double v) {
+		if(v > Byte.MAX_VALUE) {
+			throw new MessageTypeException();
+		}
 		return (byte)v;
 	}
 }
diff --git a/java/src/main/java/org/msgpack/schema/ClassGenerator.java b/java/src/main/java/org/msgpack/schema/ClassGenerator.java
index f8a13fa..a515996 100644
--- a/java/src/main/java/org/msgpack/schema/ClassGenerator.java
+++ b/java/src/main/java/org/msgpack/schema/ClassGenerator.java
@@ -77,8 +77,11 @@ public class ClassGenerator {
 			for(FieldSchema f : cs.getFields()) {
 				findSubclassSchema(dst, f.getSchema());
 			}
-		} else if(s instanceof ArraySchema) {
-			ArraySchema as = (ArraySchema)s;
+		} else if(s instanceof ListSchema) {
+			ListSchema as = (ListSchema)s;
+			findSubclassSchema(dst, as.getElementSchema(0));
+		} else if(s instanceof SetSchema) {
+			SetSchema as = (SetSchema)s;
 			findSubclassSchema(dst, as.getElementSchema(0));
 		} else if(s instanceof MapSchema) {
 			MapSchema as = (MapSchema)s;
@@ -105,7 +108,7 @@ public class ClassGenerator {
 
 	private void writeClass() throws IOException {
 		line();
-		line("public final class "+schema.getName()+" implements MessagePackable, MessageConvertable");
+		line("public final class "+schema.getClassName()+" implements MessagePackable, MessageConvertable");
 		line("{");
 		pushIndent();
 			writeSchema();
@@ -117,7 +120,7 @@ public class ClassGenerator {
 
 	private void writeSubclass() throws IOException {
 		line();
-		line("final class "+schema.getName()+" implements MessagePackable, MessageConvertable");
+		line("final class "+schema.getClassName()+" implements MessagePackable, MessageConvertable");
 		line("{");
 		pushIndent();
 			writeSchema();
@@ -135,7 +138,7 @@ public class ClassGenerator {
 	private void writeMemberVariables() throws IOException {
 		line();
 		for(FieldSchema f : schema.getFields()) {
-			line("public "+f.getSchema().getFullName()+" "+f.getName()+";");
+			line("public "+f.getSchema().getClassName()+" "+f.getName()+";");
 		}
 	}
 
@@ -156,7 +159,7 @@ public class ClassGenerator {
 
 	private void writeConstructors() throws IOException {
 		line();
-		line("public "+schema.getName()+"() { }");
+		line("public "+schema.getClassName()+"() { }");
 	}
 
 	private void writeAccessors() throws IOException {
@@ -195,7 +198,7 @@ public class ClassGenerator {
 			line("FieldSchema[] _fields = _SCHEMA.getFields();");
 			int i = 0;
 			for(FieldSchema f : schema.getFields()) {
-				line("if(_source.length <= "+i+") { return; } this."+f.getName()+" = ("+f.getSchema().getFullName()+")_fields["+i+"].getSchema().convert(_source["+i+"]);");
+				line("if(_source.length <= "+i+") { return; } this."+f.getName()+" = ("+f.getSchema().getClassName()+")_fields["+i+"].getSchema().convert(_source["+i+"]);");
 				++i;
 			}
 		popIndent();
@@ -205,13 +208,13 @@ public class ClassGenerator {
 	private void writeFactoryFunction() throws IOException {
 		line();
 		line("@SuppressWarnings(\"unchecked\")");
-		line("public static "+schema.getName()+" createFromMessage(Object[] _message)");
+		line("public static "+schema.getClassName()+" createFromMessage(Object[] _message)");
 		line("{");
 		pushIndent();
-			line(schema.getName()+" _self = new "+schema.getName()+"();");
+			line(schema.getClassName()+" _self = new "+schema.getClassName()+"();");
 			int i = 0;
 			for(FieldSchema f : schema.getFields()) {
-				line("if(_message.length <= "+i+") { return _self; } _self."+f.getName()+" = ("+f.getSchema().getFullName()+")_message["+i+"];");
+				line("if(_message.length <= "+i+") { return _self; } _self."+f.getName()+" = ("+f.getSchema().getClassName()+")_message["+i+"];");
 				++i;
 			}
 			line("return _self;");
diff --git a/java/src/main/java/org/msgpack/schema/ClassSchema.java b/java/src/main/java/org/msgpack/schema/ClassSchema.java
index cd5c008..cd59755 100644
--- a/java/src/main/java/org/msgpack/schema/ClassSchema.java
+++ b/java/src/main/java/org/msgpack/schema/ClassSchema.java
@@ -22,6 +22,7 @@ import java.util.List;
 import org.msgpack.*;
 
 public abstract class ClassSchema extends Schema implements IArraySchema {
+	protected String name;
 	protected FieldSchema[] fields;
 	protected List imports;
 	protected String namespace;
@@ -30,7 +31,7 @@ public abstract class ClassSchema extends Schema implements IArraySchema {
 	public ClassSchema(
 			String name, String namespace,
 			List imports, List fields) {
-		super(name);
+		this.name = name;
 		this.namespace = namespace;
 		this.imports = imports;  // FIXME clone?
 		this.fields = new FieldSchema[fields.size()];
@@ -42,6 +43,31 @@ public abstract class ClassSchema extends Schema implements IArraySchema {
 		}
 	}
 
+	@Override
+	public String getClassName() {
+		return name;
+	}
+
+	@Override
+	public String getExpression() {
+		StringBuffer b = new StringBuffer();
+		b.append("(class ");
+		b.append(name);
+		if(namespace != null) {
+			b.append(" (package "+namespace+")");
+		}
+		for(FieldSchema f : fields) {
+			b.append(" "+f.getExpression());
+		}
+		b.append(")");
+		return b.toString();
+	}
+
+	public boolean equals(ClassSchema o) {
+		return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) &&
+			name.equals(o.name);
+	}
+
 	public final FieldSchema[] getFields() {
 		return fields;
 	}
@@ -61,35 +87,5 @@ public abstract class ClassSchema extends Schema implements IArraySchema {
 	void setImports(List imports) {
 		this.imports = imports;  // FIXME clone?
 	}
-
-	//@Override
-	//public String getFullName()
-	//{
-	//	if(namespace == null) {
-	//		return getName();
-	//	} else {
-	//		return namespace+"."+getName();
-	//	}
-	//}
-
-	@Override
-	public String getExpression() {
-		StringBuffer b = new StringBuffer();
-		b.append("(class ");
-		b.append(getName());
-		if(namespace != null) {
-			b.append(" (package "+namespace+")");
-		}
-		for(FieldSchema f : fields) {
-			b.append(" "+f.getExpression());
-		}
-		b.append(")");
-		return b.toString();
-	}
-
-	public boolean equals(SpecificClassSchema o) {
-		return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) &&
-			getName().equals(o.getName());
-	}
 }
 
diff --git a/java/src/main/java/org/msgpack/schema/DoubleSchema.java b/java/src/main/java/org/msgpack/schema/DoubleSchema.java
index d53e47d..cb857c3 100644
--- a/java/src/main/java/org/msgpack/schema/DoubleSchema.java
+++ b/java/src/main/java/org/msgpack/schema/DoubleSchema.java
@@ -21,8 +21,11 @@ import java.io.IOException;
 import org.msgpack.*;
 
 public class DoubleSchema extends Schema {
-	public DoubleSchema() {
-		super("Double");
+	public DoubleSchema() { }
+
+	@Override
+	public String getClassName() {
+		return "Double";
 	}
 
 	@Override
@@ -32,43 +35,30 @@ public class DoubleSchema extends Schema {
 
 	@Override
 	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof Number) {
-			pk.packDouble( ((Number)obj).doubleValue() );
-
+		if(obj instanceof Double) {
+			pk.packDouble((Double)obj);
+		} else if(obj instanceof Float) {
+			pk.packFloat((Float)obj);
 		} else if(obj == null) {
 			pk.packNil();
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
 	}
 
+	public static final double convertDouble(Object obj) throws MessageTypeException {
+		if(obj instanceof Double) {
+			return (Double)obj;
+		} else if(obj instanceof Float) {
+			return ((Float)obj).doubleValue();
+		} else {
+			throw new MessageTypeException();
+		}
+	}
+
 	@Override
 	public Object convert(Object obj) throws MessageTypeException {
-		if(obj instanceof Double) {
-			return obj;
-
-		} else if(obj instanceof Number) {
-			return ((Number)obj).doubleValue();
-
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	@Override
-	public Object createFromByte(byte v) {
-		return (double)v;
-	}
-
-	@Override
-	public Object createFromShort(short v) {
-		return (double)v;
-	}
-
-	@Override
-	public Object createFromInt(int v) {
-		return (double)v;
+		return convertDouble(obj);
 	}
 
 	@Override
diff --git a/java/src/main/java/org/msgpack/schema/FloatSchema.java b/java/src/main/java/org/msgpack/schema/FloatSchema.java
index 2777521..cd73201 100644
--- a/java/src/main/java/org/msgpack/schema/FloatSchema.java
+++ b/java/src/main/java/org/msgpack/schema/FloatSchema.java
@@ -21,8 +21,11 @@ import java.io.IOException;
 import org.msgpack.*;
 
 public class FloatSchema extends Schema {
-	public FloatSchema() {
-		super("Float");
+	public FloatSchema() { }
+
+	@Override
+	public String getClassName() {
+		return "Float";
 	}
 
 	@Override
@@ -32,43 +35,30 @@ public class FloatSchema extends Schema {
 
 	@Override
 	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof Number) {
-			pk.packFloat( ((Number)obj).floatValue() );
-
+		if(obj instanceof Double) {
+			pk.packDouble((Double)obj);
+		} else if(obj instanceof Float) {
+			pk.packFloat((Float)obj);
 		} else if(obj == null) {
 			pk.packNil();
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
 	}
 
+	public static final float convertFloat(Object obj) throws MessageTypeException {
+		if(obj instanceof Double) {
+			return ((Double)obj).floatValue();
+		} else if(obj instanceof Float) {
+			return (Float)obj;
+		} else {
+			throw new MessageTypeException();
+		}
+	}
+
 	@Override
 	public Object convert(Object obj) throws MessageTypeException {
-		if(obj instanceof Float) {
-			return obj;
-
-		} else if(obj instanceof Number) {
-			return ((Number)obj).floatValue();
-
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	@Override
-	public Object createFromByte(byte v) {
-		return (float)v;
-	}
-
-	@Override
-	public Object createFromShort(short v) {
-		return (float)v;
-	}
-
-	@Override
-	public Object createFromInt(int v) {
-		return (float)v;
+		return convertFloat(obj);
 	}
 
 	@Override
diff --git a/java/src/main/java/org/msgpack/schema/GenericClassSchema.java b/java/src/main/java/org/msgpack/schema/GenericClassSchema.java
index ffdd4ab..1ab4c33 100644
--- a/java/src/main/java/org/msgpack/schema/GenericClassSchema.java
+++ b/java/src/main/java/org/msgpack/schema/GenericClassSchema.java
@@ -41,10 +41,8 @@ public class GenericClassSchema extends ClassSchema {
 				FieldSchema f = fields[i];
 				f.getSchema().pack(pk, d.get(f.getName()));
 			}
-
 		} else if(obj == null) {
 			pk.packNil();
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
@@ -55,7 +53,6 @@ public class GenericClassSchema extends ClassSchema {
 		if(obj instanceof Collection) {
 			// FIXME optimize
 			return createFromArray( ((Collection)obj).toArray() );
-
 		} else if(obj instanceof Map) {
 			HashMap m = new HashMap(fields.length);
 			Map d = (Map)obj;
@@ -65,7 +62,6 @@ public class GenericClassSchema extends ClassSchema {
 				m.put(fieldName, f.getSchema().convert(d.get(fieldName)));
 			}
 			return m;
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
diff --git a/java/src/main/java/org/msgpack/schema/GenericSchema.java b/java/src/main/java/org/msgpack/schema/GenericSchema.java
index 0adf898..f9098ed 100644
--- a/java/src/main/java/org/msgpack/schema/GenericSchema.java
+++ b/java/src/main/java/org/msgpack/schema/GenericSchema.java
@@ -22,11 +22,13 @@ import java.util.List;
 import java.util.HashMap;
 import java.io.IOException;
 import org.msgpack.*;
-//import org.msgpack.generic.*;
 
 public class GenericSchema extends Schema implements IArraySchema, IMapSchema {
-	public GenericSchema() {
-		super("Object");
+	public GenericSchema() { }
+
+	@Override
+	public String getClassName() {
+		return "Object";
 	}
 
 	@Override
@@ -123,70 +125,5 @@ public class GenericSchema extends Schema implements IArraySchema, IMapSchema {
 		}
 		return m;
 	}
-
-	/*
-   @Override
-	public Object createFromNil() {
-		return null;
-	}
-
-	@Override
-	public Object createFromBoolean(boolean v) {
-		return new GenericBoolean(v);
-	}
-
-	@Override
-	public Object createFromFromByte(byte v) {
-		return new GenericByte(v);
-	}
-
-	@Override
-	public Object createFromShort(short v) {
-		return new GenericShort(v);
-	}
-
-	@Override
-	public Object createFromInt(int v) {
-		return new GenericInt(v);
-	}
-
-	@Override
-	public Object createFromLong(long v) {
-		return new GenericLong(v);
-	}
-
-	@Override
-	public Object createFromFloat(float v) {
-		return new GenericFloat(v);
-	}
-
-	@Override
-	public Object createFromDouble(double v) {
-		return new GenericDouble(v);
-	}
-
-	@Override
-	public Object createFromRaw(byte[] b, int offset, int length) {
-		return new GenericRaw(b, offset, length);
-	}
-
-	@Override
-	public Object createFromArray(Object[] obj) {
-		// FIXME GenericArray
-		return Arrays.asList(obj);
-	}
-
-	@Override
-	public Object createFromMap(Object[] obj) {
-		GenericMap m = new GenericMap(obj.length / 2);
-		int i = 0;
-		while(i < obj.length) {
-			Object k = obj[i++];
-			Object v = obj[i++];
-			m.put(k, v);
-		}
-		return m;
-	}
-	*/
 }
 
diff --git a/java/src/main/java/org/msgpack/schema/IntSchema.java b/java/src/main/java/org/msgpack/schema/IntSchema.java
index 5a7e281..269f4fb 100644
--- a/java/src/main/java/org/msgpack/schema/IntSchema.java
+++ b/java/src/main/java/org/msgpack/schema/IntSchema.java
@@ -21,8 +21,11 @@ import java.io.IOException;
 import org.msgpack.*;
 
 public class IntSchema extends Schema {
-	public IntSchema() {
-		super("Integer");
+	public IntSchema() { }
+
+	@Override
+	public String getClassName() {
+		return "Integer";
 	}
 
 	@Override
@@ -33,27 +36,38 @@ public class IntSchema extends Schema {
 	@Override
 	public void pack(Packer pk, Object obj) throws IOException {
 		if(obj instanceof Number) {
-			pk.packInt( ((Number)obj).intValue() );
-
+			int value = ((Number)obj).intValue();
+			if(value >= Short.MAX_VALUE) {
+				long lvalue = ((Number)obj).longValue();
+				if(lvalue > Integer.MAX_VALUE) {
+					throw new MessageTypeException();
+				}
+			}
+			pk.packInt(value);
 		} else if(obj == null) {
 			pk.packNil();
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
 	}
 
+	public static final int convertInt(Object obj) throws MessageTypeException {
+		if(obj instanceof Number) {
+			int value = ((Number)obj).intValue();
+			if(value >= Integer.MAX_VALUE) {
+				long lvalue = ((Number)obj).longValue();
+				if(lvalue > Integer.MAX_VALUE) {
+					throw new MessageTypeException();
+				}
+			}
+			return value;
+		}
+		throw new MessageTypeException();
+	}
+
 	@Override
 	public Object convert(Object obj) throws MessageTypeException {
-		if(obj instanceof Integer) {
-			return obj;
-
-		} else if(obj instanceof Number) {
-			return ((Number)obj).intValue();
-
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
+		return convertInt(obj);
 	}
 
 	@Override
@@ -73,16 +87,9 @@ public class IntSchema extends Schema {
 
 	@Override
 	public Object createFromLong(long v) {
-		return (int)v;
-	}
-
-	@Override
-	public Object createFromFloat(float v) {
-		return (int)v;
-	}
-
-	@Override
-	public Object createFromDouble(double v) {
+		if(v > Integer.MAX_VALUE) {
+			throw new MessageTypeException();
+		}
 		return (int)v;
 	}
 }
diff --git a/java/src/main/java/org/msgpack/schema/ArraySchema.java b/java/src/main/java/org/msgpack/schema/ListSchema.java
similarity index 60%
rename from java/src/main/java/org/msgpack/schema/ArraySchema.java
rename to java/src/main/java/org/msgpack/schema/ListSchema.java
index fd47143..bef8cc4 100644
--- a/java/src/main/java/org/msgpack/schema/ArraySchema.java
+++ b/java/src/main/java/org/msgpack/schema/ListSchema.java
@@ -22,37 +22,32 @@ import java.util.Collection;
 import java.util.Set;
 import java.util.List;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.RandomAccess;
 import java.io.IOException;
 import org.msgpack.*;
 
-public class ArraySchema extends Schema implements IArraySchema {
+public class ListSchema extends Schema implements IArraySchema {
 	private Schema elementSchema;
 
-	public ArraySchema(Schema elementSchema)
-	{
-		super("array");
+	public ListSchema(Schema elementSchema) {
 		this.elementSchema = elementSchema;
 	}
 
 	@Override
-	public String getFullName()
-	{
-		return "List<"+elementSchema.getFullName()+">";
+	public String getClassName() {
+		return "List<"+elementSchema.getClassName()+">";
 	}
 
 	@Override
-	public String getExpression()
-	{
+	public String getExpression() {
 		return "(array "+elementSchema.getExpression()+")";
 	}
 
 	@Override
-	@SuppressWarnings("unchecked")
-	public void pack(Packer pk, Object obj) throws IOException
-	{
+	public void pack(Packer pk, Object obj) throws IOException {
 		if(obj instanceof List) {
-			ArrayList d = (ArrayList)obj;
+			List d = (List)obj;
 			pk.packArray(d.size());
 			if(obj instanceof RandomAccess) {
 				for(int i=0; i < d.size(); ++i) {
@@ -63,62 +58,53 @@ public class ArraySchema extends Schema implements IArraySchema {
 					elementSchema.pack(pk, e);
 				}
 			}
-
 		} else if(obj instanceof Set) {
 			Set d = (Set)obj;
 			pk.packArray(d.size());
 			for(Object e : d) {
 				elementSchema.pack(pk, e);
 			}
-
 		} else if(obj == null) {
 			pk.packNil();
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
 	}
 
-	@Override
 	@SuppressWarnings("unchecked")
-	public Object convert(Object obj) throws MessageTypeException
-	{
-		if(obj instanceof List) {
-			List d = (List)obj;
-			ArrayList ar = new ArrayList(d.size());
-			if(obj instanceof RandomAccess) {
-				for(int i=0; i < d.size(); ++i) {
-					ar.add( elementSchema.convert(d.get(i)) );
-				}
-			} else {
-				for(Object e : d) {
-					ar.add( elementSchema.convert(e) );
-				}
-			}
-			return ar;
-
-		} else if(obj instanceof Collection) {
-			Collection d = (Collection)obj;
-			ArrayList ar = new ArrayList(d.size());
-			for(Object e : (Collection)obj) {
-				ar.add( elementSchema.convert(e) );
-			}
-			return ar;
-
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
+	public static final  List convertList(Object obj,
+			Schema elementSchema, List dest) throws MessageTypeException {
+		if(!(obj instanceof List)) {
+			throw new MessageTypeException();
 		}
+		List d = (List)obj;
+		if(dest == null) {
+			dest = new ArrayList(d.size());
+		}
+		if(obj instanceof RandomAccess) {
+			for(int i=0; i < d.size(); ++i) {
+				dest.add( (T)elementSchema.convert(d.get(i)) );
+			}
+		} else {
+			for(Object e : d) {
+				dest.add( (T)elementSchema.convert(e) );
+			}
+		}
+		return dest;
 	}
 
 	@Override
-	public Schema getElementSchema(int index)
-	{
+	public Object convert(Object obj) throws MessageTypeException {
+		return convertList(obj, elementSchema, null);
+	}
+
+	@Override
+	public Schema getElementSchema(int index) {
 		return elementSchema;
 	}
 
 	@Override
-	public Object createFromArray(Object[] obj)
-	{
+	public Object createFromArray(Object[] obj) {
 		return Arrays.asList(obj);
 	}
 }
diff --git a/java/src/main/java/org/msgpack/schema/LongSchema.java b/java/src/main/java/org/msgpack/schema/LongSchema.java
index 83a30e3..728fa21 100644
--- a/java/src/main/java/org/msgpack/schema/LongSchema.java
+++ b/java/src/main/java/org/msgpack/schema/LongSchema.java
@@ -21,8 +21,11 @@ import java.io.IOException;
 import org.msgpack.*;
 
 public class LongSchema extends Schema {
-	public LongSchema() {
-		super("Long");
+	public LongSchema() { }
+
+	@Override
+	public String getClassName() {
+		return "Long";
 	}
 
 	@Override
@@ -33,27 +36,25 @@ public class LongSchema extends Schema {
 	@Override
 	public void pack(Packer pk, Object obj) throws IOException {
 		if(obj instanceof Number) {
-			pk.packLong( ((Number)obj).longValue() );
-
+			long value = ((Number)obj).longValue();
+			pk.packLong(value);
 		} else if(obj == null) {
 			pk.packNil();
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
 	}
 
+	public static final long convertLong(Object obj) throws MessageTypeException {
+		if(obj instanceof Number) {
+			return ((Number)obj).longValue();
+		}
+		throw new MessageTypeException();
+	}
+
 	@Override
 	public Object convert(Object obj) throws MessageTypeException {
-		if(obj instanceof Long) {
-			return obj;
-
-		} else if(obj instanceof Number) {
-			return ((Number)obj).longValue();
-
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
+		return convertLong(obj);
 	}
 
 	@Override
@@ -75,15 +76,5 @@ public class LongSchema extends Schema {
 	public Object createFromLong(long v) {
 		return (long)v;
 	}
-
-	@Override
-	public Object createFromFloat(float v) {
-		return (long)v;
-	}
-
-	@Override
-	public Object createFromDouble(double v) {
-		return (long)v;
-	}
 }
 
diff --git a/java/src/main/java/org/msgpack/schema/MapSchema.java b/java/src/main/java/org/msgpack/schema/MapSchema.java
index ba75993..339a5c2 100644
--- a/java/src/main/java/org/msgpack/schema/MapSchema.java
+++ b/java/src/main/java/org/msgpack/schema/MapSchema.java
@@ -27,14 +27,13 @@ public class MapSchema extends Schema implements IMapSchema {
 	private Schema valueSchema;
 
 	public MapSchema(Schema keySchema, Schema valueSchema) {
-		super("map");
 		this.keySchema = keySchema;
 		this.valueSchema = valueSchema;
 	}
 
 	@Override
-	public String getFullName() {
-		return "HashList<"+keySchema.getFullName()+", "+valueSchema.getFullName()+">";
+	public String getClassName() {
+		return "Map<"+keySchema.getClassName()+", "+valueSchema.getClassName()+">";
 	}
 
 	@Override
@@ -52,29 +51,33 @@ public class MapSchema extends Schema implements IMapSchema {
 				keySchema.pack(pk, e.getKey());
 				valueSchema.pack(pk, e.getValue());
 			}
-
 		} else if(obj == null) {
 			pk.packNil();
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
 	}
 
-	@Override
 	@SuppressWarnings("unchecked")
-	public Object convert(Object obj) throws MessageTypeException {
-		if(obj instanceof Map) {
-			Map d = (Map)obj;
-			Map m = new HashMap();
-			for(Map.Entry e : d.entrySet()) {
-				m.put(keySchema.convert(e.getKey()), valueSchema.convert(e.getValue()));
-			}
-			return m;
-
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
+	public static final  Map convertMap(Object obj,
+			Schema keySchema, Schema valueSchema, Map dest) throws MessageTypeException {
+		if(!(obj instanceof Map)) {
+			throw new MessageTypeException();
 		}
+		Map d = (Map)obj;
+		if(dest == null) {
+			dest = new HashMap(d.size());
+		}
+		for(Map.Entry e : d.entrySet()) {
+			dest.put((K)keySchema.convert(e.getKey()),
+					(V)valueSchema.convert(e.getValue()));
+		}
+		return (Map)d;
+	}
+
+	@Override
+	public Object convert(Object obj) throws MessageTypeException {
+		return convertMap(obj, keySchema, valueSchema, null);
 	}
 
 	@Override
@@ -90,14 +93,14 @@ public class MapSchema extends Schema implements IMapSchema {
 	@Override
 	@SuppressWarnings("unchecked")
 	public Object createFromMap(Object[] obj) {
-		HashMap m = new HashMap(obj.length / 2);
+		HashMap dest = new HashMap(obj.length / 2);
 		int i = 0;
 		while(i < obj.length) {
 			Object k = obj[i++];
 			Object v = obj[i++];
-			m.put(k, v);
+			dest.put(k, v);
 		}
-		return m;
+		return dest;
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/schema/SSchemaParser.java b/java/src/main/java/org/msgpack/schema/SSchemaParser.java
index 4ae8a4b..4345e92 100644
--- a/java/src/main/java/org/msgpack/schema/SSchemaParser.java
+++ b/java/src/main/java/org/msgpack/schema/SSchemaParser.java
@@ -140,7 +140,7 @@ public class SSchemaParser {
 			if(type.equals("string")) {
 				return new StringSchema();
 			} else if(type.equals("raw")) {
-				return new RawSchema();
+				return new ByteArraySchema();
 			} else if(type.equals("byte")) {
 				return new ByteSchema();
 			} else if(type.equals("short")) {
@@ -163,11 +163,13 @@ public class SSchemaParser {
 			if(type.equals("class")) {
 				return parseClass(exp);
 			} else if(type.equals("array")) {
-				return parseArray(exp);
+				return parseList(exp);
+			} else if(type.equals("set")) {
+				return parseSet(exp);
 			} else if(type.equals("map")) {
 				return parseMap(exp);
 			} else {
-				throw new RuntimeException("class, array or map is expected but got '"+type+"': "+exp);
+				throw new RuntimeException("class, list, set or map is expected but got '"+type+"': "+exp);
 			}
 		}
 	}
@@ -209,12 +211,20 @@ public class SSchemaParser {
 		}
 	}
 
-	private ArraySchema parseArray(SExp exp) {
+	private ListSchema parseList(SExp exp) {
 		if(exp.size() != 2) {
-			throw new RuntimeException("array is (array ELEMENT_TYPE): "+exp);
+			throw new RuntimeException("list is (list ELEMENT_TYPE): "+exp);
 		}
 		Schema elementType = readType(exp.getTuple(1));
-		return new ArraySchema(elementType);
+		return new ListSchema(elementType);
+	}
+
+	private SetSchema parseSet(SExp exp) {
+		if(exp.size() != 2) {
+			throw new RuntimeException("list is (list ELEMENT_TYPE): "+exp);
+		}
+		Schema elementType = readType(exp.getTuple(1));
+		return new SetSchema(elementType);
 	}
 
 	private MapSchema parseMap(SExp exp) {
diff --git a/java/src/main/java/org/msgpack/schema/SetSchema.java b/java/src/main/java/org/msgpack/schema/SetSchema.java
new file mode 100644
index 0000000..a3e1974
--- /dev/null
+++ b/java/src/main/java/org/msgpack/schema/SetSchema.java
@@ -0,0 +1,115 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.schema;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.RandomAccess;
+import java.io.IOException;
+import org.msgpack.*;
+
+public class SetSchema extends Schema implements IArraySchema {
+	private Schema elementSchema;
+
+	public SetSchema(Schema elementSchema) {
+		this.elementSchema = elementSchema;
+	}
+
+	@Override
+	public String getClassName() {
+		return "Set<"+elementSchema.getClassName()+">";
+	}
+
+	@Override
+	public String getExpression() {
+		return "(set "+elementSchema.getExpression()+")";
+	}
+
+	@Override
+	public void pack(Packer pk, Object obj) throws IOException {
+		if(obj instanceof List) {
+			List d = (List)obj;
+			pk.packArray(d.size());
+			if(obj instanceof RandomAccess) {
+				for(int i=0; i < d.size(); ++i) {
+					elementSchema.pack(pk, d.get(i));
+				}
+			} else {
+				for(Object e : d) {
+					elementSchema.pack(pk, e);
+				}
+			}
+		} else if(obj instanceof Set) {
+			Set d = (Set)obj;
+			pk.packArray(d.size());
+			for(Object e : d) {
+				elementSchema.pack(pk, e);
+			}
+		} else if(obj == null) {
+			pk.packNil();
+		} else {
+			throw MessageTypeException.invalidConvert(obj, this);
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	public static final  Set convertSet(Object obj,
+			Schema elementSchema, Set dest) throws MessageTypeException {
+		if(!(obj instanceof List)) {
+			throw new MessageTypeException();
+		}
+		List d = (List)obj;
+		if(dest == null) {
+			dest = new HashSet(d.size());
+		}
+		if(obj instanceof RandomAccess) {
+			for(int i=0; i < d.size(); ++i) {
+				dest.add( (T)elementSchema.convert(d.get(i)) );
+			}
+		} else {
+			for(Object e : d) {
+				dest.add( (T)elementSchema.convert(e) );
+			}
+		}
+		return dest;
+	}
+
+	@Override
+	public Object convert(Object obj) throws MessageTypeException {
+		return convertSet(obj, elementSchema, null);
+	}
+
+	@Override
+	public Schema getElementSchema(int index) {
+		return elementSchema;
+	}
+
+	@Override
+	public Object createFromArray(Object[] obj) {
+		Set m = new HashSet(obj.length);
+		for(int i=0; i < obj.length; i++) {
+			m.add(obj[i]);
+		}
+		return m;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/schema/ShortSchema.java b/java/src/main/java/org/msgpack/schema/ShortSchema.java
index f32ab41..21b9327 100644
--- a/java/src/main/java/org/msgpack/schema/ShortSchema.java
+++ b/java/src/main/java/org/msgpack/schema/ShortSchema.java
@@ -21,8 +21,11 @@ import java.io.IOException;
 import org.msgpack.*;
 
 public class ShortSchema extends Schema {
-	public ShortSchema() {
-		super("Short");
+	public ShortSchema() { }
+
+	@Override
+	public String getClassName() {
+		return "Short";
 	}
 
 	@Override
@@ -33,27 +36,32 @@ public class ShortSchema extends Schema {
 	@Override
 	public void pack(Packer pk, Object obj) throws IOException {
 		if(obj instanceof Number) {
-			pk.packShort( ((Number)obj).shortValue() );
-
+			int value = ((Number)obj).intValue();
+			if(value > Short.MAX_VALUE) {
+				throw new MessageTypeException();
+			}
+			pk.packShort((short)value);
 		} else if(obj == null) {
 			pk.packNil();
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
 	}
 
+	public static final short convertShort(Object obj) throws MessageTypeException {
+		if(obj instanceof Number) {
+			int value = ((Number)obj).intValue();
+			if(value > Short.MAX_VALUE) {
+				throw new MessageTypeException();
+			}
+			return (short)value;
+		}
+		throw new MessageTypeException();
+	}
+
 	@Override
 	public Object convert(Object obj) throws MessageTypeException {
-		if(obj instanceof Short) {
-			return obj;
-
-		} else if(obj instanceof Number) {
-			return ((Number)obj).shortValue();
-
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
+		return convertShort(obj);
 	}
 
 	@Override
@@ -68,21 +76,17 @@ public class ShortSchema extends Schema {
 
 	@Override
 	public Object createFromInt(int v) {
+		if(v > Short.MAX_VALUE) {
+			throw new MessageTypeException();
+		}
 		return (short)v;
 	}
 
 	@Override
 	public Object createFromLong(long v) {
-		return (short)v;
-	}
-
-	@Override
-	public Object createFromFloat(float v) {
-		return (short)v;
-	}
-
-	@Override
-	public Object createFromDouble(double v) {
+		if(v > Short.MAX_VALUE) {
+			throw new MessageTypeException();
+		}
 		return (short)v;
 	}
 }
diff --git a/java/src/main/java/org/msgpack/schema/StringSchema.java b/java/src/main/java/org/msgpack/schema/StringSchema.java
index 46d515b..23e4e64 100644
--- a/java/src/main/java/org/msgpack/schema/StringSchema.java
+++ b/java/src/main/java/org/msgpack/schema/StringSchema.java
@@ -23,61 +23,48 @@ import java.io.UnsupportedEncodingException;
 import org.msgpack.*;
 
 public class StringSchema extends Schema {
-	public StringSchema() {
-		super("string");
-	}
+	public StringSchema() { }
 
 	@Override
-	public String getFullName() {
+	public String getClassName() {
 		return "String";
 	}
 
+	@Override
+	public String getExpression() {
+		return "string";
+	}
+
 	@Override
 	public void pack(Packer pk, Object obj) throws IOException {
-		// FIXME instanceof GenericObject
-		if(obj instanceof String) {
+		if(obj instanceof byte[]) {
+			byte[] b = (byte[])obj;
+			pk.packRaw(b.length);
+			pk.packRawBody(b);
+		} else if(obj instanceof String) {
 			try {
-				byte[] d = ((String)obj).getBytes("UTF-8");
-				pk.packRaw(d.length);
-				pk.packRawBody(d);
+				byte[] b = ((String)obj).getBytes("UTF-8");
+				pk.packRaw(b.length);
+				pk.packRawBody(b);
 			} catch (UnsupportedEncodingException e) {
-				throw MessageTypeException.invalidConvert(obj, this);
+				throw new MessageTypeException();
 			}
-
-		} else if(obj instanceof byte[]) {
-			byte[] d = (byte[])obj;
-			pk.packRaw(d.length);
-			pk.packRawBody(d);
-
-		} else if(obj instanceof ByteBuffer) {
-			ByteBuffer d = (ByteBuffer)obj;
-			if(!d.hasArray()) {
-				throw MessageTypeException.invalidConvert(obj, this);
-			}
-			pk.packRaw(d.capacity());
-			pk.packRawBody(d.array(), d.position(), d.capacity());
-
 		} else if(obj == null) {
 			pk.packNil();
-
 		} else {
 			throw MessageTypeException.invalidConvert(obj, this);
 		}
 	}
 
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		// FIXME instanceof GenericObject
-		if(obj instanceof String) {
-			return obj;
-
-		} else if(obj instanceof byte[]) {
+	public static final String convertString(Object obj) throws MessageTypeException {
+		if(obj instanceof byte[]) {
 			try {
 				return new String((byte[])obj, "UTF-8");
 			} catch (UnsupportedEncodingException e) {
-				throw MessageTypeException.invalidConvert(obj, this);
+				throw new MessageTypeException();
 			}
-
+		} else if(obj instanceof String) {
+			return (String)obj;
 		} else if(obj instanceof ByteBuffer) {
 			ByteBuffer d = (ByteBuffer)obj;
 			try {
@@ -91,14 +78,18 @@ public class StringSchema extends Schema {
 					return new String(v, "UTF-8");
 				}
 			} catch (UnsupportedEncodingException e) {
-				throw MessageTypeException.invalidConvert(obj, this);
+				throw new MessageTypeException();
 			}
-
 		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
+			throw new MessageTypeException();
 		}
 	}
 
+	@Override
+	public Object convert(Object obj) throws MessageTypeException {
+		return convertString(obj);
+	}
+
 	@Override
 	public Object createFromRaw(byte[] b, int offset, int length) {
 		try {

From 59603b902adf2abe0d274f41520569fde387a841 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 15 Jun 2010 17:51:24 +0900
Subject: [PATCH 0530/1648] Python: add "load(s)/dump(s)" alias for
 compatibility to simplejson/marshal/pickle.

---
 python/msgpack/__init__.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/python/msgpack/__init__.py b/python/msgpack/__init__.py
index 797b29c..26bd2dd 100644
--- a/python/msgpack/__init__.py
+++ b/python/msgpack/__init__.py
@@ -1,3 +1,10 @@
 # coding: utf-8
 from _msgpack import *
 
+# alias for compatibility to simplejson/marshal/pickle.
+load = unpack
+loads = unpackb
+
+dump = pack
+dumps = packb
+

From 73ac804b4f7b8d6e1d762e5565fbf10b187c7beb Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 15 Jun 2010 17:51:24 +0900
Subject: [PATCH 0531/1648] Python: add "load(s)/dump(s)" alias for
 compatibility to simplejson/marshal/pickle.

---
 msgpack/__init__.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 797b29c..26bd2dd 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,3 +1,10 @@
 # coding: utf-8
 from _msgpack import *
 
+# alias for compatibility to simplejson/marshal/pickle.
+load = unpack
+loads = unpackb
+
+dump = pack
+dumps = packb
+

From f222f5ed9b49e9b9f4d31693969c4260ad97cef0 Mon Sep 17 00:00:00 2001
From: Naoki INADA 
Date: Tue, 15 Jun 2010 18:06:58 +0900
Subject: [PATCH 0532/1648] Python: 0.1.4

---
 python/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index 8d8a6f4..64e71ed 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -14,7 +14,7 @@ except ImportError:
     from distutils.command.build_ext import build_ext
     have_cython = False
 
-version = '0.1.3'
+version = '0.1.4'
 
 # take care of extension modules.
 if have_cython:

From f0e6d33f7965908988a8023ed734188d86f4f448 Mon Sep 17 00:00:00 2001
From: Naoki INADA 
Date: Tue, 15 Jun 2010 18:06:58 +0900
Subject: [PATCH 0533/1648] Python: 0.1.4

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 8d8a6f4..64e71ed 100755
--- a/setup.py
+++ b/setup.py
@@ -14,7 +14,7 @@ except ImportError:
     from distutils.command.build_ext import build_ext
     have_cython = False
 
-version = '0.1.3'
+version = '0.1.4'
 
 # take care of extension modules.
 if have_cython:

From fd8069342052142449d4703f8a12014cf283fedf Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Tue, 22 Jun 2010 11:15:18 +0900
Subject: [PATCH 0534/1648] erlang: tests improved and code refined.

---
 erlang/OMakefile             |   2 +-
 erlang/msgpack.erl           | 116 ++++++++++++++++++-----------------
 erlang/testcase_generator.rb |  10 +--
 3 files changed, 68 insertions(+), 60 deletions(-)

diff --git a/erlang/OMakefile b/erlang/OMakefile
index ee72f78..34c590f 100644
--- a/erlang/OMakefile
+++ b/erlang/OMakefile
@@ -36,7 +36,7 @@ msgpack.beam: msgpack.erl
 	erlc $<
 
 test: msgpack.beam
-	erl -s msgpack test -s init stop
+	erl -noshell -s msgpack test -s init stop
 
 clean:
 	-rm *.beam
diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index ca9769e..5a468a1 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -31,10 +31,52 @@
 % erl> c(msgpack).
 % erl> S = .
 % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
-
-
 -type reason() ::  enomem.
 
+% ===== external APIs ===== %
+pack(O) when is_integer(O) andalso O < 0 ->
+    pack_int_(O);
+pack(O) when is_integer(O) ->
+    pack_uint_(O);
+pack(O) when is_float(O)->
+    pack_double(O);
+pack(nil) ->
+    pack_nil();
+pack(Bool) when is_atom(Bool) ->
+    pack_bool(Bool);
+pack(Bin) when is_binary(Bin)->
+    pack_raw(Bin);
+pack(List)  when is_list(List)->
+    pack_array(List);
+pack({dict, Map})->
+    pack_map({dict, Map});
+pack(_) ->
+    undefined.
+
+% unpacking.
+% if failed in decoding and not end, get more data
+% and feed more Bin into this function.
+% TODO: error case for imcomplete format when short for any type formats.
+-spec unpack( binary() )-> {term(), binary()} | {more, non_neg_integer()} | {error, reason()}.
+unpack(Bin) when not is_binary(Bin)->
+    {error, badard};
+unpack(Bin) when bit_size(Bin) >= 8 ->
+    << Flag:8/unsigned-integer, Payload/binary >> = Bin,
+    unpack_(Flag, Payload);
+unpack(_)-> % when bit_size(Bin) < 8 ->
+    {more, 8}.
+
+unpack_all(Data)->
+    case unpack(Data) of
+	{ Term, Binary } when bit_size(Binary) =:= 0 ->
+	    [Term];
+	{ Term, Binary } when is_binary(Binary) ->
+	    [Term|unpack_all(Binary)]
+    end.
+
+
+% ===== internal APIs ===== %
+
 % positive fixnum
 pack_uint_(N) when is_integer( N ) , N < 128 ->  
     << 2#0:1, N:7 >>;
@@ -113,7 +155,7 @@ pack_array(L) when is_list(L)->
     end.
 pack_array_([])-> <<>>;
 pack_array_([Head|Tail])->
-    << (pack_object(Head))/binary, (pack_array_(Tail))/binary >>.
+    << (pack(Head))/binary, (pack_array_(Tail))/binary >>.
 
 unpack_array_(<<>>, 0)-> [];
 unpack_array_(Remain, 0) when is_binary(Remain)-> [Remain];
@@ -134,7 +176,7 @@ pack_map({dict,M})->
 
 pack_map_([])-> <<>>;
 pack_map_([{Key,Value}|Tail]) ->
-    << (pack_object(Key)),(pack_object(Value)),(pack_map_(Tail)) >>.
+    << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>.
 
 unpack_map_(<<>>, 0)-> [];
 unpack_map_(Bin,  0) when is_binary(Bin)-> [Bin];
@@ -143,38 +185,7 @@ unpack_map_(Bin, Len) when is_binary(Bin) and is_integer(Len) ->
     { Value, Rest2 } = unpack(Rest),
     [{Key,Value}|unpack_map_(Rest2,Len-1)].
 
-pack_object(O) when is_integer(O) andalso O < 0 ->
-    pack_int_(O);
-pack_object(O) when is_integer(O) ->
-    pack_uint_(O);
-pack_object(O) when is_float(O)->
-    pack_double(O);
-pack_object(nil) ->
-    pack_nil();
-pack_object(Bool) when is_atom(Bool) ->
-    pack_bool(Bool);
-pack_object(Bin) when is_binary(Bin)->
-    pack_raw(Bin);
-pack_object(List)  when is_list(List)->
-    pack_array(List);
-pack_object({dict, Map})->
-    pack_map({dict, Map});
-pack_object(_) ->
-    undefined.
-
-pack(Obj)->
-    pack_object(Obj).
-
-
-% unpacking.
-% if failed in decoding and not end, get more data 
-% and feed more Bin into this function.
-% TODO: error case for imcomplete format when short for any type formats.
--spec unpack( binary() )-> {term(), binary()} | {more, non_neg_integer()} | {error, reason()}.
-unpack(Bin) when not is_binary(Bin)->
-    {error, badard};
-unpack(Bin) when bit_size(Bin) >= 8 ->
-    << Flag:8/unsigned-integer, Payload/binary >> = Bin,
+unpack_(Flag, Payload)->
     case Flag of 
 	16#C0 ->
 	    {nil, Payload};
@@ -305,20 +316,17 @@ unpack(Bin) when bit_size(Bin) >= 8 ->
 	_Other ->
 	    erlang:display(_Other),
 	    {error, no_code_matches}
-    end;
-unpack(_)-> % when bit_size(Bin) < 8 ->
-    {more, 8}.
-    
-unpack_all(Data)->
-    case unpack(Data) of
-	{ Term, Binary } when bit_size(Binary) =:= 0 -> 
-	    [Term];
-	{ Term, Binary } when is_binary(Binary) ->
-	    [Term|unpack_all(Binary)]
     end.
 
 -ifdef(EUNIT).
 
+compare_all([], [])-> ok;
+compare_all([], R)-> {toomuchrhs, R};
+compare_all(L, [])-> {toomuchlhs, L};
+compare_all([LH|LTL], [RH|RTL]) ->
+    LH=RH,
+    compare_all(LTL, RTL).
+
 test_data()->
     [0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF,
      -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF,
@@ -339,7 +347,6 @@ port_test()->
     {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])),
     Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]),
     true = port_command(Port, msgpack:pack(Tests) ),
-    %Port ! {self, {command, msgpack:pack(Tests)}}, ... not owner
     receive
 	{Port, {data, Data}}->  {Tests, <<>>}=msgpack:unpack(Data)
     after 1024-> ?assert(false)   end,
@@ -348,24 +355,23 @@ port_test()->
 unknown_test()->
     Tests = [0, 1, 2, 123, 512, 1230, 678908,
 	     -1, -23, -512, -1230, -567898,
-%	     "hogehoge", "243546rf7g68h798j",
-	     123.123 %-234.4355, 1.0e-34, 1.0e64,
-%	     [23, 234, 0.23]
-%	     [0,42,"sum", [1,2]], [1,42, nil, [3]]
+	     <<"hogehoge">>, <<"243546rf7g68h798j">>,
+%	     123.123,  %FIXME
+%            -234.4355, 1.0e-34, 1.0e64, % FIXME
+	     [23, 234, 0.23],
+	     [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]],
+	     42
 	    ],
     Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]),
-    %Port ! {self, {command, msgpack:pack(Tests)}}, ... not owner
     receive
 	{Port, {data, Data}}->
-	    Tests=msgpack:unpack_all(Data)
-%	    io:format("~p~n", [Tests])
+	    compare_all(Tests, msgpack:unpack_all(Data))
     after 1024-> ?assert(false)   end,
     port_close(Port).
 
 test_([]) -> 0;
 test_([S|Rest])->
     Pack = msgpack:pack(S),
-%    io:format("testing: ~p => ~p~n", [S, Pack]),
     {S, <<>>} = msgpack:unpack( Pack ),
     1+test_(Rest).
 
diff --git a/erlang/testcase_generator.rb b/erlang/testcase_generator.rb
index a173790..a7c76c5 100644
--- a/erlang/testcase_generator.rb
+++ b/erlang/testcase_generator.rb
@@ -39,10 +39,12 @@ end
 
 objs = [0, 1, 2, 123, 512, 1230, 678908,
         -1, -23, -512, -1230, -567898,
-#        "hogehoge", "243546rf7g68h798j",
-        123.123, #-234.4355, 1.0e-34, 1.0e64,
-#        [23, 234, 0.23]
-#        [0,42,"sum", [1,2]], [1,42, nil, [3]]
+        "hogehoge", "243546rf7g68h798j",
+#        123.123 , #FIXME
+#       -234.4355, 1.0e-34, 1.0e64,
+        [23, 234, 0.23],
+        [0,42,"sum", [1,2]], [1,42, nil, [3]],
+        42
        ]
 begin
   objs.each do |obj|

From b1e66256ce35ea4a953d255a2a98e9c8ddbe6401 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Tue, 22 Jun 2010 11:28:36 +0900
Subject: [PATCH 0535/1648] erlang: external APIs' type/specs.

---
 erlang/msgpack.erl | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 5a468a1..f2380fc 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -18,37 +18,38 @@
 -module(msgpack).
 -author('kuenishi+msgpack@gmail.com').
 
-%% tuples, atoms are not supported.  lists, integers, double, and so on.
+%% tuples, atoms are not supported. lists, integers, double, and so on.
 %% see http://msgpack.sourceforge.jp/spec for
 %% supported formats. APIs are almost compatible
 %% for C API (http://msgpack.sourceforge.jp/c:doc)
 %% except buffering functions (both copying and zero-copying).
--export([pack/1, unpack/1, unpack_all/1, test/0]).
-
--include_lib("eunit/include/eunit.hrl").
+-export([pack/1, unpack/1, unpack_all/1]).
 
 % compile:
 % erl> c(msgpack).
 % erl> S = .
 % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
--type reason() ::  enomem.
+-type reason() ::  enomem | badarg.
+-type map() :: any(). % there's no 'dict' type...
+-type msgpack_term() :: [msgpack_term()] | integer() | float() | {dict, map()}.
 
 % ===== external APIs ===== %
+-spec pack(Term::msgpack_term()) -> binary().
 pack(O) when is_integer(O) andalso O < 0 ->
     pack_int_(O);
 pack(O) when is_integer(O) ->
     pack_uint_(O);
-pack(O) when is_float(O)->
+pack(O) when is_float(O) ->
     pack_double(O);
 pack(nil) ->
     pack_nil();
 pack(Bool) when is_atom(Bool) ->
     pack_bool(Bool);
-pack(Bin) when is_binary(Bin)->
+pack(Bin) when is_binary(Bin) ->
     pack_raw(Bin);
-pack(List)  when is_list(List)->
+pack(List)  when is_list(List) ->
     pack_array(List);
-pack({dict, Map})->
+pack({dict, Map}) ->
     pack_map({dict, Map});
 pack(_) ->
     undefined.
@@ -57,15 +58,16 @@ pack(_) ->
 % if failed in decoding and not end, get more data
 % and feed more Bin into this function.
 % TODO: error case for imcomplete format when short for any type formats.
--spec unpack( binary() )-> {term(), binary()} | {more, non_neg_integer()} | {error, reason()}.
+-spec unpack( binary() )-> {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}.
 unpack(Bin) when not is_binary(Bin)->
-    {error, badard};
+    {error, badarg};
 unpack(Bin) when bit_size(Bin) >= 8 ->
     << Flag:8/unsigned-integer, Payload/binary >> = Bin,
     unpack_(Flag, Payload);
 unpack(_)-> % when bit_size(Bin) < 8 ->
     {more, 8}.
 
+-spec unpack_all( binary() ) -> [msgpack_term()].
 unpack_all(Data)->
     case unpack(Data) of
 	{ Term, Binary } when bit_size(Binary) =:= 0 ->
@@ -318,6 +320,8 @@ unpack_(Flag, Payload)->
 	    {error, no_code_matches}
     end.
 
+% ===== test codes ===== %
+-include_lib("eunit/include/eunit.hrl").
 -ifdef(EUNIT).
 
 compare_all([], [])-> ok;

From 230ee3a03b5148d11c614fc18c7c7603c9a8af14 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Wed, 23 Jun 2010 01:26:10 +0900
Subject: [PATCH 0536/1648] erlang: too short binary to decode causes error
 {more, Int}.

---
 erlang/msgpack.erl | 211 +++++++++++++++++++++++++++------------------
 1 file changed, 125 insertions(+), 86 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index f2380fc..d2f7068 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -29,7 +29,7 @@
 % erl> c(msgpack).
 % erl> S = .
 % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
--type reason() ::  enomem | badarg.
+-type reason() ::  enomem | badarg | no_code_matches.
 -type map() :: any(). % there's no 'dict' type...
 -type msgpack_term() :: [msgpack_term()] | integer() | float() | {dict, map()}.
 
@@ -58,14 +58,15 @@ pack(_) ->
 % if failed in decoding and not end, get more data
 % and feed more Bin into this function.
 % TODO: error case for imcomplete format when short for any type formats.
--spec unpack( binary() )-> {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}.
+-spec unpack( binary() )-> 
+    {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}.
 unpack(Bin) when not is_binary(Bin)->
     {error, badarg};
 unpack(Bin) when bit_size(Bin) >= 8 ->
     << Flag:8/unsigned-integer, Payload/binary >> = Bin,
     unpack_(Flag, Payload);
-unpack(_)-> % when bit_size(Bin) < 8 ->
-    {more, 8}.
+unpack(<<>>)-> % when bit_size(Bin) < 8 ->
+    {more, 1}.
 
 -spec unpack_all( binary() ) -> [msgpack_term()].
 unpack_all(Data)->
@@ -129,12 +130,9 @@ pack_bool(false)->   << 16#C2:8 >>.
 pack_double(F) when is_float(F)->
     << 16#CB:8, F:64/big-float-unit:1 >>.
 
-power(N,0) when is_integer(N) -> 1;
-power(N,D) when is_integer(N) and is_integer(D) -> N * power(N, D-1).
-
 % raw bytes
 pack_raw(Bin) when is_binary(Bin)->
-    MaxLen = power(2,16),
+    MaxLen = 16#10000, % 65536
     case byte_size(Bin) of
 	Len when Len < 6->
 	    << 2#101:3, Len:5, Bin/binary >>;
@@ -146,7 +144,7 @@ pack_raw(Bin) when is_binary(Bin)->
 
 % list / tuple
 pack_array(L) when is_list(L)->
-    MaxLen = power(2,16),
+    MaxLen = 16#10000, %65536
     case length(L) of
  	Len when Len < 16 ->
  	    << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L))/binary >>;
@@ -159,14 +157,19 @@ pack_array_([])-> <<>>;
 pack_array_([Head|Tail])->
     << (pack(Head))/binary, (pack_array_(Tail))/binary >>.
 
-unpack_array_(<<>>, 0)-> [];
-unpack_array_(Remain, 0) when is_binary(Remain)-> [Remain];
-unpack_array_(Bin, RestLen) when is_binary(Bin)->
-    {Term, Rest} = unpack(Bin),
-    [Term|unpack_array_(Rest, RestLen-1)].
+% FIXME! this should be tail-recursive and without lists:reverse/1
+unpack_array_(<<>>, 0, RetList)   -> {lists:reverse(RetList), <<>>};
+unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain};
+unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 ->  {more, RestLen};
+unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
+    case unpack(Bin) of
+	{more, Len} -> {more, Len+RestLen-1};
+	{Term, Rest}->
+	    unpack_array_(Rest, RestLen-1, [Term|RetList])
+    end.
     
 pack_map({dict,M})->
-    MaxLen = power(2,16),
+    MaxLen = 16#10000, %65536
     case dict:size(M) of
 	Len when Len < 16 ->
  	    << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>;
@@ -180,14 +183,25 @@ pack_map_([])-> <<>>;
 pack_map_([{Key,Value}|Tail]) ->
     << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>.
 
-unpack_map_(<<>>, 0)-> [];
-unpack_map_(Bin,  0) when is_binary(Bin)-> [Bin];
-unpack_map_(Bin, Len) when is_binary(Bin) and is_integer(Len) ->
-    { Key, Rest } = unpack(Bin),
-    { Value, Rest2 } = unpack(Rest),
-    [{Key,Value}|unpack_map_(Rest2,Len-1)].
+-spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])->
+    {more, non_neg_integer()} | { any(), binary()}.
+unpack_map_(Bin,  0,  TmpMap) when is_binary(Bin) -> { dict:from_list(TmpMap), Bin};
+unpack_map_(Bin, Len, TmpMap) when is_binary(Bin) and is_integer(Len) ->
+    case unpack(Bin) of
+	{ more, MoreLen } -> { more, MoreLen+Len-1 };
+	{ Key, Rest } ->
+	    case unpack(Rest) of
+		{more, MoreLen} -> { more, MoreLen+Len-1 };
+		{ Value, Rest2 }->
+		    unpack_map_(Rest2,Len-1,[{Key,Value}|TmpMap])
+	    end
+    end.
 
+% {more, 
+-spec unpack_(Flag::integer(), Payload::binary())->
+    {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}.
 unpack_(Flag, Payload)->
+    PayloadLen = byte_size(Payload),
     case Flag of 
 	16#C0 ->
 	    {nil, Payload};
@@ -196,86 +210,101 @@ unpack_(Flag, Payload)->
 	16#C3 ->
 	    {true, Payload};
 
-	16#CA -> % 32bit float
+	16#CA when PayloadLen >= 4 -> % 32bit float
 	    << Return:32/float-unit:1, Rest/binary >> = Payload,
 	    {Return, Rest};
-	16#CB -> % 64bit float
+	16#CA ->
+	    {more, 4-PayloadLen}; % at least more
+
+	16#CB when PayloadLen >= 8 -> % 64bit float
 	    << Return:64/float-unit:1, Rest/binary >> = Payload,
 	    {Return, Rest};
+	16#CB ->
+	    {more, 8-PayloadLen};
 
-	16#CC -> % uint 8
+	16#CC when PayloadLen >= 1 -> % uint 8
 	    << Int:8/unsigned-integer, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#CD -> % uint 16
+	16#CC ->
+	    {more, 1};
+
+	16#CD when PayloadLen >= 2 -> % uint 16
 	    << Int:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#CE ->
+	16#CD ->
+	    {more, 2-PayloadLen};
+
+	16#CE when PayloadLen >= 4 ->
 	    << Int:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#CF ->
+	16#CE ->
+	    {more, 4-PayloadLen}; % at least more
+
+	16#CF when PayloadLen >= 8 ->
 	    << Int:64/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
+	16#CF ->
+	    {more, 8-PayloadLen};
 
-	16#D0 -> % int 8
+	16#D0 when PayloadLen >= 1 -> % int 8
 	    << Int:8/big-signed-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#D1 -> % int 16
+	16#D0 ->
+	    {more, 1};
+
+	16#D1 when PayloadLen >= 2 -> % int 16
 	    << Int:16/big-signed-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#D2 -> % int 32
+	16#D1 ->
+	    {more, 2-PayloadLen};
+
+	16#D2 when PayloadLen >= 4 -> % int 32
 	    << Int:32/big-signed-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#D3 -> % int 64
+	16#D2 ->
+	    {more, 4-PayloadLen};
+
+	16#D3 when PayloadLen >= 8 -> % int 64
 	    << Int:64/big-signed-integer-unit:1, Rest/binary >> = Payload,
 	    {Int, Rest};
-	16#DA -> % raw 16
+	16#D3 ->
+	    {more, 8-PayloadLen};
+
+	16#DA when PayloadLen >= 2 -> % raw 16
 	    << Len:16/unsigned-integer-unit:1, Rest/binary >> = Payload,
 	    << Return:Len/binary, Remain/binary >> = Rest,
 	    {Return, Remain};
-	16#DB -> % raw 32
+	16#DA ->
+	    {more, 16-PayloadLen};
+
+	16#DB when PayloadLen >= 4 -> % raw 32
 	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
 	    << Return:Len/binary, Remain/binary >> = Rest,
 	    {Return, Remain};
-	16#DC -> % array 16
+	16#DB ->
+	    {more, 4-PayloadLen};
+
+	16#DC when PayloadLen >= 2 -> % array 16
 	    << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    Array=unpack_array_(Rest, Len),
-	    case length(Array) of
-		Len -> {Array, <<>>};
-		_ -> 
-		    {Return, RemainRest} = lists:split(Len, Array),
-		    [Remain] = RemainRest,
-		    {Return, Remain}
-	    end;
-	16#DD -> % array 32
+	    unpack_array_(Rest, Len, []);
+	16#DC ->
+	    {more, 2-PayloadLen};
+
+	16#DD when PayloadLen >= 4 -> % array 32
 	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    Array=unpack_array_(Rest, Len),
-	    case length(Array) of
-		Len -> {Array, <<>>};
-		_ -> 
-		    {Return, RemainRest} = lists:split(Len, Array),
-		    [Remain] = RemainRest,
-		    {Return, Remain}
-	    end;
-	16#DE -> % map 16
+	    unpack_array_(Rest, Len, []);
+	16#DD ->
+	    {more, 4-PayloadLen};
+
+	16#DE when PayloadLen >= 2 -> % map 16
 	    << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    Array=unpack_map_(Rest, Len),
-	    case length(Array) of
-		Len -> { dict:from_list(Array), <<>>};
-		_ -> 
-		    {Return, RemainRest} = lists:split(Len, Array),
-		    [Remain] = RemainRest,
-		    {dict:from_list(Return), Remain}
-	    end;
-	16#DF -> % map 32
+	    unpack_map_(Rest, Len, []);
+	16#DE ->
+	    {more, 2-PayloadLen};
+
+	16#DF when PayloadLen >= 4 -> % map 32
 	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    Array=unpack_map_(Rest, Len),
-	    case length(Array) of
-		Len -> { dict:from_list(Array), <<>>};
-		_ -> 
-		    {Return, RemainRest} = lists:split(Len, Array),
-		    [Remain] = RemainRest,
-		    {dict:from_list(Return), Remain}
-	    end;
+	    unpack_map_(Rest, Len, []);
 
 	% positive fixnum
 	Code when Code >= 2#00000000, Code < 2#10000000->
@@ -294,29 +323,15 @@ unpack_(Flag, Payload)->
 	Code when Code >= 2#10010000 , Code < 2#10100000 ->
 %        1001XXXX for FixArray
 	    Len = Code rem 2#10010000,
-	    Array=unpack_array_(Payload, Len),
-	    case length(Array) of
-		Len -> { Array, <<>>};
-		_ -> 
-		    {Return, RemainRest} = lists:split(Len, Array),
-		    [Remain] = RemainRest,
-		    {Return, Remain}
-	    end;
+	    unpack_array_(Payload, Len, []);
 
 	Code when Code >= 2#10000000 , Code < 2#10010000 ->
 %        1000XXXX for FixMap
 	    Len = Code rem 2#10000000,
-	    Array=unpack_map_(Payload, Len),
-	    case length(Array) of
-		Len -> { dict:from_list(Array), <<>>};
-		_ -> 
-		    {Return, RemainRest} = lists:split(Len, Array),
-		    [Remain] = RemainRest,
-		    {dict:from_list(Return), Remain}
-	    end;
+	    unpack_map_(Payload, Len, []);
 
 	_Other ->
-	    erlang:display(_Other),
+%	    erlang:display(_Other),
 	    {error, no_code_matches}
     end.
 
@@ -356,6 +371,26 @@ port_test()->
     after 1024-> ?assert(false)   end,
     port_close(Port).
 
+test_p(Len,Term,OrigBin,Len) ->
+    {Term, <<>>}=msgpack:unpack(OrigBin);
+test_p(I,_,OrigBin,Len) ->
+    <> = OrigBin,
+    {more, N}=msgpack:unpack(Bin),
+    ?assert(0 < N),
+    ?assert(N < Len).
+
+partial_test()-> % error handling test.
+    Term = lists:seq(0, 45),
+    Bin=msgpack:pack(Term),
+    BinLen = byte_size(Bin),
+    [test_p(X, Term, Bin, BinLen) || X <- lists:seq(0,BinLen)].
+
+long_test()->
+    Longer = lists:seq(0, 655), %55),
+%%     Longest = lists:seq(0,12345),
+    {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)).
+%%     {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)).
+
 unknown_test()->
     Tests = [0, 1, 2, 123, 512, 1230, 678908,
 	     -1, -23, -512, -1230, -567898,
@@ -377,6 +412,10 @@ test_([]) -> 0;
 test_([S|Rest])->
     Pack = msgpack:pack(S),
     {S, <<>>} = msgpack:unpack( Pack ),
+%    ?debugVal( hoge ),
     1+test_(Rest).
 
+other_test()->
+    {more,1}=msgpack:unpack(<<>>).
+
 -endif.

From bc0c5f0cdca4d5244b3237b63305cfd01d1e9a9d Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Wed, 23 Jun 2010 09:02:53 +0900
Subject: [PATCH 0537/1648] erlang: (un)pack_map improved, incremental
 unpacking

---
 erlang/msgpack.erl | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index d2f7068..764de11 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -167,7 +167,8 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
 	{Term, Rest}->
 	    unpack_array_(Rest, RestLen-1, [Term|RetList])
     end.
-    
+
+% FIXME: write test for pack_map/1
 pack_map({dict,M})->
     MaxLen = 16#10000, %65536
     case dict:size(M) of
@@ -183,17 +184,18 @@ pack_map_([])-> <<>>;
 pack_map_([{Key,Value}|Tail]) ->
     << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>.
 
+% FIXME: write test for unpack_map/1
 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])->
     {more, non_neg_integer()} | { any(), binary()}.
-unpack_map_(Bin,  0,  TmpMap) when is_binary(Bin) -> { dict:from_list(TmpMap), Bin};
-unpack_map_(Bin, Len, TmpMap) when is_binary(Bin) and is_integer(Len) ->
+unpack_map_(Bin,  0,  Dict) when is_binary(Bin) -> { {dict, Dict}, Bin};
+unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) ->
     case unpack(Bin) of
 	{ more, MoreLen } -> { more, MoreLen+Len-1 };
 	{ Key, Rest } ->
 	    case unpack(Rest) of
 		{more, MoreLen} -> { more, MoreLen+Len-1 };
 		{ Value, Rest2 }->
-		    unpack_map_(Rest2,Len-1,[{Key,Value}|TmpMap])
+		    unpack_map_(Rest2,Len-1,dict:append(Key,Value,Dict))
 	    end
     end.
 
@@ -298,13 +300,13 @@ unpack_(Flag, Payload)->
 
 	16#DE when PayloadLen >= 2 -> % map 16
 	    << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_map_(Rest, Len, []);
+	    unpack_map_(Rest, Len, dict:new());
 	16#DE ->
 	    {more, 2-PayloadLen};
 
 	16#DF when PayloadLen >= 4 -> % map 32
 	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_map_(Rest, Len, []);
+	    unpack_map_(Rest, Len, dict:new());
 
 	% positive fixnum
 	Code when Code >= 2#00000000, Code < 2#10000000->
@@ -328,7 +330,7 @@ unpack_(Flag, Payload)->
 	Code when Code >= 2#10000000 , Code < 2#10010000 ->
 %        1000XXXX for FixMap
 	    Len = Code rem 2#10000000,
-	    unpack_map_(Payload, Len, []);
+	    unpack_map_(Payload, Len, dict:new());
 
 	_Other ->
 %	    erlang:display(_Other),
@@ -373,7 +375,7 @@ port_test()->
 
 test_p(Len,Term,OrigBin,Len) ->
     {Term, <<>>}=msgpack:unpack(OrigBin);
-test_p(I,_,OrigBin,Len) ->
+test_p(I,_,OrigBin,Len) when I < Len->
     <> = OrigBin,
     {more, N}=msgpack:unpack(Bin),
     ?assert(0 < N),

From 2cdfbd8970756742af7b378612756c59d184d86b Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Thu, 24 Jun 2010 07:26:34 +0900
Subject: [PATCH 0538/1648] erlang: testing pack_map/unpack_map with a silly
 bug

---
 erlang/msgpack.erl | 40 +++++++++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 764de11..dca5888 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -49,10 +49,10 @@ pack(Bin) when is_binary(Bin) ->
     pack_raw(Bin);
 pack(List)  when is_list(List) ->
     pack_array(List);
-pack({dict, Map}) ->
-    pack_map({dict, Map});
-pack(_) ->
-    undefined.
+pack(Map) when is_tuple(Map), element(1,Map)=:=dict ->
+    pack_map(Map);
+pack(_O) ->
+    {error, undefined}.
 
 % unpacking.
 % if failed in decoding and not end, get more data
@@ -169,25 +169,25 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
     end.
 
 % FIXME: write test for pack_map/1
-pack_map({dict,M})->
+pack_map(M)->
     MaxLen = 16#10000, %65536
     case dict:size(M) of
 	Len when Len < 16 ->
  	    << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>;
 	Len when Len < MaxLen ->
-	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M))) >>;
+	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>;
 	Len ->
-	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M))) >>
+	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>
     end.
 
 pack_map_([])-> <<>>;
 pack_map_([{Key,Value}|Tail]) ->
-    << (pack(Key)),(pack(Value)),(pack_map_(Tail)) >>.
+    << (pack(Key))/binary,(pack(Value))/binary,(pack_map_(Tail))/binary >>.
 
 % FIXME: write test for unpack_map/1
 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])->
     {more, non_neg_integer()} | { any(), binary()}.
-unpack_map_(Bin,  0,  Dict) when is_binary(Bin) -> { {dict, Dict}, Bin};
+unpack_map_(Bin,  0,  Dict) when is_binary(Bin) -> {Dict, Bin};
 unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) ->
     case unpack(Bin) of
 	{ more, MoreLen } -> { more, MoreLen+Len-1 };
@@ -333,7 +333,6 @@ unpack_(Flag, Payload)->
 	    unpack_map_(Payload, Len, dict:new());
 
 	_Other ->
-%	    erlang:display(_Other),
 	    {error, no_code_matches}
     end.
 
@@ -388,10 +387,25 @@ partial_test()-> % error handling test.
     [test_p(X, Term, Bin, BinLen) || X <- lists:seq(0,BinLen)].
 
 long_test()->
-    Longer = lists:seq(0, 655), %55),
-%%     Longest = lists:seq(0,12345),
-    {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)).
+    Longer = lists:seq(0, 65), %55),
+%    Longest = lists:seq(0,12345),
+    {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)),
+%    {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)).
+    ok.
+
+map_test()->
+    Ints = lists:seq(0, 65), %55),
+    Map = dict:from_list([ {X, X*2} || X <- Ints ]),
+    S=msgpack:pack(Map),
+%    ?debugVal(msgpack:unpack(S)),
+    {Map2, <<>>} = msgpack:unpack(S),
+    ?assertEqual(dict:size(Map), dict:size(Map2)),
+%    ?debugVal(dict:to_list(Map2)),
+    OrdMap = orddict:from_list( dict:to_list(Map) ),
+    OrdMap2 = orddict:from_list( dict:to_list(Map2) ),
+%    ?assertEqual(OrdMap, OrdMap2), % FIXME!! its a misery bug.
 %%     {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)).
+    ok.
 
 unknown_test()->
     Tests = [0, 1, 2, 123, 512, 1230, 678908,

From 92d192277e14be4f32135a85cbca233e7dca3182 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Fri, 25 Jun 2010 00:22:53 +0900
Subject: [PATCH 0539/1648] erlang: unpack_map's silly bug fixed. use
 dict:store/3....

---
 erlang/msgpack.erl           | 22 +++++++++-------------
 erlang/testcase_generator.rb |  5 +++--
 2 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index dca5888..0d4151b 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -195,7 +195,7 @@ unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) ->
 	    case unpack(Rest) of
 		{more, MoreLen} -> { more, MoreLen+Len-1 };
 		{ Value, Rest2 }->
-		    unpack_map_(Rest2,Len-1,dict:append(Key,Value,Dict))
+		    unpack_map_(Rest2,Len-1,dict:store(Key,Value,Dict))
 	    end
     end.
 
@@ -352,7 +352,7 @@ test_data()->
      -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF,
      123.123, -234.4355, 1.0e-34, 1.0e64,
      [23, 234, 0.23],
-     "hogehoge", "243546rf7g68h798j",
+     <<"hogehoge">>, <<"243546rf7g68h798j", 0, 23, 255>>,
      <<"hoasfdafdas][">>,
      [0,42,"sum", [1,2]], [1,42, nil, [3]]
     ].
@@ -387,34 +387,31 @@ partial_test()-> % error handling test.
     [test_p(X, Term, Bin, BinLen) || X <- lists:seq(0,BinLen)].
 
 long_test()->
-    Longer = lists:seq(0, 65), %55),
+    Longer = lists:seq(0, 655),
 %    Longest = lists:seq(0,12345),
     {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)),
 %    {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)).
     ok.
 
 map_test()->
-    Ints = lists:seq(0, 65), %55),
+    Ints = lists:seq(0, 65),
     Map = dict:from_list([ {X, X*2} || X <- Ints ]),
-    S=msgpack:pack(Map),
-%    ?debugVal(msgpack:unpack(S)),
-    {Map2, <<>>} = msgpack:unpack(S),
+    {Map2, <<>>} = msgpack:unpack(msgpack:pack(Map)),
     ?assertEqual(dict:size(Map), dict:size(Map2)),
-%    ?debugVal(dict:to_list(Map2)),
     OrdMap = orddict:from_list( dict:to_list(Map) ),
     OrdMap2 = orddict:from_list( dict:to_list(Map2) ),
-%    ?assertEqual(OrdMap, OrdMap2), % FIXME!! its a misery bug.
-%%     {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)).
+    ?assertEqual(OrdMap, OrdMap2),
     ok.
 
 unknown_test()->
     Tests = [0, 1, 2, 123, 512, 1230, 678908,
 	     -1, -23, -512, -1230, -567898,
 	     <<"hogehoge">>, <<"243546rf7g68h798j">>,
-%	     123.123,  %FIXME
-%            -234.4355, 1.0e-34, 1.0e64, % FIXME
+	     123.123,
+	     -234.4355, 1.0e-34, 1.0e64,
 	     [23, 234, 0.23],
 	     [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]],
+	     dict:from_list([{1,2},{<<"hoge">>,nil}]),
 	     42
 	    ],
     Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]),
@@ -428,7 +425,6 @@ test_([]) -> 0;
 test_([S|Rest])->
     Pack = msgpack:pack(S),
     {S, <<>>} = msgpack:unpack( Pack ),
-%    ?debugVal( hoge ),
     1+test_(Rest).
 
 other_test()->
diff --git a/erlang/testcase_generator.rb b/erlang/testcase_generator.rb
index a7c76c5..cfc36f9 100644
--- a/erlang/testcase_generator.rb
+++ b/erlang/testcase_generator.rb
@@ -40,10 +40,11 @@ end
 objs = [0, 1, 2, 123, 512, 1230, 678908,
         -1, -23, -512, -1230, -567898,
         "hogehoge", "243546rf7g68h798j",
-#        123.123 , #FIXME
-#       -234.4355, 1.0e-34, 1.0e64,
+        123.123,
+       -234.4355, 1.0e-34, 1.0e64,
         [23, 234, 0.23],
         [0,42,"sum", [1,2]], [1,42, nil, [3]],
+        { 1 => 2, "hoge" => nil },
         42
        ]
 begin

From 57f0598373ce542f7499218792a15fa0eee161f7 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Fri, 25 Jun 2010 00:44:14 +0900
Subject: [PATCH 0540/1648] erlang: code refined and tests added

---
 erlang/msgpack.erl | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 0d4151b..789ed53 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -116,8 +116,7 @@ pack_int_( N ) when is_integer( N )->
     << 16#D3:8, N:64/big-signed-integer-unit:1 >>.
 
 % nil
-pack_nil()->
-    << 16#C0:8 >>.
+pack_nil()->    << 16#C0:8 >>.
 % pack_true / pack_false
 pack_bool(true)->    << 16#C3:8 >>;
 pack_bool(false)->   << 16#C2:8 >>.
@@ -132,11 +131,10 @@ pack_double(F) when is_float(F)->
 
 % raw bytes
 pack_raw(Bin) when is_binary(Bin)->
-    MaxLen = 16#10000, % 65536
     case byte_size(Bin) of
 	Len when Len < 6->
 	    << 2#101:3, Len:5, Bin/binary >>;
-	Len when Len < MaxLen ->
+	Len when Len < 16#10000 -> % 65536
 	    << 16#DA:8, Len:16/big-unsigned-integer-unit:1, Bin/binary >>;
 	Len ->
 	    << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >>
@@ -144,11 +142,10 @@ pack_raw(Bin) when is_binary(Bin)->
 
 % list / tuple
 pack_array(L) when is_list(L)->
-    MaxLen = 16#10000, %65536
     case length(L) of
  	Len when Len < 16 ->
  	    << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L))/binary >>;
-	Len when Len < MaxLen ->
+	Len when Len < 16#10000 -> % 65536
 	    << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L))/binary >>;
 	Len ->
 	    << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L))/binary >>
@@ -164,17 +161,15 @@ unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 ->  {more, RestLen};
 unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
     case unpack(Bin) of
 	{more, Len} -> {more, Len+RestLen-1};
-	{Term, Rest}->
-	    unpack_array_(Rest, RestLen-1, [Term|RetList])
+	{Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList])
     end.
 
 % FIXME: write test for pack_map/1
 pack_map(M)->
-    MaxLen = 16#10000, %65536
     case dict:size(M) of
 	Len when Len < 16 ->
  	    << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>;
-	Len when Len < MaxLen ->
+	Len when Len < 16#10000 -> % 65536
 	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>;
 	Len ->
 	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>
@@ -348,13 +343,15 @@ compare_all([LH|LTL], [RH|RTL]) ->
     compare_all(LTL, RTL).
 
 test_data()->
-    [0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF,
+    [true, false, nil, 
+     0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF,
      -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF,
      123.123, -234.4355, 1.0e-34, 1.0e64,
      [23, 234, 0.23],
      <<"hogehoge">>, <<"243546rf7g68h798j", 0, 23, 255>>,
      <<"hoasfdafdas][">>,
-     [0,42,"sum", [1,2]], [1,42, nil, [3]]
+     [0,42, <<"sum">>, [1,2]], [1,42, nil, [3]],
+     42
     ].
 
 basic_test()->
@@ -395,7 +392,7 @@ long_test()->
 
 map_test()->
     Ints = lists:seq(0, 65),
-    Map = dict:from_list([ {X, X*2} || X <- Ints ]),
+    Map = dict:from_list([ {X, X*2} || X <- Ints ] ++ [{<<"hage">>, 324}, {43542, [nil, true, false]}]),
     {Map2, <<>>} = msgpack:unpack(msgpack:pack(Map)),
     ?assertEqual(dict:size(Map), dict:size(Map2)),
     OrdMap = orddict:from_list( dict:to_list(Map) ),

From ad052cb510e3be659df0322cbf33e4840c010b9b Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Fri, 25 Jun 2010 01:26:57 +0900
Subject: [PATCH 0541/1648] updated readme

---
 erlang/README.md | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/erlang/README.md b/erlang/README.md
index 50d446d..8616d5e 100644
--- a/erlang/README.md
+++ b/erlang/README.md
@@ -2,18 +2,8 @@ MessagePack for Erlang
 ======================
 Binary-based efficient object serialization library.
 
-## Status 
+see wiki ( http://redmine.msgpack.org/projects/msgpack/wiki/QuickStartErlang ) for details
 
-still in development.
+# Status
 
-TODOs:
- - decide string specification.
-
-## Installation
-
-## Example
-
-## License
-
-
- - 
+0.1.0 released.

From 0cca90c21deb2517de95c83a837bf90efc6b9fa0 Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Fri, 25 Jun 2010 17:32:11 +0200
Subject: [PATCH 0542/1648] Fix encoding of fixmap type. The tag value was
 wrong, and a missing /binary flag caused an error.

---
 erlang/msgpack.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 789ed53..bbf9e64 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -168,7 +168,7 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
 pack_map(M)->
     case dict:size(M) of
 	Len when Len < 16 ->
- 	    << 2#1001:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M))) >>;
+ 	    << 2#1000:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>;
 	Len when Len < 16#10000 -> % 65536
 	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>;
 	Len ->

From a1b2b41cdcb4bbc98e21bcd334b729ec6e7b90d5 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Sat, 26 Jun 2010 08:40:36 +0900
Subject: [PATCH 0543/1648] erlang: bugfix(serialization of -234 goes
 <<208,22>> while it should go int16 <<0xD1, ...>>)

---
 erlang/msgpack.erl           | 20 ++++++++++----------
 erlang/testcase_generator.rb |  1 +
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 789ed53..63d648c 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -86,15 +86,12 @@ pack_uint_(N) when is_integer( N ) , N < 128 ->
 % uint 8
 pack_uint_( N ) when is_integer( N ) andalso N < 256 ->
     << 16#CC:8, N:8 >>;
-
 % uint 16
 pack_uint_( N ) when is_integer( N ) andalso N < 65536 ->
     << 16#CD:8, N:16/big-unsigned-integer-unit:1 >>;
-
 % uint 32
 pack_uint_( N ) when is_integer( N ) andalso N < 16#FFFFFFFF->
     << 16#CE:8, N:32/big-unsigned-integer-unit:1 >>;
-
 % uint 64
 pack_uint_( N ) when is_integer( N )->
     << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>.
@@ -103,13 +100,13 @@ pack_uint_( N ) when is_integer( N )->
 pack_int_( N ) when is_integer( N ) , N >= -32->
     << 2#111:3, N:5 >>;
 % int 8
-pack_int_( N ) when is_integer( N ) , N >= -256 ->
-    << 16#D0:8, N:8 >>;
+pack_int_( N ) when is_integer( N ) , N > -128 ->
+    << 16#D0:8, N:8/big-signed-integer-unit:1 >>;
 % int 16
-pack_int_( N ) when is_integer( N ), N >= -65536 ->
+pack_int_( N ) when is_integer( N ), N > -32768 ->
     << 16#D1:8, N:16/big-signed-integer-unit:1 >>;
 % int 32
-pack_int_( N ) when is_integer( N ), N >= -16#FFFFFFFF ->
+pack_int_( N ) when is_integer( N ), N > -16#FFFFFFFF ->
     << 16#D2:8, N:32/big-signed-integer-unit:1 >>;
 % int 64
 pack_int_( N ) when is_integer( N )->
@@ -351,6 +348,7 @@ test_data()->
      <<"hogehoge">>, <<"243546rf7g68h798j", 0, 23, 255>>,
      <<"hoasfdafdas][">>,
      [0,42, <<"sum">>, [1,2]], [1,42, nil, [3]],
+     -234, -40000, -16#10000000, -16#100000000,
      42
     ].
 
@@ -409,6 +407,7 @@ unknown_test()->
 	     [23, 234, 0.23],
 	     [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]],
 	     dict:from_list([{1,2},{<<"hoge">>,nil}]),
+	     -234, -50000,
 	     42
 	    ],
     Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]),
@@ -419,9 +418,10 @@ unknown_test()->
     port_close(Port).
 
 test_([]) -> 0;
-test_([S|Rest])->
-    Pack = msgpack:pack(S),
-    {S, <<>>} = msgpack:unpack( Pack ),
+test_([Before|Rest])->
+    Pack = msgpack:pack(Before),
+    {After, <<>>} = msgpack:unpack( Pack ),
+    ?assertEqual(Before, After),
     1+test_(Rest).
 
 other_test()->
diff --git a/erlang/testcase_generator.rb b/erlang/testcase_generator.rb
index cfc36f9..8445bdd 100644
--- a/erlang/testcase_generator.rb
+++ b/erlang/testcase_generator.rb
@@ -45,6 +45,7 @@ objs = [0, 1, 2, 123, 512, 1230, 678908,
         [23, 234, 0.23],
         [0,42,"sum", [1,2]], [1,42, nil, [3]],
         { 1 => 2, "hoge" => nil },
+        -234, -50000,
         42
        ]
 begin

From 279121f87f322f3c1a2430751eb3a1032030853d Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Mon, 28 Jun 2010 11:56:12 +0200
Subject: [PATCH 0544/1648] erlang: Use a simple proplist instead of a dict.   
      A dict is overkill (code, cpu, memory) in most cases, and
 proplist<->dict conversion can easily be done by the libray user if desired. 
        This is in line with other erlang libraries I've seen for various
 encoding schemes.         The map encoder had a bug until I looked at it (see
 previous commit), so I guess it wasn't used much yet and a change is ok at
 this stage.         The chosen representation for maps is a tuple containing
 the proplist as the only element.

---
 erlang/msgpack.erl | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index bbf9e64..bc63677 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -30,8 +30,7 @@
 % erl> S = .
 % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
 -type reason() ::  enomem | badarg | no_code_matches.
--type map() :: any(). % there's no 'dict' type...
--type msgpack_term() :: [msgpack_term()] | integer() | float() | {dict, map()}.
+-type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float().
 
 % ===== external APIs ===== %
 -spec pack(Term::msgpack_term()) -> binary().
@@ -49,8 +48,10 @@ pack(Bin) when is_binary(Bin) ->
     pack_raw(Bin);
 pack(List)  when is_list(List) ->
     pack_array(List);
-pack(Map) when is_tuple(Map), element(1,Map)=:=dict ->
+pack({Map}) when is_list(Map) ->
     pack_map(Map);
+pack(Map) when is_tuple(Map), element(1,Map)=:=dict ->
+    pack_map(dict:from_list(Map));
 pack(_O) ->
     {error, undefined}.
 
@@ -166,13 +167,13 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
 
 % FIXME: write test for pack_map/1
 pack_map(M)->
-    case dict:size(M) of
+    case length(M) of
 	Len when Len < 16 ->
- 	    << 2#1000:4, Len:4/integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>;
+ 	    << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M))/binary >>;
 	Len when Len < 16#10000 -> % 65536
-	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>;
+	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M))/binary >>;
 	Len ->
-	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(dict:to_list(M)))/binary >>
+	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M))/binary >>
     end.
 
 pack_map_([])-> <<>>;
@@ -182,15 +183,15 @@ pack_map_([{Key,Value}|Tail]) ->
 % FIXME: write test for unpack_map/1
 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])->
     {more, non_neg_integer()} | { any(), binary()}.
-unpack_map_(Bin,  0,  Dict) when is_binary(Bin) -> {Dict, Bin};
-unpack_map_(Bin, Len, Dict) when is_binary(Bin) and is_integer(Len) ->
+unpack_map_(Bin,  0,  Acc) when is_binary(Bin) -> {{lists:reverse(Acc)}, Bin};
+unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) ->
     case unpack(Bin) of
 	{ more, MoreLen } -> { more, MoreLen+Len-1 };
 	{ Key, Rest } ->
 	    case unpack(Rest) of
 		{more, MoreLen} -> { more, MoreLen+Len-1 };
-		{ Value, Rest2 }->
-		    unpack_map_(Rest2,Len-1,dict:store(Key,Value,Dict))
+		{ Value, Rest2 } ->
+		    unpack_map_(Rest2,Len-1,[{Key,Value}|Acc])
 	    end
     end.
 
@@ -295,13 +296,13 @@ unpack_(Flag, Payload)->
 
 	16#DE when PayloadLen >= 2 -> % map 16
 	    << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_map_(Rest, Len, dict:new());
+	    unpack_map_(Rest, Len, []);
 	16#DE ->
 	    {more, 2-PayloadLen};
 
 	16#DF when PayloadLen >= 4 -> % map 32
 	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_map_(Rest, Len, dict:new());
+	    unpack_map_(Rest, Len, []);
 
 	% positive fixnum
 	Code when Code >= 2#00000000, Code < 2#10000000->
@@ -325,7 +326,7 @@ unpack_(Flag, Payload)->
 	Code when Code >= 2#10000000 , Code < 2#10010000 ->
 %        1000XXXX for FixMap
 	    Len = Code rem 2#10000000,
-	    unpack_map_(Payload, Len, dict:new());
+	    unpack_map_(Payload, Len, []);
 
 	_Other ->
 	    {error, no_code_matches}
@@ -392,12 +393,9 @@ long_test()->
 
 map_test()->
     Ints = lists:seq(0, 65),
-    Map = dict:from_list([ {X, X*2} || X <- Ints ] ++ [{<<"hage">>, 324}, {43542, [nil, true, false]}]),
+    Map = {[ {X, X*2} || X <- Ints ] ++ [{<<"hage">>, 324}, {43542, [nil, true, false]}]},
     {Map2, <<>>} = msgpack:unpack(msgpack:pack(Map)),
-    ?assertEqual(dict:size(Map), dict:size(Map2)),
-    OrdMap = orddict:from_list( dict:to_list(Map) ),
-    OrdMap2 = orddict:from_list( dict:to_list(Map2) ),
-    ?assertEqual(OrdMap, OrdMap2),
+    ?assertEqual(Map, Map2),
     ok.
 
 unknown_test()->

From 537322e3b59978d26fcf096433fc718c16cc8b84 Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Mon, 28 Jun 2010 14:17:44 +0200
Subject: [PATCH 0545/1648] Big speedup (around 40%) of maps and arrays
 encoding by using proper tail recursion.

---
 erlang/msgpack.erl | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index bc63677..255542b 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -59,7 +59,7 @@ pack(_O) ->
 % if failed in decoding and not end, get more data
 % and feed more Bin into this function.
 % TODO: error case for imcomplete format when short for any type formats.
--spec unpack( binary() )-> 
+-spec unpack( binary() )->
     {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}.
 unpack(Bin) when not is_binary(Bin)->
     {error, badarg};
@@ -82,7 +82,7 @@ unpack_all(Data)->
 % ===== internal APIs ===== %
 
 % positive fixnum
-pack_uint_(N) when is_integer( N ) , N < 128 ->  
+pack_uint_(N) when is_integer( N ) , N < 128 ->
     << 2#0:1, N:7 >>;
 % uint 8
 pack_uint_( N ) when is_integer( N ) andalso N < 256 ->
@@ -145,15 +145,15 @@ pack_raw(Bin) when is_binary(Bin)->
 pack_array(L) when is_list(L)->
     case length(L) of
  	Len when Len < 16 ->
- 	    << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L))/binary >>;
+ 	    << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L, <<>>))/binary >>;
 	Len when Len < 16#10000 -> % 65536
-	    << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L))/binary >>;
+	    << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >>;
 	Len ->
-	    << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L))/binary >>
+	    << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >>
     end.
-pack_array_([])-> <<>>;
-pack_array_([Head|Tail])->
-    << (pack(Head))/binary, (pack_array_(Tail))/binary >>.
+pack_array_([], Acc) -> Acc;
+pack_array_([Head|Tail], Acc) ->
+    pack_array_(Tail, <>).
 
 % FIXME! this should be tail-recursive and without lists:reverse/1
 unpack_array_(<<>>, 0, RetList)   -> {lists:reverse(RetList), <<>>};
@@ -169,16 +169,16 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
 pack_map(M)->
     case length(M) of
 	Len when Len < 16 ->
- 	    << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M))/binary >>;
+ 	    << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>;
 	Len when Len < 16#10000 -> % 65536
-	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M))/binary >>;
+	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>;
 	Len ->
-	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M))/binary >>
+	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>
     end.
 
-pack_map_([])-> <<>>;
-pack_map_([{Key,Value}|Tail]) ->
-    << (pack(Key))/binary,(pack(Value))/binary,(pack_map_(Tail))/binary >>.
+pack_map_([], Acc) -> Acc;
+pack_map_([{Key,Value}|Tail], Acc) ->
+    pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>).
 
 % FIXME: write test for unpack_map/1
 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])->
@@ -200,7 +200,7 @@ unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) ->
     {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}.
 unpack_(Flag, Payload)->
     PayloadLen = byte_size(Payload),
-    case Flag of 
+    case Flag of
 	16#C0 ->
 	    {nil, Payload};
 	16#C2 ->
@@ -313,7 +313,7 @@ unpack_(Flag, Payload)->
 	    {(Code - 16#100), Payload};
 
 	Code when Code >= 2#10100000 , Code < 2#11000000 ->
-%	 101XXXXX for FixRaw 
+%	 101XXXXX for FixRaw
 	    Len = Code rem 2#10100000,
 	    << Return:Len/binary, Remain/binary >> = Payload,
 	    {Return, Remain};
@@ -344,7 +344,7 @@ compare_all([LH|LTL], [RH|RTL]) ->
     compare_all(LTL, RTL).
 
 test_data()->
-    [true, false, nil, 
+    [true, false, nil,
      0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF,
      -1, -23, -512, -1230, -567898, -16#FFFFFFFFFF,
      123.123, -234.4355, 1.0e-34, 1.0e64,

From b471e52e281d189396a78cef3eb7ecacbab51d21 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Tue, 29 Jun 2010 00:21:47 +0900
Subject: [PATCH 0546/1648] erlang: explicit API for serializing proplists,    
     so as not to make wrong call of pack({proplists()}).

---
 erlang/msgpack.erl | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index c807065..29e1728 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -24,6 +24,7 @@
 %% for C API (http://msgpack.sourceforge.jp/c:doc)
 %% except buffering functions (both copying and zero-copying).
 -export([pack/1, unpack/1, unpack_all/1]).
+-export([pack_map/1])
 
 % compile:
 % erl> c(msgpack).
@@ -51,7 +52,7 @@ pack(List)  when is_list(List) ->
 pack({Map}) when is_list(Map) ->
     pack_map(Map);
 pack(Map) when is_tuple(Map), element(1,Map)=:=dict ->
-    pack_map(dict:from_list(Map));
+    pack_map(dict:to_list(Map));
 pack(_O) ->
     {error, undefined}.
 
@@ -78,6 +79,15 @@ unpack_all(Data)->
 	    [Term|unpack_all(Binary)]
     end.
 
+pack_map(M)->
+    case length(M) of
+	Len when Len < 16 ->
+	    << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>;
+	Len when Len < 16#10000 -> % 65536
+	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>;
+	Len ->
+	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>
+    end.
 
 % ===== internal APIs ===== %
 
@@ -162,17 +172,6 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
 	{Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList])
     end.
 
-% FIXME: write test for pack_map/1
-pack_map(M)->
-    case length(M) of
-	Len when Len < 16 ->
- 	    << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>;
-	Len when Len < 16#10000 -> % 65536
-	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>;
-	Len ->
-	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>
-    end.
-
 pack_map_([], Acc) -> Acc;
 pack_map_([{Key,Value}|Tail], Acc) ->
     pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>).
@@ -404,7 +403,7 @@ unknown_test()->
 	     -234.4355, 1.0e-34, 1.0e64,
 	     [23, 234, 0.23],
 	     [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]],
-	     dict:from_list([{1,2},{<<"hoge">>,nil}]),
+	     {[{1,2},{<<"hoge">>,nil}]},
 	     -234, -50000,
 	     42
 	    ],

From 90e305d789d231258e26a2dc546cfdb6f8c09ce8 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Tue, 29 Jun 2010 00:21:47 +0900
Subject: [PATCH 0547/1648] erlang: explicit API for serializing proplists,    
     so as not to make wrong call of pack({proplists()}).

---
 erlang/msgpack.erl | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index c807065..a168b55 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -24,6 +24,7 @@
 %% for C API (http://msgpack.sourceforge.jp/c:doc)
 %% except buffering functions (both copying and zero-copying).
 -export([pack/1, unpack/1, unpack_all/1]).
+-export([pack_map/1]).
 
 % compile:
 % erl> c(msgpack).
@@ -51,7 +52,7 @@ pack(List)  when is_list(List) ->
 pack({Map}) when is_list(Map) ->
     pack_map(Map);
 pack(Map) when is_tuple(Map), element(1,Map)=:=dict ->
-    pack_map(dict:from_list(Map));
+    pack_map(dict:to_list(Map));
 pack(_O) ->
     {error, undefined}.
 
@@ -78,6 +79,15 @@ unpack_all(Data)->
 	    [Term|unpack_all(Binary)]
     end.
 
+pack_map(M)->
+    case length(M) of
+	Len when Len < 16 ->
+	    << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>;
+	Len when Len < 16#10000 -> % 65536
+	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>;
+	Len ->
+	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>
+    end.
 
 % ===== internal APIs ===== %
 
@@ -162,17 +172,6 @@ unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
 	{Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList])
     end.
 
-% FIXME: write test for pack_map/1
-pack_map(M)->
-    case length(M) of
-	Len when Len < 16 ->
- 	    << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>;
-	Len when Len < 16#10000 -> % 65536
-	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>;
-	Len ->
-	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>
-    end.
-
 pack_map_([], Acc) -> Acc;
 pack_map_([{Key,Value}|Tail], Acc) ->
     pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>).
@@ -404,7 +403,7 @@ unknown_test()->
 	     -234.4355, 1.0e-34, 1.0e64,
 	     [23, 234, 0.23],
 	     [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]],
-	     dict:from_list([{1,2},{<<"hoge">>,nil}]),
+	     {[{1,2},{<<"hoge">>,nil}]},
 	     -234, -50000,
 	     42
 	    ],

From 8f7f23a0e5fcc595f6d6178e9e532b38b5cd1b46 Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Mon, 28 Jun 2010 18:11:52 +0200
Subject: [PATCH 0548/1648] Rewrite unpack_/1 using pattern matching to get a
 30-40% speedup. Simplify pack_* and unpack_{array,map} function clauses to
 get more readability and a minor speedup.

---
 erlang/msgpack.erl | 290 +++++++++++++++++++--------------------------
 1 file changed, 123 insertions(+), 167 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 255542b..e94262d 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -42,8 +42,10 @@ pack(O) when is_float(O) ->
     pack_double(O);
 pack(nil) ->
     pack_nil();
-pack(Bool) when is_atom(Bool) ->
-    pack_bool(Bool);
+pack(true) ->
+    pack_true();
+pack(false) ->
+    pack_false();
 pack(Bin) when is_binary(Bin) ->
     pack_raw(Bin);
 pack(List)  when is_list(List) ->
@@ -61,13 +63,8 @@ pack(_O) ->
 % TODO: error case for imcomplete format when short for any type formats.
 -spec unpack( binary() )->
     {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}.
-unpack(Bin) when not is_binary(Bin)->
-    {error, badarg};
-unpack(Bin) when bit_size(Bin) >= 8 ->
-    << Flag:8/unsigned-integer, Payload/binary >> = Bin,
-    unpack_(Flag, Payload);
-unpack(<<>>)-> % when bit_size(Bin) < 8 ->
-    {more, 1}.
+unpack(Bin) ->
+    unpack_(Bin).
 
 -spec unpack_all( binary() ) -> [msgpack_term()].
 unpack_all(Data)->
@@ -82,56 +79,52 @@ unpack_all(Data)->
 % ===== internal APIs ===== %
 
 % positive fixnum
-pack_uint_(N) when is_integer( N ) , N < 128 ->
+pack_uint_(N) when N < 128 ->
     << 2#0:1, N:7 >>;
 % uint 8
-pack_uint_( N ) when is_integer( N ) andalso N < 256 ->
+pack_uint_(N) when N < 256 ->
     << 16#CC:8, N:8 >>;
-
 % uint 16
-pack_uint_( N ) when is_integer( N ) andalso N < 65536 ->
+pack_uint_(N) when N < 65536 ->
     << 16#CD:8, N:16/big-unsigned-integer-unit:1 >>;
-
 % uint 32
-pack_uint_( N ) when is_integer( N ) andalso N < 16#FFFFFFFF->
+pack_uint_(N) when N < 16#FFFFFFFF->
     << 16#CE:8, N:32/big-unsigned-integer-unit:1 >>;
-
 % uint 64
-pack_uint_( N ) when is_integer( N )->
+pack_uint_(N) ->
     << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>.
 
 % negative fixnum
-pack_int_( N ) when is_integer( N ) , N >= -32->
+pack_int_(N) when N >= -32->
     << 2#111:3, N:5 >>;
 % int 8
-pack_int_( N ) when is_integer( N ) , N >= -256 ->
+pack_int_(N) when N >= -256 ->
     << 16#D0:8, N:8 >>;
 % int 16
-pack_int_( N ) when is_integer( N ), N >= -65536 ->
+pack_int_(N) when N >= -65536 ->
     << 16#D1:8, N:16/big-signed-integer-unit:1 >>;
 % int 32
-pack_int_( N ) when is_integer( N ), N >= -16#FFFFFFFF ->
+pack_int_(N) when N >= -16#FFFFFFFF ->
     << 16#D2:8, N:32/big-signed-integer-unit:1 >>;
 % int 64
-pack_int_( N ) when is_integer( N )->
+pack_int_(N) ->
     << 16#D3:8, N:64/big-signed-integer-unit:1 >>.
 
-% nil
-pack_nil()->    << 16#C0:8 >>.
-% pack_true / pack_false
-pack_bool(true)->    << 16#C3:8 >>;
-pack_bool(false)->   << 16#C2:8 >>.
+% nil/true/false
+pack_nil() ->  << 16#C0:8 >>.
+pack_true()->  << 16#C3:8 >>.
+pack_false()-> << 16#C2:8 >>.
 
 % float : erlang's float is always IEEE 754 64bit format.
 %pack_float(F) when is_float(F)->
 %    << 16#CA:8, F:32/big-float-unit:1 >>.
 %    pack_double(F).
 % double
-pack_double(F) when is_float(F)->
+pack_double(F) ->
     << 16#CB:8, F:64/big-float-unit:1 >>.
 
 % raw bytes
-pack_raw(Bin) when is_binary(Bin)->
+pack_raw(Bin) ->
     case byte_size(Bin) of
 	Len when Len < 6->
 	    << 2#101:3, Len:5, Bin/binary >>;
@@ -142,24 +135,22 @@ pack_raw(Bin) when is_binary(Bin)->
     end.
 
 % list / tuple
-pack_array(L) when is_list(L)->
+pack_array(L) ->
     case length(L) of
  	Len when Len < 16 ->
  	    << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L, <<>>))/binary >>;
 	Len when Len < 16#10000 -> % 65536
-	    << 16#DC:8, Len:16/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >>;
+	    << 16#DC:8, Len:16/big-unsigned-integer-unit:1, (pack_array_(L, <<>>))/binary >>;
 	Len ->
-	    << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >>
+	    << 16#DD:8, Len:32/big-unsigned-integer-unit:1, (pack_array_(L, <<>>))/binary >>
     end.
 pack_array_([], Acc) -> Acc;
 pack_array_([Head|Tail], Acc) ->
     pack_array_(Tail, <>).
 
 % FIXME! this should be tail-recursive and without lists:reverse/1
-unpack_array_(<<>>, 0, RetList)   -> {lists:reverse(RetList), <<>>};
-unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain};
-unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 ->  {more, RestLen};
-unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
+unpack_array_(Remain, 0, RetList) -> {lists:reverse(RetList), Remain};
+unpack_array_(Bin, RestLen, RetList) ->
     case unpack(Bin) of
 	{more, Len} -> {more, Len+RestLen-1};
 	{Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList])
@@ -181,10 +172,10 @@ pack_map_([{Key,Value}|Tail], Acc) ->
     pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>).
 
 % FIXME: write test for unpack_map/1
--spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])->
-    {more, non_neg_integer()} | { any(), binary()}.
-unpack_map_(Bin,  0,  Acc) when is_binary(Bin) -> {{lists:reverse(Acc)}, Bin};
-unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) ->
+-spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}]) ->
+    {more, non_neg_integer()} | {any(), binary()}.
+unpack_map_(Bin,  0,  Acc) -> {{lists:reverse(Acc)}, Bin};
+unpack_map_(Bin, Len, Acc) ->
     case unpack(Bin) of
 	{ more, MoreLen } -> { more, MoreLen+Len-1 };
 	{ Key, Rest } ->
@@ -195,142 +186,107 @@ unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) ->
 	    end
     end.
 
-% {more, 
--spec unpack_(Flag::integer(), Payload::binary())->
-    {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}.
-unpack_(Flag, Payload)->
-    PayloadLen = byte_size(Payload),
-    case Flag of
-	16#C0 ->
-	    {nil, Payload};
-	16#C2 ->
-	    {false, Payload};
-	16#C3 ->
-	    {true, Payload};
 
-	16#CA when PayloadLen >= 4 -> % 32bit float
-	    << Return:32/float-unit:1, Rest/binary >> = Payload,
-	    {Return, Rest};
-	16#CA ->
-	    {more, 4-PayloadLen}; % at least more
+-spec unpack_(Payload::binary()) -> {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}.
+unpack_(<<16#C0, Rest/binary>>) ->
+    {nil, Rest};
+unpack_(<<16#C2, Rest/binary>>) ->
+    {false, Rest};
+unpack_(<<16#C3, Rest/binary>>) ->
+    {true, Rest};
 
-	16#CB when PayloadLen >= 8 -> % 64bit float
-	    << Return:64/float-unit:1, Rest/binary >> = Payload,
-	    {Return, Rest};
-	16#CB ->
-	    {more, 8-PayloadLen};
+unpack_(<<16#CA, Return:32/float-unit:1, Rest/binary>>) -> % 32bit float
+    {Return, Rest};
+unpack_(<<16#CA, Rest/binary>>) ->
+    {more, 4-byte_size(Rest)};
+unpack_(<<16#CB, Return:64/float-unit:1, Rest/binary>>) -> % 64bit float
+    {Return, Rest};
+unpack_(<<16#CB, Rest/binary>>) ->
+    {more, 8-byte_size(Rest)};
 
-	16#CC when PayloadLen >= 1 -> % uint 8
-	    << Int:8/unsigned-integer, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#CC ->
-	    {more, 1};
+unpack_(<<16#CC, Int:8/unsigned-integer, Rest/binary>>) -> % uint 8
+    {Int, Rest};
+unpack_(<<16#CC>>) ->
+    {more, 1};
+unpack_(<<16#CD, Int:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 16
+    {Int, Rest};
+unpack_(<<16#CD, Rest/binary>>) ->
+    {more, 2-byte_size(Rest)};
+unpack_(<<16#CE, Int:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 32
+    {Int, Rest};
+unpack_(<<16#CE, Rest/binary>>) ->
+    {more, 4-byte_size(Rest)};
+unpack_(<<16#CF, Int:64/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 64
+    {Int, Rest};
+unpack_(<<16#CF, Rest/binary>>) ->
+    {more, 8-byte_size(Rest)};
 
-	16#CD when PayloadLen >= 2 -> % uint 16
-	    << Int:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#CD ->
-	    {more, 2-PayloadLen};
+unpack_(<<16#D0, Int:8/signed-integer, Rest/binary>>) -> % int 8
+    {Int, Rest};
+unpack_(<<16#D0>>) ->
+    {more, 1};
+unpack_(<<16#D1, Int:16/big-signed-integer-unit:1, Rest/binary>>) -> % int 16
+    {Int, Rest};
+unpack_(<<16#D1, Rest/binary>>) ->
+    {more, 2-byte_size(Rest)};
+unpack_(<<16#D2, Int:32/big-signed-integer-unit:1, Rest/binary>>) -> % int 32
+    {Int, Rest};
+unpack_(<<16#D2, Rest/binary>>) ->
+    {more, 4-byte_size(Rest)};
+unpack_(<<16#D3, Int:64/big-signed-integer-unit:1, Rest/binary>>) -> % int 64
+    {Int, Rest};
+unpack_(<<16#D3, Rest/binary>>) ->
+    {more, 8-byte_size(Rest)};
 
-	16#CE when PayloadLen >= 4 ->
-	    << Int:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#CE ->
-	    {more, 4-PayloadLen}; % at least more
+unpack_(<<16#DA, Len:16/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> % raw 16
+    {Val, Rest};
+unpack_(<<16#DA, Rest/binary>>) ->
+    {more, 16-byte_size(Rest)};
+unpack_(<<16#DB, Len:32/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> % raw 32
+    {Val, Rest};
+unpack_(<<16#DB, Rest/binary>>) ->
+    {more, 32-byte_size(Rest)};
 
-	16#CF when PayloadLen >= 8 ->
-	    << Int:64/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#CF ->
-	    {more, 8-PayloadLen};
+unpack_(<<16#DC, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % array 16
+    unpack_array_(Rest, Len, []);
+unpack_(<<16#DC, Rest/binary>>) ->
+    {more, 2-byte_size(Rest)};
+unpack_(<<16#DD, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % array 32
+    unpack_array_(Rest, Len, []);
+unpack_(<<16#DD, Rest/binary>>) ->
+    {more, 4-byte_size(Rest)};
 
-	16#D0 when PayloadLen >= 1 -> % int 8
-	    << Int:8/big-signed-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#D0 ->
-	    {more, 1};
+unpack_(<<16#DE, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % map 16
+    unpack_map_(Rest, Len, []);
+unpack_(<<16#DE, Rest/binary>>) ->
+    {more, 2-byte_size(Rest)};
+unpack_(<<16#DF, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % map 32
+    unpack_map_(Rest, Len, []);
+unpack_(<<16#DF, Rest/binary>>) ->
+    {more, 4-byte_size(Rest)};
 
-	16#D1 when PayloadLen >= 2 -> % int 16
-	    << Int:16/big-signed-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#D1 ->
-	    {more, 2-PayloadLen};
+unpack_(<<0:1, Value:7, Rest/binary>>) -> % positive fixnum
+    {Value, Rest};
+unpack_(<<2#111:3, Value:5, Rest/binary>>) -> % negative fixnum
+    {Value - 2#100000, Rest};
+unpack_(<<2#101:3, Len:5, Value:Len/binary, Rest/binary>>) -> % fixraw
+    {Value, Rest};
+unpack_(<<2#101:3, Len:5, Rest/binary>>) ->
+    {more, Len-byte_size(Rest)};
+unpack_(<<2#1001:4, Len:4, Rest/binary>>) -> % fixarray
+    unpack_array_(Rest, Len, []);
+unpack_(<<2#1000:4, Len:4, Rest/binary>>) -> % fixmap
+    unpack_map_(Rest, Len, []);
 
-	16#D2 when PayloadLen >= 4 -> % int 32
-	    << Int:32/big-signed-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#D2 ->
-	    {more, 4-PayloadLen};
+%unpack_(<>) when F==16#C1; F==16#C4; F==16#C5; F==16#C6; F==16#C7; F==16#C8; F==16#C9; F==16#D5; F==16#D6; F==16#D7; F==16#D8; F==16#D9->
+%    {error, {badarg, <>}}.
+%unpack_(Other) when is_binary(Bin) ->
+%    {more, 1}.
+unpack_(<<>>) ->
+    {more, 1}.
+unpack_(Other) ->
+    {error, {badarg, Other}}.
 
-	16#D3 when PayloadLen >= 8 -> % int 64
-	    << Int:64/big-signed-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#D3 ->
-	    {more, 8-PayloadLen};
-
-	16#DA when PayloadLen >= 2 -> % raw 16
-	    << Len:16/unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    << Return:Len/binary, Remain/binary >> = Rest,
-	    {Return, Remain};
-	16#DA ->
-	    {more, 16-PayloadLen};
-
-	16#DB when PayloadLen >= 4 -> % raw 32
-	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    << Return:Len/binary, Remain/binary >> = Rest,
-	    {Return, Remain};
-	16#DB ->
-	    {more, 4-PayloadLen};
-
-	16#DC when PayloadLen >= 2 -> % array 16
-	    << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_array_(Rest, Len, []);
-	16#DC ->
-	    {more, 2-PayloadLen};
-
-	16#DD when PayloadLen >= 4 -> % array 32
-	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_array_(Rest, Len, []);
-	16#DD ->
-	    {more, 4-PayloadLen};
-
-	16#DE when PayloadLen >= 2 -> % map 16
-	    << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_map_(Rest, Len, []);
-	16#DE ->
-	    {more, 2-PayloadLen};
-
-	16#DF when PayloadLen >= 4 -> % map 32
-	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_map_(Rest, Len, []);
-
-	% positive fixnum
-	Code when Code >= 2#00000000, Code < 2#10000000->
-	    {Code, Payload};
-
-	% negative fixnum
-	Code when Code >= 2#11100000 ->
-	    {(Code - 16#100), Payload};
-
-	Code when Code >= 2#10100000 , Code < 2#11000000 ->
-%	 101XXXXX for FixRaw
-	    Len = Code rem 2#10100000,
-	    << Return:Len/binary, Remain/binary >> = Payload,
-	    {Return, Remain};
-
-	Code when Code >= 2#10010000 , Code < 2#10100000 ->
-%        1001XXXX for FixArray
-	    Len = Code rem 2#10010000,
-	    unpack_array_(Payload, Len, []);
-
-	Code when Code >= 2#10000000 , Code < 2#10010000 ->
-%        1000XXXX for FixMap
-	    Len = Code rem 2#10000000,
-	    unpack_map_(Payload, Len, []);
-
-	_Other ->
-	    {error, no_code_matches}
-    end.
 
 % ===== test codes ===== %
 -include_lib("eunit/include/eunit.hrl").
@@ -419,7 +375,7 @@ unknown_test()->
 test_([]) -> 0;
 test_([S|Rest])->
     Pack = msgpack:pack(S),
-    {S, <<>>} = msgpack:unpack( Pack ),
+    ?assertEqual({S, <<>>}, msgpack:unpack(Pack)),
     1+test_(Rest).
 
 other_test()->

From 9fffa9800ae4b09180d2b892f1f70215534a874b Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 29 Jun 2010 14:54:09 +0900
Subject: [PATCH 0549/1648] ruby: fixes RDoc of Unpacker#execute and
 Unpacker#execute_impl

---
 ruby/makegem.sh | 2 ++
 ruby/unpack.c   | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ruby/makegem.sh b/ruby/makegem.sh
index 827f452..d21f06a 100755
--- a/ruby/makegem.sh
+++ b/ruby/makegem.sh
@@ -19,6 +19,8 @@ cp ../test/cases.json           test/
 
 gem build msgpack.gemspec
 
+rdoc rbinit.c pack.c unpack.c
+
 if [ $? -eq 0 ]; then
 	rm -rf ext msgpack test/msgpack_test.rb
 fi
diff --git a/ruby/unpack.c b/ruby/unpack.c
index 65ae476..151dbf4 100644
--- a/ruby/unpack.c
+++ b/ruby/unpack.c
@@ -638,7 +638,7 @@ static VALUE MessagePack_Unpacker_execute_impl(VALUE self, VALUE data,
  * Document-method: MessagePack::Unpacker#execute_limit
  *
  * call-seq:
- *   unpacker.unpack_limit(data, offset, limit) -> next offset
+ *   unpacker.execute_limit(data, offset, limit) -> next offset
  *
  * Deserializes one object over the specified buffer from _offset_ bytes upto _limit_ bytes.
  *
@@ -660,7 +660,7 @@ static VALUE MessagePack_Unpacker_execute_limit(VALUE self, VALUE data,
  * Document-method: MessagePack::Unpacker#execute
  *
  * call-seq:
- *   unpacker.unpack(data, offset) -> next offset
+ *   unpacker.execute(data, offset) -> next offset
  *
  * Deserializes one object over the specified buffer from _offset_ bytes.
  *

From 34a29cd0a50eea4a0e008fe3947c86179d536540 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 29 Jun 2010 14:54:40 +0900
Subject: [PATCH 0550/1648] ruby: fixes SEGV problem caused by GC bug at
 MessagePack_Unpacker_mark.

---
 ruby/test/test_helper.rb |  1 +
 ruby/unpack.c            | 18 +++++++++++++-----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb
index 19226ef..bf9fee8 100644
--- a/ruby/test/test_helper.rb
+++ b/ruby/test/test_helper.rb
@@ -5,3 +5,4 @@ rescue LoadError
 require File.dirname(__FILE__) + '/../lib/msgpack'
 end
 
+GC.stress = true
diff --git a/ruby/unpack.c b/ruby/unpack.c
index 151dbf4..0948151 100644
--- a/ruby/unpack.c
+++ b/ruby/unpack.c
@@ -287,6 +287,7 @@ static void MessagePack_Unpacker_mark(msgpack_unpack_t *mp)
 	unsigned int i;
 	rb_gc_mark(mp->user.stream);
 	rb_gc_mark(mp->user.streambuf);
+	rb_gc_mark_maybe(template_data(mp));
 	for(i=0; i < mp->top; ++i) {
 		rb_gc_mark(mp->stack[i].obj);
 		rb_gc_mark_maybe(mp->stack[i].map_key);
@@ -297,6 +298,17 @@ static VALUE MessagePack_Unpacker_alloc(VALUE klass)
 {
 	VALUE obj;
 	msgpack_unpack_t* mp = ALLOC_N(msgpack_unpack_t, 1);
+
+	// rb_gc_mark (not _maybe) is used for following member objects.
+	mp->user.stream = Qnil;
+	mp->user.streambuf = Qnil;
+
+	mp->user.finished = 0;
+	mp->user.offset = 0;
+	mp->user.buffer.size = 0;
+	mp->user.buffer.free = 0;
+	mp->user.buffer.ptr = NULL;
+
 	obj = Data_Wrap_Struct(klass, MessagePack_Unpacker_mark,
 			MessagePack_Unpacker_free, mp);
 	return obj;
@@ -343,14 +355,10 @@ static VALUE MessagePack_Unpacker_initialize(int argc, VALUE *argv, VALUE self)
 
 	UNPACKER(self, mp);
 	template_init(mp);
-	mp->user.finished = 0;
-	mp->user.offset = 0;
-	mp->user.buffer.size = 0;
-	mp->user.buffer.free = 0;
-	mp->user.buffer.ptr = NULL;
 	mp->user.stream = stream;
 	mp->user.streambuf = rb_str_buf_new(MSGPACK_UNPACKER_BUFFER_RESERVE_SIZE);
 	mp->user.stream_append_method = append_method_of(stream);
+
 	return self;
 }
 

From 123ae024c6d5c217f18a9444c61b292145227278 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 29 Jun 2010 15:12:52 +0900
Subject: [PATCH 0551/1648] ruby: MessagePack::VERSION constant

---
 ruby/extconf.rb               | 3 ++-
 ruby/msgpack.gemspec          | 3 ++-
 ruby/rbinit.c                 | 4 +++-
 ruby/test/test_helper.rb      | 2 +-
 ruby/test/test_pack_unpack.rb | 4 ++++
 ruby/version.rb               | 3 +++
 6 files changed, 15 insertions(+), 4 deletions(-)
 create mode 100644 ruby/version.rb

diff --git a/ruby/extconf.rb b/ruby/extconf.rb
index e6d4bd6..eb6a389 100644
--- a/ruby/extconf.rb
+++ b/ruby/extconf.rb
@@ -1,4 +1,5 @@
 require 'mkmf'
-$CFLAGS << " -I.. -Wall -O4"
+require './version.rb'
+$CFLAGS << %[ -I.. -Wall -O4 -DMESSAGEPACK_VERSION=\\"#{MessagePack::VERSION}\\"]
 create_makefile('msgpack')
 
diff --git a/ruby/msgpack.gemspec b/ruby/msgpack.gemspec
index fb6338a..95a2bd0 100644
--- a/ruby/msgpack.gemspec
+++ b/ruby/msgpack.gemspec
@@ -1,7 +1,8 @@
+require './version.rb'
 Gem::Specification.new do |s|
   s.platform = Gem::Platform::RUBY
   s.name = "msgpack"
-  s.version = "0.4.2"
+  s.version = MessagePack::VERSION
   s.summary = "MessagePack, a binary-based efficient data interchange format."
   s.author = "FURUHASHI Sadayuki"
   s.email = "frsyuki@users.sourceforge.jp"
diff --git a/ruby/rbinit.c b/ruby/rbinit.c
index ad51f6b..28a8bfe 100644
--- a/ruby/rbinit.c
+++ b/ruby/rbinit.c
@@ -43,7 +43,9 @@ static VALUE mMessagePack;
 void Init_msgpack(void)
 {
 	mMessagePack = rb_define_module("MessagePack");
+
+	rb_define_const(mMessagePack, "VERSION", rb_str_new2(MESSAGEPACK_VERSION));
+
 	Init_msgpack_unpack(mMessagePack);
 	Init_msgpack_pack(mMessagePack);
 }
-
diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb
index bf9fee8..80d7806 100644
--- a/ruby/test/test_helper.rb
+++ b/ruby/test/test_helper.rb
@@ -5,4 +5,4 @@ rescue LoadError
 require File.dirname(__FILE__) + '/../lib/msgpack'
 end
 
-GC.stress = true
+#GC.stress = true
diff --git a/ruby/test/test_pack_unpack.rb b/ruby/test/test_pack_unpack.rb
index 9dff44f..25bde81 100644
--- a/ruby/test/test_pack_unpack.rb
+++ b/ruby/test/test_pack_unpack.rb
@@ -276,6 +276,10 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase
 		assert_equal(parsed, num)
 	end
 
+	it "MessagePack::VERSION constant" do
+		p MessagePack::VERSION
+	end
+
 	private
 	def check(len, obj)
 		v = obj.to_msgpack
diff --git a/ruby/version.rb b/ruby/version.rb
new file mode 100644
index 0000000..b156620
--- /dev/null
+++ b/ruby/version.rb
@@ -0,0 +1,3 @@
+module MessagePack
+	VERSION = "0.4.3"
+end

From 20de730541475516aa7a6361af1d1b5e4ea574b8 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 29 Jun 2010 15:39:47 +0900
Subject: [PATCH 0552/1648] ruby: 0.4.3

---
 ruby/ChangeLog  | 6 ++++++
 ruby/makegem.sh | 1 +
 2 files changed, 7 insertions(+)

diff --git a/ruby/ChangeLog b/ruby/ChangeLog
index e69de29..d3a7282 100644
--- a/ruby/ChangeLog
+++ b/ruby/ChangeLog
@@ -0,0 +1,6 @@
+
+2010-06-29 version 0.4.3:
+
+  * Adds MessagePack::VERSION constant
+  * Fixes SEGV problem caused by GC bug at MessagePack_Unpacker_mark
+
diff --git a/ruby/makegem.sh b/ruby/makegem.sh
index d21f06a..bf30cd4 100755
--- a/ruby/makegem.sh
+++ b/ruby/makegem.sh
@@ -8,6 +8,7 @@ cp pack.h       ext/
 cp rbinit.c     ext/
 cp unpack.c     ext/
 cp unpack.h     ext/
+cp version.rb   ext/
 cp ../msgpack/pack_define.h     msgpack/
 cp ../msgpack/pack_template.h   msgpack/
 cp ../msgpack/unpack_define.h   msgpack/

From 33a7d56042539282e3b02d75d365dc3dfa57266c Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Tue, 29 Jun 2010 11:59:56 +0200
Subject: [PATCH 0553/1648] * Return {more,undefined} instead of
 {more,integer()}, as we can only know the "minimum bytes needed to continue"
 instead of the actually usefull "total packet size". * Merge all {more,...}
 clauses of unpack_/1 into one. * Reformat unpack_/1 for readability. * Fix
 some specs, error values, and documentation.

---
 erlang/msgpack.erl | 178 ++++++++++++++++-----------------------------
 1 file changed, 62 insertions(+), 116 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index e94262d..dc4907d 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -29,11 +29,13 @@
 % erl> c(msgpack).
 % erl> S = .
 % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
--type reason() ::  enomem | badarg | no_code_matches.
--type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float().
+-type reason() ::  enomem | {badarg, term()}.
+-type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float() | binary().
 
 % ===== external APIs ===== %
--spec pack(Term::msgpack_term()) -> binary().
+
+% @doc Pack one erlang term into an msgpack message.
+-spec pack(Term::msgpack_term()) -> binary() | reason().
 pack(O) when is_integer(O) andalso O < 0 ->
     pack_int_(O);
 pack(O) when is_integer(O) ->
@@ -54,17 +56,17 @@ pack({Map}) when is_list(Map) ->
     pack_map(Map);
 pack(Map) when is_tuple(Map), element(1,Map)=:=dict ->
     pack_map(dict:from_list(Map));
-pack(_O) ->
-    {error, undefined}.
+pack(Other) ->
+    {error, {badarg, Other}}.
 
-% unpacking.
-% if failed in decoding and not end, get more data
-% and feed more Bin into this function.
-% TODO: error case for imcomplete format when short for any type formats.
--spec unpack( binary() )->
-    {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}.
-unpack(Bin) ->
-    unpack_(Bin).
+% @doc Unpack one (possibly deeply nested) msgpack message into an erlang term.
+%      If failed in decoding and not end, get more data
+%      and feed more Bin into this function.
+-spec unpack( binary() ) -> {Decoded::msgpack_term(), Rest::binary()} | {more, undefined} | {error, reason()}.
+unpack(Bin) when is_binary(Bin) ->
+    unpack_(Bin);
+unpack(Other) ->
+    {error, {badarg, Other}}.
 
 -spec unpack_all( binary() ) -> [msgpack_term()].
 unpack_all(Data)->
@@ -134,7 +136,7 @@ pack_raw(Bin) ->
 	    << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >>
     end.
 
-% list / tuple
+% list
 pack_array(L) ->
     case length(L) of
  	Len when Len < 16 ->
@@ -152,7 +154,7 @@ pack_array_([Head|Tail], Acc) ->
 unpack_array_(Remain, 0, RetList) -> {lists:reverse(RetList), Remain};
 unpack_array_(Bin, RestLen, RetList) ->
     case unpack(Bin) of
-	{more, Len} -> {more, Len+RestLen-1};
+	{more, undefined} -> {more, undefined};
 	{Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList])
     end.
 
@@ -172,120 +174,66 @@ pack_map_([{Key,Value}|Tail], Acc) ->
     pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>).
 
 % FIXME: write test for unpack_map/1
--spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}]) ->
-    {more, non_neg_integer()} | {any(), binary()}.
+-spec unpack_map_(binary(), non_neg_integer(), [{msgpack_term(), msgpack_term()}]) -> {more, undefined} | {any(), binary()} | {error, reason()}.
 unpack_map_(Bin,  0,  Acc) -> {{lists:reverse(Acc)}, Bin};
 unpack_map_(Bin, Len, Acc) ->
     case unpack(Bin) of
-	{ more, MoreLen } -> { more, MoreLen+Len-1 };
-	{ Key, Rest } ->
+	{more, undefined} -> {more, undefined};
+	{Key, Rest} ->
 	    case unpack(Rest) of
-		{more, MoreLen} -> { more, MoreLen+Len-1 };
-		{ Value, Rest2 } ->
+		{more, undefined} -> {more, undefined};
+		{Value, Rest2} ->
 		    unpack_map_(Rest2,Len-1,[{Key,Value}|Acc])
 	    end
     end.
 
 
--spec unpack_(Payload::binary()) -> {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}.
-unpack_(<<16#C0, Rest/binary>>) ->
-    {nil, Rest};
-unpack_(<<16#C2, Rest/binary>>) ->
-    {false, Rest};
-unpack_(<<16#C3, Rest/binary>>) ->
-    {true, Rest};
+-spec unpack_(Payload::binary()) -> {more, undefined} | {msgpack_term(), binary()} | {error, reason()}.
+% Atoms
+unpack_(<<16#C0, Rest/binary>>) -> {nil, Rest};
+unpack_(<<16#C2, Rest/binary>>) -> {false, Rest};
+unpack_(<<16#C3, Rest/binary>>) -> {true, Rest};
 
-unpack_(<<16#CA, Return:32/float-unit:1, Rest/binary>>) -> % 32bit float
-    {Return, Rest};
-unpack_(<<16#CA, Rest/binary>>) ->
-    {more, 4-byte_size(Rest)};
-unpack_(<<16#CB, Return:64/float-unit:1, Rest/binary>>) -> % 64bit float
-    {Return, Rest};
-unpack_(<<16#CB, Rest/binary>>) ->
-    {more, 8-byte_size(Rest)};
+% Floats
+unpack_(<<16#CA, Val:32/float-unit:1, Rest/binary>>) -> {Val, Rest};
+unpack_(<<16#CB, Val:64/float-unit:1, Rest/binary>>) -> {Val, Rest};
 
-unpack_(<<16#CC, Int:8/unsigned-integer, Rest/binary>>) -> % uint 8
-    {Int, Rest};
-unpack_(<<16#CC>>) ->
-    {more, 1};
-unpack_(<<16#CD, Int:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 16
-    {Int, Rest};
-unpack_(<<16#CD, Rest/binary>>) ->
-    {more, 2-byte_size(Rest)};
-unpack_(<<16#CE, Int:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 32
-    {Int, Rest};
-unpack_(<<16#CE, Rest/binary>>) ->
-    {more, 4-byte_size(Rest)};
-unpack_(<<16#CF, Int:64/big-unsigned-integer-unit:1, Rest/binary>>) -> % uint 64
-    {Int, Rest};
-unpack_(<<16#CF, Rest/binary>>) ->
-    {more, 8-byte_size(Rest)};
+% Unsigned integers
+unpack_(<<16#CC, Val:8/unsigned-integer, Rest/binary>>) ->             {Val, Rest};
+unpack_(<<16#CD, Val:16/big-unsigned-integer-unit:1, Rest/binary>>) -> {Val, Rest};
+unpack_(<<16#CE, Val:32/big-unsigned-integer-unit:1, Rest/binary>>) -> {Val, Rest};
+unpack_(<<16#CF, Val:64/big-unsigned-integer-unit:1, Rest/binary>>) -> {Val, Rest};
 
-unpack_(<<16#D0, Int:8/signed-integer, Rest/binary>>) -> % int 8
-    {Int, Rest};
-unpack_(<<16#D0>>) ->
-    {more, 1};
-unpack_(<<16#D1, Int:16/big-signed-integer-unit:1, Rest/binary>>) -> % int 16
-    {Int, Rest};
-unpack_(<<16#D1, Rest/binary>>) ->
-    {more, 2-byte_size(Rest)};
-unpack_(<<16#D2, Int:32/big-signed-integer-unit:1, Rest/binary>>) -> % int 32
-    {Int, Rest};
-unpack_(<<16#D2, Rest/binary>>) ->
-    {more, 4-byte_size(Rest)};
-unpack_(<<16#D3, Int:64/big-signed-integer-unit:1, Rest/binary>>) -> % int 64
-    {Int, Rest};
-unpack_(<<16#D3, Rest/binary>>) ->
-    {more, 8-byte_size(Rest)};
+% Signed integers
+unpack_(<<16#D0, Val:8/signed-integer, Rest/binary>>) ->             {Val, Rest};
+unpack_(<<16#D1, Val:16/big-signed-integer-unit:1, Rest/binary>>) -> {Val, Rest};
+unpack_(<<16#D2, Val:32/big-signed-integer-unit:1, Rest/binary>>) -> {Val, Rest};
+unpack_(<<16#D3, Val:64/big-signed-integer-unit:1, Rest/binary>>) -> {Val, Rest};
 
-unpack_(<<16#DA, Len:16/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> % raw 16
-    {Val, Rest};
-unpack_(<<16#DA, Rest/binary>>) ->
-    {more, 16-byte_size(Rest)};
-unpack_(<<16#DB, Len:32/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> % raw 32
-    {Val, Rest};
-unpack_(<<16#DB, Rest/binary>>) ->
-    {more, 32-byte_size(Rest)};
+% Raw bytes
+unpack_(<<16#DA, Len:16/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> {Val, Rest};
+unpack_(<<16#DB, Len:32/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>>) -> {Val, Rest};
 
-unpack_(<<16#DC, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % array 16
-    unpack_array_(Rest, Len, []);
-unpack_(<<16#DC, Rest/binary>>) ->
-    {more, 2-byte_size(Rest)};
-unpack_(<<16#DD, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % array 32
-    unpack_array_(Rest, Len, []);
-unpack_(<<16#DD, Rest/binary>>) ->
-    {more, 4-byte_size(Rest)};
+% Arrays
+unpack_(<<16#DC, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> unpack_array_(Rest, Len, []);
+unpack_(<<16#DD, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> unpack_array_(Rest, Len, []);
 
-unpack_(<<16#DE, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> % map 16
-    unpack_map_(Rest, Len, []);
-unpack_(<<16#DE, Rest/binary>>) ->
-    {more, 2-byte_size(Rest)};
-unpack_(<<16#DF, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> % map 32
-    unpack_map_(Rest, Len, []);
-unpack_(<<16#DF, Rest/binary>>) ->
-    {more, 4-byte_size(Rest)};
+% Maps
+unpack_(<<16#DE, Len:16/big-unsigned-integer-unit:1, Rest/binary>>) -> unpack_map_(Rest, Len, []);
+unpack_(<<16#DF, Len:32/big-unsigned-integer-unit:1, Rest/binary>>) -> unpack_map_(Rest, Len, []);
 
-unpack_(<<0:1, Value:7, Rest/binary>>) -> % positive fixnum
-    {Value, Rest};
-unpack_(<<2#111:3, Value:5, Rest/binary>>) -> % negative fixnum
-    {Value - 2#100000, Rest};
-unpack_(<<2#101:3, Len:5, Value:Len/binary, Rest/binary>>) -> % fixraw
-    {Value, Rest};
-unpack_(<<2#101:3, Len:5, Rest/binary>>) ->
-    {more, Len-byte_size(Rest)};
-unpack_(<<2#1001:4, Len:4, Rest/binary>>) -> % fixarray
-    unpack_array_(Rest, Len, []);
-unpack_(<<2#1000:4, Len:4, Rest/binary>>) -> % fixmap
-    unpack_map_(Rest, Len, []);
+% Tag-encoded lengths (kept last, for speed)
+unpack_(<<0:1, Val:7, Rest/binary>>) ->                      {Val, Rest};                  % pos fixnum
+unpack_(<<2#111:3, Val:5, Rest/binary>>) ->                  {Val - 2#100000, Rest};       % neg fixnum
+unpack_(<<2#101:3, Len:5, Val:Len/binary, Rest/binary>>) ->  {Val, Rest};                  % fixraw
+unpack_(<<2#1001:4, Len:4, Rest/binary>>) ->                 unpack_array_(Rest, Len, []); % fixarray
+unpack_(<<2#1000:4, Len:4, Rest/binary>>) ->                 unpack_map_(Rest, Len, []);   % fixmap
 
-%unpack_(<>) when F==16#C1; F==16#C4; F==16#C5; F==16#C6; F==16#C7; F==16#C8; F==16#C9; F==16#D5; F==16#D6; F==16#D7; F==16#D8; F==16#D9->
-%    {error, {badarg, <>}}.
-%unpack_(Other) when is_binary(Bin) ->
-%    {more, 1}.
-unpack_(<<>>) ->
-    {more, 1}.
-unpack_(Other) ->
-    {error, {badarg, Other}}.
+% Incomplete / invalid data
+unpack_(<>) when F==16#C1; F==16#C4; F==16#C5; F==16#C6; F==16#C7; F==16#C8; F==16#C9; F==16#D5; F==16#D6; F==16#D7; F==16#D8; F==16#D9->
+    {error, {badarg, <>}};
+unpack_(_Bin) ->
+    {more, undefined}.
 
 
 % ===== test codes ===== %
@@ -330,9 +278,7 @@ test_p(Len,Term,OrigBin,Len) ->
     {Term, <<>>}=msgpack:unpack(OrigBin);
 test_p(I,_,OrigBin,Len) when I < Len->
     <> = OrigBin,
-    {more, N}=msgpack:unpack(Bin),
-    ?assert(0 < N),
-    ?assert(N < Len).
+    ?assertEqual({more, undefined}, msgpack:unpack(Bin)).
 
 partial_test()-> % error handling test.
     Term = lists:seq(0, 45),
@@ -379,6 +325,6 @@ test_([S|Rest])->
     1+test_(Rest).
 
 other_test()->
-    {more,1}=msgpack:unpack(<<>>).
+    ?assertEqual({more,undefined}, msgpack:unpack(<<>>)).
 
 -endif.

From 83b4b7d83d0d32b2c30d26e51439b4dfec3127f9 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Thu, 1 Jul 2010 00:58:48 +0900
Subject: [PATCH 0554/1648] erlang: more suitable variable name and removing
 unnecessary guards.

---
 erlang/msgpack.erl | 75 +++++++++++++++++++++++-----------------------
 1 file changed, 38 insertions(+), 37 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index a168b55..c99002c 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -31,20 +31,24 @@
 % erl> S = .
 % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
 -type reason() ::  enomem | badarg | no_code_matches.
--type msgpack_term() :: [msgpack_term()] | {[{msgpack_term(),msgpack_term()}]} | integer() | float().
+-type msgpack_term() :: [msgpack_term()]
+		      | {[{msgpack_term(),msgpack_term()}]}
+		      | integer() | float() | binary().
 
 % ===== external APIs ===== %
 -spec pack(Term::msgpack_term()) -> binary().
-pack(O) when is_integer(O) andalso O < 0 ->
-    pack_int_(O);
-pack(O) when is_integer(O) ->
-    pack_uint_(O);
-pack(O) when is_float(O) ->
-    pack_double(O);
+pack(I) when is_integer(I) andalso I < 0 ->
+    pack_int_(I);
+pack(I) when is_integer(I) ->
+    pack_uint_(I);
+pack(F) when is_float(F) ->
+    pack_double(F);
 pack(nil) ->
-    pack_nil();
-pack(Bool) when is_atom(Bool) ->
-    pack_bool(Bool);
+    << 16#C0:8 >>;
+pack(true) ->
+    << 16#C3:8 >>;
+pack(false) ->
+    << 16#C2:8 >>;
 pack(Bin) when is_binary(Bin) ->
     pack_raw(Bin);
 pack(List)  when is_list(List) ->
@@ -53,7 +57,7 @@ pack({Map}) when is_list(Map) ->
     pack_map(Map);
 pack(Map) when is_tuple(Map), element(1,Map)=:=dict ->
     pack_map(dict:to_list(Map));
-pack(_O) ->
+pack(_Other) ->
     {error, undefined}.
 
 % unpacking.
@@ -92,53 +96,47 @@ pack_map(M)->
 % ===== internal APIs ===== %
 
 % positive fixnum
-pack_uint_(N) when is_integer( N ) , N < 128 ->
+pack_uint_(N) when N < 128 ->
     << 2#0:1, N:7 >>;
 % uint 8
-pack_uint_( N ) when is_integer( N ) andalso N < 256 ->
+pack_uint_(N) when N < 256 ->
     << 16#CC:8, N:8 >>;
 % uint 16
-pack_uint_( N ) when is_integer( N ) andalso N < 65536 ->
+pack_uint_(N) when N < 65536 ->
     << 16#CD:8, N:16/big-unsigned-integer-unit:1 >>;
 % uint 32
-pack_uint_( N ) when is_integer( N ) andalso N < 16#FFFFFFFF->
+pack_uint_(N) when N < 16#FFFFFFFF->
     << 16#CE:8, N:32/big-unsigned-integer-unit:1 >>;
 % uint 64
-pack_uint_( N ) when is_integer( N )->
+pack_uint_(N) ->
     << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>.
 
 % negative fixnum
-pack_int_( N ) when is_integer( N ) , N >= -32->
+pack_int_(N) when is_integer(N) , N >= -32->
     << 2#111:3, N:5 >>;
 % int 8
-pack_int_( N ) when is_integer( N ) , N > -128 ->
+pack_int_(N) when N > -128 ->
     << 16#D0:8, N:8/big-signed-integer-unit:1 >>;
 % int 16
-pack_int_( N ) when is_integer( N ), N > -32768 ->
+pack_int_(N) when N > -32768 ->
     << 16#D1:8, N:16/big-signed-integer-unit:1 >>;
 % int 32
-pack_int_( N ) when is_integer( N ), N > -16#FFFFFFFF ->
+pack_int_(N) when N > -16#FFFFFFFF ->
     << 16#D2:8, N:32/big-signed-integer-unit:1 >>;
 % int 64
-pack_int_( N ) when is_integer( N )->
+pack_int_(N) ->
     << 16#D3:8, N:64/big-signed-integer-unit:1 >>.
 
-% nil
-pack_nil()->    << 16#C0:8 >>.
-% pack_true / pack_false
-pack_bool(true)->    << 16#C3:8 >>;
-pack_bool(false)->   << 16#C2:8 >>.
-
 % float : erlang's float is always IEEE 754 64bit format.
 %pack_float(F) when is_float(F)->
 %    << 16#CA:8, F:32/big-float-unit:1 >>.
 %    pack_double(F).
 % double
-pack_double(F) when is_float(F)->
+pack_double(F) ->
     << 16#CB:8, F:64/big-float-unit:1 >>.
 
 % raw bytes
-pack_raw(Bin) when is_binary(Bin)->
+pack_raw(Bin) ->
     case byte_size(Bin) of
 	Len when Len < 6->
 	    << 2#101:3, Len:5, Bin/binary >>;
@@ -149,7 +147,7 @@ pack_raw(Bin) when is_binary(Bin)->
     end.
 
 % list / tuple
-pack_array(L) when is_list(L)->
+pack_array(L) ->
     case length(L) of
  	Len when Len < 16 ->
  	    << 2#1001:4, Len:4/integer-unit:1, (pack_array_(L, <<>>))/binary >>;
@@ -165,10 +163,10 @@ pack_array_([Head|Tail], Acc) ->
 % FIXME! this should be tail-recursive and without lists:reverse/1
 unpack_array_(<<>>, 0, RetList)   -> {lists:reverse(RetList), <<>>};
 unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain};
-unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 ->  {more, RestLen};
+unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 ->  {more, undefined};
 unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
     case unpack(Bin) of
-	{more, Len} -> {more, Len+RestLen-1};
+	{more, Len} -> {more, undefined};
 	{Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList])
     end.
 
@@ -179,8 +177,8 @@ pack_map_([{Key,Value}|Tail], Acc) ->
 % FIXME: write test for unpack_map/1
 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])->
     {more, non_neg_integer()} | { any(), binary()}.
-unpack_map_(Bin,  0,  Acc) when is_binary(Bin) -> {{lists:reverse(Acc)}, Bin};
-unpack_map_(Bin, Len, Acc) when is_binary(Bin) and is_integer(Len) ->
+unpack_map_(Bin,  0,  Acc) -> {{lists:reverse(Acc)}, Bin};
+unpack_map_(Bin, Len, Acc) ->
     case unpack(Bin) of
 	{ more, MoreLen } -> { more, MoreLen+Len-1 };
 	{ Key, Rest } ->
@@ -371,9 +369,12 @@ test_p(Len,Term,OrigBin,Len) ->
     {Term, <<>>}=msgpack:unpack(OrigBin);
 test_p(I,_,OrigBin,Len) when I < Len->
     <> = OrigBin,
-    {more, N}=msgpack:unpack(Bin),
-    ?assert(0 < N),
-    ?assert(N < Len).
+    case msgpack:unpack(Bin) of
+	{more, N} when not is_integer(N) ->
+	    ?assertEqual(undefined, N);
+	{more, N} ->
+	    ?assert( N < Len )
+    end.
 
 partial_test()-> % error handling test.
     Term = lists:seq(0, 45),

From acb8fa613e87081267ec9963e5770580208e3e1f Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Thu, 1 Jul 2010 01:02:19 +0900
Subject: [PATCH 0555/1648] erlang: adding shorthand fix for {more, undefined}
 problem

---
 erlang/msgpack.erl | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index c99002c..d24220b 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -166,7 +166,7 @@ unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetLis
 unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 ->  {more, undefined};
 unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
     case unpack(Bin) of
-	{more, Len} -> {more, undefined};
+	{more, _} -> {more, undefined};
 	{Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList])
     end.
 
@@ -180,11 +180,11 @@ pack_map_([{Key,Value}|Tail], Acc) ->
 unpack_map_(Bin,  0,  Acc) -> {{lists:reverse(Acc)}, Bin};
 unpack_map_(Bin, Len, Acc) ->
     case unpack(Bin) of
-	{ more, MoreLen } -> { more, MoreLen+Len-1 };
-	{ Key, Rest } ->
+	{more, _} -> {more, undefined};
+	{Key, Rest} ->
 	    case unpack(Rest) of
-		{more, MoreLen} -> { more, MoreLen+Len-1 };
-		{ Value, Rest2 } ->
+		{more, _} -> {more, undefined};
+		{Value, Rest2} ->
 		    unpack_map_(Rest2,Len-1,[{Key,Value}|Acc])
 	    end
     end.

From 2469768a85a067e5ba425d09ebd717cd0104449a Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Thu, 1 Jul 2010 01:07:56 +0900
Subject: [PATCH 0556/1648] erlang: reducing unnecessary binary matching in
 unpack_/2    * more efficient unpack_/1 by Vincent de Phille's code. thanks.

---
 erlang/msgpack.erl | 207 +++++++++++++++------------------------------
 1 file changed, 70 insertions(+), 137 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index d24220b..703dce2 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -69,10 +69,11 @@ pack(_Other) ->
 unpack(Bin) when not is_binary(Bin)->
     {error, badarg};
 unpack(Bin) when bit_size(Bin) >= 8 ->
-    << Flag:8/unsigned-integer, Payload/binary >> = Bin,
-    unpack_(Flag, Payload);
-unpack(<<>>)-> % when bit_size(Bin) < 8 ->
-    {more, 1}.
+    unpack_(Bin);
+unpack(<<>>)->
+    {more, 1};
+unpack(_) ->
+    {more, undefined}.
 
 -spec unpack_all( binary() ) -> [msgpack_term()].
 unpack_all(Data)->
@@ -83,6 +84,7 @@ unpack_all(Data)->
 	    [Term|unpack_all(Binary)]
     end.
 
+-spec pack_map(M::[{msgpack_term(),msgpack_term()}])-> binary().
 pack_map(M)->
     case length(M) of
 	Len when Len < 16 ->
@@ -189,143 +191,74 @@ unpack_map_(Bin, Len, Acc) ->
 	    end
     end.
 
-% {more, 
--spec unpack_(Flag::integer(), Payload::binary())->
-    {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}.
-unpack_(Flag, Payload)->
-    PayloadLen = byte_size(Payload),
-    case Flag of
-	16#C0 ->
-	    {nil, Payload};
-	16#C2 ->
-	    {false, Payload};
-	16#C3 ->
-	    {true, Payload};
+-spec unpack_(Payload::binary()) -> 
+		     {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}.
+unpack_(Binary)->
+    case Binary of
+% ATOMS
+	<<16#C0, Rest/binary>> -> {nil, Rest};
+	<<16#C2, Rest/binary>> -> {false, Rest};
+	<<16#C3, Rest/binary>> -> {true, Rest};
+% Floats
+	<<16#CA, Val:32/float-unit:1, Rest/binary>> ->   {Val, Rest};
+	<<16#CB, Val:64/float-unit:1, Rest/binary>> ->   {Val, Rest};
+% Unsigned integers
+	<<16#CC, Val:8/unsigned-integer, Rest/binary>> ->             {Val, Rest};
+	<<16#CD, Val:16/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest};
+	<<16#CE, Val:32/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest};
+	<<16#CF, Val:64/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest};
+% Signed integers
+	<<16#D0, Val:8/signed-integer, Rest/binary>> ->             {Val, Rest};
+	<<16#D1, Val:16/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest};
+	<<16#D2, Val:32/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest};
+	<<16#D3, Val:64/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest};
+% Raw bytes
+	<<16#DA, Len:16/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>> -> {Val, Rest};
+	<<16#DB, Len:32/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>> -> {Val, Rest};
+% Arrays
+	<<16#DC, Len:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, Len, []);
+	<<16#DD, Len:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, Len, []);
+% Maps
+	<<16#DE, Len:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, Len, []);
+	<<16#DF, Len:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, Len, []);
 
-	16#CA when PayloadLen >= 4 -> % 32bit float
-	    << Return:32/float-unit:1, Rest/binary >> = Payload,
-	    {Return, Rest};
-	16#CA ->
-	    {more, 4-PayloadLen}; % at least more
+% Tag-encoded lengths (kept last, for speed)
+	<<0:1, Val:7, Rest/binary>> ->                      {Val, Rest};                  % positive int
+	<<2#111:3, Val:5, Rest/binary>> ->                  {Val - 2#100000, Rest};       % negative int
+	<<2#101:3, Len:5, Val:Len/binary, Rest/binary>> ->  {Val, Rest};                  % raw bytes
+	<<2#1001:4, Len:4, Rest/binary>> ->                 unpack_array_(Rest, Len, []); % array
+	<<2#1000:4, Len:4, Rest/binary>> ->                 unpack_map_(Rest, Len, []);   % map
+	
+% Incomplete / invalid data
+	<<16#CA, Rest/binary>> -> {more, 4-byte_size(Rest)};
+	<<16#CB, Rest/binary>> -> {more, 8-byte_size(Rest)};
+	<<16#CC>> ->              {more, 1};
+	<<16#CD, Rest/binary>> -> {more, 2-byte_size(Rest)};
+	<<16#CE, Rest/binary>> -> {more, 4-byte_size(Rest)};
+	<<16#CF, Rest/binary>> -> {more, 8-byte_size(Rest)};
+	<<16#D0>> ->              {more, 1};
+	<<16#D1, Rest/binary>> -> {more, 2-byte_size(Rest)};
+	<<16#D2, Rest/binary>> -> {more, 4-byte_size(Rest)};
+	<<16#D3, Rest/binary>> -> {more, 8-byte_size(Rest)};
+	<<16#DA, Rest/binary>> -> {more, 16-byte_size(Rest)};
+	<<16#DB, Rest/binary>> -> {more, 32-byte_size(Rest)};
+	<<16#DC, Rest/binary>> -> {more, 2-byte_size(Rest)};
+	<<16#DD, Rest/binary>> -> {more, 4-byte_size(Rest)};
+	<<16#DE, Rest/binary>> -> {more, 2-byte_size(Rest)};
+	<<16#DF, Rest/binary>> -> {more, 4-byte_size(Rest)};
+	<<2#101:3, Len:5, Rest/binary>> ->  {more, Len-byte_size(Rest)};
 
-	16#CB when PayloadLen >= 8 -> % 64bit float
-	    << Return:64/float-unit:1, Rest/binary >> = Payload,
-	    {Return, Rest};
-	16#CB ->
-	    {more, 8-PayloadLen};
-
-	16#CC when PayloadLen >= 1 -> % uint 8
-	    << Int:8/unsigned-integer, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#CC ->
-	    {more, 1};
-
-	16#CD when PayloadLen >= 2 -> % uint 16
-	    << Int:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#CD ->
-	    {more, 2-PayloadLen};
-
-	16#CE when PayloadLen >= 4 ->
-	    << Int:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#CE ->
-	    {more, 4-PayloadLen}; % at least more
-
-	16#CF when PayloadLen >= 8 ->
-	    << Int:64/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#CF ->
-	    {more, 8-PayloadLen};
-
-	16#D0 when PayloadLen >= 1 -> % int 8
-	    << Int:8/big-signed-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#D0 ->
-	    {more, 1};
-
-	16#D1 when PayloadLen >= 2 -> % int 16
-	    << Int:16/big-signed-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#D1 ->
-	    {more, 2-PayloadLen};
-
-	16#D2 when PayloadLen >= 4 -> % int 32
-	    << Int:32/big-signed-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#D2 ->
-	    {more, 4-PayloadLen};
-
-	16#D3 when PayloadLen >= 8 -> % int 64
-	    << Int:64/big-signed-integer-unit:1, Rest/binary >> = Payload,
-	    {Int, Rest};
-	16#D3 ->
-	    {more, 8-PayloadLen};
-
-	16#DA when PayloadLen >= 2 -> % raw 16
-	    << Len:16/unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    << Return:Len/binary, Remain/binary >> = Rest,
-	    {Return, Remain};
-	16#DA ->
-	    {more, 16-PayloadLen};
-
-	16#DB when PayloadLen >= 4 -> % raw 32
-	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    << Return:Len/binary, Remain/binary >> = Rest,
-	    {Return, Remain};
-	16#DB ->
-	    {more, 4-PayloadLen};
-
-	16#DC when PayloadLen >= 2 -> % array 16
-	    << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_array_(Rest, Len, []);
-	16#DC ->
-	    {more, 2-PayloadLen};
-
-	16#DD when PayloadLen >= 4 -> % array 32
-	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_array_(Rest, Len, []);
-	16#DD ->
-	    {more, 4-PayloadLen};
-
-	16#DE when PayloadLen >= 2 -> % map 16
-	    << Len:16/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_map_(Rest, Len, []);
-	16#DE ->
-	    {more, 2-PayloadLen};
-
-	16#DF when PayloadLen >= 4 -> % map 32
-	    << Len:32/big-unsigned-integer-unit:1, Rest/binary >> = Payload,
-	    unpack_map_(Rest, Len, []);
-
-	% positive fixnum
-	Code when Code >= 2#00000000, Code < 2#10000000->
-	    {Code, Payload};
-
-	% negative fixnum
-	Code when Code >= 2#11100000 ->
-	    {(Code - 16#100), Payload};
-
-	Code when Code >= 2#10100000 , Code < 2#11000000 ->
-%	 101XXXXX for FixRaw
-	    Len = Code rem 2#10100000,
-	    << Return:Len/binary, Remain/binary >> = Payload,
-	    {Return, Remain};
-
-	Code when Code >= 2#10010000 , Code < 2#10100000 ->
-%        1001XXXX for FixArray
-	    Len = Code rem 2#10010000,
-	    unpack_array_(Payload, Len, []);
-
-	Code when Code >= 2#10000000 , Code < 2#10010000 ->
-%        1000XXXX for FixMap
-	    Len = Code rem 2#10000000,
-	    unpack_map_(Payload, Len, []);
-
-	_Other ->
-	    {error, no_code_matches}
+	<<>> ->                  {more, 1};
+	<<2#101:3, _/binary>> -> {more, undefined};
+	<> when F==16#C1;
+				  F==16#C7; F==16#C8; F==16#C9; F==16#D5;
+				  F==16#D6; F==16#D7; F==16#D8; F==16#D9->
+	    {error, {badarg, <>}};
+	Other ->
+	    {error, {badarg, Other}}
     end.
 
+
 % ===== test codes ===== %
 -include_lib("eunit/include/eunit.hrl").
 -ifdef(EUNIT).

From 370e92b1a6d79cd6d65840cae588ded11b167fce Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Thu, 1 Jul 2010 01:14:20 +0900
Subject: [PATCH 0557/1648] erlang: just a golf.

---
 erlang/msgpack.erl | 44 ++++++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 703dce2..6d94a23 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -200,34 +200,34 @@ unpack_(Binary)->
 	<<16#C2, Rest/binary>> -> {false, Rest};
 	<<16#C3, Rest/binary>> -> {true, Rest};
 % Floats
-	<<16#CA, Val:32/float-unit:1, Rest/binary>> ->   {Val, Rest};
-	<<16#CB, Val:64/float-unit:1, Rest/binary>> ->   {Val, Rest};
+	<<16#CA, V:32/float-unit:1, Rest/binary>> ->   {V, Rest};
+	<<16#CB, V:64/float-unit:1, Rest/binary>> ->   {V, Rest};
 % Unsigned integers
-	<<16#CC, Val:8/unsigned-integer, Rest/binary>> ->             {Val, Rest};
-	<<16#CD, Val:16/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest};
-	<<16#CE, Val:32/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest};
-	<<16#CF, Val:64/big-unsigned-integer-unit:1, Rest/binary>> -> {Val, Rest};
+	<<16#CC, V:8/unsigned-integer, Rest/binary>> ->             {V, Rest};
+	<<16#CD, V:16/big-unsigned-integer-unit:1, Rest/binary>> -> {V, Rest};
+	<<16#CE, V:32/big-unsigned-integer-unit:1, Rest/binary>> -> {V, Rest};
+	<<16#CF, V:64/big-unsigned-integer-unit:1, Rest/binary>> -> {V, Rest};
 % Signed integers
-	<<16#D0, Val:8/signed-integer, Rest/binary>> ->             {Val, Rest};
-	<<16#D1, Val:16/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest};
-	<<16#D2, Val:32/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest};
-	<<16#D3, Val:64/big-signed-integer-unit:1, Rest/binary>> -> {Val, Rest};
+	<<16#D0, V:8/signed-integer, Rest/binary>> ->             {V, Rest};
+	<<16#D1, V:16/big-signed-integer-unit:1, Rest/binary>> -> {V, Rest};
+	<<16#D2, V:32/big-signed-integer-unit:1, Rest/binary>> -> {V, Rest};
+	<<16#D3, V:64/big-signed-integer-unit:1, Rest/binary>> -> {V, Rest};
 % Raw bytes
-	<<16#DA, Len:16/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>> -> {Val, Rest};
-	<<16#DB, Len:32/unsigned-integer-unit:1, Val:Len/binary, Rest/binary>> -> {Val, Rest};
+	<<16#DA, L:16/unsigned-integer-unit:1, V:L/binary, Rest/binary>> -> {V, Rest};
+	<<16#DB, L:32/unsigned-integer-unit:1, V:L/binary, Rest/binary>> -> {V, Rest};
 % Arrays
-	<<16#DC, Len:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, Len, []);
-	<<16#DD, Len:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, Len, []);
+	<<16#DC, L:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, L, []);
+	<<16#DD, L:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_array_(Rest, L, []);
 % Maps
-	<<16#DE, Len:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, Len, []);
-	<<16#DF, Len:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, Len, []);
+	<<16#DE, L:16/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, L, []);
+	<<16#DF, L:32/big-unsigned-integer-unit:1, Rest/binary>> -> unpack_map_(Rest, L, []);
 
 % Tag-encoded lengths (kept last, for speed)
-	<<0:1, Val:7, Rest/binary>> ->                      {Val, Rest};                  % positive int
-	<<2#111:3, Val:5, Rest/binary>> ->                  {Val - 2#100000, Rest};       % negative int
-	<<2#101:3, Len:5, Val:Len/binary, Rest/binary>> ->  {Val, Rest};                  % raw bytes
-	<<2#1001:4, Len:4, Rest/binary>> ->                 unpack_array_(Rest, Len, []); % array
-	<<2#1000:4, Len:4, Rest/binary>> ->                 unpack_map_(Rest, Len, []);   % map
+	<<0:1, V:7, Rest/binary>> ->                 {V, Rest};                  % positive int
+	<<2#111:3, V:5, Rest/binary>> ->             {V - 2#100000, Rest};       % negative int
+	<<2#101:3, L:5, V:L/binary, Rest/binary>> -> {V, Rest};                  % raw bytes
+	<<2#1001:4, L:4, Rest/binary>> ->            unpack_array_(Rest, L, []); % array
+	<<2#1000:4, L:4, Rest/binary>> ->            unpack_map_(Rest, L, []);   % map
 	
 % Incomplete / invalid data
 	<<16#CA, Rest/binary>> -> {more, 4-byte_size(Rest)};
@@ -246,7 +246,7 @@ unpack_(Binary)->
 	<<16#DD, Rest/binary>> -> {more, 4-byte_size(Rest)};
 	<<16#DE, Rest/binary>> -> {more, 2-byte_size(Rest)};
 	<<16#DF, Rest/binary>> -> {more, 4-byte_size(Rest)};
-	<<2#101:3, Len:5, Rest/binary>> ->  {more, Len-byte_size(Rest)};
+	<<2#101:3, L:5, Rest/binary>> ->  {more, L-byte_size(Rest)};
 
 	<<>> ->                  {more, 1};
 	<<2#101:3, _/binary>> -> {more, undefined};

From ff5d5d7cbcb7e136370f3450d67b06244d5c3bc7 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Thu, 1 Jul 2010 01:14:38 +0900
Subject: [PATCH 0558/1648] erlang: updated the comments

---
 erlang/msgpack.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 6d94a23..8a54271 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -162,7 +162,7 @@ pack_array_([], Acc) -> Acc;
 pack_array_([Head|Tail], Acc) ->
     pack_array_(Tail, <>).
 
-% FIXME! this should be tail-recursive and without lists:reverse/1
+% FIXME! this should be without lists:reverse/1
 unpack_array_(<<>>, 0, RetList)   -> {lists:reverse(RetList), <<>>};
 unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain};
 unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 ->  {more, undefined};
@@ -176,7 +176,7 @@ pack_map_([], Acc) -> Acc;
 pack_map_([{Key,Value}|Tail], Acc) ->
     pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>).
 
-% FIXME: write test for unpack_map/1
+% FIXME! this should be without lists:reverse/1
 -spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])->
     {more, non_neg_integer()} | { any(), binary()}.
 unpack_map_(Bin,  0,  Acc) -> {{lists:reverse(Acc)}, Bin};

From 584462f9b9efefa25e93600024c3d9680923b3a4 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Thu, 1 Jul 2010 01:16:25 +0900
Subject: [PATCH 0559/1648] erlang: improved spec.

---
 erlang/msgpack.erl | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 8a54271..aab07b7 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -64,8 +64,9 @@ pack(_Other) ->
 % if failed in decoding and not end, get more data
 % and feed more Bin into this function.
 % TODO: error case for imcomplete format when short for any type formats.
--spec unpack( binary() )->
-    {msgpack_term(), binary()} | {more, non_neg_integer()} | {error, reason()}.
+-spec unpack( Bin::binary() )-> {msgpack_term(), binary()} |
+				{more, non_neg_integer()} | {more, undefined} |
+				{error, reason()}.
 unpack(Bin) when not is_binary(Bin)->
     {error, badarg};
 unpack(Bin) when bit_size(Bin) >= 8 ->

From 71dd44f4308dbdda2d087130452182093262ee01 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 6 Jul 2010 12:26:21 +0900
Subject: [PATCH 0560/1648] cpp: adds operator<<(std::ostream&, const
 tuple&) (experimental)

---
 cpp/src/msgpack/type/tuple.hpp.erb | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/cpp/src/msgpack/type/tuple.hpp.erb b/cpp/src/msgpack/type/tuple.hpp.erb
index 0d9ae91..ebef816 100644
--- a/cpp/src/msgpack/type/tuple.hpp.erb
+++ b/cpp/src/msgpack/type/tuple.hpp.erb
@@ -187,5 +187,20 @@ inline void operator<< (
 
 }  // namespace msgpack
 
+
+//inline std::ostream& operator<< (std::ostream& o, const msgpack::type::tuple<>& v) {
+//  return o << "[]";
+//}
+//<%0.upto(GENERATION_LIMIT) {|i|%>
+//template , typename A<%=j%><%}%>>
+//inline std::ostream& operator<< (std::ostream& o,
+//		const msgpack::type::tuple, A<%=j%><%}%>>& v) {
+//	return o << "["
+//	<%0.upto(i) {|j|%>
+//	<<<%if j != 0 then%> ", " <<<%end%> v.template get<<%=j%>>()<%}%>
+//	<< "]";
+//}
+//<%}%>
+
 #endif /* msgpack/type/tuple.hpp */
 

From 3af10a1d000882f338529360bf35ae0e3a21ffc4 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 6 Jul 2010 17:00:58 +0900
Subject: [PATCH 0561/1648] cpp: adds MSGPACK_VERSION{,_MAJOR,_MINOR} macros
 and msgpack{,_major,_minor} functions

---
 cpp/Makefile.am              |  1 -
 cpp/configure.in             |  5 +++++
 cpp/src/Makefile.am          | 15 ++++++++++----
 cpp/src/msgpack.h            |  2 ++
 cpp/src/msgpack/version.h.in | 40 ++++++++++++++++++++++++++++++++++++
 cpp/src/version.c            | 17 +++++++++++++++
 cpp/test/Makefile.am         |  3 +++
 7 files changed, 78 insertions(+), 5 deletions(-)
 create mode 100644 cpp/src/msgpack/version.h.in
 create mode 100644 cpp/src/version.c

diff --git a/cpp/Makefile.am b/cpp/Makefile.am
index 871ff8c..7dd4891 100644
--- a/cpp/Makefile.am
+++ b/cpp/Makefile.am
@@ -12,7 +12,6 @@ EXTRA_DIST = \
 		$(DOC_FILES)
 
 doxygen:
-	./preprocess
 	./preprocess clean
 	cd src && $(MAKE) doxygen
 	./preprocess
diff --git a/cpp/configure.in b/cpp/configure.in
index 0895be4..ffe3671 100644
--- a/cpp/configure.in
+++ b/cpp/configure.in
@@ -54,5 +54,10 @@ add CFLAGS="--march=i686" and CXXFLAGS="-march=i686" options to ./configure as f
 ])
 fi
 
+major=`echo $VERSION | sed 's/\([[0-9]]*\)\.\([[0-9]]*\).*/\1/'`
+minor=`echo $VERSION | sed 's/\([[0-9]]*\)\.\([[0-9]]*\).*/\2/'`
+AC_SUBST(VERSION_MAJOR, $major)
+AC_SUBST(VERSION_MINOR, $minor)
+
 AC_OUTPUT([Makefile src/Makefile test/Makefile])
 
diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am
index 6b6457e..78d14aa 100644
--- a/cpp/src/Makefile.am
+++ b/cpp/src/Makefile.am
@@ -4,6 +4,7 @@ lib_LTLIBRARIES = libmsgpack.la
 libmsgpack_la_SOURCES = \
 		unpack.c \
 		objectc.c \
+		version.c \
 		vrefbuffer.c \
 		zone.c \
 		object.cpp
@@ -18,15 +19,12 @@ lib_LTLIBRARIES += libmsgpackc.la
 libmsgpackc_la_SOURCES = \
 		unpack.c \
 		objectc.c \
+		version.c \
 		vrefbuffer.c \
 		zone.c
 
 libmsgpackc_la_LDFLAGS = -version-info 2:0:0
 
-# work around for duplicated file name
-kumo_manager_CFLAGS = $(AM_CFLAGS)
-kumo_manager_CXXFLAGS = $(AM_CXXFLAGS)
-
 
 nobase_include_HEADERS = \
 		msgpack/pack_define.h \
@@ -44,6 +42,7 @@ nobase_include_HEADERS = \
 		msgpack/zone.h \
 		msgpack.hpp \
 		msgpack/sbuffer.hpp \
+		msgpack/version.h \
 		msgpack/vrefbuffer.hpp \
 		msgpack/zbuffer.hpp \
 		msgpack/pack.hpp \
@@ -69,11 +68,19 @@ nobase_include_HEADERS = \
 		msgpack/type/tr1/unordered_set.hpp
 
 EXTRA_DIST = \
+		msgpack/version.h.in \
 		msgpack/zone.hpp.erb \
 		msgpack/type/define.hpp.erb \
 		msgpack/type/tuple.hpp.erb
 
 
+msgpack/version.h: msgpack/version.h.in Makefile.in
+	sed -e s/VERSION_UNDEFINED/$(VERSION)/ \
+		-e s/VERSION_MAJOR_UNDEFINED/$(VERSION_MAJOR)/ \
+		-e s/VERSION_MINOR_UNDEFINED/$(VERSION_MINOR)/ \
+		$< > $@
+
+
 doxygen_c:
 	cat ../Doxyfile > Doxyfile_c
 	echo "FILE_PATTERNS      = *.h" >> Doxyfile_c
diff --git a/cpp/src/msgpack.h b/cpp/src/msgpack.h
index 0cd8a19..08f2768 100644
--- a/cpp/src/msgpack.h
+++ b/cpp/src/msgpack.h
@@ -26,3 +26,5 @@
 #include "msgpack/unpack.h"
 #include "msgpack/sbuffer.h"
 #include "msgpack/vrefbuffer.h"
+#include "msgpack/version.h"
+
diff --git a/cpp/src/msgpack/version.h.in b/cpp/src/msgpack/version.h.in
new file mode 100644
index 0000000..af292d0
--- /dev/null
+++ b/cpp/src/msgpack/version.h.in
@@ -0,0 +1,40 @@
+/*
+ * MessagePack for C version information
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_VERSION_H__
+#define MSGPACK_VERSION_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+const char* msgpack_version(void);
+int msgpack_version_major(void);
+int msgpack_version_minor(void);
+
+#define MSGPACK_VERSION "VERSION_UNDEFINED"
+#define MSGPACK_VERSION_MAJOR VERSION_MAJOR_UNDEFINED
+#define MSGPACK_VERSION_MINOR VERSION_MINOR_UNDEFINED
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* msgpack/version.h */
+
diff --git a/cpp/src/version.c b/cpp/src/version.c
new file mode 100644
index 0000000..3d956f1
--- /dev/null
+++ b/cpp/src/version.c
@@ -0,0 +1,17 @@
+#include "msgpack.h"
+
+const char* msgpack_version(void)
+{
+	return MSGPACK_VERSION;
+}
+
+int msgpack_version_major(void)
+{
+	return MSGPACK_VERSION_MAJOR;
+}
+
+int msgpack_version_minor(void)
+{
+	return MSGPACK_VERSION_MINOR;
+}
+
diff --git a/cpp/test/Makefile.am b/cpp/test/Makefile.am
index 1f52215..bb9439e 100644
--- a/cpp/test/Makefile.am
+++ b/cpp/test/Makefile.am
@@ -13,6 +13,7 @@ check_PROGRAMS = \
 		convert \
 		buffer \
 		cases \
+		version \
 		msgpackc_test \
 		msgpack_test
 
@@ -37,6 +38,8 @@ buffer_LDADD = -lz
 
 cases_SOURCES = cases.cc
 
+version_SOURCES = version.cc
+
 msgpackc_test_SOURCES = msgpackc_test.cpp
 
 msgpack_test_SOURCES = msgpack_test.cpp

From c57f6161415156328bd7039be44895b868ee6f76 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 6 Jul 2010 17:10:25 +0900
Subject: [PATCH 0562/1648] cpp: adds MSGPACK_VERSION{,_MAJOR,_MINOR} macros
 and msgpack{,_major,_minor} functions

---
 cpp/src/Makefile.am |  2 ++
 cpp/test/version.cc | 13 +++++++++++++
 2 files changed, 15 insertions(+)
 create mode 100644 cpp/test/version.cc

diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am
index 78d14aa..79b692d 100644
--- a/cpp/src/Makefile.am
+++ b/cpp/src/Makefile.am
@@ -80,6 +80,8 @@ msgpack/version.h: msgpack/version.h.in Makefile.in
 		-e s/VERSION_MINOR_UNDEFINED/$(VERSION_MINOR)/ \
 		$< > $@
 
+#version.c: msgpack/version.h
+
 
 doxygen_c:
 	cat ../Doxyfile > Doxyfile_c
diff --git a/cpp/test/version.cc b/cpp/test/version.cc
new file mode 100644
index 0000000..9357271
--- /dev/null
+++ b/cpp/test/version.cc
@@ -0,0 +1,13 @@
+#include 
+#include 
+
+TEST(version, print)
+{
+	printf("MSGPACK_VERSION          : %s\n", MSGPACK_VERSION);
+	printf("MSGPACK_VERSION_MAJOR    : %d\n", MSGPACK_VERSION_MAJOR);
+	printf("MSGPACK_VERSION_MINOR    : %d\n", MSGPACK_VERSION_MINOR);
+	printf("msgpack_version()        : %s\n", msgpack_version());
+	printf("msgpack_version_major()  : %d\n", msgpack_version_major());
+	printf("msgpack_version_minor()  : %d\n", msgpack_version_minor());
+}
+

From a2bd5ae6386af95617635fec1503640cbadb627b Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 6 Jul 2010 17:45:15 +0900
Subject: [PATCH 0563/1648] cpp: ./configure supports --disable-cxx option not
 to build/install C++ API

---
 cpp/configure.in    | 27 ++++++++++++++++++---------
 cpp/src/Makefile.am | 14 ++++++++++----
 2 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/cpp/configure.in b/cpp/configure.in
index ffe3671..04fa8e3 100644
--- a/cpp/configure.in
+++ b/cpp/configure.in
@@ -9,23 +9,31 @@ CFLAGS="-O4 -Wall $CFLAGS"
 AC_SUBST(CXXFLAGS)
 CXXFLAGS="-O4 -Wall $CXXFLAGS"
 
+
 AC_PROG_CC
-AC_PROG_CXX
+
+
+AC_MSG_CHECKING([if C++ API is enabled])
+AC_ARG_ENABLE(cxx,
+	AS_HELP_STRING([--disable-cxx],
+				   [don't build C++ API]) )
+AC_MSG_RESULT([$enable_cxx])
+if test "$enable_cxx" != "no"; then
+	AC_PROG_CXX
+	AM_PROG_CC_C_O
+fi
+AM_CONDITIONAL(ENABLE_CXX, test "$enable_cxx" != "no")
+
 
 AC_PROG_LIBTOOL
 AM_PROG_AS
-AM_PROG_CC_C_O
-
-AC_LANG_PUSH([C++])
-AC_CHECK_HEADERS(tr1/unordered_map)
-AC_CHECK_HEADERS(tr1/unordered_set)
-AC_LANG_POP([C++])
 
 
 AC_MSG_CHECKING([if debug option is enabled])
 AC_ARG_ENABLE(debug,
 	AS_HELP_STRING([--disable-debug],
-				   [disable assert macros and omit -g option.]) )
+				   [disable assert macros and omit -g option]) )
+AC_MSG_RESULT([$enable_debug])
 if test "$enable_debug" != "no"; then
 	CXXFLAGS="$CXXFLAGS -g"
 	CFLAGS="$CFLAGS -g"
@@ -33,7 +41,6 @@ else
 	CXXFLAGS="$CXXFLAGS -DNDEBUG"
 	CFLAGS="$CFLAGS -DNDEBUG"
 fi
-AC_MSG_RESULT($enable_debug)
 
 
 AC_CACHE_CHECK([for __sync_* atomic operations], msgpack_cv_atomic_ops, [
@@ -54,10 +61,12 @@ add CFLAGS="--march=i686" and CXXFLAGS="-march=i686" options to ./configure as f
 ])
 fi
 
+
 major=`echo $VERSION | sed 's/\([[0-9]]*\)\.\([[0-9]]*\).*/\1/'`
 minor=`echo $VERSION | sed 's/\([[0-9]]*\)\.\([[0-9]]*\).*/\2/'`
 AC_SUBST(VERSION_MAJOR, $major)
 AC_SUBST(VERSION_MINOR, $minor)
 
+
 AC_OUTPUT([Makefile src/Makefile test/Makefile])
 
diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am
index 79b692d..fc67385 100644
--- a/cpp/src/Makefile.am
+++ b/cpp/src/Makefile.am
@@ -6,8 +6,12 @@ libmsgpack_la_SOURCES = \
 		objectc.c \
 		version.c \
 		vrefbuffer.c \
-		zone.c \
+		zone.c
+
+if ENABLE_CXX
+libmsgpack_la_SOURCES += \
 		object.cpp
+endif
 
 # -version-info CURRENT:REVISION:AGE
 libmsgpack_la_LDFLAGS = -version-info 3:0:0
@@ -39,7 +43,10 @@ nobase_include_HEADERS = \
 		msgpack/pack.h \
 		msgpack/unpack.h \
 		msgpack/object.h \
-		msgpack/zone.h \
+		msgpack/zone.h
+
+if ENABLE_CXX
+nobase_include_HEADERS += \
 		msgpack.hpp \
 		msgpack/sbuffer.hpp \
 		msgpack/version.h \
@@ -66,6 +73,7 @@ nobase_include_HEADERS = \
 		msgpack/type/define.hpp \
 		msgpack/type/tr1/unordered_map.hpp \
 		msgpack/type/tr1/unordered_set.hpp
+endif
 
 EXTRA_DIST = \
 		msgpack/version.h.in \
@@ -80,8 +88,6 @@ msgpack/version.h: msgpack/version.h.in Makefile.in
 		-e s/VERSION_MINOR_UNDEFINED/$(VERSION_MINOR)/ \
 		$< > $@
 
-#version.c: msgpack/version.h
-
 
 doxygen_c:
 	cat ../Doxyfile > Doxyfile_c

From 39facd5dc660b90304e4d3d593244a5b9f7cd6f0 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 6 Jul 2010 17:59:07 +0900
Subject: [PATCH 0564/1648] cpp: version 0.5.1

---
 cpp/ChangeLog    | 10 ++++++++++
 cpp/configure.in |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/cpp/ChangeLog b/cpp/ChangeLog
index 53a9991..3277c13 100644
--- a/cpp/ChangeLog
+++ b/cpp/ChangeLog
@@ -1,4 +1,14 @@
 
+2010-07-06 version 0.5.1:
+
+  * Add msgpack_vrefbuffer_new and msgpack_vrefbuffer_free
+  * Add msgpack_sbuffer_new and msgpack_sbuffer_free
+  * Add msgpack_unpacker_next and msgpack_unpack_next
+  * msgpack::unpack returns void
+  * Add MSGPACK_VERSION{,_MAJOR,_MINOR} macros to check header version
+  * Add msgpack_version{,_major,_minor} functions to check library version
+  * ./configure supports --disable-cxx option not to build C++ API
+
 2010-04-29 version 0.5.0:
 
   * msgpack_object_type is changed. MSGPACK_OBJECT_NIL is now 0x00.
diff --git a/cpp/configure.in b/cpp/configure.in
index 04fa8e3..dd04ed4 100644
--- a/cpp/configure.in
+++ b/cpp/configure.in
@@ -1,6 +1,6 @@
 AC_INIT(src/object.cpp)
 AC_CONFIG_AUX_DIR(ac)
-AM_INIT_AUTOMAKE(msgpack, 0.5.0)
+AM_INIT_AUTOMAKE(msgpack, 0.5.1)
 AC_CONFIG_HEADER(config.h)
 
 AC_SUBST(CFLAGS)

From 0c331d288715b156f262cdb7841fb0dba4dc5d83 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 6 Jul 2010 18:18:28 +0900
Subject: [PATCH 0565/1648] cpp: updates vcproj

---
 cpp/msgpack_vc8.postbuild.bat | 77 ++++++++++++++++++-----------------
 cpp/msgpack_vc8.vcproj        | 40 +++++++++++++-----
 cpp/src/Makefile.am           |  2 +-
 3 files changed, 70 insertions(+), 49 deletions(-)

diff --git a/cpp/msgpack_vc8.postbuild.bat b/cpp/msgpack_vc8.postbuild.bat
index bae13f3..20fabbb 100644
--- a/cpp/msgpack_vc8.postbuild.bat
+++ b/cpp/msgpack_vc8.postbuild.bat
@@ -2,42 +2,43 @@ IF NOT EXIST include                  MKDIR include
 IF NOT EXIST include\msgpack          MKDIR include\msgpack
 IF NOT EXIST include\msgpack\type     MKDIR include\msgpack\type
 IF NOT EXIST include\msgpack\type\tr1 MKDIR include\msgpack\type\tr1
-copy msgpack\pack_define.h      include\msgpack\
-copy msgpack\pack_template.h    include\msgpack\
-copy msgpack\unpack_define.h    include\msgpack\
-copy msgpack\unpack_template.h  include\msgpack\
-copy msgpack\sysdep.h           include\msgpack\
-copy msgpack.h                     include\
-copy msgpack\sbuffer.h             include\msgpack\
-copy msgpack\vrefbuffer.h          include\msgpack\
-copy msgpack\zbuffer.h             include\msgpack\
-copy msgpack\pack.h                include\msgpack\
-copy msgpack\unpack.h              include\msgpack\
-copy msgpack\object.h              include\msgpack\
-copy msgpack\zone.h                include\msgpack\
-copy msgpack.hpp                   include\
-copy msgpack\sbuffer.hpp           include\msgpack\
-copy msgpack\vrefbuffer.hpp        include\msgpack\
-copy msgpack\zbuffer.hpp           include\msgpack\
-copy msgpack\pack.hpp              include\msgpack\
-copy msgpack\unpack.hpp            include\msgpack\
-copy msgpack\object.hpp            include\msgpack\
-copy msgpack\zone.hpp              include\msgpack\
-copy msgpack\type.hpp              include\msgpack\type\
-copy msgpack\type\bool.hpp         include\msgpack\type\
-copy msgpack\type\float.hpp        include\msgpack\type\
-copy msgpack\type\int.hpp          include\msgpack\type\
-copy msgpack\type\list.hpp         include\msgpack\type\
-copy msgpack\type\deque.hpp        include\msgpack\type\
-copy msgpack\type\map.hpp          include\msgpack\type\
-copy msgpack\type\nil.hpp          include\msgpack\type\
-copy msgpack\type\pair.hpp         include\msgpack\type\
-copy msgpack\type\raw.hpp          include\msgpack\type\
-copy msgpack\type\set.hpp          include\msgpack\type\
-copy msgpack\type\string.hpp       include\msgpack\type\
-copy msgpack\type\vector.hpp       include\msgpack\type\
-copy msgpack\type\tuple.hpp        include\msgpack\type\
-copy msgpack\type\define.hpp       include\msgpack\type\
-copy msgpack\type\tr1\unordered_map.hpp  include\msgpack\type\
-copy msgpack\type\tr1\unordered_set.hpp  include\msgpack\type\
+copy src\msgpack\pack_define.h      include\msgpack\
+copy src\msgpack\pack_template.h    include\msgpack\
+copy src\msgpack\unpack_define.h    include\msgpack\
+copy src\msgpack\unpack_template.h  include\msgpack\
+copy src\msgpack\sysdep.h           include\msgpack\
+copy src\msgpack.h                     include\
+copy src\msgpack\sbuffer.h             include\msgpack\
+copy src\msgpack\version.h             include\msgpack\
+copy src\msgpack\vrefbuffer.h          include\msgpack\
+copy src\msgpack\zbuffer.h             include\msgpack\
+copy src\msgpack\pack.h                include\msgpack\
+copy src\msgpack\unpack.h              include\msgpack\
+copy src\msgpack\object.h              include\msgpack\
+copy src\msgpack\zone.h                include\msgpack\
+copy src\msgpack.hpp                   include\
+copy src\msgpack\sbuffer.hpp           include\msgpack\
+copy src\msgpack\vrefbuffer.hpp        include\msgpack\
+copy src\msgpack\zbuffer.hpp           include\msgpack\
+copy src\msgpack\pack.hpp              include\msgpack\
+copy src\msgpack\unpack.hpp            include\msgpack\
+copy src\msgpack\object.hpp            include\msgpack\
+copy src\msgpack\zone.hpp              include\msgpack\
+copy src\msgpack\type.hpp              include\msgpack\type\
+copy src\msgpack\type\bool.hpp         include\msgpack\type\
+copy src\msgpack\type\float.hpp        include\msgpack\type\
+copy src\msgpack\type\int.hpp          include\msgpack\type\
+copy src\msgpack\type\list.hpp         include\msgpack\type\
+copy src\msgpack\type\deque.hpp        include\msgpack\type\
+copy src\msgpack\type\map.hpp          include\msgpack\type\
+copy src\msgpack\type\nil.hpp          include\msgpack\type\
+copy src\msgpack\type\pair.hpp         include\msgpack\type\
+copy src\msgpack\type\raw.hpp          include\msgpack\type\
+copy src\msgpack\type\set.hpp          include\msgpack\type\
+copy src\msgpack\type\string.hpp       include\msgpack\type\
+copy src\msgpack\type\vector.hpp       include\msgpack\type\
+copy src\msgpack\type\tuple.hpp        include\msgpack\type\
+copy src\msgpack\type\define.hpp       include\msgpack\type\
+copy src\msgpack\type\tr1\unordered_map.hpp  include\msgpack\type\
+copy src\msgpack\type\tr1\unordered_set.hpp  include\msgpack\type\
 
diff --git a/cpp/msgpack_vc8.vcproj b/cpp/msgpack_vc8.vcproj
index 5804790..ed0daa4 100644
--- a/cpp/msgpack_vc8.vcproj
+++ b/cpp/msgpack_vc8.vcproj
@@ -157,7 +157,7 @@
 			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
 			>
 			
 				
 			
 			
 				
 			
 			
 				
 			
 			
 				
 			
 			
+				
+					
+				
+				
+					
+				
+			
+			
 			
 		
@@ -247,23 +267,23 @@
 			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
 			>
 			
 			
 			
 			
 			
 			
 			
 			
 			
 			
 		
diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am
index fc67385..a6910b4 100644
--- a/cpp/src/Makefile.am
+++ b/cpp/src/Makefile.am
@@ -38,6 +38,7 @@ nobase_include_HEADERS = \
 		msgpack/sysdep.h \
 		msgpack.h \
 		msgpack/sbuffer.h \
+		msgpack/version.h \
 		msgpack/vrefbuffer.h \
 		msgpack/zbuffer.h \
 		msgpack/pack.h \
@@ -49,7 +50,6 @@ if ENABLE_CXX
 nobase_include_HEADERS += \
 		msgpack.hpp \
 		msgpack/sbuffer.hpp \
-		msgpack/version.h \
 		msgpack/vrefbuffer.hpp \
 		msgpack/zbuffer.hpp \
 		msgpack/pack.hpp \

From fe77251242f34e08f49f41fbbb2561e9278d8635 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 6 Jul 2010 19:16:49 +0900
Subject: [PATCH 0566/1648] cpp: fixes missing dependency to generate version.h

---
 cpp/src/Makefile.am | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am
index a6910b4..e12eb24 100644
--- a/cpp/src/Makefile.am
+++ b/cpp/src/Makefile.am
@@ -88,6 +88,8 @@ msgpack/version.h: msgpack/version.h.in Makefile.in
 		-e s/VERSION_MINOR_UNDEFINED/$(VERSION_MINOR)/ \
 		$< > $@
 
+version.c: msgpack/version.h
+
 
 doxygen_c:
 	cat ../Doxyfile > Doxyfile_c

From 167e2475d89cb867428b9e7b5aac7f269fd95ecb Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 6 Jul 2010 23:30:15 +0900
Subject: [PATCH 0567/1648] cpp: generate version.h using AC_OUTPUT macro in
 ./configure

---
 cpp/configure.in             | 5 ++++-
 cpp/src/Makefile.am          | 9 ---------
 cpp/src/msgpack/version.h.in | 6 +++---
 3 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/cpp/configure.in b/cpp/configure.in
index dd04ed4..ab29501 100644
--- a/cpp/configure.in
+++ b/cpp/configure.in
@@ -68,5 +68,8 @@ AC_SUBST(VERSION_MAJOR, $major)
 AC_SUBST(VERSION_MINOR, $minor)
 
 
-AC_OUTPUT([Makefile src/Makefile test/Makefile])
+AC_OUTPUT([Makefile
+		   src/Makefile
+		   src/msgpack/version.h
+		   test/Makefile])
 
diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am
index e12eb24..31096f0 100644
--- a/cpp/src/Makefile.am
+++ b/cpp/src/Makefile.am
@@ -82,15 +82,6 @@ EXTRA_DIST = \
 		msgpack/type/tuple.hpp.erb
 
 
-msgpack/version.h: msgpack/version.h.in Makefile.in
-	sed -e s/VERSION_UNDEFINED/$(VERSION)/ \
-		-e s/VERSION_MAJOR_UNDEFINED/$(VERSION_MAJOR)/ \
-		-e s/VERSION_MINOR_UNDEFINED/$(VERSION_MINOR)/ \
-		$< > $@
-
-version.c: msgpack/version.h
-
-
 doxygen_c:
 	cat ../Doxyfile > Doxyfile_c
 	echo "FILE_PATTERNS      = *.h" >> Doxyfile_c
diff --git a/cpp/src/msgpack/version.h.in b/cpp/src/msgpack/version.h.in
index af292d0..f1feb33 100644
--- a/cpp/src/msgpack/version.h.in
+++ b/cpp/src/msgpack/version.h.in
@@ -27,9 +27,9 @@ const char* msgpack_version(void);
 int msgpack_version_major(void);
 int msgpack_version_minor(void);
 
-#define MSGPACK_VERSION "VERSION_UNDEFINED"
-#define MSGPACK_VERSION_MAJOR VERSION_MAJOR_UNDEFINED
-#define MSGPACK_VERSION_MINOR VERSION_MINOR_UNDEFINED
+#define MSGPACK_VERSION "@VERSION@"
+#define MSGPACK_VERSION_MAJOR @VERSION_MAJOR@
+#define MSGPACK_VERSION_MINOR @VERSION_MINOR@
 
 
 #ifdef __cplusplus

From 45fb482ab42c7f47bf65313ade6808d0cfe3bca5 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Thu, 8 Jul 2010 23:36:18 +0900
Subject: [PATCH 0568/1648] erlang: added simple performance test.

---
 erlang/msgpack.erl | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index aab07b7..ee99311 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -359,4 +359,9 @@ test_([Before|Rest])->
 other_test()->
     {more,1}=msgpack:unpack(<<>>).
 
+benchmark_test()->
+    Data=[test_data() || _ <- lists:seq(0, 10000)],
+    S=?debugTime("  serialize", msgpack:pack(Data)),
+    {Data,<<>>}=?debugTime("deserialize", msgpack:unpack(S)).
+
 -endif.

From 485915c27a3ddf12e4ba4c9c0e27769869bb945c Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Thu, 8 Jul 2010 23:39:47 +0900
Subject: [PATCH 0569/1648] erlang: added simple performance test description.

---
 erlang/msgpack.erl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index ee99311..94fed86 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -362,6 +362,7 @@ other_test()->
 benchmark_test()->
     Data=[test_data() || _ <- lists:seq(0, 10000)],
     S=?debugTime("  serialize", msgpack:pack(Data)),
-    {Data,<<>>}=?debugTime("deserialize", msgpack:unpack(S)).
+    {Data,<<>>}=?debugTime("deserialize", msgpack:unpack(S)),
+    ?debugFmt("for ~p KB test data.", [byte_size(S) div 1024]).
 
 -endif.

From eab66a022e5b5fd9c4731ae8ba970b2146e27599 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Fri, 9 Jul 2010 01:04:09 +0900
Subject: [PATCH 0570/1648] erlang: added try-catch clause for easy error
 handling

---
 erlang/msgpack.erl | 220 +++++++++++++++++++++++----------------------
 1 file changed, 112 insertions(+), 108 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 94fed86..d4fd0ba 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -19,9 +19,8 @@
 -author('kuenishi+msgpack@gmail.com').
 
 %% tuples, atoms are not supported. lists, integers, double, and so on.
-%% see http://msgpack.sourceforge.jp/spec for
-%% supported formats. APIs are almost compatible
-%% for C API (http://msgpack.sourceforge.jp/c:doc)
+%% see http://msgpack.sourceforge.jp/spec for supported formats.
+%% APIs are almost compatible with C API (http://msgpack.sourceforge.jp/c:doc)
 %% except buffering functions (both copying and zero-copying).
 -export([pack/1, unpack/1, unpack_all/1]).
 -export([pack_map/1]).
@@ -30,51 +29,38 @@
 % erl> c(msgpack).
 % erl> S = .
 % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
--type reason() ::  enomem | badarg | no_code_matches.
+-type reason() ::  enomem | badarg | no_code_matches | undefined.
 -type msgpack_term() :: [msgpack_term()]
 		      | {[{msgpack_term(),msgpack_term()}]}
 		      | integer() | float() | binary().
 
 % ===== external APIs ===== %
--spec pack(Term::msgpack_term()) -> binary().
-pack(I) when is_integer(I) andalso I < 0 ->
-    pack_int_(I);
-pack(I) when is_integer(I) ->
-    pack_uint_(I);
-pack(F) when is_float(F) ->
-    pack_double(F);
-pack(nil) ->
-    << 16#C0:8 >>;
-pack(true) ->
-    << 16#C3:8 >>;
-pack(false) ->
-    << 16#C2:8 >>;
-pack(Bin) when is_binary(Bin) ->
-    pack_raw(Bin);
-pack(List)  when is_list(List) ->
-    pack_array(List);
-pack({Map}) when is_list(Map) ->
-    pack_map(Map);
-pack(Map) when is_tuple(Map), element(1,Map)=:=dict ->
-    pack_map(dict:to_list(Map));
-pack(_Other) ->
-    {error, undefined}.
+-spec pack(Term::msgpack_term()) -> binary() | {error, reason()}.
+pack(Term)->
+    try
+	pack_(Term)
+    catch
+	error:Error when is_tuple(Error), element(1, Error) =:= error ->
+	    Error;
+	throw:Exception ->
+	    erlang:display(Exception),
+	    {error, Exception}
+    end.
 
 % unpacking.
 % if failed in decoding and not end, get more data
 % and feed more Bin into this function.
 % TODO: error case for imcomplete format when short for any type formats.
--spec unpack( Bin::binary() )-> {msgpack_term(), binary()} |
-				{more, non_neg_integer()} | {more, undefined} |
-				{error, reason()}.
-unpack(Bin) when not is_binary(Bin)->
-    {error, badarg};
-unpack(Bin) when bit_size(Bin) >= 8 ->
-    unpack_(Bin);
-unpack(<<>>)->
-    {more, 1};
-unpack(_) ->
-    {more, undefined}.
+-spec unpack( Bin::binary() )-> {msgpack_term(), binary()} | {error, reason()}.
+unpack(Bin)->
+    try
+	unpack_(Bin)
+    catch
+	error:Error when is_tuple(Error), element(1, Error) =:= error ->
+	    Error;
+	throw:Exception ->
+	    {error, Exception}
+    end.
 
 -spec unpack_all( binary() ) -> [msgpack_term()].
 unpack_all(Data)->
@@ -85,7 +71,7 @@ unpack_all(Data)->
 	    [Term|unpack_all(Binary)]
     end.
 
--spec pack_map(M::[{msgpack_term(),msgpack_term()}])-> binary().
+-spec pack_map(M::[{msgpack_term(),msgpack_term()}])-> binary() | {error, badarg}.
 pack_map(M)->
     case length(M) of
 	Len when Len < 16 ->
@@ -98,6 +84,31 @@ pack_map(M)->
 
 % ===== internal APIs ===== %
 
+% pack them all
+-spec pack_(msgpack_term()) -> binary() | no_return().
+pack_(I) when is_integer(I) andalso I < 0 ->
+    pack_int_(I);
+pack_(I) when is_integer(I) ->
+    pack_uint_(I);
+pack_(F) when is_float(F) ->
+    pack_double(F);
+pack_(nil) ->
+    << 16#C0:8 >>;
+pack_(true) ->
+    << 16#C3:8 >>;
+pack_(false) ->
+    << 16#C2:8 >>;
+pack_(Bin) when is_binary(Bin) ->
+    pack_raw(Bin);
+pack_(List)  when is_list(List) ->
+    pack_array(List);
+pack_({Map}) when is_list(Map) ->
+    pack_map(Map);
+pack_(Map) when is_tuple(Map), element(1,Map)=:=dict ->
+    pack_map(dict:to_list(Map));
+pack_(_Other) ->
+    throw({error, undefined}).
+
 % positive fixnum
 pack_uint_(N) when N < 128 ->
     << 2#0:1, N:7 >>;
@@ -149,7 +160,7 @@ pack_raw(Bin) ->
 	    << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >>
     end.
 
-% list / tuple
+% list
 pack_array(L) ->
     case length(L) of
  	Len when Len < 16 ->
@@ -159,43 +170,40 @@ pack_array(L) ->
 	Len ->
 	    << 16#DD:8, Len:32/big-unsigned-integer-unit:1,(pack_array_(L, <<>>))/binary >>
     end.
+
 pack_array_([], Acc) -> Acc;
 pack_array_([Head|Tail], Acc) ->
-    pack_array_(Tail, <>).
+    pack_array_(Tail, <>).
 
-% FIXME! this should be without lists:reverse/1
-unpack_array_(<<>>, 0, RetList)   -> {lists:reverse(RetList), <<>>};
-unpack_array_(Remain, 0, RetList) when is_binary(Remain)-> {lists:reverse(RetList), Remain};
-unpack_array_(<<>>, RestLen, _RetList) when RestLen > 0 ->  {more, undefined};
-unpack_array_(Bin, RestLen, RetList) when is_binary(Bin)->
-    case unpack(Bin) of
-	{more, _} -> {more, undefined};
-	{Term, Rest}-> unpack_array_(Rest, RestLen-1, [Term|RetList])
-    end.
+% Users SHOULD NOT send too long list: this uses lists:reverse/1
+unpack_array_(Remain, 0,    Acc) when is_binary(Remain)-> {lists:reverse(Acc), Remain};
+unpack_array_(<<>>, RestLen,  _) when RestLen > 0 -> throw(short);
+unpack_array_(Bin, RestLen, Acc) when is_binary(Bin)->
+    {Term, Rest}=unpack_(Bin),
+    unpack_array_(Rest, RestLen-1, [Term|Acc]).
 
 pack_map_([], Acc) -> Acc;
 pack_map_([{Key,Value}|Tail], Acc) ->
-    pack_map_(Tail, << Acc/binary, (pack(Key))/binary, (pack(Value))/binary>>).
+    pack_map_(Tail, << Acc/binary, (pack_(Key))/binary, (pack_(Value))/binary>>).
 
-% FIXME! this should be without lists:reverse/1
--spec unpack_map_(binary(), non_neg_integer(), [{term(), msgpack_term()}])->
-    {more, non_neg_integer()} | { any(), binary()}.
+% Users SHOULD NOT send too long list: this uses lists:reverse/1
+-spec unpack_map_(binary(), non_neg_integer(), [{msgpack_term(), msgpack_term()}])->
+			 {[{msgpack_term(), msgpack_term()}], binary()} | no_return().
 unpack_map_(Bin,  0,  Acc) -> {{lists:reverse(Acc)}, Bin};
+unpack_map_(<<>>, _,  _ )  -> throw(short);
 unpack_map_(Bin, Len, Acc) ->
-    case unpack(Bin) of
-	{more, _} -> {more, undefined};
-	{Key, Rest} ->
-	    case unpack(Rest) of
-		{more, _} -> {more, undefined};
-		{Value, Rest2} ->
-		    unpack_map_(Rest2,Len-1,[{Key,Value}|Acc])
-	    end
-    end.
+    {Key, Rest} = unpack_(Bin),
+    {Value, Rest2} = unpack_(Rest),
+    unpack_map_(Rest2,Len-1,[{Key,Value}|Acc]).
 
--spec unpack_(Payload::binary()) -> 
-		     {more, pos_integer()} | {msgpack_term(), binary()} | {error, reason()}.
-unpack_(Binary)->
-    case Binary of
+% unpack then all
+-spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | {error, reason()} | no_return().
+unpack_(Bin) when not is_binary(Bin)->
+    throw(badarg);
+unpack_(<<>>)->
+    throw(short);
+unpack_(Bin) when bit_size(Bin) >= 8 ->
+    case Bin of
 % ATOMS
 	<<16#C0, Rest/binary>> -> {nil, Rest};
 	<<16#C2, Rest/binary>> -> {false, Rest};
@@ -231,35 +239,36 @@ unpack_(Binary)->
 	<<2#1000:4, L:4, Rest/binary>> ->            unpack_map_(Rest, L, []);   % map
 	
 % Incomplete / invalid data
-	<<16#CA, Rest/binary>> -> {more, 4-byte_size(Rest)};
-	<<16#CB, Rest/binary>> -> {more, 8-byte_size(Rest)};
-	<<16#CC>> ->              {more, 1};
-	<<16#CD, Rest/binary>> -> {more, 2-byte_size(Rest)};
-	<<16#CE, Rest/binary>> -> {more, 4-byte_size(Rest)};
-	<<16#CF, Rest/binary>> -> {more, 8-byte_size(Rest)};
-	<<16#D0>> ->              {more, 1};
-	<<16#D1, Rest/binary>> -> {more, 2-byte_size(Rest)};
-	<<16#D2, Rest/binary>> -> {more, 4-byte_size(Rest)};
-	<<16#D3, Rest/binary>> -> {more, 8-byte_size(Rest)};
-	<<16#DA, Rest/binary>> -> {more, 16-byte_size(Rest)};
-	<<16#DB, Rest/binary>> -> {more, 32-byte_size(Rest)};
-	<<16#DC, Rest/binary>> -> {more, 2-byte_size(Rest)};
-	<<16#DD, Rest/binary>> -> {more, 4-byte_size(Rest)};
-	<<16#DE, Rest/binary>> -> {more, 2-byte_size(Rest)};
-	<<16#DF, Rest/binary>> -> {more, 4-byte_size(Rest)};
-	<<2#101:3, L:5, Rest/binary>> ->  {more, L-byte_size(Rest)};
+%	<<_:16/integer, _/binary>>
+	_ -> throw(short)
+%% 	<<16#CA, _/binary>> -> {more, 4-byte_size(Rest)};
+%% 	<<16#CB, Rest/binary>> -> {more, 8-byte_size(Rest)};
+%% 	<<16#CC>> ->              {more, 1};
+%% 	<<16#CD, Rest/binary>> -> {more, 2-byte_size(Rest)};
+%% 	<<16#CE, Rest/binary>> -> {more, 4-byte_size(Rest)};
+%% 	<<16#CF, Rest/binary>> -> {more, 8-byte_size(Rest)};
+%% 	<<16#D0>> ->              {more, 1};
+%% 	<<16#D1, Rest/binary>> -> {more, 2-byte_size(Rest)};
+%% 	<<16#D2, Rest/binary>> -> {more, 4-byte_size(Rest)};
+%% 	<<16#D3, Rest/binary>> -> {more, 8-byte_size(Rest)};
+%% 	<<16#DA, Rest/binary>> -> {more, 16-byte_size(Rest)};
+%% 	<<16#DB, Rest/binary>> -> {more, 32-byte_size(Rest)};
+%% 	<<16#DC, Rest/binary>> -> {more, 2-byte_size(Rest)};
+%% 	<<16#DD, Rest/binary>> -> {more, 4-byte_size(Rest)};
+%% 	<<16#DE, Rest/binary>> -> {more, 2-byte_size(Rest)};
+%% 	<<16#DF, Rest/binary>> -> {more, 4-byte_size(Rest)};
+%% 	<<2#101:3, L:5, Rest/binary>> ->  throw(short); % {more, L-byte_size(Rest)};
 
-	<<>> ->                  {more, 1};
-	<<2#101:3, _/binary>> -> {more, undefined};
-	<> when F==16#C1;
-				  F==16#C7; F==16#C8; F==16#C9; F==16#D5;
-				  F==16#D6; F==16#D7; F==16#D8; F==16#D9->
-	    {error, {badarg, <>}};
-	Other ->
-	    {error, {badarg, Other}}
+%% 	<<>> ->   throw(short); %               {more, 1};
+%% 	<<2#101:3, _/binary>> -> {more, undefined};
+%% 	<> when F==16#C1;
+%% 				  F==16#C7; F==16#C8; F==16#C9; F==16#D5;
+%% 				  F==16#D6; F==16#D7; F==16#D8; F==16#D9->
+%% 	    throw({badarg, <>});
+%	Other ->
+%	    throw({unknown, Other})
     end.
 
-
 % ===== test codes ===== %
 -include_lib("eunit/include/eunit.hrl").
 -ifdef(EUNIT).
@@ -268,9 +277,15 @@ compare_all([], [])-> ok;
 compare_all([], R)-> {toomuchrhs, R};
 compare_all(L, [])-> {toomuchlhs, L};
 compare_all([LH|LTL], [RH|RTL]) ->
-    LH=RH,
+    ?assertEqual(LH, RH),
     compare_all(LTL, RTL).
 
+test_([]) -> 0;
+test_([Term|Rest])->
+    Pack = msgpack:pack(Term),
+    ?assertEqual({Term, <<>>}, msgpack:unpack( Pack )),
+    1+test_(Rest).
+
 test_data()->
     [true, false, nil,
      0, 1, 2, 123, 512, 1230, 678908, 16#FFFFFFFFFF,
@@ -303,12 +318,7 @@ test_p(Len,Term,OrigBin,Len) ->
     {Term, <<>>}=msgpack:unpack(OrigBin);
 test_p(I,_,OrigBin,Len) when I < Len->
     <> = OrigBin,
-    case msgpack:unpack(Bin) of
-	{more, N} when not is_integer(N) ->
-	    ?assertEqual(undefined, N);
-	{more, N} ->
-	    ?assert( N < Len )
-    end.
+    ?assertEqual({error,short}, msgpack:unpack(Bin)).
 
 partial_test()-> % error handling test.
     Term = lists:seq(0, 45),
@@ -343,21 +353,15 @@ unknown_test()->
 	     42
 	    ],
     Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]),
+    timer:sleep(1),
     receive
 	{Port, {data, Data}}->
 	    compare_all(Tests, msgpack:unpack_all(Data))
     after 1024-> ?assert(false)   end,
     port_close(Port).
 
-test_([]) -> 0;
-test_([Before|Rest])->
-    Pack = msgpack:pack(Before),
-    {After, <<>>} = msgpack:unpack( Pack ),
-    ?assertEqual(Before, After),
-    1+test_(Rest).
-
 other_test()->
-    {more,1}=msgpack:unpack(<<>>).
+    ?assertEqual({error,short},msgpack:unpack(<<>>)).
 
 benchmark_test()->
     Data=[test_data() || _ <- lists:seq(0, 10000)],

From e799082e5c4d39094c666da0f1c52ab2d6eb088c Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Fri, 9 Jul 2010 01:21:35 +0900
Subject: [PATCH 0571/1648] erlang: better test cases, except 'Broken pipe'

---
 erlang/msgpack.erl | 74 +++++++++++++++++-----------------------------
 1 file changed, 27 insertions(+), 47 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index d4fd0ba..96ea407 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -142,7 +142,7 @@ pack_int_(N) ->
     << 16#D3:8, N:64/big-signed-integer-unit:1 >>.
 
 % float : erlang's float is always IEEE 754 64bit format.
-%pack_float(F) when is_float(F)->
+% pack_float(F) when is_float(F)->
 %    << 16#CA:8, F:32/big-float-unit:1 >>.
 %    pack_double(F).
 % double
@@ -198,10 +198,7 @@ unpack_map_(Bin, Len, Acc) ->
 
 % unpack then all
 -spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | {error, reason()} | no_return().
-unpack_(Bin) when not is_binary(Bin)->
-    throw(badarg);
-unpack_(<<>>)->
-    throw(short);
+unpack_(Bin) when not is_binary(Bin)->    throw(badarg);
 unpack_(Bin) when bit_size(Bin) >= 8 ->
     case Bin of
 % ATOMS
@@ -239,43 +236,31 @@ unpack_(Bin) when bit_size(Bin) >= 8 ->
 	<<2#1000:4, L:4, Rest/binary>> ->            unpack_map_(Rest, L, []);   % map
 	
 % Incomplete / invalid data
-%	<<_:16/integer, _/binary>>
-	_ -> throw(short)
-%% 	<<16#CA, _/binary>> -> {more, 4-byte_size(Rest)};
-%% 	<<16#CB, Rest/binary>> -> {more, 8-byte_size(Rest)};
-%% 	<<16#CC>> ->              {more, 1};
-%% 	<<16#CD, Rest/binary>> -> {more, 2-byte_size(Rest)};
-%% 	<<16#CE, Rest/binary>> -> {more, 4-byte_size(Rest)};
-%% 	<<16#CF, Rest/binary>> -> {more, 8-byte_size(Rest)};
-%% 	<<16#D0>> ->              {more, 1};
-%% 	<<16#D1, Rest/binary>> -> {more, 2-byte_size(Rest)};
-%% 	<<16#D2, Rest/binary>> -> {more, 4-byte_size(Rest)};
-%% 	<<16#D3, Rest/binary>> -> {more, 8-byte_size(Rest)};
-%% 	<<16#DA, Rest/binary>> -> {more, 16-byte_size(Rest)};
-%% 	<<16#DB, Rest/binary>> -> {more, 32-byte_size(Rest)};
-%% 	<<16#DC, Rest/binary>> -> {more, 2-byte_size(Rest)};
-%% 	<<16#DD, Rest/binary>> -> {more, 4-byte_size(Rest)};
-%% 	<<16#DE, Rest/binary>> -> {more, 2-byte_size(Rest)};
-%% 	<<16#DF, Rest/binary>> -> {more, 4-byte_size(Rest)};
-%% 	<<2#101:3, L:5, Rest/binary>> ->  throw(short); % {more, L-byte_size(Rest)};
-
-%% 	<<>> ->   throw(short); %               {more, 1};
-%% 	<<2#101:3, _/binary>> -> {more, undefined};
-%% 	<> when F==16#C1;
-%% 				  F==16#C7; F==16#C8; F==16#C9; F==16#D5;
-%% 				  F==16#D6; F==16#D7; F==16#D8; F==16#D9->
-%% 	    throw({badarg, <>});
-%	Other ->
-%	    throw({unknown, Other})
-    end.
+	<> when F==16#CA; F==16#CB; F==16#CC;
+				F==16#CD; F==16#CE; F==16#CF;
+				F==16#D0; F==16#D1; F==16#D2;
+				F==16#D3; F==16#DA; F==16#DB;
+				F==16#DC; F==16#DD; F==16#DE;
+				F==16#DF ->
+	    throw(short);
+	<> when F==16#C1;
+				F==16#C7; F==16#C8; F==16#C9;
+				F==16#D5; F==16#D6; F==16#D7;
+				F==16#D8; F==16#D9 ->
+	    throw(badarg);
+	_ ->
+	    throw(short) % or unknown/badarg?
+    end;
+unpack_(<<>>)->                    throw(short);
+unpack_(<<2#101:3, _/binary>>) ->  throw(short).
 
 % ===== test codes ===== %
 -include_lib("eunit/include/eunit.hrl").
 -ifdef(EUNIT).
 
 compare_all([], [])-> ok;
-compare_all([], R)-> {toomuchrhs, R};
-compare_all(L, [])-> {toomuchlhs, L};
+compare_all([],  R)-> {toomuchrhs, R};
+compare_all(L,  [])-> {toomuchlhs, L};
 compare_all([LH|LTL], [RH|RTL]) ->
     ?assertEqual(LH, RH),
     compare_all(LTL, RTL).
@@ -305,9 +290,9 @@ basic_test()->
     Passed = length(Tests).
 
 port_test()->
+    Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]),
     Tests = test_data(),
     {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])),
-    Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]),
     true = port_command(Port, msgpack:pack(Tests) ),
     receive
 	{Port, {data, Data}}->  {Tests, <<>>}=msgpack:unpack(Data)
@@ -328,10 +313,7 @@ partial_test()-> % error handling test.
 
 long_test()->
     Longer = lists:seq(0, 655),
-%    Longest = lists:seq(0,12345),
-    {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)),
-%    {Longest, <<>>} = msgpack:unpack(msgpack:pack(Longest)).
-    ok.
+    {Longer, <<>>} = msgpack:unpack(msgpack:pack(Longer)).
 
 map_test()->
     Ints = lists:seq(0, 65),
@@ -341,6 +323,7 @@ map_test()->
     ok.
 
 unknown_test()->
+    Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]),
     Tests = [0, 1, 2, 123, 512, 1230, 678908,
 	     -1, -23, -512, -1230, -567898,
 	     <<"hogehoge">>, <<"243546rf7g68h798j">>,
@@ -348,16 +331,13 @@ unknown_test()->
 	     -234.4355, 1.0e-34, 1.0e64,
 	     [23, 234, 0.23],
 	     [0,42,<<"sum">>, [1,2]], [1,42, nil, [3]],
-	     {[{1,2},{<<"hoge">>,nil}]},
+	     {[{1,2},{<<"hoge">>,nil}]}, % map
 	     -234, -50000,
 	     42
 	    ],
-    Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]),
-    timer:sleep(1),
     receive
-	{Port, {data, Data}}->
-	    compare_all(Tests, msgpack:unpack_all(Data))
-    after 1024-> ?assert(false)   end,
+	{Port, {data, Data}}-> compare_all(Tests, msgpack:unpack_all(Data))
+    after 1024-> ?assert(false)  end,
     port_close(Port).
 
 other_test()->

From 64c36b7a8faac55d8dc80342f27929b5538c4307 Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Fri, 9 Jul 2010 13:06:57 +0200
Subject: [PATCH 0572/1648] Remove a couple of superfluous 'when' clauses.

The when clause for unpack_/1 has been moved to unpack/1 so that it is performed only once.
---
 erlang/msgpack.erl | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 1291d54..d1ba9cd 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -52,7 +52,9 @@ pack(Term)->
 % and feed more Bin into this function.
 % TODO: error case for imcomplete format when short for any type formats.
 -spec unpack( Bin::binary() )-> {msgpack_term(), binary()} | {error, reason()}.
-unpack(Bin)->
+unpack(Bin) when not is_binary(Bin) ->
+    {error, badarg};
+unpack(Bin) ->
     try
 	unpack_(Bin)
     catch
@@ -126,7 +128,7 @@ pack_uint_(N) ->
     << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>.
 
 % negative fixnum
-pack_int_(N) when is_integer(N) , N >= -32->
+pack_int_(N) when N >= -32->
     << 2#111:3, N:5 >>;
 % int 8
 pack_int_(N) when N > -128 ->
@@ -176,9 +178,9 @@ pack_array_([Head|Tail], Acc) ->
     pack_array_(Tail, <>).
 
 % Users SHOULD NOT send too long list: this uses lists:reverse/1
-unpack_array_(Remain, 0,    Acc) when is_binary(Remain)-> {lists:reverse(Acc), Remain};
-unpack_array_(<<>>, RestLen,  _) when RestLen > 0 -> throw(short);
-unpack_array_(Bin, RestLen, Acc) when is_binary(Bin)->
+unpack_array_(Remain, 0,    Acc) -> {lists:reverse(Acc), Remain};
+unpack_array_(<<>>, RestLen,  _) -> throw(short);
+unpack_array_(Bin, RestLen, Acc) ->
     {Term, Rest}=unpack_(Bin),
     unpack_array_(Rest, RestLen-1, [Term|Acc]).
 
@@ -198,8 +200,7 @@ unpack_map_(Bin, Len, Acc) ->
 
 % unpack then all
 -spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | {error, reason()} | no_return().
-unpack_(Bin) when not is_binary(Bin)->    throw(badarg);
-unpack_(Bin) when bit_size(Bin) >= 8 ->
+unpack_(Bin) ->
     case Bin of
 % ATOMS
 	<<16#C0, Rest/binary>> -> {nil, Rest};
@@ -234,7 +235,7 @@ unpack_(Bin) when bit_size(Bin) >= 8 ->
 	<<2#101:3, L:5, V:L/binary, Rest/binary>> -> {V, Rest};                  % raw bytes
 	<<2#1001:4, L:4, Rest/binary>> ->            unpack_array_(Rest, L, []); % array
 	<<2#1000:4, L:4, Rest/binary>> ->            unpack_map_(Rest, L, []);   % map
-	
+
 % Incomplete / invalid data
 	<> when F==16#CA; F==16#CB; F==16#CC;
 				F==16#CD; F==16#CE; F==16#CF;

From 6abc120279ced47d899cd8596e4d48fc2171e2cf Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Fri, 9 Jul 2010 13:29:47 +0200
Subject: [PATCH 0573/1648] erlang: Fix incomplete/invalid cases of unpack_/1

* fix list of invalid bytes was missing 3 possibilities (see type chart section of msgpack format spec)
* fix matching of invalid bytes to look at 1 byte instead of 2
* simplify 'incomplete' case : anything that's not complete or invalid is by definition incomplete
---
 erlang/msgpack.erl | 23 +++++++----------------
 1 file changed, 7 insertions(+), 16 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index d1ba9cd..1c1eae9 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -236,24 +236,15 @@ unpack_(Bin) ->
 	<<2#1001:4, L:4, Rest/binary>> ->            unpack_array_(Rest, L, []); % array
 	<<2#1000:4, L:4, Rest/binary>> ->            unpack_map_(Rest, L, []);   % map
 
-% Incomplete / invalid data
-	<> when F==16#CA; F==16#CB; F==16#CC;
-				F==16#CD; F==16#CE; F==16#CF;
-				F==16#D0; F==16#D1; F==16#D2;
-				F==16#D3; F==16#DA; F==16#DB;
-				F==16#DC; F==16#DD; F==16#DE;
-				F==16#DF ->
-	    throw(short);
-	<> when F==16#C1;
-				F==16#C7; F==16#C8; F==16#C9;
-				F==16#D5; F==16#D6; F==16#D7;
-				F==16#D8; F==16#D9 ->
+% Invalid data
+	<> when F==16#C1;
+	                     F==16#C4; F==16#C5; F==16#C6; F==16#C7; F==16#C8; F==16#C9;
+	                     F==16#D4; F==16#D5; F==16#D6; F==16#D7; F==16#D8; F==16#D9 ->
 	    throw(badarg);
+% Incomplete data (we've covered every complete/invalid case; anything left is incomplete)
 	_ ->
-	    throw(short) % or unknown/badarg?
-    end;
-unpack_(<<>>)->                    throw(short);
-unpack_(<<2#101:3, _/binary>>) ->  throw(short).
+	    throw(short)
+    end.
 
 % ===== test codes ===== %
 -include_lib("eunit/include/eunit.hrl").

From ba4a971bfaabe7da2159a634cd07977e06c61e3a Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Fri, 9 Jul 2010 13:44:02 +0200
Subject: [PATCH 0574/1648] erlang: Remove unecessary 'throw(short)' clause for
 unpack_{array,map}_/1

Unecessary because unpack_/1 will throw it anyway.
This does mean that we go a tiny bit deeper to find that we don't have enough data,
but that should be a rare code path. Keep the main code path fast and the code clean.

While at it, rename vars to match its sibling function and to avoid thinking that
RestLen is a byte count (it's an item count).
---
 erlang/msgpack.erl | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 1c1eae9..ff3eac7 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -178,11 +178,10 @@ pack_array_([Head|Tail], Acc) ->
     pack_array_(Tail, <>).
 
 % Users SHOULD NOT send too long list: this uses lists:reverse/1
-unpack_array_(Remain, 0,    Acc) -> {lists:reverse(Acc), Remain};
-unpack_array_(<<>>, RestLen,  _) -> throw(short);
-unpack_array_(Bin, RestLen, Acc) ->
-    {Term, Rest}=unpack_(Bin),
-    unpack_array_(Rest, RestLen-1, [Term|Acc]).
+unpack_array_(Bin, 0,   Acc) -> {lists:reverse(Acc), Bin};
+unpack_array_(Bin, Len, Acc) ->
+    {Term, Rest} = unpack_(Bin),
+    unpack_array_(Rest, Len-1, [Term|Acc]).
 
 pack_map_([], Acc) -> Acc;
 pack_map_([{Key,Value}|Tail], Acc) ->
@@ -191,14 +190,13 @@ pack_map_([{Key,Value}|Tail], Acc) ->
 % Users SHOULD NOT send too long list: this uses lists:reverse/1
 -spec unpack_map_(binary(), non_neg_integer(), [{msgpack_term(), msgpack_term()}])->
 			 {[{msgpack_term(), msgpack_term()}], binary()} | no_return().
-unpack_map_(Bin,  0,  Acc) -> {{lists:reverse(Acc)}, Bin};
-unpack_map_(<<>>, _,  _ )  -> throw(short);
+unpack_map_(Bin, 0,   Acc) -> {{lists:reverse(Acc)}, Bin};
 unpack_map_(Bin, Len, Acc) ->
     {Key, Rest} = unpack_(Bin),
     {Value, Rest2} = unpack_(Rest),
-    unpack_map_(Rest2,Len-1,[{Key,Value}|Acc]).
+    unpack_map_(Rest2, Len-1, [{Key,Value}|Acc]).
 
-% unpack then all
+% unpack them all
 -spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | {error, reason()} | no_return().
 unpack_(Bin) ->
     case Bin of

From a4258505a99087df2397cf8d6cfbf194311a995c Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Fri, 9 Jul 2010 23:23:00 +0900
Subject: [PATCH 0575/1648] erlang: modified wrong testcase.

---
 erlang/msgpack.erl | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index ff3eac7..e862cad 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -282,10 +282,11 @@ basic_test()->
 port_test()->
     Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]),
     Tests = test_data(),
-    {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])),
-    true = port_command(Port, msgpack:pack(Tests) ),
+    S=msgpack:pack([Tests]),
+    true = port_command(Port, S),
+    {[Tests],<<>>} = msgpack:unpack(S),
     receive
-	{Port, {data, Data}}->  {Tests, <<>>}=msgpack:unpack(Data)
+	{Port, {data, Data}}->  {[Tests], <<>>}=msgpack:unpack(Data)
     after 1024-> ?assert(false)   end,
     port_close(Port).
 

From 8a3f090684f0835958e60e6a5e3e81b4d8ff541a Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Fri, 9 Jul 2010 17:36:36 +0200
Subject: [PATCH 0576/1648] erlang: Fix some existing specs and add a few
 other.

dialyzer still complains about dict() and ?assert(false), but I don't think they're real issues.
---
 erlang/msgpack.erl | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index ff3eac7..fb9a3e1 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -29,7 +29,7 @@
 % erl> c(msgpack).
 % erl> S = .
 % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
--type reason() ::  enomem | badarg | no_code_matches | undefined.
+-type reason() ::  badarg | short.
 -type msgpack_term() :: [msgpack_term()]
 		      | {[{msgpack_term(),msgpack_term()}]}
 		      | integer() | float() | binary().
@@ -73,7 +73,7 @@ unpack_all(Data)->
 	    [Term|unpack_all(Binary)]
     end.
 
--spec pack_map(M::[{msgpack_term(),msgpack_term()}])-> binary() | {error, badarg}.
+-spec pack_map(M::[{msgpack_term(),msgpack_term()}]) -> binary() | no_return().
 pack_map(M)->
     case length(M) of
 	Len when Len < 16 ->
@@ -109,8 +109,9 @@ pack_({Map}) when is_list(Map) ->
 pack_(Map) when is_tuple(Map), element(1,Map)=:=dict ->
     pack_map(dict:to_list(Map));
 pack_(_Other) ->
-    throw({error, undefined}).
+    throw({error, badarg}).
 
+-spec pack_uint_(non_neg_integer()) -> binary().
 % positive fixnum
 pack_uint_(N) when N < 128 ->
     << 2#0:1, N:7 >>;
@@ -127,6 +128,7 @@ pack_uint_(N) when N < 16#FFFFFFFF->
 pack_uint_(N) ->
     << 16#CF:8, N:64/big-unsigned-integer-unit:1 >>.
 
+-spec pack_int_(integer()) -> binary().
 % negative fixnum
 pack_int_(N) when N >= -32->
     << 2#111:3, N:5 >>;
@@ -143,6 +145,7 @@ pack_int_(N) when N > -16#FFFFFFFF ->
 pack_int_(N) ->
     << 16#D3:8, N:64/big-signed-integer-unit:1 >>.
 
+-spec pack_double(float()) -> binary().
 % float : erlang's float is always IEEE 754 64bit format.
 % pack_float(F) when is_float(F)->
 %    << 16#CA:8, F:32/big-float-unit:1 >>.
@@ -151,6 +154,7 @@ pack_int_(N) ->
 pack_double(F) ->
     << 16#CB:8, F:64/big-float-unit:1 >>.
 
+-spec pack_raw(binary()) -> binary().
 % raw bytes
 pack_raw(Bin) ->
     case byte_size(Bin) of
@@ -162,6 +166,7 @@ pack_raw(Bin) ->
 	    << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >>
     end.
 
+-spec pack_array([msgpack_term()]) -> binary() | no_return().
 % list
 pack_array(L) ->
     case length(L) of
@@ -178,6 +183,7 @@ pack_array_([Head|Tail], Acc) ->
     pack_array_(Tail, <>).
 
 % Users SHOULD NOT send too long list: this uses lists:reverse/1
+-spec unpack_array_(binary(), non_neg_integer(), [msgpack_term()]) -> {[msgpack_term()], binary()} | no_return().
 unpack_array_(Bin, 0,   Acc) -> {lists:reverse(Acc), Bin};
 unpack_array_(Bin, Len, Acc) ->
     {Term, Rest} = unpack_(Bin),
@@ -188,8 +194,8 @@ pack_map_([{Key,Value}|Tail], Acc) ->
     pack_map_(Tail, << Acc/binary, (pack_(Key))/binary, (pack_(Value))/binary>>).
 
 % Users SHOULD NOT send too long list: this uses lists:reverse/1
--spec unpack_map_(binary(), non_neg_integer(), [{msgpack_term(), msgpack_term()}])->
-			 {[{msgpack_term(), msgpack_term()}], binary()} | no_return().
+-spec unpack_map_(binary(), non_neg_integer(), [{msgpack_term(), msgpack_term()}]) ->
+			 {{[{msgpack_term(), msgpack_term()}]}, binary()} | no_return().
 unpack_map_(Bin, 0,   Acc) -> {{lists:reverse(Acc)}, Bin};
 unpack_map_(Bin, Len, Acc) ->
     {Key, Rest} = unpack_(Bin),
@@ -197,7 +203,7 @@ unpack_map_(Bin, Len, Acc) ->
     unpack_map_(Rest2, Len-1, [{Key,Value}|Acc]).
 
 % unpack them all
--spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | {error, reason()} | no_return().
+-spec unpack_(Bin::binary()) -> {msgpack_term(), binary()} | no_return().
 unpack_(Bin) ->
     case Bin of
 % ATOMS

From 21992f1b9e43f3e0d2c3da8f1f8d94d9ee70b6fb Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Fri, 9 Jul 2010 18:53:24 +0200
Subject: [PATCH 0577/1648] erlang: fix receiving from port_command in unit
 tests

Ports can send data bit by bit; make sure we read all the port has to offer in one go.
This should fix the "broken pipe" error we sometime got during testing.
We did not previously check the return of compare_all/2, which is why the bug was not noticed.
Incidentally, this change fixes dialyzer warnings too.
---
 erlang/msgpack.erl | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index fb9a3e1..58ad414 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -261,6 +261,15 @@ compare_all([LH|LTL], [RH|RTL]) ->
     ?assertEqual(LH, RH),
     compare_all(LTL, RTL).
 
+port_receive(Port) ->
+    port_receive(Port, <<>>).
+port_receive(Port, Acc) ->
+    receive
+        {Port, {data, Data}} -> port_receive(Port, <>);
+        {Port, eof} -> Acc
+    after 1000 -> Acc
+    end.
+
 test_([]) -> 0;
 test_([Term|Rest])->
     Pack = msgpack:pack(Term),
@@ -286,13 +295,12 @@ basic_test()->
     Passed = length(Tests).
 
 port_test()->
-    Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary]),
     Tests = test_data(),
-    {[Tests],<<>>} = msgpack:unpack(msgpack:pack([Tests])),
-    true = port_command(Port, msgpack:pack(Tests) ),
-    receive
-	{Port, {data, Data}}->  {Tests, <<>>}=msgpack:unpack(Data)
-    after 1024-> ?assert(false)   end,
+    ?assertEqual({[Tests],<<>>}, msgpack:unpack(msgpack:pack([Tests]))),
+
+    Port = open_port({spawn, "ruby ../test/crosslang.rb"}, [binary, eof]),
+    true = port_command(Port, msgpack:pack(Tests)),
+    ?assertEqual({Tests, <<>>}, msgpack:unpack(port_receive(Port))),
     port_close(Port).
 
 test_p(Len,Term,OrigBin,Len) ->
@@ -319,7 +327,7 @@ map_test()->
     ok.
 
 unknown_test()->
-    Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary]),
+    Port = open_port({spawn, "ruby testcase_generator.rb"}, [binary, eof]),
     Tests = [0, 1, 2, 123, 512, 1230, 678908,
 	     -1, -23, -512, -1230, -567898,
 	     <<"hogehoge">>, <<"243546rf7g68h798j">>,
@@ -331,9 +339,7 @@ unknown_test()->
 	     -234, -50000,
 	     42
 	    ],
-    receive
-	{Port, {data, Data}}-> compare_all(Tests, msgpack:unpack_all(Data))
-    after 1024-> ?assert(false)  end,
+    ?assertEqual(ok, compare_all(Tests, msgpack:unpack_all(port_receive(Port)))),
     port_close(Port).
 
 other_test()->

From 2c29377abf1997d3a55178ef0562aa38d255f0fe Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Fri, 9 Jul 2010 20:30:17 +0200
Subject: [PATCH 0578/1648] erlang: s/short/incomplete/ and
 s/badarg/{badarg,Term}/

Nicer error returns.
---
 erlang/msgpack.erl | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 58ad414..102efed 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -29,7 +29,7 @@
 % erl> c(msgpack).
 % erl> S = .
 % erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
--type reason() ::  badarg | short.
+-type reason() :: {badarg,term()} | incomplete.
 -type msgpack_term() :: [msgpack_term()]
 		      | {[{msgpack_term(),msgpack_term()}]}
 		      | integer() | float() | binary().
@@ -43,7 +43,6 @@ pack(Term)->
 	error:Error when is_tuple(Error), element(1, Error) =:= error ->
 	    Error;
 	throw:Exception ->
-	    erlang:display(Exception),
 	    {error, Exception}
     end.
 
@@ -52,9 +51,7 @@ pack(Term)->
 % and feed more Bin into this function.
 % TODO: error case for imcomplete format when short for any type formats.
 -spec unpack( Bin::binary() )-> {msgpack_term(), binary()} | {error, reason()}.
-unpack(Bin) when not is_binary(Bin) ->
-    {error, badarg};
-unpack(Bin) ->
+unpack(Bin) when is_binary(Bin) ->
     try
 	unpack_(Bin)
     catch
@@ -62,7 +59,9 @@ unpack(Bin) ->
 	    Error;
 	throw:Exception ->
 	    {error, Exception}
-    end.
+    end;
+unpack(Other) ->
+    {error, {badarg, Other}}.
 
 -spec unpack_all( binary() ) -> [msgpack_term()].
 unpack_all(Data)->
@@ -108,8 +107,8 @@ pack_({Map}) when is_list(Map) ->
     pack_map(Map);
 pack_(Map) when is_tuple(Map), element(1,Map)=:=dict ->
     pack_map(dict:to_list(Map));
-pack_(_Other) ->
-    throw({error, badarg}).
+pack_(Other) ->
+    throw({error, {badarg, Other}}).
 
 -spec pack_uint_(non_neg_integer()) -> binary().
 % positive fixnum
@@ -241,13 +240,13 @@ unpack_(Bin) ->
 	<<2#1000:4, L:4, Rest/binary>> ->            unpack_map_(Rest, L, []);   % map
 
 % Invalid data
-	<> when F==16#C1;
+	<> when F==16#C1;
 	                     F==16#C4; F==16#C5; F==16#C6; F==16#C7; F==16#C8; F==16#C9;
 	                     F==16#D4; F==16#D5; F==16#D6; F==16#D7; F==16#D8; F==16#D9 ->
-	    throw(badarg);
+	    throw({badarg, <>});
 % Incomplete data (we've covered every complete/invalid case; anything left is incomplete)
 	_ ->
-	    throw(short)
+	    throw(incomplete)
     end.
 
 % ===== test codes ===== %
@@ -307,7 +306,7 @@ test_p(Len,Term,OrigBin,Len) ->
     {Term, <<>>}=msgpack:unpack(OrigBin);
 test_p(I,_,OrigBin,Len) when I < Len->
     <> = OrigBin,
-    ?assertEqual({error,short}, msgpack:unpack(Bin)).
+    ?assertEqual({error,incomplete}, msgpack:unpack(Bin)).
 
 partial_test()-> % error handling test.
     Term = lists:seq(0, 45),
@@ -343,7 +342,7 @@ unknown_test()->
     port_close(Port).
 
 other_test()->
-    ?assertEqual({error,short},msgpack:unpack(<<>>)).
+    ?assertEqual({error,incomplete},msgpack:unpack(<<>>)).
 
 benchmark_test()->
     Data=[test_data() || _ <- lists:seq(0, 10000)],

From 02c882bda39f65bdbe7be1b149b74f0f9a8283c6 Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Fri, 9 Jul 2010 20:34:38 +0200
Subject: [PATCH 0579/1648] erlang: Make pack_map/1 api private

---
 erlang/msgpack.erl | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 102efed..0e7fb5d 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -23,7 +23,6 @@
 %% APIs are almost compatible with C API (http://msgpack.sourceforge.jp/c:doc)
 %% except buffering functions (both copying and zero-copying).
 -export([pack/1, unpack/1, unpack_all/1]).
--export([pack_map/1]).
 
 % compile:
 % erl> c(msgpack).
@@ -72,16 +71,6 @@ unpack_all(Data)->
 	    [Term|unpack_all(Binary)]
     end.
 
--spec pack_map(M::[{msgpack_term(),msgpack_term()}]) -> binary() | no_return().
-pack_map(M)->
-    case length(M) of
-	Len when Len < 16 ->
-	    << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>;
-	Len when Len < 16#10000 -> % 65536
-	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>;
-	Len ->
-	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>
-    end.
 
 % ===== internal APIs ===== %
 
@@ -188,6 +177,17 @@ unpack_array_(Bin, Len, Acc) ->
     {Term, Rest} = unpack_(Bin),
     unpack_array_(Rest, Len-1, [Term|Acc]).
 
+-spec pack_map(M::[{msgpack_term(),msgpack_term()}]) -> binary() | no_return().
+pack_map(M)->
+    case length(M) of
+	Len when Len < 16 ->
+	    << 2#1000:4, Len:4/integer-unit:1, (pack_map_(M, <<>>))/binary >>;
+	Len when Len < 16#10000 -> % 65536
+	    << 16#DE:8, Len:16/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>;
+	Len ->
+	    << 16#DF:8, Len:32/big-unsigned-integer-unit:1, (pack_map_(M, <<>>))/binary >>
+    end.
+
 pack_map_([], Acc) -> Acc;
 pack_map_([{Key,Value}|Tail], Acc) ->
     pack_map_(Tail, << Acc/binary, (pack_(Key))/binary, (pack_(Value))/binary>>).

From e944c1ee93083a054f318e42a16eff699d9f066d Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Fri, 9 Jul 2010 20:37:06 +0200
Subject: [PATCH 0580/1648] erlang: Only handle throw() in pack/1 and unpack/1

Rationale: We only use throw/1 for error handling, never erlang:error/1.
           Caller bugs will get a nice {error,...} return while library bugs will
           bubble up in all their uglyness; that's the proper way to do things
           in erlang.
---
 erlang/msgpack.erl | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 0e7fb5d..f23ac87 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -39,8 +39,6 @@ pack(Term)->
     try
 	pack_(Term)
     catch
-	error:Error when is_tuple(Error), element(1, Error) =:= error ->
-	    Error;
 	throw:Exception ->
 	    {error, Exception}
     end.
@@ -54,8 +52,6 @@ unpack(Bin) when is_binary(Bin) ->
     try
 	unpack_(Bin)
     catch
-	error:Error when is_tuple(Error), element(1, Error) =:= error ->
-	    Error;
 	throw:Exception ->
 	    {error, Exception}
     end;

From e629e8784ff242f7c5e62066b0670f5d82afe4e2 Mon Sep 17 00:00:00 2001
From: Vincent de Phily 
Date: Mon, 12 Jul 2010 14:08:22 +0200
Subject: [PATCH 0581/1648] erlang: Improve documentation

The doc is in edoc format, generated from the source as an html file.
The makefile's default action now also generates the documentation.

I ignored unpack_all/1 and pack(dict()) for now because their future is still uncertain.
---
 erlang/OMakefile   |  7 +++--
 erlang/msgpack.erl | 70 +++++++++++++++++++++++++++++++++-------------
 2 files changed, 56 insertions(+), 21 deletions(-)

diff --git a/erlang/OMakefile b/erlang/OMakefile
index 34c590f..89b1c63 100644
--- a/erlang/OMakefile
+++ b/erlang/OMakefile
@@ -30,13 +30,16 @@
 # If so, define the subdirectory targets and uncomment this section.
 #
 
-.DEFAULT: msgpack.beam
+.DEFAULT: msgpack.beam msgpack.html
 
 msgpack.beam: msgpack.erl
 	erlc $<
 
+msgpack.html: msgpack.erl
+	erl -noshell -run edoc_run file $<
+
 test: msgpack.beam
 	erl -noshell -s msgpack test -s init stop
 
 clean:
-	-rm *.beam
+	-rm -f *.beam *.html
diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index f23ac87..aa9851d 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -15,26 +15,47 @@
 %%    See the License for the specific language governing permissions and
 %%    limitations under the License.
 
+
+%% @doc MessagePack codec for Erlang.
+%%
+%%      APIs are almost compatible with C API
+%%      except for buffering functions (both copying and zero-copying), which are unavailable.
+%%
+%%   
+%%     
+%%     
+%%     
+%%     
+%%     
+%%     
+%%     
+%%     
+%%     
+%%   
Equivalence between Erlang and Msgpack type :
    erlang                                msgpack                                      
 integer()     pos_fixnum/neg_fixnum/uint8/uint16/uint32/uint64/int8/int16/int32/int64 
 float()       float/double                                                            
 nil           nil                                                                     
 boolean()     boolean                                                                 
 binary()      fix_raw/raw16/raw32                                                     
 list()        fix_array/array16/array32                                               
 {proplist()}  fix_map/map16/map32                                                     

+%% @end
+
 -module(msgpack).
 -author('kuenishi+msgpack@gmail.com').
 
-%% tuples, atoms are not supported. lists, integers, double, and so on.
-%% see http://msgpack.sourceforge.jp/spec for supported formats.
-%% APIs are almost compatible with C API (http://msgpack.sourceforge.jp/c:doc)
-%% except buffering functions (both copying and zero-copying).
 -export([pack/1, unpack/1, unpack_all/1]).
 
-% compile:
-% erl> c(msgpack).
-% erl> S = .
-% erl> {S, <<>>} = msgpack:unpack( msgpack:pack(S) ).
--type reason() :: {badarg,term()} | incomplete.
+% @type msgpack_term() = [msgpack_term()]
+%                      | {[{msgpack_term(),msgpack_term()}]}
+%                      | integer() | float() | binary().
+% Erlang representation of msgpack data.
 -type msgpack_term() :: [msgpack_term()]
 		      | {[{msgpack_term(),msgpack_term()}]}
 		      | integer() | float() | binary().
 
-% ===== external APIs ===== %
--spec pack(Term::msgpack_term()) -> binary() | {error, reason()}.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% external APIs
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+% @doc Encode an erlang term into an msgpack binary.
+%      Returns {error, {badarg, term()}} if the input is illegal.
+% @spec pack(Term::msgpack_term()) -> binary() | {error, {badarg, term()}}
+-spec pack(Term::msgpack_term()) -> binary() | {error, {badarg, term()}}.
 pack(Term)->
     try
 	pack_(Term)
@@ -43,11 +64,12 @@ pack(Term)->
 	    {error, Exception}
     end.
 
-% unpacking.
-% if failed in decoding and not end, get more data
-% and feed more Bin into this function.
-% TODO: error case for imcomplete format when short for any type formats.
--spec unpack( Bin::binary() )-> {msgpack_term(), binary()} | {error, reason()}.
+% @doc Decode an msgpack binary into an erlang term.
+%      It only decodes the first msgpack packet contained in the binary; the rest is returned as is.
+%      Returns {error, {badarg, term()}} if the input is corrupted.
+%      Returns {error, incomplete} if the input is not a full msgpack packet (caller should gather more data and try again).
+% @spec unpack(Bin::binary()) -> {msgpack_term(), binary()} | {error, incomplete} | {error, {badarg, term()}}
+-spec unpack(Bin::binary()) -> {msgpack_term(), binary()} | {error, incomplete} | {error, {badarg, term()}}.
 unpack(Bin) when is_binary(Bin) ->
     try
 	unpack_(Bin)
@@ -58,7 +80,7 @@ unpack(Bin) when is_binary(Bin) ->
 unpack(Other) ->
     {error, {badarg, Other}}.
 
--spec unpack_all( binary() ) -> [msgpack_term()].
+-spec unpack_all(binary()) -> [msgpack_term()].
 unpack_all(Data)->
     case unpack(Data) of
 	{ Term, Binary } when bit_size(Binary) =:= 0 ->
@@ -68,7 +90,9 @@ unpack_all(Data)->
     end.
 
 
-% ===== internal APIs ===== %
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% internal APIs
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 % pack them all
 -spec pack_(msgpack_term()) -> binary() | no_return().
@@ -95,6 +119,7 @@ pack_(Map) when is_tuple(Map), element(1,Map)=:=dict ->
 pack_(Other) ->
     throw({error, {badarg, Other}}).
 
+
 -spec pack_uint_(non_neg_integer()) -> binary().
 % positive fixnum
 pack_uint_(N) when N < 128 ->
@@ -129,6 +154,7 @@ pack_int_(N) when N > -16#FFFFFFFF ->
 pack_int_(N) ->
     << 16#D3:8, N:64/big-signed-integer-unit:1 >>.
 
+
 -spec pack_double(float()) -> binary().
 % float : erlang's float is always IEEE 754 64bit format.
 % pack_float(F) when is_float(F)->
@@ -138,6 +164,7 @@ pack_int_(N) ->
 pack_double(F) ->
     << 16#CB:8, F:64/big-float-unit:1 >>.
 
+
 -spec pack_raw(binary()) -> binary().
 % raw bytes
 pack_raw(Bin) ->
@@ -150,6 +177,7 @@ pack_raw(Bin) ->
 	    << 16#DB:8, Len:32/big-unsigned-integer-unit:1, Bin/binary >>
     end.
 
+
 -spec pack_array([msgpack_term()]) -> binary() | no_return().
 % list
 pack_array(L) ->
@@ -173,6 +201,7 @@ unpack_array_(Bin, Len, Acc) ->
     {Term, Rest} = unpack_(Bin),
     unpack_array_(Rest, Len-1, [Term|Acc]).
 
+
 -spec pack_map(M::[{msgpack_term(),msgpack_term()}]) -> binary() | no_return().
 pack_map(M)->
     case length(M) of
@@ -245,7 +274,10 @@ unpack_(Bin) ->
 	    throw(incomplete)
     end.
 
-% ===== test codes ===== %
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% unit tests
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 -include_lib("eunit/include/eunit.hrl").
 -ifdef(EUNIT).
 

From ca0c844f32038329b21f92ce42c62618057ecc02 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Wed, 14 Jul 2010 09:58:05 +0900
Subject: [PATCH 0582/1648] clearly specified this distribution requires
 requires C99.

---
 perl/Makefile.PL | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index 58ab7c7..e9f9618 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -1,4 +1,6 @@
 use inc::Module::Install;
+use Config;
+
 name 'Data-MessagePack';
 all_from 'lib/Data/MessagePack.pm';
 readme_from('lib/Data/MessagePack.pm');
@@ -11,6 +13,8 @@ tests 't/*.t';
 recursive_author_tests('xt');
 use_ppport 3.19;
 
+requires_c99(); # msgpack C library requires C99.
+
 clean_files qw{
     *.stackdump
     *.gcov *.gcda *.gcno

From 9ac69337e89305a36ae3a3d3a9d89272a7453452 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Wed, 14 Jul 2010 09:58:28 +0900
Subject: [PATCH 0583/1648] perl: bump up version to 0.13!

---
 perl/Changes                 | 4 ++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index b03a9d0..f32d5f4 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.13
+
+    - clearly specify requires_c99(), because msgpack C header requires C99.
+
 0.12
 
     - PERL_NO_GET_CONTEXT makes horrible dTHXs. remove it.
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index dcc713d..ee07d35 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -4,7 +4,7 @@ use warnings;
 use XSLoader;
 use 5.008001;
 
-our $VERSION = '0.12';
+our $VERSION = '0.13';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };

From 331bf0af21cecfec1025e0f69fb4b3ffad4129fe Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Wed, 14 Jul 2010 17:02:04 +0900
Subject: [PATCH 0584/1648] cpp: type::raw_ref::str(), operator==, operator!=,
 operator< and operator> are now const

---
 cpp/src/msgpack/type/raw.hpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/cpp/src/msgpack/type/raw.hpp b/cpp/src/msgpack/type/raw.hpp
index 21d9a0d..87d188f 100644
--- a/cpp/src/msgpack/type/raw.hpp
+++ b/cpp/src/msgpack/type/raw.hpp
@@ -33,25 +33,25 @@ struct raw_ref {
 	uint32_t size;
 	const char* ptr;
 
-	std::string str() { return std::string(ptr, size); }
+	std::string str() const { return std::string(ptr, size); }
 
-	bool operator== (const raw_ref& x)
+	bool operator== (const raw_ref& x) const
 	{
 		return size == x.size && memcmp(ptr, x.ptr, size) == 0;
 	}
 
-	bool operator!= (const raw_ref& x)
+	bool operator!= (const raw_ref& x) const
 	{
 		return !(*this != x);
 	}
 
-	bool operator< (const raw_ref& x)
+	bool operator< (const raw_ref& x) const
 	{
 		if(size == x.size) { return memcmp(ptr, x.ptr, size) < 0; }
 		else { return size < x.size; }
 	}
 
-	bool operator> (const raw_ref& x)
+	bool operator> (const raw_ref& x) const
 	{
 		if(size == x.size) { return memcmp(ptr, x.ptr, size) > 0; }
 		else { return size > x.size; }

From f5453d38ec96b55e95ac745472d9b55087fc1d2f Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Wed, 14 Jul 2010 17:06:16 +0900
Subject: [PATCH 0585/1648] cpp: version 0.5.2

---
 cpp/ChangeLog    | 5 +++++
 cpp/configure.in | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/cpp/ChangeLog b/cpp/ChangeLog
index 3277c13..504ac4b 100644
--- a/cpp/ChangeLog
+++ b/cpp/ChangeLog
@@ -1,4 +1,9 @@
 
+2010-07-14 version 0.5.2:
+
+  * type::raw::str(), operator==, operator!=, operator< and operator> are now const
+  * generates version.h using AC_OUTPUT macro in ./configure
+
 2010-07-06 version 0.5.1:
 
   * Add msgpack_vrefbuffer_new and msgpack_vrefbuffer_free
diff --git a/cpp/configure.in b/cpp/configure.in
index ab29501..93174da 100644
--- a/cpp/configure.in
+++ b/cpp/configure.in
@@ -1,6 +1,6 @@
 AC_INIT(src/object.cpp)
 AC_CONFIG_AUX_DIR(ac)
-AM_INIT_AUTOMAKE(msgpack, 0.5.1)
+AM_INIT_AUTOMAKE(msgpack, 0.5.2)
 AC_CONFIG_HEADER(config.h)
 
 AC_SUBST(CFLAGS)

From 78f542f6c0205a7771f273175518db02f461f8e5 Mon Sep 17 00:00:00 2001
From: advect 
Date: Sat, 17 Jul 2010 18:46:28 +0900
Subject: [PATCH 0586/1648] Update PHP Extension

---
 php/CREDITS                   |   2 +-
 php/ChangeLog                 |  40 ++
 php/LICENSE                   |  26 ++
 php/README                    |  11 +
 php/benchmark.php             | 248 ++++++++++
 php/config.m4                 |  73 ++-
 php/config.w32                |   3 +-
 php/msgpack.c                 | 803 +++++++++-----------------------
 php/msgpack.php               |   8 +-
 php/msgpack/pack_define.h     |  26 --
 php/msgpack/pack_template.h   | 741 ------------------------------
 php/msgpack/unpack_define.h   | 129 ------
 php/msgpack/unpack_template.h | 361 ---------------
 php/msgpack_class.c           | 340 ++++++++++++++
 php/msgpack_class.h           |   7 +
 php/msgpack_pack.c            | 514 +++++++++++++++++++++
 php/msgpack_pack.h            |  17 +
 php/msgpack_unpack.c          | 829 ++++++++++++++++++++++++++++++++++
 php/msgpack_unpack.h          |  26 ++
 php/package.xml               |  98 ++++
 php/php-msgpack.spec          |  59 +++
 php/php_msgpack.h             |  85 +---
 php/test_normal.php           |   0
 php/test_streaming.php        |   9 +-
 php/tests/001.phpt            |  10 +
 php/tests/002.phpt            |  26 ++
 php/tests/003.phpt            |  31 ++
 php/tests/004.phpt            |  56 +++
 php/tests/005.phpt            |  26 ++
 php/tests/006.phpt            |  31 ++
 php/tests/007.phpt            |  72 +++
 php/tests/008.phpt            |  61 +++
 php/tests/009.phpt            | 115 +++++
 php/tests/010.phpt            |  49 ++
 php/tests/012.phpt            |  48 ++
 php/tests/013.phpt            |  54 +++
 php/tests/014.phpt            |  54 +++
 php/tests/015.phpt            |  58 +++
 php/tests/015b.phpt           |  58 +++
 php/tests/016.phpt            |  60 +++
 php/tests/017.phpt            |  48 ++
 php/tests/018.phpt            |  84 ++++
 php/tests/019.phpt            |  43 ++
 php/tests/020.phpt            |  31 ++
 php/tests/021.phpt            |  52 +++
 php/tests/022.phpt            |  45 ++
 php/tests/023.phpt            |  54 +++
 php/tests/024.phpt            | 166 +++++++
 php/tests/025.phpt            | 119 +++++
 php/tests/026.phpt            | 143 ++++++
 php/tests/027.phpt            |  73 +++
 php/tests/028.phpt            | 667 +++++++++++++++++++++++++++
 php/tests/029.phpt            |  20 +
 php/tests/030.phpt            | 233 ++++++++++
 php/tests/031.phpt            |  90 ++++
 php/tests/032.phpt            |  76 ++++
 php/tests/033.phpt            |  55 +++
 php/tests/034.phpt            |  38 ++
 php/tests/035.phpt            |  34 ++
 php/tests/040.phpt            |  43 ++
 php/tests/041.phpt            |  47 ++
 php/tests/042.phpt            |  40 ++
 php/tests/050.phpt            | 111 +++++
 php/tests/060.phpt            | 315 +++++++++++++
 php/tests/061.phpt            | 320 +++++++++++++
 php/tests/062.phpt            |  64 +++
 php/tests/063.phpt            |  68 +++
 php/tests/070.phpt            | 299 ++++++++++++
 php/tests/071.phpt            | 301 ++++++++++++
 php/tests/072.phpt            | 341 ++++++++++++++
 php/tests/test_pack.phpt      | 169 -------
 71 files changed, 7345 insertions(+), 2108 deletions(-)
 create mode 100644 php/ChangeLog
 create mode 100644 php/LICENSE
 create mode 100644 php/benchmark.php
 delete mode 100644 php/msgpack/pack_define.h
 delete mode 100644 php/msgpack/pack_template.h
 delete mode 100644 php/msgpack/unpack_define.h
 delete mode 100644 php/msgpack/unpack_template.h
 create mode 100644 php/msgpack_class.c
 create mode 100644 php/msgpack_class.h
 create mode 100644 php/msgpack_pack.c
 create mode 100644 php/msgpack_pack.h
 create mode 100644 php/msgpack_unpack.c
 create mode 100644 php/msgpack_unpack.h
 create mode 100644 php/package.xml
 create mode 100644 php/php-msgpack.spec
 mode change 100755 => 100644 php/test_normal.php
 mode change 100755 => 100644 php/test_streaming.php
 create mode 100644 php/tests/001.phpt
 create mode 100644 php/tests/002.phpt
 create mode 100644 php/tests/003.phpt
 create mode 100644 php/tests/004.phpt
 create mode 100644 php/tests/005.phpt
 create mode 100644 php/tests/006.phpt
 create mode 100644 php/tests/007.phpt
 create mode 100644 php/tests/008.phpt
 create mode 100644 php/tests/009.phpt
 create mode 100644 php/tests/010.phpt
 create mode 100644 php/tests/012.phpt
 create mode 100644 php/tests/013.phpt
 create mode 100644 php/tests/014.phpt
 create mode 100644 php/tests/015.phpt
 create mode 100644 php/tests/015b.phpt
 create mode 100644 php/tests/016.phpt
 create mode 100644 php/tests/017.phpt
 create mode 100644 php/tests/018.phpt
 create mode 100644 php/tests/019.phpt
 create mode 100644 php/tests/020.phpt
 create mode 100644 php/tests/021.phpt
 create mode 100644 php/tests/022.phpt
 create mode 100644 php/tests/023.phpt
 create mode 100644 php/tests/024.phpt
 create mode 100644 php/tests/025.phpt
 create mode 100644 php/tests/026.phpt
 create mode 100644 php/tests/027.phpt
 create mode 100644 php/tests/028.phpt
 create mode 100644 php/tests/029.phpt
 create mode 100644 php/tests/030.phpt
 create mode 100644 php/tests/031.phpt
 create mode 100644 php/tests/032.phpt
 create mode 100644 php/tests/033.phpt
 create mode 100644 php/tests/034.phpt
 create mode 100644 php/tests/035.phpt
 create mode 100644 php/tests/040.phpt
 create mode 100644 php/tests/041.phpt
 create mode 100644 php/tests/042.phpt
 create mode 100644 php/tests/050.phpt
 create mode 100644 php/tests/060.phpt
 create mode 100644 php/tests/061.phpt
 create mode 100644 php/tests/062.phpt
 create mode 100644 php/tests/063.phpt
 create mode 100644 php/tests/070.phpt
 create mode 100644 php/tests/071.phpt
 create mode 100644 php/tests/072.phpt
 delete mode 100644 php/tests/test_pack.phpt

diff --git a/php/CREDITS b/php/CREDITS
index 3aae276..b3ae8a0 100644
--- a/php/CREDITS
+++ b/php/CREDITS
@@ -1 +1 @@
-msgpack
+msgpack
\ No newline at end of file
diff --git a/php/ChangeLog b/php/ChangeLog
new file mode 100644
index 0000000..5481c1f
--- /dev/null
+++ b/php/ChangeLog
@@ -0,0 +1,40 @@
+msgpack extension changelog
+
+Version 0.2.1
+-------------
+	* Fix stream deserializer
+
+Version 0.2.0
+-------------
+	* Add stream deserializer / class MessagePackUnpacker interface
+	* Add alias functions
+	* Add class MessagePack interface
+
+Version 0.1.5
+-------------
+	* Add msgpack_pack.c
+	* Add msgpack_unpack.c
+	* Update msgpack.c
+
+Version 0.1.4
+-------------
+	* Change broken random data.
+	* Support PHP 5.2.x version.
+
+Version 0.1.3
+-------------
+	* Fix broken random data.
+	* Change arrays and objects.
+
+Version 0.1.2
+-------------
+	* Add Serializable class support.
+	* Fix arrays and objects reference.
+
+Version 0.1.1
+-------------
+	* Add session support.
+
+Version 0.1.0
+-------------
+	* Initial release.
diff --git a/php/LICENSE b/php/LICENSE
new file mode 100644
index 0000000..c0688fc
--- /dev/null
+++ b/php/LICENSE
@@ -0,0 +1,26 @@
+Copyright (c) 2010, advect
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the advect nor the names of its contributors
+      may be used to endorse or promote products derived from this software
+      without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/php/README b/php/README
index e69de29..ac8e485 100644
--- a/php/README
+++ b/php/README
@@ -0,0 +1,11 @@
+Description
+-----------
+This extension provide API for communicating with MessagePack serialization. 
+
+MessagePack is a binary-based efficient object serialization library.
+It enables to exchange structured objects between many languages like JSON.
+But unlike JSON, it is very fast and small.
+
+Resources
+---------
+ * [msgpack](http://msgpack.sourceforge.net/)
diff --git a/php/benchmark.php b/php/benchmark.php
new file mode 100644
index 0000000..559801c
--- /dev/null
+++ b/php/benchmark.php
@@ -0,0 +1,248 @@
+ md5(rand()),
+                           md5(rand()) => md5(rand()),
+                           md5(rand()) => md5(rand()),
+                           md5(rand()) => md5(rand()),
+                           md5(rand()) => md5(rand()));
+            break;
+        case 6:
+            //object
+            $value = new stdClass;
+            $value->param1 = rand();
+            $value->param2 = md5(uniqid());
+            $value->param3 = array(md5(uniqid()));
+            $value->param4 = array(md5(uniqid()) => md5(uniqid()));
+            $value->param5 = null;
+            break;
+        default:
+            //null
+            $value = null;
+    }
+
+    if (!is_numeric($retry) || empty($retry))
+    {
+        $retry = 1;
+    }
+
+    $serialize_pack = 0;
+    $serialize_unpack = 0;
+    $serialize_size = 0;
+    $serialize_status = '*NG*';
+    $json_pack = 0;
+    $json_unpack = 0;
+    $json_size = 0;
+    $json_status = '*NG*';
+    $igbinary_pack = 0;
+    $igbinary_unpack = 0;
+    $igbinary_size = 0;
+    $igbinary_status = '*NG*';
+    $msgpack_pack = 0;
+    $msgpack_unpack = 0;
+    $msgpack_size = 0;
+    $msgpack_status = '*NG*';
+
+    for ($c = 0; $c < $retry; $c++)
+    {
+        //default (serialize)
+        $pack = null;
+        $unpack = null;
+        $t = new Benchmark_Timer;
+        $t->start();
+        for ($i = 0; $i < $loop; $i++)
+        {
+            $pack = serialize($value);
+        }
+        $t->setMarker('serialize');
+        for ($i = 0; $i < $loop; $i++)
+        {
+            $unpack = unserialize($pack);
+        }
+        $t->stop();
+        //$t->display();
+        $profiling = $t->getProfiling();
+        unset($t);
+
+        $serialize_pack += $profiling[1]['diff'];
+        $serialize_unpack += $profiling[2]['diff'];
+        $serialize_size += strlen($pack);
+        if ($unpack === $value ||
+            (is_object($value) && $unpack == $value))
+        {
+            $serialize_status = 'OK';
+        }
+
+        //json
+        $pack = null;
+        $unpack = null;
+        $opt = false;
+        if (is_array($value))
+        {
+            $opt = true;
+        }
+        $t = new Benchmark_Timer;
+        $t->start();
+        for ($i = 0; $i < $loop; $i++)
+        {
+            $pack = json_encode($value);
+        }
+        $t->setMarker('json_encode');
+        for ($i = 0; $i < $loop; $i++)
+        {
+            $unpack = json_decode($pack, $opt);
+        }
+        $t->stop();
+        //$t->display();
+        $profiling = $t->getProfiling();
+        unset($t);
+
+        $json_pack += $profiling[1]['diff'];
+        $json_unpack += $profiling[2]['diff'];
+        $json_size += strlen($pack);
+        if ($unpack === $value ||
+            (is_object($value) && $unpack == $value) ||
+            (is_float($value) &&
+             number_format($value, 10, '.', '') ===
+             number_format($unpack, 10, '.', '')))
+        {
+            $json_status = 'OK';
+        }
+
+        //igbinary
+        if (extension_loaded('igbinary'))
+        {
+            $pack = null;
+            $unpack = null;
+            $t = new Benchmark_Timer;
+            $t->start();
+            for ($i = 0; $i < $loop; $i++)
+            {
+                $pack = igbinary_serialize($value);
+            }
+            $t->setMarker('igbinary_serialize');
+            for ($i = 0; $i < $loop; $i++)
+            {
+                $unpack = igbinary_unserialize($pack);
+            }
+            $t->stop();
+            //$t->display();
+            $profiling = $t->getProfiling();
+            unset($t);
+
+            $igbinary_pack += $profiling[1]['diff'];
+            $igbinary_unpack += $profiling[2]['diff'];
+            $igbinary_size += strlen($pack);
+            if ($unpack === $value ||
+                (is_object($value) && $unpack == $value))
+            {
+                $igbinary_status = 'OK';
+            }
+        }
+
+        //msgpack
+        $pack = null;
+        $unpack = null;
+        $t = new Benchmark_Timer;
+        $t->start();
+        for ($i = 0; $i < $loop; $i++)
+        {
+            $pack = msgpack_serialize($value);
+        }
+        $t->setMarker('msgpack_serialize');
+        for ($i = 0; $i < $loop; $i++)
+        {
+            $unpack = msgpack_unserialize($pack);
+        }
+        $t->stop();
+        //$t->display();
+        $profiling = $t->getProfiling();
+        unset($t);
+
+        $msgpack_pack += $profiling[1]['diff'];
+        $msgpack_unpack += $profiling[2]['diff'];
+        $msgpack_size += strlen($pack);
+        if ($unpack === $value ||
+            (is_object($value) && $unpack == $value))
+        {
+            $msgpack_status = 'OK';
+        }
+    }
+
+    $serialize_pack /= $retry;
+    $serialize_unpack /= $retry;
+    $serialize_size /= $retry;
+    $json_pack /= $retry;
+    $json_unpack /= $retry;
+    $json_size /= $retry;
+    $igbinary_pack /= $retry;
+    $igbinary_unpack /= $retry;
+    $igbinary_size /= $retry;
+    $msgpack_pack /= $retry;
+    $msgpack_unpack /= $retry;
+    $msgpack_size /= $retry;
+
+    printf("[%-10s] %13s %13s %13s %13s\n",
+           gettype($value), 'default', 'json', 'igbinary', 'msgpack');
+    printf("status     : %12s  %12s  %12s  %12s\n",
+           $serialize_status, $json_status, $igbinary_status, $msgpack_status);
+    printf("serialize  : %.4f (100%%) %.4f (%3d%%) %.4f (%3d%%) %.4f (%3d%%)\n",
+           $serialize_pack,
+           $json_pack, ($json_pack / $serialize_pack * 100),
+           $igbinary_pack, ($igbinary_pack / $serialize_pack * 100),
+           $msgpack_pack, ($msgpack_pack / $serialize_pack * 100));
+    printf("unserialize: %.4f (100%%) %.4f (%3d%%) %.4f (%3d%%) %.4f (%3d%%)\n",
+           $serialize_unpack,
+           $json_unpack, ($json_unpack / $serialize_unpack * 100),
+           $igbinary_unpack, ($igbinary_unpack / $serialize_unpack * 100),
+           $msgpack_unpack, ($msgpack_unpack / $serialize_unpack * 100));
+    printf("size       : %6d (100%%) %6d (%3d%%) %6d (%3d%%) %6d (%3d%%)\n\n",
+           $serialize_size,
+           $json_size, ($json_size / $serialize_size * 100),
+           $igbinary_size, ($igbinary_size / $serialize_size * 100),
+           $msgpack_size, ($msgpack_size / $serialize_size * 100));
+    if ($value_display === true)
+    {
+        var_dump($value);
+        echo PHP_EOL;
+    }
+}
diff --git a/php/config.m4 b/php/config.m4
index 751b471..d319684 100644
--- a/php/config.m4
+++ b/php/config.m4
@@ -1,14 +1,73 @@
-dnl $Id$
 dnl config.m4 for extension msgpack
 
-PHP_ARG_ENABLE(msgpack, whether to enable MessagePack support,
+dnl Comments in this file start with the string 'dnl'.
+dnl Remove where necessary. This file will not work
+dnl without editing.
+
+dnl Check PHP version:
+
+AC_MSG_CHECKING(PHP version)
+AC_TRY_COMPILE([#include "php/main/php_version.h"], [
+#if PHP_MAJOR_VERSION < 5 || (PHP_MAJOR_VERSION == 5 && PHP_MINOR_VERSION < 2)
+#error  this extension requires at least PHP version 5.2.0
+#endif
+],
+[AC_MSG_RESULT(ok)],
+[AC_MSG_ERROR([need at least PHP 5.2.0])])
+
+dnl If your extension references something external, use with:
+
+PHP_ARG_WITH(msgpack, for msgpack support,
 Make sure that the comment is aligned:
-[  --enable-msgpack           Enable MessagePack support])
+[  --with-msgpack             Include msgpack support])
+
+dnl PHP_ARG_ENABLE(msgpack, whether to enable msgpack support,
+dnl Make sure that the comment is aligned:
+dnl [  --enable-msgpack           Enable msgpack support])
 
 if test "$PHP_MSGPACK" != "no"; then
-  dnl AC_DEFINE([HAVE_MSGPACK],1 ,[whether to enable MessagePack support])
-  dnl AC_HEADER_STDC
+  dnl Write more examples of tests here...
 
-  PHP_NEW_EXTENSION(msgpack, msgpack.c, $ext_shared)
-  dnl PHP_SUBST(MSGPACK_SHARED_LIBADD)
+  dnl --with-msgpack -> check with-path
+  SEARCH_PATH="/usr/local /usr"     # you might want to change this
+  SEARCH_FOR="/include/msgpack.h"  # you most likely want to change this
+  if test -r $PHP_MSGPACK/$SEARCH_FOR; then # path given as parameter
+    MSGPACK_DIR=$PHP_MSGPACK
+  else # search default path list
+    AC_MSG_CHECKING([for msgpack files in default path])
+    for i in $SEARCH_PATH ; do
+      if test -r $i/$SEARCH_FOR; then
+        MSGPACK_DIR=$i
+        AC_MSG_RESULT(found in $i)
+      fi
+    done
+  fi
+
+  if test -z "$MSGPACK_DIR"; then
+    AC_MSG_RESULT([not found])
+    AC_MSG_ERROR([Please reinstall the msgpack distribution])
+  fi
+
+  dnl --with-msgpack -> add include path
+  PHP_ADD_INCLUDE($MSGPACK_DIR/include)
+
+  dnl --with-msgpack -> check for lib and symbol presence
+  LIBNAME=msgpack # you may want to change this
+  LIBSYMBOL=msgpack_pack_object # you most likely want to change this 
+
+  PHP_CHECK_LIBRARY($LIBNAME,$LIBSYMBOL,
+  [
+    PHP_ADD_LIBRARY_WITH_PATH($LIBNAME, $MSGPACK_DIR/lib, MSGPACK_SHARED_LIBADD)
+    AC_DEFINE(HAVE_MSGPACKLIB,1,[ ])
+  ],[
+    AC_MSG_ERROR([wrong msgpack lib version or lib not found])
+  ],[
+    -L$MSGPACK_DIR/lib -lm
+  ])
+
+  PHP_SUBST(MSGPACK_SHARED_LIBADD)
+
+  PHP_NEW_EXTENSION(msgpack, msgpack.c msgpack_pack.c msgpack_unpack.c msgpack_class.c, $ext_shared)
+
+  PHP_INSTALL_HEADERS([ext/msgpack], [php_msgpack.h])
 fi
diff --git a/php/config.w32 b/php/config.w32
index 50e9bec..f2153e6 100644
--- a/php/config.w32
+++ b/php/config.w32
@@ -8,6 +8,5 @@
 // ARG_ENABLE("msgpack", "enable msgpack support", "no");
 
 if (PHP_MSGPACK != "no") {
-	EXTENSION("msgpack", "msgpack.c");
+   EXTENSION("msgpack", "msgpack.c msgpack_pack.c msgpack_unpack.c msgpack_class.c");
 }
-
diff --git a/php/msgpack.c b/php/msgpack.c
index b38eb18..62fb68d 100644
--- a/php/msgpack.c
+++ b/php/msgpack.c
@@ -1,22 +1,3 @@
-/*
-  +----------------------------------------------------------------------+
-  | PHP Version 5                                                        |
-  +----------------------------------------------------------------------+
-  | Copyright (c) 1997-2007 The PHP Group                                |
-  +----------------------------------------------------------------------+
-  | This source file is subject to version 3.01 of the PHP license,      |
-  | that is bundled with this package in the file LICENSE, and is        |
-  | available through the world-wide-web at the following url:           |
-  | http://www.php.net/license/3_01.txt                                  |
-  | If you did not receive a copy of the PHP license and are unable to   |
-  | obtain it through the world-wide-web, please send a note to          |
-  | license@php.net so we can mail you a copy immediately.               |
-  +----------------------------------------------------------------------+
-  | Author: Hideyuki TAKEI                                               |
-  +----------------------------------------------------------------------+
-*/
-
-/* $Id: header 226204 2007-01-01 19:32:10Z iliaa $ */
 
 #ifdef HAVE_CONFIG_H
 #include "config.h"
@@ -26,631 +7,259 @@
 #include "php_ini.h"
 #include "ext/standard/info.h"
 #include "ext/standard/php_smart_str.h"
+#include "ext/standard/php_incomplete_class.h"
+#include "ext/standard/php_var.h"
+#include "ext/session/php_session.h"
+
 #include "php_msgpack.h"
+#include "msgpack_pack.h"
+#include "msgpack_unpack.h"
+#include "msgpack_class.h"
 
-#define PHP_EXT_VERSION "0.01"
+static ZEND_FUNCTION(msgpack_serialize);
+static ZEND_FUNCTION(msgpack_unserialize);
 
-#ifndef TRUE
-# define TRUE 1
-# define FALSE 0
-#endif
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_serialize, 0, 0, 1)
+    ZEND_ARG_INFO(0, value)
+ZEND_END_ARG_INFO()
 
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unserialize, 0, 0, 1)
+    ZEND_ARG_INFO(0, str)
+ZEND_END_ARG_INFO()
 
-/* pack */
-#include "msgpack/pack_define.h"
+PS_SERIALIZER_FUNCS(msgpack);
 
-#define msgpack_pack_inline_func(name) \
-	    static inline void msgpack_pack ## name
-
-#define msgpack_pack_inline_func_cint(name) \
-	    static inline void msgpack_pack ## name
-
-#define msgpack_pack_user smart_str*
-
-#define msgpack_pack_append_buffer(user, buf, len) \
-	    smart_str_appendl(user, (const void*)buf, len)
-
-#include "msgpack/pack_template.h"
-
-
-/* unpack */
-#include "msgpack/unpack_define.h"
-
-typedef struct {
-	    int finished;
-		char* source;
-} unpack_user;
-
-#define msgpack_unpack_struct(name) \
-	    struct template ## name
-
-#define msgpack_unpack_func(ret, name) \
-	    ret template ## name
-
-#define msgpack_unpack_callback(name) \
-	    template_callback ## name
-
-#define msgpack_unpack_object zval*
-
-#define msgpack_unpack_user unpack_user
-
-struct template_context;
-typedef struct template_context msgpack_unpack_t;
-
-static void template_init(msgpack_unpack_t* u);
-static msgpack_unpack_object template_data(msgpack_unpack_t* u);
-static int template_execute(msgpack_unpack_t* u,
-		        const char* data, size_t len, size_t* off);
-
-ZEND_BEGIN_MODULE_GLOBALS(msgpack)
-	msgpack_unpack_t *global_mp;
-ZEND_END_MODULE_GLOBALS(msgpack)
-
-#ifdef ZTS
-#define MSGPACK_G(v) TSRMG(msgpack_globals_id, zend_msgpack_globals *, v)
-#else
-#define MSGPACK_G(v) (msgpack_globals.v)
-#endif
-
-static inline msgpack_unpack_object template_callback_root(unpack_user* u)
-{
-	msgpack_unpack_object data;
-	ALLOC_INIT_ZVAL(data);
-	ZVAL_NULL(data);
-	return data;
-}
-
-static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; }
-
-static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; }
-
-static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; }
-
-static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; }
-
-static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, (long)d); return 0; }
-
-static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, (long)d); return 0; }
-
-static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, (long)d); return 0; }
-
-static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_LONG(*o, d); return 0; }
-
-static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_DOUBLE(*o, d); return 0; }
-
-static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_DOUBLE(*o, d); return 0; }
-
-static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_NULL(*o); return 0; }
-
-static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_BOOL(*o, 1); return 0; }
-
-static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); ZVAL_BOOL(*o, 0); return 0;}
-
-static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); array_init(*o); return 0; }
-
-static inline int template_callback_array_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object o)
-{ add_next_index_zval(*c, o); return 0; }
-
-static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
-{ ALLOC_INIT_ZVAL(*o); array_init(*o); return 0; }
-
-static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
-{
-	switch(k->type) {
-		case IS_LONG:
-	   	    add_index_zval(*c, Z_LVAL(*k), v);
-			break;
-		case IS_STRING:
-	   		add_assoc_zval_ex(*c, Z_STRVAL(*k), Z_STRLEN(*k)+1, v);
-			break;
-		default:
-			zend_error(E_WARNING, "[msgpack] (php_msgpack_decode) illegal offset type, skip this decoding");
-			break;
-	}
-    return 0;
-}
-			
-
-static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
-{
-	ALLOC_INIT_ZVAL(*o);
-	if (l == 0) {
-		ZVAL_STRINGL(*o, "", 0, 1);
-	} else {
-		ZVAL_STRINGL(*o, p, l, 1);
-	}
-	return 0;
-}
-
-#include "msgpack/unpack_template.h"
-
-static PHP_GINIT_FUNCTION(msgpack);
-
-ZEND_DECLARE_MODULE_GLOBALS(msgpack)
-
-/* True global resources - no need for thread safety here */
-static int le_msgpack;
-
-/* {{{ msgpack_functions[]
- *
- * Every user visible function must have an entry in msgpack_functions[].
- */
-zend_function_entry msgpack_functions[] = {
-	PHP_FE(msgpack_pack, NULL)
-	PHP_FE(msgpack_unpack, NULL)
-	PHP_FE(msgpack_unpack_limit, NULL)
-	PHP_ME(msgpack, initialize, NULL, 0)
-	PHP_ME(msgpack, execute, NULL, 0)
-	PHP_ME(msgpack, execute_limit, NULL, 0)
-	PHP_ME(msgpack, finished, NULL, 0)
-	PHP_ME(msgpack, data, NULL, 0)
-	{NULL, NULL, NULL}	/* Must be the last line in msgpack_functions[] */
+static const zend_function_entry msgpack_functions[] = {
+    ZEND_FE(msgpack_serialize, arginfo_msgpack_serialize)
+    ZEND_FE(msgpack_unserialize, arginfo_msgpack_unserialize)
+    ZEND_FALIAS(msgpack_pack, msgpack_serialize, arginfo_msgpack_serialize)
+    ZEND_FALIAS(msgpack_unpack, msgpack_unserialize,
+                arginfo_msgpack_unserialize)
+    {NULL, NULL, NULL}
 };
-/* }}} */
 
-/* {{{ msgpack_module_entry
- */
+static ZEND_MINIT_FUNCTION(msgpack)
+{
+    MSGPACK_G(error_display) = 1;
+
+#if HAVE_PHP_SESSION
+    php_session_register_serializer("msgpack",
+                                    PS_SERIALIZER_ENCODE_NAME(msgpack),
+                                    PS_SERIALIZER_DECODE_NAME(msgpack));
+#endif
+
+    msgpack_init_class(TSRMLS_CC);
+
+    return SUCCESS;
+}
+
+static ZEND_MINFO_FUNCTION(msgpack)
+{
+    php_info_print_table_start();
+    php_info_print_table_row(2, "msgpack support", "enabled");
+    php_info_print_table_row(2, "msgpack version", MSGPACK_VERSION);
+#if HAVE_PHP_SESSION
+    php_info_print_table_row(2, "msgpack Session Support", "enabled" );
+#endif
+    php_info_print_table_end();
+}
+
 zend_module_entry msgpack_module_entry = {
 #if ZEND_MODULE_API_NO >= 20010901
-	STANDARD_MODULE_HEADER,
+    STANDARD_MODULE_HEADER,
 #endif
-	"msgpack",
-	msgpack_functions,
-	PHP_MINIT(msgpack),
-	PHP_MSHUTDOWN(msgpack),
-	PHP_RINIT(msgpack),		/* Replace with NULL if there's nothing to do at request start */
-	PHP_RSHUTDOWN(msgpack),	/* Replace with NULL if there's nothing to do at request end */
-	PHP_MINFO(msgpack),
+    "msgpack",
+    msgpack_functions,
+    ZEND_MINIT(msgpack),
+    NULL,
+    NULL,
+    NULL,
+    ZEND_MINFO(msgpack),
 #if ZEND_MODULE_API_NO >= 20010901
-	"0.1", /* Replace with version number for your extension */
+    MSGPACK_VERSION,
 #endif
-	PHP_MODULE_GLOBALS(msgpack),
-	PHP_GINIT(msgpack),
-	NULL,
-	NULL,
-	STANDARD_MODULE_PROPERTIES_EX
+    STANDARD_MODULE_PROPERTIES
 };
-/* }}} */
 
 #ifdef COMPILE_DL_MSGPACK
 ZEND_GET_MODULE(msgpack)
 #endif
 
-/* {{{ PHP_GINIT_FUNCTION */
-static PHP_GINIT_FUNCTION(msgpack)
+PS_SERIALIZER_ENCODE_FUNC(msgpack)
 {
-	msgpack_globals->global_mp = NULL;
-}
-/* }}} */
+    smart_str buf = {0};
+    php_serialize_data_t var_hash;
 
-/* {{{ PHP_MINIT_FUNCTION
- */
-PHP_MINIT_FUNCTION(msgpack)
-{
-	zend_class_entry ce;
-	INIT_CLASS_ENTRY(ce, "MessagePack", msgpack_functions);
-	msgpack_ce = zend_register_internal_class(&ce TSRMLS_CC);
+    PHP_VAR_SERIALIZE_INIT(var_hash);
 
-	return SUCCESS;
-}
-/* }}} */
+    msgpack_serialize_zval(&buf, PS(http_session_vars), &var_hash TSRMLS_CC);
 
-/* {{{ PHP_MSHUTDOWN_FUNCTION
- */
-PHP_MSHUTDOWN_FUNCTION(msgpack)
-{
-	/* uncomment this line if you have INI entries
-	UNREGISTER_INI_ENTRIES();
-	*/
-	if (MSGPACK_G(global_mp)) {
-		efree(MSGPACK_G(global_mp));
-		MSGPACK_G(global_mp) = NULL;
-	}
-
-	return SUCCESS;
-}
-/* }}} */
-
-/* Remove if there's nothing to do at request start */
-/* {{{ PHP_RINIT_FUNCTION
- */
-PHP_RINIT_FUNCTION(msgpack)
-{
-	return SUCCESS;
-}
-/* }}} */
-
-/* Remove if there's nothing to do at request end */
-/* {{{ PHP_RSHUTDOWN_FUNCTION
- */
-PHP_RSHUTDOWN_FUNCTION(msgpack)
-{
-	return SUCCESS;
-}
-/* }}} */
-
-/* {{{ PHP_MINFO_FUNCTION
- */
-PHP_MINFO_FUNCTION(msgpack)
-{
-	php_info_print_table_start();
-	php_info_print_table_header(2, "msgpack support", "enabled");
-	php_info_print_table_row(2, "php extension version", PHP_EXT_VERSION);
-	php_info_print_table_row(2, "author", "Hideyuki TAKEI");
-	php_info_print_table_row(2, "homepage", "http://msgpack.sourceforge.net");
-	php_info_print_table_row(2, "open sourced by", "KLab inc.");
-	php_info_print_table_end();
-}
-/* }}} */
-
-PHP_MSGPACK_API int msgpack_determine_array_type(zval **val TSRMLS_DC)  /* {{{ */
-{
-	int i;
-	HashTable *myht = HASH_OF(*val);
-
-	i = myht ? zend_hash_num_elements(myht) : 0;
-	if (i > 0) {
-		char *key;
-		ulong index, idx;
-		uint key_len;
-		HashPosition pos;
-
-		zend_hash_internal_pointer_reset_ex(myht, &pos);
-		idx = 0;
-		for (;; zend_hash_move_forward_ex(myht, &pos)) {
-			i = zend_hash_get_current_key_ex(myht, &key, &key_len, &index, 0, &pos);
-			if (i == HASH_KEY_NON_EXISTANT)
-				break;
-			if (i == HASH_KEY_IS_STRING) {
-				return 1;
-			} else {
-				if (index != idx) {
-					return 1;
-				}
-			}
-			idx++;
-		}
-	}
-	return 0;
-}
-/* }}} */
-
-PHP_MSGPACK_API void msgpack_pack_array_hash(smart_str *pk, zval **val TSRMLS_DC) /* {{{ */	
-{
-	int i, r;
-	HashTable *myht;
-
-	if(Z_TYPE_PP(val) == IS_ARRAY){
-		myht = HASH_OF(*val);
-		r = msgpack_determine_array_type(val TSRMLS_CC);
-	}
-	else{
-		myht = Z_OBJPROP_PP(val);
-		r = 1;
-	}
-
-	i = myht ? zend_hash_num_elements(myht) : 0;
-
-	if(r == 0){
-		msgpack_pack_array(pk, i);
-	}
-	else{
-		msgpack_pack_map(pk, i);
-	}
-
-	if(i>0){
-		char *key;
-		zval **data;
-		ulong index;
-		uint key_len;
-		HashPosition pos;
-		HashTable *tmp_ht;
-		int need_comma = 0;
-
-		zend_hash_internal_pointer_reset_ex(myht, &pos);
-		for(;; zend_hash_move_forward_ex(myht, &pos)){
-			i = zend_hash_get_current_key_ex(myht, &key, &key_len, &index, 0, &pos);
-			if(i==HASH_KEY_NON_EXISTANT)
-				break;
-			if(zend_hash_get_current_data_ex(myht, (void **) &data, &pos) == SUCCESS){
-				tmp_ht = HASH_OF(*data);
-				if (tmp_ht)
-					tmp_ht->nApplyCount++;
-
-				if(r==0)
-					php_msgpack_pack(pk, *data TSRMLS_CC);
-				else if(r==1){
-					if(i==HASH_KEY_IS_STRING){
-						if(key[0]=='\0' && Z_TYPE_PP(val)==IS_OBJECT){
-							// Skip protected and private members.
-							if(tmp_ht)
-								tmp_ht->nApplyCount--;
-							continue;
-						}
-						msgpack_pack_raw(pk, key_len-1);
-						msgpack_pack_raw_body(pk, key, key_len-1);
-						php_msgpack_pack(pk, *data TSRMLS_CC);
-					}
-					else{
-						msgpack_pack_long(pk, index);
-						php_msgpack_pack(pk, *data TSRMLS_CC);
-					}
-				}
-
-				if(tmp_ht){
-					tmp_ht->nApplyCount--;
-				}
-			}
-		}
-
-	}
-}
-/* }}} */
-
-PHP_MSGPACK_API void php_msgpack_pack(smart_str *pk, zval *val TSRMLS_DC) /* {{{ */
-{
-	switch(Z_TYPE_P(val)){
-		case IS_NULL:
-			msgpack_pack_nil(pk);
-			break;
-		case IS_BOOL:
-			if (Z_BVAL_P(val))
-				msgpack_pack_true(pk);
-			else
-				msgpack_pack_false(pk);
-			break;
-		case IS_LONG:
-			msgpack_pack_long(pk, Z_LVAL_P(val));
-			break;
-		case IS_DOUBLE:
-			{
-				double dbl = Z_DVAL_P(val);
-				if (zend_isinf(dbl) || zend_isnan(dbl)) {
-					zend_error(E_WARNING, "[msgpack] (php_msgpack_pack) double %.9g does not conform to the MSGPACK spec, encoded as 0", dbl);
-					ZVAL_LONG(val, 0);
-				}
-				msgpack_pack_double(pk, Z_DVAL_P(val));
-			}
-			break;
-		case IS_STRING:
-			msgpack_pack_raw(pk, Z_STRLEN_P(val));
-			msgpack_pack_raw_body(pk, Z_STRVAL_P(val), Z_STRLEN_P(val));
-			break;
-		case IS_ARRAY:
-		case IS_OBJECT:
-			msgpack_pack_array_hash(pk, &val TSRMLS_CC);
-			break;
-		defalut:
-			zend_error(E_WARNING, "[msgpack] (php_msgpack_pack) type is unsupported, encoded as null");
-			msgpack_pack_nil(pk);
-			break;
-	}
-
-	return;
-}
-/* }}} */
-
-PHP_MSGPACK_API void php_msgpack_unpack_limit(zval *return_value, const char *buf, int len, zend_bool assoc TSRMLS_DC) /* {{{ */
-{
-	if (len<=0) {
-		RETURN_NUL();
-	}
-
-	msgpack_unpack_t mp;
-	template_init(&mp);
-	unpack_user u = {0, ""};
-
-	size_t from = 0;
-	char* dptr = (char*)buf;
-	long dlen = len;
-	int ret;
-
-	(&mp)->user.source = (char*)buf;
-	ret = template_execute(&mp, dptr, (size_t)dlen, &from);
-	(&mp)->user.source = "";
-
-	if(ret < 0) {
-		zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) parse error");
-	} else if(ret == 0) {
-		zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) insufficient bytes");
-	} else {
-		if(from < dlen) {
-			zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) extra bytes");
-		}
-
-		*return_value = *template_data(&mp);
-		FREE_ZVAL(template_data(&mp));
-	}
-}	
-/* }}} */
-
-
-PHP_FUNCTION(msgpack_pack)
-{
-	zval *parameter;
-	smart_str buf = {0};
-
-	if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "z", ¶meter) == FAILURE) {
-		return;
-	}
-
-	php_msgpack_pack(&buf, parameter TSRMLS_CC);
-
-	ZVAL_STRINGL(return_value, buf.c, buf.len, 1);
-
-	smart_str_free(&buf);
-}
-
-PHP_FUNCTION(msgpack_unpack)
-{
-	char *parameter;
-	int parameter_len;
-	zend_bool assoc = 0;
-
-	if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|b",
-	   ¶meter, ¶meter_len, &assoc) == FAILURE) {
-		return;
-	}
-
-	if (!parameter_len) {
-		RETURN_NULL();
-	}
-
-	php_msgpack_unpack_limit(return_value, parameter, parameter_len, assoc TSRMLS_CC);
-}
-
-PHP_FUNCTION(msgpack_unpack_limit)
-{
-	char *parameter;
-	int parameter_len;
-	int limit;
-	zend_bool assoc = 0;
-
-	if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sl|b",
-	   ¶meter, ¶meter_len, &limit, &assoc) == FAILURE) {
-		return;
-	}
-
-	if (!parameter_len) {
-		RETURN_NULL();
-	}
-	else if (parameter_len < limit) {
-		zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) limit greater than data_len");
-		limit = parameter_len;
-	}
-
-	php_msgpack_unpack_limit(return_value, parameter, limit, assoc TSRMLS_CC);
-}
-
-
-PHP_MSGPACK_API void php_msgpack_unpacker_execute_limit(zval *return_value, const char *buf, int off, int len, zend_bool assoc TSRMLS_DC) /* {{{ */
-{
-	if (len<=0) {
-		RETURN_NUL();
-	}
-
-	size_t from = off;
-	char* dptr = (char*)buf;
-	long dlen = len;
-	int ret;
-
-	if(from >= dlen) {
-		zend_error(E_WARNING, "[msgpack] (php_msgpack_unpacker_execute_limit) offset is bigger than data buffer size");
+    if (newlen)
+    {
+        *newlen = buf.len;
     }
 
-	MSGPACK_G(global_mp)->user.source = (char*)buf;
-	ret = template_execute(MSGPACK_G(global_mp), dptr, (size_t)dlen, &from);
-	MSGPACK_G(global_mp)->user.source = "";
+    smart_str_0(&buf);
+    *newstr = buf.c;
 
-	if(ret < 0) {
-		zend_error(E_WARNING, "[msgpack] (php_msgpack_unpacker_execute_limit) parse error");
-	} else if(ret > 0) {
-		MSGPACK_G(global_mp)->user.finished = 1;
-		RETVAL_LONG(from);
-	} else {
-		MSGPACK_G(global_mp)->user.finished = 0;
-		RETVAL_LONG(from);
-	}
+    PHP_VAR_SERIALIZE_DESTROY(var_hash);
+
+    return SUCCESS;
 }
-/* }}} */
 
-PHP_MSGPACK_API void php_msgpack_unpacker_reset(TSRMLS_D) /* {{{ */
+PS_SERIALIZER_DECODE_FUNC(msgpack)
 {
-	if(MSGPACK_G(global_mp)) {
-		efree(MSGPACK_G(global_mp));
-		MSGPACK_G(global_mp) = NULL;
-	}
-	MSGPACK_G(global_mp) = safe_emalloc(sizeof(msgpack_unpack_t), 1, 0);
+    php_unserialize_data_t var_hash;
+    int ret;
+    HashTable *tmp_hash;
+    HashPosition tmp_hash_pos;
+    char *key_str;
+    ulong key_long;
+    uint key_len;
+    zval *tmp;
+    zval **data;
+    msgpack_unserialize_data mpsd;
 
-	template_init(MSGPACK_G(global_mp));
-	unpack_user u = {0, ""};
-	MSGPACK_G(global_mp)->user = u;
-	return;
+    PHP_VAR_UNSERIALIZE_INIT(var_hash);
+
+    MAKE_STD_ZVAL(tmp);
+
+    mpsd.data = (unsigned char *)val;;
+    mpsd.length = vallen;
+    mpsd.offset = 0;
+
+    ret = msgpack_unserialize_zval(&tmp, &mpsd, &var_hash TSRMLS_CC);
+
+    switch (ret)
+    {
+        case MSGPACK_UNPACK_EXTRA_BYTES:
+        case MSGPACK_UNPACK_SUCCESS:
+            break;
+        case MSGPACK_UNPACK_PARSE_ERROR:
+        case MSGPACK_UNPACK_CONTINUE:
+        default:
+            zval_ptr_dtor(&tmp);
+            return FAILURE;
+    }
+
+    PHP_VAR_UNSERIALIZE_DESTROY(var_hash);
+
+    tmp_hash = HASH_OF(tmp);
+
+    zend_hash_internal_pointer_reset_ex(tmp_hash, &tmp_hash_pos);
+    while (zend_hash_get_current_data_ex(
+               tmp_hash, (void *)&data, &tmp_hash_pos) == SUCCESS)
+    {
+        ret = zend_hash_get_current_key_ex(
+            tmp_hash, &key_str, &key_len, &key_long, 0, &tmp_hash_pos);
+        switch (ret)
+        {
+            case HASH_KEY_IS_LONG:
+                /* ??? */
+                break;
+            case HASH_KEY_IS_STRING:
+                php_set_session_var(key_str, key_len - 1, *data, NULL TSRMLS_CC);
+                php_add_session_var(key_str, key_len - 1 TSRMLS_CC);
+                break;
+        }
+        zend_hash_move_forward_ex(tmp_hash, &tmp_hash_pos);
+    }
+
+    zval_ptr_dtor(&tmp);
+
+    return SUCCESS;
 }
-/* }}} */
 
-PHP_METHOD(msgpack, initialize)
+PHP_MSGPACK_API void php_msgpack_serialize(smart_str *buf, zval *val TSRMLS_DC)
 {
-	php_msgpack_unpacker_reset(TSRMLS_C);
-	return;
+    php_serialize_data_t var_hash;
+
+    PHP_VAR_SERIALIZE_INIT(var_hash);
+
+    msgpack_serialize_zval(buf, val, &var_hash TSRMLS_CC);
+
+    PHP_VAR_SERIALIZE_DESTROY(var_hash);
 }
 
-PHP_METHOD(msgpack, execute)
+PHP_MSGPACK_API void php_msgpack_unserialize(
+    zval *return_value, char *str, size_t str_len TSRMLS_DC)
 {
-	char *data;
-	int off;
-	int data_len;
-	zend_bool assoc = 0;
+    int ret;
+    php_unserialize_data_t var_hash;
+    msgpack_unserialize_data mpsd;
 
-	if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sl|b",
-	   &data, &data_len, &off, &assoc) == FAILURE) {
-		return;
-	}
+    if (str_len <= 0)
+    {
+        RETURN_NULL();
+    }
 
-	if (!data_len) {
-		RETURN_NULL();
-	}
+    PHP_VAR_UNSERIALIZE_INIT(var_hash);
 
-	php_msgpack_unpacker_execute_limit(return_value, data, off, data_len, assoc TSRMLS_CC);
+    mpsd.data = (unsigned char *)str;
+    mpsd.length = str_len;
+    mpsd.offset = 0;
+
+    ret = msgpack_unserialize_zval(&return_value, &mpsd, &var_hash TSRMLS_CC);
+
+    switch (ret)
+    {
+        case MSGPACK_UNPACK_PARSE_ERROR:
+            zend_error(E_WARNING,
+                       "[msgpack] (php_msgpack_unserialize) Parse error");
+            break;
+        case MSGPACK_UNPACK_CONTINUE:
+            zend_error(E_WARNING,
+                       "[msgpack] (php_msgpack_unserialize) "
+                       "Insufficient data for unserializeng");
+            break;
+        case MSGPACK_UNPACK_EXTRA_BYTES:
+            zend_error(E_WARNING,
+                       "[msgpack] (php_msgpack_unserialize) Extra bytes");
+            break;
+        case MSGPACK_UNPACK_SUCCESS:
+            break;
+        default:
+            zend_error(E_WARNING,
+                       "[msgpack] (php_msgpack_unserialize) Unknown result");
+            break;
+    }
+
+    PHP_VAR_UNSERIALIZE_DESTROY(var_hash);
 }
 
-PHP_METHOD(msgpack, execute_limit)
+static ZEND_FUNCTION(msgpack_serialize)
 {
-	char *data;
-	int off;
-	int data_len;
-	int limit;
-	zend_bool assoc = 0;
+    zval *parameter;
+    smart_str buf = {0};
 
-	if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "sll|b",
-	   &data, &data_len, &off, &limit, &assoc) == FAILURE) {
-		return;
-	}
+    if (zend_parse_parameters(
+            ZEND_NUM_ARGS() TSRMLS_CC, "z", ¶meter) == FAILURE)
+    {
+        return;
+    }
 
-	if (!data_len) {
-		RETURN_NULL();
-	}
-	else if (data_len < limit) {
-		zend_error(E_WARNING, "[msgpack] (php_msgpack_unpack) limit greater than (data+off)_len");
-		limit = data_len;
-	}
+    php_msgpack_serialize(&buf, parameter TSRMLS_CC);
 
-	php_msgpack_unpacker_execute_limit(return_value, data, off, limit, assoc TSRMLS_CC);
+    ZVAL_STRINGL(return_value, buf.c, buf.len, 1);
+
+    smart_str_free(&buf);
 }
 
-PHP_METHOD(msgpack, finished)
+static ZEND_FUNCTION(msgpack_unserialize)
 {
-	if(MSGPACK_G(global_mp)->user.finished == 1) {
-		RETURN_TRUE;
-	}
-	RETURN_FALSE;
-}
+    char *str;
+    int str_len;
 
-PHP_METHOD(msgpack, data)
-{
-	*return_value = *template_data(MSGPACK_G(global_mp));
-	return;
-}
+    if (zend_parse_parameters(
+            ZEND_NUM_ARGS() TSRMLS_CC, "s", &str, &str_len) == FAILURE)
+    {
+        return;
+    }
 
-/*
- * Local variables:
- * tab-width: 4
- * c-basic-offset: 4
- * End:
- * vim600: noet sw=4 ts=4 fdm=marker
- * vim<600: noet sw=4 ts=4
- */
+    if (!str_len)
+    {
+        RETURN_NULL();
+    }
+
+    php_msgpack_unserialize(return_value, str, str_len TSRMLS_CC);
+}
diff --git a/php/msgpack.php b/php/msgpack.php
index c2498dd..5154781 100644
--- a/php/msgpack.php
+++ b/php/msgpack.php
@@ -2,7 +2,7 @@
 $br = (php_sapi_name() == "cli")? "":"
";
 
 if(!extension_loaded('msgpack')) {
-	dl('msgpack.' . PHP_SHLIB_SUFFIX);
+    dl('msgpack.' . PHP_SHLIB_SUFFIX);
 }
 $module = 'msgpack';
 $functions = get_extension_funcs($module);
@@ -11,11 +11,11 @@ foreach($functions as $func) {
     echo $func."$br\n";
 }
 echo "$br\n";
-$function = 'confirm_' . $module . '_compiled';
+$function = $module . '_serialize';
 if (extension_loaded($module)) {
-	$str = $function($module);
+    $str = $function($module);
 } else {
-	$str = "Module $module is not compiled into PHP";
+    $str = "Module $module is not compiled into PHP";
 }
 echo "$str\n";
 ?>
diff --git a/php/msgpack/pack_define.h b/php/msgpack/pack_define.h
deleted file mode 100644
index 33408e5..0000000
--- a/php/msgpack/pack_define.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * MessagePack unpacking routine template
- *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
- *
- *    Licensed under the Apache License, Version 2.0 (the "License");
- *    you may not use this file except in compliance with the License.
- *    You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- */
-#ifndef MSGPACK_PACK_DEFINE_H__
-#define MSGPACK_PACK_DEFINE_H__
-
-#include 
-#include 
-#include 
-
-#endif /* msgpack/pack_define.h */
-
diff --git a/php/msgpack/pack_template.h b/php/msgpack/pack_template.h
deleted file mode 100644
index aa620f5..0000000
--- a/php/msgpack/pack_template.h
+++ /dev/null
@@ -1,741 +0,0 @@
-/*
- * MessagePack packing routine template
- *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
- *
- *    Licensed under the Apache License, Version 2.0 (the "License");
- *    you may not use this file except in compliance with the License.
- *    You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- */
-
-#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-#define __LITTLE_ENDIAN__
-#elif __BYTE_ORDER == __BIG_ENDIAN
-#define __BIG_ENDIAN__
-#endif
-#endif
-
-
-#ifdef __LITTLE_ENDIAN__
-
-#define STORE8_BE8(d) \
-	((uint8_t*)&d)[0]
-
-
-#define STORE16_BE8(d) \
-	((uint8_t*)&d)[0]
-
-#define STORE16_BE16(d) \
-	((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
-
-
-#define STORE32_BE8(d) \
-	((uint8_t*)&d)[0]
-
-#define STORE32_BE16(d) \
-	((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
-
-#define STORE32_BE32(d) \
-	((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
-
-
-#define STORE64_BE8(d) \
-	((uint8_t*)&d)[0]
-
-#define STORE64_BE16(d) \
-	((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
-
-#define STORE64_BE32(d) \
-	((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
-
-#define STORE64_BE64(d) \
-	((uint8_t*)&d)[7], ((uint8_t*)&d)[6], ((uint8_t*)&d)[5], ((uint8_t*)&d)[4], \
-	((uint8_t*)&d)[3], ((uint8_t*)&d)[2], ((uint8_t*)&d)[1], ((uint8_t*)&d)[0]
-
-
-#elif __BIG_ENDIAN__
-
-#define STORE8_BE8(d) \
-	((uint8_t*)&d)[0]
-
-
-#define STORE16_BE8(d) \
-	((uint8_t*)&d)[1]
-
-#define STORE16_BE16(d) \
-	((uint8_t*)&d)[0], ((uint8_t*)&d)[1]
-
-
-#define STORE32_BE8(d) \
-	((uint8_t*)&d)[3]
-
-#define STORE32_BE16(d) \
-	((uint8_t*)&d)[2], ((uint8_t*)&d)[3]
-
-#define STORE32_BE32(d) \
-	((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3]
-
-
-#define STORE64_BE8(d) \
-	((uint8_t*)&d)[7]
-
-#define STORE64_BE16(d) \
-	((uint8_t*)&d)[6], ((uint8_t*)&d)[7]
-
-#define STORE64_BE32(d) \
-	((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7]
-
-#define STORE64_BE64(d) \
-	((uint8_t*)&d)[0], ((uint8_t*)&d)[1], ((uint8_t*)&d)[2], ((uint8_t*)&d)[3], \
-	((uint8_t*)&d)[4], ((uint8_t*)&d)[5], ((uint8_t*)&d)[6], ((uint8_t*)&d)[7]
-
-#endif
-
-#ifndef msgpack_pack_inline_func
-#error msgpack_pack_inline_func template is not defined
-#endif
-
-#ifndef msgpack_pack_user
-#error msgpack_pack_user type is not defined
-#endif
-
-#ifndef msgpack_pack_append_buffer
-#error msgpack_pack_append_buffer callback is not defined
-#endif
-
-
-/*
- * Integer
- */
-
-#define msgpack_pack_real_uint8(x, d) \
-do { \
-	if(d < (1<<7)) { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \
-	} else { \
-		/* unsigned 8 */ \
-		const unsigned char buf[2] = {0xcc, STORE8_BE8(d)}; \
-		msgpack_pack_append_buffer(x, buf, 2); \
-	} \
-} while(0)
-
-#define msgpack_pack_real_uint16(x, d) \
-do { \
-	if(d < (1<<7)) { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \
-	} else if(d < (1<<8)) { \
-		/* unsigned 8 */ \
-		const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \
-		msgpack_pack_append_buffer(x, buf, 2); \
-	} else { \
-		/* unsigned 16 */ \
-		const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \
-		msgpack_pack_append_buffer(x, buf, 3); \
-	} \
-} while(0)
-
-#define msgpack_pack_real_uint32(x, d) \
-do { \
-	if(d < (1<<8)) { \
-		if(d < (1<<7)) { \
-			/* fixnum */ \
-			msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \
-		} else { \
-			/* unsigned 8 */ \
-			const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} \
-	} else { \
-		if(d < (1<<16)) { \
-			/* unsigned 16 */ \
-			const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} else { \
-			/* unsigned 32 */ \
-			const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \
-			msgpack_pack_append_buffer(x, buf, 5); \
-		} \
-	} \
-} while(0)
-
-#define msgpack_pack_real_uint64(x, d) \
-do { \
-	if(d < (1ULL<<8)) { \
-		if(d < (1<<7)) { \
-			/* fixnum */ \
-			msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \
-		} else { \
-			/* unsigned 8 */ \
-			const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} \
-	} else { \
-		if(d < (1ULL<<16)) { \
-			/* signed 16 */ \
-			const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} else if(d < (1ULL<<32)) { \
-			/* signed 32 */ \
-			const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \
-			msgpack_pack_append_buffer(x, buf, 5); \
-		} else { \
-			/* signed 64 */ \
-			const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \
-			msgpack_pack_append_buffer(x, buf, 9); \
-		} \
-	} \
-} while(0)
-
-#define msgpack_pack_real_int8(x, d) \
-do { \
-	if(d < -(1<<5)) { \
-		/* signed 8 */ \
-		const unsigned char buf[2] = {0xd0, STORE8_BE8(d)}; \
-		msgpack_pack_append_buffer(x, buf, 2); \
-	} else { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1); \
-	} \
-} while(0)
-
-#define msgpack_pack_real_int16(x, d) \
-do { \
-	if(d < -(1<<5)) { \
-		if(d < -(1<<7)) { \
-			/* signed 16 */ \
-			const unsigned char buf[3] = {0xd1, STORE16_BE16(d)}; \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} else { \
-			/* signed 8 */ \
-			const unsigned char buf[2] = {0xd0, STORE16_BE8(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} \
-	} else if(d < (1<<7)) { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &STORE16_BE8(d), 1); \
-	} else { \
-		if(d < (1<<8)) { \
-			/* unsigned 8 */ \
-			const unsigned char buf[2] = {0xcc, STORE16_BE8(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} else { \
-			/* unsigned 16 */ \
-			const unsigned char buf[3] = {0xcd, STORE16_BE16(d)}; \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} \
-	} \
-} while(0)
-
-#define msgpack_pack_real_int32(x, d) \
-do { \
-	if(d < -(1<<5)) { \
-		if(d < -(1<<15)) { \
-			/* signed 32 */ \
-			const unsigned char buf[5] = {0xd2, STORE32_BE32(d)}; \
-			msgpack_pack_append_buffer(x, buf, 5); \
-		} else if(d < -(1<<7)) { \
-			/* signed 16 */ \
-			const unsigned char buf[3] = {0xd1, STORE32_BE16(d)}; \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} else { \
-			/* signed 8 */ \
-			const unsigned char buf[2] = {0xd0, STORE32_BE8(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} \
-	} else if(d < (1<<7)) { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &STORE32_BE8(d), 1); \
-	} else { \
-		if(d < (1<<8)) { \
-			/* unsigned 8 */ \
-			const unsigned char buf[2] = {0xcc, STORE32_BE8(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} else if(d < (1<<16)) { \
-			/* unsigned 16 */ \
-			const unsigned char buf[3] = {0xcd, STORE32_BE16(d)}; \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} else { \
-			/* unsigned 32 */ \
-			const unsigned char buf[5] = {0xce, STORE32_BE32(d)}; \
-			msgpack_pack_append_buffer(x, buf, 5); \
-		} \
-	} \
-} while(0)
-
-#define msgpack_pack_real_int64(x, d) \
-do { \
-	if(d < -(1LL<<5)) { \
-		if(d < -(1LL<<15)) { \
-			if(d < -(1LL<<31)) { \
-				/* signed 64 */ \
-				const unsigned char buf[9] = {0xd3, STORE64_BE64(d)}; \
-				msgpack_pack_append_buffer(x, buf, 9); \
-			} else { \
-				/* signed 32 */ \
-				const unsigned char buf[5] = {0xd2, STORE64_BE32(d)}; \
-				msgpack_pack_append_buffer(x, buf, 5); \
-			} \
-		} else { \
-			if(d < -(1<<7)) { \
-				/* signed 16 */ \
-				const unsigned char buf[3] = {0xd1, STORE64_BE16(d)}; \
-				msgpack_pack_append_buffer(x, buf, 3); \
-			} else { \
-				/* signed 8 */ \
-				const unsigned char buf[2] = {0xd0, STORE64_BE8(d)}; \
-				msgpack_pack_append_buffer(x, buf, 2); \
-			} \
-		} \
-	} else if(d < (1<<7)) { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &STORE64_BE8(d), 1); \
-	} else { \
-		if(d < (1LL<<16)) { \
-			if(d < (1<<8)) { \
-				/* unsigned 8 */ \
-				const unsigned char buf[2] = {0xcc, STORE64_BE8(d)}; \
-				msgpack_pack_append_buffer(x, buf, 2); \
-			} else { \
-				/* unsigned 16 */ \
-				const unsigned char buf[3] = {0xcd, STORE64_BE16(d)}; \
-				msgpack_pack_append_buffer(x, buf, 3); \
-			} \
-		} else { \
-			if(d < (1LL<<32)) { \
-				/* unsigned 32 */ \
-				const unsigned char buf[5] = {0xce, STORE64_BE32(d)}; \
-				msgpack_pack_append_buffer(x, buf, 5); \
-			} else { \
-				/* unsigned 64 */ \
-				const unsigned char buf[9] = {0xcf, STORE64_BE64(d)}; \
-				msgpack_pack_append_buffer(x, buf, 9); \
-			} \
-		} \
-	} \
-} while(0)
-
-
-#ifdef msgpack_pack_inline_func_fastint
-
-msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d)
-{
-	const unsigned char buf[2] = {0xcc, STORE8_BE8(d)};
-	msgpack_pack_append_buffer(x, buf, 2);
-}
-
-msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d)
-{
-	const unsigned char buf[3] = {0xcd, STORE16_BE16(d)};
-	msgpack_pack_append_buffer(x, buf, 3);
-}
-
-msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d)
-{
-	const unsigned char buf[5] = {0xce, STORE32_BE32(d)};
-	msgpack_pack_append_buffer(x, buf, 5);
-}
-
-msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d)
-{
-	const unsigned char buf[9] = {0xcf, STORE64_BE64(d)};
-	msgpack_pack_append_buffer(x, buf, 9);
-}
-
-msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d)
-{
-	const unsigned char buf[2] = {0xd0, STORE8_BE8(d)};
-	msgpack_pack_append_buffer(x, buf, 2);
-}
-
-msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d)
-{
-	const unsigned char buf[3] = {0xd1, STORE16_BE16(d)};
-	msgpack_pack_append_buffer(x, buf, 3);
-}
-
-msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d)
-{
-	const unsigned char buf[5] = {0xd2, STORE32_BE32(d)};
-	msgpack_pack_append_buffer(x, buf, 5);
-}
-
-msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d)
-{
-	const unsigned char buf[9] = {0xd3, STORE64_BE64(d)};
-	msgpack_pack_append_buffer(x, buf, 9);
-}
-
-#undef msgpack_pack_inline_func_fastint
-#endif
-
-
-msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d)
-{
-	msgpack_pack_real_uint8(x, d);
-}
-
-msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d)
-{
-	msgpack_pack_real_uint16(x, d);
-}
-
-msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d)
-{
-	msgpack_pack_real_uint32(x, d);
-}
-
-msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d)
-{
-	msgpack_pack_real_uint64(x, d);
-}
-
-msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d)
-{
-	msgpack_pack_real_int8(x, d);
-}
-
-msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d)
-{
-	msgpack_pack_real_int16(x, d);
-}
-
-msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d)
-{
-	msgpack_pack_real_int32(x, d);
-}
-
-msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d)
-{
-	msgpack_pack_real_int64(x, d);
-}
-
-
-#ifdef msgpack_pack_inline_func_cint
-
-msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d)
-{
-#if defined(SIZEOF_SHORT) || defined(SHRT_MAX)
-#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff
-	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff
-	msgpack_pack_real_int32(x, d);
-#else
-	msgpack_pack_real_int64(x, d);
-#endif
-#else
-if(sizeof(short) == 2) {
-	msgpack_pack_real_int16(x, d);
-} else if(sizeof(short) == 4) {
-	msgpack_pack_real_int32(x, d);
-} else {
-	msgpack_pack_real_int64(x, d);
-}
-#endif
-}
-
-msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d)
-{
-#if defined(SIZEOF_INT) || defined(INT_MAX)
-#if SIZEOF_INT == 2 || INT_MAX == 0x7fff
-	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff
-	msgpack_pack_real_int32(x, d);
-#else
-	msgpack_pack_real_int64(x, d);
-#endif
-#else
-if(sizeof(int) == 2) {
-	msgpack_pack_real_int16(x, d);
-} else if(sizeof(int) == 4) {
-	msgpack_pack_real_int32(x, d);
-} else {
-	msgpack_pack_real_int64(x, d);
-}
-#endif
-}
-
-msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d)
-{
-#if defined(SIZEOF_LONG) || defined(LONG_MAX)
-#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL
-	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL
-	msgpack_pack_real_int32(x, d);
-#else
-	msgpack_pack_real_int64(x, d);
-#endif
-#else
-if(sizeof(long) == 2) {
-	msgpack_pack_real_int16(x, d);
-} else if(sizeof(long) == 4) {
-	msgpack_pack_real_int32(x, d);
-} else {
-	msgpack_pack_real_int64(x, d);
-}
-#endif
-}
-
-msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d)
-{
-#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX)
-#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL
-	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL
-	msgpack_pack_real_int32(x, d);
-#else
-	msgpack_pack_real_int64(x, d);
-#endif
-#else
-if(sizeof(long long) == 2) {
-	msgpack_pack_real_int16(x, d);
-} else if(sizeof(long long) == 4) {
-	msgpack_pack_real_int32(x, d);
-} else {
-	msgpack_pack_real_int64(x, d);
-}
-#endif
-}
-
-msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d)
-{
-#if defined(SIZEOF_SHORT) || defined(USHRT_MAX)
-#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU
-	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU
-	msgpack_pack_real_uint32(x, d);
-#else
-	msgpack_pack_real_uint64(x, d);
-#endif
-#else
-if(sizeof(unsigned short) == 2) {
-	msgpack_pack_real_uint16(x, d);
-} else if(sizeof(unsigned short) == 4) {
-	msgpack_pack_real_uint32(x, d);
-} else {
-	msgpack_pack_real_uint64(x, d);
-}
-#endif
-}
-
-msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d)
-{
-#if defined(SIZEOF_INT) || defined(UINT_MAX)
-#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU
-	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU
-	msgpack_pack_real_uint32(x, d);
-#else
-	msgpack_pack_real_uint64(x, d);
-#endif
-#else
-if(sizeof(unsigned int) == 2) {
-	msgpack_pack_real_uint16(x, d);
-} else if(sizeof(unsigned int) == 4) {
-	msgpack_pack_real_uint32(x, d);
-} else {
-	msgpack_pack_real_uint64(x, d);
-}
-#endif
-}
-
-msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d)
-{
-#if defined(SIZEOF_LONG) || defined(ULONG_MAX)
-#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL
-	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL
-	msgpack_pack_real_uint32(x, d);
-#else
-	msgpack_pack_real_uint64(x, d);
-#endif
-#else
-if(sizeof(unsigned int) == 2) {
-	msgpack_pack_real_uint16(x, d);
-} else if(sizeof(unsigned int) == 4) {
-	msgpack_pack_real_uint32(x, d);
-} else {
-	msgpack_pack_real_uint64(x, d);
-}
-#endif
-}
-
-msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d)
-{
-#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX)
-#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL
-	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL
-	msgpack_pack_real_uint32(x, d);
-#else
-	msgpack_pack_real_uint64(x, d);
-#endif
-#else
-if(sizeof(unsigned long long) == 2) {
-	msgpack_pack_real_uint16(x, d);
-} else if(sizeof(unsigned long long) == 4) {
-	msgpack_pack_real_uint32(x, d);
-} else {
-	msgpack_pack_real_uint64(x, d);
-}
-#endif
-}
-
-#undef msgpack_pack_inline_func_cint
-#endif
-
-
-
-/*
- * Float
- */
-
-msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d)
-{
-	union { char buf[4]; uint32_t num; } f;
-	*((float*)&f.buf) = d;  // FIXME
-	const unsigned char buf[5] = {0xca, STORE32_BE32(f.num)};
-	msgpack_pack_append_buffer(x, buf, 5);
-}
-
-msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d)
-{
-	union { char buf[8]; uint64_t num; } f;
-	*((double*)&f.buf) = d;  // FIXME
-	const unsigned char buf[9] = {0xcb, STORE64_BE64(f.num)};
-	msgpack_pack_append_buffer(x, buf, 9);
-}
-
-
-/*
- * Nil
- */
-
-msgpack_pack_inline_func(_nil)(msgpack_pack_user x)
-{
-	static const unsigned char d = 0xc0;
-	msgpack_pack_append_buffer(x, &d, 1);
-}
-
-
-/*
- * Boolean
- */
-
-msgpack_pack_inline_func(_true)(msgpack_pack_user x)
-{
-	static const unsigned char d = 0xc3;
-	msgpack_pack_append_buffer(x, &d, 1);
-}
-
-msgpack_pack_inline_func(_false)(msgpack_pack_user x)
-{
-	static const unsigned char d = 0xc2;
-	msgpack_pack_append_buffer(x, &d, 1);
-}
-
-
-/*
- * Array
- */
-
-msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n)
-{
-	if(n < 16) {
-		unsigned char d = 0x90 | n;
-		msgpack_pack_append_buffer(x, &d, 1);
-	} else if(n < 65536) {
-		uint16_t d = (uint16_t)n;
-		unsigned char buf[3] = {0xdc, STORE16_BE16(d)};
-		msgpack_pack_append_buffer(x, buf, 3);
-	} else {
-		uint32_t d = (uint32_t)n;
-		unsigned char buf[5] = {0xdd, STORE32_BE32(d)};
-		msgpack_pack_append_buffer(x, buf, 5);
-	}
-}
-
-
-/*
- * Map
- */
-
-msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n)
-{
-	if(n < 16) {
-		unsigned char d = 0x80 | n;
-		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1);
-	} else if(n < 65536) {
-		uint16_t d = (uint16_t)n;
-		unsigned char buf[3] = {0xde, STORE16_BE16(d)};
-		msgpack_pack_append_buffer(x, buf, 3);
-	} else {
-		uint32_t d = (uint32_t)n;
-		unsigned char buf[5] = {0xdf, STORE32_BE32(d)};
-		msgpack_pack_append_buffer(x, buf, 5);
-	}
-}
-
-
-/*
- * Raw
- */
-
-msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l)
-{
-	if(l < 32) {
-		unsigned char d = 0xa0 | l;
-		msgpack_pack_append_buffer(x, &STORE8_BE8(d), 1);
-	} else if(l < 65536) {
-		uint16_t d = (uint16_t)l;
-		unsigned char buf[3] = {0xda, STORE16_BE16(d)};
-		msgpack_pack_append_buffer(x, buf, 3);
-	} else {
-		uint32_t d = (uint32_t)l;
-		unsigned char buf[5] = {0xdb, STORE32_BE32(d)};
-		msgpack_pack_append_buffer(x, buf, 5);
-	}
-}
-
-msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l)
-{
-	msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
-}
-
-#undef msgpack_pack_inline_func
-#undef msgpack_pack_user
-#undef msgpack_pack_append_buffer
-
-#undef STORE8_BE8
-
-#undef STORE16_BE8
-#undef STORE16_BE16
-
-#undef STORE32_BE8
-#undef STORE32_BE16
-#undef STORE32_BE32
-
-#undef STORE64_BE8
-#undef STORE64_BE16
-#undef STORE64_BE32
-#undef STORE64_BE64
-
-#undef msgpack_pack_real_uint8
-#undef msgpack_pack_real_uint16
-#undef msgpack_pack_real_uint32
-#undef msgpack_pack_real_uint64
-#undef msgpack_pack_real_int8
-#undef msgpack_pack_real_int16
-#undef msgpack_pack_real_int32
-#undef msgpack_pack_real_int64
-
diff --git a/php/msgpack/unpack_define.h b/php/msgpack/unpack_define.h
deleted file mode 100644
index 63668c2..0000000
--- a/php/msgpack/unpack_define.h
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * MessagePack unpacking routine template
- *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
- *
- *    Licensed under the Apache License, Version 2.0 (the "License");
- *    you may not use this file except in compliance with the License.
- *    You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- */
-#ifndef MSGPACK_UNPACK_DEFINE_H__
-#define MSGPACK_UNPACK_DEFINE_H__
-
-#include 
-#include 
-#include 
-#include 
-#include 
-#ifndef __WIN32__
-#include 
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-#ifndef MSGPACK_MAX_STACK_SIZE
-#define MSGPACK_MAX_STACK_SIZE 16
-#endif
-
-
-#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-#define __LITTLE_ENDIAN__
-#elif __BYTE_ORDER == __BIG_ENDIAN
-#define __BIG_ENDIAN__
-#endif
-#endif
-
-#define msgpack_betoh16(x) ntohs(x)
-#define msgpack_betoh32(x) ntohl(x)
-
-#ifdef __LITTLE_ENDIAN__
-#if defined(__bswap_64)
-#  define msgpack_betoh64(x) __bswap_64(x)
-#elif defined(__DARWIN_OSSwapInt64)
-#  define msgpack_betoh64(x) __DARWIN_OSSwapInt64(x)
-#else
-static inline uint64_t msgpack_betoh64(uint64_t x) {
-	return	((x << 56) & 0xff00000000000000ULL ) |
-			((x << 40) & 0x00ff000000000000ULL ) |
-			((x << 24) & 0x0000ff0000000000ULL ) |
-			((x <<  8) & 0x000000ff00000000ULL ) |
-			((x >>  8) & 0x00000000ff000000ULL ) |
-			((x >> 24) & 0x0000000000ff0000ULL ) |
-			((x >> 40) & 0x000000000000ff00ULL ) |
-			((x >> 56) & 0x00000000000000ffULL ) ;
-}
-#endif
-#else
-#define msgpack_betoh64(x) (x)
-#endif
-
-
-typedef enum {
-	CS_HEADER            = 0x00,  // nil
-
-	//CS_                = 0x01,
-	//CS_                = 0x02,  // false
-	//CS_                = 0x03,  // true
-
-	//CS_                = 0x04,
-	//CS_                = 0x05,
-	//CS_                = 0x06,
-	//CS_                = 0x07,
-
-	//CS_                = 0x08,
-	//CS_                = 0x09,
-	CS_FLOAT             = 0x0a,
-	CS_DOUBLE            = 0x0b,
-	CS_UINT_8            = 0x0c,
-	CS_UINT_16           = 0x0d,
-	CS_UINT_32           = 0x0e,
-	CS_UINT_64           = 0x0f,
-	CS_INT_8             = 0x10,
-	CS_INT_16            = 0x11,
-	CS_INT_32            = 0x12,
-	CS_INT_64            = 0x13,
-
-	//CS_                = 0x14,
-	//CS_                = 0x15,
-	//CS_BIG_INT_16        = 0x16,
-	//CS_BIG_INT_32        = 0x17,
-	//CS_BIG_FLOAT_16      = 0x18,
-	//CS_BIG_FLOAT_32      = 0x19,
-	CS_RAW_16            = 0x1a,
-	CS_RAW_32            = 0x1b,
-	CS_ARRAY_16          = 0x1c,
-	CS_ARRAY_32          = 0x1d,
-	CS_MAP_16            = 0x1e,
-	CS_MAP_32            = 0x1f,
-
-	//ACS_BIG_INT_VALUE,
-	//ACS_BIG_FLOAT_VALUE,
-	ACS_RAW_VALUE,
-} msgpack_unpack_state;
-
-
-typedef enum {
-	CT_ARRAY_ITEM,
-	CT_MAP_KEY,
-	CT_MAP_VALUE,
-} msgpack_container_type;
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* msgpack/unpack_define.h */
-
diff --git a/php/msgpack/unpack_template.h b/php/msgpack/unpack_template.h
deleted file mode 100644
index d67fd1e..0000000
--- a/php/msgpack/unpack_template.h
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * MessagePack unpacking routine template
- *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
- *
- *    Licensed under the Apache License, Version 2.0 (the "License");
- *    you may not use this file except in compliance with the License.
- *    You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- */
-
-#ifndef msgpack_unpack_func
-#error msgpack_unpack_func template is not defined
-#endif
-
-#ifndef msgpack_unpack_callback
-#error msgpack_unpack_callback template is not defined
-#endif
-
-#ifndef msgpack_unpack_struct
-#error msgpack_unpack_struct template is not defined
-#endif
-
-#ifndef msgpack_unpack_struct_decl
-#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name)
-#endif
-
-#ifndef msgpack_unpack_object
-#error msgpack_unpack_object type is not defined
-#endif
-
-#ifndef msgpack_unpack_user
-#error msgpack_unpack_user type is not defined
-#endif
-
-
-msgpack_unpack_struct_decl(_stack) {
-	msgpack_unpack_object obj;
-	size_t count;
-	unsigned int ct;
-	msgpack_unpack_object map_key;
-};
-
-msgpack_unpack_struct_decl(_context) {
-	msgpack_unpack_user user;
-	unsigned int cs;
-	unsigned int trail;
-	unsigned int top;
-	msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE];
-};
-
-
-msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx)
-{
-	ctx->cs = CS_HEADER;
-	ctx->trail = 0;
-	ctx->top = 0;
-	ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user);
-}
-
-msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx)
-{
-	return (ctx)->stack[0].obj;
-}
-
-
-msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
-{
-	assert(len >= *off);
-
-	const unsigned char* p = (unsigned char*)data + *off;
-	const unsigned char* const pe = (unsigned char*)data + len;
-	const void* n = NULL;
-
-	unsigned int trail = ctx->trail;
-	unsigned int cs = ctx->cs;
-	unsigned int top = ctx->top;
-	msgpack_unpack_struct(_stack)* stack = ctx->stack;
-	msgpack_unpack_user* user = &ctx->user;
-
-	msgpack_unpack_object obj;
-	msgpack_unpack_struct(_stack)* c = NULL;
-
-	int ret;
-
-#define push_simple_value(func) \
-	if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \
-	goto _push
-#define push_fixed_value(func, arg) \
-	if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \
-	goto _push
-#define push_variable_value(func, base, pos, len) \
-	if(msgpack_unpack_callback(func)(user, \
-		(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
-	goto _push
-
-#define again_fixed_trail(_cs, trail_len) \
-	trail = trail_len; \
-	cs = _cs; \
-	goto _fixed_trail_again
-#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \
-	trail = trail_len; \
-	if(trail == 0) { goto ifzero; } \
-	cs = _cs; \
-	goto _fixed_trail_again
-
-#define start_container(func, count_, ct_) \
-	if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
-	if((count_) == 0) { obj = stack[top].obj; goto _push; } \
-	if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \
-	stack[top].ct = ct_; \
-	stack[top].count = count_; \
-	/*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
-	/*printf("stack push %d\n", top);*/ \
-	++top; \
-	goto _header_again
-
-#define NEXT_CS(p) \
-	((unsigned int)*p & 0x1f)
-
-#define PTR_CAST_8(ptr)   (*(uint8_t*)ptr)
-#define PTR_CAST_16(ptr)  msgpack_betoh16(*(uint16_t*)ptr)
-#define PTR_CAST_32(ptr)  msgpack_betoh32(*(uint32_t*)ptr)
-#define PTR_CAST_64(ptr)  msgpack_betoh64(*(uint64_t*)ptr)
-
-	if(p == pe) { goto _out; }
-	do {
-		switch(cs) {
-		case CS_HEADER:
-			switch(*p) {
-			case 0x00 ... 0x7f:  // Positive Fixnum
-				push_fixed_value(_uint8, *(uint8_t*)p);
-			case 0xe0 ... 0xff:  // Negative Fixnum
-				push_fixed_value(_int8, *(int8_t*)p);
-			case 0xc0 ... 0xdf:  // Variable
-				switch(*p) {
-				case 0xc0:  // nil
-					push_simple_value(_nil);
-				//case 0xc1:  // string
-				//	again_terminal_trail(NEXT_CS(p), p+1);
-				case 0xc2:  // false
-					push_simple_value(_false);
-				case 0xc3:  // true
-					push_simple_value(_true);
-				//case 0xc4:
-				//case 0xc5:
-				//case 0xc6:
-				//case 0xc7:
-				//case 0xc8:
-				//case 0xc9:
-				case 0xca:  // float
-				case 0xcb:  // double
-				case 0xcc:  // unsigned int  8
-				case 0xcd:  // unsigned int 16
-				case 0xce:  // unsigned int 32
-				case 0xcf:  // unsigned int 64
-				case 0xd0:  // signed int  8
-				case 0xd1:  // signed int 16
-				case 0xd2:  // signed int 32
-				case 0xd3:  // signed int 64
-					again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
-				//case 0xd4:
-				//case 0xd5:
-				//case 0xd6:  // big integer 16
-				//case 0xd7:  // big integer 32
-				//case 0xd8:  // big float 16
-				//case 0xd9:  // big float 32
-				case 0xda:  // raw 16
-				case 0xdb:  // raw 32
-				case 0xdc:  // array 16
-				case 0xdd:  // array 32
-				case 0xde:  // map 16
-				case 0xdf:  // map 32
-					again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
-				default:
-					goto _failed;
-				}
-			case 0xa0 ... 0xbf:  // FixRaw
-				again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
-			case 0x90 ... 0x9f:  // FixArray
-				start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM);
-			case 0x80 ... 0x8f:  // FixMap
-				start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
-
-			default:
-				goto _failed;
-			}
-			// end CS_HEADER
-
-
-		_fixed_trail_again:
-			++p;
-
-		default:
-			if((size_t)(pe - p) < trail) { goto _out; }
-			n = p;  p += trail - 1;
-			switch(cs) {
-			//case CS_
-			//case CS_
-			case CS_FLOAT: {
-					union { uint32_t num; char buf[4]; } f;
-					f.num = PTR_CAST_32(n);  // FIXME
-					push_fixed_value(_float, *((float*)f.buf)); }
-			case CS_DOUBLE: {
-					union { uint64_t num; char buf[8]; } f;
-					f.num = PTR_CAST_64(n);  // FIXME
-					push_fixed_value(_double, *((double*)f.buf)); }
-			case CS_UINT_8:
-				push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n));
-			case CS_UINT_16:
-				push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n));
-			case CS_UINT_32:
-				push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n));
-			case CS_UINT_64:
-				push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n));
-
-			case CS_INT_8:
-				push_fixed_value(_int8, (int8_t)PTR_CAST_8(n));
-			case CS_INT_16:
-				push_fixed_value(_int16, (int16_t)PTR_CAST_16(n));
-			case CS_INT_32:
-				push_fixed_value(_int32, (int32_t)PTR_CAST_32(n));
-			case CS_INT_64:
-				push_fixed_value(_int64, (int64_t)PTR_CAST_64(n));
-
-			//case CS_
-			//case CS_
-			//case CS_BIG_INT_16:
-			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero);
-			//case CS_BIG_INT_32:
-			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero);
-			//case ACS_BIG_INT_VALUE:
-			//_big_int_zero:
-			//	// FIXME
-			//	push_variable_value(_big_int, data, n, trail);
-
-			//case CS_BIG_FLOAT_16:
-			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero);
-			//case CS_BIG_FLOAT_32:
-			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero);
-			//case ACS_BIG_FLOAT_VALUE:
-			//_big_float_zero:
-			//	// FIXME
-			//	push_variable_value(_big_float, data, n, trail);
-
-			case CS_RAW_16:
-				again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero);
-			case CS_RAW_32:
-				again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero);
-			case ACS_RAW_VALUE:
-			_raw_zero:
-				push_variable_value(_raw, data, n, trail);
-
-			case CS_ARRAY_16:
-				start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM);
-			case CS_ARRAY_32:
-				/* FIXME security guard */
-				start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM);
-
-			case CS_MAP_16:
-				start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY);
-			case CS_MAP_32:
-				/* FIXME security guard */
-				start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY);
-
-			default:
-				goto _failed;
-			}
-		}
-
-_push:
-	if(top == 0) { goto _finish; }
-	c = &stack[top-1];
-	switch(c->ct) {
-	case CT_ARRAY_ITEM:
-		if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; }
-		if(--c->count == 0) {
-			obj = c->obj;
-			--top;
-			/*printf("stack pop %d\n", top);*/
-			goto _push;
-		}
-		goto _header_again;
-	case CT_MAP_KEY:
-		c->map_key = obj;
-		c->ct = CT_MAP_VALUE;
-		goto _header_again;
-	case CT_MAP_VALUE:
-		if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
-		if(--c->count == 0) {
-			obj = c->obj;
-			--top;
-			/*printf("stack pop %d\n", top);*/
-			goto _push;
-		}
-		c->ct = CT_MAP_KEY;
-		goto _header_again;
-
-	default:
-		goto _failed;
-	}
-
-_header_again:
-		cs = CS_HEADER;
-		++p;
-	} while(p != pe);
-	goto _out;
-
-
-_finish:
-	stack[0].obj = obj;
-	++p;
-	ret = 1;
-	/*printf("-- finish --\n"); */
-	goto _end;
-
-_failed:
-	/*printf("** FAILED **\n"); */
-	ret = -1;
-	goto _end;
-
-_out:
-	ret = 0;
-	goto _end;
-
-_end:
-	ctx->cs = cs;
-	ctx->trail = trail;
-	ctx->top = top;
-	*off = p - (const unsigned char*)data;
-
-	return ret;
-}
-
-
-#undef msgpack_unpack_func
-#undef msgpack_unpack_callback
-#undef msgpack_unpack_struct
-#undef msgpack_unpack_object
-#undef msgpack_unpack_user
-
-#undef push_simple_value
-#undef push_fixed_value
-#undef push_variable_value
-#undef again_fixed_trail
-#undef again_fixed_trail_if_zero
-#undef start_container
-
-#undef NEXT_CS
-#undef PTR_CAST_8
-#undef PTR_CAST_16
-#undef PTR_CAST_32
-#undef PTR_CAST_64
-
diff --git a/php/msgpack_class.c b/php/msgpack_class.c
new file mode 100644
index 0000000..b728cb5
--- /dev/null
+++ b/php/msgpack_class.c
@@ -0,0 +1,340 @@
+
+#include "php.h"
+
+#include "php_msgpack.h"
+#include "msgpack_pack.h"
+#include "msgpack_unpack.h"
+#include "msgpack_class.h"
+
+typedef struct {
+    zend_object object;
+    smart_str buffer;
+    zval *retval;
+    long offset;
+} php_msgpack_unpacker_t;
+
+#if ZEND_MODULE_API_NO >= 20060613
+#define MAGPACK_METHOD_BASE(classname, name) zim_##classname##_##name
+#else
+#define MSGPACK_METHOD_BASE(classname, name) zif_##classname##_##name
+#endif
+
+#define MSGPACK_METHOD(classname, name, retval, thisptr) \
+    MAGPACK_METHOD_BASE(classname, name)(0, retval, NULL, thisptr, 0 TSRMLS_CC)
+
+#define MSGPACK_UNPACKER_OBJECT       \
+    php_msgpack_unpacker_t *unpacker; \
+    unpacker =(php_msgpack_unpacker_t *)zend_object_store_get_object(getThis() TSRMLS_CC);
+
+/* MessagePack */
+static zend_class_entry *msgpack_ce = NULL;
+
+static ZEND_METHOD(msgpack, pack);
+static ZEND_METHOD(msgpack, unpack);
+static ZEND_METHOD(msgpack, unpacker);
+
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_base_pack, 0, 0, 1)
+    ZEND_ARG_INFO(0, value)
+ZEND_END_ARG_INFO()
+
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_base_unpack, 0, 0, 1)
+    ZEND_ARG_INFO(0, str)
+ZEND_END_ARG_INFO()
+
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_base_unpacker, 0, 0, 0)
+ZEND_END_ARG_INFO()
+
+static const zend_function_entry msgpack_base_methods[] = {
+    ZEND_ME(msgpack, pack, arginfo_msgpack_base_pack, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack, unpack, arginfo_msgpack_base_unpack, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack, unpacker, arginfo_msgpack_base_unpacker, ZEND_ACC_PUBLIC)
+    {NULL, NULL, NULL}
+};
+
+/* MessagePackUnpacker */
+static zend_class_entry *msgpack_unpacker_ce = NULL;
+
+static ZEND_METHOD(msgpack, __construct);
+static ZEND_METHOD(msgpack, __destruct);
+static ZEND_METHOD(msgpack, feed);
+static ZEND_METHOD(msgpack, execute);
+static ZEND_METHOD(msgpack, data);
+static ZEND_METHOD(msgpack, reset);
+
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unpacker___construct, 0, 0, 0)
+ZEND_END_ARG_INFO()
+
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unpacker___destruct, 0, 0, 0)
+ZEND_END_ARG_INFO()
+
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unpacker_feed, 0, 0, 1)
+    ZEND_ARG_INFO(0, str)
+ZEND_END_ARG_INFO()
+
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unpacker_execute, 1, 0, 0)
+    ZEND_ARG_INFO(0, str)
+    ZEND_ARG_INFO(1, offset)
+ZEND_END_ARG_INFO()
+
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unpacker_data, 0, 0, 0)
+ZEND_END_ARG_INFO()
+
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unpacker_reset, 0, 0, 0)
+ZEND_END_ARG_INFO()
+
+static const zend_function_entry msgpack_unpacker_methods[] = {
+    ZEND_ME(msgpack, __construct,
+            arginfo_msgpack_unpacker___construct, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack, __destruct,
+            arginfo_msgpack_unpacker___destruct, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack, feed, arginfo_msgpack_unpacker_feed, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack, execute, arginfo_msgpack_unpacker_execute, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack, data, arginfo_msgpack_unpacker_data, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack, reset, arginfo_msgpack_unpacker_reset, ZEND_ACC_PUBLIC)
+    {NULL, NULL, NULL}
+};
+
+static void php_msgpack_unpacker_free(
+    php_msgpack_unpacker_t *unpacker TSRMLS_DC)
+{
+    zend_object_std_dtor(&unpacker->object TSRMLS_CC);
+    efree(unpacker);
+}
+
+static zend_object_value php_msgpack_unpacker_new(
+    zend_class_entry *ce TSRMLS_DC)
+{
+    zend_object_value retval;
+    zval *tmp;
+    php_msgpack_unpacker_t *unpacker;
+
+    unpacker = emalloc(sizeof(php_msgpack_unpacker_t));
+
+    zend_object_std_init(&unpacker->object, ce TSRMLS_CC);
+
+    zend_hash_copy(
+        unpacker->object.properties, &ce->default_properties,
+        (copy_ctor_func_t)zval_add_ref, (void *)&tmp, sizeof(zval *));
+
+    retval.handle = zend_objects_store_put(
+        unpacker, (zend_objects_store_dtor_t)zend_objects_destroy_object,
+        (zend_objects_free_object_storage_t)php_msgpack_unpacker_free,
+        NULL TSRMLS_CC);
+    retval.handlers = zend_get_std_object_handlers();
+
+    return retval;
+}
+
+/* MessagePack */
+static ZEND_METHOD(msgpack, pack)
+{
+    zval *parameter;
+    smart_str buf = {0};
+
+    if (zend_parse_parameters(
+            ZEND_NUM_ARGS() TSRMLS_CC, "z", ¶meter) == FAILURE)
+    {
+        return;
+    }
+
+    php_msgpack_serialize(&buf, parameter TSRMLS_CC);
+
+    ZVAL_STRINGL(return_value, buf.c, buf.len, 1);
+
+    smart_str_free(&buf);
+}
+
+static ZEND_METHOD(msgpack, unpack)
+{
+    char *str;
+    int str_len;
+
+    if (zend_parse_parameters(
+            ZEND_NUM_ARGS() TSRMLS_CC, "s", &str, &str_len) == FAILURE)
+    {
+        return;
+    }
+
+    if (!str_len)
+    {
+        RETURN_NULL();
+    }
+
+    php_msgpack_unserialize(return_value, str, str_len TSRMLS_CC);
+}
+
+static ZEND_METHOD(msgpack, unpacker)
+{
+    zval temp;
+
+    object_init_ex(return_value, msgpack_unpacker_ce);
+
+    MSGPACK_METHOD(msgpack, __construct, &temp, return_value);
+}
+
+/* MessagePackUnpacker */
+static ZEND_METHOD(msgpack, __construct)
+{
+    MSGPACK_UNPACKER_OBJECT;
+
+    unpacker->buffer.c = NULL;
+    unpacker->buffer.len = 0;
+    unpacker->buffer.a = 0;
+    unpacker->retval = NULL;
+    unpacker->offset = 0;
+}
+
+static ZEND_METHOD(msgpack, __destruct)
+{
+    MSGPACK_UNPACKER_OBJECT;
+
+    smart_str_free(&unpacker->buffer);
+
+    if (unpacker->retval != NULL)
+    {
+        zval_ptr_dtor(&unpacker->retval);
+    }
+}
+
+static ZEND_METHOD(msgpack, feed)
+{
+    char *str;
+    int str_len;
+    MSGPACK_UNPACKER_OBJECT;
+
+    if (zend_parse_parameters(
+            ZEND_NUM_ARGS() TSRMLS_CC, "s", &str, &str_len) == FAILURE)
+    {
+        return;
+    }
+
+    if (!str_len)
+    {
+        RETURN_FALSE;
+    }
+
+    smart_str_appendl(&unpacker->buffer, str, str_len);
+
+    RETURN_TRUE;
+}
+
+static ZEND_METHOD(msgpack, execute)
+{
+    char *str = NULL;
+    long str_len = 0;
+    zval *offset;
+    int ret;
+    php_unserialize_data_t var_hash;
+    msgpack_unserialize_data mpsd;
+    MSGPACK_UNPACKER_OBJECT;
+
+    if (zend_parse_parameters(
+            ZEND_NUM_ARGS() TSRMLS_CC, "|sz/",
+            &str, &str_len, &offset) == FAILURE)
+    {
+        return;
+    }
+
+    if (str != NULL)
+    {
+        mpsd.data = (unsigned char *)str;
+        mpsd.length = str_len;
+        mpsd.offset = Z_LVAL_P(offset);
+    }
+    else
+    {
+        mpsd.data = (unsigned char *)unpacker->buffer.c;
+        mpsd.length = unpacker->buffer.len;
+        mpsd.offset = unpacker->offset;
+    }
+
+    if (mpsd.length <= 0 || mpsd.length == mpsd.offset)
+    {
+        RETURN_FALSE;
+    }
+
+    PHP_VAR_UNSERIALIZE_INIT(var_hash);
+
+    if (unpacker->retval == NULL)
+    {
+        ALLOC_INIT_ZVAL(unpacker->retval);
+    }
+
+    MSGPACK_G(error_display) = 0;
+
+    ret = msgpack_unserialize_zval(
+        &unpacker->retval, &mpsd, &var_hash TSRMLS_CC);
+
+    MSGPACK_G(error_display) = 1;
+
+    PHP_VAR_UNSERIALIZE_DESTROY(var_hash);
+
+    if (str != NULL)
+    {
+        ZVAL_LONG(offset, mpsd.offset);
+    }
+    else
+    {
+        unpacker->offset = mpsd.offset;
+    }
+
+    switch (ret)
+    {
+        case MSGPACK_UNPACK_EXTRA_BYTES:
+        case MSGPACK_UNPACK_SUCCESS:
+            RETURN_TRUE;
+        default:
+            RETURN_FALSE;
+    }
+}
+
+static ZEND_METHOD(msgpack, data)
+{
+    MSGPACK_UNPACKER_OBJECT;
+
+    RETURN_ZVAL(unpacker->retval, 1, 1);
+}
+
+static ZEND_METHOD(msgpack, reset)
+{
+    smart_str buffer = {0};
+    MSGPACK_UNPACKER_OBJECT;
+
+    if (unpacker->buffer.len > unpacker->offset)
+    {
+        smart_str_appendl(&buffer, unpacker->buffer.c + unpacker->offset,
+                          unpacker->buffer.len - unpacker->offset);
+    }
+
+    smart_str_free(&unpacker->buffer);
+
+    unpacker->buffer.c = NULL;
+    unpacker->buffer.len = 0;
+    unpacker->buffer.a = 0;
+    unpacker->offset = 0;
+
+    if (buffer.len > 0)
+    {
+        smart_str_appendl(&unpacker->buffer, buffer.c, buffer.len);
+    }
+
+    smart_str_free(&buffer);
+
+    if (unpacker->retval != NULL)
+    {
+        zval_ptr_dtor(&unpacker->retval);
+        unpacker->retval = NULL;
+    }
+}
+
+void msgpack_init_class(TSRMLS_DC)
+{
+    zend_class_entry ce;
+
+    INIT_CLASS_ENTRY(ce, "MessagePack", msgpack_base_methods);
+    msgpack_ce = zend_register_internal_class(&ce TSRMLS_CC);
+
+    INIT_CLASS_ENTRY(ce, "MessagePackUnpacker", msgpack_unpacker_methods);
+    msgpack_unpacker_ce = zend_register_internal_class(&ce TSRMLS_CC);
+    msgpack_unpacker_ce->create_object = php_msgpack_unpacker_new;
+}
diff --git a/php/msgpack_class.h b/php/msgpack_class.h
new file mode 100644
index 0000000..00ac9e7
--- /dev/null
+++ b/php/msgpack_class.h
@@ -0,0 +1,7 @@
+
+#ifndef MSGPACL_CLASS_H
+#define MSGPACL_CLASS_H
+
+void msgpack_init_class(TSRMLS_DC);
+
+#endif
diff --git a/php/msgpack_pack.c b/php/msgpack_pack.c
new file mode 100644
index 0000000..91a3d39
--- /dev/null
+++ b/php/msgpack_pack.c
@@ -0,0 +1,514 @@
+
+#include "php.h"
+#include "php_ini.h"
+#include "ext/standard/php_smart_str.h"
+#include "ext/standard/php_incomplete_class.h"
+#include "ext/standard/php_var.h"
+
+#include "php_msgpack.h"
+#include "msgpack_pack.h"
+
+#include "msgpack/pack_define.h"
+#define msgpack_pack_user smart_str*
+#define msgpack_pack_inline_func(name) \
+    static inline void msgpack_pack ## name
+#define msgpack_pack_inline_func_cint(name) \
+    static inline void msgpack_pack ## name
+#define msgpack_pack_append_buffer(user, buf, len) \
+    smart_str_appendl(user, (const void*)buf, len)
+#include "msgpack/pack_template.h"
+
+#if (PHP_MAJOR_VERSION == 5 && PHP_MINOR_VERSION < 3)
+#   define Z_ISREF_P(pz) PZVAL_IS_REF(pz)
+#endif
+
+inline static int msgpack_var_add(
+    HashTable *var_hash, zval *var, void *var_old TSRMLS_DC)
+{
+    ulong var_no;
+    char id[32], *p;
+    int len;
+
+    if ((Z_TYPE_P(var) == IS_OBJECT) && Z_OBJ_HT_P(var)->get_class_entry)
+    {
+        p = smart_str_print_long(
+            id + sizeof(id) - 1,
+            (((size_t)Z_OBJCE_P(var) << 5)
+             | ((size_t)Z_OBJCE_P(var) >> (sizeof(long) * 8 - 5)))
+            + (long)Z_OBJ_HANDLE_P(var));
+        len = id + sizeof(id) - 1 - p;
+    }
+    else
+    {
+        p = smart_str_print_long(id + sizeof(id) - 1, (long)var);
+        len = id + sizeof(id) - 1 - p;
+    }
+
+    if (var_old && zend_hash_find(var_hash, p, len, var_old) == SUCCESS)
+    {
+        if (!Z_ISREF_P(var))
+        {
+            var_no = -1;
+            zend_hash_next_index_insert(
+                var_hash, &var_no, sizeof(var_no), NULL);
+        }
+        return FAILURE;
+    }
+
+    var_no = zend_hash_num_elements(var_hash) + 1;
+
+    zend_hash_add(var_hash, p, len, &var_no, sizeof(var_no), NULL);
+
+    return SUCCESS;
+}
+
+inline static void msgpack_serialize_string(
+    smart_str *buf, char *str, size_t len)
+{
+    msgpack_pack_raw(buf, len);
+    msgpack_pack_raw_body(buf, str, len);
+}
+
+inline static void msgpack_serialize_class(
+    smart_str *buf, zval *val, zval *retval_ptr, HashTable *var_hash,
+    char *class_name, zend_uint name_len, zend_bool incomplete_class TSRMLS_DC)
+{
+    int count;
+    HashTable *ht = HASH_OF(retval_ptr);
+
+    count = zend_hash_num_elements(ht);
+    if (incomplete_class)
+    {
+        --count;
+    }
+
+    if (count > 0)
+    {
+        char *key;
+        zval **data, **name;
+        ulong key_index;
+        HashPosition pos;
+        int n;
+        zval nval, *nvalp;
+
+        msgpack_pack_map(buf, count + 1);
+
+        msgpack_pack_nil(buf);
+        msgpack_serialize_string(buf, class_name, name_len);
+
+        ZVAL_NULL(&nval);
+        nvalp = &nval;
+
+        zend_hash_internal_pointer_reset_ex(ht, &pos);
+
+        for (;; zend_hash_move_forward_ex(ht, &pos))
+        {
+            n = zend_hash_get_current_key_ex(
+                ht, &key, NULL, &key_index, 0, &pos);
+
+            if (n == HASH_KEY_NON_EXISTANT)
+            {
+                break;
+            }
+            if (incomplete_class && strcmp(key, MAGIC_MEMBER) == 0)
+            {
+                continue;
+            }
+
+            zend_hash_get_current_data_ex(ht, (void **)&name, &pos);
+
+            if (Z_TYPE_PP(name) != IS_STRING)
+            {
+                zend_error(E_NOTICE,
+                           "[msgpack] (msgpack_serialize_class) "
+                           "__sleep should return an array only "
+                           "containing the names of "
+                           "instance-variables to serialize.");
+
+                msgpack_pack_nil(buf);
+                continue;
+            }
+
+            if (zend_hash_find(
+                    Z_OBJPROP_P(val), Z_STRVAL_PP(name),
+                    Z_STRLEN_PP(name) + 1, (void *)&data) == SUCCESS)
+            {
+                msgpack_serialize_string(
+                    buf, Z_STRVAL_PP(name), Z_STRLEN_PP(name));
+                msgpack_serialize_zval(buf, *data, var_hash TSRMLS_CC);
+            }
+            else
+            {
+                zend_class_entry *ce;
+                ce = zend_get_class_entry(val TSRMLS_CC);
+                if (ce)
+                {
+                    char *prot_name, *priv_name;
+                    int prop_name_length;
+
+                    do
+                    {
+                        zend_mangle_property_name(
+                            &priv_name, &prop_name_length, ce->name,
+                            ce->name_length, Z_STRVAL_PP(name),
+                            Z_STRLEN_PP(name),
+                            ce->type & ZEND_INTERNAL_CLASS);
+                        if (zend_hash_find(
+                                Z_OBJPROP_P(val), priv_name,
+                                prop_name_length + 1,
+                                (void *)&data) == SUCCESS)
+                        {
+                            msgpack_serialize_string(
+                                buf, priv_name, prop_name_length);
+
+                            pefree(priv_name,
+                                   ce->type & ZEND_INTERNAL_CLASS);
+
+                            msgpack_serialize_zval(
+                                buf, *data, var_hash TSRMLS_CC);
+                            break;
+                        }
+
+                        pefree(priv_name,
+                               ce->type & ZEND_INTERNAL_CLASS);
+
+                        zend_mangle_property_name(
+                            &prot_name, &prop_name_length, "*", 1,
+                            Z_STRVAL_PP(name), Z_STRLEN_PP(name),
+                            ce->type & ZEND_INTERNAL_CLASS);
+
+                        if (zend_hash_find(
+                                Z_OBJPROP_P(val), prot_name,
+                                prop_name_length + 1,
+                                (void *)&data) == SUCCESS)
+                        {
+                            msgpack_serialize_string(
+                                buf, prot_name, prop_name_length);
+
+                            pefree(prot_name,
+                                   ce->type & ZEND_INTERNAL_CLASS);
+
+                            msgpack_serialize_zval(
+                                buf, *data, var_hash TSRMLS_CC);
+                            break;
+                        }
+
+                        pefree(prot_name, ce->type & ZEND_INTERNAL_CLASS);
+
+                        zend_error(E_NOTICE,
+                                   "[msgpack] (msgpack_serialize_class) "
+                                   "\"%s\" returned as member variable from "
+                                   "__sleep() but does not exist",
+                                   Z_STRVAL_PP(name));
+
+                        msgpack_serialize_string(
+                            buf, Z_STRVAL_PP(name), Z_STRLEN_PP(name));
+
+                        msgpack_serialize_zval(
+                            buf, nvalp, var_hash TSRMLS_CC);
+                    }
+                    while (0);
+                }
+                else
+                {
+                    msgpack_serialize_string(
+                        buf, Z_STRVAL_PP(name), Z_STRLEN_PP(name));
+
+                    msgpack_serialize_zval(buf, nvalp, var_hash TSRMLS_CC);
+                }
+            }
+        }
+    }
+}
+
+inline static void msgpack_serialize_array(
+    smart_str *buf, zval *val, HashTable *var_hash, bool object,
+    char* class_name, zend_uint name_len, zend_bool incomplete_class TSRMLS_DC)
+{
+    HashTable *ht;
+    size_t n;
+
+    if (object)
+    {
+        ht = Z_OBJPROP_P(val);
+    }
+    else
+    {
+        ht = HASH_OF(val);
+    }
+
+    if (ht)
+    {
+        n = zend_hash_num_elements(ht);
+    }
+    else
+    {
+        n = 0;
+    }
+
+    if (n > 0 && incomplete_class)
+    {
+        --n;
+    }
+
+    if (object)
+    {
+        msgpack_pack_map(buf, n + 1);
+
+        msgpack_pack_nil(buf);
+
+        msgpack_serialize_string(buf, class_name, name_len);
+    }
+    else if (n == 0)
+    {
+        msgpack_pack_array(buf, n);
+    }
+    else
+    {
+        msgpack_pack_map(buf, n);
+    }
+
+    if (n > 0)
+    {
+        char *key;
+        uint key_len;
+        int key_type;
+        ulong key_index;
+        zval **data;
+        HashPosition pos;
+
+        zend_hash_internal_pointer_reset_ex(ht, &pos);
+        for (;; zend_hash_move_forward_ex(ht, &pos))
+        {
+            key_type = zend_hash_get_current_key_ex(
+                ht, &key, &key_len, &key_index, 0, &pos);
+
+            if (key_type == HASH_KEY_NON_EXISTANT)
+            {
+                break;
+            }
+            if (incomplete_class && strcmp(key, MAGIC_MEMBER) == 0)
+            {
+                continue;
+            }
+
+            switch (key_type)
+            {
+                case HASH_KEY_IS_LONG:
+                    msgpack_pack_long(buf, key_index);
+                    break;
+                case HASH_KEY_IS_STRING:
+                    msgpack_serialize_string(buf, key, key_len - 1);
+                    break;
+                default:
+                    msgpack_serialize_string(buf, "", sizeof(""));
+
+                    zend_error(E_WARNING, "[msgpack] (msgpack_serialize_array) "
+                               "key is not string nor array");
+                    break;
+            }
+
+            if (zend_hash_get_current_data_ex(
+                    ht, (void *)&data, &pos) != SUCCESS ||
+                !data || data == &val ||
+                (Z_TYPE_PP(data) == IS_ARRAY &&
+                 Z_ARRVAL_PP(data)->nApplyCount > 1))
+            {
+                msgpack_pack_nil(buf);
+            }
+            else
+            {
+                if (Z_TYPE_PP(data) == IS_ARRAY)
+                {
+                    Z_ARRVAL_PP(data)->nApplyCount++;
+                }
+
+                msgpack_serialize_zval(buf, *data, var_hash TSRMLS_CC);
+
+                if (Z_TYPE_PP(data) == IS_ARRAY)
+                {
+                    Z_ARRVAL_PP(data)->nApplyCount--;
+                }
+            }
+        }
+    }
+}
+
+inline static void msgpack_serialize_object(
+    smart_str *buf, zval *val, HashTable *var_hash,
+    char* class_name, zend_uint name_len, zend_bool incomplete_class TSRMLS_DC)
+{
+    zval *retval_ptr = NULL;
+    zval fname;
+    int res;
+    zend_class_entry *ce = NULL;
+
+    if (Z_OBJ_HT_P(val)->get_class_entry)
+    {
+        ce = Z_OBJCE_P(val);
+    }
+
+    if (ce && ce->serialize != NULL)
+    {
+        unsigned char *serialized_data = NULL;
+        zend_uint serialized_length;
+
+        if (ce->serialize(
+                val, &serialized_data, &serialized_length,
+                (zend_serialize_data *)var_hash TSRMLS_CC) == SUCCESS &&
+            !EG(exception))
+        {
+            /* has custom handler */
+            msgpack_pack_map(buf, 2);
+
+            msgpack_pack_nil(buf);
+            msgpack_pack_long(buf, MSGPACK_SERIALIZE_TYPE_CUSTOM_OBJECT);
+
+            msgpack_serialize_string(buf, ce->name, ce->name_length);
+
+            msgpack_pack_raw(buf, serialized_length);
+            msgpack_pack_raw_body(buf, serialized_data, serialized_length);
+        }
+        else
+        {
+            msgpack_pack_nil(buf);
+        }
+
+        if (serialized_data)
+        {
+            efree(serialized_data);
+        }
+
+        return;
+    }
+
+    if (ce && ce != PHP_IC_ENTRY &&
+        zend_hash_exists(&ce->function_table, "__sleep", sizeof("__sleep")))
+    {
+        INIT_PZVAL(&fname);
+        ZVAL_STRINGL(&fname, "__sleep", sizeof("__sleep") - 1, 0);
+        res = call_user_function_ex(CG(function_table), &val, &fname,
+                                    &retval_ptr, 0, 0, 1, NULL TSRMLS_CC);
+        if (res == SUCCESS && !EG(exception))
+        {
+            if (retval_ptr)
+            {
+                if (HASH_OF(retval_ptr))
+                {
+                    msgpack_serialize_class(
+                        buf, val, retval_ptr, var_hash,
+                        class_name, name_len, incomplete_class TSRMLS_CC);
+                }
+                else
+                {
+                    zend_error(E_NOTICE,
+                               "[msgpack] (msgpack_serialize_object) "
+                               "__sleep should return an array only "
+                               "containing the names of instance-variables "
+                               "to serialize");
+                    msgpack_pack_nil(buf);
+                }
+                zval_ptr_dtor(&retval_ptr);
+            }
+            return;
+        }
+    }
+
+    if (retval_ptr)
+    {
+        zval_ptr_dtor(&retval_ptr);
+    }
+
+    msgpack_serialize_array(
+        buf, val, var_hash, true,
+        class_name, name_len, incomplete_class TSRMLS_CC);
+}
+
+void msgpack_serialize_zval(
+    smart_str *buf, zval *val, HashTable *var_hash TSRMLS_DC)
+{
+    ulong *var_already;
+
+    if (var_hash &&
+        msgpack_var_add(
+            var_hash, val, (void *)&var_already TSRMLS_CC) == FAILURE)
+    {
+        if (Z_ISREF_P(val))
+        {
+            msgpack_pack_map(buf, 2);
+
+            msgpack_pack_nil(buf);
+            msgpack_pack_long(buf, MSGPACK_SERIALIZE_TYPE_REFERENCE);
+
+            msgpack_pack_nil(buf);
+            msgpack_pack_long(buf, *var_already);
+
+            return;
+        }
+        else if (Z_TYPE_P(val) == IS_OBJECT)
+        {
+            msgpack_pack_map(buf, 2);
+
+            msgpack_pack_nil(buf);
+            msgpack_pack_long(buf, MSGPACK_SERIALIZE_TYPE_OBJECT);
+
+            msgpack_pack_nil(buf);
+            msgpack_pack_long(buf, *var_already);
+
+            return;
+        }
+    }
+
+    switch (Z_TYPE_P(val))
+    {
+        case IS_NULL:
+            msgpack_pack_nil(buf);
+            break;
+        case IS_BOOL:
+            if (Z_BVAL_P(val))
+            {
+                msgpack_pack_true(buf);
+            }
+            else
+            {
+                msgpack_pack_false(buf);
+            }
+            break;
+        case IS_LONG:
+            msgpack_pack_long(buf, Z_LVAL_P(val));
+            break;
+        case IS_DOUBLE:
+            {
+                double dbl = Z_DVAL_P(val);
+                msgpack_pack_double(buf, dbl);
+            }
+            break;
+        case IS_STRING:
+            msgpack_serialize_string(
+                buf, Z_STRVAL_P(val), Z_STRLEN_P(val));
+            break;
+        case IS_ARRAY:
+            msgpack_serialize_array(
+                buf, val, var_hash, false, NULL, 0, 0 TSRMLS_CC);
+            break;
+        case IS_OBJECT:
+            {
+                PHP_CLASS_ATTRIBUTES;
+                PHP_SET_CLASS_ATTRIBUTES(val);
+
+                msgpack_serialize_object(
+                    buf, val, var_hash, class_name, name_len,
+                    incomplete_class TSRMLS_CC);
+
+                PHP_CLEANUP_CLASS_ATTRIBUTES();
+            }
+            break;
+        default:
+            zend_error(E_WARNING,
+                       "[msgpack] (php_msgpack_serialize) "
+                       "type is unsupported, encoded as null");
+            msgpack_pack_nil(buf);
+            break;
+    }
+    return;
+}
diff --git a/php/msgpack_pack.h b/php/msgpack_pack.h
new file mode 100644
index 0000000..96cfd4e
--- /dev/null
+++ b/php/msgpack_pack.h
@@ -0,0 +1,17 @@
+
+#ifndef MSGPACL_PACK_H
+#define MSGPACL_PACK_H
+
+#include "ext/standard/php_smart_str.h"
+
+enum msgpack_serialize_type
+{
+    MSGPACK_SERIALIZE_TYPE_REFERENCE =  1,
+    MSGPACK_SERIALIZE_TYPE_OBJECT,
+    MSGPACK_SERIALIZE_TYPE_CUSTOM_OBJECT,
+};
+
+void msgpack_serialize_zval(
+    smart_str *buf, zval *val, HashTable *var_hash TSRMLS_DC);
+
+#endif
diff --git a/php/msgpack_unpack.c b/php/msgpack_unpack.c
new file mode 100644
index 0000000..c6b9b3f
--- /dev/null
+++ b/php/msgpack_unpack.c
@@ -0,0 +1,829 @@
+
+#include "php.h"
+#include "php_ini.h"
+#include "ext/standard/php_incomplete_class.h"
+#include "ext/standard/php_var.h"
+
+#include "php_msgpack.h"
+#include "msgpack_pack.h"
+#include "msgpack_unpack.h"
+
+#include "msgpack/unpack_define.h"
+
+#define VAR_ENTRIES_MAX 1024
+
+#if (PHP_MAJOR_VERSION == 5 && PHP_MINOR_VERSION < 3)
+#   define Z_ADDREF_PP(ppz)      ZVAL_ADDREF(*(ppz))
+#   define Z_SET_ISREF_PP(ppz)   (*(ppz))->is_ref = 1
+#   define Z_UNSET_ISREF_PP(ppz) (*(ppz))->is_ref = 0
+#endif
+
+typedef struct
+{
+    zval *data[VAR_ENTRIES_MAX];
+    long used_slots;
+    void *next;
+} var_entries;
+
+inline static void msgpack_var_push(
+    php_unserialize_data_t *var_hashx, zval **rval)
+{
+    var_entries *var_hash, *prev = NULL;
+
+    if (!var_hashx)
+    {
+        return;
+    }
+
+    var_hash = var_hashx->first;
+
+    while (var_hash && var_hash->used_slots == VAR_ENTRIES_MAX)
+    {
+        prev = var_hash;
+        var_hash = var_hash->next;
+    }
+
+    if (!var_hash)
+    {
+        var_hash = emalloc(sizeof(var_entries));
+        var_hash->used_slots = 0;
+        var_hash->next = 0;
+
+        if (!var_hashx->first)
+        {
+            var_hashx->first = var_hash;
+        }
+        else
+        {
+            prev->next = var_hash;
+        }
+    }
+
+    var_hash->data[var_hash->used_slots++] = *rval;
+}
+
+inline static int msgpack_var_access(
+    php_unserialize_data_t *var_hashx, long id, zval ***store)
+{
+    var_entries *var_hash = var_hashx->first;
+
+    while (id >= VAR_ENTRIES_MAX &&
+           var_hash && var_hash->used_slots == VAR_ENTRIES_MAX)
+    {
+        var_hash = var_hash->next;
+        id -= VAR_ENTRIES_MAX;
+    }
+
+    if (!var_hash)
+    {
+        return !SUCCESS;
+    }
+
+    if (id < 0 || id >= var_hash->used_slots)
+    {
+        return !SUCCESS;
+    }
+
+    *store = &var_hash->data[id];
+
+    return SUCCESS;
+}
+
+inline static int msgpack_unserialize_array(
+    zval **return_value, msgpack_unserialize_data *mpsd,
+    ulong ct, php_unserialize_data_t *var_hash TSRMLS_DC)
+{
+    ulong i;
+    HashTable *ht;
+
+    msgpack_var_push(var_hash, return_value);
+
+    if (Z_TYPE_PP(return_value) != IS_ARRAY)
+    {
+        array_init(*return_value);
+    }
+
+    ht = HASH_OF(*return_value);
+
+    for (i = 0; i < ct; i++)
+    {
+        zval *val;
+
+        /* value */
+        ALLOC_INIT_ZVAL(val);
+
+        if (msgpack_unserialize_zval(&val, mpsd, var_hash TSRMLS_CC) <= 0)
+        {
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (msgpack_unserialize_array) "
+                           "Invalid value");
+            }
+            zval_ptr_dtor(&val);
+
+            return MSGPACK_UNPACK_PARSE_ERROR;
+        }
+
+        /* update */
+        zend_hash_index_update(ht, i, &val, sizeof(zval *), NULL);
+    }
+
+    return MSGPACK_UNPACK_SUCCESS;
+}
+
+inline static int msgpack_unserialize_object_type(
+    zval **return_value, msgpack_unserialize_data *mpsd,
+    php_unserialize_data_t *var_hash, zend_bool is_ref TSRMLS_DC)
+{
+    int ret = MSGPACK_UNPACK_SUCCESS;
+    zval *key, *val, **rval;
+
+    ALLOC_INIT_ZVAL(key);
+
+    if (msgpack_unserialize_zval(&key, mpsd, NULL TSRMLS_CC) <= 0)
+    {
+        zval_ptr_dtor(&key);
+
+        ZVAL_BOOL(*return_value, 0);
+
+        return MSGPACK_UNPACK_PARSE_ERROR;
+    }
+
+    ALLOC_INIT_ZVAL(val);
+
+    if (is_ref)
+    {
+        ret = msgpack_unserialize_zval(&val, mpsd, NULL TSRMLS_CC);
+    }
+    else
+    {
+        ret = msgpack_unserialize_zval(&val, mpsd, var_hash TSRMLS_CC);
+    }
+
+    if (ret <= 0)
+    {
+        zval_ptr_dtor(&val);
+        zval_ptr_dtor(&key);
+
+        ZVAL_BOOL(*return_value, 0);
+
+        return MSGPACK_UNPACK_PARSE_ERROR;
+    }
+
+    if (!var_hash ||
+        msgpack_var_access(var_hash, Z_LVAL_P(val) - 1, &rval) != SUCCESS)
+    {
+        if (MSGPACK_G(error_display))
+        {
+            zend_error(E_WARNING,
+                       "[msgpack] (msgpack_unserialize_object_key) "
+                       "Invalid references value: %ld",
+                       Z_LVAL_P(val) - 1);
+        }
+        ret = MSGPACK_UNPACK_CONTINUE;
+    }
+    else
+    {
+        if (*return_value != NULL)
+        {
+            zval_ptr_dtor(return_value);
+        }
+
+        *return_value = *rval;
+
+        Z_ADDREF_PP(return_value);
+
+        if (is_ref)
+        {
+            Z_SET_ISREF_PP(return_value);
+        }
+        else
+        {
+            Z_UNSET_ISREF_PP(return_value);
+        }
+    }
+
+    zval_ptr_dtor(&key);
+    zval_ptr_dtor(&val);
+
+    return ret;
+}
+
+inline static int msgpack_unserialize_object(
+    zval **return_value, msgpack_unserialize_data *mpsd,
+    ulong ct, php_unserialize_data_t *var_hash TSRMLS_DC)
+{
+    int ret = MSGPACK_UNPACK_SUCCESS;
+
+    zend_class_entry *ce, **pce;
+    bool incomplete_class = false;
+    zval *user_func, *retval_ptr, **args[1], *arg_func_name;
+    HashTable *ht;
+
+    zval *key, *val;
+
+    int object = 1;
+    int custom_object = 0;
+
+    /* Get class */
+    ALLOC_INIT_ZVAL(key);
+
+    if (msgpack_unserialize_zval(&key, mpsd, NULL TSRMLS_CC) <= 0)
+    {
+        if (MSGPACK_G(error_display))
+        {
+            zend_error(E_WARNING,
+                       "[msgpack] (msgpack_unserialize_object) "
+                       "Invalid sign key");
+        }
+
+        zval_ptr_dtor(&key);
+
+        ZVAL_BOOL(*return_value, 0);
+
+        return MSGPACK_UNPACK_PARSE_ERROR;
+    }
+
+    ct--;
+
+    if (Z_TYPE_P(key) == IS_NULL)
+    {
+        ALLOC_INIT_ZVAL(val);
+
+        if (msgpack_unserialize_zval(&val, mpsd, NULL TSRMLS_CC) <= 0)
+        {
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (msgpack_unserialize_object) "
+                           "Invalid sign value");
+            }
+
+            zval_ptr_dtor(&val);
+            zval_ptr_dtor(&key);
+
+            ZVAL_BOOL(*return_value, 0);
+
+            return MSGPACK_UNPACK_PARSE_ERROR;
+        }
+
+        if (Z_TYPE_P(val) == IS_LONG)
+        {
+            switch (Z_LVAL_P(val))
+            {
+                case MSGPACK_SERIALIZE_TYPE_REFERENCE:
+                    ret = msgpack_unserialize_object_type(
+                        return_value, mpsd, var_hash, 1 TSRMLS_CC);
+
+                    zval_ptr_dtor(&key);
+                    zval_ptr_dtor(&val);
+
+                    return ret;
+                case MSGPACK_SERIALIZE_TYPE_OBJECT:
+                    ret = msgpack_unserialize_object_type(
+                        return_value, mpsd, var_hash, 0 TSRMLS_CC);
+
+                    zval_ptr_dtor(&key);
+                    zval_ptr_dtor(&val);
+
+                    return ret;
+                case MSGPACK_SERIALIZE_TYPE_CUSTOM_OBJECT:
+                    custom_object = 1;
+
+                    zval_ptr_dtor(&val);
+
+                    ALLOC_INIT_ZVAL(val);
+
+                    if (msgpack_unserialize_zval(
+                            &val, mpsd, NULL TSRMLS_CC) <= 0)
+                    {
+                        zval_ptr_dtor(&key);
+                        zval_ptr_dtor(&val);
+
+                        ZVAL_BOOL(*return_value, 0);
+
+                        return MSGPACK_UNPACK_PARSE_ERROR;
+                    }
+                    break;
+                default:
+                    zval_ptr_dtor(&key);
+                    zval_ptr_dtor(&val);
+
+                    ZVAL_BOOL(*return_value, 0);
+
+                    return MSGPACK_UNPACK_PARSE_ERROR;
+            }
+        }
+    }
+    else
+    {
+        object = 0;
+
+        msgpack_var_push(var_hash, return_value);
+
+        if (Z_TYPE_PP(return_value) != IS_ARRAY)
+        {
+            array_init(*return_value);
+        }
+
+        ht = HASH_OF(*return_value);
+
+        ALLOC_INIT_ZVAL(val);
+
+        if (msgpack_unserialize_zval(&val, mpsd, var_hash TSRMLS_CC) <= 0)
+        {
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (msgpack_unserialize_object) "
+                           "Invalid sign value");
+            }
+
+            zval_ptr_dtor(&val);
+            zval_ptr_dtor(&key);
+
+            return MSGPACK_UNPACK_PARSE_ERROR;
+        }
+
+        /* update */
+        switch (Z_TYPE_P(key))
+        {
+            case IS_LONG:
+                zend_hash_index_update(
+                    ht, Z_LVAL_P(key), &val, sizeof(zval *), NULL);
+                break;
+            case IS_STRING:
+                zend_symtable_update(
+                    ht, Z_STRVAL_P(key), Z_STRLEN_P(key) + 1,
+                    &val, sizeof(zval *), NULL);
+                break;
+            default:
+                zval_ptr_dtor(&key);
+                zval_ptr_dtor(&val);
+
+                return MSGPACK_UNPACK_PARSE_ERROR;
+        }
+    }
+
+    zval_ptr_dtor(&key);
+
+    if (object)
+    {
+        convert_to_string(val);
+
+        do {
+            /* Try to find class directly */
+            if (zend_lookup_class(
+                    Z_STRVAL_P(val), Z_STRLEN_P(val), &pce TSRMLS_CC) == SUCCESS)
+            {
+                ce = *pce;
+                break;
+            }
+
+            /* Check for unserialize callback */
+            if ((PG(unserialize_callback_func) == NULL) ||
+                (PG(unserialize_callback_func)[0] == '\0'))
+            {
+                incomplete_class = 1;
+                ce = PHP_IC_ENTRY;
+                break;
+            }
+
+            /* Call unserialize callback */
+            ALLOC_INIT_ZVAL(user_func);
+            ZVAL_STRING(user_func, PG(unserialize_callback_func), 1);
+            args[0] = &arg_func_name;
+            ALLOC_INIT_ZVAL(arg_func_name);
+            ZVAL_STRING(arg_func_name, Z_STRVAL_P(val), 1);
+            if (call_user_function_ex(
+                    CG(function_table), NULL, user_func, &retval_ptr,
+                    1, args, 0, NULL TSRMLS_CC) != SUCCESS)
+            {
+                if (MSGPACK_G(error_display))
+                {
+                    zend_error(E_WARNING,
+                               "[msgpack] (msgpack_unserialize_object) "
+                               "defined (%s) but not found", Z_STRVAL_P(val));
+                }
+
+                incomplete_class = 1;
+                ce = PHP_IC_ENTRY;
+                zval_ptr_dtor(&user_func);
+                zval_ptr_dtor(&arg_func_name);
+                break;
+            }
+            if (retval_ptr)
+            {
+                zval_ptr_dtor(&retval_ptr);
+            }
+
+            /* The callback function may have defined the class */
+            if (zend_lookup_class(
+                    Z_STRVAL_P(val), Z_STRLEN_P(val), &pce TSRMLS_CC) == SUCCESS)
+            {
+                ce = *pce;
+            }
+            else
+            {
+                if (MSGPACK_G(error_display))
+                {
+                    zend_error(E_WARNING,
+                               "[msgpack] (msgpack_unserialize_object) "
+                               "Function %s() hasn't defined the class"
+                               "it was called for", Z_STRVAL_P(val));
+                }
+
+                incomplete_class = true;
+                ce = PHP_IC_ENTRY;
+            }
+
+            zval_ptr_dtor(&user_func);
+            zval_ptr_dtor(&arg_func_name);
+        } while(0);
+
+        if (EG(exception))
+        {
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (msgpack_unserialize_object) "
+                           "Exception error");
+            }
+
+            zval_ptr_dtor(&val);
+
+            ZVAL_BOOL(*return_value, 0);
+
+            return MSGPACK_UNPACK_PARSE_ERROR;
+        }
+
+        msgpack_var_push(var_hash, return_value);
+
+        object_init_ex(*return_value, ce);
+
+        /* store incomplete class name */
+        if (incomplete_class)
+        {
+            php_store_class_name(
+                *return_value, Z_STRVAL_P(val), Z_STRLEN_P(val));
+        }
+
+        zval_ptr_dtor(&val);
+
+        /* implementing Serializable */
+        if (custom_object)
+        {
+            zval *rval;
+
+            if (ce->unserialize == NULL)
+            {
+                if (MSGPACK_G(error_display))
+                {
+                    zend_error(E_WARNING,
+                               "[msgpack] (msgpack_unserialize_object) "
+                               "Class %s has no unserializer", ce->name);
+                }
+
+                return MSGPACK_UNPACK_PARSE_ERROR;
+            }
+
+            /* value */
+            ALLOC_INIT_ZVAL(rval);
+
+            if (msgpack_unserialize_zval(&rval, mpsd, var_hash TSRMLS_CC) <= 0)
+            {
+                zval_ptr_dtor(&rval);
+
+                return MSGPACK_UNPACK_PARSE_ERROR;
+            }
+
+            ce->unserialize(
+                return_value, ce,
+                (const unsigned char *)Z_STRVAL_P(rval), Z_STRLEN_P(rval) + 1,
+                (zend_unserialize_data *)var_hash TSRMLS_CC);
+
+            zval_ptr_dtor(&key);
+            zval_ptr_dtor(&rval);
+
+            return ret;
+        }
+
+        ht = HASH_OF(*return_value);
+    }
+
+    /* object property */
+    while (ct-- > 0)
+    {
+        zval *rval;
+
+        /* key */
+        ALLOC_INIT_ZVAL(key);
+
+        if (msgpack_unserialize_zval(&key, mpsd, NULL TSRMLS_CC) <= 0)
+        {
+            zval_ptr_dtor(&key);
+
+            return MSGPACK_UNPACK_PARSE_ERROR;
+        }
+
+        /* value */
+        ALLOC_INIT_ZVAL(rval);
+
+        if (msgpack_unserialize_zval(&rval, mpsd, var_hash TSRMLS_CC) <= 0)
+        {
+            zval_ptr_dtor(&rval);
+            zval_ptr_dtor(&key);
+
+            return MSGPACK_UNPACK_PARSE_ERROR;
+
+        }
+
+        /* update */
+        switch (Z_TYPE_P(key))
+        {
+            case IS_LONG:
+                zend_hash_index_update(
+                    ht, Z_LVAL_P(key), &rval, sizeof(zval *), NULL);
+                break;
+            case IS_STRING:
+                zend_symtable_update(
+                    ht, Z_STRVAL_P(key), Z_STRLEN_P(key) + 1,
+                    &rval, sizeof(zval *), NULL);
+                break;
+            default:
+                zval_ptr_dtor(&key);
+                zval_ptr_dtor(&rval);
+
+                return MSGPACK_UNPACK_PARSE_ERROR;
+        }
+
+        zval_ptr_dtor(&key);
+    }
+
+    /* wakeup */
+    if (object && Z_OBJCE_PP(return_value) != PHP_IC_ENTRY &&
+        zend_hash_exists(&Z_OBJCE_PP(return_value)->function_table,
+                         "__wakeup", sizeof("__wakeup")))
+    {
+        zval f, *h = NULL;
+
+        INIT_PZVAL(&f);
+        ZVAL_STRINGL(&f, "__wakeup", sizeof("__wakeup") - 1, 0);
+        call_user_function_ex(
+            CG(function_table), return_value, &f, &h, 0, 0, 1, NULL TSRMLS_CC);
+        if (h)
+        {
+            zval_ptr_dtor(&h);
+        }
+
+        if (EG(exception))
+        {
+            ret = MSGPACK_UNPACK_PARSE_ERROR;
+        }
+    }
+
+    return ret;
+}
+
+int msgpack_unserialize_zval(
+    zval **return_value, msgpack_unserialize_data *mpsd,
+    php_unserialize_data_t *var_hash TSRMLS_DC)
+{
+    const unsigned char* data = mpsd->data;
+    const unsigned char* const pe = mpsd->data + mpsd->length;
+    const void* n = NULL;
+
+    unsigned int trail = 0;
+    unsigned int cs = CS_HEADER;
+
+    int ret;
+    unsigned int ct;
+
+#define next_cs(p) ((unsigned int)*p & 0x1f)
+
+#define finish_zval_long(val_) \
+    msgpack_var_push(var_hash, return_value); \
+    ZVAL_LONG(*return_value, val_); \
+    goto _finish
+
+#define finish_zval_null() \
+    msgpack_var_push(var_hash, return_value); \
+    ZVAL_NULL(*return_value); \
+    goto _finish
+
+#define finish_zval_bool(val_) \
+    msgpack_var_push(var_hash, return_value); \
+    ZVAL_BOOL(*return_value, val_); \
+    goto _finish
+
+#define finish_zval_double(val_) \
+    msgpack_var_push(var_hash, return_value); \
+    ZVAL_DOUBLE(*return_value, val_); \
+    goto _finish
+
+#define finish_zval_string(val_, len_) \
+    msgpack_var_push(var_hash, return_value); \
+    if (len_ == 0) { ZVAL_STRINGL(*return_value, "", 0, 1); } \
+    else { ZVAL_STRINGL(*return_value, val_, len_, 1); } \
+    goto _finish
+
+#define finish_zval_array(count_) \
+    ct = count_; \
+    cs = CS_HEADER; \
+    ++(mpsd->data); \
+    mpsd->length = (pe - mpsd->data); \
+    if (msgpack_unserialize_array(return_value, mpsd, ct, var_hash TSRMLS_CC) <= 0) { goto _failed; } \
+    goto _finish_end
+
+#define finish_zval_object(count_) \
+    ct = count_; \
+    cs = CS_HEADER; \
+    ++(mpsd->data); \
+    mpsd->length = (pe - mpsd->data); \
+    if (msgpack_unserialize_object(return_value, mpsd, ct, var_hash TSRMLS_CC) <= 0) { goto _failed; } \
+    goto _finish_end
+
+#define again_fixed_trail(_cs, trail_len) \
+    trail = trail_len; \
+    cs = _cs; \
+    goto _fixed_trail_again
+
+#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \
+    trail = trail_len; \
+    if(trail == 0) { goto ifzero; } \
+    cs = _cs; \
+    goto _fixed_trail_again
+
+    if (mpsd->length <= 0)
+    {
+        goto _failed;
+    }
+
+    if (mpsd->data == pe)
+    {
+        goto _out;
+    }
+    do
+    {
+        switch (cs)
+        {
+            case CS_HEADER:
+                switch (*(mpsd->data))
+                {
+                    case 0x00 ... 0x7f: /* Positive Fixnum */
+                        finish_zval_long(*(uint8_t*)mpsd->data);
+                    case 0xe0 ... 0xff: /* Negative Fixnum */
+                        finish_zval_long(*(int8_t*)mpsd->data);
+                    case 0xc0 ... 0xdf: /* Variable */
+                        switch (*(mpsd->data))
+                        {
+                            case 0xc0:  /* nil */
+                                finish_zval_null();
+                            case 0xc2:  /* false */
+                                finish_zval_bool(0);
+                            case 0xc3:  /* true */
+                                finish_zval_bool(1);
+                            case 0xca:  /* float */
+                            case 0xcb:  /* double */
+                            case 0xcc:  /* unsigned int  8 */
+                            case 0xcd:  /* unsigned int 16 */
+                            case 0xce:  /* unsigned int 32 */
+                            case 0xcf:  /* unsigned int 64 */
+                            case 0xd0:  /* signed int  8 */
+                            case 0xd1:  /* signed int 16 */
+                            case 0xd2:  /* signed int 32 */
+                            case 0xd3:  /* signed int 64 */
+                                again_fixed_trail(
+                                    next_cs(mpsd->data),
+                                    1 << (((unsigned int)*(mpsd->data)) & 0x03));
+                            case 0xda:  /* raw 16 */
+                            case 0xdb:  /* raw 32 */
+                            case 0xdc:  /* array 16 */
+                            case 0xdd:  /* array 32 */
+                            case 0xde:  /* map 16 */
+                            case 0xdf:  /* map 32 */
+                                again_fixed_trail(
+                                    next_cs(mpsd->data),
+                                    2 << (((unsigned int)*(mpsd->data)) & 0x01));
+                            default:
+                                goto _failed;
+                        }
+                    case 0xa0 ... 0xbf: /* FixRaw */
+                        again_fixed_trail_if_zero(
+                            ACS_RAW_VALUE, (unsigned int)*(mpsd->data) & 0x1f,
+                            _raw_zero);
+                    case 0x90 ... 0x9f: /* FixArray */
+                        finish_zval_array(((unsigned int)*(mpsd->data)) & 0x0f);
+                    case 0x80 ... 0x8f: /* FixMap */
+                        finish_zval_object(((unsigned int)*(mpsd->data)) & 0x0f);
+                    default:
+                        goto _failed;
+                }
+_fixed_trail_again:
+                ++(mpsd->data);
+            default:
+                if ((size_t)(pe - (mpsd->data)) < trail)
+                {
+                    goto _out;
+                }
+                n = (mpsd->data);
+                mpsd->data += trail - 1;
+                switch (cs)
+                {
+                    case CS_FLOAT:
+                    {
+                        union { uint32_t i; float f; } mem;
+                        mem.i = _msgpack_load32(uint32_t, n);
+                        finish_zval_double(mem.f);
+                    }
+                    case CS_DOUBLE:
+                    {
+                        union { uint64_t i; double f; } mem;
+                        mem.i = _msgpack_load64(uint64_t, n);
+                        finish_zval_double(mem.f);
+                    }
+                    case CS_UINT_8:
+                        finish_zval_long(*(uint8_t*)n);
+                    case CS_UINT_16:
+                        finish_zval_long(_msgpack_load16(uint16_t, n));
+                    case CS_UINT_32:
+                        finish_zval_long(_msgpack_load32(uint32_t, n));
+                    case CS_UINT_64:
+                        finish_zval_long(_msgpack_load64(uint64_t, n));
+                    case CS_INT_8:
+                        finish_zval_long(*(int8_t*)n);
+                    case CS_INT_16:
+                        finish_zval_long(_msgpack_load16(int16_t, n));
+                    case CS_INT_32:
+                        finish_zval_long(_msgpack_load32(int32_t, n));
+                    case CS_INT_64:
+                        finish_zval_long(_msgpack_load64(int64_t, n));
+                    case CS_RAW_16:
+                        again_fixed_trail_if_zero(
+                            ACS_RAW_VALUE, _msgpack_load16(uint16_t, n),
+                            _raw_zero);
+                    case CS_RAW_32:
+                        again_fixed_trail_if_zero(
+                            ACS_RAW_VALUE, _msgpack_load32(uint32_t, n),
+                            _raw_zero);
+                    case ACS_RAW_VALUE:
+_raw_zero:
+                        finish_zval_string(n, trail);
+                    case CS_ARRAY_16:
+                        finish_zval_array(_msgpack_load16(uint16_t, n));
+                    case CS_ARRAY_32:
+                        finish_zval_array(_msgpack_load32(uint32_t, n));
+                    /* FIXME security guard */
+                    case CS_MAP_16:
+                        finish_zval_object(_msgpack_load16(uint16_t, n));
+                    case CS_MAP_32:
+                        finish_zval_object(_msgpack_load32(uint32_t, n));
+                    /* FIXME security guard */
+                    default:
+                        goto _failed;
+                }
+        }
+        cs = CS_HEADER;
+        ++(mpsd->data);
+    }
+    while (mpsd->data != pe);
+    goto _out;
+
+_finish:
+    ++(mpsd->data);
+_finish_end:
+    ret = MSGPACK_UNPACK_EXTRA_BYTES;
+    goto _end;
+
+_failed:
+    ret = MSGPACK_UNPACK_PARSE_ERROR;
+    goto _end;
+
+_out:
+    ret = MSGPACK_UNPACK_CONTINUE;
+    goto _end;
+
+_end:
+    mpsd->offset = mpsd->data - data;
+    mpsd->length = pe - mpsd->data;
+
+    if (ret == MSGPACK_UNPACK_EXTRA_BYTES && mpsd->length == 0)
+    {
+        ret = MSGPACK_UNPACK_SUCCESS;
+    }
+
+    return ret;
+}
+
+#undef finish_zval_long
+#undef finish_zval_null
+#undef finish_zval_bool
+#undef finish_zval_double
+#undef finish_zval_string
+#undef finish_zval_array
+#undef finish_zval_object
+#undef again_fixed_trail
+#undef again_fixed_trail_if_zero
+
+#undef next_cs
diff --git a/php/msgpack_unpack.h b/php/msgpack_unpack.h
new file mode 100644
index 0000000..6c85d73
--- /dev/null
+++ b/php/msgpack_unpack.h
@@ -0,0 +1,26 @@
+
+#ifndef MSGPACL_UNPACK_H
+#define MSGPACL_UNPACK_H
+
+#include "ext/standard/php_var.h"
+
+typedef enum
+{
+    MSGPACK_UNPACK_SUCCESS     =  2,
+    MSGPACK_UNPACK_EXTRA_BYTES =  1,
+    MSGPACK_UNPACK_CONTINUE    =  0,
+    MSGPACK_UNPACK_PARSE_ERROR = -1,
+} msgpack_unpack_return;
+
+typedef struct
+{
+    unsigned char *data;
+    size_t length;
+    size_t offset;
+} msgpack_unserialize_data;
+
+int msgpack_unserialize_zval(
+    zval **return_value, msgpack_unserialize_data *mpsd,
+    php_unserialize_data_t *var_hash TSRMLS_DC);
+
+#endif
diff --git a/php/package.xml b/php/package.xml
new file mode 100644
index 0000000..4869770
--- /dev/null
+++ b/php/package.xml
@@ -0,0 +1,98 @@
+
+
+  msgpack
+  pecl.php.net
+  PHP extension for interfacing with MessagePack
+  This extension provide API for communicating with MessagePack serialization.
+  
+    Advect
+    advect
+    advect@gmail.com
+    yes
+  
+  2010-07-17
+  
+  
+    0.2.1
+    0.2.1
+  
+  
+    beta
+    beta
+  
+  New BSD
+  Initial release.
+  
+    
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+    
+  
+  
+    
+      
+        5.2.0
+      
+      
+        1.4.3
+      
+    
+  
+  msgpack
+  
+
diff --git a/php/php-msgpack.spec b/php/php-msgpack.spec
new file mode 100644
index 0000000..5726c43
--- /dev/null
+++ b/php/php-msgpack.spec
@@ -0,0 +1,59 @@
+%define php_apiver %((echo 0; php -i 2>/dev/null | sed -n 's/^PHP API => //p') | tail -1)
+%{!?php_extdir: %{expand: %%define php_extdir %(php-config --extension-dir)}}
+
+Summary: PHP extension for interfacing with MessagePack
+Name: php-msgpack
+Version: 0.2.1
+Release: 1%{?dist}
+Source: php-msgpack-%{version}.tar.gz
+License: New BSD License
+Group: Development/Libraries
+Packager: advect 
+Provides: php-pecl-msgpack
+BuildRoot: %{_tmppath}/%{name}-%{version}-root
+BuildRequires: php-devel, msgpack-devel
+Requires: msgpack
+%if 0%{?php_zend_api}
+Requires: php(zend-abi) = %{php_zend_api}
+Requires: php(api) = %{php_core_api}
+%else
+Requires: php-api = %{php_apiver}
+%endif
+
+%description
+PHP extension for interfacing with MessagePack.
+
+%prep
+%setup -q -n php-msgpack
+
+%build
+phpize
+%configure
+%{__make}
+
+%install
+%makeinstall INSTALL_ROOT=%{buildroot}
+
+%{__install} -d %{buildroot}%{_sysconfdir}/php.d
+%{__cat} > %{buildroot}%{_sysconfdir}/php.d/msgpack.ini <= 4
+#   define PHP_MSGPACK_API __attribute__ ((visibility("default")))
 #else
-#define PHP_MSGPACK_API
+#   define PHP_MSGPACK_API
 #endif
 
 #ifdef ZTS
 #include "TSRM.h"
 #endif
 
-PHP_MINIT_FUNCTION(msgpack);
-PHP_MSHUTDOWN_FUNCTION(msgpack);
-PHP_RINIT_FUNCTION(msgpack);
-PHP_RSHUTDOWN_FUNCTION(msgpack);
-PHP_MINFO_FUNCTION(msgpack);
-
-PHP_FUNCTION(msgpack_pack);
-PHP_FUNCTION(msgpack_unpack);
-PHP_FUNCTION(msgpack_unpack_limit);
-
-PHP_METHOD(msgpack, initialize);
-PHP_METHOD(msgpack, execute);
-PHP_METHOD(msgpack, execute_limit);
-PHP_METHOD(msgpack, finished);
-PHP_METHOD(msgpack, data);
-
-static zend_class_entry *msgpack_ce;
-
-/* 
-  	Declare any global variables you may need between the BEGIN
-	and END macros here:     
-
 ZEND_BEGIN_MODULE_GLOBALS(msgpack)
-	long  global_value;
-	char *global_string;
+    zend_bool error_display;
 ZEND_END_MODULE_GLOBALS(msgpack)
-*/
 
-/* In every utility function you add that needs to use variables 
-   in php_msgpack_globals, call TSRMLS_FETCH(); after declaring other 
-   variables used by that function, or better yet, pass in TSRMLS_CC
-   after the last function argument and declare your utility function
-   with TSRMLS_DC after the last declared argument.  Always refer to
-   the globals in your function as MSGPACK_G(variable).  You are 
-   encouraged to rename these macros something shorter, see
-   examples in any other php module directory.
-*/
+ZEND_DECLARE_MODULE_GLOBALS(msgpack)
 
+#ifdef ZTS
+#define MSGPACK_G(v) TSRMG(msgpack_globals_id, zend_msgpack_globals *, v)
+#else
+#define MSGPACK_G(v) (msgpack_globals.v)
+#endif
 
-#endif	/* PHP_MSGPACK_H */
+PHP_MSGPACK_API void php_msgpack_serialize(
+    smart_str *buf, zval *val TSRMLS_DC);
+PHP_MSGPACK_API void php_msgpack_unserialize(
+    zval *return_value, char *str, size_t str_len TSRMLS_DC);
 
-
-/*
- * Local variables:
- * tab-width: 4
- * c-basic-offset: 4
- * End:
- * vim600: noet sw=4 ts=4 fdm=marker
- * vim<600: noet sw=4 ts=4
- */
+#endif  /* PHP_MSGPACK_H */
diff --git a/php/test_normal.php b/php/test_normal.php
old mode 100755
new mode 100644
diff --git a/php/test_streaming.php b/php/test_streaming.php
old mode 100755
new mode 100644
index 8f89f4c..f2f470e
--- a/php/test_streaming.php
+++ b/php/test_streaming.php
@@ -4,8 +4,7 @@
  $msgs = array(pack("C*", 0x93, 0x01, 0x02, 0x03, 0x92), pack("C*", 0x03, 0x09, 0x04));
 
  // streaming deserialize
- $unpacker = new MessagePack();
- $unpacker->initialize();
+ $unpacker = new MessagePackUnpacker();
  $buffer = "";
  $nread = 0;
 
@@ -13,13 +12,11 @@
     $buffer = $buffer . $msg;
 
     while(true){
-        $nread = $unpacker->execute($buffer, $nread);
-
-        if($unpacker->finished()){
+        if($unpacker->execute($buffer, $nread)){
             $msg = $unpacker->data();
             var_dump($msg);
 
-            $unpacker->initialize();
+            $unpacker->reset();
             $buffer = substr($buffer, $nread);
             $nread = 0;
 
diff --git a/php/tests/001.phpt b/php/tests/001.phpt
new file mode 100644
index 0000000..840ae46
--- /dev/null
+++ b/php/tests/001.phpt
@@ -0,0 +1,10 @@
+--TEST--
+Check for msgpack presence
+--SKIPIF--
+
+--FILE--
+
+--EXPECT--
+msgpack extension is available
diff --git a/php/tests/002.phpt b/php/tests/002.phpt
new file mode 100644
index 0000000..75b9488
--- /dev/null
+++ b/php/tests/002.phpt
@@ -0,0 +1,26 @@
+--TEST--
+Check for null serialisation
+--SKIPIF--
+--FILE--
+
+--EXPECT--
+null
+c0
+NULL
+OK
diff --git a/php/tests/003.phpt b/php/tests/003.phpt
new file mode 100644
index 0000000..8c85f90
--- /dev/null
+++ b/php/tests/003.phpt
@@ -0,0 +1,31 @@
+--TEST--
+Check for bool serialisation
+--SKIPIF--
+--FILE--
+
+--EXPECT--
+bool true
+c3
+bool(true)
+OK
+bool false
+c2
+bool(false)
+OK
diff --git a/php/tests/004.phpt b/php/tests/004.phpt
new file mode 100644
index 0000000..8c352ec
--- /dev/null
+++ b/php/tests/004.phpt
@@ -0,0 +1,56 @@
+--TEST--
+Check for integer serialisation
+--SKIPIF--
+--FILE--
+
+--EXPECT--
+zero: 0
+00
+int(0)
+OK
+small: 1
+01
+int(1)
+OK
+small: -1
+ff
+int(-1)
+OK
+medium: 1000
+cd03e8
+int(1000)
+OK
+medium: -1000
+d1fc18
+int(-1000)
+OK
+large: 100000
+ce000186a0
+int(100000)
+OK
+large: -100000
+d2fffe7960
+int(-100000)
+OK
diff --git a/php/tests/005.phpt b/php/tests/005.phpt
new file mode 100644
index 0000000..2156ffa
--- /dev/null
+++ b/php/tests/005.phpt
@@ -0,0 +1,26 @@
+--TEST--
+Check for double serialisation
+--SKIPIF--
+--FILE--
+
+--EXPECT--
+double: 123.456
+cb405edd2f1a9fbe77
+float(123.456)
+OK
diff --git a/php/tests/006.phpt b/php/tests/006.phpt
new file mode 100644
index 0000000..6832355
--- /dev/null
+++ b/php/tests/006.phpt
@@ -0,0 +1,31 @@
+--TEST--
+Check for simple string serialization
+--SKIPIF--
+--FILE--
+
+--EXPECT--
+empty: ""
+a0
+string(0) ""
+OK
+string: "foobar"
+a6666f6f626172
+string(6) "foobar"
+OK
diff --git a/php/tests/007.phpt b/php/tests/007.phpt
new file mode 100644
index 0000000..db41185
--- /dev/null
+++ b/php/tests/007.phpt
@@ -0,0 +1,72 @@
+--TEST--
+Check for simple array serialization
+--SKIPIF--
+--FILE--
+
+--EXPECT--
+empty array:
+90
+array(0) {
+}
+OK
+array(1, 2, 3)
+83000101020203
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(array(1, 2, 3), arr...
+83008300010102020301830004010502060283000701080209
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
diff --git a/php/tests/008.phpt b/php/tests/008.phpt
new file mode 100644
index 0000000..661c1cd
--- /dev/null
+++ b/php/tests/008.phpt
@@ -0,0 +1,61 @@
+--TEST--
+Check for array+string serialization
+--SKIPIF--
+--FILE--
+ 1, "two" => 2))', array("one" => 1, "two" => 2));
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"));
+test('array("" => "empty")', array("" => "empty"));
+?>
+--EXPECT--
+array("foo", "foo", "foo")
+8300a3666f6f01a3666f6f02a3666f6f
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array("one" => 1, "two" => 2))
+82a36f6e6501a374776f02
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array("kek" => "lol", "lol" => "kek")
+82a36b656ba36c6f6ca36c6f6ca36b656b
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array("" => "empty")
+81a0a5656d707479
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
diff --git a/php/tests/009.phpt b/php/tests/009.phpt
new file mode 100644
index 0000000..6baad5e
--- /dev/null
+++ b/php/tests/009.phpt
@@ -0,0 +1,115 @@
+--TEST--
+Check for reference serialisation
+--SKIPIF--
+--FILE--
+
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(&$a, &$a)
+82008100a3666f6f0182c001c002
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+cyclic
+81008100810082c001c002
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      &array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      &array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
diff --git a/php/tests/010.phpt b/php/tests/010.phpt
new file mode 100644
index 0000000..9eb77a6
--- /dev/null
+++ b/php/tests/010.phpt
@@ -0,0 +1,49 @@
+--TEST--
+Array test
+--SKIPIF--
+--FILE--
+ array(
+        'b' => 'c',
+        'd' => 'e'
+    ),
+    'f' => array(
+        'g' => 'h'
+    )
+);
+
+test('array', $a, false);
+?>
+--EXPECT--
+array
+82a16182a162a163a164a165a16681a167a168
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
diff --git a/php/tests/012.phpt b/php/tests/012.phpt
new file mode 100644
index 0000000..8121c96
--- /dev/null
+++ b/php/tests/012.phpt
@@ -0,0 +1,48 @@
+--TEST--
+Object test
+--SKIPIF--
+--FILE--
+a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+$o = new Obj(1, 2, 3);
+
+
+test('object', $o, false);
+?>
+--EXPECTF--
+object
+84c0a34f626aa16101a4002a006202a6004f626a006303
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
diff --git a/php/tests/013.phpt b/php/tests/013.phpt
new file mode 100644
index 0000000..73f7d33
--- /dev/null
+++ b/php/tests/013.phpt
@@ -0,0 +1,54 @@
+--TEST--
+Object-Array test
+--SKIPIF--
+--FILE--
+a = $a;
+        $this->b = $b;
+    }
+}
+
+$o = array(new Obj(1, 2), new Obj(3, 4));
+
+
+test('object', $o, false);
+?>
+--EXPECTF--
+object
+820083c0a34f626aa16101a162020183c0a34f626aa16103a16204
+array(2) {
+  [0]=>
+  object(Obj)#%d (2) {
+    ["a"]=>
+    int(1)
+    ["b"]=>
+    int(2)
+  }
+  [1]=>
+  object(Obj)#%d (2) {
+    ["a"]=>
+    int(3)
+    ["b"]=>
+    int(4)
+  }
+}
+OK
diff --git a/php/tests/014.phpt b/php/tests/014.phpt
new file mode 100644
index 0000000..c663579
--- /dev/null
+++ b/php/tests/014.phpt
@@ -0,0 +1,54 @@
+--TEST--
+Object-Reference test
+--SKIPIF--
+--FILE--
+a = $a;
+        $this->b = $b;
+    }
+}
+
+$o = new Obj(1, 2);
+$a = array(&$o, &$o);
+
+test('object', $a, false);
+?>
+--EXPECTF--
+object
+820083c0a34f626aa16101a162020182c001c002
+array(2) {
+  [0]=>
+  &object(Obj)#%d (2) {
+    ["a"]=>
+    int(1)
+    ["b"]=>
+    int(2)
+  }
+  [1]=>
+  &object(Obj)#%d (2) {
+    ["a"]=>
+    int(1)
+    ["b"]=>
+    int(2)
+  }
+}
+OK
diff --git a/php/tests/015.phpt b/php/tests/015.phpt
new file mode 100644
index 0000000..828800a
--- /dev/null
+++ b/php/tests/015.phpt
@@ -0,0 +1,58 @@
+--TEST--
+Check for serialization handler
+--SKIPIF--
+--FILE--
+
+--EXPECT--
+2
+81a3666f6f02
+array(1) {
+  ["foo"]=>
+  int(2)
+}
diff --git a/php/tests/015b.phpt b/php/tests/015b.phpt
new file mode 100644
index 0000000..829584b
--- /dev/null
+++ b/php/tests/015b.phpt
@@ -0,0 +1,58 @@
+--TEST--
+Check for serialization handler, ini-directive
+--SKIPIF--
+--INI--
+session.serialize_handler=msgpack
+--FILE--
+
+--EXPECT--
+2
+81a3666f6f02
+array(1) {
+  ["foo"]=>
+  int(2)
+}
diff --git a/php/tests/016.phpt b/php/tests/016.phpt
new file mode 100644
index 0000000..f8f4779
--- /dev/null
+++ b/php/tests/016.phpt
@@ -0,0 +1,60 @@
+--TEST--
+Object test, __sleep
+--SKIPIF--
+--FILE--
+a = $a;
+        $this->b = $b;
+        $this->c = $c;
+        $this->d = $d;
+    }
+
+    function __sleep() {
+        return array('a', 'b', 'c');
+    }
+
+#   function __wakeup() {
+#       $this->d = $this->a + $this->b + $this->c;
+#   }
+}
+
+$o = new Obj(1, 2, 3, 4);
+
+
+test('object', $o, true);
+?>
+--EXPECTF--
+object
+84c0a34f626aa16101a4002a006202a6004f626a006303
+object(Obj)#%d (4) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+  ["d"]=>
+  NULL
+}
+OK
diff --git a/php/tests/017.phpt b/php/tests/017.phpt
new file mode 100644
index 0000000..6fbbd90
--- /dev/null
+++ b/php/tests/017.phpt
@@ -0,0 +1,48 @@
+--TEST--
+Object test, __wakeup
+--SKIPIF--
+--FILE--
+b == 3 ? 'OK' : 'ERROR', PHP_EOL;
+}
+
+class Obj {
+    var $a;
+    var $b;
+
+    function __construct($a, $b) {
+        $this->a = $a;
+        $this->b = $b;
+    }
+
+    function __wakeup() {
+        $this->b = $this->a * 3;
+    }
+}
+
+$o = new Obj(1, 2);
+
+
+test('object', $o, false);
+?>
+--EXPECTF--
+object
+83c0a34f626aa16101a16202
+object(Obj)#%d (2) {
+  ["a"]=>
+  int(1)
+  ["b"]=>
+  int(3)
+}
+OK
diff --git a/php/tests/018.phpt b/php/tests/018.phpt
new file mode 100644
index 0000000..84ccafa
--- /dev/null
+++ b/php/tests/018.phpt
@@ -0,0 +1,84 @@
+--TEST--
+Object test, __sleep error cases
+--SKIPIF--
+--FILE--
+a = $a;
+        $this->b = $b;
+    }
+
+    function __sleep() {
+        return array('c');
+    }
+
+#   function __wakeup() {
+#       $this->b = $this->a * 3;
+#   }
+}
+
+class Opj {
+    var $a;
+    var $b;
+
+    function __construct($a, $b) {
+        $this->a = $a;
+        $this->b = $b;
+    }
+
+    function __sleep() {
+        return array(1);
+    }
+
+#   function __wakeup() {
+#
+#   }
+}
+
+$o = new Obj(1, 2);
+$p = new Opj(1, 2);
+
+test('nonexisting', $o, true);
+test('wrong', $p, true);
+?>
+--EXPECTF--
+nonexisting
+82c0a34f626aa163c0
+object(Obj)#%d (3) {
+  ["a"]=>
+  NULL
+  ["b"]=>
+  NULL
+  ["c"]=>
+  NULL
+}
+OK
+wrong
+82c0a34f706ac0
+object(Opj)#%d (2) {
+  ["a"]=>
+  NULL
+  ["b"]=>
+  NULL
+}
+OK
diff --git a/php/tests/019.phpt b/php/tests/019.phpt
new file mode 100644
index 0000000..46cccb3
--- /dev/null
+++ b/php/tests/019.phpt
@@ -0,0 +1,43 @@
+--TEST--
+Object test, __autoload
+--SKIPIF--
+--FILE--
+b == 2 ? 'OK' : 'ERROR', PHP_EOL;
+}
+
+function __autoload($classname) {
+    class Obj {
+        var $a;
+        var $b;
+
+        function __construct($a, $b) {
+            $this->a = $a;
+            $this->b = $b;
+        }
+    }
+}
+
+test('autoload', '83c0a34f626aa16101a16202', false);
+?>
+--EXPECTF--
+autoload
+83c0a34f626aa16101a16202
+object(Obj)#%d (2) {
+  ["a"]=>
+  int(1)
+  ["b"]=>
+  int(2)
+}
+OK
diff --git a/php/tests/020.phpt b/php/tests/020.phpt
new file mode 100644
index 0000000..4ed30e8
--- /dev/null
+++ b/php/tests/020.phpt
@@ -0,0 +1,31 @@
+--TEST--
+Object test, incomplete class
+--SKIPIF--
+--FILE--
+
+--EXPECTF--
+incom
+83c0a34f626aa16101a16202
+object(__PHP_Incomplete_Class)#%d (3) {
+  ["__PHP_Incomplete_Class_Name"]=>
+  string(3) "Obj"
+  ["a"]=>
+  int(1)
+  ["b"]=>
+  int(2)
+}
diff --git a/php/tests/021.phpt b/php/tests/021.phpt
new file mode 100644
index 0000000..7960a1f
--- /dev/null
+++ b/php/tests/021.phpt
@@ -0,0 +1,52 @@
+--TEST--
+Object Serializable interface
+--SKIPIF--
+--FILE--
+a = $a;
+        $this->b = $b;
+    }
+
+    public function serialize() {
+        return pack('NN', $this->a, $this->b);
+    }
+
+    public function unserialize($serialized) {
+        $tmp = unpack('N*', $serialized);
+        $this->__construct($tmp[1], $tmp[2]);
+    }
+}
+
+$o = new Obj(1, 2);
+
+test('object', $o, false);
+?>
+--EXPECTF--
+object
+82c003a34f626aa80000000100000002
+object(Obj)#%d (2) {
+  ["a"]=>
+  int(1)
+  ["b"]=>
+  int(2)
+}
+OK
diff --git a/php/tests/022.phpt b/php/tests/022.phpt
new file mode 100644
index 0000000..7eab4b8
--- /dev/null
+++ b/php/tests/022.phpt
@@ -0,0 +1,45 @@
+--TEST--
+Object test, unserialize_callback_func
+--SKIPIF--
+--INI--
+unserialize_callback_func=autoload
+--FILE--
+b == 2 ? 'OK' : 'ERROR', PHP_EOL;
+}
+
+function autoload($classname) {
+    class Obj {
+        var $a;
+        var $b;
+
+        function __construct($a, $b) {
+            $this->a = $a;
+            $this->b = $b;
+        }
+    }
+}
+
+test('autoload', '83c0a34f626aa16101a16202', false);
+?>
+--EXPECTF--
+autoload
+83c0a34f626aa16101a16202
+object(Obj)#%d (2) {
+  ["a"]=>
+  int(1)
+  ["b"]=>
+  int(2)
+}
+OK
diff --git a/php/tests/023.phpt b/php/tests/023.phpt
new file mode 100644
index 0000000..46d9ade
--- /dev/null
+++ b/php/tests/023.phpt
@@ -0,0 +1,54 @@
+--TEST--
+Resource
+--SKIPIF--
+
+--FILE--
+open('db.txt');
+}
+
+test('resource', $res, false);
+
+switch ($test) {
+    case 'curl':
+        curl_close($res);
+        break;
+    case 'sqlite':
+        if (isset($sqlite)) {
+            $sqlite->close();
+        }
+        @unlink('db.txt');
+        break;
+}
+?>
+--EXPECT--
+resource
+c0
+NULL
+OK
diff --git a/php/tests/024.phpt b/php/tests/024.phpt
new file mode 100644
index 0000000..30fab75
--- /dev/null
+++ b/php/tests/024.phpt
@@ -0,0 +1,166 @@
+--TEST--
+Recursive objects
+--SKIPIF--
+--FILE--
+a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+class Obj2 {
+    public $aa;
+    protected $bb;
+    private $cc;
+    private $obj;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+
+        $this->obj = new Obj($a, $b, $c);
+    }
+}
+
+class Obj3 {
+    private $objs;
+
+    function __construct($a, $b, $c) {
+        $this->objs = array();
+
+        for ($i = $a; $i < $c; $i += $b) {
+            $this->objs[] = new Obj($a, $i, $c);
+        }
+    }
+}
+
+class Obj4 {
+    private $a;
+    private $obj;
+
+    function __construct($a) {
+        $this->a = $a;
+    }
+
+    public function set($obj) {
+        $this->obj = $obj;
+    }
+}
+
+$o2 = new Obj2(1, 2, 3);
+test('objectrec', $o2, false);
+
+$o3 = new Obj3(0, 1, 4);
+test('objectrecarr', $o3, false);
+
+$o4 = new Obj4(100);
+$o4->set($o4);
+test('objectselfrec', $o4, true);
+?>
+--EXPECTF--
+objectrec
+88c0a44f626a32a26161c0a5002a006262c0a8004f626a32006363c0a9004f626a32006f626a84c0a34f626aa16101a4002a006202a6004f626a006303a16101a16202a16303
+object(Obj2)#%d (7) {
+  ["aa"]=>
+  NULL
+  [%r"?bb"?:protected"?%r]=>
+  NULL
+  [%r"?cc"?:("Obj2":)?private"?%r]=>
+  NULL
+  [%r"?obj"?:("Obj2":)?private"?%r]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  ["a"]=>
+  int(1)
+  ["b"]=>
+  int(2)
+  ["c"]=>
+  int(3)
+}
+OK
+objectrecarr
+82c0a44f626a33aa004f626a33006f626a73840084c0a34f626aa16100a4002a006200a6004f626a0063040184c0a34f626aa16100a4002a006201a6004f626a0063040284c0a34f626aa16100a4002a006202a6004f626a0063040384c0a34f626aa16100a4002a006203a6004f626a006304
+object(Obj3)#%d (1) {
+  [%r"?objs"?:("Obj3":)?private"?%r]=>
+  array(4) {
+    [0]=>
+    object(Obj)#%d (3) {
+      ["a"]=>
+      int(0)
+      [%r"?b"?:protected"?%r]=>
+      int(0)
+      [%r"?c"?:("Obj":)?private"?%r]=>
+      int(4)
+    }
+    [1]=>
+    object(Obj)#%d (3) {
+      ["a"]=>
+      int(0)
+      [%r"?b"?:protected"?%r]=>
+      int(1)
+      [%r"?c"?:("Obj":)?private"?%r]=>
+      int(4)
+    }
+    [2]=>
+    object(Obj)#%d (3) {
+      ["a"]=>
+      int(0)
+      [%r"?b"?:protected"?%r]=>
+      int(2)
+      [%r"?c"?:("Obj":)?private"?%r]=>
+      int(4)
+    }
+    [3]=>
+    object(Obj)#%d (3) {
+      ["a"]=>
+      int(0)
+      [%r"?b"?:protected"?%r]=>
+      int(3)
+      [%r"?c"?:("Obj":)?private"?%r]=>
+      int(4)
+    }
+  }
+}
+OK
+objectselfrec
+83c0a44f626a34a7004f626a34006164a9004f626a34006f626a82c002c001
+object(Obj4)#%d (2) {
+  [%r"?a"?:("Obj4":)?private"?%r]=>
+  int(100)
+  [%r"?obj"?:("Obj4":)?private"?%r]=>
+  object(Obj4)#%d (2) {
+    [%r"?a"?:("Obj4":)?private"?%r]=>
+    int(100)
+    [%r"?obj"?:("Obj4":)?private"?%r]=>
+    *RECURSION*
+  }
+}
+OK
diff --git a/php/tests/025.phpt b/php/tests/025.phpt
new file mode 100644
index 0000000..234539d
--- /dev/null
+++ b/php/tests/025.phpt
@@ -0,0 +1,119 @@
+--TEST--
+Object test, array of objects with __sleep
+--SKIPIF--
+--FILE--
+a = $a;
+        $this->b = $b;
+        $this->c = $c;
+        $this->d = $d;
+    }
+
+    function __sleep() {
+        return array('a', 'b', 'c');
+    }
+
+#   function __wakeup() {
+#       $this->d = $this->a + $this->b + $this->c;
+#   }
+}
+
+$array = array(
+    new Obj("aa", "bb", "cc", "dd"),
+    new Obj("ee", "ff", "gg", "hh"),
+    new Obj(1, 2, 3, 4),
+);
+
+
+test('array', $array, true);
+?>
+--EXPECTF--
+array(3) {
+  [0]=>
+  object(Obj)#1 (4) {
+    ["a"]=>
+    string(2) "aa"
+    [%r"?b"?:protected"?%r]=>
+    string(2) "bb"
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    string(2) "cc"
+    ["d"]=>
+    string(2) "dd"
+  }
+  [1]=>
+  object(Obj)#2 (4) {
+    ["a"]=>
+    string(2) "ee"
+    [%r"?b"?:protected"?%r]=>
+    string(2) "ff"
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    string(2) "gg"
+    ["d"]=>
+    string(2) "hh"
+  }
+  [2]=>
+  object(Obj)#3 (4) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+    ["d"]=>
+    int(4)
+  }
+}
+array(3) {
+  [0]=>
+  object(Obj)#4 (4) {
+    ["a"]=>
+    string(2) "aa"
+    [%r"?b"?:protected"?%r]=>
+    string(2) "bb"
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    string(2) "cc"
+    ["d"]=>
+    NULL
+  }
+  [1]=>
+  object(Obj)#5 (4) {
+    ["a"]=>
+    string(2) "ee"
+    [%r"?b"?:protected"?%r]=>
+    string(2) "ff"
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    string(2) "gg"
+    ["d"]=>
+    NULL
+  }
+  [2]=>
+  object(Obj)#6 (4) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+    ["d"]=>
+    NULL
+  }
+}
diff --git a/php/tests/026.phpt b/php/tests/026.phpt
new file mode 100644
index 0000000..2d3bb73
--- /dev/null
+++ b/php/tests/026.phpt
@@ -0,0 +1,143 @@
+--TEST--
+Cyclic array test
+--INI--
+--SKIPIF--
+--FILE--
+ array(
+        'b' => 'c',
+        'd' => 'e'
+    ),
+);
+
+$a['f'] = &$a;
+
+test('array', $a, true);
+
+$a = array("foo" => &$b);
+$b = array(1, 2, $a);
+var_dump($a);
+var_dump($k = msgpack_unserialize(msgpack_serialize($a)));
+
+$k["foo"][1] = "b";
+var_dump($k);
+?>
+--EXPECT--
+array
+82a16182a162a163a164a165a16682a16182a162a163a164a165a16682c001c005
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  &array(2) {
+    ["a"]=>
+    array(2) {
+      ["b"]=>
+      string(1) "c"
+      ["d"]=>
+      string(1) "e"
+    }
+    ["f"]=>
+    &array(2) {
+      ["a"]=>
+      array(2) {
+        ["b"]=>
+        string(1) "c"
+        ["d"]=>
+        string(1) "e"
+      }
+      ["f"]=>
+      *RECURSION*
+    }
+  }
+}
+OK
+array(1) {
+  ["foo"]=>
+  &array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    array(1) {
+      ["foo"]=>
+      &array(3) {
+        [0]=>
+        int(1)
+        [1]=>
+        int(2)
+        [2]=>
+        *RECURSION*
+      }
+    }
+  }
+}
+array(1) {
+  ["foo"]=>
+  &array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    array(1) {
+      ["foo"]=>
+      &array(3) {
+        [0]=>
+        int(1)
+        [1]=>
+        int(2)
+        [2]=>
+        array(1) {
+          ["foo"]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
+array(1) {
+  ["foo"]=>
+  &array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    string(1) "b"
+    [2]=>
+    array(1) {
+      ["foo"]=>
+      &array(3) {
+        [0]=>
+        int(1)
+        [1]=>
+        string(1) "b"
+        [2]=>
+        array(1) {
+          ["foo"]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
diff --git a/php/tests/027.phpt b/php/tests/027.phpt
new file mode 100644
index 0000000..ecb8784
--- /dev/null
+++ b/php/tests/027.phpt
@@ -0,0 +1,73 @@
+--TEST--
+Check for serialization handler
+--SKIPIF--
+--FILE--
+
+--EXPECT--
+bool(true)
+read
+wrote: 82a3666f6f01a474657374a6666f6f626172
+array(2) {
+  ["foo"]=>
+  int(1)
+  ["test"]=>
+  string(6) "foobar"
+}
diff --git a/php/tests/028.phpt b/php/tests/028.phpt
new file mode 100644
index 0000000..56805f9
--- /dev/null
+++ b/php/tests/028.phpt
@@ -0,0 +1,667 @@
+--TEST--
+Serialize object into session, full set
+--SKIPIF--
+--FILE--
+d1 = $foo;
+        $this->d2 = $foo;
+        $this->d3 = $foo;
+    }
+}
+
+class Bar {
+    private static $s1 = array();
+    protected static $s2 = array();
+    public static $s3 = array();
+
+    public $d1;
+    private $d2;
+    protected $d3;
+
+    public function __construct() {
+    }
+
+    public function set($foo) {
+        $this->d1 = $foo;
+        $this->d2 = $foo;
+        $this->d3 = $foo;
+    }
+}
+
+$output = '';
+
+function open($path, $name) {
+    return true;
+}
+
+function close() {
+    return true;
+}
+
+function read($id) {
+    global $output;
+    $output .= "read" . PHP_EOL;
+    $a = new Bar();
+    $b = new Foo($a);
+    $a->set($b);
+    $session = array('old' => $b);
+    return msgpack_serialize($session);
+}
+
+function write($id, $data) {
+    global $output;
+    $output .= "write: ";
+    $output .= bin2hex($data) . PHP_EOL;
+    return true;
+}
+
+function destroy($id) {
+    return true;
+}
+
+function gc($time) {
+    return true;
+}
+
+ini_set('session.serialize_handler', 'msgpack');
+
+session_set_save_handler('open', 'close', 'read', 'write', 'destroy', 'gc');
+
+session_start();
+
+$_SESSION['test'] = "foobar";
+$a = new Bar();
+$b = new Foo($a);
+$a->set($b);
+$_SESSION['new'] = $a;
+
+session_write_close();
+
+echo $output;
+var_dump($_SESSION);
+?>
+--EXPECTF--
+read
+write: 83a36f6c6484c0a3466f6fa700466f6f00643184c0a3426172a2643182c002c002a70042617200643282c002c002a5002a00643382c002c002a5002a00643282c002c003a2643382c002c003a474657374a6666f6f626172a36e657784c0a3426172a2643184c0a3466f6fa700466f6f00643182c002c00aa5002a00643282c002c00aa2643382c002c00aa70042617200643282c002c00ba5002a00643382c002c00b
+array(3) {
+  ["old"]=>
+  object(Foo)#3 (3) {
+    [%r"?d1"?:("Foo":)?private"?%r]=>
+    object(Bar)#4 (3) {
+      ["d1"]=>
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
+      [%r"?d2"?:("Bar":)?private"?%r]=>
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
+      [%r"?d3"?:protected"?%r]=>
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
+    }
+    [%r"?d2"?:protected"?%r]=>
+    object(Bar)#4 (3) {
+      ["d1"]=>
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
+      [%r"?d2"?:("Bar":)?private"?%r]=>
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
+      [%r"?d3"?:protected"?%r]=>
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
+    }
+    ["d3"]=>
+    object(Bar)#4 (3) {
+      ["d1"]=>
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
+      [%r"?d2"?:("Bar":)?private"?%r]=>
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
+      [%r"?d3"?:protected"?%r]=>
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+  ["test"]=>
+  string(6) "foobar"
+  ["new"]=>
+  object(Bar)#5 (3) {
+    ["d1"]=>
+    object(Foo)#6 (3) {
+      [%r"?d1"?:("Foo":)?private"?%r]=>
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
+      [%r"?d2"?:protected"?%r]=>
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
+      ["d3"]=>
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
+    }
+    [%r"?d2"?:("Bar":)?private"?%r]=>
+    object(Foo)#6 (3) {
+      [%r"?d1"?:("Foo":)?private"?%r]=>
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
+      [%r"?d2"?:protected"?%r]=>
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
+      ["d3"]=>
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
+    }
+    [%r"?d3"?:protected"?%r]=>
+    object(Foo)#6 (3) {
+      [%r"?d1"?:("Foo":)?private"?%r]=>
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
+      [%r"?d2"?:protected"?%r]=>
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
+      ["d3"]=>
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
diff --git a/php/tests/029.phpt b/php/tests/029.phpt
new file mode 100644
index 0000000..3bd1e9d
--- /dev/null
+++ b/php/tests/029.phpt
@@ -0,0 +1,20 @@
+--TEST--
+Msgpack module info
+--SKIPIF--
+
+--FILE--
+ enabled
+msgpack version => %s
+msgpack Session Support => enabled
diff --git a/php/tests/030.phpt b/php/tests/030.phpt
new file mode 100644
index 0000000..ec8622d
--- /dev/null
+++ b/php/tests/030.phpt
@@ -0,0 +1,233 @@
+--TEST--
+Unserialize invalid data
+--SKIPIF--
+--FILE--
+ 10, "foo"),
+    true,
+    false,
+    0.187182,
+    "dakjdh98389\000",
+    null,
+    (object)array(1,2,3),
+);
+
+error_reporting(0);
+
+foreach ($datas as $data) {
+    $str = msgpack_serialize($data);
+    $len = strlen($str);
+
+    // truncated
+    for ($i = 0; $i < $len - 1; $i++) {
+        $v = msgpack_unserialize(substr($str, 0, $i));
+
+        if (is_object($data) || is_array($data)) {
+            if ($v !== null && $v !== false && $v != $data) {
+                echo "output at $i:\n";
+                var_dump($v);
+            }
+        } else if ($v !== null && $v == $data) {
+            continue;
+        } else if ($v !== null && $v !== $data) {
+            echo "output at $i:\n";
+            var_dump($v);
+            echo "vs.\n";
+            var_dump($data);
+        }
+    }
+
+    // padded
+    $str .= "98398afa\000y21_ ";
+    $v = msgpack_unserialize($str);
+    if ($v !== $data && !(is_object($data) && $v == $data)) {
+        echo "padded should get original\n";
+        var_dump($v);
+        echo "vs.\n";
+        var_dump($data);
+    }
+}
+?>
+--EXPECTF--
+output at 2:
+array(0) {
+}
+output at 3:
+array(1) {
+  [0]=>
+  int(1)
+}
+output at 4:
+array(1) {
+  [0]=>
+  int(1)
+}
+output at 5:
+array(2) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+}
+output at 6:
+array(2) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+}
+output at 7:
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+output at 8:
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+output at 9:
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+output at 10:
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+output at 11:
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+output at 12:
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+output at 13:
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+output at 14:
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+output at 15:
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+output at 16:
+array(4) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+  ["testing"]=>
+  int(10)
+}
+output at 17:
+array(4) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+  ["testing"]=>
+  int(10)
+}
+output at 18:
+array(4) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+  ["testing"]=>
+  int(10)
+}
+output at 19:
+array(4) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+  ["testing"]=>
+  int(10)
+}
+output at 11:
+object(stdClass)#2 (0) {
+}
+output at 12:
+object(stdClass)#3 (0) {
+}
+output at 13:
+object(stdClass)#2 (1) {
+  [0]=>
+  int(1)
+}
+output at 14:
+object(stdClass)#3 (1) {
+  [0]=>
+  int(1)
+}
+output at 15:
+object(stdClass)#2 (2) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+}
diff --git a/php/tests/031.phpt b/php/tests/031.phpt
new file mode 100644
index 0000000..ce3ba49
--- /dev/null
+++ b/php/tests/031.phpt
@@ -0,0 +1,90 @@
+--TEST--
+Object Serializable interface throws exceptions
+--SKIPIF--
+--FILE--
+a = $a;
+        $this->b = $b;
+    }
+
+    public function serialize() {
+        $c = self::$count++;
+        echo "call serialize, ", ($this->a ? "throw" : "no throw"), PHP_EOL;
+        if ($this->a) {
+            throw new Exception("exception in serialize $c");
+        }
+        return pack('NN', $this->a, $this->b);
+    }
+
+    public function unserialize($serialized) {
+        $tmp = unpack('N*', $serialized);
+        $this->__construct($tmp[1], $tmp[2]);
+        $c = self::$count++;
+        echo "call unserialize, ", ($this->b ? "throw" : "no throw"), PHP_EOL;
+        if ($this->b) {
+            throw new Exception("exception in unserialize $c");
+        }
+    }
+}
+
+$a = new Obj(1, 0);
+$a = new Obj(0, 0);
+$b = new Obj(0, 0);
+$c = new Obj(1, 0);
+$d = new Obj(0, 1);
+
+echo "a, a, c", PHP_EOL;
+try {
+    test(array($a, $a, $c));
+} catch (Exception $e) {
+    if (version_compare(phpversion(), "5.3.0", ">=")) {
+        if ($e->getPrevious()) {
+            $e = $e->getPrevious();
+        }
+    }
+
+    echo $e->getMessage(), PHP_EOL;
+}
+
+echo "b, b, d", PHP_EOL;
+
+try {
+    test(array($b, $b, $d));
+} catch (Exception $e) {
+    if (version_compare(phpversion(), "5.3.0", ">=")) {
+        if ($e->getPrevious()) {
+            $e = $e->getPrevious();
+        }
+    }
+
+    echo $e->getMessage(), PHP_EOL;
+}
+?>
+--EXPECT--
+a, a, c
+call serialize, no throw
+call serialize, throw
+exception in serialize 2
+b, b, d
+call serialize, no throw
+call serialize, no throw
+call unserialize, no throw
+call unserialize, throw
+exception in unserialize 6
diff --git a/php/tests/032.phpt b/php/tests/032.phpt
new file mode 100644
index 0000000..b120ea9
--- /dev/null
+++ b/php/tests/032.phpt
@@ -0,0 +1,76 @@
+--TEST--
+Object test, __sleep and __wakeup exceptions
+--SKIPIF--
+--FILE--
+a = $a;
+        $this->b = $b;
+    }
+
+    function __sleep() {
+        $c = self::$count++;
+        if ($this->a) {
+            throw new Exception("exception in __sleep $c");
+        }
+        return array('a', 'b');
+    }
+
+    function __wakeup() {
+        $c = self::$count++;
+        if ($this->b) {
+            throw new Exception("exception in __wakeup $c");
+        }
+        $this->b = $this->a * 3;
+    }
+}
+
+
+$a = new Obj(1, 0);
+$b = new Obj(0, 1);
+$c = new Obj(0, 0);
+
+try {
+    test($a);
+} catch (Exception $e) {
+    echo $e->getMessage(), PHP_EOL;
+}
+
+try {
+    test($b);
+} catch (Exception $e) {
+    echo $e->getMessage(), PHP_EOL;
+}
+
+try {
+    test($c);
+} catch (Exception $e) {
+    echo $e->getMessage(), PHP_EOL;
+}
+?>
+--EXPECTF--
+exception in __sleep 0
+exception in __wakeup 2
+object(Obj)#%d (2) {
+  ["a"]=>
+  int(0)
+  ["b"]=>
+  int(0)
+}
diff --git a/php/tests/033.phpt b/php/tests/033.phpt
new file mode 100644
index 0000000..682fd4d
--- /dev/null
+++ b/php/tests/033.phpt
@@ -0,0 +1,55 @@
+--TEST--
+Object test, cyclic references
+--SKIPIF--
+--FILE--
+parent = null;
+        $this->children = array();
+    }
+
+    public function addChild(Foo $obj) {
+        $this->children[] = $obj;
+        $obj->setParent($this);
+    }
+
+    public function setParent(Foo $obj) {
+        $this->parent = $obj;
+    }
+}
+
+$obj1 = new Foo();
+
+for ($i = 0; $i < 10; $i++) {
+    $obj = new Foo();
+    $obj1->addChild($obj);
+}
+
+$o = msgpack_unserialize(msgpack_serialize($obj1->children));
+
+foreach ($obj1->children as $k => $v) {
+    $obj_v = $v;
+    $o_v = $o[$k];
+
+    echo gettype($obj_v), "  ", gettype($o_v), PHP_EOL;
+}
+?>
+--EXPECT--
+object  object
+object  object
+object  object
+object  object
+object  object
+object  object
+object  object
+object  object
+object  object
+object  object
diff --git a/php/tests/034.phpt b/php/tests/034.phpt
new file mode 100644
index 0000000..6610c7c
--- /dev/null
+++ b/php/tests/034.phpt
@@ -0,0 +1,38 @@
+--TEST--
+Unserialize invalid random data
+--SKIPIF--
+--FILE--
+ 10, "foo"),
+	true,
+	false,
+	0.187182,
+	"dakjdh98389\000",
+	null,
+	(object)array(1,2,3),
+);
+
+error_reporting(0);
+
+foreach ($datas as $data) {
+	$str = msgpack_serialize($data);
+	$len = strlen($str);
+
+	for ($j = 0; $j < 200; $j++) {
+		for ($i = 0; $i < $len - 1; $i++) {
+			$sub = substr($str, 0, $i);
+			$sub .= mcrypt_create_iv(30, MCRYPT_DEV_URANDOM);
+			$php_errormsg = null;
+			$v = msgpack_unserialize($sub);
+		}
+	}
+}
+
+--EXPECT--
diff --git a/php/tests/035.phpt b/php/tests/035.phpt
new file mode 100644
index 0000000..d680565
--- /dev/null
+++ b/php/tests/035.phpt
@@ -0,0 +1,34 @@
+--TEST--
+Profiling perf test.
+--SKIPIF--
+
+--FILE--
+ 400 ? "GOOD" : "BAD"));
+}
+?>
+--EXPECTF--
+%d iterations took %f seconds: %d/s (GOOD)
diff --git a/php/tests/040.phpt b/php/tests/040.phpt
new file mode 100644
index 0000000..9a2d038
--- /dev/null
+++ b/php/tests/040.phpt
@@ -0,0 +1,43 @@
+--TEST--
+b0rked random data test
+--SKIPIF--
+--FILE--
+
+--EXPECT--
diff --git a/php/tests/041.phpt b/php/tests/041.phpt
new file mode 100644
index 0000000..6400fd9
--- /dev/null
+++ b/php/tests/041.phpt
@@ -0,0 +1,47 @@
+--TEST--
+Check for double NaN, Inf, -Inf, 0, and -0
+--FILE--
+
+--EXPECT--
+empty array:
+90
+array(0) {
+}
+array(1, 2, 3)
+93010203
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+array(array(1, 2, 3), arr...
+93930102039304050693070809
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+array("foo", "FOO", "Foo")
+93a3666f6fa3464f4fa3466f6f
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "FOO"
+  [2]=>
+  string(3) "Foo"
+}
+array(1, 123.45,  true, ...
+9701cb405edccccccccccdc3c293010293090807c0a3666f6f
+array(7) {
+  [0]=>
+  int(1)
+  [1]=>
+  float(123.45)
+  [2]=>
+  bool(true)
+  [3]=>
+  bool(false)
+  [4]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    array(3) {
+      [0]=>
+      int(9)
+      [1]=>
+      int(8)
+      [2]=>
+      int(7)
+    }
+  }
+  [5]=>
+  NULL
+  [6]=>
+  string(3) "foo"
+}
diff --git a/php/tests/060.phpt b/php/tests/060.phpt
new file mode 100644
index 0000000..c0fda4c
--- /dev/null
+++ b/php/tests/060.phpt
@@ -0,0 +1,315 @@
+--TEST--
+Check for buffered streaming unserialization
+--SKIPIF--
+--FILE--
+feed($str);
+        if ($unpacker->execute())
+        {
+            $unserialized = $unpacker->data();
+            var_dump($unserialized);
+            $unpacker->reset();
+        }
+
+        $i += $len;
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      &array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/061.phpt b/php/tests/061.phpt
new file mode 100644
index 0000000..ded494e
--- /dev/null
+++ b/php/tests/061.phpt
@@ -0,0 +1,320 @@
+--TEST--
+Check for unbuffered streaming unserialization
+--SKIPIF--
+--FILE--
+execute($str, $offset))
+        {
+            $unserialized = $unpacker->data();
+            var_dump($unserialized);
+
+            $unpacker->reset();
+            $str = "";
+            $offset = 0;
+        }
+
+        $i += $len;
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      &array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/062.phpt b/php/tests/062.phpt
new file mode 100644
index 0000000..159e00d
--- /dev/null
+++ b/php/tests/062.phpt
@@ -0,0 +1,64 @@
+--TEST--
+Extra bytes buffered streaming unserialization
+--SKIPIF--
+--FILE--
+feed($str);
+
+            while (true) {
+                if ($unpacker->execute()) {
+                    $unserialized = $unpacker->data();
+                    var_dump($unserialized);
+                    $unpacker->reset();
+                } else {
+                    break;
+                }
+            }
+            $i += $len;
+        }
+    }
+}
+
+test('array(1, 2, 3)', array('9301020392'));
+test('array(1, 2, 3), array(3, 9), 4', array('9301020392', '030904'));
+--EXPECTF--
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+array(2) {
+  [0]=>
+  int(3)
+  [1]=>
+  int(9)
+}
+int(4)
diff --git a/php/tests/063.phpt b/php/tests/063.phpt
new file mode 100644
index 0000000..5be7e09
--- /dev/null
+++ b/php/tests/063.phpt
@@ -0,0 +1,68 @@
+--TEST--
+Extra bytes unbuffered streaming unserialization
+--SKIPIF--
+--FILE--
+execute($str, $offset)) {
+                    $unserialized = $unpacker->data();
+                    var_dump($unserialized);
+
+                    $unpacker->reset();
+                    $str = substr($str, $offset);
+                    $offset = 0;
+                } else {
+                    break;
+                }
+            }
+            $i += $len;
+        }
+    }
+}
+
+test('array(1, 2, 3)', array('9301020392'));
+test('array(1, 2, 3), array(3, 9), 4', array('9301020392', '030904'));
+--EXPECTF--
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+array(2) {
+  [0]=>
+  int(3)
+  [1]=>
+  int(9)
+}
+int(4)
diff --git a/php/tests/070.phpt b/php/tests/070.phpt
new file mode 100644
index 0000000..4a00ddf
--- /dev/null
+++ b/php/tests/070.phpt
@@ -0,0 +1,299 @@
+--TEST--
+Check for alias functions
+--SKIPIF--
+--FILE--
+ 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      &array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/071.phpt b/php/tests/071.phpt
new file mode 100644
index 0000000..cca351a
--- /dev/null
+++ b/php/tests/071.phpt
@@ -0,0 +1,301 @@
+--TEST--
+Check for class methods
+--SKIPIF--
+--FILE--
+pack($variable);
+    $unserialized = $msgpack->unpack($serialized);
+
+    var_dump($unserialized);
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      &array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/072.phpt b/php/tests/072.phpt
new file mode 100644
index 0000000..d8cb1b9
--- /dev/null
+++ b/php/tests/072.phpt
@@ -0,0 +1,341 @@
+--TEST--
+Check for class methods unpacker
+--SKIPIF--
+--FILE--
+pack($variable);
+    $unpacker = $msgpack->unpacker();
+
+    $length = strlen($serialized);
+
+    if (rand(0, 1))
+    {
+        for ($i = 0; $i < $length;) {
+            $len = rand(1, 10);
+            $str = substr($serialized, $i, $len);
+
+            $unpacker->feed($str);
+            if ($unpacker->execute())
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+                $unpacker->reset();
+            }
+
+            $i += $len;
+        }
+    }
+    else
+    {
+        $str = "";
+        $offset = 0;
+
+        for ($i = 0; $i < $length;) {
+            $len = rand(1, 10);
+            $str .= substr($serialized, $i, $len);
+
+            if ($unpacker->execute($str, $offset))
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+
+                $unpacker->reset();
+                $str = "";
+                $offset = 0;
+            }
+
+            $i += $len;
+        }
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      &array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/test_pack.phpt b/php/tests/test_pack.phpt
deleted file mode 100644
index 595e4d0..0000000
--- a/php/tests/test_pack.phpt
+++ /dev/null
@@ -1,169 +0,0 @@
---TEST--
-Test msgpack_pack() function : basic functionality
---SKIPIF--
-
---FILE--
-
-===DONE===
---EXPECT--
-*** Testing msgpack_pack() : basic functionality ***
--- Iteration 1 --
-string(4) "0xc0"
--- Iteration 2 --
-string(4) "0xc0"
--- Iteration 3 --
-string(4) "0xc2"
--- Iteration 4 --
-string(4) "0xc2"
--- Iteration 5 --
-string(4) "0xc3"
--- Iteration 6 --
-string(4) "0xc3"
--- Iteration 7 --
-string(4) "0x00"
--- Iteration 8 --
-string(4) "0x7f"
--- Iteration 9 --
-string(6) "0xcc80"
--- Iteration 10 --
-string(8) "0xcd0100"
--- Iteration 11 --
-string(4) "0xff"
--- Iteration 12 --
-string(6) "0xd0df"
--- Iteration 13 --
-string(8) "0xd1ff7f"
--- Iteration 14 --
-string(8) "0x810101"
--- Iteration 15 --
-string(20) "0xcb3ff0000000000000"
--- Iteration 16 --
-string(4) "0x90"
--- Iteration 17 --
-string(34) "0x9f000102030405060708090a0b0c0d0e"
--- Iteration 18 --
-string(40) "0xdc0010000102030405060708090a0b0c0d0e0f"
--- Iteration 19 --
-string(64) "0x8f0100020103020403050406050706080709080a090b0a0c0b0d0c0e0d0f0e"
--- Iteration 20 --
-string(72) "0xde00100100020103020403050406050706080709080a090b0a0c0b0d0c0e0d0f0e100f"
-===DONE===

From 7b152640d962434192496e4d96788622d4e90886 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Sun, 18 Jul 2010 23:40:25 +0900
Subject: [PATCH 0587/1648] erlang: 'edoc' document generation

---
 erlang/.gitignore      | 1 +
 erlang/OMakefile       | 7 +++++--
 erlang/edoc/.gitignore | 4 ++++
 erlang/msgpack.erl     | 2 +-
 4 files changed, 11 insertions(+), 3 deletions(-)
 create mode 100644 erlang/edoc/.gitignore

diff --git a/erlang/.gitignore b/erlang/.gitignore
index 7d6d324..0f7faad 100644
--- a/erlang/.gitignore
+++ b/erlang/.gitignore
@@ -2,3 +2,4 @@ MANIFEST
 *.beam
 .omakedb*
 *.omc
+*~
\ No newline at end of file
diff --git a/erlang/OMakefile b/erlang/OMakefile
index 89b1c63..d213444 100644
--- a/erlang/OMakefile
+++ b/erlang/OMakefile
@@ -30,10 +30,10 @@
 # If so, define the subdirectory targets and uncomment this section.
 #
 
-.DEFAULT: msgpack.beam msgpack.html
+.DEFAULT: msgpack.beam
 
 msgpack.beam: msgpack.erl
-	erlc $<
+	erlc -Wall +debug_info $<
 
 msgpack.html: msgpack.erl
 	erl -noshell -run edoc_run file $<
@@ -41,5 +41,8 @@ msgpack.html: msgpack.erl
 test: msgpack.beam
 	erl -noshell -s msgpack test -s init stop
 
+edoc: msgpack.erl
+	erl -noshell -eval 'ok=edoc:files(["msgpack.erl"], [{dir, "edoc"}]).' -s init stop
+
 clean:
 	-rm -f *.beam *.html
diff --git a/erlang/edoc/.gitignore b/erlang/edoc/.gitignore
new file mode 100644
index 0000000..97f4246
--- /dev/null
+++ b/erlang/edoc/.gitignore
@@ -0,0 +1,4 @@
+*.html
+*.css
+*.png
+edoc-info
diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index aa9851d..a697483 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -18,7 +18,7 @@
 
 %% @doc MessagePack codec for Erlang.
 %%
-%%      APIs are almost compatible with C API
+%%      APIs are almost compatible with C API
 %%      except for buffering functions (both copying and zero-copying), which are unavailable.
 %%
 %%   

From dad7a03d1975c28574fb768e10a63355832a0474 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Sun, 18 Jul 2010 23:42:23 +0900
Subject: [PATCH 0588/1648] erlang: stopped support for dict() type.

---
 erlang/msgpack.erl | 2 --
 1 file changed, 2 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index a697483..13bb8e1 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -114,8 +114,6 @@ pack_(List)  when is_list(List) ->
     pack_array(List);
 pack_({Map}) when is_list(Map) ->
     pack_map(Map);
-pack_(Map) when is_tuple(Map), element(1,Map)=:=dict ->
-    pack_map(dict:to_list(Map));
 pack_(Other) ->
     throw({error, {badarg, Other}}).
 

From 6cabad19d5a576e3e44e069c0a2b107bd33d30ef Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Sun, 18 Jul 2010 23:48:20 +0900
Subject: [PATCH 0589/1648] erlang: unpack_all/1 improve, error handling added.

---
 erlang/msgpack.erl | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index 13bb8e1..d4a0bc8 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -80,16 +80,22 @@ unpack(Bin) when is_binary(Bin) ->
 unpack(Other) ->
     {error, {badarg, Other}}.
 
--spec unpack_all(binary()) -> [msgpack_term()].
+-spec unpack_all(binary()) -> [msgpack_term()] | {error, incomplete} | {error, {badarg, term()}}.
 unpack_all(Data)->
-    case unpack(Data) of
-	{ Term, Binary } when bit_size(Binary) =:= 0 ->
+    try
+	unpack_all_(Data)
+    catch
+	throw:Exception ->
+	    {error, Exception}
+    end.
+unpack_all_(Data)->
+    case unpack_(Data) of
+	{ Term, <<>> } ->
 	    [Term];
 	{ Term, Binary } when is_binary(Binary) ->
-	    [Term|unpack_all(Binary)]
+	    [Term|unpack_all_(Binary)]
     end.
 
-
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 % internal APIs
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

From 8a3ac6d9bd374acc6b134d330bf7aeb906fa1a80 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Sun, 18 Jul 2010 23:50:29 +0900
Subject: [PATCH 0590/1648] erlang: omake menus added.

---
 erlang/OMakefile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/erlang/OMakefile b/erlang/OMakefile
index d213444..2107940 100644
--- a/erlang/OMakefile
+++ b/erlang/OMakefile
@@ -22,7 +22,7 @@
 # Phony targets are scoped, so you probably want to declare them first.
 #
 
-.PHONY: all clean test #install
+.PHONY: all clean test edoc dialyzer #install
 
 ########################################################################
 # Subdirectories.
@@ -44,5 +44,8 @@ test: msgpack.beam
 edoc: msgpack.erl
 	erl -noshell -eval 'ok=edoc:files(["msgpack.erl"], [{dir, "edoc"}]).' -s init stop
 
+dialyzer: msgpack.erl
+	dialyzer --src $<
+
 clean:
 	-rm -f *.beam *.html

From dcbcf5842f58b7ead3f524ac027576ce0c55e9ab Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Sun, 18 Jul 2010 23:55:07 +0900
Subject: [PATCH 0591/1648] erlang: msgpack:unpack_all/1 doc.

---
 erlang/msgpack.erl | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index d4a0bc8..b54874d 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -80,6 +80,11 @@ unpack(Bin) when is_binary(Bin) ->
 unpack(Other) ->
     {error, {badarg, Other}}.
 
+% @doc Decode an msgpack binary into an erlang terms.
+%      It only decodes ALL msgpack packets contained in the binary. No packets should not remain.
+%      Returns {error, {badarg, term()}} if the input is corrupted.
+%      Returns {error, incomplete} if the input is not a full msgpack packet (caller should gather more data and try again).
+% @spec unpack_all(binary()) -> [msgpack_term()] | {error, incomplete} | {error, {badarg, term()}}
 -spec unpack_all(binary()) -> [msgpack_term()] | {error, incomplete} | {error, {badarg, term()}}.
 unpack_all(Data)->
     try

From 227c168b65be72e5e3a843af38d9382b59fc858a Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sat, 24 Jul 2010 18:07:22 +0900
Subject: [PATCH 0592/1648] java: fixes fatal offset calculation bugs on
 BufferedUnpackerIMPL.unpackInt()

---
 java/src/main/java/org/msgpack/BufferedUnpackerImpl.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java
index cc6604d..f4ed35b 100644
--- a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java
+++ b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java
@@ -103,7 +103,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl {
 		case 0xcc:  // unsigned int  8
 			more(2);
 			advance(2);
-			return (int)((short)buffer[offset+1] & 0xff);
+			return (int)((short)(buffer[offset-1]) & 0xff);
 		case 0xcd:  // unsigned int 16
 			more(3);
 			castBuffer.rewind();
@@ -137,7 +137,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl {
 		case 0xd0:  // signed int  8
 			more(2);
 			advance(2);
-			return (int)buffer[offset+1];
+			return (int)buffer[offset-1];
 		case 0xd1:  // signed int 16
 			more(3);
 			castBuffer.rewind();
@@ -178,7 +178,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl {
 		case 0xcc:  // unsigned int  8
 			more(2);
 			advance(2);
-			return (long)((short)buffer[offset+1] & 0xff);
+			return (long)((short)(buffer[offset-1]) & 0xff);
 		case 0xcd:  // unsigned int 16
 			more(3);
 			castBuffer.rewind();
@@ -207,7 +207,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl {
 		case 0xd0:  // signed int  8
 			more(2);
 			advance(2);
-			return (long)buffer[offset+1];
+			return (long)buffer[offset-1];
 		case 0xd1:  // signed int 16
 			more(3);
 			castBuffer.rewind();

From 2aef495d62d19b2f1721989225700942ea71e582 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sat, 24 Jul 2010 18:08:00 +0900
Subject: [PATCH 0593/1648] java: adds MessagePackObject class

---
 .../java/org/msgpack/MessagePackObject.java   | 130 ++++++++++++++++++
 .../java/org/msgpack/object/ArrayType.java    |  48 +++++++
 .../msgpack/object/BigIntegerTypeIMPL.java    |  92 +++++++++++++
 .../java/org/msgpack/object/BooleanType.java  |  39 ++++++
 .../org/msgpack/object/DoubleTypeIMPL.java    |  71 ++++++++++
 .../java/org/msgpack/object/FloatType.java    |  28 ++++
 .../org/msgpack/object/FloatTypeIMPL.java     |  70 ++++++++++
 .../java/org/msgpack/object/IntegerType.java  |  49 +++++++
 .../msgpack/object/LongIntegerTypeIMPL.java   |  89 ++++++++++++
 .../main/java/org/msgpack/object/MapType.java |  48 +++++++
 .../main/java/org/msgpack/object/NilType.java |  28 ++++
 .../msgpack/object/ShortIntegerTypeIMPL.java  |  91 ++++++++++++
 12 files changed, 783 insertions(+)
 create mode 100644 java/src/main/java/org/msgpack/MessagePackObject.java
 create mode 100644 java/src/main/java/org/msgpack/object/ArrayType.java
 create mode 100644 java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
 create mode 100644 java/src/main/java/org/msgpack/object/BooleanType.java
 create mode 100644 java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
 create mode 100644 java/src/main/java/org/msgpack/object/FloatType.java
 create mode 100644 java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
 create mode 100644 java/src/main/java/org/msgpack/object/IntegerType.java
 create mode 100644 java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
 create mode 100644 java/src/main/java/org/msgpack/object/MapType.java
 create mode 100644 java/src/main/java/org/msgpack/object/NilType.java
 create mode 100644 java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java

diff --git a/java/src/main/java/org/msgpack/MessagePackObject.java b/java/src/main/java/org/msgpack/MessagePackObject.java
new file mode 100644
index 0000000..aba27e4
--- /dev/null
+++ b/java/src/main/java/org/msgpack/MessagePackObject.java
@@ -0,0 +1,130 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.util.List;
+import java.util.Set;
+import java.util.Map;
+import java.math.BigInteger;
+
+public abstract class MessagePackObject {
+	public boolean isNull() {
+		return false;
+	}
+
+	public boolean isBooleanType() {
+		return false;
+	}
+
+	public boolean isIntegerType() {
+		return false;
+	}
+
+	public boolean isFloatType() {
+		return false;
+	}
+
+	public boolean isArrayType() {
+		return false;
+	}
+
+	public boolean isMapType() {
+		return false;
+	}
+
+	public boolean isRawType() {
+		return false;
+	}
+
+	public boolean asBoolean() {
+		throw new MessageTypeException("type error");
+	}
+
+	public byte asByte() {
+		throw new MessageTypeException("type error");
+	}
+
+	public short asShort() {
+		throw new MessageTypeException("type error");
+	}
+
+	public int asInt() {
+		throw new MessageTypeException("type error");
+	}
+
+	public long asLong() {
+		throw new MessageTypeException("type error");
+	}
+
+	public BigInteger asBigInteger() {
+		throw new MessageTypeException("type error");
+	}
+
+	public float asFloat() {
+		throw new MessageTypeException("type error");
+	}
+
+	public double asDouble() {
+		throw new MessageTypeException("type error");
+	}
+
+	public byte[] asByteArray() {
+		throw new MessageTypeException("type error");
+	}
+
+	public String asString() {
+		throw new MessageTypeException("type error");
+	}
+
+	public MessagePackObject[] asArray() {
+		throw new MessageTypeException("type error");
+	}
+
+	public List asList() {
+		throw new MessageTypeException("type error");
+	}
+
+	public Map asMap() {
+		throw new MessageTypeException("type error");
+	}
+
+	public byte byteValue() {
+		throw new MessageTypeException("type error");
+	}
+
+	public short shortValue() {
+		throw new MessageTypeException("type error");
+	}
+
+	public int intValue() {
+		throw new MessageTypeException("type error");
+	}
+
+	public long longValue() {
+		throw new MessageTypeException("type error");
+	}
+
+	public float floatValue() {
+		throw new MessageTypeException("type error");
+	}
+
+	public double doubleValue() {
+		throw new MessageTypeException("type error");
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/object/ArrayType.java b/java/src/main/java/org/msgpack/object/ArrayType.java
new file mode 100644
index 0000000..06e9b16
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/ArrayType.java
@@ -0,0 +1,48 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import java.util.List;
+import java.util.Set;
+import java.util.Map;
+import java.util.Arrays;
+import org.msgpack.*;
+
+public class ArrayType extends MessagePackObject {
+	private MessagePackObject[] array;
+
+	public ArrayType(MessagePackObject[] array) {
+		this.array = array;
+	}
+
+	@Override
+	public boolean isArrayType() {
+		return true;
+	}
+
+	@Override
+	public MessagePackObject[] asArray() {
+		return array;
+	}
+
+	@Override
+	public List asList() {
+		return Arrays.asList(array);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
new file mode 100644
index 0000000..1d638c9
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
@@ -0,0 +1,92 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import java.math.BigInteger;
+import org.msgpack.*;
+
+class BigIntegerTypeIMPL extends IntegerType {
+	private BigInteger value;
+
+	BigIntegerTypeIMPL(BigInteger vlaue) {
+		this.value = value;
+	}
+
+	@Override
+	public byte asByte() {
+		if(value.compareTo(BigInteger.valueOf((long)Byte.MAX_VALUE)) > 0) {
+			throw new MessageTypeException("type error");
+		}
+		return value.byteValue();
+	}
+
+	@Override
+	public short asShort() {
+		if(value.compareTo(BigInteger.valueOf((long)Short.MAX_VALUE)) > 0) {
+			throw new MessageTypeException("type error");
+		}
+		return value.shortValue();
+	}
+
+	@Override
+	public int asInt() {
+		if(value.compareTo(BigInteger.valueOf((long)Integer.MAX_VALUE)) > 0) {
+			throw new MessageTypeException("type error");
+		}
+		return value.intValue();
+	}
+
+	@Override
+	public long asLong() {
+		if(value.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) {
+			throw new MessageTypeException("type error");
+		}
+		return value.longValue();
+	}
+
+	@Override
+	public byte byteValue() {
+		return value.byteValue();
+	}
+
+	@Override
+	public short shortValue() {
+		return value.shortValue();
+	}
+
+	@Override
+	public int intValue() {
+		return value.intValue();
+	}
+
+	@Override
+	public long longValue() {
+		return value.longValue();
+	}
+
+	@Override
+	public float floatValue() {
+		return value.floatValue();
+	}
+
+	@Override
+	public double doubleValue() {
+		return value.doubleValue();
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/object/BooleanType.java b/java/src/main/java/org/msgpack/object/BooleanType.java
new file mode 100644
index 0000000..c9e84b6
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/BooleanType.java
@@ -0,0 +1,39 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import org.msgpack.*;
+
+public class BooleanType extends MessagePackObject {
+	private boolean value;
+
+	public BooleanType(boolean value) {
+		this.value = value;
+	}
+
+	@Override
+	public boolean isBooleanType() {
+		return false;
+	}
+
+	@Override
+	public boolean asBoolean() {
+		return value;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
new file mode 100644
index 0000000..0e33d5b
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
@@ -0,0 +1,71 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import java.math.BigInteger;
+import org.msgpack.*;
+
+class DoubleTypeIMPL extends FloatType {
+	private double value;
+
+	public DoubleTypeIMPL(double vlaue) {
+		this.value = value;
+	}
+
+	@Override
+	public float asFloat() {
+		// FIXME check overflow, underflow?
+		return (float)value;
+	}
+
+	@Override
+	public double asDouble() {
+		return value;
+	}
+
+	@Override
+	public byte byteValue() {
+		return (byte)value;
+	}
+
+	@Override
+	public short shortValue() {
+		return (short)value;
+	}
+
+	@Override
+	public int intValue() {
+		return (int)value;
+	}
+
+	@Override
+	public long longValue() {
+		return (long)value;
+	}
+
+	@Override
+	public float floatValue() {
+		return (float)value;
+	}
+
+	@Override
+	public double doubleValue() {
+		return (double)value;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/object/FloatType.java b/java/src/main/java/org/msgpack/object/FloatType.java
new file mode 100644
index 0000000..2782dda
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/FloatType.java
@@ -0,0 +1,28 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import org.msgpack.*;
+
+public abstract class FloatType extends MessagePackObject {
+	@Override
+	public boolean isFloatType() {
+		return true;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
new file mode 100644
index 0000000..75a5070
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
@@ -0,0 +1,70 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import java.math.BigInteger;
+import org.msgpack.*;
+
+class FloatTypeIMPL extends FloatType {
+	private float value;
+
+	public FloatTypeIMPL(float vlaue) {
+		this.value = value;
+	}
+
+	@Override
+	public float asFloat() {
+		return value;
+	}
+
+	@Override
+	public double asDouble() {
+		return (double)value;
+	}
+
+	@Override
+	public byte byteValue() {
+		return (byte)value;
+	}
+
+	@Override
+	public short shortValue() {
+		return (short)value;
+	}
+
+	@Override
+	public int intValue() {
+		return (int)value;
+	}
+
+	@Override
+	public long longValue() {
+		return (long)value;
+	}
+
+	@Override
+	public float floatValue() {
+		return (float)value;
+	}
+
+	@Override
+	public double doubleValue() {
+		return (double)value;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/object/IntegerType.java b/java/src/main/java/org/msgpack/object/IntegerType.java
new file mode 100644
index 0000000..d6a9b54
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/IntegerType.java
@@ -0,0 +1,49 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import java.math.BigInteger;
+import org.msgpack.*;
+
+public abstract class IntegerType extends MessagePackObject {
+	public static IntegerType create(byte value) {
+		return new ShortIntegerTypeIMPL((int)value);
+	}
+
+	public static IntegerType create(short value) {
+		return new ShortIntegerTypeIMPL((int)value);
+	}
+
+	public static IntegerType create(int value) {
+		return new ShortIntegerTypeIMPL(value);
+	}
+
+	public static IntegerType create(long value) {
+		return new LongIntegerTypeIMPL(value);
+	}
+
+	public static IntegerType create(BigInteger value) {
+		return new BigIntegerTypeIMPL(value);
+	}
+
+	@Override
+	public boolean isIntegerType() {
+		return true;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
new file mode 100644
index 0000000..c914e91
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
@@ -0,0 +1,89 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import java.math.BigInteger;
+import org.msgpack.*;
+
+class LongIntegerTypeIMPL extends IntegerType {
+	private long value;
+
+	LongIntegerTypeIMPL(long value) {
+		this.value = value;
+	}
+
+	@Override
+	public byte asByte() {
+		if(value > (long)Byte.MAX_VALUE) {
+			throw new MessageTypeException("type error");
+		}
+		return (byte)value;
+	}
+
+	@Override
+	public short asShort() {
+		if(value > (long)Short.MAX_VALUE) {
+			throw new MessageTypeException("type error");
+		}
+		return (short)value;
+	}
+
+	@Override
+	public int asInt() {
+		if(value > (long)Integer.MAX_VALUE) {
+			throw new MessageTypeException("type error");
+		}
+		return (int)value;
+	}
+
+	@Override
+	public long asLong() {
+		return value;
+	}
+
+	@Override
+	public byte byteValue() {
+		return (byte)value;
+	}
+
+	@Override
+	public short shortValue() {
+		return (short)value;
+	}
+
+	@Override
+	public int intValue() {
+		return (int)value;
+	}
+
+	@Override
+	public long longValue() {
+		return (long)value;
+	}
+
+	@Override
+	public float floatValue() {
+		return (float)value;
+	}
+
+	@Override
+	public double doubleValue() {
+		return (double)value;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/object/MapType.java b/java/src/main/java/org/msgpack/object/MapType.java
new file mode 100644
index 0000000..dbd145b
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/MapType.java
@@ -0,0 +1,48 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.msgpack.*;
+
+public class MapType extends MessagePackObject {
+	MessagePackObject[] map;
+
+	public MapType(MessagePackObject[] map) {
+		this.map = map;
+	}
+
+	@Override
+	public boolean isMapType() {
+		return false;
+	}
+
+	@Override
+	public Map asMap() {
+		HashMap m = new HashMap(map.length / 2);
+		int i = 0;
+		while(i < map.length) {
+			MessagePackObject k = map[i++];
+			MessagePackObject v = map[i++];
+			m.put(k, v);
+		}
+		return m;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/object/NilType.java b/java/src/main/java/org/msgpack/object/NilType.java
new file mode 100644
index 0000000..c36ff05
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/NilType.java
@@ -0,0 +1,28 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import org.msgpack.*;
+
+public class NilType extends MessagePackObject {
+	@Override
+	public boolean isNull() {
+		return true;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java
new file mode 100644
index 0000000..a725950
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java
@@ -0,0 +1,91 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import java.math.BigInteger;
+import org.msgpack.*;
+
+class ShortIntegerTypeIMPL extends IntegerType {
+	private int value;
+
+	ShortIntegerTypeIMPL(int value) {
+		this.value = value;
+	}
+
+	@Override
+	public byte asByte() {
+		if(value > (int)Byte.MAX_VALUE) {
+			throw new MessageTypeException("type error");
+		}
+		return (byte)value;
+	}
+
+	@Override
+	public short asShort() {
+		if(value > (int)Short.MAX_VALUE) {
+			throw new MessageTypeException("type error");
+		}
+		return (short)value;
+	}
+
+	@Override
+	public int asInt() {
+		return value;
+	}
+
+	@Override
+	public long asLong() {
+		return value;
+	}
+
+	@Override
+	public BigInteger asBigInteger() {
+		return BigInteger.valueOf((long)value);
+	}
+
+	@Override
+	public byte byteValue() {
+		return (byte)value;
+	}
+
+	@Override
+	public short shortValue() {
+		return (short)value;
+	}
+
+	@Override
+	public int intValue() {
+		return (int)value;
+	}
+
+	@Override
+	public long longValue() {
+		return (long)value;
+	}
+
+	@Override
+	public float floatValue() {
+		return (float)value;
+	}
+
+	@Override
+	public double doubleValue() {
+		return (double)value;
+	}
+}
+

From 02ae247536ec5570c3a150de8283ef399aaf82eb Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sat, 24 Jul 2010 18:20:00 +0900
Subject: [PATCH 0594/1648] java: adds MessagePackObject class 2

---
 .../msgpack/object/BigIntegerTypeIMPL.java    |  2 +-
 .../java/org/msgpack/object/BooleanType.java  |  2 +-
 .../org/msgpack/object/DoubleTypeIMPL.java    |  2 +-
 .../java/org/msgpack/object/FloatType.java    |  8 ++++
 .../org/msgpack/object/FloatTypeIMPL.java     |  2 +-
 .../java/org/msgpack/object/IntegerType.java  | 10 ++--
 .../main/java/org/msgpack/object/MapType.java |  4 +-
 .../main/java/org/msgpack/object/RawType.java | 48 +++++++++++++++++++
 8 files changed, 67 insertions(+), 11 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/object/RawType.java

diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
index 1d638c9..1ebb83d 100644
--- a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
@@ -23,7 +23,7 @@ import org.msgpack.*;
 class BigIntegerTypeIMPL extends IntegerType {
 	private BigInteger value;
 
-	BigIntegerTypeIMPL(BigInteger vlaue) {
+	BigIntegerTypeIMPL(BigInteger value) {
 		this.value = value;
 	}
 
diff --git a/java/src/main/java/org/msgpack/object/BooleanType.java b/java/src/main/java/org/msgpack/object/BooleanType.java
index c9e84b6..d272b6f 100644
--- a/java/src/main/java/org/msgpack/object/BooleanType.java
+++ b/java/src/main/java/org/msgpack/object/BooleanType.java
@@ -28,7 +28,7 @@ public class BooleanType extends MessagePackObject {
 
 	@Override
 	public boolean isBooleanType() {
-		return false;
+		return true;
 	}
 
 	@Override
diff --git a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
index 0e33d5b..8bbc52a 100644
--- a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
@@ -23,7 +23,7 @@ import org.msgpack.*;
 class DoubleTypeIMPL extends FloatType {
 	private double value;
 
-	public DoubleTypeIMPL(double vlaue) {
+	public DoubleTypeIMPL(double value) {
 		this.value = value;
 	}
 
diff --git a/java/src/main/java/org/msgpack/object/FloatType.java b/java/src/main/java/org/msgpack/object/FloatType.java
index 2782dda..514efd5 100644
--- a/java/src/main/java/org/msgpack/object/FloatType.java
+++ b/java/src/main/java/org/msgpack/object/FloatType.java
@@ -24,5 +24,13 @@ public abstract class FloatType extends MessagePackObject {
 	public boolean isFloatType() {
 		return true;
 	}
+
+	public static FloatType create(float value) {
+		return new FloatTypeIMPL(value);
+	}
+
+	public static FloatType create(double value) {
+		return new DoubleTypeIMPL(value);
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
index 75a5070..8821640 100644
--- a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
@@ -23,7 +23,7 @@ import org.msgpack.*;
 class FloatTypeIMPL extends FloatType {
 	private float value;
 
-	public FloatTypeIMPL(float vlaue) {
+	public FloatTypeIMPL(float value) {
 		this.value = value;
 	}
 
diff --git a/java/src/main/java/org/msgpack/object/IntegerType.java b/java/src/main/java/org/msgpack/object/IntegerType.java
index d6a9b54..43357e8 100644
--- a/java/src/main/java/org/msgpack/object/IntegerType.java
+++ b/java/src/main/java/org/msgpack/object/IntegerType.java
@@ -21,6 +21,11 @@ import java.math.BigInteger;
 import org.msgpack.*;
 
 public abstract class IntegerType extends MessagePackObject {
+	@Override
+	public boolean isIntegerType() {
+		return true;
+	}
+
 	public static IntegerType create(byte value) {
 		return new ShortIntegerTypeIMPL((int)value);
 	}
@@ -40,10 +45,5 @@ public abstract class IntegerType extends MessagePackObject {
 	public static IntegerType create(BigInteger value) {
 		return new BigIntegerTypeIMPL(value);
 	}
-
-	@Override
-	public boolean isIntegerType() {
-		return true;
-	}
 }
 
diff --git a/java/src/main/java/org/msgpack/object/MapType.java b/java/src/main/java/org/msgpack/object/MapType.java
index dbd145b..359ebe6 100644
--- a/java/src/main/java/org/msgpack/object/MapType.java
+++ b/java/src/main/java/org/msgpack/object/MapType.java
@@ -22,7 +22,7 @@ import java.util.Map;
 import org.msgpack.*;
 
 public class MapType extends MessagePackObject {
-	MessagePackObject[] map;
+	private MessagePackObject[] map;
 
 	public MapType(MessagePackObject[] map) {
 		this.map = map;
@@ -30,7 +30,7 @@ public class MapType extends MessagePackObject {
 
 	@Override
 	public boolean isMapType() {
-		return false;
+		return true;
 	}
 
 	@Override
diff --git a/java/src/main/java/org/msgpack/object/RawType.java b/java/src/main/java/org/msgpack/object/RawType.java
new file mode 100644
index 0000000..107ba27
--- /dev/null
+++ b/java/src/main/java/org/msgpack/object/RawType.java
@@ -0,0 +1,48 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.object;
+
+import org.msgpack.*;
+
+class RawType extends MessagePackObject {
+	private byte[] bytes;
+
+	public RawType(byte[] bytes) {
+		this.bytes = bytes;
+	}
+
+	@Override
+	public boolean isRawType() {
+		return true;
+	}
+
+	@Override
+	public byte[] asByteArray() {
+		return bytes;
+	}
+
+	@Override
+	public String asString() {
+		try {
+			return new String(bytes, "UTF-8");
+		} catch (Exception e) {
+			throw new MessageTypeException("type error");
+		}
+	}
+}
+

From cd83388f8b035102f260f2ab45a551233dd593da Mon Sep 17 00:00:00 2001
From: Kazuki Ohta 
Date: Tue, 27 Jul 2010 08:59:09 +0900
Subject: [PATCH 0595/1648] java: fixed repository location.
 msgpack.sourceforge.net => msgpack.org

---
 java/pom.xml | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/java/pom.xml b/java/pom.xml
index 9b74b4c..959d2df 100755
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -7,7 +7,7 @@
   MessagePack for Java
 
   MessagePack for Java
-  http://msgpack.sourceforge.net/
+  http://msgpack.org/
 
   
     
@@ -97,29 +97,26 @@
 
   
     
-      msgpack.sourceforge.net
+      msgpack.org
       MessagePack Maven2 Repository
-      http://msgpack.sourceforge.net/maven2
-    
-    
-      msgpack.sourceforge.net
-      MessagePack Maven2 Snapshot Repository
-      http://msgpack.sourceforge.net/maven2-snapshot
+      http://msgpack.org/maven2
     
   
 
   
     
       false
-      shell.sourceforge.net
-      Repository at sourceforge.net
-      scp://shell.sourceforge.net/home/groups/m/ms/msgpack/htdocs/maven2/
+      msgpack.org
+      Repository at msgpack.org
+       
+      file:///Users/otakazuki/soft/website/maven2/
     
     
        true
-       shell.sourceforge.net
-       Repository Name
-       scp://shell.sourceforge.net/home/groups/m/ms/msgpack/htdocs/maven2-snapshot/
+       msgpack.org
+       Repository at msgpack.org
+       
+       file:///Users/otakazuki/soft/website/maven2/
     
   
 

From cba47b635a5a3bab2050fb4e0d099c23f8455867 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 27 Jul 2010 09:50:57 +0900
Subject: [PATCH 0596/1648] java: changed deploy path to
 ./target/website/maven2 directory.

---
 java/pom.xml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/java/pom.xml b/java/pom.xml
index 959d2df..44806d5 100755
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -108,15 +108,13 @@
       false
       msgpack.org
       Repository at msgpack.org
-       
-      file:///Users/otakazuki/soft/website/maven2/
+	  file://${project.build.directory}/website/maven2/
     
     
        true
        msgpack.org
        Repository at msgpack.org
-       
-       file:///Users/otakazuki/soft/website/maven2/
+	   file://${project.build.directory}/website/maven2/
     
   
 

From 6c91b862c9da56ce0914d430fea2cb388e32d5d5 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Wed, 28 Jul 2010 01:17:20 +0900
Subject: [PATCH 0597/1648] java: MessagePackObject implements Cloneable and
 MessagePackable interfaces

---
 .../java/org/msgpack/MessagePackObject.java   |  4 ++-
 java/src/main/java/org/msgpack/Packer.java    | 22 +++++++++++++++
 .../java/org/msgpack/object/ArrayType.java    | 28 +++++++++++++++++--
 .../msgpack/object/BigIntegerTypeIMPL.java    | 19 +++++++++++++
 .../java/org/msgpack/object/BooleanType.java  | 19 +++++++++++++
 .../org/msgpack/object/DoubleTypeIMPL.java    | 19 +++++++++++++
 .../org/msgpack/object/FloatTypeIMPL.java     | 19 +++++++++++++
 .../msgpack/object/LongIntegerTypeIMPL.java   | 19 +++++++++++++
 .../main/java/org/msgpack/object/MapType.java | 27 ++++++++++++++++++
 .../main/java/org/msgpack/object/NilType.java | 19 +++++++++++++
 .../main/java/org/msgpack/object/RawType.java | 23 ++++++++++++++-
 .../msgpack/object/ShortIntegerTypeIMPL.java  | 19 +++++++++++++
 12 files changed, 233 insertions(+), 4 deletions(-)

diff --git a/java/src/main/java/org/msgpack/MessagePackObject.java b/java/src/main/java/org/msgpack/MessagePackObject.java
index aba27e4..b1a6fab 100644
--- a/java/src/main/java/org/msgpack/MessagePackObject.java
+++ b/java/src/main/java/org/msgpack/MessagePackObject.java
@@ -22,7 +22,7 @@ import java.util.Set;
 import java.util.Map;
 import java.math.BigInteger;
 
-public abstract class MessagePackObject {
+public abstract class MessagePackObject implements Cloneable, MessagePackable {
 	public boolean isNull() {
 		return false;
 	}
@@ -126,5 +126,7 @@ public abstract class MessagePackObject {
 	public double doubleValue() {
 		throw new MessageTypeException("type error");
 	}
+
+	abstract public Object clone();
 }
 
diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index dd510f3..fbf7e35 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.List;
 import java.util.Map;
+import java.math.BigInteger;
 
 /**
  * Packer enables you to serialize objects into OutputStream.
@@ -194,6 +195,27 @@ public class Packer {
 		return this;
 	}
 
+	public Packer packBigInteger(BigInteger d) throws IOException {
+		if(d.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0) {
+			return packLong(d.longValue());
+		} else if(d.bitLength() <= 64) {
+			castBytes[0] = (byte)0xcf;
+			byte[] barray = d.toByteArray();
+			castBytes[1] = barray[0];
+			castBytes[2] = barray[1];
+			castBytes[3] = barray[2];
+			castBytes[4] = barray[3];
+			castBytes[5] = barray[4];
+			castBytes[6] = barray[5];
+			castBytes[7] = barray[6];
+			castBytes[8] = barray[7];
+			out.write(castBytes);
+			return this;
+		} else {
+			throw new MessageTypeException("can't BigInteger larger than 0xffffffffffffffff");
+		}
+	}
+
 	public Packer packFloat(float d) throws IOException {
 		castBytes[0] = (byte)0xca;
 		castBuffer.putFloat(1, d);
diff --git a/java/src/main/java/org/msgpack/object/ArrayType.java b/java/src/main/java/org/msgpack/object/ArrayType.java
index 06e9b16..694f53f 100644
--- a/java/src/main/java/org/msgpack/object/ArrayType.java
+++ b/java/src/main/java/org/msgpack/object/ArrayType.java
@@ -18,9 +18,8 @@
 package org.msgpack.object;
 
 import java.util.List;
-import java.util.Set;
-import java.util.Map;
 import java.util.Arrays;
+import java.io.IOException;
 import org.msgpack.*;
 
 public class ArrayType extends MessagePackObject {
@@ -44,5 +43,30 @@ public class ArrayType extends MessagePackObject {
 	public List asList() {
 		return Arrays.asList(array);
 	}
+
+	@Override
+	public void messagePack(Packer pk) throws IOException {
+		pk.packArray(array.length);
+		for(int i=0; i < array.length; i++) {
+			array[i].messagePack(pk);
+		}
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(obj.getClass() != getClass()) {
+			return false;
+		}
+		return Arrays.equals(((ArrayType)obj).array, array);
+	}
+
+	@Override
+	public Object clone() {
+		MessagePackObject[] copy = new MessagePackObject[array.length];
+		for(int i=0; i < array.length; i++) {
+			copy[i] = (MessagePackObject)array[i].clone();
+		}
+		return copy;
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
index 1ebb83d..f7c73ae 100644
--- a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
@@ -18,6 +18,7 @@
 package org.msgpack.object;
 
 import java.math.BigInteger;
+import java.io.IOException;
 import org.msgpack.*;
 
 class BigIntegerTypeIMPL extends IntegerType {
@@ -88,5 +89,23 @@ class BigIntegerTypeIMPL extends IntegerType {
 	public double doubleValue() {
 		return value.doubleValue();
 	}
+
+	@Override
+	public void messagePack(Packer pk) throws IOException {
+		pk.packBigInteger(value);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(obj.getClass() != getClass()) {
+			return false;
+		}
+		return ((BigIntegerTypeIMPL)obj).value.equals(value);
+	}
+
+	@Override
+	public Object clone() {
+		return new BigIntegerTypeIMPL(value);
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/object/BooleanType.java b/java/src/main/java/org/msgpack/object/BooleanType.java
index d272b6f..c6e4f30 100644
--- a/java/src/main/java/org/msgpack/object/BooleanType.java
+++ b/java/src/main/java/org/msgpack/object/BooleanType.java
@@ -17,6 +17,7 @@
 //
 package org.msgpack.object;
 
+import java.io.IOException;
 import org.msgpack.*;
 
 public class BooleanType extends MessagePackObject {
@@ -35,5 +36,23 @@ public class BooleanType extends MessagePackObject {
 	public boolean asBoolean() {
 		return value;
 	}
+
+	@Override
+	public void messagePack(Packer pk) throws IOException {
+		pk.packBoolean(value);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(obj.getClass() != getClass()) {
+			return false;
+		}
+		return ((BooleanType)obj).value == value;
+	}
+
+	@Override
+	public Object clone() {
+		return new BooleanType(value);
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
index 8bbc52a..dafe540 100644
--- a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
@@ -18,6 +18,7 @@
 package org.msgpack.object;
 
 import java.math.BigInteger;
+import java.io.IOException;
 import org.msgpack.*;
 
 class DoubleTypeIMPL extends FloatType {
@@ -67,5 +68,23 @@ class DoubleTypeIMPL extends FloatType {
 	public double doubleValue() {
 		return (double)value;
 	}
+
+	@Override
+	public void messagePack(Packer pk) throws IOException {
+		pk.packDouble(value);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(obj.getClass() != getClass()) {
+			return false;
+		}
+		return ((DoubleTypeIMPL)obj).value == value;
+	}
+
+	@Override
+	public Object clone() {
+		return new DoubleTypeIMPL(value);
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
index 8821640..234b2ad 100644
--- a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
@@ -18,6 +18,7 @@
 package org.msgpack.object;
 
 import java.math.BigInteger;
+import java.io.IOException;
 import org.msgpack.*;
 
 class FloatTypeIMPL extends FloatType {
@@ -66,5 +67,23 @@ class FloatTypeIMPL extends FloatType {
 	public double doubleValue() {
 		return (double)value;
 	}
+
+	@Override
+	public void messagePack(Packer pk) throws IOException {
+		pk.packFloat(value);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(obj.getClass() != getClass()) {
+			return false;
+		}
+		return ((FloatTypeIMPL)obj).value == value;
+	}
+
+	@Override
+	public Object clone() {
+		return new FloatTypeIMPL(value);
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
index c914e91..0ce22a2 100644
--- a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
@@ -18,6 +18,7 @@
 package org.msgpack.object;
 
 import java.math.BigInteger;
+import java.io.IOException;
 import org.msgpack.*;
 
 class LongIntegerTypeIMPL extends IntegerType {
@@ -85,5 +86,23 @@ class LongIntegerTypeIMPL extends IntegerType {
 	public double doubleValue() {
 		return (double)value;
 	}
+
+	@Override
+	public void messagePack(Packer pk) throws IOException {
+		pk.packLong(value);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(obj.getClass() != getClass()) {
+			return false;
+		}
+		return ((LongIntegerTypeIMPL)obj).value == value;
+	}
+
+	@Override
+	public Object clone() {
+		return new LongIntegerTypeIMPL(value);
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/object/MapType.java b/java/src/main/java/org/msgpack/object/MapType.java
index 359ebe6..d456e78 100644
--- a/java/src/main/java/org/msgpack/object/MapType.java
+++ b/java/src/main/java/org/msgpack/object/MapType.java
@@ -19,6 +19,8 @@ package org.msgpack.object;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Arrays;
+import java.io.IOException;
 import org.msgpack.*;
 
 public class MapType extends MessagePackObject {
@@ -44,5 +46,30 @@ public class MapType extends MessagePackObject {
 		}
 		return m;
 	}
+
+	@Override
+	public void messagePack(Packer pk) throws IOException {
+		pk.packMap(map.length / 2);
+		for(int i=0; i < map.length; i++) {
+			map[i].messagePack(pk);
+		}
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(obj.getClass() != getClass()) {
+			return false;
+		}
+		return Arrays.equals(((MapType)obj).map, map);
+	}
+
+	@Override
+	public Object clone() {
+		MessagePackObject[] copy = new MessagePackObject[map.length];
+		for(int i=0; i < map.length; i++) {
+			copy[i] = (MessagePackObject)map[i].clone();
+		}
+		return copy;
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/object/NilType.java b/java/src/main/java/org/msgpack/object/NilType.java
index c36ff05..7a46376 100644
--- a/java/src/main/java/org/msgpack/object/NilType.java
+++ b/java/src/main/java/org/msgpack/object/NilType.java
@@ -17,6 +17,7 @@
 //
 package org.msgpack.object;
 
+import java.io.IOException;
 import org.msgpack.*;
 
 public class NilType extends MessagePackObject {
@@ -24,5 +25,23 @@ public class NilType extends MessagePackObject {
 	public boolean isNull() {
 		return true;
 	}
+
+	@Override
+	public void messagePack(Packer pk) throws IOException {
+		pk.packNil();
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(obj.getClass() != getClass()) {
+			return false;
+		}
+		return true;
+	}
+
+	@Override
+	public Object clone() {
+		return new NilType();
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/object/RawType.java b/java/src/main/java/org/msgpack/object/RawType.java
index 107ba27..18a419d 100644
--- a/java/src/main/java/org/msgpack/object/RawType.java
+++ b/java/src/main/java/org/msgpack/object/RawType.java
@@ -17,9 +17,11 @@
 //
 package org.msgpack.object;
 
+import java.util.Arrays;
+import java.io.IOException;
 import org.msgpack.*;
 
-class RawType extends MessagePackObject {
+public class RawType extends MessagePackObject {
 	private byte[] bytes;
 
 	public RawType(byte[] bytes) {
@@ -44,5 +46,24 @@ class RawType extends MessagePackObject {
 			throw new MessageTypeException("type error");
 		}
 	}
+
+	@Override
+	public void messagePack(Packer pk) throws IOException {
+		pk.packRaw(bytes.length);
+		pk.packRawBody(bytes);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(obj.getClass() != getClass()) {
+			return false;
+		}
+		return ((RawType)obj).bytes.equals(bytes);
+	}
+
+	@Override
+	public Object clone() {
+		return new RawType((byte[])bytes.clone());
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java
index a725950..83a4daf 100644
--- a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java
@@ -18,6 +18,7 @@
 package org.msgpack.object;
 
 import java.math.BigInteger;
+import java.io.IOException;
 import org.msgpack.*;
 
 class ShortIntegerTypeIMPL extends IntegerType {
@@ -87,5 +88,23 @@ class ShortIntegerTypeIMPL extends IntegerType {
 	public double doubleValue() {
 		return (double)value;
 	}
+
+	@Override
+	public void messagePack(Packer pk) throws IOException {
+		pk.packInt(value);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(obj.getClass() != getClass()) {
+			return false;
+		}
+		return ((ShortIntegerTypeIMPL)obj).value == value;
+	}
+
+	@Override
+	public Object clone() {
+		return new ShortIntegerTypeIMPL(value);
+	}
 }
 

From d3bb37d113575e8b36f44c06ac9eeb6b406aa298 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 10 Aug 2010 14:11:25 +0900
Subject: [PATCH 0598/1648] java: fixes MapSchema

---
 java/src/main/java/org/msgpack/schema/MapSchema.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/src/main/java/org/msgpack/schema/MapSchema.java b/java/src/main/java/org/msgpack/schema/MapSchema.java
index 339a5c2..2e09af3 100644
--- a/java/src/main/java/org/msgpack/schema/MapSchema.java
+++ b/java/src/main/java/org/msgpack/schema/MapSchema.java
@@ -72,7 +72,7 @@ public class MapSchema extends Schema implements IMapSchema {
 			dest.put((K)keySchema.convert(e.getKey()),
 					(V)valueSchema.convert(e.getValue()));
 		}
-		return (Map)d;
+		return dest;
 	}
 
 	@Override

From 057f73a73e0c3ddabe92f1cd2c394fe4afa13514 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 10 Aug 2010 14:11:44 +0900
Subject: [PATCH 0599/1648] java: implements MessagePackObject::hashCode()

---
 java/src/main/java/org/msgpack/object/ArrayType.java     | 5 +++++
 .../main/java/org/msgpack/object/BigIntegerTypeIMPL.java | 5 +++++
 java/src/main/java/org/msgpack/object/BooleanType.java   | 9 +++++++++
 .../src/main/java/org/msgpack/object/DoubleTypeIMPL.java | 6 ++++++
 java/src/main/java/org/msgpack/object/FloatTypeIMPL.java | 5 +++++
 .../java/org/msgpack/object/LongIntegerTypeIMPL.java     | 5 +++++
 java/src/main/java/org/msgpack/object/MapType.java       | 5 +++++
 java/src/main/java/org/msgpack/object/NilType.java       | 5 +++++
 java/src/main/java/org/msgpack/object/RawType.java       | 5 +++++
 .../java/org/msgpack/object/ShortIntegerTypeIMPL.java    | 5 +++++
 10 files changed, 55 insertions(+)

diff --git a/java/src/main/java/org/msgpack/object/ArrayType.java b/java/src/main/java/org/msgpack/object/ArrayType.java
index 694f53f..350ce32 100644
--- a/java/src/main/java/org/msgpack/object/ArrayType.java
+++ b/java/src/main/java/org/msgpack/object/ArrayType.java
@@ -60,6 +60,11 @@ public class ArrayType extends MessagePackObject {
 		return Arrays.equals(((ArrayType)obj).array, array);
 	}
 
+	@Override
+	public int hashCode() {
+		return array.hashCode();
+	}
+
 	@Override
 	public Object clone() {
 		MessagePackObject[] copy = new MessagePackObject[array.length];
diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
index f7c73ae..fd517e7 100644
--- a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
@@ -103,6 +103,11 @@ class BigIntegerTypeIMPL extends IntegerType {
 		return ((BigIntegerTypeIMPL)obj).value.equals(value);
 	}
 
+	@Override
+	public int hashCode() {
+		return value.hashCode();
+	}
+
 	@Override
 	public Object clone() {
 		return new BigIntegerTypeIMPL(value);
diff --git a/java/src/main/java/org/msgpack/object/BooleanType.java b/java/src/main/java/org/msgpack/object/BooleanType.java
index c6e4f30..1d12c1c 100644
--- a/java/src/main/java/org/msgpack/object/BooleanType.java
+++ b/java/src/main/java/org/msgpack/object/BooleanType.java
@@ -50,6 +50,15 @@ public class BooleanType extends MessagePackObject {
 		return ((BooleanType)obj).value == value;
 	}
 
+	@Override
+	public int hashCode() {
+		if(value) {
+			return 1231;
+		} else {
+			return 1237;
+		}
+	}
+
 	@Override
 	public Object clone() {
 		return new BooleanType(value);
diff --git a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
index dafe540..b47a709 100644
--- a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
@@ -82,6 +82,12 @@ class DoubleTypeIMPL extends FloatType {
 		return ((DoubleTypeIMPL)obj).value == value;
 	}
 
+	@Override
+	public int hashCode() {
+		long v = Double.doubleToLongBits(value);
+		return (int)(v^(v>>>32));
+	}
+
 	@Override
 	public Object clone() {
 		return new DoubleTypeIMPL(value);
diff --git a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
index 234b2ad..1d79961 100644
--- a/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/FloatTypeIMPL.java
@@ -81,6 +81,11 @@ class FloatTypeIMPL extends FloatType {
 		return ((FloatTypeIMPL)obj).value == value;
 	}
 
+	@Override
+	public int hashCode() {
+		return Float.floatToIntBits(value);
+	}
+
 	@Override
 	public Object clone() {
 		return new FloatTypeIMPL(value);
diff --git a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
index 0ce22a2..940ab6f 100644
--- a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
@@ -100,6 +100,11 @@ class LongIntegerTypeIMPL extends IntegerType {
 		return ((LongIntegerTypeIMPL)obj).value == value;
 	}
 
+	@Override
+	public int hashCode() {
+		return (int)(value^(value>>>32));
+	}
+
 	@Override
 	public Object clone() {
 		return new LongIntegerTypeIMPL(value);
diff --git a/java/src/main/java/org/msgpack/object/MapType.java b/java/src/main/java/org/msgpack/object/MapType.java
index d456e78..619d388 100644
--- a/java/src/main/java/org/msgpack/object/MapType.java
+++ b/java/src/main/java/org/msgpack/object/MapType.java
@@ -63,6 +63,11 @@ public class MapType extends MessagePackObject {
 		return Arrays.equals(((MapType)obj).map, map);
 	}
 
+	@Override
+	public int hashCode() {
+		return map.hashCode();
+	}
+
 	@Override
 	public Object clone() {
 		MessagePackObject[] copy = new MessagePackObject[map.length];
diff --git a/java/src/main/java/org/msgpack/object/NilType.java b/java/src/main/java/org/msgpack/object/NilType.java
index 7a46376..ece62f0 100644
--- a/java/src/main/java/org/msgpack/object/NilType.java
+++ b/java/src/main/java/org/msgpack/object/NilType.java
@@ -39,6 +39,11 @@ public class NilType extends MessagePackObject {
 		return true;
 	}
 
+	@Override
+	public int hashCode() {
+		return 0;
+	}
+
 	@Override
 	public Object clone() {
 		return new NilType();
diff --git a/java/src/main/java/org/msgpack/object/RawType.java b/java/src/main/java/org/msgpack/object/RawType.java
index 18a419d..3a39486 100644
--- a/java/src/main/java/org/msgpack/object/RawType.java
+++ b/java/src/main/java/org/msgpack/object/RawType.java
@@ -61,6 +61,11 @@ public class RawType extends MessagePackObject {
 		return ((RawType)obj).bytes.equals(bytes);
 	}
 
+	@Override
+	public int hashCode() {
+		return bytes.hashCode();
+	}
+
 	@Override
 	public Object clone() {
 		return new RawType((byte[])bytes.clone());
diff --git a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java
index 83a4daf..60e92b8 100644
--- a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java
@@ -102,6 +102,11 @@ class ShortIntegerTypeIMPL extends IntegerType {
 		return ((ShortIntegerTypeIMPL)obj).value == value;
 	}
 
+	@Override
+	public int hashCode() {
+		return value;
+	}
+
 	@Override
 	public Object clone() {
 		return new ShortIntegerTypeIMPL(value);

From 8c67087a154da0e7cdc32c0b676d2956ce1d0f47 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Wed, 18 Aug 2010 16:32:42 +0900
Subject: [PATCH 0600/1648] java: adds MessagePackObject.bigIntegerValue(),
 asBigInteger() and equals()

---
 .../main/java/org/msgpack/MessagePackObject.java  |  4 ++++
 .../org/msgpack/object/BigIntegerTypeIMPL.java    | 15 +++++++++++++++
 .../java/org/msgpack/object/DoubleTypeIMPL.java   |  5 +++++
 .../org/msgpack/object/LongIntegerTypeIMPL.java   | 15 +++++++++++++++
 .../org/msgpack/object/ShortIntegerTypeIMPL.java  | 10 ++++++++++
 5 files changed, 49 insertions(+)

diff --git a/java/src/main/java/org/msgpack/MessagePackObject.java b/java/src/main/java/org/msgpack/MessagePackObject.java
index b1a6fab..6181f7a 100644
--- a/java/src/main/java/org/msgpack/MessagePackObject.java
+++ b/java/src/main/java/org/msgpack/MessagePackObject.java
@@ -119,6 +119,10 @@ public abstract class MessagePackObject implements Cloneable, MessagePackable {
 		throw new MessageTypeException("type error");
 	}
 
+	public BigInteger bigIntegerValue() {
+		throw new MessageTypeException("type error");
+	}
+
 	public float floatValue() {
 		throw new MessageTypeException("type error");
 	}
diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
index fd517e7..7b060ee 100644
--- a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
@@ -60,6 +60,11 @@ class BigIntegerTypeIMPL extends IntegerType {
 		return value.longValue();
 	}
 
+	@Override
+	public BigInteger asBigInteger() {
+		return value;
+	}
+
 	@Override
 	public byte byteValue() {
 		return value.byteValue();
@@ -80,6 +85,11 @@ class BigIntegerTypeIMPL extends IntegerType {
 		return value.longValue();
 	}
 
+	@Override
+	public BigInteger bigIntegerValue() {
+		return value;
+	}
+
 	@Override
 	public float floatValue() {
 		return value.floatValue();
@@ -98,6 +108,11 @@ class BigIntegerTypeIMPL extends IntegerType {
 	@Override
 	public boolean equals(Object obj) {
 		if(obj.getClass() != getClass()) {
+			if(obj.getClass() == ShortIntegerTypeIMPL.class) {
+				return BigInteger.valueOf((long)((ShortIntegerTypeIMPL)obj).shortValue()).equals(value);
+			} else if(obj.getClass() == LongIntegerTypeIMPL.class) {
+				return BigInteger.valueOf(((LongIntegerTypeIMPL)obj).longValue()).equals(value);
+			}
 			return false;
 		}
 		return ((BigIntegerTypeIMPL)obj).value.equals(value);
diff --git a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
index b47a709..fd38089 100644
--- a/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/DoubleTypeIMPL.java
@@ -59,6 +59,11 @@ class DoubleTypeIMPL extends FloatType {
 		return (long)value;
 	}
 
+	@Override
+	public BigInteger bigIntegerValue() {
+		return BigInteger.valueOf((long)value);
+	}
+
 	@Override
 	public float floatValue() {
 		return (float)value;
diff --git a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
index 940ab6f..3928a29 100644
--- a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
@@ -57,6 +57,11 @@ class LongIntegerTypeIMPL extends IntegerType {
 		return value;
 	}
 
+	@Override
+	public BigInteger asBigInteger() {
+		return BigInteger.valueOf(value);
+	}
+
 	@Override
 	public byte byteValue() {
 		return (byte)value;
@@ -77,6 +82,11 @@ class LongIntegerTypeIMPL extends IntegerType {
 		return (long)value;
 	}
 
+	@Override
+	public BigInteger bigIntegerValue() {
+		return BigInteger.valueOf(value);
+	}
+
 	@Override
 	public float floatValue() {
 		return (float)value;
@@ -95,6 +105,11 @@ class LongIntegerTypeIMPL extends IntegerType {
 	@Override
 	public boolean equals(Object obj) {
 		if(obj.getClass() != getClass()) {
+			if(obj.getClass() == ShortIntegerTypeIMPL.class) {
+				return value == ((ShortIntegerTypeIMPL)obj).longValue();
+			} else if(obj.getClass() == BigIntegerTypeIMPL.class) {
+				return (long)value == ((BigIntegerTypeIMPL)obj).longValue();
+			}
 			return false;
 		}
 		return ((LongIntegerTypeIMPL)obj).value == value;
diff --git a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java
index 60e92b8..dbed426 100644
--- a/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/ShortIntegerTypeIMPL.java
@@ -79,6 +79,11 @@ class ShortIntegerTypeIMPL extends IntegerType {
 		return (long)value;
 	}
 
+	@Override
+	public BigInteger bigIntegerValue() {
+		return BigInteger.valueOf((long)value);
+	}
+
 	@Override
 	public float floatValue() {
 		return (float)value;
@@ -97,6 +102,11 @@ class ShortIntegerTypeIMPL extends IntegerType {
 	@Override
 	public boolean equals(Object obj) {
 		if(obj.getClass() != getClass()) {
+			if(obj.getClass() == LongIntegerTypeIMPL.class) {
+				return (long)value == ((LongIntegerTypeIMPL)obj).longValue();
+			} else if(obj.getClass() == BigIntegerTypeIMPL.class) {
+				return ((BigIntegerTypeIMPL)obj).bigIntegerValue().equals(BigInteger.valueOf((long)value));
+			}
 			return false;
 		}
 		return ((ShortIntegerTypeIMPL)obj).value == value;

From 8b79e6d3c72a02f4dc039799e3cd370c06e966b0 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Wed, 18 Aug 2010 18:10:30 +0900
Subject: [PATCH 0601/1648] java: uses MessagePackObject instead of Object for
 type of deserialized objects

---
 .../org/msgpack/BufferedUnpackerImpl.java     |   8 +-
 .../org/msgpack/MessageTypeException.java     |  10 -
 java/src/main/java/org/msgpack/Packer.java    |   6 -
 java/src/main/java/org/msgpack/Schema.java    |  78 ----
 .../main/java/org/msgpack/UnpackIterator.java |   4 +-
 .../main/java/org/msgpack/UnpackResult.java   |   6 +-
 java/src/main/java/org/msgpack/Unpacker.java  |  30 +-
 .../main/java/org/msgpack/UnpackerImpl.java   | 132 ++----
 .../org/msgpack/schema/BooleanSchema.java     |  64 ---
 .../org/msgpack/schema/ByteArraySchema.java   |  97 ----
 .../java/org/msgpack/schema/ByteSchema.java   |  96 ----
 .../org/msgpack/schema/ClassGenerator.java    | 244 ----------
 .../java/org/msgpack/schema/ClassSchema.java  |  91 ----
 .../java/org/msgpack/schema/DoubleSchema.java |  74 ---
 .../java/org/msgpack/schema/FieldSchema.java  |  43 --
 .../java/org/msgpack/schema/FloatSchema.java  |  74 ---
 .../msgpack/schema/GenericClassSchema.java    |  87 ----
 .../org/msgpack/schema/GenericSchema.java     | 129 ------
 .../java/org/msgpack/schema/IArraySchema.java |  26 --
 .../java/org/msgpack/schema/IMapSchema.java   |  27 --
 .../java/org/msgpack/schema/IntSchema.java    |  96 ----
 .../java/org/msgpack/schema/ListSchema.java   | 111 -----
 .../java/org/msgpack/schema/LongSchema.java   |  80 ----
 .../java/org/msgpack/schema/MapSchema.java    | 106 -----
 .../msgpack/schema/ReflectionClassSchema.java |  64 ---
 .../org/msgpack/schema/SSchemaParser.java     | 264 -----------
 .../java/org/msgpack/schema/SetSchema.java    | 115 -----
 .../java/org/msgpack/schema/ShortSchema.java  |  93 ----
 .../msgpack/schema/SpecificClassSchema.java   | 122 -----
 .../java/org/msgpack/schema/StringSchema.java | 102 ----
 .../test/java/org/msgpack/TestPackUnpack.java | 435 +++++++++---------
 31 files changed, 270 insertions(+), 2644 deletions(-)
 delete mode 100644 java/src/main/java/org/msgpack/Schema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/BooleanSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/ByteArraySchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/ByteSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/ClassGenerator.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/ClassSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/DoubleSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/FieldSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/FloatSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/GenericClassSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/GenericSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/IArraySchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/IMapSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/IntSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/ListSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/LongSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/MapSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/SSchemaParser.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/SetSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/ShortSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/SpecificClassSchema.java
 delete mode 100644 java/src/main/java/org/msgpack/schema/StringSchema.java

diff --git a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java
index f4ed35b..9496238 100644
--- a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java
+++ b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java
@@ -47,7 +47,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl {
 			offset = noffset;
 		} while(!super.isFinished());
 
-		Object obj = super.getData();
+		MessagePackObject obj = super.getData();
 		super.reset();
 		result.done(obj);
 
@@ -198,7 +198,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl {
 			{
 				long o = castBuffer.getLong(0);
 				if(o < 0) {
-					// FIXME
+					// FIXME unpackBigInteger
 					throw new MessageTypeException("uint 64 bigger than 0x7fffffff is not supported");
 				}
 				advance(9);
@@ -231,6 +231,8 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl {
 		}
 	}
 
+	// FIXME unpackBigInteger
+
 	final float unpackFloat() throws IOException, MessageTypeException {
 		more(1);
 		int b = buffer[offset];
@@ -414,7 +416,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl {
 		return s;
 	}
 
-	final Object unpackObject() throws IOException {
+	final MessagePackObject unpackObject() throws IOException {
 		UnpackResult result = new UnpackResult();
 		if(!next(result)) {
 			super.reset();
diff --git a/java/src/main/java/org/msgpack/MessageTypeException.java b/java/src/main/java/org/msgpack/MessageTypeException.java
index 0fa37b7..698ef6d 100644
--- a/java/src/main/java/org/msgpack/MessageTypeException.java
+++ b/java/src/main/java/org/msgpack/MessageTypeException.java
@@ -23,15 +23,5 @@ public class MessageTypeException extends RuntimeException {
 	public MessageTypeException(String s) {
 		super(s);
 	}
-
-	public static MessageTypeException invalidConvert(Object from, Schema to) {
-		return new MessageTypeException(from.getClass().getName()+" cannot be convert to "+to.getExpression());
-	}
-
-	/* FIXME
-	public static MessageTypeException schemaMismatch(Schema to) {
-		return new MessageTypeException("schema mismatch "+to.getExpression());
-	}
-	*/
 }
 
diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index fbf7e35..98af3d6 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -308,12 +308,6 @@ public class Packer {
 	}
 
 
-	public Packer packWithSchema(Object o, Schema s) throws IOException {
-		s.pack(this, o);
-		return this;
-	}
-
-
 	public Packer packString(String s) throws IOException {
 		byte[] b = ((String)s).getBytes("UTF-8");
 		packRaw(b.length);
diff --git a/java/src/main/java/org/msgpack/Schema.java b/java/src/main/java/org/msgpack/Schema.java
deleted file mode 100644
index 25e10f9..0000000
--- a/java/src/main/java/org/msgpack/Schema.java
+++ /dev/null
@@ -1,78 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack;
-
-import java.io.Writer;
-import java.io.IOException;
-import org.msgpack.schema.SSchemaParser;
-//import org.msgpack.schema.ClassGenerator;
-
-public abstract class Schema {
-	public Schema() { }
-
-	public abstract String getClassName();
-	public abstract String getExpression();
-
-	public static Schema parse(String source) {
-		return SSchemaParser.parse(source);
-	}
-
-	public static Schema load(String source) {
-		return SSchemaParser.load(source);
-	}
-
-	public abstract void pack(Packer pk, Object obj) throws IOException;
-	public abstract Object convert(Object obj) throws MessageTypeException;
-
-	public Object createFromNil() {
-		return null;
-	}
-
-	public Object createFromBoolean(boolean v) {
-		throw new MessageTypeException("type error");
-	}
-
-	public Object createFromByte(byte v) {
-		throw new MessageTypeException("type error");
-	}
-
-	public Object createFromShort(short v) {
-		throw new MessageTypeException("type error");
-	}
-
-	public Object createFromInt(int v) {
-		throw new MessageTypeException("type error");
-	}
-
-	public Object createFromLong(long v) {
-		throw new MessageTypeException("type error");
-	}
-
-	public Object createFromFloat(float v) {
-		throw new MessageTypeException("type error");
-	}
-
-	public Object createFromDouble(double v) {
-		throw new MessageTypeException("type error");
-	}
-
-	public Object createFromRaw(byte[] b, int offset, int length) {
-		throw new MessageTypeException("type error");
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/UnpackIterator.java b/java/src/main/java/org/msgpack/UnpackIterator.java
index f17e229..8c778b6 100644
--- a/java/src/main/java/org/msgpack/UnpackIterator.java
+++ b/java/src/main/java/org/msgpack/UnpackIterator.java
@@ -21,7 +21,7 @@ import java.io.IOException;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
 
-public class UnpackIterator extends UnpackResult implements Iterator {
+public class UnpackIterator extends UnpackResult implements Iterator {
 	private Unpacker pac;
 
 	UnpackIterator(Unpacker pac) {
@@ -38,7 +38,7 @@ public class UnpackIterator extends UnpackResult implements Iterator {
 		}
 	}
 
-	public Object next() {
+	public MessagePackObject next() {
 		if(!finished && !hasNext()) {
 			throw new NoSuchElementException();
 		}
diff --git a/java/src/main/java/org/msgpack/UnpackResult.java b/java/src/main/java/org/msgpack/UnpackResult.java
index cec18a1..bb981c1 100644
--- a/java/src/main/java/org/msgpack/UnpackResult.java
+++ b/java/src/main/java/org/msgpack/UnpackResult.java
@@ -19,13 +19,13 @@ package org.msgpack;
 
 public class UnpackResult {
 	protected boolean finished = false;
-	protected Object data = null;
+	protected MessagePackObject data = null;
 
 	public boolean isFinished() {
 		return finished;
 	}
 
-	public Object getData() {
+	public MessagePackObject getData() {
 		return data;
 	}
 
@@ -34,7 +34,7 @@ public class UnpackResult {
 		data = null;
 	}
 
-	void done(Object obj) {
+	void done(MessagePackObject obj) {
 		finished = true;
 		data = obj;
 	}
diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index 32bab64..3a95243 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -26,8 +26,8 @@ import java.nio.ByteBuffer;
 /**
  * Unpacker enables you to deserialize objects from stream.
  *
- * Unpacker provides Buffered API, Unbuffered API, Schema API
- * and Direct Conversion API.
+ * Unpacker provides Buffered API, Unbuffered API and
+ * Direct Conversion API.
  *
  * Buffered API uses the internal buffer of the Unpacker.
  * Following code uses Buffered API with an InputStream:
@@ -39,7 +39,7 @@ import java.nio.ByteBuffer;
  * UnpackResult result = pac.next();
  *
  * // use an iterator.
- * for(Object obj : pac) {
+ * for(MessagePackObject obj : pac) {
  *   // use MessageConvertable interface to convert the
  *   // the generic object to the specific type.
  * }
@@ -56,7 +56,7 @@ import java.nio.ByteBuffer;
  * pac.feed(input_bytes);
  *
  * // use next() method or iterators.
- * for(Object obj : pac) {
+ * for(MessagePackObject obj : pac) {
  *   // ...
  * }
  * 
@@ -75,7 +75,7 @@ import java.nio.ByteBuffer;
  * System.in.read(pac.getBuffer(), pac.getBufferOffset(), pac.getBufferCapacity());
  *
  * // use next() method or iterators.
- * for(Object obj : pac) {
+ * for(MessagePackObject obj : pac) {
  *     // ...
  * }
  * 
@@ -96,12 +96,12 @@ import java.nio.ByteBuffer;
  *
  * // take out object if deserialized object is ready.
  * if(pac.isFinished()) {
- *     Object obj = pac.getData();
+ *     MessagePackObject obj = pac.getData();
  *     // ...
  * }
  * 
  */
-public class Unpacker implements Iterable {
+public class Unpacker implements Iterable {
 
 	// buffer:
 	// +---------------------------------------------+
@@ -170,16 +170,6 @@ public class Unpacker implements Iterable {
 		this.stream = stream;
 	}
 
-	/**
-	 * Sets schema to convert deserialized object into specific type.
-	 * Default schema is {@link GenericSchema} that leaves objects for generic type. Use {@link MessageConvertable#messageConvert(Object)} method to convert the generic object.
-	 * @param s schem to use
-	 */
-	public Unpacker useSchema(Schema s) {
-		impl.setSchema(s);
-		return this;
-	}
-
 
 	/**
 	 * Gets the input stream.
@@ -255,7 +245,7 @@ public class Unpacker implements Iterable {
 	/**
 	 * Returns the iterator that calls {@link next()} method repeatedly.
 	 */
-	public Iterator iterator() {
+	public Iterator iterator() {
 		return new UnpackIterator(this);
 	}
 
@@ -387,7 +377,7 @@ public class Unpacker implements Iterable {
 	/**
 	 * Gets the object deserialized by {@link execute(byte[])} method.
 	 */
-	public Object getData() {
+	public MessagePackObject getData() {
 		return impl.getData();
 	}
 
@@ -557,7 +547,7 @@ public class Unpacker implements Iterable {
 	 * Gets one {@code Object} value from the buffer.
 	 * This method calls {@link fill()} method if needed.
 	 */
-	final public Object unpackObject() throws IOException {
+	final public MessagePackObject unpackObject() throws IOException {
 		return impl.unpackObject();
 	}
 
diff --git a/java/src/main/java/org/msgpack/UnpackerImpl.java b/java/src/main/java/org/msgpack/UnpackerImpl.java
index 10cf5f0..f004e6c 100644
--- a/java/src/main/java/org/msgpack/UnpackerImpl.java
+++ b/java/src/main/java/org/msgpack/UnpackerImpl.java
@@ -18,11 +18,7 @@
 package org.msgpack;
 
 import java.nio.ByteBuffer;
-//import java.math.BigInteger;
-import org.msgpack.*;
-import org.msgpack.schema.GenericSchema;
-import org.msgpack.schema.IMapSchema;
-import org.msgpack.schema.IArraySchema;
+import org.msgpack.object.*;
 
 public class UnpackerImpl {
 	static final int CS_HEADER      = 0x00;
@@ -55,30 +51,19 @@ public class UnpackerImpl {
 	private int[]    stack_ct       = new int[MAX_STACK_SIZE];
 	private int[]    stack_count    = new int[MAX_STACK_SIZE];
 	private Object[] stack_obj      = new Object[MAX_STACK_SIZE];
-	private Schema[] stack_schema   = new Schema[MAX_STACK_SIZE];
 	private int top_ct;
 	private int top_count;
 	private Object top_obj;
-	private Schema top_schema;
 	private ByteBuffer castBuffer   = ByteBuffer.allocate(8);
 	private boolean finished = false;
-	private Object data = null;
-
-	private static final Schema GENERIC_SCHEMA = new GenericSchema();
-	private Schema rootSchema;
+	private MessagePackObject data = null;
 
 	public UnpackerImpl()
 	{
-		setSchema(GENERIC_SCHEMA);
-	}
-
-	public void setSchema(Schema schema)
-	{
-		this.rootSchema = schema;
 		reset();
 	}
 
-	public final Object getData()
+	public final MessagePackObject getData()
 	{
 		return data;
 	}
@@ -94,7 +79,6 @@ public class UnpackerImpl {
 		top_ct = 0;
 		top_count = 0;
 		top_obj = null;
-		top_schema = rootSchema;
 	}
 
 	public final void reset()
@@ -127,20 +111,20 @@ public class UnpackerImpl {
 	
 					if((b & 0x80) == 0) {  // Positive Fixnum
 						//System.out.println("positive fixnum "+b);
-						obj = top_schema.createFromByte((byte)b);
+						obj = IntegerType.create((byte)b);
 						break _push;
 					}
 	
 					if((b & 0xe0) == 0xe0) {  // Negative Fixnum
 						//System.out.println("negative fixnum "+b);
-						obj = top_schema.createFromByte((byte)b);
+						obj = IntegerType.create((byte)b);
 						break _push;
 					}
 	
 					if((b & 0xe0) == 0xa0) {  // FixRaw
 						trail = b & 0x1f;
 						if(trail == 0) {
-							obj = top_schema.createFromRaw(new byte[0], 0, 0);
+							obj = new RawType(new byte[0]);
 							break _push;
 						}
 						cs = ACS_RAW_VALUE;
@@ -151,25 +135,20 @@ public class UnpackerImpl {
 						if(top >= MAX_STACK_SIZE) {
 							throw new UnpackException("parse error");
 						}
-						if(!(top_schema instanceof IArraySchema)) {
-							throw new RuntimeException("type error");
-						}
 						count = b & 0x0f;
 						//System.out.println("fixarray count:"+count);
-						obj = new Object[count];
+						obj = new MessagePackObject[count];
 						if(count == 0) {
-							obj = ((IArraySchema)top_schema).createFromArray((Object[])obj);
+							obj = new ArrayType((MessagePackObject[])obj);
 							break _push;
 						}
 						++top;
 						stack_obj[top]    = top_obj;
 						stack_ct[top]     = top_ct;
 						stack_count[top]  = top_count;
-						stack_schema[top] = top_schema;
 						top_obj    = obj;
 						top_ct     = CT_ARRAY_ITEM;
 						top_count  = count;
-						top_schema = ((IArraySchema)top_schema).getElementSchema(0);
 						break _header_again;
 					}
 	
@@ -177,13 +156,10 @@ public class UnpackerImpl {
 						if(top >= MAX_STACK_SIZE) {
 							throw new UnpackException("parse error");
 						}
-						if(!(top_schema instanceof IMapSchema)) {
-							throw new RuntimeException("type error");
-						}
 						count = b & 0x0f;
-						obj = new Object[count*2];
+						obj = new MessagePackObject[count*2];
 						if(count == 0) {
-							obj = ((IMapSchema)top_schema).createFromMap((Object[])obj);
+							obj = new MapType((MessagePackObject[])obj);
 							break _push;
 						}
 						//System.out.println("fixmap count:"+count);
@@ -191,23 +167,21 @@ public class UnpackerImpl {
 						stack_obj[top]    = top_obj;
 						stack_ct[top]     = top_ct;
 						stack_count[top]  = top_count;
-						stack_schema[top] = top_schema;
 						top_obj    = obj;
 						top_ct     = CT_MAP_KEY;
 						top_count  = count;
-						top_schema = ((IMapSchema)top_schema).getKeySchema();
 						break _header_again;
 					}
 	
 					switch(b & 0xff) {    // FIXME
 					case 0xc0:  // nil
-						obj = top_schema.createFromNil();
+						obj = new NilType();
 						break _push;
 					case 0xc2:  // false
-						obj = top_schema.createFromBoolean(false);
+						obj = new BooleanType(false);
 						break _push;
 					case 0xc3:  // true
-						obj = top_schema.createFromBoolean(true);
+						obj = new BooleanType(true);
 						break _push;
 					case 0xca:  // float
 					case 0xcb:  // double
@@ -251,13 +225,13 @@ public class UnpackerImpl {
 					case CS_FLOAT:
 						castBuffer.rewind();
 						castBuffer.put(src, n, 4);
-						obj = top_schema.createFromFloat( castBuffer.getFloat(0) );
+						obj = FloatType.create( castBuffer.getFloat(0) );
 						//System.out.println("float "+obj);
 						break _push;
 					case CS_DOUBLE:
 						castBuffer.rewind();
 						castBuffer.put(src, n, 8);
-						obj = top_schema.createFromDouble( castBuffer.getDouble(0) );
+						obj = FloatType.create( castBuffer.getDouble(0) );
 						//System.out.println("double "+obj);
 						break _push;
 					case CS_UINT_8:
@@ -265,7 +239,7 @@ public class UnpackerImpl {
 						//System.out.println(src[n]);
 						//System.out.println(src[n+1]);
 						//System.out.println(src[n-1]);
-						obj = top_schema.createFromShort( (short)((src[n]) & 0xff) );
+						obj = IntegerType.create( (short)((src[n]) & 0xff) );
 						//System.out.println("uint8 "+obj);
 						break _push;
 					case CS_UINT_16:
@@ -273,13 +247,13 @@ public class UnpackerImpl {
 						//System.out.println(src[n+1]);
 						castBuffer.rewind();
 						castBuffer.put(src, n, 2);
-						obj = top_schema.createFromInt( ((int)castBuffer.getShort(0)) & 0xffff );
+						obj = IntegerType.create( ((int)castBuffer.getShort(0)) & 0xffff );
 						//System.out.println("uint 16 "+obj);
 						break _push;
 					case CS_UINT_32:
 						castBuffer.rewind();
 						castBuffer.put(src, n, 4);
-						obj = top_schema.createFromLong( ((long)castBuffer.getInt(0)) & 0xffffffffL );
+						obj = IntegerType.create( ((long)castBuffer.getInt(0)) & 0xffffffffL );
 						//System.out.println("uint 32 "+obj);
 						break _push;
 					case CS_UINT_64:
@@ -292,34 +266,34 @@ public class UnpackerImpl {
 								//obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31);
 								throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported");
 							} else {
-								obj = top_schema.createFromLong( o );
+								obj = IntegerType.create(o);
 							}
 						}
 						break _push;
 					case CS_INT_8:
-						obj = top_schema.createFromByte(  src[n] );
+						obj = IntegerType.create( src[n] );
 						break _push;
 					case CS_INT_16:
 						castBuffer.rewind();
 						castBuffer.put(src, n, 2);
-						obj = top_schema.createFromShort( castBuffer.getShort(0) );
+						obj = IntegerType.create( castBuffer.getShort(0) );
 						break _push;
 					case CS_INT_32:
 						castBuffer.rewind();
 						castBuffer.put(src, n, 4);
-						obj = top_schema.createFromInt( castBuffer.getInt(0) );
+						obj = IntegerType.create( castBuffer.getInt(0) );
 						break _push;
 					case CS_INT_64:
 						castBuffer.rewind();
 						castBuffer.put(src, n, 8);
-						obj = top_schema.createFromLong( castBuffer.getLong(0) );
+						obj = IntegerType.create( castBuffer.getLong(0) );
 						break _push;
 					case CS_RAW_16:
 						castBuffer.rewind();
 						castBuffer.put(src, n, 2);
 						trail = ((int)castBuffer.getShort(0)) & 0xffff;
 						if(trail == 0) {
-							obj = top_schema.createFromRaw(new byte[0], 0, 0);
+							obj = new RawType(new byte[0]);
 							break _push;
 						}
 						cs = ACS_RAW_VALUE;
@@ -330,77 +304,67 @@ public class UnpackerImpl {
 						// FIXME overflow check
 						trail = castBuffer.getInt(0) & 0x7fffffff;
 						if(trail == 0) {
-							obj = top_schema.createFromRaw(new byte[0], 0, 0);
+							obj = new RawType(new byte[0]);
 							break _push;
 						}
 						cs = ACS_RAW_VALUE;
-					case ACS_RAW_VALUE:
-						obj = top_schema.createFromRaw(src, n, trail);
+					case ACS_RAW_VALUE: {
+							byte[] raw = new byte[trail];
+							System.arraycopy(src, n, raw, 0, trail);
+							obj = new RawType(raw);
+						}
 						break _push;
 					case CS_ARRAY_16:
 						if(top >= MAX_STACK_SIZE) {
 							throw new UnpackException("parse error");
 						}
-						if(!(top_schema instanceof IArraySchema)) {
-							throw new RuntimeException("type error");
-						}
 						castBuffer.rewind();
 						castBuffer.put(src, n, 2);
 						count = ((int)castBuffer.getShort(0)) & 0xffff;
-						obj = new Object[count];
+						obj = new MessagePackObject[count];
 						if(count == 0) {
-							obj = ((IArraySchema)top_schema).createFromArray((Object[])obj);
+							obj = new ArrayType((MessagePackObject[])obj);
 							break _push;
 						}
 						++top;
 						stack_obj[top]    = top_obj;
 						stack_ct[top]     = top_ct;
 						stack_count[top]  = top_count;
-						stack_schema[top] = top_schema;
 						top_obj    = obj;
 						top_ct     = CT_ARRAY_ITEM;
 						top_count  = count;
-						top_schema = ((IArraySchema)top_schema).getElementSchema(0);
 						break _header_again;
 					case CS_ARRAY_32:
 						if(top >= MAX_STACK_SIZE) {
 							throw new UnpackException("parse error");
 						}
-						if(!(top_schema instanceof IArraySchema)) {
-							throw new RuntimeException("type error");
-						}
 						castBuffer.rewind();
 						castBuffer.put(src, n, 4);
 						// FIXME overflow check
 						count = castBuffer.getInt(0) & 0x7fffffff;
-						obj = new Object[count];
+						obj = new MessagePackObject[count];
 						if(count == 0) {
-							obj = ((IArraySchema)top_schema).createFromArray((Object[])obj);
+							obj = new ArrayType((MessagePackObject[])obj);
 							break _push;
 						}
 						++top;
 						stack_obj[top]    = top_obj;
 						stack_ct[top]     = top_ct;
 						stack_count[top]  = top_count;
-						stack_schema[top] = top_schema;
 						top_obj    = obj;
 						top_ct     = CT_ARRAY_ITEM;
 						top_count  = count;
-						top_schema = ((IArraySchema)top_schema).getElementSchema(0);
 						break _header_again;
 					case CS_MAP_16:
 						if(top >= MAX_STACK_SIZE) {
 							throw new UnpackException("parse error");
 						}
-						if(!(top_schema instanceof IMapSchema)) {
-							throw new RuntimeException("type error");
-						}
 						castBuffer.rewind();
 						castBuffer.put(src, n, 2);
 						count = ((int)castBuffer.getShort(0)) & 0xffff;
-						obj = new Object[count*2];
+						obj = new MessagePackObject[count*2];
 						if(count == 0) {
-							obj = ((IMapSchema)top_schema).createFromMap((Object[])obj);
+							obj = new MapType((MessagePackObject[])obj);
 							break _push;
 						}
 						//System.out.println("fixmap count:"+count);
@@ -408,26 +372,21 @@ public class UnpackerImpl {
 						stack_obj[top]    = top_obj;
 						stack_ct[top]     = top_ct;
 						stack_count[top]  = top_count;
-						stack_schema[top] = top_schema;
 						top_obj    = obj;
 						top_ct     = CT_MAP_KEY;
 						top_count  = count;
-						top_schema = ((IMapSchema)top_schema).getKeySchema();
 						break _header_again;
 					case CS_MAP_32:
 						if(top >= MAX_STACK_SIZE) {
 							throw new UnpackException("parse error");
 						}
-						if(!(top_schema instanceof IMapSchema)) {
-							throw new RuntimeException("type error");
-						}
 						castBuffer.rewind();
 						castBuffer.put(src, n, 4);
 						// FIXME overflow check
 						count = castBuffer.getInt(0) & 0x7fffffff;
-						obj = new Object[count*2];
+						obj = new MessagePackObject[count*2];
 						if(count == 0) {
-							obj = ((IMapSchema)top_schema).createFromMap((Object[])obj);
+							obj = new MapType((MessagePackObject[])obj);
 							break _push;
 						}
 						//System.out.println("fixmap count:"+count);
@@ -435,11 +394,9 @@ public class UnpackerImpl {
 						stack_obj[top]    = top_obj;
 						stack_ct[top]     = top_ct;
 						stack_count[top]  = top_count;
-						stack_schema[top] = top_schema;
 						top_obj    = obj;
 						top_ct     = CT_MAP_KEY;
 						top_count  = count;
-						top_schema = ((IMapSchema)top_schema).getKeySchema();
 						break _header_again;
 					default:
 						throw new UnpackException("parse error");
@@ -454,7 +411,7 @@ public class UnpackerImpl {
 				//System.out.println("push top:"+top);
 				if(top == -1) {
 					++i;
-					data = obj;
+					data = (MessagePackObject)obj;
 					finished = true;
 					break _out;
 				}
@@ -468,14 +425,10 @@ public class UnpackerImpl {
 							top_obj    = stack_obj[top];
 							top_ct     = stack_ct[top];
 							top_count  = stack_count[top];
-							top_schema = stack_schema[top];
-							obj = ((IArraySchema)top_schema).createFromArray(ar);
+							obj = new ArrayType((MessagePackObject[])ar);
 							stack_obj[top] = null;
-							stack_schema[top] = null;
 							--top;
 							break _push;
-						} else {
-							top_schema = ((IArraySchema)stack_schema[top]).getElementSchema(ar.length - top_count);
 						}
 						break _header_again;
 					}
@@ -484,7 +437,6 @@ public class UnpackerImpl {
 						Object[] mp = (Object[])top_obj;
 						mp[mp.length - top_count*2] = obj;
 						top_ct = CT_MAP_VALUE;
-						top_schema = ((IMapSchema)stack_schema[top]).getValueSchema();
 						break _header_again;
 					}
 				case CT_MAP_VALUE: {
@@ -495,10 +447,8 @@ public class UnpackerImpl {
 							top_obj    = stack_obj[top];
 							top_ct     = stack_ct[top];
 							top_count  = stack_count[top];
-							top_schema = stack_schema[top];
-							obj = ((IMapSchema)top_schema).createFromMap(mp);
+							obj = new MapType((MessagePackObject[])mp);
 							stack_obj[top] = null;
-							stack_schema[top] = null;
 							--top;
 							break _push;
 						}
diff --git a/java/src/main/java/org/msgpack/schema/BooleanSchema.java b/java/src/main/java/org/msgpack/schema/BooleanSchema.java
deleted file mode 100644
index 2c325f1..0000000
--- a/java/src/main/java/org/msgpack/schema/BooleanSchema.java
+++ /dev/null
@@ -1,64 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.io.IOException;
-import org.msgpack.*;
-
-public class BooleanSchema extends Schema {
-	public BooleanSchema() { }
-
-	@Override
-	public String getClassName() {
-		return "Boolean";
-	}
-
-	@Override
-	public String getExpression() {
-		return "boolean";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof Boolean) {
-			pk.packBoolean((Boolean)obj);
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	public static final boolean convertBoolean(Object obj) throws MessageTypeException {
-		if(obj instanceof Boolean) {
-			return (Boolean)obj;
-		}
-		throw new MessageTypeException();
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertBoolean(obj);
-	}
-
-	@Override
-	public Object createFromBoolean(boolean v) {
-		return v;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/ByteArraySchema.java b/java/src/main/java/org/msgpack/schema/ByteArraySchema.java
deleted file mode 100644
index af9c0ed..0000000
--- a/java/src/main/java/org/msgpack/schema/ByteArraySchema.java
+++ /dev/null
@@ -1,97 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.nio.ByteBuffer;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import org.msgpack.*;
-
-public class ByteArraySchema extends Schema {
-	public ByteArraySchema() { }
-
-	@Override
-	public String getClassName() {
-		return "byte[]";
-	}
-
-	@Override
-	public String getExpression() {
-		return "raw";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof byte[]) {
-			byte[] b = (byte[])obj;
-			pk.packRaw(b.length);
-			pk.packRawBody(b);
-		} else if(obj instanceof String) {
-			try {
-				byte[] b = ((String)obj).getBytes("UTF-8");
-				pk.packRaw(b.length);
-				pk.packRawBody(b);
-			} catch (UnsupportedEncodingException e) {
-				throw new MessageTypeException();
-			}
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	public static final byte[] convertByteArray(Object obj) throws MessageTypeException {
-		if(obj instanceof byte[]) {
-			// FIXME copy?
-			//byte[] d = (byte[])obj;
-			//byte[] v = new byte[d.length];
-			//System.arraycopy(d, 0, v, 0, d.length);
-			//return v;
-			return (byte[])obj;
-		} else if(obj instanceof ByteBuffer) {
-			ByteBuffer d = (ByteBuffer)obj;
-			byte[] v = new byte[d.capacity()];
-			int pos = d.position();
-			d.get(v);
-			d.position(pos);
-			return v;
-		} else if(obj instanceof String) {
-			try {
-				return ((String)obj).getBytes("UTF-8");
-			} catch (UnsupportedEncodingException e) {
-				throw new MessageTypeException();
-			}
-		} else {
-			throw new MessageTypeException();
-		}
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertByteArray(obj);
-	}
-
-	@Override
-	public Object createFromRaw(byte[] b, int offset, int length) {
-		byte[] d = new byte[length];
-		System.arraycopy(b, offset, d, 0, length);
-		return d;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/ByteSchema.java b/java/src/main/java/org/msgpack/schema/ByteSchema.java
deleted file mode 100644
index 6003a0f..0000000
--- a/java/src/main/java/org/msgpack/schema/ByteSchema.java
+++ /dev/null
@@ -1,96 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.io.IOException;
-import org.msgpack.*;
-
-public class ByteSchema extends Schema {
-	public ByteSchema() { }
-
-	@Override
-	public String getClassName() {
-		return "Byte";
-	}
-
-	@Override
-	public String getExpression() {
-		return "byte";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof Number) {
-			short value = ((Number)obj).shortValue();
-			if(value > Byte.MAX_VALUE) {
-				throw new MessageTypeException();
-			}
-			pk.packByte((byte)value);
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	public static final byte convertByte(Object obj) throws MessageTypeException {
-		if(obj instanceof Number) {
-			short value = ((Number)obj).shortValue();
-			if(value > Byte.MAX_VALUE) {
-				throw new MessageTypeException();
-			}
-			return (byte)value;
-		}
-		throw new MessageTypeException();
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertByte(obj);
-	}
-
-	@Override
-	public Object createFromByte(byte v) {
-		return (byte)v;
-	}
-
-	@Override
-	public Object createFromShort(short v) {
-		if(v > Byte.MAX_VALUE) {
-			throw new MessageTypeException();
-		}
-		return (byte)v;
-	}
-
-	@Override
-	public Object createFromInt(int v) {
-		if(v > Byte.MAX_VALUE) {
-			throw new MessageTypeException();
-		}
-		return (byte)v;
-	}
-
-	@Override
-	public Object createFromLong(long v) {
-		if(v > Byte.MAX_VALUE) {
-			throw new MessageTypeException();
-		}
-		return (byte)v;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/ClassGenerator.java b/java/src/main/java/org/msgpack/schema/ClassGenerator.java
deleted file mode 100644
index a515996..0000000
--- a/java/src/main/java/org/msgpack/schema/ClassGenerator.java
+++ /dev/null
@@ -1,244 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.io.IOException;
-import java.io.File;
-import java.io.Writer;
-import org.msgpack.*;
-
-public class ClassGenerator {
-	private ClassSchema schema;
-	private Writer writer;
-	private int indent;
-
-	private ClassGenerator(Writer writer) {
-		this.writer = writer;
-		this.indent = 0;
-	}
-
-	public static void write(Schema schema, Writer dest) throws IOException {
-		if(!(schema instanceof ClassSchema)) {
-			throw new RuntimeException("schema is not class schema");
-		}
-		ClassSchema cs = (ClassSchema)schema;
-		new ClassGenerator(dest).run(cs);
-	}
-
-	private void run(ClassSchema cs) throws IOException {
-		List subclasses = new ArrayList();
-		for(FieldSchema f : cs.getFields()) {
-			findSubclassSchema(subclasses, f.getSchema());
-		}
-
-		for(ClassSchema sub : subclasses) {
-			sub.setNamespace(cs.getNamespace());
-			sub.setImports(cs.getImports());
-		}
-
-		this.schema = cs;
-
-		writeHeader();
-
-		writeClass();
-
-		for(ClassSchema sub : subclasses) {
-			this.schema = sub;
-			writeSubclass();
-		}
-
-		writeFooter();
-
-		this.schema = null;
-		writer.flush();
-	}
-
-	private void findSubclassSchema(List dst, Schema s) {
-		if(s instanceof ClassSchema) {
-			ClassSchema cs = (ClassSchema)s;
-			if(!dst.contains(cs)) { dst.add(cs); }
-			for(FieldSchema f : cs.getFields()) {
-				findSubclassSchema(dst, f.getSchema());
-			}
-		} else if(s instanceof ListSchema) {
-			ListSchema as = (ListSchema)s;
-			findSubclassSchema(dst, as.getElementSchema(0));
-		} else if(s instanceof SetSchema) {
-			SetSchema as = (SetSchema)s;
-			findSubclassSchema(dst, as.getElementSchema(0));
-		} else if(s instanceof MapSchema) {
-			MapSchema as = (MapSchema)s;
-			findSubclassSchema(dst, as.getKeySchema());
-			findSubclassSchema(dst, as.getValueSchema());
-		}
-	}
-
-	private void writeHeader() throws IOException {
-		if(schema.getNamespace() != null) {
-			line("package "+schema.getNamespace()+";");
-			line();
-		}
-		line("import java.util.*;");
-		line("import java.io.*;");
-		line("import org.msgpack.*;");
-		line("import org.msgpack.schema.ClassSchema;");
-		line("import org.msgpack.schema.FieldSchema;");
-	}
-
-	private void writeFooter() throws IOException {
-		line();
-	}
-
-	private void writeClass() throws IOException {
-		line();
-		line("public final class "+schema.getClassName()+" implements MessagePackable, MessageConvertable");
-		line("{");
-		pushIndent();
-			writeSchema();
-			writeMemberVariables();
-			writeMemberFunctions();
-		popIndent();
-		line("}");
-	}
-
-	private void writeSubclass() throws IOException {
-		line();
-		line("final class "+schema.getClassName()+" implements MessagePackable, MessageConvertable");
-		line("{");
-		pushIndent();
-			writeSchema();
-			writeMemberVariables();
-			writeMemberFunctions();
-		popIndent();
-		line("}");
-	}
-
-	private void writeSchema() throws IOException {
-		line("private static final ClassSchema _SCHEMA = (ClassSchema)Schema.load(\""+schema.getExpression()+"\");");
-		line("public static ClassSchema getSchema() { return _SCHEMA; }");
-	}
-
-	private void writeMemberVariables() throws IOException {
-		line();
-		for(FieldSchema f : schema.getFields()) {
-			line("public "+f.getSchema().getClassName()+" "+f.getName()+";");
-		}
-	}
-
-	private void writeMemberFunctions() throws IOException {
-		// void messagePack(Packer pk)
-		// boolean equals(Object obj)
-		// int hashCode()
-		// void set(int _index, Object _value)
-		// Object get(int _index);
-		// getXxx()
-		// setXxx(Xxx xxx)
-		writeConstructors();
-		writeAccessors();
-		writePackFunction();
-		writeConvertFunction();
-		writeFactoryFunction();
-	}
-
-	private void writeConstructors() throws IOException {
-		line();
-		line("public "+schema.getClassName()+"() { }");
-	}
-
-	private void writeAccessors() throws IOException {
-		// FIXME
-		//line();
-		//for(FieldSchema f : schema.getFields()) {
-		//	line("");
-		//}
-	}
-
-	private void writePackFunction() throws IOException {
-		line();
-		line("@Override");
-		line("public void messagePack(Packer _pk) throws IOException");
-		line("{");
-		pushIndent();
-			line("_pk.packArray("+schema.getFields().length+");");
-			line("FieldSchema[] _fields = _SCHEMA.getFields();");
-			int i = 0;
-			for(FieldSchema f : schema.getFields()) {
-				line("_fields["+i+"].getSchema().pack(_pk, "+f.getName()+");");
-				++i;
-			}
-		popIndent();
-		line("}");
-	}
-
-	private void writeConvertFunction() throws IOException {
-		line();
-		line("@Override");
-		line("@SuppressWarnings(\"unchecked\")");
-		line("public void messageConvert(Object obj) throws MessageTypeException");
-		line("{");
-		pushIndent();
-			line("Object[] _source = ((List)obj).toArray();");
-			line("FieldSchema[] _fields = _SCHEMA.getFields();");
-			int i = 0;
-			for(FieldSchema f : schema.getFields()) {
-				line("if(_source.length <= "+i+") { return; } this."+f.getName()+" = ("+f.getSchema().getClassName()+")_fields["+i+"].getSchema().convert(_source["+i+"]);");
-				++i;
-			}
-		popIndent();
-		line("}");
-	}
-
-	private void writeFactoryFunction() throws IOException {
-		line();
-		line("@SuppressWarnings(\"unchecked\")");
-		line("public static "+schema.getClassName()+" createFromMessage(Object[] _message)");
-		line("{");
-		pushIndent();
-			line(schema.getClassName()+" _self = new "+schema.getClassName()+"();");
-			int i = 0;
-			for(FieldSchema f : schema.getFields()) {
-				line("if(_message.length <= "+i+") { return _self; } _self."+f.getName()+" = ("+f.getSchema().getClassName()+")_message["+i+"];");
-				++i;
-			}
-			line("return _self;");
-		popIndent();
-		line("}");
-	}
-
-	private void line(String str) throws IOException {
-		for(int i=0; i < indent; ++i) {
-			writer.write("\t");
-		}
-		writer.write(str+"\n");
-	}
-
-	private void line() throws IOException {
-		writer.write("\n");
-	}
-
-	private void pushIndent() {
-		indent += 1;
-	}
-
-	private void popIndent() {
-		indent -= 1;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/ClassSchema.java b/java/src/main/java/org/msgpack/schema/ClassSchema.java
deleted file mode 100644
index cd59755..0000000
--- a/java/src/main/java/org/msgpack/schema/ClassSchema.java
+++ /dev/null
@@ -1,91 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.util.Arrays;
-import java.util.List;
-import org.msgpack.*;
-
-public abstract class ClassSchema extends Schema implements IArraySchema {
-	protected String name;
-	protected FieldSchema[] fields;
-	protected List imports;
-	protected String namespace;
-	protected String fqdn;
-
-	public ClassSchema(
-			String name, String namespace,
-			List imports, List fields) {
-		this.name = name;
-		this.namespace = namespace;
-		this.imports = imports;  // FIXME clone?
-		this.fields = new FieldSchema[fields.size()];
-		System.arraycopy(fields.toArray(), 0, this.fields, 0, fields.size());
-		if(namespace == null) {
-			this.fqdn = name;
-		} else {
-			this.fqdn = namespace+"."+name;
-		}
-	}
-
-	@Override
-	public String getClassName() {
-		return name;
-	}
-
-	@Override
-	public String getExpression() {
-		StringBuffer b = new StringBuffer();
-		b.append("(class ");
-		b.append(name);
-		if(namespace != null) {
-			b.append(" (package "+namespace+")");
-		}
-		for(FieldSchema f : fields) {
-			b.append(" "+f.getExpression());
-		}
-		b.append(")");
-		return b.toString();
-	}
-
-	public boolean equals(ClassSchema o) {
-		return (namespace != null ? namespace.equals(o.getNamespace()) : o.getNamespace() == null) &&
-			name.equals(o.name);
-	}
-
-	public final FieldSchema[] getFields() {
-		return fields;
-	}
-
-	String getNamespace() {
-		return namespace;
-	}
-
-	List getImports() {
-		return imports;
-	}
-
-	void setNamespace(String namespace) {
-		this.namespace = namespace;
-	}
-
-	void setImports(List imports) {
-		this.imports = imports;  // FIXME clone?
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/DoubleSchema.java b/java/src/main/java/org/msgpack/schema/DoubleSchema.java
deleted file mode 100644
index cb857c3..0000000
--- a/java/src/main/java/org/msgpack/schema/DoubleSchema.java
+++ /dev/null
@@ -1,74 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.io.IOException;
-import org.msgpack.*;
-
-public class DoubleSchema extends Schema {
-	public DoubleSchema() { }
-
-	@Override
-	public String getClassName() {
-		return "Double";
-	}
-
-	@Override
-	public String getExpression() {
-		return "double";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof Double) {
-			pk.packDouble((Double)obj);
-		} else if(obj instanceof Float) {
-			pk.packFloat((Float)obj);
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	public static final double convertDouble(Object obj) throws MessageTypeException {
-		if(obj instanceof Double) {
-			return (Double)obj;
-		} else if(obj instanceof Float) {
-			return ((Float)obj).doubleValue();
-		} else {
-			throw new MessageTypeException();
-		}
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertDouble(obj);
-	}
-
-	@Override
-	public Object createFromFloat(float v) {
-		return (double)v;
-	}
-
-	@Override
-	public Object createFromDouble(double v) {
-		return (double)v;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/FieldSchema.java b/java/src/main/java/org/msgpack/schema/FieldSchema.java
deleted file mode 100644
index 66c2ff2..0000000
--- a/java/src/main/java/org/msgpack/schema/FieldSchema.java
+++ /dev/null
@@ -1,43 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import org.msgpack.Schema;
-
-public class FieldSchema {
-	private String name;
-	private Schema schema;
-
-	public FieldSchema(String name, Schema schema) {
-		this.name = name;
-		this.schema = schema;
-	}
-
-	public final String getName() {
-		return name;
-	}
-
-	public final Schema getSchema() {
-		return schema;
-	}
-
-	public String getExpression() {
-		return "(field "+name+" "+schema.getExpression()+")";
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/FloatSchema.java b/java/src/main/java/org/msgpack/schema/FloatSchema.java
deleted file mode 100644
index cd73201..0000000
--- a/java/src/main/java/org/msgpack/schema/FloatSchema.java
+++ /dev/null
@@ -1,74 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.io.IOException;
-import org.msgpack.*;
-
-public class FloatSchema extends Schema {
-	public FloatSchema() { }
-
-	@Override
-	public String getClassName() {
-		return "Float";
-	}
-
-	@Override
-	public String getExpression() {
-		return "float";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof Double) {
-			pk.packDouble((Double)obj);
-		} else if(obj instanceof Float) {
-			pk.packFloat((Float)obj);
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	public static final float convertFloat(Object obj) throws MessageTypeException {
-		if(obj instanceof Double) {
-			return ((Double)obj).floatValue();
-		} else if(obj instanceof Float) {
-			return (Float)obj;
-		} else {
-			throw new MessageTypeException();
-		}
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertFloat(obj);
-	}
-
-	@Override
-	public Object createFromFloat(float v) {
-		return (float)v;
-	}
-
-	@Override
-	public Object createFromDouble(double v) {
-		return (float)v;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/GenericClassSchema.java b/java/src/main/java/org/msgpack/schema/GenericClassSchema.java
deleted file mode 100644
index 1ab4c33..0000000
--- a/java/src/main/java/org/msgpack/schema/GenericClassSchema.java
+++ /dev/null
@@ -1,87 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
-import java.io.IOException;
-import org.msgpack.*;
-
-public class GenericClassSchema extends ClassSchema {
-	public GenericClassSchema(
-			String name, String namespace,
-			List imports, List fields) {
-		super(name, namespace, imports, fields);
-	}
-
-	@Override
-	@SuppressWarnings("unchecked")
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof Map) {
-			Map d = (Map)obj;
-			pk.packArray(fields.length);
-			for(int i=0; i < fields.length; ++i) {
-				FieldSchema f = fields[i];
-				f.getSchema().pack(pk, d.get(f.getName()));
-			}
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		if(obj instanceof Collection) {
-			// FIXME optimize
-			return createFromArray( ((Collection)obj).toArray() );
-		} else if(obj instanceof Map) {
-			HashMap m = new HashMap(fields.length);
-			Map d = (Map)obj;
-			for(int i=0; i < fields.length; ++i) {
-				FieldSchema f = fields[i];
-				String fieldName = f.getName();
-				m.put(fieldName, f.getSchema().convert(d.get(fieldName)));
-			}
-			return m;
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	public Schema getElementSchema(int index) {
-		// FIXME check index < fields.length
-		return fields[index].getSchema();
-	}
-
-	public Object createFromArray(Object[] obj) {
-		HashMap m = new HashMap(fields.length);
-		int i=0;
-		for(; i < obj.length; ++i) {
-			m.put(fields[i].getName(), obj[i]);
-		}
-		for(; i < fields.length; ++i) {
-			m.put(fields[i].getName(), null);
-		}
-		return m;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/GenericSchema.java b/java/src/main/java/org/msgpack/schema/GenericSchema.java
deleted file mode 100644
index f9098ed..0000000
--- a/java/src/main/java/org/msgpack/schema/GenericSchema.java
+++ /dev/null
@@ -1,129 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.HashMap;
-import java.io.IOException;
-import org.msgpack.*;
-
-public class GenericSchema extends Schema implements IArraySchema, IMapSchema {
-	public GenericSchema() { }
-
-	@Override
-	public String getClassName() {
-		return "Object";
-	}
-
-	@Override
-	public String getExpression() {
-		return "object";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		pk.pack(obj);
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return obj;
-	}
-
-	@Override
-	public Schema getElementSchema(int index) {
-		return this;
-	}
-
-	@Override
-	public Schema getKeySchema() {
-		return this;
-	}
-
-	@Override
-	public Schema getValueSchema() {
-		return this;
-	}
-
-	@Override
-	public Object createFromNil() {
-		return null;
-	}
-
-	@Override
-	public Object createFromBoolean(boolean v) {
-		return v;
-	}
-
-	@Override
-	public Object createFromByte(byte v) {
-		return v;
-	}
-
-	@Override
-	public Object createFromShort(short v) {
-		return v;
-	}
-
-	@Override
-	public Object createFromInt(int v) {
-		return v;
-	}
-
-	@Override
-	public Object createFromLong(long v) {
-		return v;
-	}
-
-	@Override
-	public Object createFromFloat(float v) {
-		return v;
-	}
-
-	@Override
-	public Object createFromDouble(double v) {
-		return v;
-	}
-
-	@Override
-	public Object createFromRaw(byte[] b, int offset, int length) {
-		byte[] bytes = new byte[length];
-		System.arraycopy(b, offset, bytes, 0, length);
-		return bytes;
-	}
-
-	@Override
-	public Object createFromArray(Object[] obj) {
-		return Arrays.asList(obj);
-	}
-
-	@Override
-	@SuppressWarnings("unchecked")
-	public Object createFromMap(Object[] obj) {
-		HashMap m = new HashMap(obj.length / 2);
-		int i = 0;
-		while(i < obj.length) {
-			Object k = obj[i++];
-			Object v = obj[i++];
-			m.put(k, v);
-		}
-		return m;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/IArraySchema.java b/java/src/main/java/org/msgpack/schema/IArraySchema.java
deleted file mode 100644
index 67e9f55..0000000
--- a/java/src/main/java/org/msgpack/schema/IArraySchema.java
+++ /dev/null
@@ -1,26 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import org.msgpack.Schema;
-
-public interface IArraySchema {
-	public Schema getElementSchema(int index);
-	public Object createFromArray(Object[] obj);
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/IMapSchema.java b/java/src/main/java/org/msgpack/schema/IMapSchema.java
deleted file mode 100644
index 3a2f556..0000000
--- a/java/src/main/java/org/msgpack/schema/IMapSchema.java
+++ /dev/null
@@ -1,27 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import org.msgpack.Schema;
-
-public interface IMapSchema {
-	public Schema getKeySchema();
-	public Schema getValueSchema();
-	public Object createFromMap(Object[] obj);
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/IntSchema.java b/java/src/main/java/org/msgpack/schema/IntSchema.java
deleted file mode 100644
index 269f4fb..0000000
--- a/java/src/main/java/org/msgpack/schema/IntSchema.java
+++ /dev/null
@@ -1,96 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.io.IOException;
-import org.msgpack.*;
-
-public class IntSchema extends Schema {
-	public IntSchema() { }
-
-	@Override
-	public String getClassName() {
-		return "Integer";
-	}
-
-	@Override
-	public String getExpression() {
-		return "int";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof Number) {
-			int value = ((Number)obj).intValue();
-			if(value >= Short.MAX_VALUE) {
-				long lvalue = ((Number)obj).longValue();
-				if(lvalue > Integer.MAX_VALUE) {
-					throw new MessageTypeException();
-				}
-			}
-			pk.packInt(value);
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	public static final int convertInt(Object obj) throws MessageTypeException {
-		if(obj instanceof Number) {
-			int value = ((Number)obj).intValue();
-			if(value >= Integer.MAX_VALUE) {
-				long lvalue = ((Number)obj).longValue();
-				if(lvalue > Integer.MAX_VALUE) {
-					throw new MessageTypeException();
-				}
-			}
-			return value;
-		}
-		throw new MessageTypeException();
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertInt(obj);
-	}
-
-	@Override
-	public Object createFromByte(byte v) {
-		return (int)v;
-	}
-
-	@Override
-	public Object createFromShort(short v) {
-		return (int)v;
-	}
-
-	@Override
-	public Object createFromInt(int v) {
-		return (int)v;
-	}
-
-	@Override
-	public Object createFromLong(long v) {
-		if(v > Integer.MAX_VALUE) {
-			throw new MessageTypeException();
-		}
-		return (int)v;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/ListSchema.java b/java/src/main/java/org/msgpack/schema/ListSchema.java
deleted file mode 100644
index bef8cc4..0000000
--- a/java/src/main/java/org/msgpack/schema/ListSchema.java
+++ /dev/null
@@ -1,111 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Set;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.RandomAccess;
-import java.io.IOException;
-import org.msgpack.*;
-
-public class ListSchema extends Schema implements IArraySchema {
-	private Schema elementSchema;
-
-	public ListSchema(Schema elementSchema) {
-		this.elementSchema = elementSchema;
-	}
-
-	@Override
-	public String getClassName() {
-		return "List<"+elementSchema.getClassName()+">";
-	}
-
-	@Override
-	public String getExpression() {
-		return "(array "+elementSchema.getExpression()+")";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof List) {
-			List d = (List)obj;
-			pk.packArray(d.size());
-			if(obj instanceof RandomAccess) {
-				for(int i=0; i < d.size(); ++i) {
-					elementSchema.pack(pk, d.get(i));
-				}
-			} else {
-				for(Object e : d) {
-					elementSchema.pack(pk, e);
-				}
-			}
-		} else if(obj instanceof Set) {
-			Set d = (Set)obj;
-			pk.packArray(d.size());
-			for(Object e : d) {
-				elementSchema.pack(pk, e);
-			}
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	@SuppressWarnings("unchecked")
-	public static final  List convertList(Object obj,
-			Schema elementSchema, List dest) throws MessageTypeException {
-		if(!(obj instanceof List)) {
-			throw new MessageTypeException();
-		}
-		List d = (List)obj;
-		if(dest == null) {
-			dest = new ArrayList(d.size());
-		}
-		if(obj instanceof RandomAccess) {
-			for(int i=0; i < d.size(); ++i) {
-				dest.add( (T)elementSchema.convert(d.get(i)) );
-			}
-		} else {
-			for(Object e : d) {
-				dest.add( (T)elementSchema.convert(e) );
-			}
-		}
-		return dest;
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertList(obj, elementSchema, null);
-	}
-
-	@Override
-	public Schema getElementSchema(int index) {
-		return elementSchema;
-	}
-
-	@Override
-	public Object createFromArray(Object[] obj) {
-		return Arrays.asList(obj);
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/LongSchema.java b/java/src/main/java/org/msgpack/schema/LongSchema.java
deleted file mode 100644
index 728fa21..0000000
--- a/java/src/main/java/org/msgpack/schema/LongSchema.java
+++ /dev/null
@@ -1,80 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.io.IOException;
-import org.msgpack.*;
-
-public class LongSchema extends Schema {
-	public LongSchema() { }
-
-	@Override
-	public String getClassName() {
-		return "Long";
-	}
-
-	@Override
-	public String getExpression() {
-		return "long";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof Number) {
-			long value = ((Number)obj).longValue();
-			pk.packLong(value);
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	public static final long convertLong(Object obj) throws MessageTypeException {
-		if(obj instanceof Number) {
-			return ((Number)obj).longValue();
-		}
-		throw new MessageTypeException();
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertLong(obj);
-	}
-
-	@Override
-	public Object createFromByte(byte v) {
-		return (long)v;
-	}
-
-	@Override
-	public Object createFromShort(short v) {
-		return (long)v;
-	}
-
-	@Override
-	public Object createFromInt(int v) {
-		return (long)v;
-	}
-
-	@Override
-	public Object createFromLong(long v) {
-		return (long)v;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/MapSchema.java b/java/src/main/java/org/msgpack/schema/MapSchema.java
deleted file mode 100644
index 2e09af3..0000000
--- a/java/src/main/java/org/msgpack/schema/MapSchema.java
+++ /dev/null
@@ -1,106 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.util.Map;
-import java.util.HashMap;
-import java.io.IOException;
-import org.msgpack.*;
-
-public class MapSchema extends Schema implements IMapSchema {
-	private Schema keySchema;
-	private Schema valueSchema;
-
-	public MapSchema(Schema keySchema, Schema valueSchema) {
-		this.keySchema = keySchema;
-		this.valueSchema = valueSchema;
-	}
-
-	@Override
-	public String getClassName() {
-		return "Map<"+keySchema.getClassName()+", "+valueSchema.getClassName()+">";
-	}
-
-	@Override
-	public String getExpression() {
-		return "(map "+keySchema.getExpression()+" "+valueSchema.getExpression()+")";
-	}
-
-	@Override
-	@SuppressWarnings("unchecked")
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof Map) {
-			Map d = (Map)obj;
-			pk.packMap(d.size());
-			for(Map.Entry e : d.entrySet()) {
-				keySchema.pack(pk, e.getKey());
-				valueSchema.pack(pk, e.getValue());
-			}
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	@SuppressWarnings("unchecked")
-	public static final  Map convertMap(Object obj,
-			Schema keySchema, Schema valueSchema, Map dest) throws MessageTypeException {
-		if(!(obj instanceof Map)) {
-			throw new MessageTypeException();
-		}
-		Map d = (Map)obj;
-		if(dest == null) {
-			dest = new HashMap(d.size());
-		}
-		for(Map.Entry e : d.entrySet()) {
-			dest.put((K)keySchema.convert(e.getKey()),
-					(V)valueSchema.convert(e.getValue()));
-		}
-		return dest;
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertMap(obj, keySchema, valueSchema, null);
-	}
-
-	@Override
-	public Schema getKeySchema() {
-		return keySchema;
-	}
-
-	@Override
-	public Schema getValueSchema() {
-		return valueSchema;
-	}
-
-	@Override
-	@SuppressWarnings("unchecked")
-	public Object createFromMap(Object[] obj) {
-		HashMap dest = new HashMap(obj.length / 2);
-		int i = 0;
-		while(i < obj.length) {
-			Object k = obj[i++];
-			Object v = obj[i++];
-			dest.put(k, v);
-		}
-		return dest;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java b/java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java
deleted file mode 100644
index fb94adf..0000000
--- a/java/src/main/java/org/msgpack/schema/ReflectionClassSchema.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package org.msgpack.schema;
-
-import java.util.Arrays;
-import java.util.List;
-import java.lang.reflect.*;
-import org.msgpack.*;
-
-// FIXME
-public abstract class ReflectionClassSchema extends ClassSchema {
-	private Constructor constructorCache;
-
-	public ReflectionClassSchema(String name, List fields, String namespace, List imports) {
-		super(name, namespace, imports, fields);
-	}
-
-	/*
-	Schema getElementSchema(int index)
-	{
-		// FIXME check index < fields.length
-		fields[index].getSchema();
-	}
-
-	Object createFromArray(Object[] obj)
-	{
-		Object o = newInstance();
-		((MessageConvertable)o).messageConvert(obj);
-		return o;
-	}
-
-	Object newInstance()
-	{
-		if(constructorCache == null) {
-			cacheConstructor();
-		}
-		try {
-			return constructorCache.newInstance((Object[])null);
-		} catch (InvocationTargetException e) {
-			throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage());
-		} catch (InstantiationException e) {
-			throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage());
-		} catch (IllegalAccessException e) {
-			throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage());
-		}
-	}
-
-	private void cacheConstructor()
-	{
-		try {
-			Class c = Class.forName(fqdn);
-			int index = 0;
-			for(SpecificFieldSchema f : fields) {
-				f.cacheField(c, index++);
-			}
-			constructorCache = c.getDeclaredConstructor((Class[])null);
-			constructorCache.setAccessible(true);
-		} catch(ClassNotFoundException e) {
-			throw new RuntimeException("class not found: "+fqdn);
-		} catch (NoSuchMethodException e) {
-			throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage());
-		}
-	}
-	*/
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/SSchemaParser.java b/java/src/main/java/org/msgpack/schema/SSchemaParser.java
deleted file mode 100644
index 4345e92..0000000
--- a/java/src/main/java/org/msgpack/schema/SSchemaParser.java
+++ /dev/null
@@ -1,264 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Stack;
-import java.util.regex.Pattern;
-import java.util.regex.Matcher;
-import org.msgpack.*;
-
-// FIXME exception class
-
-public class SSchemaParser {
-	public static Schema parse(String source) {
-		return new SSchemaParser(false).run(source);
-	}
-
-	public static Schema load(String source) {
-		return new SSchemaParser(true).run(source);
-	}
-
-	private static abstract class SExp {
-		boolean isAtom() { return false; }
-		public String getAtom() { return null; }
-
-		boolean isTuple() { return false; }
-		public SExp getTuple(int i) { return null; }
-		public int size() { return 0; }
-		public boolean empty() { return size() == 0; }
-		Iterator iterator(int offset) { return null; }
-	}
-
-	private static class SAtom extends SExp {
-		private String atom;
-
-		SAtom(String atom) { this.atom = atom; }
-
-		boolean isAtom() { return true; }
-		public String getAtom() { return atom; }
-
-		public String toString() { return atom; }
-	}
-
-	private static class STuple extends SExp {
-		private List tuple;
-
-		STuple() { this.tuple = new ArrayList(); }
-
-		public void add(SExp e) { tuple.add(e); }
-
-		boolean isTuple() { return true; }
-		public SExp getTuple(int i) { return tuple.get(i); }
-		public int size() { return tuple.size(); }
-
-		Iterator iterator(int skip) {
-			Iterator i = tuple.iterator();
-			for(int s=0; s < skip; ++s) { i.next(); }
-			return i;
-		}
-
-		public String toString() {
-			if(tuple.isEmpty()) { return "()"; }
-			Iterator i = tuple.iterator();
-			StringBuffer o = new StringBuffer();
-			o.append("(").append(i.next());
-			while(i.hasNext()) { o.append(" ").append(i.next()); }
-			o.append(")");
-			return o.toString();
-		}
-	}
-
-	boolean specificClass;
-
-	private SSchemaParser(boolean specificClass) {
-		this.specificClass = specificClass;
-	}
-
-	private static Pattern pattern = Pattern.compile(
-			"(?:\\s+)|([\\(\\)]|[\\d\\w\\.]+)");
-
-	private Schema run(String source) {
-		Matcher m = pattern.matcher(source);
-
-		Stack stack = new Stack();
-		String token;
-
-		while(true) {
-			while(true) {
-				if(!m.find()) { throw new RuntimeException("unexpected end of file"); }
-				token = m.group(1);
-				if(token != null) { break; }
-			}
-
-			if(token.equals("(")) {
-				stack.push(new STuple());
-			} else if(token.equals(")")) {
-				STuple top = stack.pop();
-				if(stack.empty()) {
-					stack.push(top);
-					break;
-				}
-				stack.peek().add(top);
-			} else {
-				if(stack.empty()) {
-					throw new RuntimeException("unexpected token '"+token+"'");
-				}
-				stack.peek().add(new SAtom(token));
-			}
-		}
-
-		while(true) {
-			if(!m.find()) { break; }
-			token = m.group(1);
-			if(token != null) { throw new RuntimeException("unexpected token '"+token+"'"); }
-		}
-
-		return readType( stack.pop() );
-	}
-
-	private Schema readType(SExp exp) {
-		if(exp.isAtom()) {
-			String type = exp.getAtom();
-			if(type.equals("string")) {
-				return new StringSchema();
-			} else if(type.equals("raw")) {
-				return new ByteArraySchema();
-			} else if(type.equals("byte")) {
-				return new ByteSchema();
-			} else if(type.equals("short")) {
-				return new ShortSchema();
-			} else if(type.equals("int")) {
-				return new IntSchema();
-			} else if(type.equals("long")) {
-				return new LongSchema();
-			} else if(type.equals("float")) {
-				return new FloatSchema();
-			} else if(type.equals("double")) {
-				return new DoubleSchema();
-			} else if(type.equals("object")) {
-				return new GenericSchema();
-			} else {
-				throw new RuntimeException("byte, short, int, long, float, double, raw, string or object is expected but got '"+type+"': "+exp);
-			}
-		} else {
-			String type = exp.getTuple(0).getAtom();
-			if(type.equals("class")) {
-				return parseClass(exp);
-			} else if(type.equals("array")) {
-				return parseList(exp);
-			} else if(type.equals("set")) {
-				return parseSet(exp);
-			} else if(type.equals("map")) {
-				return parseMap(exp);
-			} else {
-				throw new RuntimeException("class, list, set or map is expected but got '"+type+"': "+exp);
-			}
-		}
-	}
-
-	private ClassSchema parseClass(SExp exp) {
-		if(exp.size() < 3 || !exp.getTuple(1).isAtom()) {
-			throw new RuntimeException("class is (class NAME CLASS_BODY): "+exp);
-		}
-
-		String namespace = null;
-		List imports = new ArrayList();
-		String name = exp.getTuple(1).getAtom();
-		List fields = new ArrayList();
-
-		for(Iterator i=exp.iterator(2); i.hasNext();) {
-			SExp subexp = i.next();
-			if(!subexp.isTuple() || subexp.empty() || !subexp.getTuple(0).isAtom()) {
-				throw new RuntimeException("field, package or import is expected: "+subexp);
-			}
-			String type = subexp.getTuple(0).getAtom();
-			if(type.equals("field")) {
-				fields.add( parseField(subexp) );
-			} else if(type.equals("package")) {
-				if(namespace != null) {
-					throw new RuntimeException("duplicated package definition: "+subexp);
-				}
-				namespace = parseNamespace(subexp);
-			} else if(type.equals("import")) {
-				imports.add( parseImport(subexp) );
-			} else {
-				throw new RuntimeException("field, package or import is expected but got '"+type+"': "+subexp);
-			}
-		}
-
-		if(specificClass) {
-			return new SpecificClassSchema(name, namespace, imports, fields);
-		} else {
-			return new GenericClassSchema(name, namespace, imports, fields);
-		}
-	}
-
-	private ListSchema parseList(SExp exp) {
-		if(exp.size() != 2) {
-			throw new RuntimeException("list is (list ELEMENT_TYPE): "+exp);
-		}
-		Schema elementType = readType(exp.getTuple(1));
-		return new ListSchema(elementType);
-	}
-
-	private SetSchema parseSet(SExp exp) {
-		if(exp.size() != 2) {
-			throw new RuntimeException("list is (list ELEMENT_TYPE): "+exp);
-		}
-		Schema elementType = readType(exp.getTuple(1));
-		return new SetSchema(elementType);
-	}
-
-	private MapSchema parseMap(SExp exp) {
-		if(exp.size() != 3 || !exp.getTuple(1).isAtom()) {
-			throw new RuntimeException("map is (map KEY_TYPE VALUE_TYPE): "+exp);
-		}
-		Schema keyType   = readType(exp.getTuple(1));
-		Schema valueType = readType(exp.getTuple(2));
-		return new MapSchema(keyType, valueType);
-	}
-
-	private String parseNamespace(SExp exp) {
-		if(exp.size() != 2 || !exp.getTuple(1).isAtom()) {
-			throw new RuntimeException("package is (package NAME): "+exp);
-		}
-		String name = exp.getTuple(1).getAtom();
-		return name;
-	}
-
-	private String parseImport(SExp exp) {
-		if(exp.size() != 2 || !exp.getTuple(1).isAtom()) {
-			throw new RuntimeException("import is (import NAME): "+exp);
-		}
-		String name = exp.getTuple(1).getAtom();
-		return name;
-	}
-
-	private FieldSchema parseField(SExp exp) {
-		if(exp.size() != 3 || !exp.getTuple(1).isAtom()) {
-			throw new RuntimeException("field is (field NAME TYPE): "+exp);
-		}
-		String name = exp.getTuple(1).getAtom();
-		Schema type = readType(exp.getTuple(2));
-		return new FieldSchema(name, type);
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/SetSchema.java b/java/src/main/java/org/msgpack/schema/SetSchema.java
deleted file mode 100644
index a3e1974..0000000
--- a/java/src/main/java/org/msgpack/schema/SetSchema.java
+++ /dev/null
@@ -1,115 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Set;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.RandomAccess;
-import java.io.IOException;
-import org.msgpack.*;
-
-public class SetSchema extends Schema implements IArraySchema {
-	private Schema elementSchema;
-
-	public SetSchema(Schema elementSchema) {
-		this.elementSchema = elementSchema;
-	}
-
-	@Override
-	public String getClassName() {
-		return "Set<"+elementSchema.getClassName()+">";
-	}
-
-	@Override
-	public String getExpression() {
-		return "(set "+elementSchema.getExpression()+")";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof List) {
-			List d = (List)obj;
-			pk.packArray(d.size());
-			if(obj instanceof RandomAccess) {
-				for(int i=0; i < d.size(); ++i) {
-					elementSchema.pack(pk, d.get(i));
-				}
-			} else {
-				for(Object e : d) {
-					elementSchema.pack(pk, e);
-				}
-			}
-		} else if(obj instanceof Set) {
-			Set d = (Set)obj;
-			pk.packArray(d.size());
-			for(Object e : d) {
-				elementSchema.pack(pk, e);
-			}
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	@SuppressWarnings("unchecked")
-	public static final  Set convertSet(Object obj,
-			Schema elementSchema, Set dest) throws MessageTypeException {
-		if(!(obj instanceof List)) {
-			throw new MessageTypeException();
-		}
-		List d = (List)obj;
-		if(dest == null) {
-			dest = new HashSet(d.size());
-		}
-		if(obj instanceof RandomAccess) {
-			for(int i=0; i < d.size(); ++i) {
-				dest.add( (T)elementSchema.convert(d.get(i)) );
-			}
-		} else {
-			for(Object e : d) {
-				dest.add( (T)elementSchema.convert(e) );
-			}
-		}
-		return dest;
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertSet(obj, elementSchema, null);
-	}
-
-	@Override
-	public Schema getElementSchema(int index) {
-		return elementSchema;
-	}
-
-	@Override
-	public Object createFromArray(Object[] obj) {
-		Set m = new HashSet(obj.length);
-		for(int i=0; i < obj.length; i++) {
-			m.add(obj[i]);
-		}
-		return m;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/ShortSchema.java b/java/src/main/java/org/msgpack/schema/ShortSchema.java
deleted file mode 100644
index 21b9327..0000000
--- a/java/src/main/java/org/msgpack/schema/ShortSchema.java
+++ /dev/null
@@ -1,93 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.io.IOException;
-import org.msgpack.*;
-
-public class ShortSchema extends Schema {
-	public ShortSchema() { }
-
-	@Override
-	public String getClassName() {
-		return "Short";
-	}
-
-	@Override
-	public String getExpression() {
-		return "short";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof Number) {
-			int value = ((Number)obj).intValue();
-			if(value > Short.MAX_VALUE) {
-				throw new MessageTypeException();
-			}
-			pk.packShort((short)value);
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	public static final short convertShort(Object obj) throws MessageTypeException {
-		if(obj instanceof Number) {
-			int value = ((Number)obj).intValue();
-			if(value > Short.MAX_VALUE) {
-				throw new MessageTypeException();
-			}
-			return (short)value;
-		}
-		throw new MessageTypeException();
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertShort(obj);
-	}
-
-	@Override
-	public Object createFromByte(byte v) {
-		return (short)v;
-	}
-
-	@Override
-	public Object createFromShort(short v) {
-		return (short)v;
-	}
-
-	@Override
-	public Object createFromInt(int v) {
-		if(v > Short.MAX_VALUE) {
-			throw new MessageTypeException();
-		}
-		return (short)v;
-	}
-
-	@Override
-	public Object createFromLong(long v) {
-		if(v > Short.MAX_VALUE) {
-			throw new MessageTypeException();
-		}
-		return (short)v;
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java b/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java
deleted file mode 100644
index 850f621..0000000
--- a/java/src/main/java/org/msgpack/schema/SpecificClassSchema.java
+++ /dev/null
@@ -1,122 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.util.Collection;
-import java.util.List;
-import java.lang.reflect.*;
-import java.io.IOException;
-import org.msgpack.*;
-
-public class SpecificClassSchema extends ClassSchema {
-	private Class classCache;
-	private Method factoryCache;
-	private Constructor constructorCache;
-
-	public SpecificClassSchema(
-			String name, String namespace,
-			List imports, List fields) {
-		super(name, namespace, imports, fields);
-	}
-
-	@Override
-	@SuppressWarnings("unchecked")
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj == null) {
-			pk.packNil();
-			return;
-		}
-		if(classCache == null) {
-			cacheFactory();
-		}
-		if(classCache.isInstance(obj)) {
-			((MessagePackable)obj).messagePack(pk);
-		} else {
-			// FIXME Map
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		if(obj instanceof Collection) {
-			if(constructorCache == null) {
-				cacheConstructor();
-			}
-			try {
-				MessageConvertable o = (MessageConvertable)constructorCache.newInstance((Object[])null);
-				o.messageConvert(obj);
-				return o;
-			} catch (InvocationTargetException e) {
-				throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage());
-			} catch (InstantiationException e) {
-				throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage());
-			} catch (IllegalAccessException e) {
-				throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage());
-			}
-
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	public Schema getElementSchema(int index) {
-		// FIXME check index < fields.length
-		return fields[index].getSchema();
-	}
-
-	public Object createFromArray(Object[] obj) {
-		if(factoryCache == null) {
-			cacheFactory();
-		}
-		try {
-			return factoryCache.invoke(null, new Object[]{obj});
-		} catch (InvocationTargetException e) {
-			throw new RuntimeException("can't instantiate "+fqdn+": "+e.getCause());
-		} catch (IllegalAccessException e) {
-			throw new RuntimeException("can't instantiate "+fqdn+": "+e.getMessage());
-		}
-	}
-
-	@SuppressWarnings("unchecked")
-	private void cacheFactory() {
-		try {
-			classCache = Class.forName(fqdn);
-			factoryCache = classCache.getDeclaredMethod("createFromMessage", new Class[]{Object[].class});
-			factoryCache.setAccessible(true);
-		} catch(ClassNotFoundException e) {
-			throw new RuntimeException("class not found: "+fqdn);
-		} catch (NoSuchMethodException e) {
-			throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage());
-		}
-	}
-
-	@SuppressWarnings("unchecked")
-	private void cacheConstructor() {
-		try {
-			classCache = Class.forName(fqdn);
-			constructorCache = classCache.getDeclaredConstructor((Class[])null);
-			constructorCache.setAccessible(true);
-		} catch(ClassNotFoundException e) {
-			throw new RuntimeException("class not found: "+fqdn);
-		} catch (NoSuchMethodException e) {
-			throw new RuntimeException("class not found: "+fqdn+": "+e.getMessage());
-		}
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/schema/StringSchema.java b/java/src/main/java/org/msgpack/schema/StringSchema.java
deleted file mode 100644
index 23e4e64..0000000
--- a/java/src/main/java/org/msgpack/schema/StringSchema.java
+++ /dev/null
@@ -1,102 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack.schema;
-
-import java.nio.ByteBuffer;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import org.msgpack.*;
-
-public class StringSchema extends Schema {
-	public StringSchema() { }
-
-	@Override
-	public String getClassName() {
-		return "String";
-	}
-
-	@Override
-	public String getExpression() {
-		return "string";
-	}
-
-	@Override
-	public void pack(Packer pk, Object obj) throws IOException {
-		if(obj instanceof byte[]) {
-			byte[] b = (byte[])obj;
-			pk.packRaw(b.length);
-			pk.packRawBody(b);
-		} else if(obj instanceof String) {
-			try {
-				byte[] b = ((String)obj).getBytes("UTF-8");
-				pk.packRaw(b.length);
-				pk.packRawBody(b);
-			} catch (UnsupportedEncodingException e) {
-				throw new MessageTypeException();
-			}
-		} else if(obj == null) {
-			pk.packNil();
-		} else {
-			throw MessageTypeException.invalidConvert(obj, this);
-		}
-	}
-
-	public static final String convertString(Object obj) throws MessageTypeException {
-		if(obj instanceof byte[]) {
-			try {
-				return new String((byte[])obj, "UTF-8");
-			} catch (UnsupportedEncodingException e) {
-				throw new MessageTypeException();
-			}
-		} else if(obj instanceof String) {
-			return (String)obj;
-		} else if(obj instanceof ByteBuffer) {
-			ByteBuffer d = (ByteBuffer)obj;
-			try {
-				if(d.hasArray()) {
-					return new String(d.array(), d.position(), d.capacity(), "UTF-8");
-				} else {
-					byte[] v = new byte[d.capacity()];
-					int pos = d.position();
-					d.get(v);
-					d.position(pos);
-					return new String(v, "UTF-8");
-				}
-			} catch (UnsupportedEncodingException e) {
-				throw new MessageTypeException();
-			}
-		} else {
-			throw new MessageTypeException();
-		}
-	}
-
-	@Override
-	public Object convert(Object obj) throws MessageTypeException {
-		return convertString(obj);
-	}
-
-	@Override
-	public Object createFromRaw(byte[] b, int offset, int length) {
-		try {
-			return new String(b, offset, length, "UTF-8");
-		} catch (Exception e) {
-			throw new RuntimeException(e.getMessage());
-		}
-	}
-}
-
diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java
index b02bbb4..ca3d235 100644
--- a/java/src/test/java/org/msgpack/TestPackUnpack.java
+++ b/java/src/test/java/org/msgpack/TestPackUnpack.java
@@ -8,240 +8,223 @@ import org.junit.Test;
 import static org.junit.Assert.*;
 
 public class TestPackUnpack {
-    protected Object unpackOne(ByteArrayOutputStream out) {
-        return unpackOne(out, null);
-    }
-    protected Object unpackOne(ByteArrayOutputStream out, Schema schema) {
-        ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-        Unpacker upk = new Unpacker(in);
-        if (schema != null)
-            upk = upk.useSchema(schema);
-        Iterator it = upk.iterator();
-        assertEquals(true, it.hasNext());
-        Object obj = it.next();
-        assertEquals(false, it.hasNext());
-        return obj;
-    }
+	public MessagePackObject unpackOne(ByteArrayOutputStream out) {
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker upk = new Unpacker(in);
+		Iterator it = upk.iterator();
+		assertEquals(true, it.hasNext());
+		MessagePackObject obj = it.next();
+		assertEquals(false, it.hasNext());
+		return obj;
+	}
 
-    @Test
-    public void testInt() throws Exception {
-        testInt(0);
-        testInt(-1);
-        testInt(1);
-        testInt(Integer.MIN_VALUE);
-        testInt(Integer.MAX_VALUE);
-        Random rand = new Random();
-        for (int i = 0; i < 1000; i++)
-            testInt(rand.nextInt());
-    }
-    public void testInt(int val) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        Object obj = unpackOne(out);
-        if (obj instanceof Byte)
-            assertEquals(val, ((Byte)obj).intValue());
-        else if (obj instanceof Integer)
-            assertEquals(val, ((Integer)obj).intValue());
-        else if (obj instanceof Short)
-            assertEquals(val, ((Short)obj).intValue());
-        else if (obj instanceof Long)
-            assertEquals(val, ((Long)obj).intValue());
-        else {
-            System.out.println("Got unexpected class: " + obj.getClass());
-            assertTrue(false);
-        }
-    }
+	public void testInt(int val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		MessagePackObject obj = unpackOne(out);
+		assertEquals(val, obj.asInt());
+	}
+	@Test
+	public void testInt() throws Exception {
+		testInt(0);
+		testInt(-1);
+		testInt(1);
+		testInt(Integer.MIN_VALUE);
+		testInt(Integer.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testInt(rand.nextInt());
+	}
 
-    @Test
-    public void testFloat() throws Exception {
-        testFloat((float)0.0);
-        testFloat((float)-0.0);
-        testFloat((float)1.0);
-        testFloat((float)-1.0);
-        testFloat((float)Float.MAX_VALUE);
-        testFloat((float)Float.MIN_VALUE);
-        testFloat((float)Float.NaN);
-        testFloat((float)Float.NEGATIVE_INFINITY);
-        testFloat((float)Float.POSITIVE_INFINITY);
-        Random rand = new Random();
-        for (int i = 0; i < 1000; i++)
-            testFloat(rand.nextFloat());
-    }
-    public void testFloat(float val) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        Object obj = unpackOne(out);
-        if (obj instanceof Float)
-            assertEquals(val, ((Float)obj).floatValue(), 10e-10);
-        else {
-            System.out.println("Got unexpected class: " + obj.getClass());
-            assertTrue(false);
-        }
-    }
+	public void testFloat(float val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		MessagePackObject obj = unpackOne(out);
+		assertEquals(val, obj.asFloat(), 10e-10);
+	}
+	@Test
+	public void testFloat() throws Exception {
+		testFloat((float)0.0);
+		testFloat((float)-0.0);
+		testFloat((float)1.0);
+		testFloat((float)-1.0);
+		testFloat((float)Float.MAX_VALUE);
+		testFloat((float)Float.MIN_VALUE);
+		testFloat((float)Float.NaN);
+		testFloat((float)Float.NEGATIVE_INFINITY);
+		testFloat((float)Float.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testFloat(rand.nextFloat());
+	}
 
-    @Test
-    public void testDouble() throws Exception {
-        testDouble((double)0.0);
-        testDouble((double)-0.0);
-        testDouble((double)1.0);
-        testDouble((double)-1.0);
-        testDouble((double)Double.MAX_VALUE);
-        testDouble((double)Double.MIN_VALUE);
-        testDouble((double)Double.NaN);
-        testDouble((double)Double.NEGATIVE_INFINITY);
-        testDouble((double)Double.POSITIVE_INFINITY);
-        Random rand = new Random();
-        for (int i = 0; i < 1000; i++)
-            testDouble(rand.nextDouble());
-    }
-    public void testDouble(double val) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        Object obj = unpackOne(out);
-        if (obj instanceof Double)
-            assertEquals(val, ((Double)obj).doubleValue(), 10e-10);
-        else {
-            System.out.println("Got unexpected class: " + obj.getClass());
-            assertTrue(false);
-        }
-    }
+	public void testDouble(double val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		MessagePackObject obj = unpackOne(out);
+		assertEquals(val, obj.asDouble(), 10e-10);
+	}
+	@Test
+	public void testDouble() throws Exception {
+		testDouble((double)0.0);
+		testDouble((double)-0.0);
+		testDouble((double)1.0);
+		testDouble((double)-1.0);
+		testDouble((double)Double.MAX_VALUE);
+		testDouble((double)Double.MIN_VALUE);
+		testDouble((double)Double.NaN);
+		testDouble((double)Double.NEGATIVE_INFINITY);
+		testDouble((double)Double.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testDouble(rand.nextDouble());
+	}
 
-    @Test
-    public void testNil() throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).packNil();
-        Object obj = unpackOne(out);
-        assertEquals(null, obj);
-    }
+	@Test
+	public void testNil() throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).packNil();
+		MessagePackObject obj = unpackOne(out);
+		assertTrue(obj.isNull());
+	}
 
-    @Test
-    public void testBoolean() throws Exception {
-        testBoolean(false);
-        testBoolean(true);
-    }
-    public void testBoolean(boolean val) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        Object obj = unpackOne(out);
-        if (obj instanceof Boolean)
-            assertEquals(val, ((Boolean)obj).booleanValue());
-        else {
-            System.out.println("Got unexpected class: " + obj.getClass());
-            assertTrue(false);
-        }
-    }
+	@Test
+	public void testBoolean() throws Exception {
+		testBoolean(false);
+		testBoolean(true);
+	}
+	public void testBoolean(boolean val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		MessagePackObject obj = unpackOne(out);
+		assertEquals(val, obj.asBoolean());
+	}
 
-    @Test
-    public void testString() throws Exception {
-        testString("");
-        testString("a");
-        testString("ab");
-        testString("abc");
-        // small size string
-        for (int i = 0; i < 100; i++) {
-            StringBuilder sb = new StringBuilder();
-            int len = (int)Math.random() % 31 + 1;
-            for (int j = 0; j < len; j++)
-                sb.append('a' + ((int)Math.random()) & 26);
-            testString(sb.toString());
-        }
-        // medium size string
-        for (int i = 0; i < 100; i++) {
-            StringBuilder sb = new StringBuilder();
-            int len = (int)Math.random() % 100 + (1 << 15);
-            for (int j = 0; j < len; j++)
-                sb.append('a' + ((int)Math.random()) & 26);
-            testString(sb.toString());
-        }
-        // large size string
-        for (int i = 0; i < 10; i++) {
-            StringBuilder sb = new StringBuilder();
-            int len = (int)Math.random() % 100 + (1 << 31);
-            for (int j = 0; j < len; j++)
-                sb.append('a' + ((int)Math.random()) & 26);
-            testString(sb.toString());
-        }
-    }
-    public void testString(String val) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        Object obj = unpackOne(out);
-        if (obj instanceof byte[])
-            assertEquals(val, new String((byte[])obj));
-        else {
-            System.out.println("obj=" + obj);
-            System.out.println("Got unexpected class: " + obj.getClass());
-            assertTrue(false);
-        }
-    }
+	public void testString(String val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		MessagePackObject obj = unpackOne(out);
+		assertEquals(val, obj.asString());
+	}
+	@Test
+	public void testString() throws Exception {
+		testString("");
+		testString("a");
+		testString("ab");
+		testString("abc");
 
-    @Test
-    public void testArray() throws Exception {
-        List emptyList = new ArrayList();
-        testArray(emptyList, Schema.parse("(array int)"));
+		// small size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int)Math.random() % 31 + 1;
+			for (int j = 0; j < len; j++)
+				sb.append('a' + ((int)Math.random()) & 26);
+			testString(sb.toString());
+		}
 
-        for (int i = 0; i < 1000; i++) {
-            Schema schema = Schema.parse("(array int)");
-            List l = new ArrayList();
-            int len = (int)Math.random() % 1000 + 1;
-            for (int j = 0; j < len; j++)
-                l.add(j);
-            testArray(l, schema);
-        }
-        for (int i = 0; i < 1000; i++) {
-            Schema schema = Schema.parse("(array string)");
-            List l = new ArrayList();
-            int len = (int)Math.random() % 1000 + 1;
-            for (int j = 0; j < len; j++)
-                l.add(Integer.toString(j));
-            testArray(l, schema);
-        }
-    }
-    public void testArray(List val, Schema schema) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        Object obj = unpackOne(out, schema);
-        if (obj instanceof List)
-            assertTrue(val.equals(obj));
-        else {
-            System.out.println("obj=" + obj);
-            System.out.println("Got unexpected class: " + obj.getClass());
-            assertTrue(false);
-        }
-    }
+		// medium size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int)Math.random() % 100 + (1 << 15);
+			for (int j = 0; j < len; j++)
+				sb.append('a' + ((int)Math.random()) & 26);
+			testString(sb.toString());
+		}
 
-    @Test
-    public void testMap() throws Exception {
-        Map emptyMap = new HashMap();
-        testMap(emptyMap, Schema.parse("(map int int)"));
+		// large size string
+		for (int i = 0; i < 10; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int)Math.random() % 100 + (1 << 31);
+			for (int j = 0; j < len; j++)
+				sb.append('a' + ((int)Math.random()) & 26);
+			testString(sb.toString());
+		}
+	}
 
-        for (int i = 0; i < 1000; i++) {
-            Schema schema = Schema.parse("(map int int)");
-            Map m = new HashMap();
-            int len = (int)Math.random() % 1000 + 1;
-            for (int j = 0; j < len; j++)
-                m.put(j, j);
-            testMap(m, schema);
-        }
-        for (int i = 0; i < 1000; i++) {
-            Schema schema = Schema.parse("(map string int)");
-            Map m = new HashMap();
-            int len = (int)Math.random() % 1000 + 1;
-            for (int j = 0; j < len; j++)
-                m.put(Integer.toString(j), j);
-            testMap(m, schema);
-        }
-    }
-    public void testMap(Map val, Schema schema) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        Object obj = unpackOne(out, schema);
-        if (obj instanceof Map)
-            assertTrue(val.equals(obj));
-        else {
-            System.out.println("obj=" + obj);
-            System.out.println("Got unexpected class: " + obj.getClass());
-            assertTrue(false);
-        }
-    }
+	@Test
+	public void testArray() throws Exception {
+		List emptyList = new ArrayList();
+		{
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(emptyList);
+			MessagePackObject obj = unpackOne(out);
+			assertEquals(emptyList, obj.asList());
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			List l = new ArrayList();
+			int len = (int)Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++)
+				l.add(j);
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(l);
+			MessagePackObject obj = unpackOne(out);
+			List list = obj.asList();
+			assertEquals(l.size(), list.size());
+			for (int j = 0; j < len; j++) {
+				assertEquals(l.get(j).intValue(), list.get(j).asInt());
+			}
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			List l = new ArrayList();
+			int len = (int)Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++)
+				l.add(Integer.toString(j));
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(l);
+			MessagePackObject obj = unpackOne(out);
+			List list = obj.asList();
+			assertEquals(l.size(), list.size());
+			for (int j = 0; j < len; j++) {
+				assertEquals(l.get(j), list.get(j).asString());
+			}
+		}
+	}
+
+	@Test
+	public void testMap() throws Exception {
+		Map emptyMap = new HashMap();
+		{
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(emptyMap);
+			MessagePackObject obj = unpackOne(out);
+			assertEquals(emptyMap, obj.asMap());
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			Map m = new HashMap();
+			int len = (int)Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++)
+				m.put(j, j);
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(m);
+			MessagePackObject obj = unpackOne(out);
+			Map map = obj.asMap();
+			assertEquals(m.size(), map.size());
+			for (Map.Entry pair : map.entrySet()) {
+				Integer val = m.get(pair.getKey().asInt());
+				assertNotNull(val);
+				assertEquals(val.intValue(), pair.getValue().asInt());
+			}
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			Map m = new HashMap();
+			int len = (int)Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++)
+				m.put(Integer.toString(j), j);
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(m);
+			MessagePackObject obj = unpackOne(out);
+			Map map = obj.asMap();
+			assertEquals(m.size(), map.size());
+			for (Map.Entry pair : map.entrySet()) {
+				Integer val = m.get(pair.getKey().asString());
+				assertNotNull(val);
+				assertEquals(val.intValue(), pair.getValue().asInt());
+			}
+		}
+	}
 };
+

From 5658ca5b903b294dcccad0dfa6fa6a7bcd9acc12 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Wed, 18 Aug 2010 22:24:51 +0900
Subject: [PATCH 0602/1648] java: adds ObjectEquals test

---
 .../main/java/org/msgpack/UnpackerImpl.java   | 30 +++---
 .../java/org/msgpack/object/ArrayType.java    |  6 +-
 .../msgpack/object/BigIntegerTypeIMPL.java    |  2 +-
 .../java/org/msgpack/object/BooleanType.java  |  6 +-
 .../msgpack/object/LongIntegerTypeIMPL.java   |  2 +-
 .../main/java/org/msgpack/object/MapType.java |  6 +-
 .../main/java/org/msgpack/object/NilType.java |  6 ++
 .../main/java/org/msgpack/object/RawType.java | 20 +++-
 .../java/org/msgpack/TestObjectEquals.java    | 96 +++++++++++++++++++
 9 files changed, 152 insertions(+), 22 deletions(-)
 create mode 100644 java/src/test/java/org/msgpack/TestObjectEquals.java

diff --git a/java/src/main/java/org/msgpack/UnpackerImpl.java b/java/src/main/java/org/msgpack/UnpackerImpl.java
index f004e6c..cfd3d22 100644
--- a/java/src/main/java/org/msgpack/UnpackerImpl.java
+++ b/java/src/main/java/org/msgpack/UnpackerImpl.java
@@ -124,7 +124,7 @@ public class UnpackerImpl {
 					if((b & 0xe0) == 0xa0) {  // FixRaw
 						trail = b & 0x1f;
 						if(trail == 0) {
-							obj = new RawType(new byte[0]);
+							obj = RawType.create(new byte[0]);
 							break _push;
 						}
 						cs = ACS_RAW_VALUE;
@@ -139,7 +139,7 @@ public class UnpackerImpl {
 						//System.out.println("fixarray count:"+count);
 						obj = new MessagePackObject[count];
 						if(count == 0) {
-							obj = new ArrayType((MessagePackObject[])obj);
+							obj = ArrayType.create((MessagePackObject[])obj);
 							break _push;
 						}
 						++top;
@@ -159,7 +159,7 @@ public class UnpackerImpl {
 						count = b & 0x0f;
 						obj = new MessagePackObject[count*2];
 						if(count == 0) {
-							obj = new MapType((MessagePackObject[])obj);
+							obj = MapType.create((MessagePackObject[])obj);
 							break _push;
 						}
 						//System.out.println("fixmap count:"+count);
@@ -175,13 +175,13 @@ public class UnpackerImpl {
 	
 					switch(b & 0xff) {    // FIXME
 					case 0xc0:  // nil
-						obj = new NilType();
+						obj = NilType.create();
 						break _push;
 					case 0xc2:  // false
-						obj = new BooleanType(false);
+						obj = BooleanType.create(false);
 						break _push;
 					case 0xc3:  // true
-						obj = new BooleanType(true);
+						obj = BooleanType.create(true);
 						break _push;
 					case 0xca:  // float
 					case 0xcb:  // double
@@ -293,7 +293,7 @@ public class UnpackerImpl {
 						castBuffer.put(src, n, 2);
 						trail = ((int)castBuffer.getShort(0)) & 0xffff;
 						if(trail == 0) {
-							obj = new RawType(new byte[0]);
+							obj = RawType.create(new byte[0]);
 							break _push;
 						}
 						cs = ACS_RAW_VALUE;
@@ -304,14 +304,14 @@ public class UnpackerImpl {
 						// FIXME overflow check
 						trail = castBuffer.getInt(0) & 0x7fffffff;
 						if(trail == 0) {
-							obj = new RawType(new byte[0]);
+							obj = RawType.create(new byte[0]);
 							break _push;
 						}
 						cs = ACS_RAW_VALUE;
 					case ACS_RAW_VALUE: {
 							byte[] raw = new byte[trail];
 							System.arraycopy(src, n, raw, 0, trail);
-							obj = new RawType(raw);
+							obj = RawType.create(raw);
 						}
 						break _push;
 					case CS_ARRAY_16:
@@ -323,7 +323,7 @@ public class UnpackerImpl {
 						count = ((int)castBuffer.getShort(0)) & 0xffff;
 						obj = new MessagePackObject[count];
 						if(count == 0) {
-							obj = new ArrayType((MessagePackObject[])obj);
+							obj = ArrayType.create((MessagePackObject[])obj);
 							break _push;
 						}
 						++top;
@@ -344,7 +344,7 @@ public class UnpackerImpl {
 						count = castBuffer.getInt(0) & 0x7fffffff;
 						obj = new MessagePackObject[count];
 						if(count == 0) {
-							obj = new ArrayType((MessagePackObject[])obj);
+							obj = ArrayType.create((MessagePackObject[])obj);
 							break _push;
 						}
 						++top;
@@ -364,7 +364,7 @@ public class UnpackerImpl {
 						count = ((int)castBuffer.getShort(0)) & 0xffff;
 						obj = new MessagePackObject[count*2];
 						if(count == 0) {
-							obj = new MapType((MessagePackObject[])obj);
+							obj = MapType.create((MessagePackObject[])obj);
 							break _push;
 						}
 						//System.out.println("fixmap count:"+count);
@@ -386,7 +386,7 @@ public class UnpackerImpl {
 						count = castBuffer.getInt(0) & 0x7fffffff;
 						obj = new MessagePackObject[count*2];
 						if(count == 0) {
-							obj = new MapType((MessagePackObject[])obj);
+							obj = MapType.create((MessagePackObject[])obj);
 							break _push;
 						}
 						//System.out.println("fixmap count:"+count);
@@ -425,7 +425,7 @@ public class UnpackerImpl {
 							top_obj    = stack_obj[top];
 							top_ct     = stack_ct[top];
 							top_count  = stack_count[top];
-							obj = new ArrayType((MessagePackObject[])ar);
+							obj = ArrayType.create((MessagePackObject[])ar);
 							stack_obj[top] = null;
 							--top;
 							break _push;
@@ -447,7 +447,7 @@ public class UnpackerImpl {
 							top_obj    = stack_obj[top];
 							top_ct     = stack_ct[top];
 							top_count  = stack_count[top];
-							obj = new MapType((MessagePackObject[])mp);
+							obj = MapType.create((MessagePackObject[])mp);
 							stack_obj[top] = null;
 							--top;
 							break _push;
diff --git a/java/src/main/java/org/msgpack/object/ArrayType.java b/java/src/main/java/org/msgpack/object/ArrayType.java
index 350ce32..36134dc 100644
--- a/java/src/main/java/org/msgpack/object/ArrayType.java
+++ b/java/src/main/java/org/msgpack/object/ArrayType.java
@@ -25,10 +25,14 @@ import org.msgpack.*;
 public class ArrayType extends MessagePackObject {
 	private MessagePackObject[] array;
 
-	public ArrayType(MessagePackObject[] array) {
+	ArrayType(MessagePackObject[] array) {
 		this.array = array;
 	}
 
+	public static ArrayType create(MessagePackObject[] array) {
+		return new ArrayType(array);
+	}
+
 	@Override
 	public boolean isArrayType() {
 		return true;
diff --git a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
index 7b060ee..b29879f 100644
--- a/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/BigIntegerTypeIMPL.java
@@ -109,7 +109,7 @@ class BigIntegerTypeIMPL extends IntegerType {
 	public boolean equals(Object obj) {
 		if(obj.getClass() != getClass()) {
 			if(obj.getClass() == ShortIntegerTypeIMPL.class) {
-				return BigInteger.valueOf((long)((ShortIntegerTypeIMPL)obj).shortValue()).equals(value);
+				return BigInteger.valueOf(((ShortIntegerTypeIMPL)obj).longValue()).equals(value);
 			} else if(obj.getClass() == LongIntegerTypeIMPL.class) {
 				return BigInteger.valueOf(((LongIntegerTypeIMPL)obj).longValue()).equals(value);
 			}
diff --git a/java/src/main/java/org/msgpack/object/BooleanType.java b/java/src/main/java/org/msgpack/object/BooleanType.java
index 1d12c1c..65bd42b 100644
--- a/java/src/main/java/org/msgpack/object/BooleanType.java
+++ b/java/src/main/java/org/msgpack/object/BooleanType.java
@@ -23,10 +23,14 @@ import org.msgpack.*;
 public class BooleanType extends MessagePackObject {
 	private boolean value;
 
-	public BooleanType(boolean value) {
+	BooleanType(boolean value) {
 		this.value = value;
 	}
 
+	public static BooleanType create(boolean value) {
+		return new BooleanType(value);
+	}
+
 	@Override
 	public boolean isBooleanType() {
 		return true;
diff --git a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
index 3928a29..d052e77 100644
--- a/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
+++ b/java/src/main/java/org/msgpack/object/LongIntegerTypeIMPL.java
@@ -108,7 +108,7 @@ class LongIntegerTypeIMPL extends IntegerType {
 			if(obj.getClass() == ShortIntegerTypeIMPL.class) {
 				return value == ((ShortIntegerTypeIMPL)obj).longValue();
 			} else if(obj.getClass() == BigIntegerTypeIMPL.class) {
-				return (long)value == ((BigIntegerTypeIMPL)obj).longValue();
+				return BigInteger.valueOf(value).equals(((BigIntegerTypeIMPL)obj).bigIntegerValue());
 			}
 			return false;
 		}
diff --git a/java/src/main/java/org/msgpack/object/MapType.java b/java/src/main/java/org/msgpack/object/MapType.java
index 619d388..148c0bf 100644
--- a/java/src/main/java/org/msgpack/object/MapType.java
+++ b/java/src/main/java/org/msgpack/object/MapType.java
@@ -26,10 +26,14 @@ import org.msgpack.*;
 public class MapType extends MessagePackObject {
 	private MessagePackObject[] map;
 
-	public MapType(MessagePackObject[] map) {
+	MapType(MessagePackObject[] map) {
 		this.map = map;
 	}
 
+	public static MapType create(MessagePackObject[] map) {
+		return new MapType(map);
+	}
+
 	@Override
 	public boolean isMapType() {
 		return true;
diff --git a/java/src/main/java/org/msgpack/object/NilType.java b/java/src/main/java/org/msgpack/object/NilType.java
index ece62f0..d0572f1 100644
--- a/java/src/main/java/org/msgpack/object/NilType.java
+++ b/java/src/main/java/org/msgpack/object/NilType.java
@@ -21,6 +21,12 @@ import java.io.IOException;
 import org.msgpack.*;
 
 public class NilType extends MessagePackObject {
+	private static NilType instance = new NilType();
+
+	public static NilType create() {
+		return instance;
+	}
+
 	@Override
 	public boolean isNull() {
 		return true;
diff --git a/java/src/main/java/org/msgpack/object/RawType.java b/java/src/main/java/org/msgpack/object/RawType.java
index 3a39486..26f6e0d 100644
--- a/java/src/main/java/org/msgpack/object/RawType.java
+++ b/java/src/main/java/org/msgpack/object/RawType.java
@@ -24,10 +24,26 @@ import org.msgpack.*;
 public class RawType extends MessagePackObject {
 	private byte[] bytes;
 
-	public RawType(byte[] bytes) {
+	RawType(byte[] bytes) {
 		this.bytes = bytes;
 	}
 
+	RawType(String str) {
+		try {
+			this.bytes = str.getBytes("UTF-8");
+		} catch (Exception e) {
+			throw new MessageTypeException("type error");
+		}
+	}
+
+	public static RawType create(byte[] bytes) {
+		return new RawType(bytes);
+	}
+
+	public static RawType create(String str) {
+		return new RawType(str);
+	}
+
 	@Override
 	public boolean isRawType() {
 		return true;
@@ -58,7 +74,7 @@ public class RawType extends MessagePackObject {
 		if(obj.getClass() != getClass()) {
 			return false;
 		}
-		return ((RawType)obj).bytes.equals(bytes);
+		return Arrays.equals(((RawType)obj).bytes, bytes);
 	}
 
 	@Override
diff --git a/java/src/test/java/org/msgpack/TestObjectEquals.java b/java/src/test/java/org/msgpack/TestObjectEquals.java
new file mode 100644
index 0000000..b2b018d
--- /dev/null
+++ b/java/src/test/java/org/msgpack/TestObjectEquals.java
@@ -0,0 +1,96 @@
+package org.msgpack;
+
+import org.msgpack.*;
+import org.msgpack.object.*;
+import java.math.BigInteger;
+import java.util.*;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestObjectEquals {
+	public void testInt(int val) throws Exception {
+		MessagePackObject objInt = IntegerType.create(val);
+		MessagePackObject objLong = IntegerType.create((long)val);
+		MessagePackObject objBigInt = IntegerType.create(BigInteger.valueOf((long)val));
+		assertTrue(objInt.equals(objInt));
+		assertTrue(objInt.equals(objLong));
+		assertTrue(objInt.equals(objBigInt));
+		assertTrue(objLong.equals(objInt));
+		assertTrue(objLong.equals(objLong));
+		assertTrue(objLong.equals(objBigInt));
+		assertTrue(objBigInt.equals(objInt));
+		assertTrue(objBigInt.equals(objLong));
+		assertTrue(objBigInt.equals(objBigInt));
+	}
+	@Test
+	public void testInt() throws Exception {
+		testInt(0);
+		testInt(-1);
+		testInt(1);
+		testInt(Integer.MIN_VALUE);
+		testInt(Integer.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testInt(rand.nextInt());
+	}
+
+	public void testLong(long val) throws Exception {
+		MessagePackObject objInt = IntegerType.create((int)val);
+		MessagePackObject objLong = IntegerType.create(val);
+		MessagePackObject objBigInt = IntegerType.create(BigInteger.valueOf(val));
+		if(val > (long)Integer.MAX_VALUE || val < (long)Integer.MIN_VALUE) {
+			assertTrue(objInt.equals(objInt));
+			assertFalse(objInt.equals(objLong));
+			assertFalse(objInt.equals(objBigInt));
+			assertFalse(objLong.equals(objInt));
+			assertTrue(objLong.equals(objLong));
+			assertTrue(objLong.equals(objBigInt));
+			assertFalse(objBigInt.equals(objInt));
+			assertTrue(objBigInt.equals(objLong));
+			assertTrue(objBigInt.equals(objBigInt));
+		} else {
+			assertTrue(objInt.equals(objInt));
+			assertTrue(objInt.equals(objLong));
+			assertTrue(objInt.equals(objBigInt));
+			assertTrue(objLong.equals(objInt));
+			assertTrue(objLong.equals(objLong));
+			assertTrue(objLong.equals(objBigInt));
+			assertTrue(objBigInt.equals(objInt));
+			assertTrue(objBigInt.equals(objLong));
+			assertTrue(objBigInt.equals(objBigInt));
+		}
+	}
+	@Test
+	public void testLong() throws Exception {
+		testLong(0);
+		testLong(-1);
+		testLong(1);
+		testLong(Integer.MIN_VALUE);
+		testLong(Integer.MAX_VALUE);
+		testLong(Long.MIN_VALUE);
+		testLong(Long.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testLong(rand.nextLong());
+	}
+
+	@Test
+	public void testNil() throws Exception {
+		assertTrue(NilType.create().equals(NilType.create()));
+		assertFalse(NilType.create().equals(IntegerType.create(0)));
+		assertFalse(NilType.create().equals(BooleanType.create(false)));
+	}
+
+	public void testString(String str) throws Exception {
+		assertTrue(RawType.create(str).equals(RawType.create(str)));
+	}
+	@Test
+	public void testString() throws Exception {
+		testString("");
+		testString("a");
+		testString("ab");
+		testString("abc");
+	}
+}
+

From fdfabc9f8862da5982830017e21b0628a00356eb Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Wed, 18 Aug 2010 22:51:23 +0900
Subject: [PATCH 0603/1648] java: adds cross-language test case

---
 .../main/java/org/msgpack/UnpackerImpl.java   |   1 +
 java/src/test/java/org/msgpack/TestCases.java |  46 ++++++++++++++++++
 java/src/test/resources/cases.json            |   1 +
 java/src/test/resources/cases.mpac            | Bin 0 -> 213 bytes
 java/src/test/resources/cases_compact.mpac    | Bin 0 -> 116 bytes
 5 files changed, 48 insertions(+)
 create mode 100644 java/src/test/java/org/msgpack/TestCases.java
 create mode 100644 java/src/test/resources/cases.json
 create mode 100644 java/src/test/resources/cases.mpac
 create mode 100644 java/src/test/resources/cases_compact.mpac

diff --git a/java/src/main/java/org/msgpack/UnpackerImpl.java b/java/src/main/java/org/msgpack/UnpackerImpl.java
index cfd3d22..d4f99e3 100644
--- a/java/src/main/java/org/msgpack/UnpackerImpl.java
+++ b/java/src/main/java/org/msgpack/UnpackerImpl.java
@@ -308,6 +308,7 @@ public class UnpackerImpl {
 							break _push;
 						}
 						cs = ACS_RAW_VALUE;
+						break _fixed_trail_again;
 					case ACS_RAW_VALUE: {
 							byte[] raw = new byte[trail];
 							System.arraycopy(src, n, raw, 0, trail);
diff --git a/java/src/test/java/org/msgpack/TestCases.java b/java/src/test/java/org/msgpack/TestCases.java
new file mode 100644
index 0000000..632645f
--- /dev/null
+++ b/java/src/test/java/org/msgpack/TestCases.java
@@ -0,0 +1,46 @@
+package org.msgpack;
+
+import java.io.*;
+import java.util.*;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestCases {
+	public void feedFile(Unpacker pac, String path) throws Exception {
+		FileInputStream input = new FileInputStream(path);
+		byte[] buffer = new byte[32*1024];
+		while(true) {
+			int count = input.read(buffer);
+			if(count < 0) {
+				break;
+			}
+			pac.feed(buffer, 0, count);
+		}
+	}
+
+	@Test
+	public void testCases() throws Exception {
+		System.out.println( new File(".").getAbsoluteFile().getParent() );
+
+
+		Unpacker pac = new Unpacker();
+		Unpacker pac_compact = new Unpacker();
+
+		feedFile(pac, "src/test/resources/cases.mpac");
+		feedFile(pac_compact, "src/test/resources/cases_compact.mpac");
+
+		UnpackResult result = new UnpackResult();
+		while(pac.next(result)) {
+			UnpackResult result_compact = new UnpackResult();
+			assertTrue( pac_compact.next(result_compact) );
+			System.out.println("obj: "+result_compact.getData());
+			if(!result.getData().equals(result_compact.getData())) {
+				System.out.println("compact: "+result_compact.getData().asString());
+				System.out.println("data   : "+result.getData().asString());
+			}
+			assertTrue( result.getData().equals(result_compact.getData()) );
+		}
+	}
+};
+
diff --git a/java/src/test/resources/cases.json b/java/src/test/resources/cases.json
new file mode 100644
index 0000000..fd390d4
--- /dev/null
+++ b/java/src/test/resources/cases.json
@@ -0,0 +1 @@
+[false,true,null,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,127,127,255,65535,4294967295,-32,-32,-128,-32768,-2147483648,0.0,-0.0,1.0,-1.0,"a","a","a","","","",[0],[0],[0],[],[],[],{},{},{},{"a":97},{"a":97},{"a":97},[[]],[["a"]]]
\ No newline at end of file
diff --git a/java/src/test/resources/cases.mpac b/java/src/test/resources/cases.mpac
new file mode 100644
index 0000000000000000000000000000000000000000..5ec08c6a9af42d9568eb429a209a639616e94711
GIT binary patch
literal 213
zcmXYp!4<+V3`3R4n8lOSY|v~#CV>aXw2-v7Qc6c)17ihrkbe}**V_dHM&J(DgGLop
zU?R;l%8FI9$y_sy>V|HFdDW~{neAn-roN|Wd+OcH160;F91fo!
Date: Wed, 18 Aug 2010 22:55:50 +0900
Subject: [PATCH 0604/1648] java: fixes cross-language test case

---
 java/src/test/java/org/msgpack/TestCases.java | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/java/src/test/java/org/msgpack/TestCases.java b/java/src/test/java/org/msgpack/TestCases.java
index 632645f..c368972 100644
--- a/java/src/test/java/org/msgpack/TestCases.java
+++ b/java/src/test/java/org/msgpack/TestCases.java
@@ -21,9 +21,6 @@ public class TestCases {
 
 	@Test
 	public void testCases() throws Exception {
-		System.out.println( new File(".").getAbsoluteFile().getParent() );
-
-
 		Unpacker pac = new Unpacker();
 		Unpacker pac_compact = new Unpacker();
 
@@ -34,13 +31,10 @@ public class TestCases {
 		while(pac.next(result)) {
 			UnpackResult result_compact = new UnpackResult();
 			assertTrue( pac_compact.next(result_compact) );
-			System.out.println("obj: "+result_compact.getData());
-			if(!result.getData().equals(result_compact.getData())) {
-				System.out.println("compact: "+result_compact.getData().asString());
-				System.out.println("data   : "+result.getData().asString());
-			}
 			assertTrue( result.getData().equals(result_compact.getData()) );
 		}
+
+		assertFalse( pac_compact.next(result) );
 	}
 };
 

From 40dc9de6c9bc90a95e55d5d2999f7e45d34cabc8 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Wed, 18 Aug 2010 23:37:47 +0900
Subject: [PATCH 0605/1648] java: supports packing/unpacking of BigInteger less
 than 0xffffffffffffffff

---
 .../java/org/msgpack/MessageConvertable.java  |  2 +-
 java/src/main/java/org/msgpack/Packer.java    | 22 ++++++++-------
 .../main/java/org/msgpack/UnpackerImpl.java   |  5 ++--
 .../test/java/org/msgpack/TestPackUnpack.java | 27 +++++++++++++++++++
 4 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/java/src/main/java/org/msgpack/MessageConvertable.java b/java/src/main/java/org/msgpack/MessageConvertable.java
index da251dc..8acf1f2 100644
--- a/java/src/main/java/org/msgpack/MessageConvertable.java
+++ b/java/src/main/java/org/msgpack/MessageConvertable.java
@@ -18,6 +18,6 @@
 package org.msgpack;
 
 public interface MessageConvertable {
-	public void messageConvert(Object obj) throws MessageTypeException;
+	public void messageConvert(MessagePackObject obj) throws MessageTypeException;
 }
 
diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index 98af3d6..60e04bf 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -196,19 +196,19 @@ public class Packer {
 	}
 
 	public Packer packBigInteger(BigInteger d) throws IOException {
-		if(d.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0) {
+		if(d.bitLength() <= 63) {
 			return packLong(d.longValue());
-		} else if(d.bitLength() <= 64) {
+		} else if(d.bitLength() <= 64 && d.signum() >= 0) {
 			castBytes[0] = (byte)0xcf;
 			byte[] barray = d.toByteArray();
-			castBytes[1] = barray[0];
-			castBytes[2] = barray[1];
-			castBytes[3] = barray[2];
-			castBytes[4] = barray[3];
-			castBytes[5] = barray[4];
-			castBytes[6] = barray[5];
-			castBytes[7] = barray[6];
-			castBytes[8] = barray[7];
+			castBytes[1] = barray[barray.length-8];
+			castBytes[2] = barray[barray.length-7];
+			castBytes[3] = barray[barray.length-6];
+			castBytes[4] = barray[barray.length-5];
+			castBytes[5] = barray[barray.length-4];
+			castBytes[6] = barray[barray.length-3];
+			castBytes[7] = barray[barray.length-2];
+			castBytes[8] = barray[barray.length-1];
 			out.write(castBytes);
 			return this;
 		} else {
@@ -436,6 +436,8 @@ public class Packer {
 			return packFloat((Float)o);
 		} else if(o instanceof Double) {
 			return packDouble((Double)o);
+		} else if(o instanceof BigInteger) {
+			return packBigInteger((BigInteger)o);
 		} else {
 			throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")");
 		}
diff --git a/java/src/main/java/org/msgpack/UnpackerImpl.java b/java/src/main/java/org/msgpack/UnpackerImpl.java
index d4f99e3..6a7085f 100644
--- a/java/src/main/java/org/msgpack/UnpackerImpl.java
+++ b/java/src/main/java/org/msgpack/UnpackerImpl.java
@@ -18,6 +18,7 @@
 package org.msgpack;
 
 import java.nio.ByteBuffer;
+import java.math.BigInteger;
 import org.msgpack.object.*;
 
 public class UnpackerImpl {
@@ -262,9 +263,7 @@ public class UnpackerImpl {
 						{
 							long o = castBuffer.getLong(0);
 							if(o < 0) {
-								// FIXME
-								//obj = GenericBigInteger.valueOf(o & 0x7fffffffL).setBit(31);
-								throw new UnpackException("uint 64 bigger than 0x7fffffff is not supported");
+								obj = IntegerType.create(new BigInteger(1, castBuffer.array()));
 							} else {
 								obj = IntegerType.create(o);
 							}
diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java
index ca3d235..75c5fe7 100644
--- a/java/src/test/java/org/msgpack/TestPackUnpack.java
+++ b/java/src/test/java/org/msgpack/TestPackUnpack.java
@@ -3,6 +3,7 @@ package org.msgpack;
 import org.msgpack.*;
 import java.io.*;
 import java.util.*;
+import java.math.BigInteger;
 
 import org.junit.Test;
 import static org.junit.Assert.*;
@@ -36,6 +37,32 @@ public class TestPackUnpack {
 			testInt(rand.nextInt());
 	}
 
+	public void testBigInteger(BigInteger val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		MessagePackObject obj = unpackOne(out);
+		if(!val.equals(obj.asBigInteger())) {
+			System.out.println("expect: "+val);
+			System.out.println("but   : "+obj.asBigInteger());
+		}
+		assertEquals(val, obj.asBigInteger());
+	}
+	@Test
+	public void testBigInteger() throws Exception {
+		testBigInteger(BigInteger.valueOf(0));
+		testBigInteger(BigInteger.valueOf(-1));
+		testBigInteger(BigInteger.valueOf(1));
+		testBigInteger(BigInteger.valueOf(Integer.MIN_VALUE));
+		testBigInteger(BigInteger.valueOf(Integer.MAX_VALUE));
+		testBigInteger(BigInteger.valueOf(Long.MIN_VALUE));
+		testBigInteger(BigInteger.valueOf(Long.MAX_VALUE));
+		BigInteger max = BigInteger.valueOf(Long.MAX_VALUE).setBit(63);
+		testBigInteger(max);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testBigInteger( max.subtract(BigInteger.valueOf( Math.abs(rand.nextLong()) )) );
+	}
+
 	public void testFloat(float val) throws Exception {
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		new Packer(out).pack(val);

From 48da2b8353a640de6bdc4e59c7fe91b640321b27 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Wed, 18 Aug 2010 23:47:31 +0900
Subject: [PATCH 0606/1648] java: adds Packer.pack()

---
 java/src/main/java/org/msgpack/Packer.java    | 103 ++++++++++++------
 .../test/java/org/msgpack/TestPackUnpack.java |  24 +++-
 2 files changed, 89 insertions(+), 38 deletions(-)

diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index 60e04bf..0ac8d89 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -315,39 +315,36 @@ public class Packer {
 	}
 
 
-	public Packer pack(String o) throws IOException {
-		if(o == null) { return packNil(); }
-		return packString(o);
-	}
-
-	public Packer pack(MessagePackable o) throws IOException {
-		if(o == null) { return packNil(); }
-		o.messagePack(this);
-		return this;
-	}
-
-	public Packer pack(byte[] o) throws IOException {
-		if(o == null) { return packNil(); }
-		packRaw(o.length);
-		return packRawBody(o);
-	}
-
-	public Packer pack(List o) throws IOException {
-		if(o == null) { return packNil(); }
-		packArray(o.size());
-		for(Object i : o) { pack(i); }
-		return this;
-	}
-
-	@SuppressWarnings("unchecked")
-	public Packer pack(Map o) throws IOException {
-		if(o == null) { return packNil(); }
-		packMap(o.size());
-		for(Map.Entry e : ((Map)o).entrySet()) {
-			pack(e.getKey());
-			pack(e.getValue());
+	public Packer pack(boolean o) throws IOException {
+		if(o) {
+			return packTrue();
+		} else {
+			return packFalse();
 		}
-		return this;
+	}
+
+	public Packer pack(byte o) throws IOException {
+		return packByte(o);
+	}
+
+	public Packer pack(short o) throws IOException {
+		return packShort(o);
+	}
+
+	public Packer pack(int o) throws IOException {
+		return packInt(o);
+	}
+
+	public Packer pack(long o) throws IOException {
+		return packLong(o);
+	}
+
+	public Packer pack(float o) throws IOException {
+		return packFloat(o);
+	}
+
+	public Packer pack(double o) throws IOException {
+		return packDouble(o);
 	}
 
 	public Packer pack(Boolean o) throws IOException {
@@ -379,6 +376,11 @@ public class Packer {
 		return packLong(o);
 	}
 
+	public Packer pack(BigInteger o) throws IOException {
+		if(o == null) { return packNil(); }
+		return packBigInteger(o);
+	}
+
 	public Packer pack(Float o) throws IOException {
 		if(o == null) { return packNil(); }
 		return packFloat(o);
@@ -389,8 +391,41 @@ public class Packer {
 		return packDouble(o);
 	}
 
+	public Packer pack(String o) throws IOException {
+		if(o == null) { return packNil(); }
+		return packString(o);
+	}
+
+	public Packer pack(MessagePackable o) throws IOException {
+		if(o == null) { return packNil(); }
+		o.messagePack(this);
+		return this;
+	}
+
+	public Packer pack(byte[] o) throws IOException {
+		if(o == null) { return packNil(); }
+		packRaw(o.length);
+		return packRawBody(o);
+	}
+
+	public Packer pack(List o) throws IOException {
+		if(o == null) { return packNil(); }
+		packArray(o.size());
+		for(Object i : o) { pack(i); }
+		return this;
+	}
+
+	public Packer pack(Map o) throws IOException {
+		if(o == null) { return packNil(); }
+		packMap(o.size());
+		for(Map.Entry e : ((Map)o).entrySet()) {
+			pack(e.getKey());
+			pack(e.getValue());
+		}
+		return this;
+	}
+
 
-	@SuppressWarnings("unchecked")
 	public Packer pack(Object o) throws IOException {
 		if(o == null) {
 			return packNil();
@@ -413,7 +448,7 @@ public class Packer {
 		} else if(o instanceof Map) {
 			Map m = (Map)o;
 			packMap(m.size());
-			for(Map.Entry e : m.entrySet()) {
+			for(Map.Entry e : m.entrySet()) {
 				pack(e.getKey());
 				pack(e.getValue());
 			}
diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java
index 75c5fe7..8163678 100644
--- a/java/src/test/java/org/msgpack/TestPackUnpack.java
+++ b/java/src/test/java/org/msgpack/TestPackUnpack.java
@@ -37,14 +37,30 @@ public class TestPackUnpack {
 			testInt(rand.nextInt());
 	}
 
+	public void testLong(long val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		MessagePackObject obj = unpackOne(out);
+		assertEquals(val, obj.asLong());
+	}
+	@Test
+	public void testLong() throws Exception {
+		testLong(0);
+		testLong(-1);
+		testLong(1);
+		testLong(Integer.MIN_VALUE);
+		testLong(Integer.MAX_VALUE);
+		testLong(Long.MIN_VALUE);
+		testLong(Long.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testLong(rand.nextLong());
+	}
+
 	public void testBigInteger(BigInteger val) throws Exception {
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		new Packer(out).pack(val);
 		MessagePackObject obj = unpackOne(out);
-		if(!val.equals(obj.asBigInteger())) {
-			System.out.println("expect: "+val);
-			System.out.println("but   : "+obj.asBigInteger());
-		}
 		assertEquals(val, obj.asBigInteger());
 	}
 	@Test

From 193a739749687e45a71a6821e625815587c4c4bf Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Thu, 19 Aug 2010 00:05:48 +0900
Subject: [PATCH 0607/1648] java: updates TestDirectConversion

---
 cpp/test/cases.cc                             |   2 +
 .../org/msgpack/TestDirectConversion.java     | 369 +++++++++++-------
 .../java/org/msgpack/TestObjectEquals.java    |  48 +--
 .../test/java/org/msgpack/TestPackUnpack.java |  54 +--
 4 files changed, 292 insertions(+), 181 deletions(-)

diff --git a/cpp/test/cases.cc b/cpp/test/cases.cc
index b408876..eb1286c 100644
--- a/cpp/test/cases.cc
+++ b/cpp/test/cases.cc
@@ -32,5 +32,7 @@ TEST(cases, format)
 		EXPECT_TRUE( pac_compact.next(&result_compact) );
 		EXPECT_EQ(result_compact.get(), result.get());
 	}
+
+	EXPECT_FALSE( pac_compact.next(&result) );
 }
 
diff --git a/java/src/test/java/org/msgpack/TestDirectConversion.java b/java/src/test/java/org/msgpack/TestDirectConversion.java
index 77bbc58..dbacf01 100644
--- a/java/src/test/java/org/msgpack/TestDirectConversion.java
+++ b/java/src/test/java/org/msgpack/TestDirectConversion.java
@@ -8,141 +8,248 @@ import org.junit.Test;
 import static org.junit.Assert.*;
 
 public class TestDirectConversion {
-    @Test
-    public void testInt() throws Exception {
-        testInt(0);
-        testInt(-1);
-        testInt(1);
-        testInt(Integer.MIN_VALUE);
-        testInt(Integer.MAX_VALUE);
-        Random rand = new Random();
-        for (int i = 0; i < 1000; i++)
-            testInt(rand.nextInt());
-    }
-    public void testInt(int val) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-        Unpacker upk = new Unpacker(in);
-        assertEquals(val, upk.unpackInt());
-    }
+	@Test
+	public void testInt() throws Exception {
+		testInt(0);
+		testInt(-1);
+		testInt(1);
+		testInt(Integer.MIN_VALUE);
+		testInt(Integer.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testInt(rand.nextInt());
+	}
+	public void testInt(int val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		assertEquals(val, pac.unpackInt());
+	}
 
-    @Test
-    public void testFloat() throws Exception {
-        testFloat((float)0.0);
-        testFloat((float)-0.0);
-        testFloat((float)1.0);
-        testFloat((float)-1.0);
-        testFloat((float)Float.MAX_VALUE);
-        testFloat((float)Float.MIN_VALUE);
-        testFloat((float)Float.NaN);
-        testFloat((float)Float.NEGATIVE_INFINITY);
-        testFloat((float)Float.POSITIVE_INFINITY);
-        Random rand = new Random();
-        for (int i = 0; i < 1000; i++)
-            testFloat(rand.nextFloat());
-    }
-    public void testFloat(float val) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-        Unpacker upk = new Unpacker(in);
-        float f = upk.unpackFloat();
-        if(Float.isNaN(val)) {
-            assertTrue(Float.isNaN(f));
-        } else {
-            assertEquals(val, f, 10e-10);
-        }
-    }
+	@Test
+	public void testLong() throws Exception {
+		testLong(0);
+		testLong(-1);
+		testLong(1);
+		testLong(Integer.MIN_VALUE);
+		testLong(Integer.MAX_VALUE);
+		testLong(Long.MIN_VALUE);
+		testLong(Long.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testLong(rand.nextLong());
+	}
+	public void testLong(long val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		assertEquals(val, pac.unpackLong());
+	}
 
-    @Test
-    public void testDouble() throws Exception {
-        testDouble((double)0.0);
-        testDouble((double)-0.0);
-        testDouble((double)1.0);
-        testDouble((double)-1.0);
-        testDouble((double)Double.MAX_VALUE);
-        testDouble((double)Double.MIN_VALUE);
-        testDouble((double)Double.NaN);
-        testDouble((double)Double.NEGATIVE_INFINITY);
-        testDouble((double)Double.POSITIVE_INFINITY);
-        Random rand = new Random();
-        for (int i = 0; i < 1000; i++)
-            testDouble(rand.nextDouble());
-    }
-    public void testDouble(double val) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-        Unpacker upk = new Unpacker(in);
-        double f = upk.unpackDouble();
-        if(Double.isNaN(val)) {
-            assertTrue(Double.isNaN(f));
-        } else {
-            assertEquals(val, f, 10e-10);
-        }
-    }
+	@Test
+	public void testFloat() throws Exception {
+		testFloat((float)0.0);
+		testFloat((float)-0.0);
+		testFloat((float)1.0);
+		testFloat((float)-1.0);
+		testFloat((float)Float.MAX_VALUE);
+		testFloat((float)Float.MIN_VALUE);
+		testFloat((float)Float.NaN);
+		testFloat((float)Float.NEGATIVE_INFINITY);
+		testFloat((float)Float.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testFloat(rand.nextFloat());
+	}
+	public void testFloat(float val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		assertEquals(val, pac.unpackFloat(), 10e-10);
+	}
 
-    @Test
-    public void testNil() throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).packNil();
-        ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-        Unpacker upk = new Unpacker(in);
-        assertEquals(null, upk.unpackNull());
-    }
+	@Test
+	public void testDouble() throws Exception {
+		testDouble((double)0.0);
+		testDouble((double)-0.0);
+		testDouble((double)1.0);
+		testDouble((double)-1.0);
+		testDouble((double)Double.MAX_VALUE);
+		testDouble((double)Double.MIN_VALUE);
+		testDouble((double)Double.NaN);
+		testDouble((double)Double.NEGATIVE_INFINITY);
+		testDouble((double)Double.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testDouble(rand.nextDouble());
+	}
+	public void testDouble(double val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		assertEquals(val, pac.unpackDouble(), 10e-10);
+	}
 
-    @Test
-    public void testBoolean() throws Exception {
-        testBoolean(false);
-        testBoolean(true);
-    }
-    public void testBoolean(boolean val) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-        Unpacker upk = new Unpacker(in);
-        assertEquals(val, upk.unpackBoolean());
-    }
+	@Test
+	public void testNil() throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).packNil();
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		assertEquals(null, pac.unpackNull());
+	}
 
-    @Test
-    public void testString() throws Exception {
-        testString("");
-        testString("a");
-        testString("ab");
-        testString("abc");
-        // small size string
-        for (int i = 0; i < 100; i++) {
-            StringBuilder sb = new StringBuilder();
-            int len = (int)Math.random() % 31 + 1;
-            for (int j = 0; j < len; j++)
-                sb.append('a' + ((int)Math.random()) & 26);
-            testString(sb.toString());
-        }
-        // medium size string
-        for (int i = 0; i < 100; i++) {
-            StringBuilder sb = new StringBuilder();
-            int len = (int)Math.random() % 100 + (1 << 15);
-            for (int j = 0; j < len; j++)
-                sb.append('a' + ((int)Math.random()) & 26);
-            testString(sb.toString());
-        }
-        // large size string
-        for (int i = 0; i < 10; i++) {
-            StringBuilder sb = new StringBuilder();
-            int len = (int)Math.random() % 100 + (1 << 31);
-            for (int j = 0; j < len; j++)
-                sb.append('a' + ((int)Math.random()) & 26);
-            testString(sb.toString());
-        }
-    }
-    public void testString(String val) throws Exception {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        new Packer(out).pack(val);
-        ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-        Unpacker upk = new Unpacker(in);
-        assertEquals(val, upk.unpackString());
-    }
+	@Test
+	public void testBoolean() throws Exception {
+		testBoolean(false);
+		testBoolean(true);
+	}
+	public void testBoolean(boolean val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		assertEquals(val, pac.unpackBoolean());
+	}
 
-    // FIXME container types
+	@Test
+	public void testString() throws Exception {
+		testString("");
+		testString("a");
+		testString("ab");
+		testString("abc");
+
+		// small size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int)Math.random() % 31 + 1;
+			for (int j = 0; j < len; j++)
+				sb.append('a' + ((int)Math.random()) & 26);
+			testString(sb.toString());
+		}
+
+		// medium size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int)Math.random() % 100 + (1 << 15);
+			for (int j = 0; j < len; j++)
+				sb.append('a' + ((int)Math.random()) & 26);
+			testString(sb.toString());
+		}
+
+		// large size string
+		for (int i = 0; i < 10; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int)Math.random() % 100 + (1 << 31);
+			for (int j = 0; j < len; j++)
+				sb.append('a' + ((int)Math.random()) & 26);
+			testString(sb.toString());
+		}
+	}
+	public void testString(String val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		assertEquals(val, pac.unpackString());
+	}
+
+	@Test
+	public void testArray() throws Exception {
+		List emptyList = new ArrayList();
+		{
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(emptyList);
+			ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+			Unpacker pac = new Unpacker(in);
+			int ulen = pac.unpackArray();
+			assertEquals(0, ulen);
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			List l = new ArrayList();
+			int len = (int)Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++)
+				l.add(j);
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(l);
+			ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+			Unpacker pac = new Unpacker(in);
+			int ulen = pac.unpackArray();
+			assertEquals(len, ulen);
+			for (int j = 0; j < len; j++) {
+				assertEquals(l.get(j).intValue(), pac.unpackInt());
+			}
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			List l = new ArrayList();
+			int len = (int)Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++)
+				l.add(Integer.toString(j));
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(l);
+			ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+			Unpacker pac = new Unpacker(in);
+			int ulen = pac.unpackArray();
+			assertEquals(len, ulen);
+			for (int j = 0; j < len; j++) {
+				assertEquals(l.get(j), pac.unpackString());
+			}
+		}
+	}
+
+	@Test
+	public void testMap() throws Exception {
+		Map emptyMap = new HashMap();
+		{
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(emptyMap);
+			ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+			Unpacker pac = new Unpacker(in);
+			int ulen = pac.unpackMap();
+			assertEquals(0, ulen);
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			Map m = new HashMap();
+			int len = (int)Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++)
+				m.put(j, j);
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(m);
+			ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+			Unpacker pac = new Unpacker(in);
+			int ulen = pac.unpackMap();
+			assertEquals(len, ulen);
+			for (int j = 0; j < len; j++) {
+				Integer val = m.get(pac.unpackInt());
+				assertNotNull(val);
+				assertEquals(val.intValue(), pac.unpackInt());
+			}
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			Map m = new HashMap();
+			int len = (int)Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++)
+				m.put(Integer.toString(j), j);
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(m);
+			ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+			Unpacker pac = new Unpacker(in);
+			int ulen = pac.unpackMap();
+			assertEquals(len, ulen);
+			for (int j = 0; j < len; j++) {
+				Integer val = m.get(pac.unpackString());
+				assertNotNull(val);
+				assertEquals(val.intValue(), pac.unpackInt());
+			}
+		}
+	}
 };
+
diff --git a/java/src/test/java/org/msgpack/TestObjectEquals.java b/java/src/test/java/org/msgpack/TestObjectEquals.java
index b2b018d..25fe927 100644
--- a/java/src/test/java/org/msgpack/TestObjectEquals.java
+++ b/java/src/test/java/org/msgpack/TestObjectEquals.java
@@ -9,6 +9,17 @@ import org.junit.Test;
 import static org.junit.Assert.*;
 
 public class TestObjectEquals {
+	@Test
+	public void testInt() throws Exception {
+		testInt(0);
+		testInt(-1);
+		testInt(1);
+		testInt(Integer.MIN_VALUE);
+		testInt(Integer.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testInt(rand.nextInt());
+	}
 	public void testInt(int val) throws Exception {
 		MessagePackObject objInt = IntegerType.create(val);
 		MessagePackObject objLong = IntegerType.create((long)val);
@@ -23,18 +34,20 @@ public class TestObjectEquals {
 		assertTrue(objBigInt.equals(objLong));
 		assertTrue(objBigInt.equals(objBigInt));
 	}
+
 	@Test
-	public void testInt() throws Exception {
-		testInt(0);
-		testInt(-1);
-		testInt(1);
-		testInt(Integer.MIN_VALUE);
-		testInt(Integer.MAX_VALUE);
+	public void testLong() throws Exception {
+		testLong(0);
+		testLong(-1);
+		testLong(1);
+		testLong(Integer.MIN_VALUE);
+		testLong(Integer.MAX_VALUE);
+		testLong(Long.MIN_VALUE);
+		testLong(Long.MAX_VALUE);
 		Random rand = new Random();
 		for (int i = 0; i < 1000; i++)
-			testInt(rand.nextInt());
+			testLong(rand.nextLong());
 	}
-
 	public void testLong(long val) throws Exception {
 		MessagePackObject objInt = IntegerType.create((int)val);
 		MessagePackObject objLong = IntegerType.create(val);
@@ -61,19 +74,6 @@ public class TestObjectEquals {
 			assertTrue(objBigInt.equals(objBigInt));
 		}
 	}
-	@Test
-	public void testLong() throws Exception {
-		testLong(0);
-		testLong(-1);
-		testLong(1);
-		testLong(Integer.MIN_VALUE);
-		testLong(Integer.MAX_VALUE);
-		testLong(Long.MIN_VALUE);
-		testLong(Long.MAX_VALUE);
-		Random rand = new Random();
-		for (int i = 0; i < 1000; i++)
-			testLong(rand.nextLong());
-	}
 
 	@Test
 	public void testNil() throws Exception {
@@ -82,9 +82,6 @@ public class TestObjectEquals {
 		assertFalse(NilType.create().equals(BooleanType.create(false)));
 	}
 
-	public void testString(String str) throws Exception {
-		assertTrue(RawType.create(str).equals(RawType.create(str)));
-	}
 	@Test
 	public void testString() throws Exception {
 		testString("");
@@ -92,5 +89,8 @@ public class TestObjectEquals {
 		testString("ab");
 		testString("abc");
 	}
+	public void testString(String str) throws Exception {
+		assertTrue(RawType.create(str).equals(RawType.create(str)));
+	}
 }
 
diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java
index 8163678..7edd411 100644
--- a/java/src/test/java/org/msgpack/TestPackUnpack.java
+++ b/java/src/test/java/org/msgpack/TestPackUnpack.java
@@ -11,20 +11,14 @@ import static org.junit.Assert.*;
 public class TestPackUnpack {
 	public MessagePackObject unpackOne(ByteArrayOutputStream out) {
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker upk = new Unpacker(in);
-		Iterator it = upk.iterator();
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
 		assertEquals(true, it.hasNext());
 		MessagePackObject obj = it.next();
 		assertEquals(false, it.hasNext());
 		return obj;
 	}
 
-	public void testInt(int val) throws Exception {
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(val);
-		MessagePackObject obj = unpackOne(out);
-		assertEquals(val, obj.asInt());
-	}
 	@Test
 	public void testInt() throws Exception {
 		testInt(0);
@@ -36,13 +30,13 @@ public class TestPackUnpack {
 		for (int i = 0; i < 1000; i++)
 			testInt(rand.nextInt());
 	}
-
-	public void testLong(long val) throws Exception {
+	public void testInt(int val) throws Exception {
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		new Packer(out).pack(val);
 		MessagePackObject obj = unpackOne(out);
-		assertEquals(val, obj.asLong());
+		assertEquals(val, obj.asInt());
 	}
+
 	@Test
 	public void testLong() throws Exception {
 		testLong(0);
@@ -56,13 +50,13 @@ public class TestPackUnpack {
 		for (int i = 0; i < 1000; i++)
 			testLong(rand.nextLong());
 	}
-
-	public void testBigInteger(BigInteger val) throws Exception {
+	public void testLong(long val) throws Exception {
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		new Packer(out).pack(val);
 		MessagePackObject obj = unpackOne(out);
-		assertEquals(val, obj.asBigInteger());
+		assertEquals(val, obj.asLong());
 	}
+
 	@Test
 	public void testBigInteger() throws Exception {
 		testBigInteger(BigInteger.valueOf(0));
@@ -78,13 +72,13 @@ public class TestPackUnpack {
 		for (int i = 0; i < 1000; i++)
 			testBigInteger( max.subtract(BigInteger.valueOf( Math.abs(rand.nextLong()) )) );
 	}
-
-	public void testFloat(float val) throws Exception {
+	public void testBigInteger(BigInteger val) throws Exception {
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		new Packer(out).pack(val);
 		MessagePackObject obj = unpackOne(out);
-		assertEquals(val, obj.asFloat(), 10e-10);
+		assertEquals(val, obj.asBigInteger());
 	}
+
 	@Test
 	public void testFloat() throws Exception {
 		testFloat((float)0.0);
@@ -100,13 +94,14 @@ public class TestPackUnpack {
 		for (int i = 0; i < 1000; i++)
 			testFloat(rand.nextFloat());
 	}
-
-	public void testDouble(double val) throws Exception {
+	public void testFloat(float val) throws Exception {
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		new Packer(out).pack(val);
 		MessagePackObject obj = unpackOne(out);
-		assertEquals(val, obj.asDouble(), 10e-10);
+		float f = obj.asFloat();
+		assertEquals(val, f, 10e-10);
 	}
+
 	@Test
 	public void testDouble() throws Exception {
 		testDouble((double)0.0);
@@ -122,6 +117,13 @@ public class TestPackUnpack {
 		for (int i = 0; i < 1000; i++)
 			testDouble(rand.nextDouble());
 	}
+	public void testDouble(double val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		MessagePackObject obj = unpackOne(out);
+		double f = obj.asDouble();
+		assertEquals(val, f, 10e-10);
+	}
 
 	@Test
 	public void testNil() throws Exception {
@@ -143,12 +145,6 @@ public class TestPackUnpack {
 		assertEquals(val, obj.asBoolean());
 	}
 
-	public void testString(String val) throws Exception {
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(val);
-		MessagePackObject obj = unpackOne(out);
-		assertEquals(val, obj.asString());
-	}
 	@Test
 	public void testString() throws Exception {
 		testString("");
@@ -183,6 +179,12 @@ public class TestPackUnpack {
 			testString(sb.toString());
 		}
 	}
+	public void testString(String val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		MessagePackObject obj = unpackOne(out);
+		assertEquals(val, obj.asString());
+	}
 
 	@Test
 	public void testArray() throws Exception {

From 1d17836b7d2eb4e5e0f25d2466df70e137642061 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Thu, 19 Aug 2010 00:54:23 +0900
Subject: [PATCH 0608/1648] java: NilType::create() returns NilType's one and
 only instance

---
 java/src/main/java/org/msgpack/MessagePackObject.java |  2 +-
 java/src/main/java/org/msgpack/object/NilType.java    | 10 ++++++----
 java/src/test/java/org/msgpack/TestPackUnpack.java    |  2 +-
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/java/src/main/java/org/msgpack/MessagePackObject.java b/java/src/main/java/org/msgpack/MessagePackObject.java
index 6181f7a..2424446 100644
--- a/java/src/main/java/org/msgpack/MessagePackObject.java
+++ b/java/src/main/java/org/msgpack/MessagePackObject.java
@@ -23,7 +23,7 @@ import java.util.Map;
 import java.math.BigInteger;
 
 public abstract class MessagePackObject implements Cloneable, MessagePackable {
-	public boolean isNull() {
+	public boolean isNil() {
 		return false;
 	}
 
diff --git a/java/src/main/java/org/msgpack/object/NilType.java b/java/src/main/java/org/msgpack/object/NilType.java
index d0572f1..c443db1 100644
--- a/java/src/main/java/org/msgpack/object/NilType.java
+++ b/java/src/main/java/org/msgpack/object/NilType.java
@@ -21,14 +21,16 @@ import java.io.IOException;
 import org.msgpack.*;
 
 public class NilType extends MessagePackObject {
-	private static NilType instance = new NilType();
+	private final static NilType INSTANCE = new NilType();
 
 	public static NilType create() {
-		return instance;
+		return INSTANCE;
 	}
 
+	private NilType() { }
+
 	@Override
-	public boolean isNull() {
+	public boolean isNil() {
 		return true;
 	}
 
@@ -52,7 +54,7 @@ public class NilType extends MessagePackObject {
 
 	@Override
 	public Object clone() {
-		return new NilType();
+		return INSTANCE;
 	}
 }
 
diff --git a/java/src/test/java/org/msgpack/TestPackUnpack.java b/java/src/test/java/org/msgpack/TestPackUnpack.java
index 7edd411..494c8a8 100644
--- a/java/src/test/java/org/msgpack/TestPackUnpack.java
+++ b/java/src/test/java/org/msgpack/TestPackUnpack.java
@@ -130,7 +130,7 @@ public class TestPackUnpack {
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		new Packer(out).packNil();
 		MessagePackObject obj = unpackOne(out);
-		assertTrue(obj.isNull());
+		assertTrue(obj.isNil());
 	}
 
 	@Test

From b4c98584db002cbb4d3960b6ed025212248dcce8 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Thu, 19 Aug 2010 01:19:34 +0900
Subject: [PATCH 0609/1648] java: adds Unpacker.unpackBigInteger()

---
 .../org/msgpack/BufferedUnpackerImpl.java     | 25 ++++++++++++++++---
 java/src/main/java/org/msgpack/Unpacker.java  | 10 ++++++++
 .../org/msgpack/TestDirectConversion.java     | 24 ++++++++++++++++++
 3 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java
index 9496238..5b449c7 100644
--- a/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java
+++ b/java/src/main/java/org/msgpack/BufferedUnpackerImpl.java
@@ -19,7 +19,7 @@ package org.msgpack;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-//import java.math.BigInteger;
+import java.math.BigInteger;
 
 abstract class BufferedUnpackerImpl extends UnpackerImpl {
 	int offset = 0;
@@ -198,8 +198,7 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl {
 			{
 				long o = castBuffer.getLong(0);
 				if(o < 0) {
-					// FIXME unpackBigInteger
-					throw new MessageTypeException("uint 64 bigger than 0x7fffffff is not supported");
+					throw new MessageTypeException();
 				}
 				advance(9);
 				return o;
@@ -231,7 +230,25 @@ abstract class BufferedUnpackerImpl extends UnpackerImpl {
 		}
 	}
 
-	// FIXME unpackBigInteger
+	final BigInteger unpackBigInteger() throws IOException, MessageTypeException {
+		more(1);
+		int b = buffer[offset];
+		if((b & 0xff) != 0xcf) {
+			return BigInteger.valueOf(unpackLong());
+		}
+
+		// unsigned int 64
+		more(9);
+		castBuffer.rewind();
+		castBuffer.put(buffer, offset+1, 8);
+		advance(9);
+		long o = castBuffer.getLong(0);
+		if(o < 0) {
+			return new BigInteger(1, castBuffer.array());
+		} else {
+			return BigInteger.valueOf(o);
+		}
+	}
 
 	final float unpackFloat() throws IOException, MessageTypeException {
 		more(1);
diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index 3a95243..3cae502 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -22,6 +22,7 @@ import java.io.InputStream;
 import java.io.IOException;
 import java.util.Iterator;
 import java.nio.ByteBuffer;
+import java.math.BigInteger;
 
 /**
  * Unpacker enables you to deserialize objects from stream.
@@ -452,6 +453,15 @@ public class Unpacker implements Iterable {
 		return impl.unpackLong();
 	}
 
+	/**
+	 * Gets one {@code BigInteger} value from the buffer.
+	 * This method calls {@link fill()} method if needed.
+	 * @throws MessageTypeException the first value of the buffer is not a {@code BigInteger}.
+	 */
+	public BigInteger unpackBigInteger() throws IOException, MessageTypeException {
+		return impl.unpackBigInteger();
+	}
+
 	/**
 	 * Gets one {@code float} value from the buffer.
 	 * This method calls {@link fill()} method if needed.
diff --git a/java/src/test/java/org/msgpack/TestDirectConversion.java b/java/src/test/java/org/msgpack/TestDirectConversion.java
index dbacf01..1822ecb 100644
--- a/java/src/test/java/org/msgpack/TestDirectConversion.java
+++ b/java/src/test/java/org/msgpack/TestDirectConversion.java
@@ -3,6 +3,7 @@ package org.msgpack;
 import org.msgpack.*;
 import java.io.*;
 import java.util.*;
+import java.math.BigInteger;
 
 import org.junit.Test;
 import static org.junit.Assert.*;
@@ -48,6 +49,29 @@ public class TestDirectConversion {
 		assertEquals(val, pac.unpackLong());
 	}
 
+	@Test
+	public void testBigInteger() throws Exception {
+		testBigInteger(BigInteger.valueOf(0));
+		testBigInteger(BigInteger.valueOf(-1));
+		testBigInteger(BigInteger.valueOf(1));
+		testBigInteger(BigInteger.valueOf(Integer.MIN_VALUE));
+		testBigInteger(BigInteger.valueOf(Integer.MAX_VALUE));
+		testBigInteger(BigInteger.valueOf(Long.MIN_VALUE));
+		testBigInteger(BigInteger.valueOf(Long.MAX_VALUE));
+		BigInteger max = BigInteger.valueOf(Long.MAX_VALUE).setBit(63);
+		testBigInteger(max);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			testBigInteger( max.subtract(BigInteger.valueOf( Math.abs(rand.nextLong()) )) );
+	}
+	public void testBigInteger(BigInteger val) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(val);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		assertEquals(val, pac.unpackBigInteger());
+	}
+
 	@Test
 	public void testFloat() throws Exception {
 		testFloat((float)0.0);

From c8e351b31e28042c10f7406238636c92a64a696c Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sat, 21 Aug 2010 03:36:44 +0900
Subject: [PATCH 0610/1648] java: adds TestMessageUnpackable test

---
 java/src/main/java/org/msgpack/Packer.java    |  2 +-
 java/src/test/java/org/msgpack/Image.java     | 50 +++++++++++++++++++
 .../org/msgpack/TestMessageUnpackable.java    | 34 +++++++++++++
 3 files changed, 85 insertions(+), 1 deletion(-)
 create mode 100644 java/src/test/java/org/msgpack/Image.java
 create mode 100644 java/src/test/java/org/msgpack/TestMessageUnpackable.java

diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index 0ac8d89..139b3b1 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -212,7 +212,7 @@ public class Packer {
 			out.write(castBytes);
 			return this;
 		} else {
-			throw new MessageTypeException("can't BigInteger larger than 0xffffffffffffffff");
+			throw new MessageTypeException("can't pack BigInteger larger than 0xffffffffffffffff");
 		}
 	}
 
diff --git a/java/src/test/java/org/msgpack/Image.java b/java/src/test/java/org/msgpack/Image.java
new file mode 100644
index 0000000..3fcfe38
--- /dev/null
+++ b/java/src/test/java/org/msgpack/Image.java
@@ -0,0 +1,50 @@
+package org.msgpack;
+
+import org.msgpack.*;
+import java.io.*;
+import java.util.*;
+
+public class Image implements MessagePackable, MessageUnpackable {
+	public String uri = "";
+	public String title = "";
+	public int width = 0;
+	public int height = 0;
+	public int size = 0;
+
+	public void messagePack(Packer pk) throws IOException {
+		pk.packArray(5);
+		pk.pack(uri);
+		pk.pack(title);
+		pk.pack(width);
+		pk.pack(height);
+		pk.pack(size);
+	}
+
+	public void messageUnpack(Unpacker pac) throws IOException, MessageTypeException {
+		int length = pac.unpackArray();
+		if(length != 5) {
+			throw new MessageTypeException();
+		}
+		uri = pac.unpackString();
+		title = pac.unpackString();
+		width = pac.unpackInt();
+		height = pac.unpackInt();
+		size = pac.unpackInt();
+	}
+
+	public boolean equals(Image obj) {
+		return uri.equals(obj.uri) &&
+			title.equals(obj.title) &&
+			width == obj.width &&
+			height == obj.height &&
+			size == obj.size;
+	}
+
+	public boolean equals(Object obj) {
+		if(obj.getClass() != Image.class) {
+			return false;
+		}
+		return equals((Image)obj);
+	}
+}
+
diff --git a/java/src/test/java/org/msgpack/TestMessageUnpackable.java b/java/src/test/java/org/msgpack/TestMessageUnpackable.java
new file mode 100644
index 0000000..9099f91
--- /dev/null
+++ b/java/src/test/java/org/msgpack/TestMessageUnpackable.java
@@ -0,0 +1,34 @@
+package org.msgpack;
+
+import org.msgpack.*;
+import java.io.*;
+import java.util.*;
+import java.math.BigInteger;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestMessageUnpackable {
+	@Test
+	public void testImage() throws Exception {
+		Image src = new Image();
+		src.title = "msgpack";
+		src.uri = "http://msgpack.org/";
+		src.width = 2560;
+		src.height = 1600;
+		src.size = 4096000;
+
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		src.messagePack(new Packer(out));
+
+		Image dst = new Image();
+
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+
+		dst.messageUnpack(pac);
+
+		assertEquals(src, dst);
+	}
+}
+

From a91c1ec6d9b86a3f8f71504e0889191da13a210e Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Sat, 21 Aug 2010 16:02:23 +0900
Subject: [PATCH 0611/1648] fixed segv on cyclic reference(patch by dankogai)

---
 perl/pack.c       | 28 +++++++++++++++++-----------
 perl/t/08_cycle.t | 25 +++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 11 deletions(-)
 create mode 100644 perl/t/08_cycle.t

diff --git a/perl/pack.c b/perl/pack.c
index 589cee8..af6669c 100644
--- a/perl/pack.c
+++ b/perl/pack.c
@@ -49,6 +49,8 @@ static void need(enc_t *enc, STRLEN len);
 #  error  "msgpack only supports IVSIZE = 8,4,2 environment."
 #endif
 
+#define ERR_NESTING_EXCEEDED "perl structure exceeds maximum nesting level (max_depth set too low?)"
+
 
 static void need(enc_t *enc, STRLEN len)
 {
@@ -146,9 +148,10 @@ static int try_int(enc_t* enc, const char *p, size_t len) {
 }
 
 
-static void _msgpack_pack_rv(enc_t *enc, SV* sv);
+static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
 
-static void _msgpack_pack_sv(enc_t *enc, SV* sv) {
+static void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) {
+    if (!depth) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
     SvGETMAGIC(sv);
 
     if (sv==NULL) {
@@ -171,7 +174,7 @@ static void _msgpack_pack_sv(enc_t *enc, SV* sv) {
     } else if (SvIOKp(sv)) {
         PACK_IV(enc, SvIV(sv));
     } else if (SvROK(sv)) {
-        _msgpack_pack_rv(enc, SvRV(sv));
+        _msgpack_pack_rv(enc, SvRV(sv), depth-1);
     } else if (!SvOK(sv)) {
         msgpack_pack_nil(enc);
     } else if (isGV(sv)) {
@@ -182,8 +185,9 @@ static void _msgpack_pack_sv(enc_t *enc, SV* sv) {
     }
 }
 
-static void _msgpack_pack_rv(enc_t *enc, SV* sv) {
+static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
     svtype svt;
+    if (!depth) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
     SvGETMAGIC(sv);
     svt = SvTYPE(sv);
 
@@ -207,8 +211,8 @@ static void _msgpack_pack_rv(enc_t *enc, SV* sv) {
         msgpack_pack_map(enc, count);
 
         while (he = hv_iternext(hval)) {
-            _msgpack_pack_sv(enc, hv_iterkeysv(he));
-            _msgpack_pack_sv(enc, HeVAL(he));
+            _msgpack_pack_sv(enc, hv_iterkeysv(he), depth);
+            _msgpack_pack_sv(enc, HeVAL(he), depth);
         }
     } else if (svt == SVt_PVAV) {
         AV* ary = (AV*)sv;
@@ -218,7 +222,7 @@ static void _msgpack_pack_rv(enc_t *enc, SV* sv) {
         for (i=0; ipack($dat)");
+    if (items < 2) {
+        Perl_croak(aTHX_ "Usage: Data::MessagePack->pack($dat [,$max_depth])");
     }
 
-    SV* val = ST(1);
+    SV* val   = ST(1);
+    int depth = 512;
+    if (items >= 3) depth = SvIV(ST(2));
 
     enc_t enc;
     enc.sv        = sv_2mortal(NEWSV(0, INIT_SIZE));
@@ -256,7 +262,7 @@ XS(xs_pack) {
     enc.end       = SvEND(enc.sv);
     SvPOK_only(enc.sv);
 
-    _msgpack_pack_sv(&enc, val);
+    _msgpack_pack_sv(&enc, val, depth);
 
     SvCUR_set(enc.sv, enc.cur - SvPVX (enc.sv));
     *SvEND (enc.sv) = 0; /* many xs functions expect a trailing 0 for text strings */
diff --git a/perl/t/08_cycle.t b/perl/t/08_cycle.t
new file mode 100644
index 0000000..55d8427
--- /dev/null
+++ b/perl/t/08_cycle.t
@@ -0,0 +1,25 @@
+use t::Util;
+use Test::More;
+use Data::MessagePack;
+
+plan tests => 5;
+
+my $aref = [0];
+$aref->[1] = $aref;
+eval { Data::MessagePack->pack($aref) };
+ok $@, $@;
+
+my $href = {};
+$href->{cycle} = $href;
+eval { Data::MessagePack->pack($aref) };
+ok $@, $@;
+
+$aref = [0,[1,2]];
+eval { Data::MessagePack->pack($aref) };
+ok !$@;
+
+eval { Data::MessagePack->pack($aref, 3) };
+ok !$@;
+
+eval { Data::MessagePack->pack($aref, 2) };
+ok $@, $@;

From 8de1f764fdcab4e9ec3f5739b2fb0a8ae2a02df9 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Sat, 21 Aug 2010 16:09:30 +0900
Subject: [PATCH 0612/1648] Perl: bump up version to 0.14

---
 perl/Changes                 | 5 +++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index f32d5f4..13fc98b 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.14
+
+    - fixed segv on serializing cyclic reference
+      (Dan Kogai)
+
 0.13
 
     - clearly specify requires_c99(), because msgpack C header requires C99.
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index ee07d35..3c38a79 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -4,7 +4,7 @@ use warnings;
 use XSLoader;
 use 5.008001;
 
-our $VERSION = '0.13';
+our $VERSION = '0.14';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };

From 18c712cd99d1317d69f628fe58fd6ff9d10865c5 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Wed, 25 Aug 2010 21:35:52 +0900
Subject: [PATCH 0613/1648] object serialization with reflection and with
 dynamic code generation

---
 java/pom.xml                                  |  12 +
 .../org/msgpack/tmp/DynamicCodeGenerator.java | 209 ++++++++++++++++++
 .../src/main/java/org/msgpack/tmp/Image1.java |  47 ++++
 .../src/main/java/org/msgpack/tmp/Image2.java |  85 +++++++
 .../src/main/java/org/msgpack/tmp/Image3.java |  38 ++++
 .../msgpack/tmp/ImagePackUnpackPerfTest.java  | 134 +++++++++++
 6 files changed, 525 insertions(+)
 create mode 100644 java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java
 create mode 100644 java/src/main/java/org/msgpack/tmp/Image1.java
 create mode 100644 java/src/main/java/org/msgpack/tmp/Image2.java
 create mode 100644 java/src/main/java/org/msgpack/tmp/Image3.java
 create mode 100644 java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java

diff --git a/java/pom.xml b/java/pom.xml
index 44806d5..6591a72 100755
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -29,6 +29,12 @@
       4.8.1
       test
     
+
+    
+      jboss
+      javassist
+      3.7.ga
+    
   
 
   
@@ -101,6 +107,12 @@
       MessagePack Maven2 Repository
       http://msgpack.org/maven2
     
+
+    
+      repo2.maven.org
+      repo2.maven.org Maven2 Repository
+      http://repo2.maven.org/maven2
+    
   
 
   
diff --git a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java
new file mode 100644
index 0000000..9e4336d
--- /dev/null
+++ b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java
@@ -0,0 +1,209 @@
+package org.msgpack.tmp;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.lang.reflect.Field;
+import java.util.HashMap;
+
+import javassist.ClassPool;
+import javassist.CtClass;
+import javassist.CtConstructor;
+import javassist.CtMethod;
+import javassist.CtNewConstructor;
+import javassist.CtNewMethod;
+
+import org.msgpack.Packer;
+import org.msgpack.Unpacker;
+
+public class DynamicCodeGenerator {
+
+	private static final String ENHANCED_CLASS_NAME_POSTFIX = "_$$_Enhanced";
+
+	private static final String SPACE = " ";
+
+	private static final String MODIFIER_PUBLIC = "public";
+
+	private static final String METHOD_NAME_MESSAGEPACK = "messagePack";
+
+	private static final String METHOD_NAME_MESSAGEUNPACK = "messageUnpack";
+
+	private static final String PACKER_CLASS_TYPE_NAME = Packer.class.getName();
+
+	private static final String PACKER_OBJECT_NAME = "pk";
+
+	private static final String UNPACKER_CLASS_TYPE_NAME = Unpacker.class
+			.getName();
+
+	private static final String UNPACKER_OBJECT_NAME = "pk";
+
+	private HashMap> classMap;
+	
+	private HashMap objectCacheMap;
+
+	private ClassPool pool;
+
+	public DynamicCodeGenerator() {
+		classMap = new HashMap>();
+		objectCacheMap = new HashMap();
+		pool = ClassPool.getDefault();
+	}
+
+	public Object newEnhancedInstance(Class targetClass) throws Exception {
+		String targetClassName = targetClass.getName();
+		//Class enhancedClass = classMap.get(targetClassName);
+		Object enhancedObject = objectCacheMap.get(targetClassName);
+		//if (enhancedClass == null) {
+		if (enhancedObject == null) {
+			CtClass enhancedCtClass = createEnhancedCtClass(targetClassName);
+//			System.out.println("enhanced class name: "
+//					+ enhancedCtClass.getName());
+			addSuperclass(enhancedCtClass, targetClassName);
+			addConstructor(enhancedCtClass);
+			createMessagePackMethod(enhancedCtClass, targetClass);
+			createMessageUnpackMethod(enhancedCtClass, targetClass);
+			Class enhancedClass = loadEnhancedClass(enhancedCtClass);
+			//classMap.put(targetClassName, enhancedClass);
+			enhancedObject = enhancedClass.newInstance();
+			objectCacheMap.put(targetClassName, enhancedObject);
+		}
+		//return newEnhancedInstance0(enhancedClass);
+		return enhancedObject;
+	}
+
+	private CtClass createEnhancedCtClass(final String targetClassName)
+			throws Exception {
+		return pool.makeClass(targetClassName + ENHANCED_CLASS_NAME_POSTFIX);
+	}
+
+	private void addSuperclass(CtClass enhancedCtClass,
+			final String targetClassName) throws Exception {
+		CtClass targetCtClass = pool.get(targetClassName);
+		enhancedCtClass.setSuperclass(targetCtClass);
+	}
+
+	private void addConstructor(CtClass enhancedCtClass) throws Exception {
+		StringBuilder sb = new StringBuilder();
+		sb.append(MODIFIER_PUBLIC).append(SPACE).append(
+		// enhancedCtClass.getName()).append("(").append(")")
+				"Image3_$$_Enhanced").append("(").append(")").append(SPACE)
+				.append("{ super(); }"); // TODO
+//		System.out.println("cons: " + sb.toString());
+		CtConstructor newCtConstructor = CtNewConstructor.make(sb.toString(),
+				enhancedCtClass);
+		enhancedCtClass.addConstructor(newCtConstructor);
+	}
+
+	private void createMessagePackMethod(CtClass enhancedCtClass,
+			Class targetClass) throws Exception {
+		StringBuilder sb = new StringBuilder();
+		sb.append(MODIFIER_PUBLIC).append(SPACE).append("void").append(SPACE)
+				.append(METHOD_NAME_MESSAGEPACK).append("(").append(
+						PACKER_CLASS_TYPE_NAME).append(SPACE).append(
+						PACKER_OBJECT_NAME).append(")").append(SPACE).append(
+						"throws").append(SPACE).append("java.io.IOException")
+				.append(SPACE).append("{");
+		Field[] fields = targetClass.getFields();
+		sb.append(PACKER_OBJECT_NAME).append(".").append("packArray").append(
+				"(").append(fields.length).append(")").append(";");
+		for (Field field : fields) {
+			insertCodeOfMessagePack(field, sb);
+		}
+		sb.append("}");
+//		System.out.println("messagePack method: " + sb.toString());
+		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass);
+		enhancedCtClass.addMethod(newCtMethod);
+	}
+
+	private void insertCodeOfMessagePack(Field field, StringBuilder sb) {
+		Class type = field.getType();
+		if (type.equals(int.class)) {
+			sb.append(PACKER_OBJECT_NAME).append(".").append("pack")
+					.append("(").append(field.getName()).append(")")
+					.append(";");
+		} else if (type.equals(String.class)) {
+			sb.append(PACKER_OBJECT_NAME).append(".").append("pack")
+					.append("(").append(field.getName()).append(")")
+					.append(";");
+		} else {
+			throw new UnsupportedOperationException();
+		}
+	}
+
+	private void createMessageUnpackMethod(CtClass enhancedCtClass,
+			Class targetClass) throws Exception {
+		StringBuilder sb = new StringBuilder();
+		sb
+				.append(MODIFIER_PUBLIC)
+				.append(SPACE)
+				.append("void")
+				.append(SPACE)
+				.append(METHOD_NAME_MESSAGEUNPACK)
+				.append("(")
+				.append(UNPACKER_CLASS_TYPE_NAME)
+				.append(SPACE)
+				.append(UNPACKER_OBJECT_NAME)
+				.append(")")
+				.append(SPACE)
+				.append("throws")
+				.append(SPACE)
+				.append("org.msgpack.MessageTypeException, java.io.IOException")
+				.append(SPACE).append("{");
+		Field[] fields = targetClass.getFields();
+		sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()").append(";");
+		// TODO
+		for (Field field : fields) {
+			insertCodeOfMessageUnpack(field, sb);
+		}
+		sb.append("}");
+//		System.out.println("messageUnpack method: " + sb.toString());
+		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass);
+		enhancedCtClass.addMethod(newCtMethod);
+	}
+
+	private void insertCodeOfMessageUnpack(Field field, StringBuilder sb) {
+		Class type = field.getType();
+		if (type.equals(int.class)) {
+			sb.append(field.getName()).append(SPACE).append("=").append(SPACE)
+					.append(PACKER_OBJECT_NAME).append(".").append("unpackInt")
+					.append("(").append(")").append(";");
+		} else if (type.equals(String.class)) {
+			sb.append(field.getName()).append(SPACE).append("=").append(SPACE)
+					.append(PACKER_OBJECT_NAME).append(".").append(
+							"unpackString").append("(").append(")").append(";");
+		} else {
+			throw new UnsupportedOperationException();
+		}
+	}
+
+	private Class loadEnhancedClass(CtClass enhancedCtClass)
+			throws Exception {
+		return enhancedCtClass.toClass(null, null);
+	}
+
+	private Object newEnhancedInstance0(Class enhancedClass) {
+		try {
+			return enhancedClass.newInstance();
+		} catch (Exception e) {
+			throw new UnsupportedOperationException();
+		}
+	}
+
+	public static void main(String[] args) throws Exception {
+		DynamicCodeGenerator gen = new DynamicCodeGenerator();
+		Image3 src = (Image3) gen.newEnhancedInstance(Image3.class);
+		src.title = "msgpack";
+		src.uri = "http://msgpack.org/";
+		src.width = 2560;
+		src.height = 1600;
+		src.size = 4096000;
+
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		src.messagePack(new Packer(out));
+		Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		dst.messageUnpack(pac);
+
+//		System.out.println(dst.equals(src));
+	}
+}
diff --git a/java/src/main/java/org/msgpack/tmp/Image1.java b/java/src/main/java/org/msgpack/tmp/Image1.java
new file mode 100644
index 0000000..f1819ed
--- /dev/null
+++ b/java/src/main/java/org/msgpack/tmp/Image1.java
@@ -0,0 +1,47 @@
+package org.msgpack.tmp;
+
+import org.msgpack.*;
+import java.io.*;
+
+public class Image1 implements MessagePackable, MessageUnpackable {
+	public String uri = "";
+	public String title = "";
+	public int width = 0;
+	public int height = 0;
+	public int size = 0;
+
+	public void messagePack(Packer pk) throws IOException {
+		pk.packArray(5);
+		pk.pack(uri);
+		pk.pack(title);
+		pk.pack(width);
+		pk.pack(height);
+		pk.pack(size);
+	}
+
+	public void messageUnpack(Unpacker pac) throws IOException,
+			MessageTypeException {
+		int length = pac.unpackArray();
+		if (length != 5) {
+			throw new MessageTypeException();
+		}
+		uri = pac.unpackString();
+		title = pac.unpackString();
+		width = pac.unpackInt();
+		height = pac.unpackInt();
+		size = pac.unpackInt();
+	}
+
+	public boolean equals(Image1 obj) {
+		return uri.equals(obj.uri) && title.equals(obj.title)
+				&& width == obj.width && height == obj.height
+				&& size == obj.size;
+	}
+
+	public boolean equals(Object obj) {
+		if (obj.getClass() != Image1.class) {
+			return false;
+		}
+		return equals((Image1) obj);
+	}
+}
diff --git a/java/src/main/java/org/msgpack/tmp/Image2.java b/java/src/main/java/org/msgpack/tmp/Image2.java
new file mode 100644
index 0000000..9c357bf
--- /dev/null
+++ b/java/src/main/java/org/msgpack/tmp/Image2.java
@@ -0,0 +1,85 @@
+package org.msgpack.tmp;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+
+import org.msgpack.MessageTypeException;
+import org.msgpack.Packer;
+import org.msgpack.Unpacker;
+
+public class Image2 {
+	public String uri = "";
+	public String title = "";
+	public int width = 0;
+	public int height = 0;
+	public int size = 0;
+
+	public void messagePack(Packer pk) throws IOException {
+		messagePackWithReflection(pk);
+	}
+
+	public void messagePackWithReflection(Packer pk) throws IOException {
+		Class cl = this.getClass();
+		Field[] fields = cl.getFields();
+		pk.packArray(fields.length);
+		for (Field field : fields) {
+			try {
+				Object obj = field.get(this);
+				pk.pack(obj);
+			} catch (IllegalArgumentException e) {
+				e.printStackTrace();
+			} catch (IllegalAccessException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+
+	public void messageUnpack(Unpacker pac) throws IOException,
+			MessageTypeException {
+		messageUnpackWithReflection(pac);
+	}
+	
+	public void messageUnpackWithReflection(Unpacker pac) throws IOException,
+			MessageTypeException {
+		Class cl = this.getClass();
+		Field[] fields = cl.getFields();
+		int length = pac.unpackArray();
+		if (length != fields.length) {
+			throw new MessageTypeException();
+		}
+		for (Field field : fields) {
+			try {
+				field.set(this, unpack(pac, field.getType()));
+			} catch (IOException e) {
+				e.printStackTrace();
+			} catch (IllegalArgumentException e) {
+				e.printStackTrace();
+			} catch (IllegalAccessException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+	
+	public static Object unpack(Unpacker pk, Class cl) throws IOException {
+		if (cl == int.class) {
+			return pk.unpackInt();
+		} else if (cl == String.class) {
+			return pk.unpackString();
+		} else {
+			throw new UnsupportedOperationException();
+		}
+	}
+
+	public boolean equals(Image2 obj) {
+		return uri.equals(obj.uri) && title.equals(obj.title)
+				&& width == obj.width && height == obj.height
+				&& size == obj.size;
+	}
+
+	public boolean equals(Object obj) {
+		if (obj.getClass() != Image2.class) {
+			return false;
+		}
+		return equals((Image2) obj);
+	}
+}
diff --git a/java/src/main/java/org/msgpack/tmp/Image3.java b/java/src/main/java/org/msgpack/tmp/Image3.java
new file mode 100644
index 0000000..a28ff53
--- /dev/null
+++ b/java/src/main/java/org/msgpack/tmp/Image3.java
@@ -0,0 +1,38 @@
+package org.msgpack.tmp;
+
+import java.io.IOException;
+import org.msgpack.MessagePackable;
+import org.msgpack.MessageTypeException;
+import org.msgpack.MessageUnpackable;
+import org.msgpack.Packer;
+import org.msgpack.Unpacker;
+
+public class Image3 implements MessagePackable, MessageUnpackable {
+	public String uri = "";
+	public String title = "";
+	public int width = 0;
+	public int height = 0;
+	public int size = 0;
+
+	public void messagePack(Packer pk) throws IOException {
+		// empty
+	}
+
+	public void messageUnpack(Unpacker pac) throws IOException,
+			MessageTypeException {
+		// empty
+	}
+
+	public boolean equals(Image3 obj) {
+		return uri.equals(obj.uri) && title.equals(obj.title)
+				&& width == obj.width && height == obj.height
+				&& size == obj.size;
+	}
+
+	public boolean equals(Object obj) {
+		if (obj.getClass() != Image3.class) {
+			return false;
+		}
+		return equals((Image3) obj);
+	}
+}
\ No newline at end of file
diff --git a/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java b/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java
new file mode 100644
index 0000000..1f27c1b
--- /dev/null
+++ b/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java
@@ -0,0 +1,134 @@
+package org.msgpack.tmp;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+
+import org.msgpack.Packer;
+import org.msgpack.Unpacker;
+
+public class ImagePackUnpackPerfTest {
+
+	private static int BIG_LOOP = 5;
+
+	private static int SMALL_LOOP = 50000;
+
+	private static DynamicCodeGenerator gen;
+
+	private static void doGc() {
+		try {
+			Thread.sleep(50L);
+		} catch (InterruptedException ie) {
+			System.err
+					.println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()");
+		}
+		System.gc();
+		try { // longer sleep afterwards (not needed by GC, but may help with
+			// scheduling)
+			Thread.sleep(200L);
+		} catch (InterruptedException ie) {
+			System.err
+					.println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()");
+		}
+	}
+
+	public void doIt(int versionID) {
+		try {
+			doIt0(versionID);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+		try {
+			for (int j = 0; j < BIG_LOOP; ++j) {
+				long t = System.currentTimeMillis();
+				doIt0(versionID);
+				t = System.currentTimeMillis() - t;
+				System.out.println("time: " + t);
+			}
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+	public void doIt0(int versionID) throws Exception {
+		for (int i = 0; i < SMALL_LOOP; ++i) {
+			switch (versionID) {
+			case 0:
+				doCurrentVersion();
+				break;
+			case 1:
+				doWithReflection();
+				break;
+			case 2:
+				doWithDynamicCodeGeneration();
+				break;
+			default:
+				throw new RuntimeException();
+			}
+		}
+	}
+
+	public void doCurrentVersion() throws Exception {
+		Image1 src = new Image1();
+		src.title = "msgpack";
+		src.uri = "http://msgpack.org/";
+		src.width = 2560;
+		src.height = 1600;
+		src.size = 4096000;
+
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		src.messagePack(new Packer(out));
+		Image1 dst = new Image1();
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		dst.messageUnpack(pac);
+	}
+
+	public void doWithReflection() throws Exception {
+		Image2 src = new Image2();
+		src.title = "msgpack";
+		src.uri = "http://msgpack.org/";
+		src.width = 2560;
+		src.height = 1600;
+		src.size = 4096000;
+
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		src.messagePack(new Packer(out));
+		Image2 dst = new Image2();
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		dst.messageUnpack(pac);
+	}
+
+	public void doWithDynamicCodeGeneration() throws Exception {
+		Image3 src = (Image3) gen.newEnhancedInstance(Image3.class);
+		src.title = "msgpack";
+		src.uri = "http://msgpack.org/";
+		src.width = 2560;
+		src.height = 1600;
+		src.size = 4096000;
+
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		src.messagePack(new Packer(out));
+		Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		dst.messageUnpack(pac);
+	}
+
+	public static void main(String[] args) {
+		ImagePackUnpackPerfTest test = new ImagePackUnpackPerfTest();
+
+		doGc();
+		System.out.println("test current version");
+		test.doIt(0);
+
+		doGc();
+		System.out.println("test with reflection");
+		test.doIt(1);
+
+		doGc();
+		System.out.println("test with dynamic codegen");
+		gen = new DynamicCodeGenerator();
+		test.doIt(2);
+	}
+}

From ff0e1bbbc070e95be3631459815b0f765c3aab8d Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 26 Aug 2010 17:53:29 +0900
Subject: [PATCH 0614/1648] change the part for creating a new constructor
 within DynamicCodeGenerator

---
 .../org/msgpack/tmp/DynamicCodeGenerator.java | 43 +++++++++----------
 1 file changed, 20 insertions(+), 23 deletions(-)

diff --git a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java
index 9e4336d..78238b4 100644
--- a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java
+++ b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java
@@ -29,6 +29,8 @@ public class DynamicCodeGenerator {
 
 	private static final String PACKER_CLASS_TYPE_NAME = Packer.class.getName();
 
+	private static final String VOID_TYPE_NAME = "void";
+
 	private static final String PACKER_OBJECT_NAME = "pk";
 
 	private static final String UNPACKER_CLASS_TYPE_NAME = Unpacker.class
@@ -37,7 +39,7 @@ public class DynamicCodeGenerator {
 	private static final String UNPACKER_OBJECT_NAME = "pk";
 
 	private HashMap> classMap;
-	
+
 	private HashMap objectCacheMap;
 
 	private ClassPool pool;
@@ -50,23 +52,23 @@ public class DynamicCodeGenerator {
 
 	public Object newEnhancedInstance(Class targetClass) throws Exception {
 		String targetClassName = targetClass.getName();
-		//Class enhancedClass = classMap.get(targetClassName);
+		// Class enhancedClass = classMap.get(targetClassName);
 		Object enhancedObject = objectCacheMap.get(targetClassName);
-		//if (enhancedClass == null) {
+		// if (enhancedClass == null) {
 		if (enhancedObject == null) {
 			CtClass enhancedCtClass = createEnhancedCtClass(targetClassName);
-//			System.out.println("enhanced class name: "
-//					+ enhancedCtClass.getName());
+			// System.out.println("enhanced class name: "
+			// + enhancedCtClass.getName());
 			addSuperclass(enhancedCtClass, targetClassName);
 			addConstructor(enhancedCtClass);
 			createMessagePackMethod(enhancedCtClass, targetClass);
 			createMessageUnpackMethod(enhancedCtClass, targetClass);
 			Class enhancedClass = loadEnhancedClass(enhancedCtClass);
-			//classMap.put(targetClassName, enhancedClass);
+			// classMap.put(targetClassName, enhancedClass);
 			enhancedObject = enhancedClass.newInstance();
 			objectCacheMap.put(targetClassName, enhancedObject);
 		}
-		//return newEnhancedInstance0(enhancedClass);
+		// return newEnhancedInstance0(enhancedClass);
 		return enhancedObject;
 	}
 
@@ -82,24 +84,18 @@ public class DynamicCodeGenerator {
 	}
 
 	private void addConstructor(CtClass enhancedCtClass) throws Exception {
-		StringBuilder sb = new StringBuilder();
-		sb.append(MODIFIER_PUBLIC).append(SPACE).append(
-		// enhancedCtClass.getName()).append("(").append(")")
-				"Image3_$$_Enhanced").append("(").append(")").append(SPACE)
-				.append("{ super(); }"); // TODO
-//		System.out.println("cons: " + sb.toString());
-		CtConstructor newCtConstructor = CtNewConstructor.make(sb.toString(),
-				enhancedCtClass);
+		CtConstructor newCtConstructor = CtNewConstructor
+				.defaultConstructor(enhancedCtClass);
 		enhancedCtClass.addConstructor(newCtConstructor);
 	}
 
 	private void createMessagePackMethod(CtClass enhancedCtClass,
 			Class targetClass) throws Exception {
 		StringBuilder sb = new StringBuilder();
-		sb.append(MODIFIER_PUBLIC).append(SPACE).append("void").append(SPACE)
-				.append(METHOD_NAME_MESSAGEPACK).append("(").append(
-						PACKER_CLASS_TYPE_NAME).append(SPACE).append(
-						PACKER_OBJECT_NAME).append(")").append(SPACE).append(
+		sb.append(MODIFIER_PUBLIC).append(SPACE).append(VOID_TYPE_NAME).append(
+				SPACE).append(METHOD_NAME_MESSAGEPACK).append("(").append(
+				PACKER_CLASS_TYPE_NAME).append(SPACE)
+				.append(PACKER_OBJECT_NAME).append(")").append(SPACE).append(
 						"throws").append(SPACE).append("java.io.IOException")
 				.append(SPACE).append("{");
 		Field[] fields = targetClass.getFields();
@@ -109,7 +105,7 @@ public class DynamicCodeGenerator {
 			insertCodeOfMessagePack(field, sb);
 		}
 		sb.append("}");
-//		System.out.println("messagePack method: " + sb.toString());
+		// System.out.println("messagePack method: " + sb.toString());
 		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass);
 		enhancedCtClass.addMethod(newCtMethod);
 	}
@@ -135,7 +131,7 @@ public class DynamicCodeGenerator {
 		sb
 				.append(MODIFIER_PUBLIC)
 				.append(SPACE)
-				.append("void")
+				.append(VOID_TYPE_NAME)
 				.append(SPACE)
 				.append(METHOD_NAME_MESSAGEUNPACK)
 				.append("(")
@@ -149,13 +145,14 @@ public class DynamicCodeGenerator {
 				.append("org.msgpack.MessageTypeException, java.io.IOException")
 				.append(SPACE).append("{");
 		Field[] fields = targetClass.getFields();
-		sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()").append(";");
+		sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()")
+				.append(";");
 		// TODO
 		for (Field field : fields) {
 			insertCodeOfMessageUnpack(field, sb);
 		}
 		sb.append("}");
-//		System.out.println("messageUnpack method: " + sb.toString());
+		// System.out.println("messageUnpack method: " + sb.toString());
 		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass);
 		enhancedCtClass.addMethod(newCtMethod);
 	}

From 59ba8dec4ee082e8777047e6ae72e8b6998cdc79 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Fri, 27 Aug 2010 16:45:48 +0900
Subject: [PATCH 0615/1648] cpp: fixes include paths

---
 cpp/src/msgpack/object.h       |  2 +-
 cpp/src/msgpack/object.hpp     |  6 +++---
 cpp/src/msgpack/pack.h         |  9 ++++++---
 cpp/src/msgpack/pack.hpp       |  4 ++--
 cpp/src/msgpack/sbuffer.hpp    |  2 +-
 cpp/src/msgpack/type.hpp       | 28 ++++++++++++++--------------
 cpp/src/msgpack/unpack.h       |  4 ++--
 cpp/src/msgpack/unpack.hpp     |  6 +++---
 cpp/src/msgpack/vrefbuffer.h   |  2 +-
 cpp/src/msgpack/vrefbuffer.hpp |  2 +-
 cpp/src/msgpack/zbuffer.h      |  2 +-
 cpp/src/msgpack/zbuffer.hpp    |  2 +-
 cpp/src/msgpack/zone.h         |  2 +-
 cpp/src/msgpack/zone.hpp.erb   |  2 +-
 14 files changed, 38 insertions(+), 35 deletions(-)

diff --git a/cpp/src/msgpack/object.h b/cpp/src/msgpack/object.h
index cf0b4c1..baeeb9b 100644
--- a/cpp/src/msgpack/object.h
+++ b/cpp/src/msgpack/object.h
@@ -18,7 +18,7 @@
 #ifndef MSGPACK_OBJECT_H__
 #define MSGPACK_OBJECT_H__
 
-#include "msgpack/zone.h"
+#include "zone.h"
 #include 
 
 #ifdef __cplusplus
diff --git a/cpp/src/msgpack/object.hpp b/cpp/src/msgpack/object.hpp
index f80a390..96c026e 100644
--- a/cpp/src/msgpack/object.hpp
+++ b/cpp/src/msgpack/object.hpp
@@ -18,9 +18,9 @@
 #ifndef MSGPACK_OBJECT_HPP__
 #define MSGPACK_OBJECT_HPP__
 
-#include "msgpack/object.h"
-#include "msgpack/pack.hpp"
-#include "msgpack/zone.hpp"
+#include "object.h"
+#include "pack.hpp"
+#include "zone.hpp"
 #include 
 #include 
 #include 
diff --git a/cpp/src/msgpack/pack.h b/cpp/src/msgpack/pack.h
index 1252895..9c4ce59 100644
--- a/cpp/src/msgpack/pack.h
+++ b/cpp/src/msgpack/pack.h
@@ -18,8 +18,8 @@
 #ifndef MSGPACK_PACK_H__
 #define MSGPACK_PACK_H__
 
-#include "msgpack/pack_define.h"
-#include "msgpack/object.h"
+#include "pack_define.h"
+#include "object.h"
 #include 
 
 #ifdef __cplusplus
@@ -96,12 +96,15 @@ int msgpack_pack_object(msgpack_packer* pk, msgpack_object d);
 #define msgpack_pack_inline_func_cint(name) \
 	inline int msgpack_pack ## name
 
+#define msgpack_pack_inline_func_cint(name) \
+	inline int msgpack_pack ## name
+
 #define msgpack_pack_user msgpack_packer*
 
 #define msgpack_pack_append_buffer(user, buf, len) \
 	return (*(user)->callback)((user)->data, (const char*)buf, len)
 
-#include "msgpack/pack_template.h"
+#include "pack_template.h"
 
 inline void msgpack_packer_init(msgpack_packer* pk, void* data, msgpack_packer_write callback)
 {
diff --git a/cpp/src/msgpack/pack.hpp b/cpp/src/msgpack/pack.hpp
index ee90690..2f7dfa0 100644
--- a/cpp/src/msgpack/pack.hpp
+++ b/cpp/src/msgpack/pack.hpp
@@ -18,7 +18,7 @@
 #ifndef MSGPACK_PACK_HPP__
 #define MSGPACK_PACK_HPP__
 
-#include "msgpack/pack_define.h"
+#include "pack_define.h"
 #include 
 #include 
 
@@ -137,7 +137,7 @@ inline void pack(Stream& s, const T& v)
 
 #define msgpack_pack_append_buffer append_buffer
 
-#include "msgpack/pack_template.h"
+#include "pack_template.h"
 
 
 template 
diff --git a/cpp/src/msgpack/sbuffer.hpp b/cpp/src/msgpack/sbuffer.hpp
index e4a3f96..a9efc6d 100644
--- a/cpp/src/msgpack/sbuffer.hpp
+++ b/cpp/src/msgpack/sbuffer.hpp
@@ -18,7 +18,7 @@
 #ifndef MSGPACK_SBUFFER_HPP__
 #define MSGPACK_SBUFFER_HPP__
 
-#include "msgpack/sbuffer.h"
+#include "sbuffer.h"
 #include 
 
 namespace msgpack {
diff --git a/cpp/src/msgpack/type.hpp b/cpp/src/msgpack/type.hpp
index fafa674..a55c68e 100644
--- a/cpp/src/msgpack/type.hpp
+++ b/cpp/src/msgpack/type.hpp
@@ -1,15 +1,15 @@
-#include "msgpack/type/bool.hpp"
-#include "msgpack/type/float.hpp"
-#include "msgpack/type/int.hpp"
-#include "msgpack/type/list.hpp"
-#include "msgpack/type/deque.hpp"
-#include "msgpack/type/map.hpp"
-#include "msgpack/type/nil.hpp"
-#include "msgpack/type/pair.hpp"
-#include "msgpack/type/raw.hpp"
-#include "msgpack/type/set.hpp"
-#include "msgpack/type/string.hpp"
-#include "msgpack/type/vector.hpp"
-#include "msgpack/type/tuple.hpp"
-#include "msgpack/type/define.hpp"
+#include "type/bool.hpp"
+#include "type/float.hpp"
+#include "type/int.hpp"
+#include "type/list.hpp"
+#include "type/deque.hpp"
+#include "type/map.hpp"
+#include "type/nil.hpp"
+#include "type/pair.hpp"
+#include "type/raw.hpp"
+#include "type/set.hpp"
+#include "type/string.hpp"
+#include "type/vector.hpp"
+#include "type/tuple.hpp"
+#include "type/define.hpp"
 
diff --git a/cpp/src/msgpack/unpack.h b/cpp/src/msgpack/unpack.h
index 82698fc..bea7d92 100644
--- a/cpp/src/msgpack/unpack.h
+++ b/cpp/src/msgpack/unpack.h
@@ -18,8 +18,8 @@
 #ifndef MSGPACK_UNPACKER_H__
 #define MSGPACK_UNPACKER_H__
 
-#include "msgpack/zone.h"
-#include "msgpack/object.h"
+#include "zone.h"
+#include "object.h"
 #include 
 
 #ifdef __cplusplus
diff --git a/cpp/src/msgpack/unpack.hpp b/cpp/src/msgpack/unpack.hpp
index 7b45017..51580b6 100644
--- a/cpp/src/msgpack/unpack.hpp
+++ b/cpp/src/msgpack/unpack.hpp
@@ -18,9 +18,9 @@
 #ifndef MSGPACK_UNPACK_HPP__
 #define MSGPACK_UNPACK_HPP__
 
-#include "msgpack/unpack.h"
-#include "msgpack/object.hpp"
-#include "msgpack/zone.hpp"
+#include "unpack.h"
+#include "object.hpp"
+#include "zone.hpp"
 #include 
 #include 
 
diff --git a/cpp/src/msgpack/vrefbuffer.h b/cpp/src/msgpack/vrefbuffer.h
index 123499d..0643927 100644
--- a/cpp/src/msgpack/vrefbuffer.h
+++ b/cpp/src/msgpack/vrefbuffer.h
@@ -18,7 +18,7 @@
 #ifndef MSGPACK_VREFBUFFER_H__
 #define MSGPACK_VREFBUFFER_H__
 
-#include "msgpack/zone.h"
+#include "zone.h"
 #include 
 
 #ifndef _WIN32
diff --git a/cpp/src/msgpack/vrefbuffer.hpp b/cpp/src/msgpack/vrefbuffer.hpp
index 7e0ffb2..8233527 100644
--- a/cpp/src/msgpack/vrefbuffer.hpp
+++ b/cpp/src/msgpack/vrefbuffer.hpp
@@ -18,7 +18,7 @@
 #ifndef MSGPACK_VREFBUFFER_HPP__
 #define MSGPACK_VREFBUFFER_HPP__
 
-#include "msgpack/vrefbuffer.h"
+#include "vrefbuffer.h"
 #include 
 
 namespace msgpack {
diff --git a/cpp/src/msgpack/zbuffer.h b/cpp/src/msgpack/zbuffer.h
index abb9c50..efdd304 100644
--- a/cpp/src/msgpack/zbuffer.h
+++ b/cpp/src/msgpack/zbuffer.h
@@ -18,7 +18,7 @@
 #ifndef MSGPACK_ZBUFFER_H__
 #define MSGPACK_ZBUFFER_H__
 
-#include "msgpack/sysdep.h"
+#include "sysdep.h"
 #include 
 #include 
 #include 
diff --git a/cpp/src/msgpack/zbuffer.hpp b/cpp/src/msgpack/zbuffer.hpp
index 278f076..08e6fd0 100644
--- a/cpp/src/msgpack/zbuffer.hpp
+++ b/cpp/src/msgpack/zbuffer.hpp
@@ -18,7 +18,7 @@
 #ifndef MSGPACK_ZBUFFER_HPP__
 #define MSGPACK_ZBUFFER_HPP__
 
-#include "msgpack/zbuffer.h"
+#include "zbuffer.h"
 #include 
 
 namespace msgpack {
diff --git a/cpp/src/msgpack/zone.h b/cpp/src/msgpack/zone.h
index 0e811df..d8c60b6 100644
--- a/cpp/src/msgpack/zone.h
+++ b/cpp/src/msgpack/zone.h
@@ -18,7 +18,7 @@
 #ifndef MSGPACK_ZONE_H__
 #define MSGPACK_ZONE_H__
 
-#include "msgpack/sysdep.h"
+#include "sysdep.h"
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/cpp/src/msgpack/zone.hpp.erb b/cpp/src/msgpack/zone.hpp.erb
index 8e69aa4..1cef05e 100644
--- a/cpp/src/msgpack/zone.hpp.erb
+++ b/cpp/src/msgpack/zone.hpp.erb
@@ -18,7 +18,7 @@
 #ifndef MSGPACK_ZONE_HPP__
 #define MSGPACK_ZONE_HPP__
 
-#include "msgpack/zone.h"
+#include "zone.h"
 #include 
 #include 
 #include 

From fe2a0f5089ebfc5c03db783a1f85b1c7c217128a Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Fri, 27 Aug 2010 17:42:05 +0900
Subject: [PATCH 0616/1648] cpp: adds fixed length serialization for integers

---
 cpp/src/Makefile.am             |  3 +-
 cpp/src/msgpack/pack.h          | 12 +++++
 cpp/src/msgpack/pack.hpp        | 56 +++++++++++++++++++++
 cpp/src/msgpack/type.hpp        |  3 +-
 cpp/src/msgpack/type/fixint.hpp | 89 +++++++++++++++++++++++++++++++++
 cpp/test/Makefile.am            |  6 +++
 cpp/test/fixint.cc              | 24 +++++++++
 cpp/test/fixint_c.cc            | 32 ++++++++++++
 8 files changed, 223 insertions(+), 2 deletions(-)
 create mode 100644 cpp/src/msgpack/type/fixint.hpp
 create mode 100644 cpp/test/fixint.cc
 create mode 100644 cpp/test/fixint_c.cc

diff --git a/cpp/src/Makefile.am b/cpp/src/Makefile.am
index 31096f0..0979d23 100644
--- a/cpp/src/Makefile.am
+++ b/cpp/src/Makefile.am
@@ -58,10 +58,11 @@ nobase_include_HEADERS += \
 		msgpack/zone.hpp \
 		msgpack/type.hpp \
 		msgpack/type/bool.hpp \
+		msgpack/type/deque.hpp \
 		msgpack/type/float.hpp \
+		msgpack/type/fixint.hpp \
 		msgpack/type/int.hpp \
 		msgpack/type/list.hpp \
-		msgpack/type/deque.hpp \
 		msgpack/type/map.hpp \
 		msgpack/type/nil.hpp \
 		msgpack/type/pair.hpp \
diff --git a/cpp/src/msgpack/pack.h b/cpp/src/msgpack/pack.h
index 9c4ce59..c156496 100644
--- a/cpp/src/msgpack/pack.h
+++ b/cpp/src/msgpack/pack.h
@@ -70,6 +70,15 @@ static int msgpack_pack_int16(msgpack_packer* pk, int16_t d);
 static int msgpack_pack_int32(msgpack_packer* pk, int32_t d);
 static int msgpack_pack_int64(msgpack_packer* pk, int64_t d);
 
+static int msgpack_pack_fix_uint8(msgpack_packer* pk, uint8_t d);
+static int msgpack_pack_fix_uint16(msgpack_packer* pk, uint16_t d);
+static int msgpack_pack_fix_uint32(msgpack_packer* pk, uint32_t d);
+static int msgpack_pack_fix_uint64(msgpack_packer* pk, uint64_t d);
+static int msgpack_pack_fix_int8(msgpack_packer* pk, int8_t d);
+static int msgpack_pack_fix_int16(msgpack_packer* pk, int16_t d);
+static int msgpack_pack_fix_int32(msgpack_packer* pk, int32_t d);
+static int msgpack_pack_fix_int64(msgpack_packer* pk, int64_t d);
+
 static int msgpack_pack_float(msgpack_packer* pk, float d);
 static int msgpack_pack_double(msgpack_packer* pk, double d);
 
@@ -99,6 +108,9 @@ int msgpack_pack_object(msgpack_packer* pk, msgpack_object d);
 #define msgpack_pack_inline_func_cint(name) \
 	inline int msgpack_pack ## name
 
+#define msgpack_pack_inline_func_fixint(name) \
+	inline int msgpack_pack_fix ## name
+
 #define msgpack_pack_user msgpack_packer*
 
 #define msgpack_pack_append_buffer(user, buf, len) \
diff --git a/cpp/src/msgpack/pack.hpp b/cpp/src/msgpack/pack.hpp
index 2f7dfa0..0090b96 100644
--- a/cpp/src/msgpack/pack.hpp
+++ b/cpp/src/msgpack/pack.hpp
@@ -45,6 +45,15 @@ public:
 	packer& pack_int32(int32_t d);
 	packer& pack_int64(int64_t d);
 
+	packer& pack_fix_uint8(uint8_t d);
+	packer& pack_fix_uint16(uint16_t d);
+	packer& pack_fix_uint32(uint32_t d);
+	packer& pack_fix_uint64(uint64_t d);
+	packer& pack_fix_int8(int8_t d);
+	packer& pack_fix_int16(int16_t d);
+	packer& pack_fix_int32(int32_t d);
+	packer& pack_fix_int64(int64_t d);
+
 	packer& pack_short(short d);
 	packer& pack_int(int d);
 	packer& pack_long(long d);
@@ -78,6 +87,15 @@ private:
 	static void _pack_int32(Stream& x, int32_t d);
 	static void _pack_int64(Stream& x, int64_t d);
 
+	static void _pack_fix_uint8(Stream& x, uint8_t d);
+	static void _pack_fix_uint16(Stream& x, uint16_t d);
+	static void _pack_fix_uint32(Stream& x, uint32_t d);
+	static void _pack_fix_uint64(Stream& x, uint64_t d);
+	static void _pack_fix_int8(Stream& x, int8_t d);
+	static void _pack_fix_int16(Stream& x, int16_t d);
+	static void _pack_fix_int32(Stream& x, int32_t d);
+	static void _pack_fix_int64(Stream& x, int64_t d);
+
 	static void _pack_short(Stream& x, short d);
 	static void _pack_int(Stream& x, int d);
 	static void _pack_long(Stream& x, long d);
@@ -133,6 +151,10 @@ inline void pack(Stream& s, const T& v)
 	template  \
 	inline void packer::_pack ## name
 
+#define msgpack_pack_inline_func_fixint(name) \
+	template  \
+	inline void packer::_pack_fix ## name
+
 #define msgpack_pack_user Stream&
 
 #define msgpack_pack_append_buffer append_buffer
@@ -149,6 +171,7 @@ packer::packer(Stream& s) : m_stream(s) { }
 template 
 packer::~packer() { }
 
+
 template 
 inline packer& packer::pack_uint8(uint8_t d)
 { _pack_uint8(m_stream, d); return *this; }
@@ -182,6 +205,39 @@ inline packer& packer::pack_int64(int64_t d)
 { _pack_int64(m_stream, d); return *this;}
 
 
+template 
+inline packer& packer::pack_fix_uint8(uint8_t d)
+{ _pack_fix_uint8(m_stream, d); return *this; }
+
+template 
+inline packer& packer::pack_fix_uint16(uint16_t d)
+{ _pack_fix_uint16(m_stream, d); return *this; }
+
+template 
+inline packer& packer::pack_fix_uint32(uint32_t d)
+{ _pack_fix_uint32(m_stream, d); return *this; }
+
+template 
+inline packer& packer::pack_fix_uint64(uint64_t d)
+{ _pack_fix_uint64(m_stream, d); return *this; }
+
+template 
+inline packer& packer::pack_fix_int8(int8_t d)
+{ _pack_fix_int8(m_stream, d); return *this; }
+
+template 
+inline packer& packer::pack_fix_int16(int16_t d)
+{ _pack_fix_int16(m_stream, d); return *this; }
+
+template 
+inline packer& packer::pack_fix_int32(int32_t d)
+{ _pack_fix_int32(m_stream, d); return *this; }
+
+template 
+inline packer& packer::pack_fix_int64(int64_t d)
+{ _pack_fix_int64(m_stream, d); return *this;}
+
+
 template 
 inline packer& packer::pack_short(short d)
 { _pack_short(m_stream, d); return *this; }
diff --git a/cpp/src/msgpack/type.hpp b/cpp/src/msgpack/type.hpp
index a55c68e..bca69bf 100644
--- a/cpp/src/msgpack/type.hpp
+++ b/cpp/src/msgpack/type.hpp
@@ -1,8 +1,9 @@
 #include "type/bool.hpp"
+#include "type/deque.hpp"
+#include "type/fixint.hpp"
 #include "type/float.hpp"
 #include "type/int.hpp"
 #include "type/list.hpp"
-#include "type/deque.hpp"
 #include "type/map.hpp"
 #include "type/nil.hpp"
 #include "type/pair.hpp"
diff --git a/cpp/src/msgpack/type/fixint.hpp b/cpp/src/msgpack/type/fixint.hpp
new file mode 100644
index 0000000..da58415
--- /dev/null
+++ b/cpp/src/msgpack/type/fixint.hpp
@@ -0,0 +1,89 @@
+//
+// MessagePack for C++ static resolution routine
+//
+// Copyright (C) 2020 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+#ifndef MSGPACK_TYPE_FIXINT_HPP__
+#define MSGPACK_TYPE_FIXINT_HPP__
+
+#include "msgpack/object.hpp"
+
+namespace msgpack {
+
+namespace type {
+
+
+template 
+struct fix_int {
+	fix_int() : value(0) { }
+	fix_int(T value) : value(value) { }
+	operator T() const { return value; }
+	T get() const { return value; }
+private:
+	const T value;
+};
+
+
+typedef fix_int  fix_uint8;
+typedef fix_int fix_uint16;
+typedef fix_int fix_uint32;
+typedef fix_int fix_uint64;
+
+typedef fix_int  fix_int8;
+typedef fix_int fix_int16;
+typedef fix_int fix_int32;
+typedef fix_int fix_int64;
+
+
+}  // namespace type
+
+
+template 
+inline packer& operator<< (packer& o, const type::fix_int8& v)
+	{ o.pack_fix_int8(v); return o; }
+
+template 
+inline packer& operator<< (packer& o, const type::fix_int16& v)
+	{ o.pack_fix_int16(v); return o; }
+
+template 
+inline packer& operator<< (packer& o, const type::fix_int32& v)
+	{ o.pack_fix_int32(v); return o; }
+
+template 
+inline packer& operator<< (packer& o, const type::fix_int64& v)
+	{ o.pack_fix_int64(v); return o; }
+
+template 
+inline packer& operator<< (packer& o, const type::fix_uint8& v)
+	{ o.pack_fix_uint8(v); return o; }
+
+template 
+inline packer& operator<< (packer& o, const type::fix_uint16& v)
+	{ o.pack_fix_uint16(v); return o; }
+
+template 
+inline packer& operator<< (packer& o, const type::fix_uint32& v)
+	{ o.pack_fix_uint32(v); return o; }
+
+template 
+inline packer& operator<< (packer& o, const type::fix_uint64& v)
+	{ o.pack_fix_uint64(v); return o; }
+
+
+}  // namespace msgpack
+
+#endif /* msgpack/type/fixint.hpp */
+
diff --git a/cpp/test/Makefile.am b/cpp/test/Makefile.am
index bb9439e..5225f28 100644
--- a/cpp/test/Makefile.am
+++ b/cpp/test/Makefile.am
@@ -13,6 +13,8 @@ check_PROGRAMS = \
 		convert \
 		buffer \
 		cases \
+		fixint \
+		fixint_c \
 		version \
 		msgpackc_test \
 		msgpack_test
@@ -38,6 +40,10 @@ buffer_LDADD = -lz
 
 cases_SOURCES = cases.cc
 
+fixint_SOURCES = fixint.cc
+
+fixint_c_SOURCES = fixint_c.cc
+
 version_SOURCES = version.cc
 
 msgpackc_test_SOURCES = msgpackc_test.cpp
diff --git a/cpp/test/fixint.cc b/cpp/test/fixint.cc
new file mode 100644
index 0000000..64a39ac
--- /dev/null
+++ b/cpp/test/fixint.cc
@@ -0,0 +1,24 @@
+#include 
+#include 
+
+template 
+void check_size(size_t size) {
+	T v(0);
+	msgpack::sbuffer sbuf;
+	msgpack::pack(sbuf, v);
+	EXPECT_EQ(size, sbuf.size());
+}
+
+TEST(fixint, size)
+{
+	check_size(2);
+	check_size(3);
+	check_size(5);
+	check_size(9);
+
+	check_size(2);
+	check_size(3);
+	check_size(5);
+	check_size(9);
+}
+
diff --git a/cpp/test/fixint_c.cc b/cpp/test/fixint_c.cc
new file mode 100644
index 0000000..caa4d26
--- /dev/null
+++ b/cpp/test/fixint_c.cc
@@ -0,0 +1,32 @@
+#include 
+#include 
+
+TEST(fixint, size)
+{
+	msgpack_sbuffer* sbuf = msgpack_sbuffer_new();
+	msgpack_packer* pk = msgpack_packer_new(sbuf, msgpack_sbuffer_write);
+
+	size_t sum = 0;
+
+	EXPECT_EQ(0, msgpack_pack_fix_int8(pk, 0));
+	EXPECT_EQ(sum+=2, sbuf->size);
+	EXPECT_EQ(0, msgpack_pack_fix_int16(pk, 0));
+	EXPECT_EQ(sum+=3, sbuf->size);
+	EXPECT_EQ(0, msgpack_pack_fix_int32(pk, 0));
+	EXPECT_EQ(sum+=5, sbuf->size);
+	EXPECT_EQ(0, msgpack_pack_fix_int64(pk, 0));
+	EXPECT_EQ(sum+=9, sbuf->size);
+
+	EXPECT_EQ(0, msgpack_pack_fix_uint8(pk, 0));
+	EXPECT_EQ(sum+=2, sbuf->size);
+	EXPECT_EQ(0, msgpack_pack_fix_uint16(pk, 0));
+	EXPECT_EQ(sum+=3, sbuf->size);
+	EXPECT_EQ(0, msgpack_pack_fix_uint32(pk, 0));
+	EXPECT_EQ(sum+=5, sbuf->size);
+	EXPECT_EQ(0, msgpack_pack_fix_uint64(pk, 0));
+	EXPECT_EQ(sum+=9, sbuf->size);
+
+	msgpack_sbuffer_free(sbuf);
+	msgpack_packer_free(pk);
+}
+

From 2c7573a032b7aa3b0588bace376d3c24b53fbc2d Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Fri, 27 Aug 2010 17:53:02 +0900
Subject: [PATCH 0617/1648] cpp: updates msgpack_vc8.postbuild.bat

---
 cpp/msgpack_vc8.postbuild.bat | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cpp/msgpack_vc8.postbuild.bat b/cpp/msgpack_vc8.postbuild.bat
index 20fabbb..33ff232 100644
--- a/cpp/msgpack_vc8.postbuild.bat
+++ b/cpp/msgpack_vc8.postbuild.bat
@@ -26,10 +26,11 @@ copy src\msgpack\object.hpp            include\msgpack\
 copy src\msgpack\zone.hpp              include\msgpack\
 copy src\msgpack\type.hpp              include\msgpack\type\
 copy src\msgpack\type\bool.hpp         include\msgpack\type\
+copy src\msgpack\type\deque.hpp        include\msgpack\type\
+copy src\msgpack\type\fixint.hpp       include\msgpack\type\
 copy src\msgpack\type\float.hpp        include\msgpack\type\
 copy src\msgpack\type\int.hpp          include\msgpack\type\
 copy src\msgpack\type\list.hpp         include\msgpack\type\
-copy src\msgpack\type\deque.hpp        include\msgpack\type\
 copy src\msgpack\type\map.hpp          include\msgpack\type\
 copy src\msgpack\type\nil.hpp          include\msgpack\type\
 copy src\msgpack\type\pair.hpp         include\msgpack\type\

From 421bee38719ee66d79cd8c376c871678dbb55a1d Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Fri, 27 Aug 2010 17:53:19 +0900
Subject: [PATCH 0618/1648] cpp: version 0.5.3

---
 cpp/ChangeLog    | 7 +++++++
 cpp/configure.in | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/cpp/ChangeLog b/cpp/ChangeLog
index 504ac4b..2756236 100644
--- a/cpp/ChangeLog
+++ b/cpp/ChangeLog
@@ -1,4 +1,11 @@
 
+2010-07-27 version 0.5.3:
+
+  * adds type::fix_{u,}int{8,16,32,64} types
+  * adds msgpack_pack_fix_{u,}int{8,16,32,64} functions
+  * adds packer::pack_fix_{u,}int{8,16,32,64} functions
+  * fixes include paths
+
 2010-07-14 version 0.5.2:
 
   * type::raw::str(), operator==, operator!=, operator< and operator> are now const
diff --git a/cpp/configure.in b/cpp/configure.in
index 93174da..0104ef9 100644
--- a/cpp/configure.in
+++ b/cpp/configure.in
@@ -1,6 +1,6 @@
 AC_INIT(src/object.cpp)
 AC_CONFIG_AUX_DIR(ac)
-AM_INIT_AUTOMAKE(msgpack, 0.5.2)
+AM_INIT_AUTOMAKE(msgpack, 0.5.3)
 AC_CONFIG_HEADER(config.h)
 
 AC_SUBST(CFLAGS)

From c87f7cb9ac87525c9531ddf6b2fd16499535967c Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Fri, 27 Aug 2010 20:52:40 +0900
Subject: [PATCH 0619/1648] cpp: fixes fix_int; updates test/fixint.cc

---
 cpp/src/msgpack/type/fixint.hpp | 85 ++++++++++++++++++++++++++++++++-
 cpp/test/fixint.cc              | 31 ++++++++++++
 2 files changed, 115 insertions(+), 1 deletion(-)

diff --git a/cpp/src/msgpack/type/fixint.hpp b/cpp/src/msgpack/type/fixint.hpp
index da58415..ebe2696 100644
--- a/cpp/src/msgpack/type/fixint.hpp
+++ b/cpp/src/msgpack/type/fixint.hpp
@@ -19,6 +19,7 @@
 #define MSGPACK_TYPE_FIXINT_HPP__
 
 #include "msgpack/object.hpp"
+#include "msgpack/type/int.hpp"
 
 namespace msgpack {
 
@@ -29,10 +30,13 @@ template 
 struct fix_int {
 	fix_int() : value(0) { }
 	fix_int(T value) : value(value) { }
+
 	operator T() const { return value; }
+
 	T get() const { return value; }
+
 private:
-	const T value;
+	T value;
 };
 
 
@@ -50,6 +54,32 @@ typedef fix_int fix_int64;
 }  // namespace type
 
 
+inline type::fix_int8& operator>> (object o, type::fix_int8& v)
+	{ v = type::detail::convert_integer(o); return v; }
+
+inline type::fix_int16& operator>> (object o, type::fix_int16& v)
+	{ v = type::detail::convert_integer(o); return v; }
+
+inline type::fix_int32& operator>> (object o, type::fix_int32& v)
+	{ v = type::detail::convert_integer(o); return v; }
+
+inline type::fix_int64& operator>> (object o, type::fix_int64& v)
+	{ v = type::detail::convert_integer(o); return v; }
+
+
+inline type::fix_uint8& operator>> (object o, type::fix_uint8& v)
+	{ v = type::detail::convert_integer(o); return v; }
+
+inline type::fix_uint16& operator>> (object o, type::fix_uint16& v)
+	{ v = type::detail::convert_integer(o); return v; }
+
+inline type::fix_uint32& operator>> (object o, type::fix_uint32& v)
+	{ v = type::detail::convert_integer(o); return v; }
+
+inline type::fix_uint64& operator>> (object o, type::fix_uint64& v)
+	{ v = type::detail::convert_integer(o); return v; }
+
+
 template 
 inline packer& operator<< (packer& o, const type::fix_int8& v)
 	{ o.pack_fix_int8(v); return o; }
@@ -66,6 +96,7 @@ template 
 inline packer& operator<< (packer& o, const type::fix_int64& v)
 	{ o.pack_fix_int64(v); return o; }
 
+
 template 
 inline packer& operator<< (packer& o, const type::fix_uint8& v)
 	{ o.pack_fix_uint8(v); return o; }
@@ -83,6 +114,58 @@ inline packer& operator<< (packer& o, const type::fix_uint64& v)
 	{ o.pack_fix_uint64(v); return o; }
 
 
+inline void operator<< (object& o, type::fix_int8 v)
+	{ v.get() < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v.get() : o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); }
+
+inline void operator<< (object& o, type::fix_int16 v)
+	{ v.get() < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v.get() : o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); }
+
+inline void operator<< (object& o, type::fix_int32 v)
+	{ v.get() < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v.get() : o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); }
+
+inline void operator<< (object& o, type::fix_int64 v)
+	{ v.get() < 0 ? o.type = type::NEGATIVE_INTEGER, o.via.i64 = v.get() : o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); }
+
+
+inline void operator<< (object& o, type::fix_uint8 v)
+	{ o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); }
+
+inline void operator<< (object& o, type::fix_uint16 v)
+	{ o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); }
+
+inline void operator<< (object& o, type::fix_uint32 v)
+	{ o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); }
+
+inline void operator<< (object& o, type::fix_uint64 v)
+	{ o.type = type::POSITIVE_INTEGER, o.via.u64 = v.get(); }
+
+
+inline void operator<< (object::with_zone& o, type::fix_int8 v)
+	{ static_cast(o) << v; }
+
+inline void operator<< (object::with_zone& o, type::fix_int16 v)
+	{ static_cast(o) << v; }
+
+inline void operator<< (object::with_zone& o, type::fix_int32 v)
+	{ static_cast(o) << v; }
+
+inline void operator<< (object::with_zone& o, type::fix_int64 v)
+	{ static_cast(o) << v; }
+
+
+inline void operator<< (object::with_zone& o, type::fix_uint8 v)
+	{ static_cast(o) << v; }
+
+inline void operator<< (object::with_zone& o, type::fix_uint16 v)
+	{ static_cast(o) << v; }
+
+inline void operator<< (object::with_zone& o, type::fix_uint32 v)
+	{ static_cast(o) << v; }
+
+inline void operator<< (object::with_zone& o, type::fix_uint64 v)
+	{ static_cast(o) << v; }
+
+
 }  // namespace msgpack
 
 #endif /* msgpack/type/fixint.hpp */
diff --git a/cpp/test/fixint.cc b/cpp/test/fixint.cc
index 64a39ac..63288a1 100644
--- a/cpp/test/fixint.cc
+++ b/cpp/test/fixint.cc
@@ -22,3 +22,34 @@ TEST(fixint, size)
 	check_size(9);
 }
 
+
+template 
+void check_convert() {
+	T v1(-11);
+	msgpack::sbuffer sbuf;
+	msgpack::pack(sbuf, v1);
+
+	msgpack::unpacked msg;
+	msgpack::unpack(&msg, sbuf.data(), sbuf.size());
+
+	T v2;
+	msg.get().convert(&v2);
+
+	EXPECT_EQ(v1.get(), v2.get());
+
+	EXPECT_EQ(msg.get(), msgpack::object(T(v1.get())));
+}
+
+TEST(fixint, convert)
+{
+	check_convert();
+	check_convert();
+	check_convert();
+	check_convert();
+
+	check_convert();
+	check_convert();
+	check_convert();
+	check_convert();
+}
+

From c42cba1d5453f718a85f49f7441387ed4d0a7210 Mon Sep 17 00:00:00 2001
From: UENISHI Kota 
Date: Fri, 27 Aug 2010 23:02:16 +0900
Subject: [PATCH 0620/1648] erlang: fixed bug around error case when
 serializing atom.

---
 erlang/msgpack.erl | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/erlang/msgpack.erl b/erlang/msgpack.erl
index b54874d..8c85458 100644
--- a/erlang/msgpack.erl
+++ b/erlang/msgpack.erl
@@ -126,7 +126,7 @@ pack_(List)  when is_list(List) ->
 pack_({Map}) when is_list(Map) ->
     pack_map(Map);
 pack_(Other) ->
-    throw({error, {badarg, Other}}).
+    throw({badarg, Other}).
 
 
 -spec pack_uint_(non_neg_integer()) -> binary().
@@ -387,4 +387,9 @@ benchmark_test()->
     {Data,<<>>}=?debugTime("deserialize", msgpack:unpack(S)),
     ?debugFmt("for ~p KB test data.", [byte_size(S) div 1024]).
 
+error_test()->
+    ?assertEqual({error,{badarg, atom}}, msgpack:pack(atom)),
+    Term = {"hoge", "hage", atom},
+    ?assertEqual({error,{badarg, Term}}, msgpack:pack(Term)).
+
 -endif.

From c44c9ab74da304c70f78fbfda7fcad206435ff0e Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 29 Aug 2010 18:23:16 +0900
Subject: [PATCH 0621/1648] cpp: adds msgpack_vc2008.vcproj file in source
 package

---
 cpp/Makefile.am                                             | 4 +++-
 cpp/{msgpack_vc8.postbuild.bat => msgpack_vc.postbuild.bat} | 0
 cpp/msgpack_vc8.vcproj                                      | 4 ++--
 cpp/preprocess                                              | 3 +++
 4 files changed, 8 insertions(+), 3 deletions(-)
 rename cpp/{msgpack_vc8.postbuild.bat => msgpack_vc.postbuild.bat} (100%)

diff --git a/cpp/Makefile.am b/cpp/Makefile.am
index 7dd4891..ecec1b5 100644
--- a/cpp/Makefile.am
+++ b/cpp/Makefile.am
@@ -6,7 +6,9 @@ DOC_FILES = \
 		NOTICE \
 		msgpack_vc8.vcproj \
 		msgpack_vc8.sln \
-		msgpack_vc8.postbuild.bat
+		msgpack_vc2008.vcproj \
+		msgpack_vc2008.sln \
+		msgpack_vc.postbuild.bat
 
 EXTRA_DIST = \
 		$(DOC_FILES)
diff --git a/cpp/msgpack_vc8.postbuild.bat b/cpp/msgpack_vc.postbuild.bat
similarity index 100%
rename from cpp/msgpack_vc8.postbuild.bat
rename to cpp/msgpack_vc.postbuild.bat
diff --git a/cpp/msgpack_vc8.vcproj b/cpp/msgpack_vc8.vcproj
index ed0daa4..72d47b6 100644
--- a/cpp/msgpack_vc8.vcproj
+++ b/cpp/msgpack_vc8.vcproj
@@ -28,7 +28,7 @@
 			
 			
 			 msgpack_vc2008.vcproj
+sed -e 's/9\.00/10.00/' -e 's/msgpack_vc8/msgpack_vc2008/' < msgpack_vc8.sln > msgpack_vc2008.sln
+

From 3c75361e5a195eba9622927cac7aebe0944f9232 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 29 Aug 2010 18:24:32 +0900
Subject: [PATCH 0622/1648] cpp: updates README.md

---
 cpp/README.md                                 | 45 +++++++++----------
 .../org/msgpack/TestMessageUnpackable.java    |  4 +-
 msgpack/pack_template.h                       | 20 ++++-----
 3 files changed, 31 insertions(+), 38 deletions(-)

diff --git a/cpp/README.md b/cpp/README.md
index 454ce1a..eac7793 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -13,9 +13,10 @@ On UNIX-like platform, run ./configure && make && sudo make install:
     $ make
     $ sudo make install
 
-On Windows, open msgpack_vc8.vcproj file and build it using batch build. DLLs are built on lib folder, and the headers are built on include folder.
+On Windows, open msgpack_vc8.vcproj or msgpack_vc2008 file and build it using batch build. DLLs are built on lib folder,
+and the headers are built on include folder.
 
-To use the library in your program, include msgpack.hpp header and link msgpack and msgpackc library.
+To use the library in your program, include msgpack.hpp header and link "msgpack" library.
 
 
 ## Example
@@ -34,15 +35,9 @@ To use the library in your program, include msgpack.hpp header and link msgpack
         msgpack::pack(&buffer, target);
     
         // Deserialize the serialized data.
-        msgpack::zone mempool;    // this manages the life of deserialized object
-        msgpack::object obj;
-        msgpack::unpack_return ret =
-                msgpack::unpack(buffer.data, buffer.size, NULL, &mempool, &obj);
-    
-        if(ret != msgapck::UNPACK_SUCCESS) {
-            // error check
-            exit(1);
-        }
+        msgpack::unpacked msg;    // includes memory pool and deserialized object
+        msgpack::unpack(&msg, sbuf.data(), sbuf.size());
+        msgpack::object obj = msg.get();
     
         // Print the deserialized object to stdout.
         std::cout << obj << std::endl;    // ["Hello," "World!"]
@@ -55,24 +50,24 @@ To use the library in your program, include msgpack.hpp header and link msgpack
         obj.as();  // type is mismatched, msgpack::type_error is thrown
     }
 
-API document and other example codes are available at the [wiki.](http://msgpack.sourceforge.net/start)
+API documents and other example codes are available at the [wiki.](http://redmine.msgpack.org/projects/msgpack/wiki)
 
 
 ## License
 
-Copyright (C) 2008-2010 FURUHASHI Sadayuki
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+    Copyright (C) 2008-2010 FURUHASHI Sadayuki
+    
+       Licensed under the Apache License, Version 2.0 (the "License");
+       you may not use this file except in compliance with the License.
+       You may obtain a copy of the License at
+    
+           http://www.apache.org/licenses/LICENSE-2.0
+    
+       Unless required by applicable law or agreed to in writing, software
+       distributed under the License is distributed on an "AS IS" BASIS,
+       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+       See the License for the specific language governing permissions and
+       limitations under the License.
 
 See also NOTICE file.
 
diff --git a/java/src/test/java/org/msgpack/TestMessageUnpackable.java b/java/src/test/java/org/msgpack/TestMessageUnpackable.java
index 9099f91..32917c7 100644
--- a/java/src/test/java/org/msgpack/TestMessageUnpackable.java
+++ b/java/src/test/java/org/msgpack/TestMessageUnpackable.java
@@ -24,9 +24,7 @@ public class TestMessageUnpackable {
 		Image dst = new Image();
 
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-
-		dst.messageUnpack(pac);
+		dst.messageUnpack(new Unpacker(in));
 
 		assertEquals(src, dst);
 	}
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index daa8f81..b636967 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -272,63 +272,63 @@ do { \
 } while(0)
 
 
-#ifdef msgpack_pack_inline_func_fastint
+#ifdef msgpack_pack_inline_func_fixint
 
-msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d)
+msgpack_pack_inline_func_fixint(_uint8)(msgpack_pack_user x, uint8_t d)
 {
 	unsigned char buf[2] = {0xcc, TAKE8_8(d)};
 	msgpack_pack_append_buffer(x, buf, 2);
 }
 
-msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d)
+msgpack_pack_inline_func_fixint(_uint16)(msgpack_pack_user x, uint16_t d)
 {
 	unsigned char buf[3];
 	buf[0] = 0xcd; _msgpack_store16(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 3);
 }
 
-msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d)
+msgpack_pack_inline_func_fixint(_uint32)(msgpack_pack_user x, uint32_t d)
 {
 	unsigned char buf[5];
 	buf[0] = 0xce; _msgpack_store32(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 5);
 }
 
-msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d)
+msgpack_pack_inline_func_fixint(_uint64)(msgpack_pack_user x, uint64_t d)
 {
 	unsigned char buf[9];
 	buf[0] = 0xcf; _msgpack_store64(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 9);
 }
 
-msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d)
+msgpack_pack_inline_func_fixint(_int8)(msgpack_pack_user x, int8_t d)
 {
 	unsigned char buf[2] = {0xd0, TAKE8_8(d)};
 	msgpack_pack_append_buffer(x, buf, 2);
 }
 
-msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d)
+msgpack_pack_inline_func_fixint(_int16)(msgpack_pack_user x, int16_t d)
 {
 	unsigned char buf[3];
 	buf[0] = 0xd1; _msgpack_store16(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 3);
 }
 
-msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d)
+msgpack_pack_inline_func_fixint(_int32)(msgpack_pack_user x, int32_t d)
 {
 	unsigned char buf[5];
 	buf[0] = 0xd2; _msgpack_store32(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 5);
 }
 
-msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d)
+msgpack_pack_inline_func_fixint(_int64)(msgpack_pack_user x, int64_t d)
 {
 	unsigned char buf[9];
 	buf[0] = 0xd3; _msgpack_store64(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 9);
 }
 
-#undef msgpack_pack_inline_func_fastint
+#undef msgpack_pack_inline_func_fixint
 #endif
 
 

From 9684c8664fe7643274fe546ccbe0009c741ebc72 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 29 Aug 2010 18:27:10 +0900
Subject: [PATCH 0623/1648] cpp: version 0.5.4

---
 cpp/ChangeLog    | 7 ++++++-
 cpp/configure.in | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/cpp/ChangeLog b/cpp/ChangeLog
index 2756236..71c7d5b 100644
--- a/cpp/ChangeLog
+++ b/cpp/ChangeLog
@@ -1,5 +1,10 @@
 
-2010-07-27 version 0.5.3:
+2010-08-29 version 0.5.4:
+
+  * includes msgpack_vc2008.vcproj file in source package
+  * fixes type::fix_int types
+
+2010-08-27 version 0.5.3:
 
   * adds type::fix_{u,}int{8,16,32,64} types
   * adds msgpack_pack_fix_{u,}int{8,16,32,64} functions
diff --git a/cpp/configure.in b/cpp/configure.in
index 0104ef9..2dd92d1 100644
--- a/cpp/configure.in
+++ b/cpp/configure.in
@@ -1,6 +1,6 @@
 AC_INIT(src/object.cpp)
 AC_CONFIG_AUX_DIR(ac)
-AM_INIT_AUTOMAKE(msgpack, 0.5.3)
+AM_INIT_AUTOMAKE(msgpack, 0.5.4)
 AC_CONFIG_HEADER(config.h)
 
 AC_SUBST(CFLAGS)

From a1bd14e516a0baef6f96b441da70e29e5be7d175 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 31 Aug 2010 06:27:15 +0900
Subject: [PATCH 0624/1648] template: casts integer types explicitly

---
 msgpack/pack_template.h | 44 ++++++++++++++++++++---------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index b636967..da54c36 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -69,7 +69,7 @@ do { \
 	} else { \
 		/* unsigned 16 */ \
 		unsigned char buf[3]; \
-		buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+		buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 		msgpack_pack_append_buffer(x, buf, 3); \
 	} \
 } while(0)
@@ -89,12 +89,12 @@ do { \
 		if(d < (1<<16)) { \
 			/* unsigned 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* unsigned 32 */ \
 			unsigned char buf[5]; \
-			buf[0] = 0xce; _msgpack_store32(&buf[1], d); \
+			buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} \
 	} \
@@ -103,7 +103,7 @@ do { \
 #define msgpack_pack_real_uint64(x, d) \
 do { \
 	if(d < (1ULL<<8)) { \
-		if(d < (1<<7)) { \
+		if(d < (1ULL<<7)) { \
 			/* fixnum */ \
 			msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
 		} else { \
@@ -115,12 +115,12 @@ do { \
 		if(d < (1ULL<<16)) { \
 			/* unsigned 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else if(d < (1ULL<<32)) { \
 			/* unsigned 32 */ \
 			unsigned char buf[5]; \
-			buf[0] = 0xce; _msgpack_store32(&buf[1], d); \
+			buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} else { \
 			/* unsigned 64 */ \
@@ -149,7 +149,7 @@ do { \
 		if(d < -(1<<7)) { \
 			/* signed 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \
+			buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* signed 8 */ \
@@ -167,7 +167,7 @@ do { \
 		} else { \
 			/* unsigned 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} \
 	} \
@@ -179,12 +179,12 @@ do { \
 		if(d < -(1<<15)) { \
 			/* signed 32 */ \
 			unsigned char buf[5]; \
-			buf[0] = 0xd2; _msgpack_store32(&buf[1], d); \
+			buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} else if(d < -(1<<7)) { \
 			/* signed 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \
+			buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* signed 8 */ \
@@ -202,12 +202,12 @@ do { \
 		} else if(d < (1<<16)) { \
 			/* unsigned 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* unsigned 32 */ \
 			unsigned char buf[5]; \
-			buf[0] = 0xce; _msgpack_store32(&buf[1], d); \
+			buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} \
 	} \
@@ -225,14 +225,14 @@ do { \
 			} else { \
 				/* signed 32 */ \
 				unsigned char buf[5]; \
-				buf[0] = 0xd2; _msgpack_store32(&buf[1], d); \
+				buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
 				msgpack_pack_append_buffer(x, buf, 5); \
 			} \
 		} else { \
 			if(d < -(1<<7)) { \
 				/* signed 16 */ \
 				unsigned char buf[3]; \
-				buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \
+				buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
 				msgpack_pack_append_buffer(x, buf, 3); \
 			} else { \
 				/* signed 8 */ \
@@ -252,14 +252,14 @@ do { \
 			} else { \
 				/* unsigned 16 */ \
 				unsigned char buf[3]; \
-				buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+				buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 				msgpack_pack_append_buffer(x, buf, 3); \
 			} \
 		} else { \
 			if(d < (1LL<<32)) { \
 				/* unsigned 32 */ \
 				unsigned char buf[5]; \
-				buf[0] = 0xce; _msgpack_store32(&buf[1], d); \
+				buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
 				msgpack_pack_append_buffer(x, buf, 5); \
 			} else { \
 				/* unsigned 64 */ \
@@ -690,11 +690,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n)
 		msgpack_pack_append_buffer(x, &d, 1);
 	} else if(n < 65536) {
 		unsigned char buf[3];
-		buf[0] = 0xdc; _msgpack_store16(&buf[1], n);
+		buf[0] = 0xdc; _msgpack_store16(&buf[1], (uint16_t)n);
 		msgpack_pack_append_buffer(x, buf, 3);
 	} else {
 		unsigned char buf[5];
-		buf[0] = 0xdd; _msgpack_store32(&buf[1], n);
+		buf[0] = 0xdd; _msgpack_store32(&buf[1], (uint32_t)n);
 		msgpack_pack_append_buffer(x, buf, 5);
 	}
 }
@@ -711,11 +711,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n)
 		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
 	} else if(n < 65536) {
 		unsigned char buf[3];
-		buf[0] = 0xde; _msgpack_store16(&buf[1], n);
+		buf[0] = 0xde; _msgpack_store16(&buf[1], (uint16_t)n);
 		msgpack_pack_append_buffer(x, buf, 3);
 	} else {
 		unsigned char buf[5];
-		buf[0] = 0xdf; _msgpack_store32(&buf[1], n);
+		buf[0] = 0xdf; _msgpack_store32(&buf[1], (uint32_t)n);
 		msgpack_pack_append_buffer(x, buf, 5);
 	}
 }
@@ -732,11 +732,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l)
 		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
 	} else if(l < 65536) {
 		unsigned char buf[3];
-		buf[0] = 0xda; _msgpack_store16(&buf[1], l);
+		buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l);
 		msgpack_pack_append_buffer(x, buf, 3);
 	} else {
 		unsigned char buf[5];
-		buf[0] = 0xdb; _msgpack_store32(&buf[1], l);
+		buf[0] = 0xdb; _msgpack_store32(&buf[1], (uint32_t)l);
 		msgpack_pack_append_buffer(x, buf, 5);
 	}
 }

From b5c78de2ddf82783a6f80a199b68927d1a1747ca Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 31 Aug 2010 06:30:16 +0900
Subject: [PATCH 0625/1648] ruby: converts encodings into UTF-8 on Ruby 1.9

---
 ruby/encoding.h            | 33 ++++++++++++++++++
 ruby/pack.c                | 21 ++++++++++--
 ruby/rbinit.c              | 15 +++++++++
 ruby/test/test_encoding.rb | 68 ++++++++++++++++++++++++++++++++++++++
 ruby/test/test_helper.rb   |  4 ++-
 ruby/unpack.c              | 38 +++------------------
 6 files changed, 141 insertions(+), 38 deletions(-)
 create mode 100644 ruby/encoding.h
 create mode 100644 ruby/test/test_encoding.rb

diff --git a/ruby/encoding.h b/ruby/encoding.h
new file mode 100644
index 0000000..2ad3fd7
--- /dev/null
+++ b/ruby/encoding.h
@@ -0,0 +1,33 @@
+/*
+ * MessagePack for Ruby
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef ENCODING_H__
+#define ENCODING_H__
+
+
+#ifdef HAVE_RUBY_ENCODING_H
+#include "ruby/encoding.h"
+#define MSGPACK_RUBY_ENCODING
+extern int s_enc_utf8;
+extern int s_enc_ascii8bit;
+extern int s_enc_usascii;
+extern VALUE s_enc_utf8_value;
+#endif
+
+
+#endif /* encoding.h */
+
diff --git a/ruby/pack.c b/ruby/pack.c
index bbeac4a..35878c7 100644
--- a/ruby/pack.c
+++ b/ruby/pack.c
@@ -16,6 +16,8 @@
  *    limitations under the License.
  */
 #include "ruby.h"
+#include "encoding.h"
+
 #include "msgpack/pack_define.h"
 
 static ID s_to_msgpack;
@@ -131,7 +133,6 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self)
 static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
-	// FIXME bignum
 	if(RBIGNUM_SIGN(self)) {  // positive
 		msgpack_pack_uint64(out, rb_big2ull(self));
 	} else {  // negative
@@ -168,6 +169,14 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self)
 static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
+#ifdef MSGPACK_RUBY_ENCODING
+	int enc = ENCODING_GET(self);
+	if(enc != s_enc_utf8 && enc != s_enc_ascii8bit && enc != s_enc_usascii) {
+		if(!ENC_CODERANGE_ASCIIONLY(self)) {
+			self = rb_str_encode(self, s_enc_utf8_value, 0, Qnil);
+		}
+	}
+#endif
 	msgpack_pack_raw(out, RSTRING_LEN(self));
 	msgpack_pack_raw_body(out, RSTRING_PTR(self), RSTRING_LEN(self));
 	return out;
@@ -184,12 +193,16 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self)
  */
 static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
+#ifdef MSGPACK_RUBY_ENCODING
+	return MessagePack_String_to_msgpack(argc, argv, rb_id2str(SYM2ID(self)));
+#else
 	ARG_BUFFER(out, argc, argv);
 	const char* name = rb_id2name(SYM2ID(self));
 	size_t len = strlen(name);
 	msgpack_pack_raw(out, len);
 	msgpack_pack_raw_body(out, name, len);
 	return out;
+#endif
 }
 
 
@@ -205,7 +218,8 @@ static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self)
 static VALUE MessagePack_Array_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
-	msgpack_pack_array(out, RARRAY_LEN(self));
+	// FIXME check sizeof(long) > sizeof(unsigned int) && RARRAY_LEN(self) > UINT_MAX
+	msgpack_pack_array(out, (unsigned int)RARRAY_LEN(self));
 	VALUE* p = RARRAY_PTR(self);
 	VALUE* const pend = p + RARRAY_LEN(self);
 	for(;p != pend; ++p) {
@@ -239,7 +253,8 @@ static int MessagePack_Hash_to_msgpack_foreach(VALUE key, VALUE value, VALUE out
 static VALUE MessagePack_Hash_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
-	msgpack_pack_map(out, RHASH_SIZE(self));
+	// FIXME check sizeof(st_index_t) > sizeof(unsigned int) && RARRAY_LEN(self) > UINT_MAX
+	msgpack_pack_map(out, (unsigned int)RHASH_SIZE(self));
 	rb_hash_foreach(self, MessagePack_Hash_to_msgpack_foreach, out);
 	return out;
 }
diff --git a/ruby/rbinit.c b/ruby/rbinit.c
index 28a8bfe..4678159 100644
--- a/ruby/rbinit.c
+++ b/ruby/rbinit.c
@@ -17,9 +17,17 @@
  */
 #include "pack.h"
 #include "unpack.h"
+#include "encoding.h"
 
 static VALUE mMessagePack;
 
+#ifdef MSGPACK_RUBY_ENCODING
+int s_enc_utf8;
+int s_enc_ascii8bit;
+int s_enc_usascii;
+VALUE s_enc_utf8_value;
+#endif
+
 /**
  * Document-module: MessagePack
  *
@@ -46,6 +54,13 @@ void Init_msgpack(void)
 
 	rb_define_const(mMessagePack, "VERSION", rb_str_new2(MESSAGEPACK_VERSION));
 
+#ifdef MSGPACK_RUBY_ENCODING
+	s_enc_ascii8bit = rb_ascii8bit_encindex();
+	s_enc_utf8 = rb_utf8_encindex();
+	s_enc_usascii = rb_usascii_encindex();
+	s_enc_utf8_value = rb_enc_from_encoding(rb_utf8_encoding());
+#endif
+
 	Init_msgpack_unpack(mMessagePack);
 	Init_msgpack_pack(mMessagePack);
 }
diff --git a/ruby/test/test_encoding.rb b/ruby/test/test_encoding.rb
new file mode 100644
index 0000000..2cf0767
--- /dev/null
+++ b/ruby/test/test_encoding.rb
@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__)+'/test_helper'
+
+if RUBY_VERSION < "1.9"
+	exit
+end
+
+class MessagePackTestEncoding < Test::Unit::TestCase
+	def self.it(name, &block)
+		define_method("test_#{name}", &block)
+	end
+
+	it "US-ASCII" do
+		check_unpack "abc".force_encoding("US-ASCII")
+	end
+
+	it "UTF-8 ascii" do
+		check_unpack "abc".force_encoding("UTF-8")
+	end
+
+	it "UTF-8 mbstr" do
+		check_unpack "\xE3\x81\x82".force_encoding("UTF-8")
+	end
+
+	it "UTF-8 invalid" do
+		check_unpack "\xD0".force_encoding("UTF-8")
+	end
+
+	it "ASCII-8BIT" do
+		check_unpack "\xD0".force_encoding("ASCII-8BIT")
+	end
+
+	it "EUC-JP" do
+		x = "\xA4\xA2".force_encoding("EUC-JP")
+		check_unpack(x)
+	end
+
+	it "EUC-JP invalid" do
+		begin
+			"\xD0".force_encoding("EUC-JP").to_msgpack
+			assert(false)
+		rescue Encoding::InvalidByteSequenceError
+			assert(true)
+		end
+	end
+
+	private
+	def check_unpack(str)
+		if str.encoding.to_s == "ASCII-8BIT"
+			should_str = str.dup.force_encoding("UTF-8")
+		else
+			should_str = str.encode("UTF-8")
+		end
+
+		raw = str.to_msgpack
+		r = MessagePack.unpack(str.to_msgpack)
+		assert_equal(r.encoding.to_s, "UTF-8")
+		assert_equal(r, should_str.force_encoding("UTF-8"))
+
+		if str.valid_encoding?
+			sym = str.to_sym
+			r = MessagePack.unpack(sym.to_msgpack)
+			assert_equal(r.encoding.to_s, "UTF-8")
+			assert_equal(r, should_str.force_encoding("UTF-8"))
+		end
+	end
+end
+
diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb
index 80d7806..4def861 100644
--- a/ruby/test/test_helper.rb
+++ b/ruby/test/test_helper.rb
@@ -5,4 +5,6 @@ rescue LoadError
 require File.dirname(__FILE__) + '/../lib/msgpack'
 end
 
-#GC.stress = true
+if ENV["GC_STRESS"]
+	GC.stress = true
+end
diff --git a/ruby/unpack.c b/ruby/unpack.c
index 0948151..3c5e350 100644
--- a/ruby/unpack.c
+++ b/ruby/unpack.c
@@ -16,17 +16,13 @@
  *    limitations under the License.
  */
 #include "ruby.h"
+#include "encoding.h"
 
 #include "msgpack/unpack_define.h"
 
 static ID s_sysread;
 static ID s_readpartial;
 
-#ifdef HAVE_RUBY_ENCODING_H
-#include "ruby/encoding.h"
-int s_ascii_8bit;
-#endif
-
 struct unpack_buffer {
 	size_t size;
 	size_t free;
@@ -136,6 +132,9 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha
 	} else {
 		*o = rb_str_substr(u->source, p - b, l);
 	}
+#ifdef MSGPACK_RUBY_ENCODING
+	ENCODING_SET(*o, s_enc_utf8);
+#endif
 	return 0;
 }
 
@@ -163,16 +162,6 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha
 #endif
 
 
-#ifdef HAVE_RUBY_ENCODING_H
-static VALUE template_execute_rescue_enc(VALUE data)
-{
-	rb_gc_enable();
-	VALUE* resc = (VALUE*)data;
-	rb_enc_set_index(resc[0], (int)resc[1]);
-	RERAISE;
-}
-#endif
-
 static VALUE template_execute_rescue(VALUE nouse)
 {
 	rb_gc_enable();
@@ -203,31 +192,16 @@ static int template_execute_wrap(msgpack_unpack_t* mp,
 		(VALUE)from,
 	};
 
-#ifdef HAVE_RUBY_ENCODING_H
-	int enc_orig = rb_enc_get_index(str);
-	rb_enc_set_index(str, s_ascii_8bit);
-#endif
-
 	// FIXME execute実行中ã¯mp->topãŒæ›´æ–°ã•ã‚Œãªã„ã®ã§GC markãŒæ©Ÿèƒ½ã—ãªã„
 	rb_gc_disable();
 
 	mp->user.source = str;
 
-#ifdef HAVE_RUBY_ENCODING_H
-	VALUE resc[2] = {str, enc_orig};
-	int ret = (int)rb_rescue(template_execute_do, (VALUE)args,
-			template_execute_rescue_enc, (VALUE)resc);
-#else
 	int ret = (int)rb_rescue(template_execute_do, (VALUE)args,
 			template_execute_rescue, Qnil);
-#endif
 
 	rb_gc_enable();
 
-#ifdef HAVE_RUBY_ENCODING_H
-	rb_enc_set_index(str, enc_orig);
-#endif
-
 	return ret;
 }
 
@@ -746,10 +720,6 @@ void Init_msgpack_unpack(VALUE mMessagePack)
 	s_sysread = rb_intern("sysread");
 	s_readpartial = rb_intern("readpartial");
 
-#ifdef HAVE_RUBY_ENCODING_H
-	s_ascii_8bit = rb_enc_find_index("ASCII-8BIT");
-#endif
-
 	eUnpackError = rb_define_class_under(mMessagePack, "UnpackError", rb_eStandardError);
 	cUnpacker = rb_define_class_under(mMessagePack, "Unpacker", rb_cObject);
 	rb_define_alloc_func(cUnpacker, MessagePack_Unpacker_alloc);

From 09b47cc536ebd951c231fd5e09b4382a25b98020 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 31 Aug 2010 07:00:19 +0900
Subject: [PATCH 0626/1648] ruby: fixes compatibility with ruby-1.8.5

---
 ruby/{encoding.h => compat.h} | 35 +++++++++++++++++++++++++++++++----
 ruby/pack.c                   |  6 +++---
 ruby/rbinit.c                 |  6 +++---
 ruby/test/test_pack_unpack.rb | 12 ++++++++----
 ruby/unpack.c                 | 12 +++---------
 5 files changed, 48 insertions(+), 23 deletions(-)
 rename ruby/{encoding.h => compat.h} (60%)

diff --git a/ruby/encoding.h b/ruby/compat.h
similarity index 60%
rename from ruby/encoding.h
rename to ruby/compat.h
index 2ad3fd7..98c8881 100644
--- a/ruby/encoding.h
+++ b/ruby/compat.h
@@ -15,19 +15,46 @@
  *    See the License for the specific language governing permissions and
  *    limitations under the License.
  */
-#ifndef ENCODING_H__
-#define ENCODING_H__
+#ifndef COMPAT_H__
+#define COMPAT_H__
 
 
 #ifdef HAVE_RUBY_ENCODING_H
 #include "ruby/encoding.h"
-#define MSGPACK_RUBY_ENCODING
+#define COMPAT_HAVE_ENCODING
 extern int s_enc_utf8;
 extern int s_enc_ascii8bit;
 extern int s_enc_usascii;
 extern VALUE s_enc_utf8_value;
 #endif
 
+#ifdef RUBY_VM
+#define COMPAT_RERAISE rb_exc_raise(rb_errinfo())
+#else
+#define COMPAT_RERAISE rb_exc_raise(ruby_errinfo)
+#endif
 
-#endif /* encoding.h */
+
+/* ruby 1.8.5 */
+#ifndef RSTRING_PTR
+#define RSTRING_PTR(s) (RSTRING(s)->ptr)
+#endif
+
+/* ruby 1.8.5 */
+#ifndef RSTRING_LEN
+#define RSTRING_LEN(s) (RSTRING(s)->len)
+#endif
+
+/* ruby 1.8.5 */
+#ifndef RARRAY_PTR
+#define RARRAY_PTR(s) (RARRAY(s)->ptr)
+#endif
+
+/* ruby 1.8.5 */
+#ifndef RARRAY_LEN
+#define RARRAY_LEN(s) (RARRAY(s)->len)
+#endif
+
+
+#endif /* compat.h */
 
diff --git a/ruby/pack.c b/ruby/pack.c
index 35878c7..49b69fc 100644
--- a/ruby/pack.c
+++ b/ruby/pack.c
@@ -16,7 +16,7 @@
  *    limitations under the License.
  */
 #include "ruby.h"
-#include "encoding.h"
+#include "compat.h"
 
 #include "msgpack/pack_define.h"
 
@@ -169,7 +169,7 @@ static VALUE MessagePack_Float_to_msgpack(int argc, VALUE *argv, VALUE self)
 static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
-#ifdef MSGPACK_RUBY_ENCODING
+#ifdef COMPAT_HAVE_ENCODING
 	int enc = ENCODING_GET(self);
 	if(enc != s_enc_utf8 && enc != s_enc_ascii8bit && enc != s_enc_usascii) {
 		if(!ENC_CODERANGE_ASCIIONLY(self)) {
@@ -193,7 +193,7 @@ static VALUE MessagePack_String_to_msgpack(int argc, VALUE *argv, VALUE self)
  */
 static VALUE MessagePack_Symbol_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
-#ifdef MSGPACK_RUBY_ENCODING
+#ifdef COMPAT_HAVE_ENCODING
 	return MessagePack_String_to_msgpack(argc, argv, rb_id2str(SYM2ID(self)));
 #else
 	ARG_BUFFER(out, argc, argv);
diff --git a/ruby/rbinit.c b/ruby/rbinit.c
index 4678159..1d1cbc6 100644
--- a/ruby/rbinit.c
+++ b/ruby/rbinit.c
@@ -17,11 +17,11 @@
  */
 #include "pack.h"
 #include "unpack.h"
-#include "encoding.h"
+#include "compat.h"
 
 static VALUE mMessagePack;
 
-#ifdef MSGPACK_RUBY_ENCODING
+#ifdef COMPAT_HAVE_ENCODING
 int s_enc_utf8;
 int s_enc_ascii8bit;
 int s_enc_usascii;
@@ -54,7 +54,7 @@ void Init_msgpack(void)
 
 	rb_define_const(mMessagePack, "VERSION", rb_str_new2(MESSAGEPACK_VERSION));
 
-#ifdef MSGPACK_RUBY_ENCODING
+#ifdef COMPAT_HAVE_ENCODING
 	s_enc_ascii8bit = rb_ascii8bit_encindex();
 	s_enc_utf8 = rb_utf8_encindex();
 	s_enc_usascii = rb_usascii_encindex();
diff --git a/ruby/test/test_pack_unpack.rb b/ruby/test/test_pack_unpack.rb
index 25bde81..545e593 100644
--- a/ruby/test/test_pack_unpack.rb
+++ b/ruby/test/test_pack_unpack.rb
@@ -153,7 +153,8 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase
 	end
 
 	it "{1=>1}" do
-		match ({1=>1}), "\x81\x01\x01"
+		obj = {1=>1}
+		match obj, "\x81\x01\x01"
 	end
 
 	it "1.0" do
@@ -165,15 +166,18 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase
 	end
 
 	it "[0, 1, ..., 14]" do
-		match (0..14).to_a, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"
+		obj = (0..14).to_a
+		match obj, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"
 	end
 
 	it "[0, 1, ..., 15]" do
-		match (0..15).to_a, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+		obj = (0..15).to_a
+		match obj, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
 	end
 
 	it "{}" do
-		match ({}), "\x80"
+		obj = {}
+		match obj, "\x80"
 	end
 
 ## FIXME
diff --git a/ruby/unpack.c b/ruby/unpack.c
index 3c5e350..2d10e75 100644
--- a/ruby/unpack.c
+++ b/ruby/unpack.c
@@ -16,7 +16,7 @@
  *    limitations under the License.
  */
 #include "ruby.h"
-#include "encoding.h"
+#include "compat.h"
 
 #include "msgpack/unpack_define.h"
 
@@ -132,7 +132,7 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha
 	} else {
 		*o = rb_str_substr(u->source, p - b, l);
 	}
-#ifdef MSGPACK_RUBY_ENCODING
+#ifdef COMPAT_HAVE_ENCODING
 	ENCODING_SET(*o, s_enc_utf8);
 #endif
 	return 0;
@@ -155,17 +155,11 @@ static inline int template_callback_raw(unpack_user* u, const char* b, const cha
 		rb_raise(rb_eTypeError, "instance of String needed"); \
 	}
 
-#ifdef RUBY_VM
-#define RERAISE rb_exc_raise(rb_errinfo())
-#else
-#define RERAISE rb_exc_raise(ruby_errinfo)
-#endif
-
 
 static VALUE template_execute_rescue(VALUE nouse)
 {
 	rb_gc_enable();
-	RERAISE;
+	COMPAT_RERAISE;
 }
 
 static VALUE template_execute_do(VALUE argv)

From 71a1cb01842787e2fa897f023addd88337542915 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Tue, 31 Aug 2010 09:29:01 +0900
Subject: [PATCH 0627/1648] fixes compatibility with Rubinius

---
 ruby/compat.h   | 17 +++++++++++++++++
 ruby/extconf.rb |  2 +-
 ruby/pack.c     |  8 ++------
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/ruby/compat.h b/ruby/compat.h
index 98c8881..d7a2ca7 100644
--- a/ruby/compat.h
+++ b/ruby/compat.h
@@ -35,6 +35,23 @@ extern VALUE s_enc_utf8_value;
 #endif
 
 
+/* ruby 1.8 and Rubinius */
+#ifndef RBIGNUM_POSITIVE_P
+# ifdef RUBINIUS
+#  define RBIGNUM_POSITIVE_P(b) (rb_funcall(b, rb_intern(">="), 1, INT2FIX(0)) == Qtrue)
+# else
+#  define RBIGNUM_POSITIVE_P(b) (RBIGNUM(b)->sign)
+# endif
+#endif
+
+
+/* Rubinius */
+#ifdef RUBINIUS
+static inline void rb_gc_enable() { return; }
+static inline void rb_gc_disable() { return; }
+#endif
+
+
 /* ruby 1.8.5 */
 #ifndef RSTRING_PTR
 #define RSTRING_PTR(s) (RSTRING(s)->ptr)
diff --git a/ruby/extconf.rb b/ruby/extconf.rb
index eb6a389..f1d44ec 100644
--- a/ruby/extconf.rb
+++ b/ruby/extconf.rb
@@ -1,5 +1,5 @@
 require 'mkmf'
 require './version.rb'
-$CFLAGS << %[ -I.. -Wall -O4 -DMESSAGEPACK_VERSION=\\"#{MessagePack::VERSION}\\"]
+$CFLAGS << %[ -I.. -Wall -O4 -DMESSAGEPACK_VERSION=\\"#{MessagePack::VERSION}\\" -g]
 create_makefile('msgpack')
 
diff --git a/ruby/pack.c b/ruby/pack.c
index 49b69fc..8ce46aa 100644
--- a/ruby/pack.c
+++ b/ruby/pack.c
@@ -118,10 +118,6 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self)
 }
 
 
-#ifndef RBIGNUM_SIGN  // Ruby 1.8
-#define RBIGNUM_SIGN(b) (RBIGNUM(b)->sign)
-#endif
-
 /*
  * Document-method: Bignum#to_msgpack
  *
@@ -133,9 +129,9 @@ static VALUE MessagePack_Fixnum_to_msgpack(int argc, VALUE *argv, VALUE self)
 static VALUE MessagePack_Bignum_to_msgpack(int argc, VALUE *argv, VALUE self)
 {
 	ARG_BUFFER(out, argc, argv);
-	if(RBIGNUM_SIGN(self)) {  // positive
+	if(RBIGNUM_POSITIVE_P(self)) {
 		msgpack_pack_uint64(out, rb_big2ull(self));
-	} else {  // negative
+	} else {
 		msgpack_pack_int64(out, rb_big2ll(self));
 	}
 	return out;

From 23a7137e6a6d7f2910fef2dde305f45f83416194 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Tue, 31 Aug 2010 23:42:32 +0900
Subject: [PATCH 0628/1648] Perl: better argument validation(patch from
 dankogai)

---
 perl/pack.c       | 4 ++--
 perl/t/08_cycle.t | 5 ++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/perl/pack.c b/perl/pack.c
index af6669c..93b2e2f 100644
--- a/perl/pack.c
+++ b/perl/pack.c
@@ -151,7 +151,7 @@ static int try_int(enc_t* enc, const char *p, size_t len) {
 static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
 
 static void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) {
-    if (!depth) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
+    if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
     SvGETMAGIC(sv);
 
     if (sv==NULL) {
@@ -187,7 +187,7 @@ static void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) {
 
 static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
     svtype svt;
-    if (!depth) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
+    if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
     SvGETMAGIC(sv);
     svt = SvTYPE(sv);
 
diff --git a/perl/t/08_cycle.t b/perl/t/08_cycle.t
index 55d8427..2bd66c1 100644
--- a/perl/t/08_cycle.t
+++ b/perl/t/08_cycle.t
@@ -2,7 +2,7 @@ use t::Util;
 use Test::More;
 use Data::MessagePack;
 
-plan tests => 5;
+plan tests => 6;
 
 my $aref = [0];
 $aref->[1] = $aref;
@@ -23,3 +23,6 @@ ok !$@;
 
 eval { Data::MessagePack->pack($aref, 2) };
 ok $@, $@;
+
+eval { Data::MessagePack->pack($aref, -1) };
+ok $@, $@;

From 558e9c21edf3cee5813aaa0e7797509eec5d43fb Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Wed, 1 Sep 2010 08:19:05 +0900
Subject: [PATCH 0629/1648] Perl: 0.15

---
 perl/Changes                 | 5 +++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index 13fc98b..189990a 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.15
+
+    - better argument validation.
+      (Dan Kogai)
+
 0.14
 
     - fixed segv on serializing cyclic reference
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 3c38a79..276353a 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -4,7 +4,7 @@ use warnings;
 use XSLoader;
 use 5.008001;
 
-our $VERSION = '0.14';
+our $VERSION = '0.15';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };

From 712b8eec3d90f7e61908cb32c4433ee38a5f1848 Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Wed, 1 Sep 2010 11:22:43 +0900
Subject: [PATCH 0630/1648] added pp version

---
 perl/lib/Data/MessagePack/PP.pm | 556 ++++++++++++++++++++++++++++++++
 1 file changed, 556 insertions(+)
 create mode 100644 perl/lib/Data/MessagePack/PP.pm

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
new file mode 100644
index 0000000..f4f1060
--- /dev/null
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -0,0 +1,556 @@
+package Data::MessagePack::PP;
+
+use 5.008000;
+use strict;
+use B ();
+use Scalar::Util qw( blessed );
+use Carp ();
+
+our $VERSION = '0.03';
+
+
+# copied from Data::MessagePack
+our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };
+our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" };
+
+sub true  () { $true  }
+sub false () { $false }
+
+our $PreferInteger;
+
+# See also
+# http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec
+# http://cpansearch.perl.org/src/YAPPO/Data-Model-0.00006/lib/Data/Model/Driver/Memcached.pm
+# http://frox25.no-ip.org/~mtve/wiki/MessagePack.html : reference to using CORE::pack, CORE::unpack
+
+
+BEGIN {
+    # for pack and unpack compatibility
+    if ( $] < 5.010 ) {
+        require Data::Float;
+        *pack_double = sub {
+            my $float_hex = Data::Float::float_hex( $_[0] );
+            my ( $sign, $sgnf, $exp ) = $float_hex =~ /^([-+])0x1\.([a-z0-9]+)p([-+][\d]+)$/;
+            my @bits;
+
+            $sign = $sign eq '-' ? 1 : 0;
+            $exp  = sprintf( '%011b', 1023 + $exp );
+
+            my $bit  = $sign . $exp . join( '', map { unpack('B4', pack('H', $_) ) } split //, $sgnf );
+
+            while ( $bit =~ /(.{8})/g ) {
+                push @bits, $1;
+            }
+
+             return pack( 'C*', 0xcb, map { unpack( 'C', pack("B*", $_ ) ) } @bits );
+        };
+        *unpack_double = sub {
+            my $bits = join('', map { sprintf('%08b', $_) } unpack( 'C*', substr( $_[0], $_[1], 8 ) ) );
+            my $sign = substr($bits, 0, 1) ? '-' : '+';
+            my $sgnf = substr($bits, 12, 52);
+            my $exp  = substr($bits, 1, 11);
+            $bits = '';
+            while ( $sgnf =~ /(.{4})/g ) {
+                $bits .= unpack('H',pack('B4', $1));
+            }
+            $exp = ((unpack("C*",(pack("B8", (substr('00000'.$exp,0,8) )))) <<8 )
+                    + unpack("C*",(pack("B8", (substr('00000'.$exp,8,8) ))))) - 1023;
+            return Data::Float::hex_float( $sign . '0x1.' . $bits . 'p' . $exp ) + 0.0;
+        };
+        *unpack_float  = sub { Carp::croak("unpack_float is disable in less than Perl 5.10"); };
+        *unpack_int16  = sub {
+            my $v = unpack 'n', substr( $_[0], $_[1], 2 );
+            return $v ? $v - 0x10000 : 0;
+        };
+        *unpack_int32  = sub {
+            my $v = unpack 'N', substr( $_[0], $_[1], 4 );
+            return $v ? -(~$v + 1) : $v;
+        };
+        *unpack_int64  = sub { Carp::croak("unpack_int64 is disable in less than Perl 5.10"); };
+    }
+    else {
+        *pack_double   = sub { return pack 'Cd>', 0xcb, $_[0]; };
+        *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); };
+        *unpack_float  = sub { return unpack( 'f>', substr( $_[0], $_[1], 4 ) ); };
+        *unpack_int16  = sub { return unpack 'n!',  substr( $_[0], $_[1], 2 ); };
+        *unpack_int32  = sub { return unpack 'N!',  substr( $_[0], $_[1], 4 ); };
+        *unpack_int64  = sub { return unpack 'Q>',  substr( $_[0], $_[1], 8 ); };
+    }
+    # for 5.8 etc.
+    unless ( defined &utf8::is_utf8 ) {
+       require Encode;
+       *utf8::is_utf8 = *Encode::is_utf8;
+    }
+}
+
+
+#
+# PACK
+#
+
+{
+    my $max_depth;
+
+sub pack {
+    Carp::croak('Usage: Data::MessagePack->pack($dat [,$max_depth])') if @_ < 2;
+    $max_depth = defined $_[2] ? $_[2] : 512; # init
+    return _pack( $_[1] );
+}
+
+
+sub _pack {
+    my ( $value ) = @_;
+
+    return pack( 'C', 0xc0 ) if ( not defined $value );
+
+    my $b_obj = B::svref_2object( ref $value ? $value : \$value );
+
+    if ( $b_obj->isa('B::AV') ) {
+        my $num = @$value;
+        my $header = 
+              $num < 16          ? pack( 'C',  0x90 + $num )
+            : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xdc,  $num )
+            : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdd,  $num )
+            : die "" # don't arrivie here
+        ;
+        if ( --$max_depth <= 0 ) {
+            Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)");
+        }
+        return join( '', $header, map { _pack( $_ ) } @$value );
+    }
+
+    elsif ( $b_obj->isa('B::HV') ) {
+        my $num = keys %$value;
+        my $header = 
+              $num < 16          ? pack( 'C',  0x80 + $num )
+            : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xde,  $num )
+            : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdf,  $num )
+            : die "" # don't arrivie here
+        ;
+        if ( --$max_depth <= 0 ) {
+            Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)");
+        }
+        return join( '', $header, map { _pack( $_ ) } %$value );
+    }
+
+    elsif ( blessed( $value ) eq 'Data::MessagePack::Boolean' ) {
+        return  pack( 'C', $$value ? 0xc3 : 0xc2 );
+    }
+
+    my $flags = $b_obj->FLAGS;
+
+    if ( $flags & ( B::SVf_IOK | B::SVp_IOK ) ) {
+
+        if ($value >= 0) {
+            return    $value <= 127 ?    pack 'C',        $value
+                    : $value < 2 ** 8 ?  pack 'CC', 0xcc, $value
+                    : $value < 2 ** 16 ? pack 'Cn', 0xcd, $value
+                    : $value < 2 ** 32 ? pack 'CN', 0xce, $value
+                    : pack 'CQ>', 0xcf, $value;
+        }
+        else {
+            return    -$value <= 32 ?      pack 'C',        $value
+                    : -$value <= 2 **  7 ? pack 'Cc', 0xd0, $value
+                    : -$value <= 2 ** 15 ? pack 'Cn', 0xd1, $value
+                    : -$value <= 2 ** 31 ? pack 'CN', 0xd2, $value
+                    : pack 'Cq>', 0xd3, $value;
+        }
+
+    }
+
+    elsif ( $flags & B::SVf_POK ) { # raw / check needs before dboule
+
+        if ( $PreferInteger ) {
+            if ( $value =~ /^-?[0-9]+$/ ) { # ok?
+                my $value2 = 0 + $value;
+                if (  0 + $value != B::svref_2object( \$value2 )->int_value ) {
+                    local $PreferInteger;      # avoid for PV => NV
+                    return _pack( "$value" );
+                }
+                return _pack( $value + 0 );
+            }
+        }
+
+        utf8::encode( $value ) if utf8::is_utf8( $value );
+
+        my $num = length $value;
+        my $header = 
+              $num < 32          ? pack( 'C',  0xa0 + $num )
+            : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xda, $num )
+            : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdb, $num )
+            : die "" # don't arrivie here
+        ;
+
+        return $header . $value;
+
+    }
+
+    elsif ( $flags & ( B::SVf_NOK | B::SVp_NOK ) ) { # double only
+        return pack_double( $value );
+    }
+
+    else {
+        die "???";
+    }
+
+}
+
+} # PACK
+
+
+#
+# UNPACK
+#
+
+{
+    my $p; # position variables for speed.
+
+sub unpack {
+    $p = 0; # init
+    _unpack( $_[1] );
+}
+
+
+sub _unpack {
+    my ( $value ) = @_;
+    my $byte = unpack( 'C', substr( $value, $p++, 1 ) ); # get header
+
+    die "invalid data" unless defined $byte;
+
+    if ( ( $byte >= 0x90 and $byte <= 0x9f ) or $byte == 0xdc or $byte == 0xdd ) {
+        my $num;
+        if ( $byte == 0xdc ) { # array 16
+            $num = unpack 'n', substr( $value, $p, 2 );
+            $p += 2;
+        }
+        elsif ( $byte == 0xdd ) { # array 32
+            $num = unpack 'N', substr( $value, $p, 4 );
+            $p += 4;
+        }
+        else { # fix array
+            $num = $byte & ~0x90;
+        }
+        my @array;
+        push @array, _unpack( $value ) while $num-- > 0;
+        return \@array;
+    }
+
+    elsif ( ( $byte >= 0x80 and $byte <= 0x8f ) or $byte == 0xde or $byte == 0xdf ) {
+        my $num;
+        if ( $byte == 0xde ) { # map 16
+            $num = unpack 'n', substr( $value, $p, 2 );
+            $p += 2;
+        }
+        elsif ( $byte == 0xdf ) { # map 32
+            $num = unpack 'N', substr( $value, $p, 4 );
+            $p += 4;
+        }
+        else { # fix map
+            $num = $byte & ~0x80;
+        }
+        my %map;
+        for ( 0 .. $num - 1 ) {
+            my $key = _unpack( $value );
+            my $val = _unpack( $value );
+            $map{ $key } = $val;
+        }
+        return \%map;
+    }
+
+    elsif ( $byte >= 0x00 and $byte <= 0x7f ) { # positive fixnum
+        return $byte;
+    }
+    elsif ( $byte == 0xcc ) { # uint8
+        unpack( 'C', substr( $value, $p++, 1 ) );
+    }
+    elsif ( $byte == 0xcd ) { # uint16
+        $p += 2;
+        return unpack 'n', substr( $value, $p - 2, 2 );
+    }
+    elsif ( $byte == 0xce ) { # unit32
+        $p += 4;
+        return unpack 'N', substr( $value, $p - 4, 4 );
+    }
+    elsif ( $byte == 0xcf ) { # unit64
+        $p += 8;
+        return unpack 'Q>', substr( $value, $p - 8, 8 );
+    }
+    elsif ( $byte == 0xd3 ) { # int64
+        $p += 8;
+        return unpack_int64( $value, $p - 8 );
+        return unpack 'q>',  substr( $value, $p - 8, 8 );
+    }
+    elsif ( $byte == 0xd2 ) { # int32
+        $p += 4;
+        return unpack_int32( $value, $p - 4 );
+    }
+    elsif ( $byte == 0xd1 ) { # int16
+        $p += 2;
+        return unpack_int16( $value, $p - 2 );
+    }
+    elsif ( $byte == 0xd0 ) { # int8
+        return unpack 'c',  substr( $value, $p++, 1 ); # c / C
+    }
+    elsif ( $byte >= 0xe0 and $byte <= 0xff ) { # negative fixnum
+        return $byte - 256;
+    }
+
+    elsif ( ( $byte >= 0xa0 and $byte <= 0xbf ) or $byte == 0xda or $byte == 0xdb ) { # raw
+        my $num;
+        if ( $byte == 0xda ) {
+            $num = unpack 'n', substr( $value, $p, 2 );
+            $p += 2 + $num;
+        }
+        elsif ( $byte == 0xdb ) {
+            $num = unpack 'N', substr( $value, $p, 4 );
+            $p += 4 + $num;
+        }
+        else { # fix raw
+            $num = $byte & ~0xa0;
+            $p += $num;
+        }
+        return substr( $value, $p - $num, $num );
+    }
+
+    elsif ( $byte == 0xc0 ) { # nil
+        return undef;
+    }
+    elsif ( $byte == 0xc2 ) { # boolean
+        return false;
+    }
+    elsif ( $byte == 0xc3 ) { # boolean
+        return true;
+    }
+
+    elsif ( $byte == 0xcb ) { # double
+        $p += 8;
+        return unpack_double( $value, $p - 8 );
+    }
+
+    elsif ( $byte == 0xca ) { # float
+        $p += 4;
+        return unpack_float( $value, $p - 4 );
+    }
+
+    else {
+        die "???";
+    }
+
+}
+
+
+} # UNPACK
+
+
+#
+# Data::MessagePack::Unpacker
+#
+
+package Data::MessagePack::PP::Unpacker;
+
+use strict;
+
+sub new {
+    bless { stack => [] }, shift;
+}
+
+
+sub execute_limit {
+    execute( @_ );
+}
+
+
+{
+    my $p;
+    #my $r; # remained data.
+
+sub execute {
+    my ( $self, $data, $offset, $limit ) = @_;
+    #my $value = ( defined $self->{ remain } ? $self->{ remain } : '' ) . substr( $data, $offset, $limit );
+    my $value = substr( $data, $offset, $limit ? $limit : length $data );
+    my $len   = length $value;
+
+    $p = 0;
+    #$r = 0;
+
+    while ( $len > $p ) {
+        _count( $self, $value ) or last;
+
+        if ( @{ $self->{stack} } > 0 ) {
+            $self->{stack}->[-1];
+            pop @{ $self->{stack} } if --$self->{stack}->[-1] == 0;
+        }
+    }
+
+    if ( $len == $p ) {
+        $self->{ data } .= substr( $value, 0, $p );
+        $self->{ remain } = undef;
+    }
+    else { # I thought this feature is needed. but XS version can't do so
+        #$self->{ remain } = substr( $value, 0, $p + $r );
+    }
+
+    return $p;
+}
+
+
+sub _count {
+    my ( $self, $value ) = @_;
+    my $byte = unpack( 'C', substr( $value, $p++, 1 ) ); # get header
+
+    if ( ( $byte >= 0x90 and $byte <= 0x9f ) or $byte == 0xdc or $byte == 0xdd ) {
+        my $num;
+        if ( $byte == 0xdc ) { # array 16
+            # I thought this feature is needed. but XS version can't do so. So commented out.
+            #my $len = length substr( $value, $p, 2 );
+            #if ( $len != 2 ) {
+            #    $r = $len;
+            #    return 0;
+            #}
+            $num = unpack 'n', substr( $value, $p, 2 );
+            $p += 2;
+        }
+        elsif ( $byte == 0xdd ) { # array 32
+            $num = unpack 'N', substr( $value, $p, 4 );
+            $p += 4;
+        }
+        else { # fix array
+            $num = $byte & ~0x90;
+        }
+
+        push @{ $self->{stack} }, $num + 1;
+
+        return 1;
+    }
+
+    elsif ( ( $byte >= 0x80 and $byte <= 0x8f ) or $byte == 0xde or $byte == 0xdf ) {
+        my $num;
+        if ( $byte == 0xde ) { # map 16
+            $num = unpack 'n', substr( $value, $p, 2 );
+            $p += 2;
+        }
+        elsif ( $byte == 0xdf ) { # map 32
+            $num = unpack 'N', substr( $value, $p, 4 );
+            $p += 4;
+        }
+        else { # fix map
+            $num = $byte & ~0x80;
+        }
+
+        push @{ $self->{stack} }, $num * 2 + 1; # a pair
+
+        return 1;
+    }
+
+    elsif ( $byte == 0xc0 or $byte == 0xc2 or $byte == 0xc3 ) { # nil, false, true
+        return 1;
+    }
+
+    elsif ( $byte >= 0x00 and $byte <= 0x7f ) { # positive fixnum
+        return 1;
+    }
+
+    elsif ( $byte >= 0xcc and $byte <= 0xcf ) { # uint
+        $p += $byte == 0xcc ? 1
+            : $byte == 0xcd ? 2
+            : $byte == 0xce ? 4
+            : $byte == 0xcf ? 8
+            : die;
+        return 1;
+    }
+
+    elsif ( $byte >= 0xd0 and $byte <= 0xd3 ) { # int
+        $p += $byte == 0xd0 ? 1
+            : $byte == 0xd1 ? 2
+            : $byte == 0xd2 ? 4
+            : $byte == 0xd3 ? 8
+            : die;
+        return 1;
+    }
+
+    elsif ( $byte >= 0xe0 and $byte <= 0xff ) { # negative fixnum
+        return 1;
+    }
+
+    elsif ( $byte >= 0xca and $byte <= 0xcb ) { # float, double
+        $p += $byte == 0xca ? 4 : 8;
+        return 1;
+    }
+
+    elsif ( ( $byte >= 0xa0 and $byte <= 0xbf ) or $byte == 0xda or $byte == 0xdb ) {
+        my $num;
+        if ( $byte == 0xda ) {
+            $num = unpack 'n', substr( $value, $p, 2 );
+            $p += 2;
+        }
+        elsif ( $byte == 0xdb ) {
+            $num = unpack 'N', substr( $value, $p, 4 );
+            $p += 4;
+        }
+        else { # fix raw
+            $num = $byte & ~0xa0;
+        }
+        $p += $num;
+        return 1;
+    }
+
+    else {
+        die "???";
+    }
+
+    return 0;
+}
+
+} # execute
+
+
+sub data {
+    my $data = Data::MessagePack->unpack( $_[0]->{ data } );
+    $_[0]->reset;
+    return $data;
+}
+
+
+sub is_finished {
+    my ( $self ) = @_;
+    ( scalar( @{ $self->{stack} } ) or defined $self->{ remain } ) ? 0 : 1;
+}
+
+
+sub reset {
+    $_[0]->{ stack }  = [];
+    $_[0]->{ data }   = undef;
+    $_[0]->{ remain } = undef;
+}
+
+1;
+__END__
+
+=pod
+
+=head1 NAME
+
+Data::MessagePack::PP - the pure perl version of Data::MessagePack
+
+=head1 LIMITATION
+
+Currently this module works completely in Perl 5.10 or later.
+In Perl 5.8.x, it requires L and cannot unpack int64 and float (pack int64 too).
+
+
+=head1 SEE ALSO
+
+L,
+L,
+L
+
+=head1 AUTHOR
+
+makamaka
+
+=head1 COPYRIGHT AND LICENSE
+
+This library is free software; you can redistribute it and/or modify
+it under the same terms as Perl itself. 
+
+=cut

From a0705a6c67e852154e92bb16876ac9e950a8f044 Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Wed, 1 Sep 2010 11:59:01 +0900
Subject: [PATCH 0631/1648] added PP backend switch into Data::MessagePack

---
 perl/Changes                    |  6 ++++++
 perl/lib/Data/MessagePack.pm    | 15 +++++++++++++--
 perl/lib/Data/MessagePack/PP.pm | 33 +++++++++++++++------------------
 3 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/perl/Changes b/perl/Changes
index 189990a..a8a4298 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,9 @@
+
+0.1x
+
+    - added PP version.
+      (makamaka)
+
 0.15
 
     - better argument validation.
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 276353a..a3f8264 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -1,7 +1,6 @@
 package Data::MessagePack;
 use strict;
 use warnings;
-use XSLoader;
 use 5.008001;
 
 our $VERSION = '0.15';
@@ -12,7 +11,19 @@ our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" };
 sub true  () { $true  }
 sub false () { $false }
 
-XSLoader::load(__PACKAGE__, $VERSION);
+if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate
+    if ( $ENV{ PERL_DATA_MESSAGEPACK } !~ /\b pp \b/xms ) {
+        eval {
+            require XSLoader;
+            XSLoader::load(__PACKAGE__, $VERSION);
+        };
+        die $@ if $@ && $ENV{ PERL_DATA_MESSAGEPACK } =~ /\b xs \b/xms; # force XS
+    }
+    if ( !__PACKAGE__->can('pack') ) {
+        print "PP\n";
+        require 'Data/MessagePack/PP.pm';
+    }
+}
 
 1;
 __END__
diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index f4f1060..ecb97b4 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -2,21 +2,9 @@ package Data::MessagePack::PP;
 
 use 5.008000;
 use strict;
-use B ();
-use Scalar::Util qw( blessed );
 use Carp ();
 
-our $VERSION = '0.03';
-
-
-# copied from Data::MessagePack
-our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };
-our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" };
-
-sub true  () { $true  }
-sub false () { $false }
-
-our $PreferInteger;
+our $VERSION = '0.15';
 
 # See also
 # http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec
@@ -24,6 +12,13 @@ our $PreferInteger;
 # http://frox25.no-ip.org/~mtve/wiki/MessagePack.html : reference to using CORE::pack, CORE::unpack
 
 
+package
+    Data::MessagePack;
+
+use Scalar::Util qw( blessed );
+use strict;
+use B ();
+
 BEGIN {
     # for pack and unpack compatibility
     if ( $] < 5.010 ) {
@@ -160,11 +155,11 @@ sub _pack {
 
     elsif ( $flags & B::SVf_POK ) { # raw / check needs before dboule
 
-        if ( $PreferInteger ) {
+        if ( $Data::MessagePack::PreferInteger ) {
             if ( $value =~ /^-?[0-9]+$/ ) { # ok?
                 my $value2 = 0 + $value;
                 if (  0 + $value != B::svref_2object( \$value2 )->int_value ) {
-                    local $PreferInteger;      # avoid for PV => NV
+                    local $Data::MessagePack::PreferInteger; # avoid for PV => NV
                     return _pack( "$value" );
                 }
                 return _pack( $value + 0 );
@@ -346,7 +341,8 @@ sub _unpack {
 # Data::MessagePack::Unpacker
 #
 
-package Data::MessagePack::PP::Unpacker;
+package
+    Data::MessagePack::Unpacker;
 
 use strict;
 
@@ -530,7 +526,7 @@ __END__
 
 =head1 NAME
 
-Data::MessagePack::PP - the pure perl version of Data::MessagePack
+Data::MessagePack::PP - Pure Perl version of Data::MessagePack
 
 =head1 LIMITATION
 
@@ -540,9 +536,10 @@ In Perl 5.8.x, it requires L and cannot unpack int64 and float (pac
 
 =head1 SEE ALSO
 
+L,
 L,
+L,
 L,
-L
 
 =head1 AUTHOR
 

From af83a624743735e1f4404bcd3942e98eee36ce2a Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Wed, 1 Sep 2010 16:04:25 +0900
Subject: [PATCH 0632/1648] modified some codes for test warnings

---
 perl/lib/Data/MessagePack.pm    |  5 ++-
 perl/lib/Data/MessagePack/PP.pm | 74 +++++++++++++++++----------------
 2 files changed, 41 insertions(+), 38 deletions(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index a3f8264..f8d1625 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -12,12 +12,13 @@ sub true  () { $true  }
 sub false () { $false }
 
 if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate
-    if ( $ENV{ PERL_DATA_MESSAGEPACK } !~ /\b pp \b/xms ) {
+    my $backend = $ENV{ PERL_DATA_MESSAGEPACK } || '';
+    if ( $backend !~ /\b pp \b/xms ) {
         eval {
             require XSLoader;
             XSLoader::load(__PACKAGE__, $VERSION);
         };
-        die $@ if $@ && $ENV{ PERL_DATA_MESSAGEPACK } =~ /\b xs \b/xms; # force XS
+        die $@ if $@ && $backend =~ /\b xs \b/xms; # force XS
     }
     if ( !__PACKAGE__->can('pack') ) {
         print "PP\n";
diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index ecb97b4..1e05bab 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -84,6 +84,8 @@ BEGIN {
 #
 
 {
+    no warnings 'recursion';
+
     my $max_depth;
 
 sub pack {
@@ -96,16 +98,16 @@ sub pack {
 sub _pack {
     my ( $value ) = @_;
 
-    return pack( 'C', 0xc0 ) if ( not defined $value );
+    return CORE::pack( 'C', 0xc0 ) if ( not defined $value );
 
     my $b_obj = B::svref_2object( ref $value ? $value : \$value );
 
     if ( $b_obj->isa('B::AV') ) {
         my $num = @$value;
         my $header = 
-              $num < 16          ? pack( 'C',  0x90 + $num )
-            : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xdc,  $num )
-            : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdd,  $num )
+              $num < 16          ? CORE::pack( 'C',  0x90 + $num )
+            : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xdc,  $num )
+            : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdd,  $num )
             : die "" # don't arrivie here
         ;
         if ( --$max_depth <= 0 ) {
@@ -117,9 +119,9 @@ sub _pack {
     elsif ( $b_obj->isa('B::HV') ) {
         my $num = keys %$value;
         my $header = 
-              $num < 16          ? pack( 'C',  0x80 + $num )
-            : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xde,  $num )
-            : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdf,  $num )
+              $num < 16          ? CORE::pack( 'C',  0x80 + $num )
+            : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xde,  $num )
+            : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdf,  $num )
             : die "" # don't arrivie here
         ;
         if ( --$max_depth <= 0 ) {
@@ -128,8 +130,8 @@ sub _pack {
         return join( '', $header, map { _pack( $_ ) } %$value );
     }
 
-    elsif ( blessed( $value ) eq 'Data::MessagePack::Boolean' ) {
-        return  pack( 'C', $$value ? 0xc3 : 0xc2 );
+    elsif ( blessed( $value ) and blessed( $value ) eq 'Data::MessagePack::Boolean' ) {
+        return  CORE::pack( 'C', $$value ? 0xc3 : 0xc2 );
     }
 
     my $flags = $b_obj->FLAGS;
@@ -137,18 +139,18 @@ sub _pack {
     if ( $flags & ( B::SVf_IOK | B::SVp_IOK ) ) {
 
         if ($value >= 0) {
-            return    $value <= 127 ?    pack 'C',        $value
-                    : $value < 2 ** 8 ?  pack 'CC', 0xcc, $value
-                    : $value < 2 ** 16 ? pack 'Cn', 0xcd, $value
-                    : $value < 2 ** 32 ? pack 'CN', 0xce, $value
-                    : pack 'CQ>', 0xcf, $value;
+            return    $value <= 127 ?    CORE::pack 'C',        $value
+                    : $value < 2 ** 8 ?  CORE::pack 'CC', 0xcc, $value
+                    : $value < 2 ** 16 ? CORE::pack 'Cn', 0xcd, $value
+                    : $value < 2 ** 32 ? CORE::pack 'CN', 0xce, $value
+                    : CORE::pack 'CQ>', 0xcf, $value;
         }
         else {
-            return    -$value <= 32 ?      pack 'C',        $value
-                    : -$value <= 2 **  7 ? pack 'Cc', 0xd0, $value
-                    : -$value <= 2 ** 15 ? pack 'Cn', 0xd1, $value
-                    : -$value <= 2 ** 31 ? pack 'CN', 0xd2, $value
-                    : pack 'Cq>', 0xd3, $value;
+            return    -$value <= 32 ?      CORE::pack 'C', ($value & 255)
+                    : -$value <= 2 **  7 ? CORE::pack 'Cc', 0xd0, $value
+                    : -$value <= 2 ** 15 ? CORE::pack 'Cn', 0xd1, $value
+                    : -$value <= 2 ** 31 ? CORE::pack 'CN', 0xd2, $value
+                    : CORE::pack 'Cq>', 0xd3, $value;
         }
 
     }
@@ -170,9 +172,9 @@ sub _pack {
 
         my $num = length $value;
         my $header = 
-              $num < 32          ? pack( 'C',  0xa0 + $num )
-            : $num < 2 ** 16 - 1 ? pack( 'Cn', 0xda, $num )
-            : $num < 2 ** 32 - 1 ? pack( 'CN', 0xdb, $num )
+              $num < 32          ? CORE::pack( 'C',  0xa0 + $num )
+            : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xda, $num )
+            : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdb, $num )
             : die "" # don't arrivie here
         ;
 
@@ -198,6 +200,7 @@ sub _pack {
 #
 
 {
+
     my $p; # position variables for speed.
 
 sub unpack {
@@ -208,18 +211,18 @@ sub unpack {
 
 sub _unpack {
     my ( $value ) = @_;
-    my $byte = unpack( 'C', substr( $value, $p++, 1 ) ); # get header
+    my $byte = CORE::unpack( 'C', substr( $value, $p++, 1 ) ); # get header
 
     die "invalid data" unless defined $byte;
 
     if ( ( $byte >= 0x90 and $byte <= 0x9f ) or $byte == 0xdc or $byte == 0xdd ) {
         my $num;
         if ( $byte == 0xdc ) { # array 16
-            $num = unpack 'n', substr( $value, $p, 2 );
+            $num = CORE::unpack 'n', substr( $value, $p, 2 );
             $p += 2;
         }
         elsif ( $byte == 0xdd ) { # array 32
-            $num = unpack 'N', substr( $value, $p, 4 );
+            $num = CORE::unpack 'N', substr( $value, $p, 4 );
             $p += 4;
         }
         else { # fix array
@@ -233,11 +236,11 @@ sub _unpack {
     elsif ( ( $byte >= 0x80 and $byte <= 0x8f ) or $byte == 0xde or $byte == 0xdf ) {
         my $num;
         if ( $byte == 0xde ) { # map 16
-            $num = unpack 'n', substr( $value, $p, 2 );
+            $num = CORE::unpack 'n', substr( $value, $p, 2 );
             $p += 2;
         }
         elsif ( $byte == 0xdf ) { # map 32
-            $num = unpack 'N', substr( $value, $p, 4 );
+            $num = CORE::unpack 'N', substr( $value, $p, 4 );
             $p += 4;
         }
         else { # fix map
@@ -245,6 +248,7 @@ sub _unpack {
         }
         my %map;
         for ( 0 .. $num - 1 ) {
+            no warnings; # for undef key case
             my $key = _unpack( $value );
             my $val = _unpack( $value );
             $map{ $key } = $val;
@@ -256,24 +260,23 @@ sub _unpack {
         return $byte;
     }
     elsif ( $byte == 0xcc ) { # uint8
-        unpack( 'C', substr( $value, $p++, 1 ) );
+        CORE::unpack( 'C', substr( $value, $p++, 1 ) );
     }
     elsif ( $byte == 0xcd ) { # uint16
         $p += 2;
-        return unpack 'n', substr( $value, $p - 2, 2 );
+        return CORE::unpack 'n', substr( $value, $p - 2, 2 );
     }
     elsif ( $byte == 0xce ) { # unit32
         $p += 4;
-        return unpack 'N', substr( $value, $p - 4, 4 );
+        return CORE::unpack 'N', substr( $value, $p - 4, 4 );
     }
     elsif ( $byte == 0xcf ) { # unit64
         $p += 8;
-        return unpack 'Q>', substr( $value, $p - 8, 8 );
+        return CORE::unpack 'Q>', substr( $value, $p - 8, 8 );
     }
     elsif ( $byte == 0xd3 ) { # int64
         $p += 8;
         return unpack_int64( $value, $p - 8 );
-        return unpack 'q>',  substr( $value, $p - 8, 8 );
     }
     elsif ( $byte == 0xd2 ) { # int32
         $p += 4;
@@ -284,7 +287,7 @@ sub _unpack {
         return unpack_int16( $value, $p - 2 );
     }
     elsif ( $byte == 0xd0 ) { # int8
-        return unpack 'c',  substr( $value, $p++, 1 ); # c / C
+        return CORE::unpack 'c',  substr( $value, $p++, 1 ); # c / C
     }
     elsif ( $byte >= 0xe0 and $byte <= 0xff ) { # negative fixnum
         return $byte - 256;
@@ -293,11 +296,11 @@ sub _unpack {
     elsif ( ( $byte >= 0xa0 and $byte <= 0xbf ) or $byte == 0xda or $byte == 0xdb ) { # raw
         my $num;
         if ( $byte == 0xda ) {
-            $num = unpack 'n', substr( $value, $p, 2 );
+            $num = CORE::unpack 'n', substr( $value, $p, 2 );
             $p += 2 + $num;
         }
         elsif ( $byte == 0xdb ) {
-            $num = unpack 'N', substr( $value, $p, 4 );
+            $num = CORE::unpack 'N', substr( $value, $p, 4 );
             $p += 4 + $num;
         }
         else { # fix raw
@@ -373,7 +376,6 @@ sub execute {
         _count( $self, $value ) or last;
 
         if ( @{ $self->{stack} } > 0 ) {
-            $self->{stack}->[-1];
             pop @{ $self->{stack} } if --$self->{stack}->[-1] == 0;
         }
     }

From 4a15d8b6d2b69bdc1de0b0a7f643b02e00100e66 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 01:29:57 +0900
Subject: [PATCH 0633/1648] python: Support Python3.

---
 python/msgpack/__init__.py  |  2 +-
 python/msgpack/_msgpack.pyx | 37 +++++++++++++++++++------------------
 python/msgpack/unpack.h     |  2 +-
 3 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/python/msgpack/__init__.py b/python/msgpack/__init__.py
index 797b29c..9593714 100644
--- a/python/msgpack/__init__.py
+++ b/python/msgpack/__init__.py
@@ -1,3 +1,3 @@
 # coding: utf-8
-from _msgpack import *
+from msgpack._msgpack import *
 
diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx
index 61ae36b..6a0b1a5 100644
--- a/python/msgpack/_msgpack.pyx
+++ b/python/msgpack/_msgpack.pyx
@@ -1,26 +1,24 @@
 # coding: utf-8
 
-import cStringIO
-
 cdef extern from "Python.h":
     ctypedef char* const_char_ptr "const char*"
     ctypedef struct PyObject
 
-    cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
+    cdef object PyBytes_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
     cdef PyObject* Py_True
     cdef PyObject* Py_False
 
-    cdef char* PyString_AsString(object o)
     cdef long long PyLong_AsLongLong(object o)
     cdef unsigned long long PyLong_AsUnsignedLongLong(object o)
 
-    cdef int PyMapping_Check(object o)
-    cdef int PySequence_Check(object o)
-    cdef int PyLong_Check(object o)
-    cdef int PyInt_Check(object o)
-    cdef int PyFloat_Check(object o)
-    cdef int PyString_Check(object o)
-    cdef int PyUnicode_Check(object o)
+    cdef bint PyBool_Check(object o)
+    cdef bint PyMapping_Check(object o)
+    cdef bint PySequence_Check(object o)
+    cdef bint PyLong_Check(object o)
+    cdef bint PyInt_Check(object o)
+    cdef bint PyFloat_Check(object o)
+    cdef bint PyBytes_Check(object o)
+    cdef bint PyUnicode_Check(object o)
 
 cdef extern from "stdlib.h":
     void* malloc(size_t)
@@ -81,10 +79,12 @@ cdef class Packer(object):
 
         if o is None:
             ret = msgpack_pack_nil(&self.pk)
-        elif o == Py_True:
-            ret = msgpack_pack_true(&self.pk)
-        elif o == Py_False:
-            ret = msgpack_pack_false(&self.pk)
+            #elif PyBool_Check(o):
+        elif isinstance(o, bool):
+            if o:
+                ret = msgpack_pack_true(&self.pk)
+            else:
+                ret = msgpack_pack_false(&self.pk)
         elif PyLong_Check(o):
             if o > 0:
                 ullval = PyLong_AsUnsignedLongLong(o)
@@ -98,7 +98,7 @@ cdef class Packer(object):
         elif PyFloat_Check(o):
             fval = o
             ret = msgpack_pack_double(&self.pk, fval)
-        elif PyString_Check(o):
+        elif PyBytes_Check(o):
             rawval = o
             ret = msgpack_pack_raw(&self.pk, len(o))
             if ret == 0:
@@ -133,7 +133,7 @@ cdef class Packer(object):
         ret = self.__pack(obj)
         if ret:
             raise TypeError
-        buf = PyString_FromStringAndSize(self.pk.buf, self.pk.length)
+        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
         self.pk.length = 0
         return buf
 
@@ -262,10 +262,11 @@ cdef class Unpacker(object):
         cdef char* buf = self.buf
         cdef Py_ssize_t tail = self.buf_tail
         cdef Py_ssize_t l
+        cdef bytes b
 
         for b in self.waiting_bytes:
             l = len(b)
-            memcpy(buf + tail, PyString_AsString(b), l)
+            memcpy(buf + tail, (b), l)
             tail += l
         self.buf_tail = tail
         del self.waiting_bytes[:]
diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h
index 61a3786..9eb8ce7 100644
--- a/python/msgpack/unpack.h
+++ b/python/msgpack/unpack.h
@@ -175,7 +175,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje
 static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
 {
     PyObject *py;
-    py = PyString_FromStringAndSize(p, l);
+    py = PyBytes_FromStringAndSize(p, l);
     if (!py)
         return -1;
     *o = py;

From 0dc8938dbfff67760834bd97150fee5fdb8e875c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 01:29:57 +0900
Subject: [PATCH 0634/1648] python: Support Python3.

---
 msgpack/__init__.py  |  2 +-
 msgpack/_msgpack.pyx | 37 +++++++++++++++++++------------------
 msgpack/unpack.h     |  2 +-
 3 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 797b29c..9593714 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,3 +1,3 @@
 # coding: utf-8
-from _msgpack import *
+from msgpack._msgpack import *
 
diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 61ae36b..6a0b1a5 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -1,26 +1,24 @@
 # coding: utf-8
 
-import cStringIO
-
 cdef extern from "Python.h":
     ctypedef char* const_char_ptr "const char*"
     ctypedef struct PyObject
 
-    cdef object PyString_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
+    cdef object PyBytes_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
     cdef PyObject* Py_True
     cdef PyObject* Py_False
 
-    cdef char* PyString_AsString(object o)
     cdef long long PyLong_AsLongLong(object o)
     cdef unsigned long long PyLong_AsUnsignedLongLong(object o)
 
-    cdef int PyMapping_Check(object o)
-    cdef int PySequence_Check(object o)
-    cdef int PyLong_Check(object o)
-    cdef int PyInt_Check(object o)
-    cdef int PyFloat_Check(object o)
-    cdef int PyString_Check(object o)
-    cdef int PyUnicode_Check(object o)
+    cdef bint PyBool_Check(object o)
+    cdef bint PyMapping_Check(object o)
+    cdef bint PySequence_Check(object o)
+    cdef bint PyLong_Check(object o)
+    cdef bint PyInt_Check(object o)
+    cdef bint PyFloat_Check(object o)
+    cdef bint PyBytes_Check(object o)
+    cdef bint PyUnicode_Check(object o)
 
 cdef extern from "stdlib.h":
     void* malloc(size_t)
@@ -81,10 +79,12 @@ cdef class Packer(object):
 
         if o is None:
             ret = msgpack_pack_nil(&self.pk)
-        elif o == Py_True:
-            ret = msgpack_pack_true(&self.pk)
-        elif o == Py_False:
-            ret = msgpack_pack_false(&self.pk)
+            #elif PyBool_Check(o):
+        elif isinstance(o, bool):
+            if o:
+                ret = msgpack_pack_true(&self.pk)
+            else:
+                ret = msgpack_pack_false(&self.pk)
         elif PyLong_Check(o):
             if o > 0:
                 ullval = PyLong_AsUnsignedLongLong(o)
@@ -98,7 +98,7 @@ cdef class Packer(object):
         elif PyFloat_Check(o):
             fval = o
             ret = msgpack_pack_double(&self.pk, fval)
-        elif PyString_Check(o):
+        elif PyBytes_Check(o):
             rawval = o
             ret = msgpack_pack_raw(&self.pk, len(o))
             if ret == 0:
@@ -133,7 +133,7 @@ cdef class Packer(object):
         ret = self.__pack(obj)
         if ret:
             raise TypeError
-        buf = PyString_FromStringAndSize(self.pk.buf, self.pk.length)
+        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
         self.pk.length = 0
         return buf
 
@@ -262,10 +262,11 @@ cdef class Unpacker(object):
         cdef char* buf = self.buf
         cdef Py_ssize_t tail = self.buf_tail
         cdef Py_ssize_t l
+        cdef bytes b
 
         for b in self.waiting_bytes:
             l = len(b)
-            memcpy(buf + tail, PyString_AsString(b), l)
+            memcpy(buf + tail, (b), l)
             tail += l
         self.buf_tail = tail
         del self.waiting_bytes[:]
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 61a3786..9eb8ce7 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -175,7 +175,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje
 static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
 {
     PyObject *py;
-    py = PyString_FromStringAndSize(p, l);
+    py = PyBytes_FromStringAndSize(p, l);
     if (!py)
         return -1;
     *o = py;

From 2146f5f623ce4ee2afda660c77bce7852d4a1530 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 02:02:47 +0900
Subject: [PATCH 0635/1648] python: Fix Unpacker.feed doesn't accept bytes on
 Python3.

---
 python/msgpack/_msgpack.pyx | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx
index 6a0b1a5..85d717e 100644
--- a/python/msgpack/_msgpack.pyx
+++ b/python/msgpack/_msgpack.pyx
@@ -253,9 +253,7 @@ cdef class Unpacker(object):
         template_init(&self.ctx)
         self.ctx.user.use_list = use_list
 
-    def feed(self, next_bytes):
-        if not isinstance(next_bytes, str):
-           raise ValueError, "Argument must be bytes object"
+    def feed(self, bytes next_bytes):
         self.waiting_bytes.append(next_bytes)
 
     cdef append_buffer(self):

From 1e8eeb8ebed50e8d39c69be653df06a10730631e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 02:02:47 +0900
Subject: [PATCH 0636/1648] python: Fix Unpacker.feed doesn't accept bytes on
 Python3.

---
 msgpack/_msgpack.pyx | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 6a0b1a5..85d717e 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -253,9 +253,7 @@ cdef class Unpacker(object):
         template_init(&self.ctx)
         self.ctx.user.use_list = use_list
 
-    def feed(self, next_bytes):
-        if not isinstance(next_bytes, str):
-           raise ValueError, "Argument must be bytes object"
+    def feed(self, bytes next_bytes):
         self.waiting_bytes.append(next_bytes)
 
     cdef append_buffer(self):

From 8d0d2bd3fca404906aef8982a99a81b2acdc50a0 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 02:16:28 +0900
Subject: [PATCH 0637/1648] python: Add test for python3 and fix found
 problems.

---
 python/msgpack/_msgpack.pyx    |  20 ++++---
 python/test3/test_case.py      | 102 +++++++++++++++++++++++++++++++++
 python/test3/test_except.py    |  14 +++++
 python/test3/test_format.py    |  75 ++++++++++++++++++++++++
 python/test3/test_pack.py      |  28 +++++++++
 python/test3/test_sequnpack.py |  36 ++++++++++++
 6 files changed, 266 insertions(+), 9 deletions(-)
 create mode 100644 python/test3/test_case.py
 create mode 100644 python/test3/test_except.py
 create mode 100644 python/test3/test_format.py
 create mode 100644 python/test3/test_pack.py
 create mode 100644 python/test3/test_sequnpack.py

diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx
index 85d717e..c887127 100644
--- a/python/msgpack/_msgpack.pyx
+++ b/python/msgpack/_msgpack.pyx
@@ -12,7 +12,7 @@ cdef extern from "Python.h":
     cdef unsigned long long PyLong_AsUnsignedLongLong(object o)
 
     cdef bint PyBool_Check(object o)
-    cdef bint PyMapping_Check(object o)
+    cdef bint PyDict_Check(object o)
     cdef bint PySequence_Check(object o)
     cdef bint PyLong_Check(object o)
     cdef bint PyInt_Check(object o)
@@ -69,13 +69,14 @@ cdef class Packer(object):
     def __dealloc__(self):
         free(self.pk.buf);
 
-    cdef int __pack(self, object o) except -1:
+    cdef int _pack(self, object o) except -1:
         cdef long long llval
         cdef unsigned long long ullval
         cdef long longval
         cdef double fval
         cdef char* rawval 
         cdef int ret
+        cdef dict d
 
         if o is None:
             ret = msgpack_pack_nil(&self.pk)
@@ -109,19 +110,20 @@ cdef class Packer(object):
             ret = msgpack_pack_raw(&self.pk, len(o))
             if ret == 0:
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
-        elif PyMapping_Check(o):
-            ret = msgpack_pack_map(&self.pk, len(o))
+        elif PyDict_Check(o):
+            d = o
+            ret = msgpack_pack_map(&self.pk, len(d))
             if ret == 0:
-                for k,v in o.iteritems():
-                    ret = self.__pack(k)
+                for k,v in d.items():
+                    ret = self._pack(k)
                     if ret != 0: break
-                    ret = self.__pack(v)
+                    ret = self._pack(v)
                     if ret != 0: break
         elif PySequence_Check(o):
             ret = msgpack_pack_array(&self.pk, len(o))
             if ret == 0:
                 for v in o:
-                    ret = self.__pack(v)
+                    ret = self._pack(v)
                     if ret != 0: break
         else:
             # TODO: Serialize with defalt() like simplejson.
@@ -130,7 +132,7 @@ cdef class Packer(object):
 
     def pack(self, object obj):
         cdef int ret
-        ret = self.__pack(obj)
+        ret = self._pack(obj)
         if ret:
             raise TypeError
         buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
diff --git a/python/test3/test_case.py b/python/test3/test_case.py
new file mode 100644
index 0000000..53dfcaf
--- /dev/null
+++ b/python/test3/test_case.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import packs, unpacks
+
+
+def check(length, obj):
+    v = packs(obj)
+    assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v)))
+    assert_equal(unpacks(v), obj)
+
+def test_1():
+    for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
+              -((1<<5)-1), -(1<<5)]:
+        check(1, o)
+
+def test_2():
+    for o in [1 << 7, (1 << 8) - 1,
+              -((1<<5)+1), -(1<<7)
+             ]:
+        check(2, o)
+
+def test_3():
+    for o in [1 << 8, (1 << 16) - 1,
+              -((1<<7)+1), -(1<<15)]:
+        check(3, o)
+
+def test_5():
+    for o in [1 << 16, (1 << 32) - 1,
+              -((1<<15)+1), -(1<<31)]:
+        check(5, o)
+
+def test_9():
+    for o in [1 << 32, (1 << 64) - 1,
+              -((1<<31)+1), -(1<<63),
+              1.0, 0.1, -0.1, -1.0]:
+        check(9, o)
+
+
+def check_raw(overhead, num):
+    check(num + overhead, b" " * num)
+
+def test_fixraw():
+    check_raw(1, 0)
+    check_raw(1, (1<<5) - 1)
+
+def test_raw16():
+    check_raw(3, 1<<5)
+    check_raw(3, (1<<16) - 1)
+
+def test_raw32():
+    check_raw(5, 1<<16)
+
+
+def check_array(overhead, num):
+    check(num + overhead, (None,) * num)
+
+def test_fixarray():
+    check_array(1, 0)
+    check_array(1, (1 << 4) - 1)
+
+def test_array16():
+    check_array(3, 1 << 4)
+    check_array(3, (1<<16)-1)
+
+def test_array32():
+    check_array(5, (1<<16))
+
+
+def match(obj, buf):
+    assert_equal(packs(obj), buf)
+    assert_equal(unpacks(buf), obj)
+
+def test_match():
+    cases = [
+        (None, b'\xc0'),
+        (False, b'\xc2'),
+        (True, b'\xc3'),
+        (0, b'\x00'),
+        (127, b'\x7f'),
+        (128, b'\xcc\x80'),
+        (256, b'\xcd\x01\x00'),
+        (-1, b'\xff'),
+        (-33, b'\xd0\xdf'),
+        (-129, b'\xd1\xff\x7f'),
+        ({1:1}, b'\x81\x01\x01'),
+        (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"),
+        ((), b'\x90'),
+        (tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"),
+        (tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"),
+        ({}, b'\x80'),
+        (dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'),
+        (dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'),
+        ]
+
+    for v, p in cases:
+        match(v, p)
+
+if __name__ == '__main__':
+    main()
diff --git a/python/test3/test_except.py b/python/test3/test_except.py
new file mode 100644
index 0000000..574728f
--- /dev/null
+++ b/python/test3/test_except.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose.tools import *
+from msgpack import packs, unpacks
+
+import datetime
+
+def test_raise_on_find_unsupported_value():
+    assert_raises(TypeError, packs, datetime.datetime.now())
+
+if __name__ == '__main__':
+    from nose import main
+    main()
diff --git a/python/test3/test_format.py b/python/test3/test_format.py
new file mode 100644
index 0000000..022e680
--- /dev/null
+++ b/python/test3/test_format.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import unpacks
+
+def check(src, should):
+    assert_equal(unpacks(src), should)
+
+def testSimpleValue():
+    check(b"\x93\xc0\xc2\xc3", 
+            (None, False, True,))
+
+def testFixnum():
+    check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff",
+          ((0,64,127,), (-32,-16,-1,),)
+          )
+
+def testFixArray():
+    check(b"\x92\x90\x91\x91\xc0",
+          ((),((None,),),),
+          )
+
+def testFixRaw():
+    check(b"\x94\xa0\xa1a\xa2bc\xa3def",
+          (b"", b"a", b"bc", b"def",),
+          )
+
+def testFixMap():
+    check(
+          b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
+          {False: {None: None}, True:{None:{}}},
+          )
+
+def testUnsignedInt():
+    check(
+          b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
+          b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
+          b"\xce\xff\xff\xff\xff",
+          (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,),
+          )
+
+def testSignedInt():
+    check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
+          b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
+          b"\xd2\xff\xff\xff\xff",
+          (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,))
+
+def testRaw():
+    check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
+        b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
+        (b"", b"a", b"ab", b"", b"a", b"ab"))
+
+def testArray():
+    check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
+        b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02"
+        b"\xc2\xc3",
+        ((), (None,), (False,True), (), (None,), (False,True))
+        )
+
+def testMap():
+    check(
+        b"\x96"
+            b"\xde\x00\x00"
+            b"\xde\x00\x01\xc0\xc2"
+            b"\xde\x00\x02\xc0\xc2\xc3\xc2"
+            b"\xdf\x00\x00\x00\x00"
+            b"\xdf\x00\x00\x00\x01\xc0\xc2"
+            b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
+        ({}, {None: False}, {True: False, None: False}, {},
+            {None: False}, {True: False, None: False}))
+
+if __name__ == '__main__':
+    main()
diff --git a/python/test3/test_pack.py b/python/test3/test_pack.py
new file mode 100644
index 0000000..c861704
--- /dev/null
+++ b/python/test3/test_pack.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+
+from msgpack import packs, unpacks
+
+def check(data):
+    re = unpacks(packs(data))
+    assert_equal(re, data)
+
+def testPack():
+    test_data = [
+            0, 1, 127, 128, 255, 256, 65535, 65536,
+            -1, -32, -33, -128, -129, -32768, -32769,
+            1.0,
+        b"", b"a", b"a"*31, b"a"*32,
+        None, True, False,
+        (), ((),), ((), None,), 
+        {None: 0}, 
+        (1<<23), 
+        ]
+    for td in test_data:
+        check(td)
+
+if __name__ == '__main__':
+    main()
diff --git a/python/test3/test_sequnpack.py b/python/test3/test_sequnpack.py
new file mode 100644
index 0000000..5fd377c
--- /dev/null
+++ b/python/test3/test_sequnpack.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+
+
+from msgpack import Unpacker
+
+def test_foobar():
+    unpacker = Unpacker(read_size=3)
+    unpacker.feed(b'foobar')
+    assert unpacker.unpack() == ord(b'f')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'b')
+    assert unpacker.unpack() == ord(b'a')
+    assert unpacker.unpack() == ord(b'r')
+    try:
+        o = unpacker.unpack()
+        print(("Oops!", o))
+        assert 0
+    except StopIteration:
+        assert 1
+    else:
+        assert 0
+    unpacker.feed(b'foo')
+    unpacker.feed(b'bar')
+
+    k = 0
+    for o, e in zip(unpacker, b'foobarbaz'):
+        assert o == e
+        k += 1
+    assert k == len(b'foobar')
+
+if __name__ == '__main__':
+    test_foobar()
+

From 039542ebcb8ca923c4414a414ecd62df43ff3f24 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 02:16:28 +0900
Subject: [PATCH 0638/1648] python: Add test for python3 and fix found
 problems.

---
 msgpack/_msgpack.pyx    |  20 ++++----
 test3/test_case.py      | 102 ++++++++++++++++++++++++++++++++++++++++
 test3/test_except.py    |  14 ++++++
 test3/test_format.py    |  75 +++++++++++++++++++++++++++++
 test3/test_pack.py      |  28 +++++++++++
 test3/test_sequnpack.py |  36 ++++++++++++++
 6 files changed, 266 insertions(+), 9 deletions(-)
 create mode 100644 test3/test_case.py
 create mode 100644 test3/test_except.py
 create mode 100644 test3/test_format.py
 create mode 100644 test3/test_pack.py
 create mode 100644 test3/test_sequnpack.py

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 85d717e..c887127 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -12,7 +12,7 @@ cdef extern from "Python.h":
     cdef unsigned long long PyLong_AsUnsignedLongLong(object o)
 
     cdef bint PyBool_Check(object o)
-    cdef bint PyMapping_Check(object o)
+    cdef bint PyDict_Check(object o)
     cdef bint PySequence_Check(object o)
     cdef bint PyLong_Check(object o)
     cdef bint PyInt_Check(object o)
@@ -69,13 +69,14 @@ cdef class Packer(object):
     def __dealloc__(self):
         free(self.pk.buf);
 
-    cdef int __pack(self, object o) except -1:
+    cdef int _pack(self, object o) except -1:
         cdef long long llval
         cdef unsigned long long ullval
         cdef long longval
         cdef double fval
         cdef char* rawval 
         cdef int ret
+        cdef dict d
 
         if o is None:
             ret = msgpack_pack_nil(&self.pk)
@@ -109,19 +110,20 @@ cdef class Packer(object):
             ret = msgpack_pack_raw(&self.pk, len(o))
             if ret == 0:
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
-        elif PyMapping_Check(o):
-            ret = msgpack_pack_map(&self.pk, len(o))
+        elif PyDict_Check(o):
+            d = o
+            ret = msgpack_pack_map(&self.pk, len(d))
             if ret == 0:
-                for k,v in o.iteritems():
-                    ret = self.__pack(k)
+                for k,v in d.items():
+                    ret = self._pack(k)
                     if ret != 0: break
-                    ret = self.__pack(v)
+                    ret = self._pack(v)
                     if ret != 0: break
         elif PySequence_Check(o):
             ret = msgpack_pack_array(&self.pk, len(o))
             if ret == 0:
                 for v in o:
-                    ret = self.__pack(v)
+                    ret = self._pack(v)
                     if ret != 0: break
         else:
             # TODO: Serialize with defalt() like simplejson.
@@ -130,7 +132,7 @@ cdef class Packer(object):
 
     def pack(self, object obj):
         cdef int ret
-        ret = self.__pack(obj)
+        ret = self._pack(obj)
         if ret:
             raise TypeError
         buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
diff --git a/test3/test_case.py b/test3/test_case.py
new file mode 100644
index 0000000..53dfcaf
--- /dev/null
+++ b/test3/test_case.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import packs, unpacks
+
+
+def check(length, obj):
+    v = packs(obj)
+    assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v)))
+    assert_equal(unpacks(v), obj)
+
+def test_1():
+    for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
+              -((1<<5)-1), -(1<<5)]:
+        check(1, o)
+
+def test_2():
+    for o in [1 << 7, (1 << 8) - 1,
+              -((1<<5)+1), -(1<<7)
+             ]:
+        check(2, o)
+
+def test_3():
+    for o in [1 << 8, (1 << 16) - 1,
+              -((1<<7)+1), -(1<<15)]:
+        check(3, o)
+
+def test_5():
+    for o in [1 << 16, (1 << 32) - 1,
+              -((1<<15)+1), -(1<<31)]:
+        check(5, o)
+
+def test_9():
+    for o in [1 << 32, (1 << 64) - 1,
+              -((1<<31)+1), -(1<<63),
+              1.0, 0.1, -0.1, -1.0]:
+        check(9, o)
+
+
+def check_raw(overhead, num):
+    check(num + overhead, b" " * num)
+
+def test_fixraw():
+    check_raw(1, 0)
+    check_raw(1, (1<<5) - 1)
+
+def test_raw16():
+    check_raw(3, 1<<5)
+    check_raw(3, (1<<16) - 1)
+
+def test_raw32():
+    check_raw(5, 1<<16)
+
+
+def check_array(overhead, num):
+    check(num + overhead, (None,) * num)
+
+def test_fixarray():
+    check_array(1, 0)
+    check_array(1, (1 << 4) - 1)
+
+def test_array16():
+    check_array(3, 1 << 4)
+    check_array(3, (1<<16)-1)
+
+def test_array32():
+    check_array(5, (1<<16))
+
+
+def match(obj, buf):
+    assert_equal(packs(obj), buf)
+    assert_equal(unpacks(buf), obj)
+
+def test_match():
+    cases = [
+        (None, b'\xc0'),
+        (False, b'\xc2'),
+        (True, b'\xc3'),
+        (0, b'\x00'),
+        (127, b'\x7f'),
+        (128, b'\xcc\x80'),
+        (256, b'\xcd\x01\x00'),
+        (-1, b'\xff'),
+        (-33, b'\xd0\xdf'),
+        (-129, b'\xd1\xff\x7f'),
+        ({1:1}, b'\x81\x01\x01'),
+        (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"),
+        ((), b'\x90'),
+        (tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"),
+        (tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"),
+        ({}, b'\x80'),
+        (dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'),
+        (dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'),
+        ]
+
+    for v, p in cases:
+        match(v, p)
+
+if __name__ == '__main__':
+    main()
diff --git a/test3/test_except.py b/test3/test_except.py
new file mode 100644
index 0000000..574728f
--- /dev/null
+++ b/test3/test_except.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose.tools import *
+from msgpack import packs, unpacks
+
+import datetime
+
+def test_raise_on_find_unsupported_value():
+    assert_raises(TypeError, packs, datetime.datetime.now())
+
+if __name__ == '__main__':
+    from nose import main
+    main()
diff --git a/test3/test_format.py b/test3/test_format.py
new file mode 100644
index 0000000..022e680
--- /dev/null
+++ b/test3/test_format.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import unpacks
+
+def check(src, should):
+    assert_equal(unpacks(src), should)
+
+def testSimpleValue():
+    check(b"\x93\xc0\xc2\xc3", 
+            (None, False, True,))
+
+def testFixnum():
+    check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff",
+          ((0,64,127,), (-32,-16,-1,),)
+          )
+
+def testFixArray():
+    check(b"\x92\x90\x91\x91\xc0",
+          ((),((None,),),),
+          )
+
+def testFixRaw():
+    check(b"\x94\xa0\xa1a\xa2bc\xa3def",
+          (b"", b"a", b"bc", b"def",),
+          )
+
+def testFixMap():
+    check(
+          b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
+          {False: {None: None}, True:{None:{}}},
+          )
+
+def testUnsignedInt():
+    check(
+          b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
+          b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
+          b"\xce\xff\xff\xff\xff",
+          (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,),
+          )
+
+def testSignedInt():
+    check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
+          b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
+          b"\xd2\xff\xff\xff\xff",
+          (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,))
+
+def testRaw():
+    check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
+        b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
+        (b"", b"a", b"ab", b"", b"a", b"ab"))
+
+def testArray():
+    check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
+        b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02"
+        b"\xc2\xc3",
+        ((), (None,), (False,True), (), (None,), (False,True))
+        )
+
+def testMap():
+    check(
+        b"\x96"
+            b"\xde\x00\x00"
+            b"\xde\x00\x01\xc0\xc2"
+            b"\xde\x00\x02\xc0\xc2\xc3\xc2"
+            b"\xdf\x00\x00\x00\x00"
+            b"\xdf\x00\x00\x00\x01\xc0\xc2"
+            b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
+        ({}, {None: False}, {True: False, None: False}, {},
+            {None: False}, {True: False, None: False}))
+
+if __name__ == '__main__':
+    main()
diff --git a/test3/test_pack.py b/test3/test_pack.py
new file mode 100644
index 0000000..c861704
--- /dev/null
+++ b/test3/test_pack.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+
+from msgpack import packs, unpacks
+
+def check(data):
+    re = unpacks(packs(data))
+    assert_equal(re, data)
+
+def testPack():
+    test_data = [
+            0, 1, 127, 128, 255, 256, 65535, 65536,
+            -1, -32, -33, -128, -129, -32768, -32769,
+            1.0,
+        b"", b"a", b"a"*31, b"a"*32,
+        None, True, False,
+        (), ((),), ((), None,), 
+        {None: 0}, 
+        (1<<23), 
+        ]
+    for td in test_data:
+        check(td)
+
+if __name__ == '__main__':
+    main()
diff --git a/test3/test_sequnpack.py b/test3/test_sequnpack.py
new file mode 100644
index 0000000..5fd377c
--- /dev/null
+++ b/test3/test_sequnpack.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+
+
+from msgpack import Unpacker
+
+def test_foobar():
+    unpacker = Unpacker(read_size=3)
+    unpacker.feed(b'foobar')
+    assert unpacker.unpack() == ord(b'f')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'b')
+    assert unpacker.unpack() == ord(b'a')
+    assert unpacker.unpack() == ord(b'r')
+    try:
+        o = unpacker.unpack()
+        print(("Oops!", o))
+        assert 0
+    except StopIteration:
+        assert 1
+    else:
+        assert 0
+    unpacker.feed(b'foo')
+    unpacker.feed(b'bar')
+
+    k = 0
+    for o, e in zip(unpacker, b'foobarbaz'):
+        assert o == e
+        k += 1
+    assert k == len(b'foobar')
+
+if __name__ == '__main__':
+    test_foobar()
+

From 8fa64e3ab2350faadfe40e0828bafafadac0990a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 09:54:38 +0900
Subject: [PATCH 0639/1648] Add msgpack.version as version tuple.

---
 python/.gitignore          | 1 +
 python/Makefile            | 7 ++++++-
 python/msgpack/__init__.py | 1 +
 python/setup.py            | 8 ++++++--
 4 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/python/.gitignore b/python/.gitignore
index 430c633..8531de3 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -3,3 +3,4 @@ build/*
 dist/*
 *.pyc
 *.pyo
+msgpack/__version__.py
diff --git a/python/Makefile b/python/Makefile
index e06794d..245c09c 100644
--- a/python/Makefile
+++ b/python/Makefile
@@ -1,7 +1,12 @@
+.PHONY: test all python3
+
 all:
 	python setup.py build_ext -i -f
 	python setup.py build sdist
 
-.PHONY: test
+python3:
+	python3 setup.py build_ext -i -f
+	python3 setup.py build sdist
+
 test:
 	nosetests test
diff --git a/python/msgpack/__init__.py b/python/msgpack/__init__.py
index 86786a2..cdf045f 100644
--- a/python/msgpack/__init__.py
+++ b/python/msgpack/__init__.py
@@ -1,4 +1,5 @@
 # coding: utf-8
+from msgpack.__version__ import *
 from msgpack._msgpack import *
 
 # alias for compatibility to simplejson/marshal/pickle.
diff --git a/python/setup.py b/python/setup.py
index 64e71ed..c79c148 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
+version = (0, 1, 5, 'dev')
 
 import os
 from glob import glob
@@ -14,7 +15,10 @@ except ImportError:
     from distutils.command.build_ext import build_ext
     have_cython = False
 
-version = '0.1.4'
+# make msgpack/__verison__.py
+f = open('msgpack/__version__.py', 'w')
+f.write("version = %r\n" % (version,))
+f.close()
 
 # take care of extension modules.
 if have_cython:
@@ -53,7 +57,7 @@ What's MessagePack? (from http://msgpack.sourceforge.net/)
 setup(name='msgpack-python',
       author='INADA Naoki',
       author_email='songofacandy@gmail.com',
-      version=version,
+      version=''.join(str(x) for x in version),
       cmdclass={'build_ext': build_ext, 'sdist': Sdist},
       ext_modules=[msgpack_mod],
       packages=['msgpack'],

From 50ffd2c489fdab81da2d994794b9582e0cadf5f4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 09:54:38 +0900
Subject: [PATCH 0640/1648] Add msgpack.version as version tuple.

---
 .gitignore          | 1 +
 Makefile            | 7 ++++++-
 msgpack/__init__.py | 1 +
 setup.py            | 8 ++++++--
 4 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index 430c633..8531de3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@ build/*
 dist/*
 *.pyc
 *.pyo
+msgpack/__version__.py
diff --git a/Makefile b/Makefile
index e06794d..245c09c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,12 @@
+.PHONY: test all python3
+
 all:
 	python setup.py build_ext -i -f
 	python setup.py build sdist
 
-.PHONY: test
+python3:
+	python3 setup.py build_ext -i -f
+	python3 setup.py build sdist
+
 test:
 	nosetests test
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 86786a2..cdf045f 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,4 +1,5 @@
 # coding: utf-8
+from msgpack.__version__ import *
 from msgpack._msgpack import *
 
 # alias for compatibility to simplejson/marshal/pickle.
diff --git a/setup.py b/setup.py
index 64e71ed..c79c148 100755
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
+version = (0, 1, 5, 'dev')
 
 import os
 from glob import glob
@@ -14,7 +15,10 @@ except ImportError:
     from distutils.command.build_ext import build_ext
     have_cython = False
 
-version = '0.1.4'
+# make msgpack/__verison__.py
+f = open('msgpack/__version__.py', 'w')
+f.write("version = %r\n" % (version,))
+f.close()
 
 # take care of extension modules.
 if have_cython:
@@ -53,7 +57,7 @@ What's MessagePack? (from http://msgpack.sourceforge.net/)
 setup(name='msgpack-python',
       author='INADA Naoki',
       author_email='songofacandy@gmail.com',
-      version=version,
+      version=''.join(str(x) for x in version),
       cmdclass={'build_ext': build_ext, 'sdist': Sdist},
       ext_modules=[msgpack_mod],
       packages=['msgpack'],

From 138d232149e82daabb13d5b6f750aef831ce4398 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 09:58:50 +0900
Subject: [PATCH 0641/1648] python: vesion 0.1.5

---
 python/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index c79c148..67ff74c 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 5, 'dev')
+version = (0, 1, 5, 'final')
 
 import os
 from glob import glob

From 742ca5c341ac1d51680d662b8833cb2a6cf9064d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 09:58:50 +0900
Subject: [PATCH 0642/1648] python: vesion 0.1.5

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index c79c148..67ff74c 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 5, 'dev')
+version = (0, 1, 5, 'final')
 
 import os
 from glob import glob

From a62aefe74bf4017ea90c3dadc86230d12eea5b43 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 10:10:34 +0900
Subject: [PATCH 0643/1648] python: Release 0.1.6 - Fix wrong version string.

---
 python/setup.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/python/setup.py b/python/setup.py
index 67ff74c..d079e3e 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 5, 'final')
+version = (0, 1, 6, 'final')
 
 import os
 from glob import glob
@@ -19,6 +19,9 @@ except ImportError:
 f = open('msgpack/__version__.py', 'w')
 f.write("version = %r\n" % (version,))
 f.close()
+version_str = '.'.join(str(x) for x in version[:3])
+if len(version) > 3 and version[3] != 'final':
+    version_str += version[3]
 
 # take care of extension modules.
 if have_cython:
@@ -57,7 +60,7 @@ What's MessagePack? (from http://msgpack.sourceforge.net/)
 setup(name='msgpack-python',
       author='INADA Naoki',
       author_email='songofacandy@gmail.com',
-      version=''.join(str(x) for x in version),
+      version=version_str,
       cmdclass={'build_ext': build_ext, 'sdist': Sdist},
       ext_modules=[msgpack_mod],
       packages=['msgpack'],

From b68f61cae998c2d690501393563ef6a29ac35154 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 10:10:34 +0900
Subject: [PATCH 0644/1648] python: Release 0.1.6 - Fix wrong version string.

---
 setup.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 67ff74c..d079e3e 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 5, 'final')
+version = (0, 1, 6, 'final')
 
 import os
 from glob import glob
@@ -19,6 +19,9 @@ except ImportError:
 f = open('msgpack/__version__.py', 'w')
 f.write("version = %r\n" % (version,))
 f.close()
+version_str = '.'.join(str(x) for x in version[:3])
+if len(version) > 3 and version[3] != 'final':
+    version_str += version[3]
 
 # take care of extension modules.
 if have_cython:
@@ -57,7 +60,7 @@ What's MessagePack? (from http://msgpack.sourceforge.net/)
 setup(name='msgpack-python',
       author='INADA Naoki',
       author_email='songofacandy@gmail.com',
-      version=''.join(str(x) for x in version),
+      version=version_str,
       cmdclass={'build_ext': build_ext, 'sdist': Sdist},
       ext_modules=[msgpack_mod],
       packages=['msgpack'],

From bf0cb4058634cb28450036c296d18185d7d8867a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 10:13:49 +0900
Subject: [PATCH 0645/1648] python: Add python3 category.

---
 python/setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/setup.py b/python/setup.py
index d079e3e..ac7ece5 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -69,6 +69,7 @@ setup(name='msgpack-python',
       url='http://msgpack.sourceforge.net/',
       download_url='http://pypi.python.org/pypi/msgpack/',
       classifiers=[
+          'Programming Language :: Python :: 3',
           'Development Status :: 4 - Beta',
           'Intended Audience :: Developers',
           'License :: OSI Approved :: Apache Software License',

From ff5709718bd8516e895cc461f4db57ff601abc49 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 2 Sep 2010 10:13:49 +0900
Subject: [PATCH 0646/1648] python: Add python3 category.

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index d079e3e..ac7ece5 100755
--- a/setup.py
+++ b/setup.py
@@ -69,6 +69,7 @@ setup(name='msgpack-python',
       url='http://msgpack.sourceforge.net/',
       download_url='http://pypi.python.org/pypi/msgpack/',
       classifiers=[
+          'Programming Language :: Python :: 3',
           'Development Status :: 4 - Beta',
           'Intended Audience :: Developers',
           'License :: OSI Approved :: Apache Software License',

From 8fc86ce7fa588657ce841a9cf30ea868c461c4e1 Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Thu, 2 Sep 2010 14:33:59 +0900
Subject: [PATCH 0647/1648] removed commented out codes

---
 perl/lib/Data/MessagePack.pm    |  1 -
 perl/lib/Data/MessagePack/PP.pm | 12 ------------
 perl/{ => xs-src}/MessagePack.c |  0
 perl/{ => xs-src}/pack.c        |  0
 perl/{ => xs-src}/unpack.c      |  0
 5 files changed, 13 deletions(-)
 rename perl/{ => xs-src}/MessagePack.c (100%)
 rename perl/{ => xs-src}/pack.c (100%)
 rename perl/{ => xs-src}/unpack.c (100%)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index f8d1625..785f275 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -21,7 +21,6 @@ if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate
         die $@ if $@ && $backend =~ /\b xs \b/xms; # force XS
     }
     if ( !__PACKAGE__->can('pack') ) {
-        print "PP\n";
         require 'Data/MessagePack/PP.pm';
     }
 }
diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 1e05bab..540b416 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -361,16 +361,13 @@ sub execute_limit {
 
 {
     my $p;
-    #my $r; # remained data.
 
 sub execute {
     my ( $self, $data, $offset, $limit ) = @_;
-    #my $value = ( defined $self->{ remain } ? $self->{ remain } : '' ) . substr( $data, $offset, $limit );
     my $value = substr( $data, $offset, $limit ? $limit : length $data );
     my $len   = length $value;
 
     $p = 0;
-    #$r = 0;
 
     while ( $len > $p ) {
         _count( $self, $value ) or last;
@@ -384,9 +381,6 @@ sub execute {
         $self->{ data } .= substr( $value, 0, $p );
         $self->{ remain } = undef;
     }
-    else { # I thought this feature is needed. but XS version can't do so
-        #$self->{ remain } = substr( $value, 0, $p + $r );
-    }
 
     return $p;
 }
@@ -399,12 +393,6 @@ sub _count {
     if ( ( $byte >= 0x90 and $byte <= 0x9f ) or $byte == 0xdc or $byte == 0xdd ) {
         my $num;
         if ( $byte == 0xdc ) { # array 16
-            # I thought this feature is needed. but XS version can't do so. So commented out.
-            #my $len = length substr( $value, $p, 2 );
-            #if ( $len != 2 ) {
-            #    $r = $len;
-            #    return 0;
-            #}
             $num = unpack 'n', substr( $value, $p, 2 );
             $p += 2;
         }
diff --git a/perl/MessagePack.c b/perl/xs-src/MessagePack.c
similarity index 100%
rename from perl/MessagePack.c
rename to perl/xs-src/MessagePack.c
diff --git a/perl/pack.c b/perl/xs-src/pack.c
similarity index 100%
rename from perl/pack.c
rename to perl/xs-src/pack.c
diff --git a/perl/unpack.c b/perl/xs-src/unpack.c
similarity index 100%
rename from perl/unpack.c
rename to perl/xs-src/unpack.c

From 918dbd1926589a9b70f34037e35e98f3194302fc Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Thu, 2 Sep 2010 14:37:22 +0900
Subject: [PATCH 0648/1648] made Makefile.PL XS/PP configurable

---
 perl/Makefile.PL    | 69 +++++++++++++++++++++++++++++++++++++++------
 perl/t/00_compile.t |  2 +-
 2 files changed, 62 insertions(+), 9 deletions(-)

diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index e9f9618..7440a46 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -5,15 +5,29 @@ name 'Data-MessagePack';
 all_from 'lib/Data/MessagePack.pm';
 readme_from('lib/Data/MessagePack.pm');
 
-perl_version '5.008005';
+perl_version '5.008000';
 license 'perl';
-can_cc or die "This module requires a C compiler";
 
 tests 't/*.t';
 recursive_author_tests('xt');
-use_ppport 3.19;
 
-requires_c99(); # msgpack C library requires C99.
+my $use_xs = want_xs();
+
+if ( $] >= 5.008005 and $use_xs ) {
+    can_cc or die "This module requires a C compiler";
+    use_ppport 3.19;
+    requires_c99(); # msgpack C library requires C99.
+    cc_src_paths('xs-src');
+    if ($ENV{DEBUG}) {
+        cc_append_to_ccflags '-g';
+    }
+    # for author's test_pp
+    requires 'Data::Float' => 0 if ( $Module::Install::AUTHOR and $] < 5.010 );
+}
+else { # for Data::MessagePack::PP
+    print "configure PP version\n";
+    requires 'Data::Float' => 0 if ( $] < 5.010 );
+}
 
 clean_files qw{
     *.stackdump
@@ -23,10 +37,6 @@ clean_files qw{
     cover_db
 };
 
-if ($ENV{DEBUG}) {
-    cc_append_to_ccflags '-g';
-}
-
 # copy modules
 if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) {
     mkdir 'msgpack' unless -d 'msgpack';
@@ -39,7 +49,50 @@ if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) {
 requires 'Test::More' => 0.94; # done_testing
 test_requires('Test::Requires');
 
+test_with_env( test_pp => PERL_DATA_MESSAGEPACK => 'pp' );
+
+if($Module::Install::AUTHOR) {
+    postamble qq{test :: test_pp\n\n};
+}
+
 auto_set_repository();
 auto_include;
 WriteAll;
 
+# copied from Makefile.PL in Text::Xslate.
+sub test_with_env {
+    my($name, %env) = @_;
+
+    my $dir = '.testenv';
+    if(not -e $dir) {
+        mkdir $dir or die "Cannot mkdir '.testenv': $!";
+    }
+    clean_files($dir);
+
+    {
+        open my $out, '>', "$dir/$name.pl"
+            or die "Cannot open '$dir/$name.pl' for writing: $!";
+       print $out "# This file sets the env for 'make $name', \n";
+       print $out "# generated by $0 at ", scalar(localtime), ".\n";
+       print $out "# DO NOT EDIT THIS FILE DIRECTLY.\n";
+       print $out "\n";
+
+        while(my($name, $value) = each %env) {
+            printf $out '$ENV{q{%s}} = q{%s};'."\n", $name, $value;
+        }
+    }
+
+    # repeat testing for pure Perl mode
+    # see also ExtUtils::MM_Any::test_via_harness()
+
+    my $t =  q{$(FULLPERLRUN) -MExtUtils::Command::MM -e}
+            .q{ "do q[%s]; test_harness($(TEST_VERBOSE), '$(INST_LIB)', '$(INST_ARCHLIB)')"}
+            .q{ $(TEST_FILES)};
+
+    postamble qq{$name :: pure_all\n}
+            . qq{\t} . q{$(NOECHO) $(ECHO) TESTING: } . $name . qq{\n}
+            . qq{\t} . sprintf($t, "$dir/$name.pl") . qq{\n\n}
+
+            . qq{testall :: $name\n\n};
+    return;
+}
diff --git a/perl/t/00_compile.t b/perl/t/00_compile.t
index 66fe8f0..f91b29e 100644
--- a/perl/t/00_compile.t
+++ b/perl/t/00_compile.t
@@ -3,4 +3,4 @@ use warnings;
 use Test::More tests => 1;
 
 use_ok 'Data::MessagePack';
-
+diag ( $INC{'Data/MessagePack/PP.pm'} ? 'PP' : 'XS' );

From 8f43e033a49aaf1bacb8fb887a0f7b7a538c4031 Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Thu, 2 Sep 2010 23:45:05 +0900
Subject: [PATCH 0649/1648] removed dependency on Data::Float

---
 perl/Makefile.PL                |  5 +--
 perl/lib/Data/MessagePack/PP.pm | 70 +++++++++++++--------------------
 2 files changed, 28 insertions(+), 47 deletions(-)

diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index 7440a46..b786485 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -21,12 +21,9 @@ if ( $] >= 5.008005 and $use_xs ) {
     if ($ENV{DEBUG}) {
         cc_append_to_ccflags '-g';
     }
-    # for author's test_pp
-    requires 'Data::Float' => 0 if ( $Module::Install::AUTHOR and $] < 5.010 );
 }
-else { # for Data::MessagePack::PP
+else {
     print "configure PP version\n";
-    requires 'Data::Float' => 0 if ( $] < 5.010 );
 }
 
 clean_files qw{
diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 540b416..8658373 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -22,37 +22,19 @@ use B ();
 BEGIN {
     # for pack and unpack compatibility
     if ( $] < 5.010 ) {
-        require Data::Float;
-        *pack_double = sub {
-            my $float_hex = Data::Float::float_hex( $_[0] );
-            my ( $sign, $sgnf, $exp ) = $float_hex =~ /^([-+])0x1\.([a-z0-9]+)p([-+][\d]+)$/;
-            my @bits;
-
-            $sign = $sign eq '-' ? 1 : 0;
-            $exp  = sprintf( '%011b', 1023 + $exp );
-
-            my $bit  = $sign . $exp . join( '', map { unpack('B4', pack('H', $_) ) } split //, $sgnf );
-
-            while ( $bit =~ /(.{8})/g ) {
-                push @bits, $1;
-            }
-
-             return pack( 'C*', 0xcb, map { unpack( 'C', pack("B*", $_ ) ) } @bits );
-        };
-        *unpack_double = sub {
-            my $bits = join('', map { sprintf('%08b', $_) } unpack( 'C*', substr( $_[0], $_[1], 8 ) ) );
-            my $sign = substr($bits, 0, 1) ? '-' : '+';
-            my $sgnf = substr($bits, 12, 52);
-            my $exp  = substr($bits, 1, 11);
-            $bits = '';
-            while ( $sgnf =~ /(.{4})/g ) {
-                $bits .= unpack('H',pack('B4', $1));
-            }
-            $exp = ((unpack("C*",(pack("B8", (substr('00000'.$exp,0,8) )))) <<8 )
-                    + unpack("C*",(pack("B8", (substr('00000'.$exp,8,8) ))))) - 1023;
-            return Data::Float::hex_float( $sign . '0x1.' . $bits . 'p' . $exp ) + 0.0;
-        };
-        *unpack_float  = sub { Carp::croak("unpack_float is disable in less than Perl 5.10"); };
+        my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE
+        *pack_double = $bo_is_le ? sub {
+            my @v = unpack( 'V2', pack( 'd', $_[0] ) );
+            return pack 'CN2', 0xcb, @v[1,0];
+        }  : sub { pack 'Cd', 0xcb, $_[0]; };
+        *unpack_float = $bo_is_le ? sub {
+            my @v = unpack( 'v2', substr( $_[0], $_[1], 4 ) );
+            return unpack( 'f', pack( 'n2', @v[1,0] ) );
+        } : sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); };
+        *unpack_double = $bo_is_le ? sub {
+            my @v = unpack( 'V2', substr( $_[0], $_[1], 8 ) );
+            return unpack( 'd', pack( 'N2', @v[1,0] ) );
+        } : sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); };
         *unpack_int16  = sub {
             my $v = unpack 'n', substr( $_[0], $_[1], 2 );
             return $v ? $v - 0x10000 : 0;
@@ -62,14 +44,16 @@ BEGIN {
             return $v ? -(~$v + 1) : $v;
         };
         *unpack_int64  = sub { Carp::croak("unpack_int64 is disable in less than Perl 5.10"); };
+        *unpack_uint64 = sub { Carp::croak("unpack_uint64 is disable in less than Perl 5.10"); };
     }
     else {
         *pack_double   = sub { return pack 'Cd>', 0xcb, $_[0]; };
-        *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); };
         *unpack_float  = sub { return unpack( 'f>', substr( $_[0], $_[1], 4 ) ); };
-        *unpack_int16  = sub { return unpack 'n!',  substr( $_[0], $_[1], 2 ); };
-        *unpack_int32  = sub { return unpack 'N!',  substr( $_[0], $_[1], 4 ); };
-        *unpack_int64  = sub { return unpack 'Q>',  substr( $_[0], $_[1], 8 ); };
+        *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); };
+        *unpack_int16  = sub { return unpack( 'n!', substr( $_[0], $_[1], 2 ) ); };
+        *unpack_int32  = sub { return unpack( 'N!', substr( $_[0], $_[1], 4 ) ); };
+        *unpack_int64  = sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); };
+        *unpack_uint64 = sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); };
     }
     # for 5.8 etc.
     unless ( defined &utf8::is_utf8 ) {
@@ -272,7 +256,7 @@ sub _unpack {
     }
     elsif ( $byte == 0xcf ) { # unit64
         $p += 8;
-        return CORE::unpack 'Q>', substr( $value, $p - 8, 8 );
+        return pack_uint64( $value, $p - 8 );
     }
     elsif ( $byte == 0xd3 ) { # int64
         $p += 8;
@@ -516,25 +500,25 @@ __END__
 
 =head1 NAME
 
-Data::MessagePack::PP - Pure Perl version of Data::MessagePack
-
-=head1 LIMITATION
-
-Currently this module works completely in Perl 5.10 or later.
-In Perl 5.8.x, it requires L and cannot unpack int64 and float (pack int64 too).
+Data::MessagePack::PP - Pure Perl implementation of Data::MessagePack
 
 
 =head1 SEE ALSO
 
 L,
 L,
-L,
 L,
 
 =head1 AUTHOR
 
 makamaka
 
+=head1 LIMITATION
+
+Currently this module works completely in Perl 5.10 or later.
+In Perl 5.8.x, it cannot C uint64 and int64.
+
+
 =head1 COPYRIGHT AND LICENSE
 
 This library is free software; you can redistribute it and/or modify

From 4cc6c3e535e1181dcd9810fd862b80954246f9b7 Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Thu, 2 Sep 2010 23:48:57 +0900
Subject: [PATCH 0650/1648] modified t/05_preferred_int.t for Win32

---
 perl/t/05_preferred_int.t | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t
index 9860711..67d11aa 100644
--- a/perl/t/05_preferred_int.t
+++ b/perl/t/05_preferred_int.t
@@ -16,6 +16,7 @@ sub pis ($$) {
     # is(Dumper(Data::MessagePack->unpack(Data::MessagePack->pack($_[0]))), Dumper($_[0]));
 }
 
+my $is_win = $^O eq 'MSWin32';
 my @dat = (
     '',      'a0',
     '0',     '00',
@@ -29,12 +30,16 @@ my @dat = (
     ''.0xFFFFFF => 'ce 00 ff ff ff',
     ''.0xFFFFFFFF => 'ce ff ff ff ff',
     ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35',
-    ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34',
+    ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => $is_win ?
+                                            'b5 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 30 33 34'
+                                          : 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34',
     '-'.0x8000000 => 'd2 f8 00 00 00',
     '-'.0x80000000 => 'd2 80 00 00 00',
     '-'.0x800000000 => 'ac 2d 33 34 33 35 39 37 33 38 33 36 38',
     '-'.0x8000000000 => 'ad 2d 35 34 39 37 35 35 38 31 33 38 38 38',
-    '-'.0x800000000000000000000000000000 => 'b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35',
+    '-'.0x800000000000000000000000000000 => $is_win ?
+                                            'b6 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 30 33 35'
+                                          : 'b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35',
     {'0' => '1'}, '81 00 01',
     {'abc' => '1'}, '81 a3 61 62 63 01',
 );

From cdc09a7d30e3390dba17db64df121a2dc34c8f04 Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Thu, 2 Sep 2010 23:52:36 +0900
Subject: [PATCH 0651/1648] Changes

---
 perl/Changes | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index a8a4298..a9bb2db 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,7 +1,10 @@
 
 0.1x
 
-    - added PP version.
+    - added PP version (used in cases PERL_DATA_MESSAGEPACK=pp or fail to load XS).
+    - made Makefile.PL PP configurable.
+    - test_pp in author's test
+    - modified t/05_preferred_int.t for Win32
       (makamaka)
 
 0.15

From 2b75d54ce14521b70b63d7aa808a005ac8dafdfa Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Thu, 2 Sep 2010 23:56:55 +0900
Subject: [PATCH 0652/1648] modified pod

---
 perl/lib/Data/MessagePack/PP.pm | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 8658373..270db34 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -509,16 +509,16 @@ L,
 L,
 L,
 
-=head1 AUTHOR
-
-makamaka
-
 =head1 LIMITATION
 
 Currently this module works completely in Perl 5.10 or later.
 In Perl 5.8.x, it cannot C uint64 and int64.
 
 
+=head1 AUTHOR
+
+makamaka
+
 =head1 COPYRIGHT AND LICENSE
 
 This library is free software; you can redistribute it and/or modify

From f91728561fe9c374edb93262e7c9a7c1d819d284 Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Thu, 2 Sep 2010 23:58:40 +0900
Subject: [PATCH 0653/1648] ouch, modified pod

---
 perl/lib/Data/MessagePack/PP.pm | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 270db34..5d95607 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -502,6 +502,11 @@ __END__
 
 Data::MessagePack::PP - Pure Perl implementation of Data::MessagePack
 
+=head1 LIMITATION
+
+Currently this module works completely in Perl 5.10 or later.
+In Perl 5.8.x, it cannot C uint64 and int64.
+
 
 =head1 SEE ALSO
 
@@ -509,12 +514,6 @@ L,
 L,
 L,
 
-=head1 LIMITATION
-
-Currently this module works completely in Perl 5.10 or later.
-In Perl 5.8.x, it cannot C uint64 and int64.
-
-
 =head1 AUTHOR
 
 makamaka

From b97baf4d4713580e89e0dca3bad350339618923e Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Fri, 3 Sep 2010 12:53:56 +0900
Subject: [PATCH 0654/1648] added some comments in Data::MessagePack::PP

---
 perl/lib/Data/MessagePack/PP.pm | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 5d95607..e01b797 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -22,7 +22,10 @@ use B ();
 BEGIN {
     # for pack and unpack compatibility
     if ( $] < 5.010 ) {
+        # require $Config{byteorder}; my $bo_is_le = ( $Config{byteorder} =~ /^1234/ );
+        # which better?
         my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE
+        # In really, since 5.9.2 '>' is introduced.
         *pack_double = $bo_is_le ? sub {
             my @v = unpack( 'V2', pack( 'd', $_[0] ) );
             return pack 'CN2', 0xcb, @v[1,0];

From 1fe4109a42d717aea41ea7ffd7a3193208711e77 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Fri, 3 Sep 2010 14:50:01 +0900
Subject: [PATCH 0655/1648] fixed tests on 64bit machines with -Duselongdouble
 #60625

---
 perl/t/05_preferred_int.t | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t
index 9860711..fe14ef6 100644
--- a/perl/t/05_preferred_int.t
+++ b/perl/t/05_preferred_int.t
@@ -12,7 +12,11 @@ sub packit {
 }
 
 sub pis ($$) {
-    is packit($_[0]), $_[1], 'dump ' . $_[1];
+    if (ref $_[1]) {
+        like packit($_[0]), $_[1], 'dump ' . $_[1];
+    } else {
+        is packit($_[0]), $_[1], 'dump ' . $_[1];
+    }
     # is(Dumper(Data::MessagePack->unpack(Data::MessagePack->pack($_[0]))), Dumper($_[0]));
 }
 
@@ -29,12 +33,12 @@ my @dat = (
     ''.0xFFFFFF => 'ce 00 ff ff ff',
     ''.0xFFFFFFFF => 'ce ff ff ff ff',
     ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35',
-    ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => 'b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34',
+    ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => qr{^(b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34|b7 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 34 32 31 65 2b 33 34)$},
     '-'.0x8000000 => 'd2 f8 00 00 00',
     '-'.0x80000000 => 'd2 80 00 00 00',
     '-'.0x800000000 => 'ac 2d 33 34 33 35 39 37 33 38 33 36 38',
     '-'.0x8000000000 => 'ad 2d 35 34 39 37 35 35 38 31 33 38 38 38',
-    '-'.0x800000000000000000000000000000 => 'b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35',
+    '-'.0x800000000000000000000000000000 => qr{^(b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35|b8 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 37 39 33 36 65 2b 33 35)},
     {'0' => '1'}, '81 00 01',
     {'abc' => '1'}, '81 a3 61 62 63 01',
 );

From adfadc542a98dcc7d838778797b512ccf8bd78f2 Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Sat, 4 Sep 2010 14:35:24 +0900
Subject: [PATCH 0656/1648] enable PP to pack/unpack int64 in less than Perl
 5.10

---
 perl/lib/Data/MessagePack/PP.pm | 39 +++++++++++++++++++++++----------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index e01b797..bd37ad7 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -26,6 +26,14 @@ BEGIN {
         # which better?
         my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE
         # In really, since 5.9.2 '>' is introduced.
+        *pack_uint64 = $bo_is_le ? sub {
+            my @v = unpack( 'V2', pack( 'Q', $_[0] ) );
+            return pack 'CN2', 0xcf, @v[1,0];
+        }  : sub { pack 'CQ', 0xcf, $_[0]; };
+        *pack_int64 = $bo_is_le ? sub {
+            my @v = unpack( 'V2', pack( 'q', $_[0] ) );
+            return pack 'CN2', 0xd3, @v[1,0];
+        }  : sub { pack 'Cq', 0xd3, $_[0]; };
         *pack_double = $bo_is_le ? sub {
             my @v = unpack( 'V2', pack( 'd', $_[0] ) );
             return pack 'CN2', 0xcb, @v[1,0];
@@ -43,13 +51,22 @@ BEGIN {
             return $v ? $v - 0x10000 : 0;
         };
         *unpack_int32  = sub {
+            no warnings; # avoid for warning about Hexadecimal number
             my $v = unpack 'N', substr( $_[0], $_[1], 4 );
-            return $v ? -(~$v + 1) : $v;
+            return $v ? $v - 0x100000000 : 0;
+        };
+        *unpack_int64 = sub {
+            my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) );
+            return unpack( 'q', pack( 'N2', @v[1,0] ) );
+        };
+        *unpack_uint64 = sub {
+            my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) );
+            return unpack( 'Q', pack( 'N2', @v[1,0] ) );
         };
-        *unpack_int64  = sub { Carp::croak("unpack_int64 is disable in less than Perl 5.10"); };
-        *unpack_uint64 = sub { Carp::croak("unpack_uint64 is disable in less than Perl 5.10"); };
     }
     else {
+        *pack_uint64   = sub { return pack 'CQ>', 0xcf, $_[0]; };
+        *pack_int64    = sub { return pack 'Cq>', 0xd3, $_[0]; };
         *pack_double   = sub { return pack 'Cd>', 0xcb, $_[0]; };
         *unpack_float  = sub { return unpack( 'f>', substr( $_[0], $_[1], 4 ) ); };
         *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); };
@@ -130,14 +147,14 @@ sub _pack {
                     : $value < 2 ** 8 ?  CORE::pack 'CC', 0xcc, $value
                     : $value < 2 ** 16 ? CORE::pack 'Cn', 0xcd, $value
                     : $value < 2 ** 32 ? CORE::pack 'CN', 0xce, $value
-                    : CORE::pack 'CQ>', 0xcf, $value;
+                    : pack_uint64( $value );
         }
         else {
             return    -$value <= 32 ?      CORE::pack 'C', ($value & 255)
                     : -$value <= 2 **  7 ? CORE::pack 'Cc', 0xd0, $value
                     : -$value <= 2 ** 15 ? CORE::pack 'Cn', 0xd1, $value
                     : -$value <= 2 ** 31 ? CORE::pack 'CN', 0xd2, $value
-                    : CORE::pack 'Cq>', 0xd3, $value;
+                    : pack_int64( $value );
         }
 
     }
@@ -147,7 +164,9 @@ sub _pack {
         if ( $Data::MessagePack::PreferInteger ) {
             if ( $value =~ /^-?[0-9]+$/ ) { # ok?
                 my $value2 = 0 + $value;
-                if (  0 + $value != B::svref_2object( \$value2 )->int_value ) {
+                if ( $value > 0xFFFFFFFF or $value < '-'.0x80000000 or # <- needless but for XS compat
+                    0 + $value != B::svref_2object( \$value2 )->int_value
+                ) {
                     local $Data::MessagePack::PreferInteger; # avoid for PV => NV
                     return _pack( "$value" );
                 }
@@ -259,7 +278,7 @@ sub _unpack {
     }
     elsif ( $byte == 0xcf ) { # unit64
         $p += 8;
-        return pack_uint64( $value, $p - 8 );
+        return unpack_uint64( $value, $p - 8 );
     }
     elsif ( $byte == 0xd3 ) { # int64
         $p += 8;
@@ -505,11 +524,9 @@ __END__
 
 Data::MessagePack::PP - Pure Perl implementation of Data::MessagePack
 
-=head1 LIMITATION
-
-Currently this module works completely in Perl 5.10 or later.
-In Perl 5.8.x, it cannot C uint64 and int64.
+=head1 DESCRIPTION
 
+This module is used by L internally.
 
 =head1 SEE ALSO
 

From 25531d83936a1253a9dc5ee1b0f4f771d301317d Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Sat, 4 Sep 2010 19:54:12 +0900
Subject: [PATCH 0657/1648] modified t/05_preferred_int.t for Win32

---
 perl/t/05_preferred_int.t | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/perl/t/05_preferred_int.t b/perl/t/05_preferred_int.t
index 9eb223a..084df31 100644
--- a/perl/t/05_preferred_int.t
+++ b/perl/t/05_preferred_int.t
@@ -34,12 +34,16 @@ my @dat = (
     ''.0xFFFFFF => 'ce 00 ff ff ff',
     ''.0xFFFFFFFF => 'ce ff ff ff ff',
     ''.0xFFFFFFFFF => 'ab 36 38 37 31 39 34 37 36 37 33 35',
-    ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => qr{^(b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34|b7 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 34 32 31 65 2b 33 34)$},
+    ''.0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFF => $is_win ? 
+                                            qr{^(b5 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 30 33 34|b8 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 34 32 31 65 2b 30 33 34)$}
+                                          : qr{^(b4 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 65 2b 33 34|b7 38 2e 33 30 37 36 37 34 39 37 33 36 35 35 37 32 34 32 31 65 2b 33 34)$},
     '-'.0x8000000 => 'd2 f8 00 00 00',
     '-'.0x80000000 => 'd2 80 00 00 00',
     '-'.0x800000000 => 'ac 2d 33 34 33 35 39 37 33 38 33 36 38',
     '-'.0x8000000000 => 'ad 2d 35 34 39 37 35 35 38 31 33 38 38 38',
-    '-'.0x800000000000000000000000000000 => qr{^(b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35|b8 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 37 39 33 36 65 2b 33 35)},
+    '-'.0x800000000000000000000000000000 => $is_win ? 
+                                              qr{^(b6 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 30 33 35|b9 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 37 39 33 36 65 2b 30 33 35)}
+                                            : qr{^(b5 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 38 65 2b 33 35|b8 2d 36 2e 36 34 36 31 33 39 39 37 38 39 32 34 35 37 39 33 36 65 2b 33 35)},
     {'0' => '1'}, '81 00 01',
     {'abc' => '1'}, '81 a3 61 62 63 01',
 );

From 84123f544524d6ffd118b91cd7e053d0a6d8bbe4 Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Sat, 4 Sep 2010 20:02:46 +0900
Subject: [PATCH 0658/1648] fallback PP configuration with c99 unspport
 compiler

---
 perl/Makefile.PL | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index b786485..783e658 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -11,19 +11,31 @@ license 'perl';
 tests 't/*.t';
 recursive_author_tests('xt');
 
-my $use_xs = want_xs();
 
-if ( $] >= 5.008005 and $use_xs ) {
-    can_cc or die "This module requires a C compiler";
-    use_ppport 3.19;
-    requires_c99(); # msgpack C library requires C99.
-    cc_src_paths('xs-src');
-    if ($ENV{DEBUG}) {
-        cc_append_to_ccflags '-g';
+if ( $] >= 5.008005 and want_xs() ) {
+    can_cc or die "This module requires a C compiler. Please retry with --pp";
+
+    my $has_c99 = c99_available(); # msgpack C library requires C99.
+
+    if ( $has_c99 ) {
+        use_ppport 3.19;
+        cc_src_paths('xs-src');
+        if ($ENV{DEBUG}) {
+            cc_append_to_ccflags '-g';
+        }
+    }
+    else {
+        print <
Date: Sun, 5 Sep 2010 01:54:44 +0900
Subject: [PATCH 0659/1648] modified begin process about byte order

---
 perl/lib/Data/MessagePack/PP.pm | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index bd37ad7..9e32299 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -25,19 +25,19 @@ BEGIN {
         # require $Config{byteorder}; my $bo_is_le = ( $Config{byteorder} =~ /^1234/ );
         # which better?
         my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE
-        # In really, since 5.9.2 '>' is introduced.
+        # In really, since 5.9.2 '>' is introduced. but 'n!' and 'N!'?
         *pack_uint64 = $bo_is_le ? sub {
             my @v = unpack( 'V2', pack( 'Q', $_[0] ) );
             return pack 'CN2', 0xcf, @v[1,0];
-        }  : sub { pack 'CQ', 0xcf, $_[0]; };
+        } : sub { pack 'CQ', 0xcf, $_[0]; };
         *pack_int64 = $bo_is_le ? sub {
             my @v = unpack( 'V2', pack( 'q', $_[0] ) );
             return pack 'CN2', 0xd3, @v[1,0];
-        }  : sub { pack 'Cq', 0xd3, $_[0]; };
+        } : sub { pack 'Cq', 0xd3, $_[0]; };
         *pack_double = $bo_is_le ? sub {
             my @v = unpack( 'V2', pack( 'd', $_[0] ) );
             return pack 'CN2', 0xcb, @v[1,0];
-        }  : sub { pack 'Cd', 0xcb, $_[0]; };
+        } : sub { pack 'Cd', 0xcb, $_[0]; };
         *unpack_float = $bo_is_le ? sub {
             my @v = unpack( 'v2', substr( $_[0], $_[1], 4 ) );
             return unpack( 'f', pack( 'n2', @v[1,0] ) );
@@ -55,14 +55,14 @@ BEGIN {
             my $v = unpack 'N', substr( $_[0], $_[1], 4 );
             return $v ? $v - 0x100000000 : 0;
         };
-        *unpack_int64 = sub {
+        *unpack_int64 = $bo_is_le ? sub {
             my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) );
             return unpack( 'q', pack( 'N2', @v[1,0] ) );
-        };
-        *unpack_uint64 = sub {
+        } : sub { pack 'q', substr( $_[0], $_[1], 8 ); };
+        *unpack_uint64 = $bo_is_le ? sub {
             my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) );
             return unpack( 'Q', pack( 'N2', @v[1,0] ) );
-        };
+        } : sub { pack 'Q', substr( $_[0], $_[1], 8 ); };
     }
     else {
         *pack_uint64   = sub { return pack 'CQ>', 0xcf, $_[0]; };

From b9bca2a19fc6519296bcda2c6af5f82cc744e005 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Sun, 5 Sep 2010 16:17:19 +0900
Subject: [PATCH 0660/1648] bump to 0.16

---
 perl/Changes                 | 5 +++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index 189990a..448130b 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.16
+
+    - tests on 64bit machines with -Duselongdouble
+      (reported by andk)
+
 0.15
 
     - better argument validation.
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 276353a..94f28c1 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -4,7 +4,7 @@ use warnings;
 use XSLoader;
 use 5.008001;
 
-our $VERSION = '0.15';
+our $VERSION = '0.16';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };

From 80db9971b5a579a1388e0e110baa4a8ec3d1ea7c Mon Sep 17 00:00:00 2001
From: Hideyuki Tanaka 
Date: Mon, 6 Sep 2010 01:32:00 +0900
Subject: [PATCH 0661/1648] pure haskell implementation.

---
 haskell/LICENSE                               |   2 +-
 haskell/cbits/msgpack.c                       | 137 ----
 haskell/msgpack.cabal                         |  55 +-
 haskell/src/Data/MessagePack.hs               | 130 ++--
 haskell/src/Data/MessagePack/Base.hsc         | 584 ------------------
 haskell/src/Data/MessagePack/Feed.hs          |  62 --
 haskell/src/Data/MessagePack/Monad.hs         | 156 -----
 .../Data/MessagePack/{Class.hs => Object.hs}  |  38 +-
 haskell/src/Data/MessagePack/Packer.hs        | 147 +++++
 haskell/src/Data/MessagePack/Put.hs           | 202 ++++++
 haskell/src/Data/MessagePack/Stream.hs        |  82 ---
 haskell/test/Monad.hs                         |  15 +-
 haskell/test/Stream.hs                        |  14 -
 haskell/test/Test.hs                          |  69 ++-
 14 files changed, 543 insertions(+), 1150 deletions(-)
 delete mode 100644 haskell/cbits/msgpack.c
 delete mode 100644 haskell/src/Data/MessagePack/Base.hsc
 delete mode 100644 haskell/src/Data/MessagePack/Feed.hs
 delete mode 100644 haskell/src/Data/MessagePack/Monad.hs
 rename haskell/src/Data/MessagePack/{Class.hs => Object.hs} (77%)
 create mode 100644 haskell/src/Data/MessagePack/Packer.hs
 create mode 100644 haskell/src/Data/MessagePack/Put.hs
 delete mode 100644 haskell/src/Data/MessagePack/Stream.hs
 delete mode 100644 haskell/test/Stream.hs

diff --git a/haskell/LICENSE b/haskell/LICENSE
index 2de30f6..3cb4d8c 100644
--- a/haskell/LICENSE
+++ b/haskell/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2009, Hideyuki Tanaka
+Copyright (c) 2009-2010, Hideyuki Tanaka
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/haskell/cbits/msgpack.c b/haskell/cbits/msgpack.c
deleted file mode 100644
index be44592..0000000
--- a/haskell/cbits/msgpack.c
+++ /dev/null
@@ -1,137 +0,0 @@
-#include 
-
-void msgpack_sbuffer_init_wrap(msgpack_sbuffer* sbuf)
-{
-  msgpack_sbuffer_init(sbuf);
-}
-
-void msgpack_sbuffer_destroy_wrap(msgpack_sbuffer* sbuf)
-{
-  msgpack_sbuffer_destroy(sbuf);
-}
-
-int msgpack_sbuffer_write_wrap(void* data, const char* buf, unsigned int len)
-{
-  return msgpack_sbuffer_write(data, buf, len);
-}
-
-msgpack_packer* msgpack_packer_new_wrap(void *data, msgpack_packer_write callback)
-{
-  return msgpack_packer_new(data, callback);
-}
-
-void msgpack_packer_free_wrap(msgpack_packer* pk)
-{
-  msgpack_packer_free(pk);
-}
-
-int msgpack_pack_uint8_wrap(msgpack_packer* pk, uint8_t d)
-{
-  return msgpack_pack_uint8(pk, d);
-}
-
-int msgpack_pack_uint16_wrap(msgpack_packer* pk, uint16_t d)
-{
-  return msgpack_pack_uint16(pk, d);
-}
-
-int msgpack_pack_uint32_wrap(msgpack_packer* pk, uint32_t d)
-{
-  return msgpack_pack_uint32(pk, d);
-}
-
-int msgpack_pack_uint64_wrap(msgpack_packer* pk, uint64_t d)
-{
-  return msgpack_pack_uint64(pk, d);
-}
-
-int msgpack_pack_int8_wrap(msgpack_packer* pk, int8_t d)
-{
-  return msgpack_pack_int8(pk, d);
-}
-
-int msgpack_pack_int16_wrap(msgpack_packer* pk, int16_t d)
-{
-  return msgpack_pack_int16(pk, d);
-}
-
-int msgpack_pack_int32_wrap(msgpack_packer* pk, int32_t d)
-{
-  return msgpack_pack_int32(pk, d);
-}
-
-int msgpack_pack_int64_wrap(msgpack_packer* pk, int64_t d)
-{
-  return msgpack_pack_int64(pk, d);
-}
-
-int msgpack_pack_double_wrap(msgpack_packer* pk, double d)
-{
-  return msgpack_pack_double(pk, d);
-}
-
-int msgpack_pack_nil_wrap(msgpack_packer* pk)
-{
-  return msgpack_pack_nil(pk);
-}
-
-int msgpack_pack_true_wrap(msgpack_packer* pk)
-{
-  return msgpack_pack_true(pk);
-}
-
-int msgpack_pack_false_wrap(msgpack_packer* pk)
-{
-  return msgpack_pack_false(pk);
-}
-
-int msgpack_pack_array_wrap(msgpack_packer* pk, unsigned int n)
-{
-  return msgpack_pack_array(pk, n);
-}
-
-int msgpack_pack_map_wrap(msgpack_packer* pk, unsigned int n)
-{
-  return msgpack_pack_map(pk, n);
-}
-
-int msgpack_pack_raw_wrap(msgpack_packer* pk, size_t l)
-{
-  return msgpack_pack_raw(pk, l);
-}
-
-int msgpack_pack_raw_body_wrap(msgpack_packer* pk, const void *b, size_t l)
-{
-  return msgpack_pack_raw_body(pk, b, l);
-}
-
-bool msgpack_unpacker_reserve_buffer_wrap(msgpack_unpacker *mpac, size_t size)
-{
-  return msgpack_unpacker_reserve_buffer(mpac, size);
-}
-
-char *msgpack_unpacker_buffer_wrap(msgpack_unpacker *mpac)
-{
-  return msgpack_unpacker_buffer(mpac);
-}
-
-size_t msgpack_unpacker_buffer_capacity_wrap(const msgpack_unpacker *mpac)
-{
-  return msgpack_unpacker_buffer_capacity(mpac);
-}
-
-void msgpack_unpacker_buffer_consumed_wrap(msgpack_unpacker *mpac, size_t size)
-{
-  msgpack_unpacker_buffer_consumed(mpac, size);
-}
-
-void msgpack_unpacker_data_wrap(msgpack_unpacker *mpac, msgpack_object *obj)
-{
-  *obj=msgpack_unpacker_data(mpac);
-}
-
-size_t msgpack_unpacker_message_size_wrap(const msgpack_unpacker *mpac)
-{
-  return msgpack_unpacker_message_size(mpac);
-}
-
diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal
index 82cdb52..8346c1f 100644
--- a/haskell/msgpack.cabal
+++ b/haskell/msgpack.cabal
@@ -1,32 +1,35 @@
-Name:                msgpack
-Version:             0.2.2
-License:             BSD3
-License-File:        LICENSE
-Author:              Hideyuki Tanaka
-Maintainer:          Hideyuki Tanaka 
-Category:            Data
-Synopsis:            A Haskell binding to MessagePack
+Name:               msgpack
+Version:            0.3.0
+Synopsis:           A Haskell binding to MessagePack
 Description:
   A Haskell binding to MessagePack 
-Homepage:	     http://github.com/tanakh/hsmsgpack
-Stability:	     Experimental
-Tested-with:	     GHC==6.10.4
-Cabal-Version:	     >=1.2
-Build-Type:          Simple
 
-library
-  build-depends:	base>=4 && <5, mtl, bytestring
-  ghc-options:		-O2 -Wall
-  hs-source-dirs:	src
-  extra-libraries:	msgpackc
+License:            BSD3
+License-File:       LICENSE
+Category:           Data
+Author:             Hideyuki Tanaka
+Maintainer:         Hideyuki Tanaka 
+Homepage:           http://github.com/tanakh/hsmsgpack
+Stability:          Experimental
+Tested-with:        GHC == 6.12.3
+Cabal-Version:      >= 1.2
+Build-Type:         Simple
+
+Library
+  Build-depends:    base >=4 && <5,
+                    transformers >= 0.2.1 && < 0.2.2,
+                    MonadCatchIO-transformers >= 0.2.2 && < 0.2.3,
+                    bytestring >= 0.9 && < 0.10,
+                    vector >= 0.6.0 && < 0.6.1,
+                    iteratee >= 0.4 && < 0.5,
+                    attoparsec >= 0.8.1 && < 0.8.2,
+                    binary >= 0.5.0 && < 0.5.1,
+                    data-binary-ieee754 >= 0.4 && < 0.5
+  Ghc-options:      -Wall -O2
+  Hs-source-dirs:   src
 
   Exposed-modules:
     Data.MessagePack
-    Data.MessagePack.Base
-    Data.MessagePack.Class
-    Data.MessagePack.Feed
-    Data.MessagePack.Monad
-    Data.MessagePack.Stream
-
-  C-Sources:
-    cbits/msgpack.c
+    Data.MessagePack.Object
+    Data.MessagePack.Put
+    Data.MessagePack.Parser
diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs
index 2949e60..010eaab 100644
--- a/haskell/src/Data/MessagePack.hs
+++ b/haskell/src/Data/MessagePack.hs
@@ -1,7 +1,7 @@
 --------------------------------------------------------------------
 -- |
 -- Module    : Data.MessagePack
--- Copyright : (c) Hideyuki Tanaka, 2009
+-- Copyright : (c) Hideyuki Tanaka, 2009-2010
 -- License   : BSD3
 --
 -- Maintainer:  tanaka.hideyuki@gmail.com
@@ -13,51 +13,105 @@
 --------------------------------------------------------------------
 
 module Data.MessagePack(
-  module Data.MessagePack.Base,
-  module Data.MessagePack.Class,
-  module Data.MessagePack.Feed,
-  module Data.MessagePack.Monad,
-  module Data.MessagePack.Stream,
+  module Data.MessagePack.Object,
+  module Data.MessagePack.Put,
+  module Data.MessagePack.Parser,
   
-  -- * Pack and Unpack
-  packb,
-  unpackb,
+  -- * Simple functions of Pack and Unpack
+  pack,
+  unpack,
+  
+  -- * Pack functions
+  packToString,
+  packToHandle,
+  packToFile,
+  
+  -- * Unpack functions
+  unpackFromString,
+  unpackFromHandle,
+  unpackFromFile,
   
-  -- * Pure version of Pack and Unpack
-  packb',
-  unpackb',
   ) where
 
-import Data.ByteString (ByteString)
-import System.IO.Unsafe
+import qualified Control.Monad.CatchIO as CIO
+import Control.Monad.IO.Class
+import qualified Data.Attoparsec as A
+import Data.Binary.Put
+import qualified Data.ByteString as B
+import qualified Data.ByteString.Lazy as L
+import Data.Functor.Identity
+import qualified Data.Iteratee as I
+import qualified Data.Iteratee.IO as I
+import System.IO
 
-import Data.MessagePack.Base
-import Data.MessagePack.Class
-import Data.MessagePack.Feed
-import Data.MessagePack.Monad
-import Data.MessagePack.Stream
+import Data.MessagePack.Object
+import Data.MessagePack.Put
+import Data.MessagePack.Parser
+
+bufferSize :: Int
+bufferSize = 4 * 1024
+
+class IsByteString s where
+  toBS :: s -> B.ByteString
+
+instance IsByteString B.ByteString where
+  toBS = id
+
+instance IsByteString L.ByteString where
+  toBS = B.concat . L.toChunks
 
 -- | Pack Haskell data to MessagePack string.
-packb :: OBJECT a => a -> IO ByteString
-packb dat = do
-  sb <- newSimpleBuffer
-  pc <- newPacker sb
-  pack pc dat
-  simpleBufferData sb
+pack :: ObjectPut a => a -> L.ByteString
+pack = packToString . put
 
 -- | Unpack MessagePack string to Haskell data.
-unpackb :: OBJECT a => ByteString -> IO (Result a)
-unpackb bs = do
-  withZone $ \z -> do
-    r <- unpackObject z bs
-    return $ case r of
-      Left err -> Left (show err)
-      Right (_, dat) -> fromObject dat
+unpack :: (ObjectGet a, IsByteString s) => s -> a
+unpack bs =
+  runIdentity $ I.run $ I.joinIM $ I.enumPure1Chunk (toBS bs) (parserToIteratee get)
 
--- | Pure version of 'packb'.
-packb' :: OBJECT a => a -> ByteString
-packb' dat = unsafePerformIO $ packb dat
+-- TODO: tryUnpack
 
--- | Pure version of 'unpackb'.
-unpackb' :: OBJECT a => ByteString -> Result a
-unpackb' bs = unsafePerformIO $ unpackb bs
+-- | Pack to ByteString.
+packToString :: Put -> L.ByteString
+packToString = runPut
+
+-- | Pack to Handle
+packToHandle :: Handle -> Put -> IO ()
+packToHandle h = L.hPutStr h . packToString
+
+-- | Pack to File
+packToFile :: FilePath -> Put -> IO ()
+packToFile path = L.writeFile path . packToString
+
+-- | Unpack from ByteString
+unpackFromString :: (Monad m, IsByteString s) => s -> A.Parser a -> m a
+unpackFromString bs =
+  I.run . I.joinIM . I.enumPure1Chunk (toBS bs) . parserToIteratee
+
+-- | Unpack from Handle
+unpackFromHandle :: CIO.MonadCatchIO m => Handle -> A.Parser a -> m a
+unpackFromHandle h =
+  I.run . I.joinIM . I.enumHandle bufferSize h . parserToIteratee
+
+-- | Unpack from File
+unpackFromFile :: CIO.MonadCatchIO m => FilePath -> A.Parser a -> m a
+unpackFromFile path p =
+  CIO.bracket
+  (liftIO $ openBinaryFile path ReadMode)
+  (liftIO . hClose)
+  (flip unpackFromHandle p)
+
+parserToIteratee :: Monad m => A.Parser a -> I.Iteratee B.ByteString m a
+parserToIteratee p = I.icont (itr (A.parse p)) Nothing
+  where
+    itr pcont s = case s of
+      I.EOF _ ->
+        I.throwErr (I.setEOF s)
+      I.Chunk bs ->
+        case pcont bs of
+          A.Fail _ _ msg ->
+            I.throwErr (I.iterStrExc msg)
+          A.Partial cont ->
+            I.icont (itr cont) Nothing
+          A.Done remain ret ->
+            I.idone ret (I.Chunk remain)
diff --git a/haskell/src/Data/MessagePack/Base.hsc b/haskell/src/Data/MessagePack/Base.hsc
deleted file mode 100644
index b6cdc28..0000000
--- a/haskell/src/Data/MessagePack/Base.hsc
+++ /dev/null
@@ -1,584 +0,0 @@
-{-# LANGUAGE CPP #-}
-{-# LANGUAGE ForeignFunctionInterface #-}
-
---------------------------------------------------------------------
--- |
--- Module    : Data.MessagePack.Base
--- Copyright : (c) Hideyuki Tanaka, 2009
--- License   : BSD3
---
--- Maintainer:  tanaka.hideyuki@gmail.com
--- Stability :  experimental
--- Portability: portable
---
--- Low Level Interface to MessagePack C API
---
---------------------------------------------------------------------
-
-module Data.MessagePack.Base(
-  -- * Simple Buffer
-  SimpleBuffer,
-  newSimpleBuffer,
-  simpleBufferData,
-  
-  -- * Serializer
-  Packer,
-  newPacker,
-  
-  packU8,
-  packU16,
-  packU32,
-  packU64,  
-  packS8,
-  packS16,
-  packS32,
-  packS64,
-  
-  packTrue,
-  packFalse,
-  
-  packInt,
-  packDouble,
-  packNil,
-  packBool,
-  
-  packArray,
-  packMap,
-  packRAW,
-  packRAWBody,
-  packRAW',
-  
-  -- * Stream Deserializer
-  Unpacker,
-  defaultInitialBufferSize,
-  newUnpacker,
-  unpackerReserveBuffer,
-  unpackerBuffer,
-  unpackerBufferCapacity,
-  unpackerBufferConsumed,
-  unpackerFeed,
-  unpackerExecute,
-  unpackerData,
-  unpackerReleaseZone,
-  unpackerResetZone,
-  unpackerReset,
-  unpackerMessageSize,
-  
-  -- * MessagePack Object
-  Object(..),
-  packObject,
-  
-  UnpackReturn(..),
-  unpackObject,
-  
-  -- * Memory Zone
-  Zone,
-  newZone,
-  freeZone,
-  withZone,
-  ) where
-
-import Control.Exception
-import Control.Monad
-import Data.ByteString (ByteString)
-import qualified Data.ByteString as BS hiding (pack, unpack)
-import Data.Int
-import Data.Word
-import Foreign.C
-import Foreign.Concurrent
-import Foreign.ForeignPtr hiding (newForeignPtr)
-import Foreign.Marshal.Alloc
-import Foreign.Marshal.Array
-import Foreign.Ptr
-import Foreign.Storable
-
-#include 
-
-type SimpleBuffer = ForeignPtr ()
-
-type WriteCallback = Ptr () -> CString -> CUInt -> IO CInt
-
--- | Create a new Simple Buffer. It will be deleted automatically.
-newSimpleBuffer :: IO SimpleBuffer
-newSimpleBuffer = do
-  ptr <- mallocBytes (#size msgpack_sbuffer)
-  fptr <- newForeignPtr ptr $ do
-    msgpack_sbuffer_destroy ptr
-    free ptr
-  withForeignPtr fptr $ \p ->
-    msgpack_sbuffer_init p
-  return fptr
-
--- | Get data of Simple Buffer.
-simpleBufferData :: SimpleBuffer -> IO ByteString
-simpleBufferData sb =
-  withForeignPtr sb $ \ptr -> do
-    size <- (#peek msgpack_sbuffer, size) ptr
-    dat  <- (#peek msgpack_sbuffer, data) ptr
-    BS.packCStringLen (dat, fromIntegral (size :: CSize))
-
-foreign import ccall "msgpack_sbuffer_init_wrap" msgpack_sbuffer_init ::
-  Ptr () -> IO ()
-
-foreign import ccall "msgpack_sbuffer_destroy_wrap" msgpack_sbuffer_destroy ::
-  Ptr () -> IO ()
-
-foreign import ccall "msgpack_sbuffer_write_wrap" msgpack_sbuffer_write ::
-  WriteCallback
-
-type Packer = ForeignPtr ()
-
--- | Create new Packer. It will be deleted automatically.
-newPacker :: SimpleBuffer -> IO Packer
-newPacker sbuf = do
-  cb <- wrap_callback msgpack_sbuffer_write
-  ptr <- withForeignPtr sbuf $ \ptr ->
-    msgpack_packer_new ptr cb
-  fptr <- newForeignPtr ptr $ do
-    msgpack_packer_free ptr
-  return fptr
-
-foreign import ccall "msgpack_packer_new_wrap" msgpack_packer_new ::
-  Ptr () -> FunPtr WriteCallback -> IO (Ptr ())
-
-foreign import ccall "msgpack_packer_free_wrap" msgpack_packer_free ::
-  Ptr () -> IO ()
-
-foreign import ccall "wrapper" wrap_callback ::
-  WriteCallback -> IO (FunPtr WriteCallback)
-
-packU8 :: Packer -> Word8 -> IO Int
-packU8 pc n =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_uint8 ptr n
-
-foreign import ccall "msgpack_pack_uint8_wrap" msgpack_pack_uint8 ::
-  Ptr () -> Word8 -> IO CInt
-
-packU16 :: Packer -> Word16 -> IO Int
-packU16 pc n =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_uint16 ptr n
-
-foreign import ccall "msgpack_pack_uint16_wrap" msgpack_pack_uint16 ::
-  Ptr () -> Word16 -> IO CInt
-
-packU32 :: Packer -> Word32 -> IO Int
-packU32 pc n =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_uint32 ptr n
-
-foreign import ccall "msgpack_pack_uint32_wrap" msgpack_pack_uint32 ::
-  Ptr () -> Word32 -> IO CInt
-
-packU64 :: Packer -> Word64 -> IO Int
-packU64 pc n =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_uint64 ptr n
-
-foreign import ccall "msgpack_pack_uint64_wrap" msgpack_pack_uint64 ::
-  Ptr () -> Word64 -> IO CInt
-
-packS8 :: Packer -> Int8 -> IO Int
-packS8 pc n =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_int8 ptr n
-
-foreign import ccall "msgpack_pack_int8_wrap" msgpack_pack_int8 ::
-  Ptr () -> Int8 -> IO CInt
-
-packS16 :: Packer -> Int16 -> IO Int
-packS16 pc n =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_int16 ptr n
-
-foreign import ccall "msgpack_pack_int16_wrap" msgpack_pack_int16 ::
-  Ptr () -> Int16 -> IO CInt
-
-packS32 :: Packer -> Int32 -> IO Int
-packS32 pc n =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_int32 ptr n
-
-foreign import ccall "msgpack_pack_int32_wrap" msgpack_pack_int32 ::
-  Ptr () -> Int32 -> IO CInt
-
-packS64 :: Packer -> Int64 -> IO Int
-packS64 pc n =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_int64 ptr n
-
-foreign import ccall "msgpack_pack_int64_wrap" msgpack_pack_int64 ::
-  Ptr () -> Int64 -> IO CInt
-
--- | Pack an integral data.
-packInt :: Integral a => Packer -> a -> IO Int
-packInt pc n = packS64 pc $ fromIntegral n
-
--- | Pack a double data.
-packDouble :: Packer -> Double -> IO Int
-packDouble pc d =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_double ptr (realToFrac d)
-
-foreign import ccall "msgpack_pack_double_wrap" msgpack_pack_double ::
-  Ptr () -> CDouble -> IO CInt
-
--- | Pack a nil.
-packNil :: Packer -> IO Int
-packNil pc =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_nil ptr
-
-foreign import ccall "msgpack_pack_nil_wrap" msgpack_pack_nil ::
-  Ptr () -> IO CInt
-
-packTrue :: Packer -> IO Int
-packTrue pc =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_true ptr
-
-foreign import ccall "msgpack_pack_true_wrap" msgpack_pack_true ::
-  Ptr () -> IO CInt
-
-packFalse :: Packer -> IO Int
-packFalse pc =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_false ptr
-
-foreign import ccall "msgpack_pack_false_wrap" msgpack_pack_false ::
-  Ptr () -> IO CInt
-
--- | Pack a bool data.
-packBool :: Packer -> Bool -> IO Int
-packBool pc True  = packTrue pc
-packBool pc False = packFalse pc
-
--- | 'packArray' @p n@ starts packing an array. 
--- Next @n@ data will consist this array.
-packArray :: Packer -> Int -> IO Int
-packArray pc n =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_array ptr (fromIntegral n)
-
-foreign import ccall "msgpack_pack_array_wrap" msgpack_pack_array ::
-  Ptr () -> CUInt -> IO CInt
-
--- | 'packMap' @p n@ starts packing a map. 
--- Next @n@ pairs of data (2*n data) will consist this map.
-packMap :: Packer -> Int -> IO Int
-packMap pc n =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_map ptr (fromIntegral n)
-
-foreign import ccall "msgpack_pack_map_wrap" msgpack_pack_map ::
-  Ptr () -> CUInt -> IO CInt
-
--- | 'packRAW' @p n@ starts packing a byte sequence. 
--- Next total @n@ bytes of 'packRAWBody' call will consist this sequence.
-packRAW :: Packer -> Int -> IO Int
-packRAW pc n =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-    msgpack_pack_raw ptr (fromIntegral n)
-
-foreign import ccall "msgpack_pack_raw_wrap" msgpack_pack_raw ::
-  Ptr () -> CSize -> IO CInt
-
--- | Pack a byte sequence.
-packRAWBody :: Packer -> ByteString -> IO Int
-packRAWBody pc bs =
-  liftM fromIntegral $ withForeignPtr pc $ \ptr ->
-  BS.useAsCStringLen bs $ \(str, len) ->
-    msgpack_pack_raw_body ptr (castPtr str) (fromIntegral len)
-
-foreign import ccall "msgpack_pack_raw_body_wrap" msgpack_pack_raw_body ::
-  Ptr () -> Ptr () -> CSize -> IO CInt
-
--- | Pack a single byte stream. It calls 'packRAW' and 'packRAWBody'.
-packRAW' :: Packer -> ByteString -> IO Int
-packRAW' pc bs = do
-  _ <- packRAW pc (BS.length bs)
-  packRAWBody pc bs
-
-type Unpacker = ForeignPtr ()
-
-defaultInitialBufferSize :: Int
-defaultInitialBufferSize = 32 * 1024 -- #const MSGPACK_UNPACKER_DEFAULT_INITIAL_BUFFER_SIZE
-
--- | 'newUnpacker' @initialBufferSize@ creates a new Unpacker. It will be deleted automatically.
-newUnpacker :: Int -> IO Unpacker
-newUnpacker initialBufferSize = do
-  ptr <- msgpack_unpacker_new (fromIntegral initialBufferSize)
-  fptr <- newForeignPtr ptr $ do
-    msgpack_unpacker_free ptr
-  return fptr
-
-foreign import ccall "msgpack_unpacker_new" msgpack_unpacker_new ::
-  CSize -> IO (Ptr ())
-
-foreign import ccall "msgpack_unpacker_free" msgpack_unpacker_free ::
-  Ptr() -> IO ()
-
--- | 'unpackerReserveBuffer' @up size@ reserves at least @size@ bytes of buffer.
-unpackerReserveBuffer :: Unpacker -> Int -> IO Bool
-unpackerReserveBuffer up size =
-  withForeignPtr up $ \ptr ->
-  liftM (/=0) $ msgpack_unpacker_reserve_buffer ptr (fromIntegral size)
-
-foreign import ccall "msgpack_unpacker_reserve_buffer_wrap" msgpack_unpacker_reserve_buffer ::
-  Ptr () -> CSize -> IO CChar
-
--- | Get a pointer of unpacker buffer.
-unpackerBuffer :: Unpacker -> IO (Ptr CChar)
-unpackerBuffer up =
-  withForeignPtr up $ \ptr ->
-  msgpack_unpacker_buffer ptr
-
-foreign import ccall "msgpack_unpacker_buffer_wrap" msgpack_unpacker_buffer ::
-  Ptr () -> IO (Ptr CChar)
-
--- | Get size of allocated buffer.
-unpackerBufferCapacity :: Unpacker -> IO Int
-unpackerBufferCapacity up =
-  withForeignPtr up $ \ptr ->
-  liftM fromIntegral $ msgpack_unpacker_buffer_capacity ptr
-
-foreign import ccall "msgpack_unpacker_buffer_capacity_wrap" msgpack_unpacker_buffer_capacity ::
-  Ptr () -> IO CSize
-
--- | 'unpackerBufferConsumed' @up size@ notices that writed @size@ bytes to buffer.
-unpackerBufferConsumed :: Unpacker -> Int -> IO ()
-unpackerBufferConsumed up size =
-  withForeignPtr up $ \ptr ->
-  msgpack_unpacker_buffer_consumed ptr (fromIntegral size)
-
-foreign import ccall "msgpack_unpacker_buffer_consumed_wrap" msgpack_unpacker_buffer_consumed ::
-  Ptr () -> CSize -> IO ()
-
--- | Write byte sequence to Unpacker. It is utility funciton, calls 'unpackerReserveBuffer', 'unpackerBuffer' and 'unpackerBufferConsumed'.
-unpackerFeed :: Unpacker -> ByteString -> IO ()
-unpackerFeed up bs =
-  BS.useAsCStringLen bs $ \(str, len) -> do
-    True <- unpackerReserveBuffer up len
-    ptr <- unpackerBuffer up
-    copyArray ptr str len
-    unpackerBufferConsumed up len
-
--- | Execute deserializing. It returns 0 when buffer contains not enough bytes, returns 1 when succeeded, returns negative value when it failed.
-unpackerExecute :: Unpacker -> IO Int
-unpackerExecute up =
-  withForeignPtr up $ \ptr ->
-  liftM fromIntegral $ msgpack_unpacker_execute ptr
-
-foreign import ccall "msgpack_unpacker_execute" msgpack_unpacker_execute ::
-  Ptr () -> IO CInt
-
--- | Returns a deserialized object when 'unpackerExecute' returned 1.
-unpackerData :: Unpacker -> IO Object
-unpackerData up =
-  withForeignPtr up $ \ptr ->
-  allocaBytes (#size msgpack_object) $ \pobj -> do
-    msgpack_unpacker_data ptr pobj
-    peekObject pobj
-
-foreign import ccall "msgpack_unpacker_data_wrap" msgpack_unpacker_data ::
-  Ptr () -> Ptr () -> IO ()
-
--- | Release memory zone. The returned zone must be freed by calling 'freeZone'.
-unpackerReleaseZone :: Unpacker -> IO Zone
-unpackerReleaseZone up =
-  withForeignPtr up $ \ptr ->
-  msgpack_unpacker_release_zone ptr
-
-foreign import ccall "msgpack_unpacker_release_zone" msgpack_unpacker_release_zone ::
-  Ptr () -> IO (Ptr ())
-
--- | Free memory zone used by Unapcker.
-unpackerResetZone :: Unpacker -> IO ()
-unpackerResetZone up =
-  withForeignPtr up $ \ptr ->
-  msgpack_unpacker_reset_zone ptr
-
-foreign import ccall "msgpack_unpacker_reset_zone" msgpack_unpacker_reset_zone ::
-  Ptr () -> IO ()
-
--- | Reset Unpacker state except memory zone.
-unpackerReset :: Unpacker -> IO ()
-unpackerReset up =
-  withForeignPtr up $ \ptr ->
-  msgpack_unpacker_reset ptr
-
-foreign import ccall "msgpack_unpacker_reset" msgpack_unpacker_reset ::
-  Ptr () -> IO ()
-
--- | Returns number of bytes of sequence of deserializing object.
-unpackerMessageSize :: Unpacker -> IO Int
-unpackerMessageSize up =
-  withForeignPtr up $ \ptr ->
-  liftM fromIntegral $ msgpack_unpacker_message_size ptr
-
-foreign import ccall "msgpack_unpacker_message_size_wrap" msgpack_unpacker_message_size ::
-  Ptr () -> IO CSize
-
-type Zone = Ptr ()
-
--- | Create a new memory zone. It must be freed manually.
-newZone :: IO Zone
-newZone =
-  msgpack_zone_new (#const MSGPACK_ZONE_CHUNK_SIZE)
-
--- | Free a memory zone.
-freeZone :: Zone -> IO ()
-freeZone z =
-  msgpack_zone_free z
-
--- | Create a memory zone, then execute argument, then free memory zone.
-withZone :: (Zone -> IO a) -> IO a
-withZone z =
-  bracket newZone freeZone z
-
-foreign import ccall "msgpack_zone_new" msgpack_zone_new ::
-  CSize -> IO Zone
-
-foreign import ccall "msgpack_zone_free" msgpack_zone_free ::
-  Zone -> IO ()
-
--- | Object Representation of MessagePack data.
-data Object =
-  ObjectNil
-  | ObjectBool Bool
-  | ObjectInteger Int
-  | ObjectDouble Double
-  | ObjectRAW ByteString
-  | ObjectArray [Object]
-  | ObjectMap [(Object, Object)]
-  deriving (Show)
-
-peekObject :: Ptr a -> IO Object
-peekObject ptr = do
-  typ <- (#peek msgpack_object, type) ptr
-  case (typ :: CInt) of
-    (#const MSGPACK_OBJECT_NIL) ->
-      return ObjectNil
-    (#const MSGPACK_OBJECT_BOOLEAN) ->
-      peekObjectBool ptr
-    (#const MSGPACK_OBJECT_POSITIVE_INTEGER) ->
-      peekObjectPositiveInteger ptr
-    (#const MSGPACK_OBJECT_NEGATIVE_INTEGER) ->
-      peekObjectNegativeInteger ptr
-    (#const MSGPACK_OBJECT_DOUBLE) ->
-      peekObjectDouble ptr
-    (#const MSGPACK_OBJECT_RAW) ->
-      peekObjectRAW ptr
-    (#const MSGPACK_OBJECT_ARRAY) ->
-      peekObjectArray ptr
-    (#const MSGPACK_OBJECT_MAP) ->
-      peekObjectMap ptr
-    _ ->
-      fail $ "peekObject: unknown object type (" ++ show typ ++ ")"
-
-peekObjectBool :: Ptr a -> IO Object
-peekObjectBool ptr = do
-  b <- (#peek msgpack_object, via.boolean) ptr
-  return $ ObjectBool $ (b :: CUChar) /= 0
-
-peekObjectPositiveInteger :: Ptr a -> IO Object
-peekObjectPositiveInteger ptr = do
-  n <- (#peek msgpack_object, via.u64) ptr
-  return $ ObjectInteger $ fromIntegral (n :: Word64)
-
-peekObjectNegativeInteger :: Ptr a -> IO Object
-peekObjectNegativeInteger ptr = do
-  n <- (#peek msgpack_object, via.i64) ptr
-  return $ ObjectInteger $ fromIntegral (n :: Int64)
-
-peekObjectDouble :: Ptr a -> IO Object
-peekObjectDouble ptr = do
-  d <- (#peek msgpack_object, via.dec) ptr
-  return $ ObjectDouble $ realToFrac (d :: CDouble)
-
-peekObjectRAW :: Ptr a -> IO Object
-peekObjectRAW ptr = do
-  size <- (#peek msgpack_object, via.raw.size) ptr
-  p    <- (#peek msgpack_object, via.raw.ptr) ptr
-  bs   <- BS.packCStringLen (p, fromIntegral (size :: Word32))
-  return $ ObjectRAW bs
-
-peekObjectArray :: Ptr a -> IO Object
-peekObjectArray ptr = do
-  csize <- (#peek msgpack_object, via.array.size) ptr
-  let size = fromIntegral (csize :: Word32)
-  p     <- (#peek msgpack_object, via.array.ptr) ptr
-  objs  <- mapM (\i -> peekObject $ p `plusPtr`
-                      ((#size msgpack_object) * i))
-           [0..size-1]
-  return $ ObjectArray objs
-
-peekObjectMap :: Ptr a -> IO Object
-peekObjectMap ptr = do
-  csize <- (#peek msgpack_object, via.map.size) ptr
-  let size = fromIntegral (csize :: Word32)
-  p     <- (#peek msgpack_object, via.map.ptr) ptr
-  dat   <- mapM (\i -> peekObjectKV $ p `plusPtr`
-                      ((#size msgpack_object_kv) * i))
-           [0..size-1]
-  return $ ObjectMap dat
-
-peekObjectKV :: Ptr a -> IO (Object, Object)
-peekObjectKV ptr = do
-  k <- peekObject $ ptr `plusPtr` (#offset msgpack_object_kv, key)
-  v <- peekObject $ ptr `plusPtr` (#offset msgpack_object_kv, val)
-  return (k, v)
-
--- | Pack a Object.
-packObject :: Packer -> Object -> IO ()
-packObject pc ObjectNil = packNil pc >> return ()
-
-packObject pc (ObjectBool b) = packBool pc b >> return ()
-
-packObject pc (ObjectInteger n) = packInt pc n >> return ()
-
-packObject pc (ObjectDouble d) = packDouble pc d >> return ()
-
-packObject pc (ObjectRAW bs) = packRAW' pc bs >> return ()
-
-packObject pc (ObjectArray ls) = do
-  _ <- packArray pc (length ls)
-  mapM_ (packObject pc) ls
-
-packObject pc (ObjectMap ls) = do
-  _ <- packMap pc (length ls)
-  mapM_ (\(a, b) -> packObject pc a >> packObject pc b) ls
-
-data UnpackReturn =
-  UnpackContinue     -- ^ not enough bytes to unpack object
-  | UnpackParseError -- ^ got invalid bytes
-  | UnpackError      -- ^ other error
-  deriving (Eq, Show)
-
--- | Unpack a single MessagePack object from byte sequence.
-unpackObject :: Zone -> ByteString -> IO (Either UnpackReturn (Int, Object))
-unpackObject z dat =
-  allocaBytes (#size msgpack_object) $ \ptr ->
-  BS.useAsCStringLen dat $ \(str, len) ->
-  alloca $ \poff -> do
-    poke poff 0
-    ret <- msgpack_unpack str (fromIntegral len) poff z ptr
-    case ret of
-      (#const MSGPACK_UNPACK_SUCCESS) -> do
-        off <- peek poff
-        obj <- peekObject ptr
-        return $ Right (fromIntegral off, obj)
-      (#const MSGPACK_UNPACK_EXTRA_BYTES) -> do
-        off <- peek poff
-        obj <- peekObject ptr
-        return $ Right (fromIntegral off, obj)
-      (#const MSGPACK_UNPACK_CONTINUE) ->
-        return $ Left UnpackContinue
-      (#const MSGPACK_UNPACK_PARSE_ERROR) ->
-        return $ Left UnpackParseError
-      _ ->
-        return $ Left UnpackError
-
-foreign import ccall "msgpack_unpack" msgpack_unpack ::
-  Ptr CChar -> CSize -> Ptr CSize -> Zone -> Ptr () -> IO CInt
diff --git a/haskell/src/Data/MessagePack/Feed.hs b/haskell/src/Data/MessagePack/Feed.hs
deleted file mode 100644
index 4b48639..0000000
--- a/haskell/src/Data/MessagePack/Feed.hs
+++ /dev/null
@@ -1,62 +0,0 @@
---------------------------------------------------------------------
--- |
--- Module    : Data.MessagePack.Feed
--- Copyright : (c) Hideyuki Tanaka, 2009
--- License   : BSD3
---
--- Maintainer:  tanaka.hideyuki@gmail.com
--- Stability :  experimental
--- Portability: portable
---
--- Feeders for Stream Deserializers
---
---------------------------------------------------------------------
-
-module Data.MessagePack.Feed(
-  -- * Feeder type
-  Feeder,
-  -- * Feeders
-  feederFromHandle,
-  feederFromFile,
-  feederFromString,
-  ) where
-
-import Data.ByteString (ByteString)
-import qualified Data.ByteString as BS
-import Data.IORef
-import System.IO
-
--- | Feeder returns Just ByteString when bytes remains, otherwise Nothing.
-type Feeder = IO (Maybe ByteString)
-
--- | Feeder from Handle
-feederFromHandle :: Handle -> IO Feeder
-feederFromHandle h = return $ do
-  bs <- BS.hGetNonBlocking h bufSize
-  if BS.length bs > 0
-    then do return $ Just bs
-    else do
-    c <- BS.hGet h 1
-    if BS.length c > 0
-      then do return $ Just c
-      else do
-      hClose h
-      return Nothing
-  where
-    bufSize = 4096
-
--- | Feeder from File
-feederFromFile :: FilePath -> IO Feeder
-feederFromFile path =
-  openFile path ReadMode >>= feederFromHandle
-
--- | Feeder from ByteString
-feederFromString :: ByteString -> IO Feeder
-feederFromString bs = do
-  r <- newIORef (Just bs)
-  return $ f r
-  where
-    f r = do
-      mb <- readIORef r
-      writeIORef r Nothing
-      return mb
diff --git a/haskell/src/Data/MessagePack/Monad.hs b/haskell/src/Data/MessagePack/Monad.hs
deleted file mode 100644
index 15f21fe..0000000
--- a/haskell/src/Data/MessagePack/Monad.hs
+++ /dev/null
@@ -1,156 +0,0 @@
---------------------------------------------------------------------
--- |
--- Module    : Data.MessagePack.Monad
--- Copyright : (c) Hideyuki Tanaka, 2009
--- License   : BSD3
---
--- Maintainer:  tanaka.hideyuki@gmail.com
--- Stability :  experimental
--- Portability: portable
---
--- Monadic Stream Serializers and Deserializers
---
---------------------------------------------------------------------
-
-module Data.MessagePack.Monad(
-  -- * Classes
-  MonadPacker(..),
-  MonadUnpacker(..),
-  
-  -- * Packer and Unpacker type
-  PackerT(..),
-  UnpackerT(..),
-  
-  -- * Packers
-  packToString,
-  packToHandle,
-  packToFile,
-  
-  -- * Unpackers
-  unpackFrom,
-  unpackFromString,
-  unpackFromHandle,
-  unpackFromFile,
-  ) where
-
-import Control.Monad
-import Control.Monad.Trans
-import Data.ByteString (ByteString)
-import qualified Data.ByteString as BS
-import System.IO
-
-import Data.MessagePack.Base hiding (Unpacker)
-import qualified Data.MessagePack.Base as Base
-import Data.MessagePack.Class
-import Data.MessagePack.Feed
-
-class Monad m => MonadPacker m where
-  -- | Serialize a object
-  put :: OBJECT a => a -> m ()
-
-class Monad m => MonadUnpacker m where
-  -- | Deserialize a object
-  get :: OBJECT a => m a
-
--- | Serializer Type
-newtype PackerT m r = PackerT { runPackerT :: Base.Packer -> m r }
-
-instance Monad m => Monad (PackerT m) where
-  a >>= b =
-    PackerT $ \pc -> do
-      r <- runPackerT a pc
-      runPackerT (b r) pc
-  
-  return r =
-    PackerT $ \_ -> return r
-
-instance MonadTrans PackerT where
-  lift m = PackerT $ \_ -> m
-
-instance MonadIO m => MonadIO (PackerT m) where
-  liftIO = lift . liftIO
-
-instance MonadIO m => MonadPacker (PackerT m) where
-  put v = PackerT $ \pc -> liftIO $ do
-    pack pc v
-
--- | Execute given serializer and returns byte sequence.
-packToString :: MonadIO m => PackerT m r -> m ByteString
-packToString m = do
-  sb <- liftIO $ newSimpleBuffer
-  pc <- liftIO $ newPacker sb
-  _ <- runPackerT m pc
-  liftIO $ simpleBufferData sb
-
--- | Execute given serializer and write byte sequence to Handle.
-packToHandle :: MonadIO m => Handle -> PackerT m r -> m ()
-packToHandle h m = do
-  sb <- packToString m
-  liftIO $ BS.hPut h sb
-  liftIO $ hFlush h
-
--- | Execute given serializer and write byte sequence to file.
-packToFile :: MonadIO m => FilePath -> PackerT m r -> m ()
-packToFile p m = do
-  sb <- packToString m
-  liftIO $ BS.writeFile p sb
-
--- | Deserializer type
-newtype UnpackerT m r = UnpackerT { runUnpackerT :: Base.Unpacker -> Feeder -> m r }
-
-instance Monad m => Monad (UnpackerT m) where
-  a >>= b =
-    UnpackerT $ \up feed -> do
-      r <- runUnpackerT a up feed
-      runUnpackerT (b r) up feed
-  
-  return r =
-    UnpackerT $ \_ _ -> return r
-
-instance MonadTrans UnpackerT where
-  lift m = UnpackerT $ \_ _ -> m
-
-instance MonadIO m => MonadIO (UnpackerT m) where
-  liftIO = lift . liftIO
-
-instance MonadIO m => MonadUnpacker (UnpackerT m) where
-  get = UnpackerT $ \up feed -> liftIO $ do
-    executeOne up feed
-    obj <- unpackerData up
-    freeZone =<< unpackerReleaseZone up
-    unpackerReset up
-    let Right r = fromObject obj
-    return r
-    
-    where
-      executeOne up feed = do
-        resp <- unpackerExecute up
-        guard $ resp>=0
-        when (resp==0) $ do
-          Just bs <- feed
-          unpackerFeed up bs
-          executeOne up feed
-
--- | Execute deserializer using given feeder.
-unpackFrom :: MonadIO m => Feeder -> UnpackerT m r -> m r
-unpackFrom f m = do
-  up <- liftIO $ newUnpacker defaultInitialBufferSize
-  runUnpackerT m up f
-
--- | Execute deserializer using given handle.
-unpackFromHandle :: MonadIO m => Handle -> UnpackerT m r -> m r
-unpackFromHandle h m =
-  flip unpackFrom m =<< liftIO (feederFromHandle h)
-
--- | Execute deserializer using given file content.
-unpackFromFile :: MonadIO m => FilePath -> UnpackerT m r -> m r
-unpackFromFile p m = do
-  h <- liftIO $ openFile p ReadMode
-  r <- flip unpackFrom m =<< liftIO (feederFromHandle h)
-  liftIO $ hClose h
-  return r
-
--- | Execute deserializer from given byte sequence.
-unpackFromString :: MonadIO m => ByteString -> UnpackerT m r -> m r
-unpackFromString bs m = do
-  flip unpackFrom m =<< liftIO (feederFromString bs)
diff --git a/haskell/src/Data/MessagePack/Class.hs b/haskell/src/Data/MessagePack/Object.hs
similarity index 77%
rename from haskell/src/Data/MessagePack/Class.hs
rename to haskell/src/Data/MessagePack/Object.hs
index 365acc5..19a3aeb 100644
--- a/haskell/src/Data/MessagePack/Class.hs
+++ b/haskell/src/Data/MessagePack/Object.hs
@@ -1,38 +1,50 @@
 {-# LANGUAGE TypeSynonymInstances #-}
 {-# LANGUAGE FlexibleInstances #-}
-{-# LANGUAGE OverlappingInstances #-}
-{-# LANGUAGE IncoherentInstances #-}
 
 --------------------------------------------------------------------
 -- |
--- Module    : Data.MessagePack.Class
--- Copyright : (c) Hideyuki Tanaka, 2009
+-- Module    : Data.MessagePack.Object
+-- Copyright : (c) Hideyuki Tanaka, 2009-2010
 -- License   : BSD3
 --
 -- Maintainer:  tanaka.hideyuki@gmail.com
 -- Stability :  experimental
 -- Portability: portable
 --
--- Serializing Haskell values to and from MessagePack Objects.
+-- MessagePack object definition
 --
 --------------------------------------------------------------------
 
-module Data.MessagePack.Class(
+module Data.MessagePack.Object(
+  -- * MessagePack Object
+  Object(..),
+  
   -- * Serialization to and from Object
   OBJECT(..),
   Result,
-  pack,
   ) where
 
-import Control.Monad.Error
-import Data.ByteString.Char8 (ByteString)
+import Control.Monad
+import Control.Monad.Trans.Error ()
+import qualified Data.ByteString as B
 import qualified Data.ByteString.Char8 as C8
 
-import Data.MessagePack.Base
+-- | Object Representation of MessagePack data.
+data Object =
+  ObjectNil
+  | ObjectBool Bool
+  | ObjectInteger Int
+  | ObjectDouble Double
+  | ObjectRAW B.ByteString
+  | ObjectArray [Object]
+  | ObjectMap [(Object, Object)]
+  deriving (Show)
 
 -- | The class of types serializable to and from MessagePack object
 class OBJECT a where
+  -- | Encode a value to MessagePack object
   toObject :: a -> Object
+  -- | Decode a value from MessagePack object
   fromObject :: Object -> Result a
 
 -- | A type for parser results
@@ -65,7 +77,7 @@ instance OBJECT Double where
   fromObject (ObjectDouble d) = Right d
   fromObject _ = Left fromObjectError
 
-instance OBJECT ByteString where
+instance OBJECT B.ByteString where
   toObject = ObjectRAW
   fromObject (ObjectRAW bs) = Right bs
   fromObject _ = Left fromObjectError
@@ -95,7 +107,3 @@ instance OBJECT a => OBJECT (Maybe a) where
   
   fromObject ObjectNil = return Nothing
   fromObject obj = liftM Just $ fromObject obj
-
--- | Pack a serializable Haskell value.
-pack :: OBJECT a => Packer -> a -> IO ()
-pack pc = packObject pc . toObject
diff --git a/haskell/src/Data/MessagePack/Packer.hs b/haskell/src/Data/MessagePack/Packer.hs
new file mode 100644
index 0000000..9c10f5e
--- /dev/null
+++ b/haskell/src/Data/MessagePack/Packer.hs
@@ -0,0 +1,147 @@
+{-# Language FlexibleInstances #-}
+{-# Language OverlappingInstances #-}
+
+module Data.MessagePack.Packer(
+  ObjectPut(..),
+  ) where
+
+import Data.Binary.Put
+import Data.Binary.IEEE754
+import Data.Bits
+import qualified Data.ByteString as B
+
+import Data.MessagePack.Object
+
+class ObjectPut a where
+  put :: a -> Put
+
+instance ObjectPut Object where
+  put = putObject
+
+instance ObjectPut Int where
+  put = putInteger
+
+instance ObjectPut () where
+  put _ = putNil
+
+instance ObjectPut Bool where
+  put = putBool
+
+instance ObjectPut Double where
+  put = putDouble
+
+instance ObjectPut B.ByteString where
+  put = putRAW
+
+instance ObjectPut a => ObjectPut [a] where
+  put = putArray
+
+instance (ObjectPut k, ObjectPut v) => ObjectPut [(k, v)] where
+  put = putMap
+
+putObject :: Object -> Put
+putObject obj =
+  case obj of
+    ObjectInteger n ->
+      putInteger n
+    ObjectNil ->
+      putNil
+    ObjectBool b ->
+      putBool b
+    ObjectDouble d ->
+      putDouble d
+    ObjectRAW raw ->
+      putRAW raw
+    ObjectArray arr ->
+      putArray arr
+    ObjectMap m ->
+      putMap m
+
+putInteger :: Int -> Put      
+putInteger n =
+  case n of
+    _ | n >= 0 && n <= 127 ->
+      putWord8 $ fromIntegral n
+    _ | n >= -32 && n <= -1 ->
+      putWord8 $ fromIntegral n
+    _ | n >= 0 && n < 0x100 -> do
+      putWord8 0xCC
+      putWord8 $ fromIntegral n
+    _ | n >= 0 && n < 0x10000 -> do
+      putWord8 0xCD
+      putWord16be $ fromIntegral n
+    _ | n >= 0 && n < 0x100000000 -> do
+      putWord8 0xCE
+      putWord32be $ fromIntegral n
+    _ | n >= 0 -> do
+      putWord8 0xCF
+      putWord64be $ fromIntegral n
+    _ | n >= -0x100 -> do
+      putWord8 0xD0
+      putWord8 $ fromIntegral n
+    _ | n >= -0x10000 -> do
+      putWord8 0xD1
+      putWord16be $ fromIntegral n
+    _ | n >= -0x100000000 -> do
+      putWord8 0xD2
+      putWord32be $ fromIntegral n
+    _ -> do
+      putWord8 0xD3
+      putWord64be $ fromIntegral n
+
+putNil :: Put
+putNil = putWord8 0xC0
+
+putBool :: Bool -> Put
+putBool True = putWord8 0xC3
+putBool False = putWord8 0xC2
+
+putDouble :: Double -> Put
+putDouble d = do
+  putWord8 0xCB
+  putFloat64be d
+
+putRAW :: B.ByteString -> Put
+putRAW bs = do
+  case len of
+    _ | len <= 31 -> do
+      putWord8 $ 0xA0 .|. fromIntegral len
+    _ | len < 0x10000 -> do
+      putWord8 0xDA
+      putWord16be $ fromIntegral len
+    _ -> do
+      putWord8 0xDB
+      putWord32be $ fromIntegral len
+  putByteString bs
+  where
+    len = B.length bs
+
+putArray :: ObjectPut a => [a] -> Put
+putArray arr = do
+  case len of
+    _ | len <= 15 ->
+      putWord8 $ 0x90 .|. fromIntegral len
+    _ | len < 0x10000 -> do
+      putWord8 0xDC
+      putWord16be $ fromIntegral len
+    _ -> do
+      putWord8 0xDD
+      putWord32be $ fromIntegral len
+  mapM_ put arr
+  where
+    len = length arr
+
+putMap :: (ObjectPut k, ObjectPut v) => [(k, v)] -> Put
+putMap m = do
+  case len of
+    _ | len <= 15 ->
+      putWord8 $ 0x80 .|. fromIntegral len
+    _ | len < 0x10000 -> do
+      putWord8 0xDE
+      putWord16be $ fromIntegral len
+    _ -> do
+      putWord8 0xDF
+      putWord16be $ fromIntegral len
+  mapM_ (\(k, v) -> put k >> put v) m
+  where
+    len = length m
diff --git a/haskell/src/Data/MessagePack/Put.hs b/haskell/src/Data/MessagePack/Put.hs
new file mode 100644
index 0000000..8d0af2b
--- /dev/null
+++ b/haskell/src/Data/MessagePack/Put.hs
@@ -0,0 +1,202 @@
+{-# Language FlexibleInstances #-}
+{-# Language IncoherentInstances #-}
+{-# Language OverlappingInstances #-}
+
+--------------------------------------------------------------------
+-- |
+-- Module    : Data.MessagePack.Put
+-- Copyright : (c) Hideyuki Tanaka, 2009-2010
+-- License   : BSD3
+--
+-- Maintainer:  tanaka.hideyuki@gmail.com
+-- Stability :  experimental
+-- Portability: portable
+--
+-- MessagePack Serializer using @Data.Binary.Put@
+--
+--------------------------------------------------------------------
+
+module Data.MessagePack.Put(
+  -- * Serializable class
+  ObjectPut(..),
+  ) where
+
+import Data.Binary.Put
+import Data.Binary.IEEE754
+import Data.Bits
+import qualified Data.ByteString as B
+import qualified Data.Vector as V
+
+import Data.MessagePack.Object
+
+-- | Serializable class
+class ObjectPut a where
+  -- | Serialize a value
+  put :: a -> Put
+
+instance ObjectPut Object where
+  put = putObject
+
+instance ObjectPut Int where
+  put = putInteger
+
+instance ObjectPut () where
+  put _ = putNil
+
+instance ObjectPut Bool where
+  put = putBool
+
+instance ObjectPut Double where
+  put = putDouble
+
+instance ObjectPut B.ByteString where
+  put = putRAW
+
+instance ObjectPut a => ObjectPut [a] where
+  put = putArray
+
+instance ObjectPut a => ObjectPut (V.Vector a) where
+  put = putArrayVector
+
+instance (ObjectPut k, ObjectPut v) => ObjectPut [(k, v)] where
+  put = putMap
+
+instance (ObjectPut k, ObjectPut v) => ObjectPut (V.Vector (k, v)) where
+  put = putMapVector
+
+putObject :: Object -> Put
+putObject obj =
+  case obj of
+    ObjectInteger n ->
+      putInteger n
+    ObjectNil ->
+      putNil
+    ObjectBool b ->
+      putBool b
+    ObjectDouble d ->
+      putDouble d
+    ObjectRAW raw ->
+      putRAW raw
+    ObjectArray arr ->
+      putArray arr
+    ObjectMap m ->
+      putMap m
+
+putInteger :: Int -> Put      
+putInteger n =
+  case n of
+    _ | n >= 0 && n <= 127 ->
+      putWord8 $ fromIntegral n
+    _ | n >= -32 && n <= -1 ->
+      putWord8 $ fromIntegral n
+    _ | n >= 0 && n < 0x100 -> do
+      putWord8 0xCC
+      putWord8 $ fromIntegral n
+    _ | n >= 0 && n < 0x10000 -> do
+      putWord8 0xCD
+      putWord16be $ fromIntegral n
+    _ | n >= 0 && n < 0x100000000 -> do
+      putWord8 0xCE
+      putWord32be $ fromIntegral n
+    _ | n >= 0 -> do
+      putWord8 0xCF
+      putWord64be $ fromIntegral n
+    _ | n >= -0x80 -> do
+      putWord8 0xD0
+      putWord8 $ fromIntegral n
+    _ | n >= -0x8000 -> do
+      putWord8 0xD1
+      putWord16be $ fromIntegral n
+    _ | n >= -0x80000000 -> do
+      putWord8 0xD2
+      putWord32be $ fromIntegral n
+    _ -> do
+      putWord8 0xD3
+      putWord64be $ fromIntegral n
+
+putNil :: Put
+putNil = putWord8 0xC0
+
+putBool :: Bool -> Put
+putBool True = putWord8 0xC3
+putBool False = putWord8 0xC2
+
+putDouble :: Double -> Put
+putDouble d = do
+  putWord8 0xCB
+  putFloat64be d
+
+putRAW :: B.ByteString -> Put
+putRAW bs = do
+  case len of
+    _ | len <= 31 -> do
+      putWord8 $ 0xA0 .|. fromIntegral len
+    _ | len < 0x10000 -> do
+      putWord8 0xDA
+      putWord16be $ fromIntegral len
+    _ -> do
+      putWord8 0xDB
+      putWord32be $ fromIntegral len
+  putByteString bs
+  where
+    len = B.length bs
+
+putArray :: ObjectPut a => [a] -> Put
+putArray arr = do
+  case len of
+    _ | len <= 15 ->
+      putWord8 $ 0x90 .|. fromIntegral len
+    _ | len < 0x10000 -> do
+      putWord8 0xDC
+      putWord16be $ fromIntegral len
+    _ -> do
+      putWord8 0xDD
+      putWord32be $ fromIntegral len
+  mapM_ put arr
+  where
+    len = length arr
+
+putArrayVector :: ObjectPut a => V.Vector a -> Put
+putArrayVector arr = do
+  case len of
+    _ | len <= 15 ->
+      putWord8 $ 0x90 .|. fromIntegral len
+    _ | len < 0x10000 -> do
+      putWord8 0xDC
+      putWord16be $ fromIntegral len
+    _ -> do
+      putWord8 0xDD
+      putWord32be $ fromIntegral len
+  V.mapM_ put arr
+  where
+    len = V.length arr
+
+putMap :: (ObjectPut k, ObjectPut v) => [(k, v)] -> Put
+putMap m = do
+  case len of
+    _ | len <= 15 ->
+      putWord8 $ 0x80 .|. fromIntegral len
+    _ | len < 0x10000 -> do
+      putWord8 0xDE
+      putWord16be $ fromIntegral len
+    _ -> do
+      putWord8 0xDF
+      putWord32be $ fromIntegral len
+  mapM_ (\(k, v) -> put k >> put v) m
+  where
+    len = length m
+
+putMapVector :: (ObjectPut k, ObjectPut v) => V.Vector (k, v) -> Put
+putMapVector m = do
+  case len of
+    _ | len <= 15 ->
+      putWord8 $ 0x80 .|. fromIntegral len
+    _ | len < 0x10000 -> do
+      putWord8 0xDE
+      putWord16be $ fromIntegral len
+    _ -> do
+      putWord8 0xDF
+      putWord32be $ fromIntegral len
+  V.mapM_ (\(k, v) -> put k >> put v) m
+  where
+    len = V.length m
diff --git a/haskell/src/Data/MessagePack/Stream.hs b/haskell/src/Data/MessagePack/Stream.hs
deleted file mode 100644
index c56fe8d..0000000
--- a/haskell/src/Data/MessagePack/Stream.hs
+++ /dev/null
@@ -1,82 +0,0 @@
---------------------------------------------------------------------
--- |
--- Module    : Data.MessagePack.Stream
--- Copyright : (c) Hideyuki Tanaka, 2009
--- License   : BSD3
---
--- Maintainer:  tanaka.hideyuki@gmail.com
--- Stability :  experimental
--- Portability: portable
---
--- Lazy Stream Serializers and Deserializers
---
---------------------------------------------------------------------
-
-module Data.MessagePack.Stream(
-  unpackObjects,
-  unpackObjectsFromFile,
-  unpackObjectsFromHandle,
-  unpackObjectsFromString,
-  ) where
-
-import Data.ByteString (ByteString)
-import System.IO
-import System.IO.Unsafe
-
-import Data.MessagePack.Base
-import Data.MessagePack.Feed
-
--- | Unpack objects using given feeder.
-unpackObjects :: Feeder -> IO [Object]
-unpackObjects feeder = do
-  up <- newUnpacker defaultInitialBufferSize
-  f up
-  where
-  f up = unsafeInterleaveIO $ do
-    mbo <- unpackOnce up
-    case mbo of
-      Just o -> do
-        os <- f up
-        return $ o:os
-      Nothing ->
-        return []
-
-  unpackOnce up = do
-    resp <- unpackerExecute up
-    case resp of
-      0 -> do
-        r <- feedOnce up
-        if r
-          then unpackOnce up
-          else return Nothing
-      1 -> do
-        obj <- unpackerData up
-        freeZone =<< unpackerReleaseZone up
-        unpackerReset up
-        return $ Just obj
-      _ ->
-        error $ "unpackerExecute fails: " ++ show resp
-
-  feedOnce up = do
-    dat <- feeder
-    case dat of
-      Nothing ->
-        return False
-      Just bs -> do
-        unpackerFeed up bs
-        return True
-
--- | Unpack objects from file.
-unpackObjectsFromFile :: FilePath -> IO [Object]
-unpackObjectsFromFile fname =
-  unpackObjects =<< feederFromFile fname
-
--- | Unpack objects from handle.
-unpackObjectsFromHandle :: Handle -> IO [Object]
-unpackObjectsFromHandle h =
-  unpackObjects =<< feederFromHandle h
-  
--- | Unpack oobjects from given byte sequence.
-unpackObjectsFromString :: ByteString -> IO [Object]
-unpackObjectsFromString bs =
-  unpackObjects =<< feederFromString bs
diff --git a/haskell/test/Monad.hs b/haskell/test/Monad.hs
index 4bee5c5..2ec4093 100644
--- a/haskell/test/Monad.hs
+++ b/haskell/test/Monad.hs
@@ -1,16 +1,21 @@
-import Control.Monad.Trans
+{-# Language OverloadedStrings #-}
+
+import Control.Monad.IO.Class
+import qualified Data.ByteString as B
 import Data.MessagePack
 
 main = do
-  sb <- packToString $ do
+  sb <- return $ packToString $ do
     put [1,2,3::Int]
     put (3.14 :: Double)
-    put "Hoge"
+    put ("Hoge" :: B.ByteString)
   
   print sb
   
-  unpackFromString sb $ do
+  r <- unpackFromString sb $ do
     arr <- get
     dbl <- get
     str <- get
-    liftIO $ print (arr :: [Int], dbl :: Double, str :: String)
+    return (arr :: [Int], dbl :: Double, str :: B.ByteString)
+  
+  print r
diff --git a/haskell/test/Stream.hs b/haskell/test/Stream.hs
deleted file mode 100644
index ce060de..0000000
--- a/haskell/test/Stream.hs
+++ /dev/null
@@ -1,14 +0,0 @@
-import Control.Applicative
-import qualified Data.ByteString as BS
-import Data.MessagePack
-
-main = do
-  sb <- newSimpleBuffer
-  pc <- newPacker sb
-  pack pc [1,2,3::Int]
-  pack pc True
-  pack pc "hoge"
-  bs <- simpleBufferData sb
-  
-  os <- unpackObjectsFromString bs
-  mapM_ print os
diff --git a/haskell/test/Test.hs b/haskell/test/Test.hs
index 4e713ba..1bb551c 100644
--- a/haskell/test/Test.hs
+++ b/haskell/test/Test.hs
@@ -1,36 +1,45 @@
+import Test.Framework
+import Test.Framework.Providers.QuickCheck2
+import Test.QuickCheck
+
 import Control.Monad
+import qualified Data.ByteString.Char8 as B
 import Data.MessagePack
 
-{-
-main = do
-  sb <- newSimpleBuffer
-  pc <- newPacker sb
-  
-  pack pc [(1,2),(2,3),(3::Int,4::Int)]
-  pack pc [4,5,6::Int]
-  pack pc "hoge"
-  
-  bs <- simpleBufferData sb
-  print bs
-  
-  up <- newUnpacker defaultInitialBufferSize
-  
-  unpackerFeed up bs
+mid :: (ObjectGet a, ObjectPut a) => a -> a
+mid = unpack . pack
 
-  let f = do
-        res <- unpackerExecute up
-        when (res==1) $ do
-          obj <- unpackerData up
-          print obj
-          f
-  
-  f
+prop_mid_int a = a == mid a
+  where types = a :: Int
+prop_mid_nil a = a == mid a
+  where types = a :: ()
+prop_mid_bool a = a == mid a
+  where types = a :: Bool
+prop_mid_double a = a == mid a
+  where types = a :: Double
+prop_mid_string a = a == B.unpack (mid (B.pack a))
+  where types = a :: String
+prop_mid_array_int a = a == mid a
+  where types = a :: [Int]
+prop_mid_array_string a = a == map B.unpack (mid (map B.pack a))
+  where types = a :: [String]
+prop_mid_map_int_double a = a == mid a
+  where types = a :: [(Int, Double)]
+prop_mid_map_string_string a = a == map (\(x, y) -> (B.unpack x, B.unpack y)) (mid (map (\(x, y) -> (B.pack x, B.pack y)) a))
+  where types = a :: [(String, String)]
 
-  return ()
--}
+tests =
+  [ testGroup "simple"
+    [ testProperty "int" prop_mid_int
+    , testProperty "nil" prop_mid_nil
+    , testProperty "bool" prop_mid_bool
+    , testProperty "double" prop_mid_double
+    , testProperty "string" prop_mid_string
+    , testProperty "[int]" prop_mid_array_int
+    , testProperty "[string]" prop_mid_array_string
+    , testProperty "[(int, double)]" prop_mid_map_int_double
+    , testProperty "[(string, string)]" prop_mid_map_string_string
+    ]
+  ]
 
-main = do
-  bs <- packb [(1,2),(2,3),(3::Int,4::Int)]
-  print bs
-  dat <- unpackb bs
-  print (dat :: Result [(Int, Int)])
+main = defaultMain tests

From 0368a70dd70a91598507bc7baad8291adc1309fa Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Mon, 6 Sep 2010 13:55:34 +0900
Subject: [PATCH 0662/1648] forgot to add file

---
 haskell/src/Data/MessagePack/Parser.hs | 259 +++++++++++++++++++++++++
 1 file changed, 259 insertions(+)
 create mode 100644 haskell/src/Data/MessagePack/Parser.hs

diff --git a/haskell/src/Data/MessagePack/Parser.hs b/haskell/src/Data/MessagePack/Parser.hs
new file mode 100644
index 0000000..d0cd084
--- /dev/null
+++ b/haskell/src/Data/MessagePack/Parser.hs
@@ -0,0 +1,259 @@
+{-# Language FlexibleInstances #-}
+{-# Language IncoherentInstances #-}
+{-# Language OverlappingInstances #-}
+
+--------------------------------------------------------------------
+-- |
+-- Module    : Data.MessagePack.Parser
+-- Copyright : (c) Hideyuki Tanaka, 2009-2010
+-- License   : BSD3
+--
+-- Maintainer:  tanaka.hideyuki@gmail.com
+-- Stability :  experimental
+-- Portability: portable
+--
+-- MessagePack Deserializer using @Data.Attoparsec@
+--
+--------------------------------------------------------------------
+
+module Data.MessagePack.Parser(
+  -- * MessagePack deserializer
+  ObjectGet(..),
+  ) where
+
+import Control.Monad
+import qualified Data.Attoparsec as A
+import Data.Binary.Get
+import Data.Binary.IEEE754
+import Data.Bits
+import qualified Data.ByteString as B
+import qualified Data.ByteString.Lazy as L
+import Data.Int
+import qualified Data.Vector as V
+import Data.Word
+import Text.Printf
+
+import Data.MessagePack.Object
+
+-- | Deserializable class
+class ObjectGet a where
+  -- | Deserialize a value
+  get :: A.Parser a
+
+instance ObjectGet Int where
+  get = parseInt
+
+instance ObjectGet () where
+  get = parseNil
+
+instance ObjectGet Bool where
+  get = parseBool
+
+instance ObjectGet Double where
+  get = parseDouble
+
+instance ObjectGet B.ByteString where
+  get = parseRAW
+
+instance ObjectGet a => ObjectGet [a] where
+  get = parseArray
+
+instance ObjectGet a => ObjectGet (V.Vector a) where
+  get = parseArrayVector
+
+instance (ObjectGet k, ObjectGet v) => ObjectGet [(k, v)] where
+  get = parseMap
+
+instance (ObjectGet k, ObjectGet v) => ObjectGet (V.Vector (k, v)) where
+  get = parseMapVector
+
+instance ObjectGet Object where
+  get = parseObject
+
+parseInt :: A.Parser Int
+parseInt = do
+  c <- A.anyWord8
+  case c of
+    _ | c .&. 0x80 == 0x00 ->
+      return $ fromIntegral c
+    _ | c .&. 0xE0 == 0xE0 ->
+      return $ fromIntegral (fromIntegral c :: Int8)      
+    0xCC ->
+      return . fromIntegral =<< A.anyWord8
+    0xCD ->
+      return . fromIntegral =<< parseUint16
+    0xCE ->
+      return . fromIntegral =<< parseUint32
+    0xCF ->
+      return . fromIntegral =<< parseUint64
+    0xD0 ->
+      return . fromIntegral =<< parseInt8
+    0xD1 ->
+      return . fromIntegral =<< parseInt16
+    0xD2 ->
+      return . fromIntegral =<< parseInt32
+    0xD3 ->
+      return . fromIntegral =<< parseInt64
+    _ ->
+      fail $ printf "invlid integer tag: 0x%02X" c
+
+parseNil :: A.Parser ()
+parseNil = do
+  _ <- A.word8 0xC0
+  return ()
+
+parseBool :: A.Parser Bool
+parseBool = do
+  c <- A.anyWord8
+  case c of
+    0xC3 ->
+      return True
+    0xC2 ->
+      return False
+    _ ->
+      fail $ printf "invlid bool tag: 0x%02X" c
+
+parseDouble :: A.Parser Double
+parseDouble = do
+  c <- A.anyWord8
+  case c of
+    0xCA ->
+      return . realToFrac . runGet getFloat32be . toLBS =<< A.take 4
+    0xCB ->
+      return . runGet getFloat64be . toLBS =<< A.take 8
+    _ ->
+      fail $ printf "invlid double tag: 0x%02X" c
+
+parseRAW :: A.Parser B.ByteString
+parseRAW = do
+  c <- A.anyWord8
+  case c of
+    _ | c .&. 0xE0 == 0xA0 ->
+      A.take . fromIntegral $ c .&. 0x1F
+    0xDA ->
+      A.take . fromIntegral =<< parseUint16
+    0xDB ->
+      A.take . fromIntegral =<< parseUint32
+    _ ->
+      fail $ printf "invlid raw tag: 0x%02X" c
+  
+parseArray :: ObjectGet a => A.Parser [a]
+parseArray = do
+  c <- A.anyWord8
+  case c of
+    _ | c .&. 0xF0 == 0x90 ->
+      flip replicateM get . fromIntegral $ c .&. 0x0F
+    0xDC ->
+      flip replicateM get . fromIntegral =<< parseUint16
+    0xDD ->
+      flip replicateM get . fromIntegral =<< parseUint32
+    _ ->
+      fail $ printf "invlid array tag: 0x%02X" c
+
+parseArrayVector :: ObjectGet a => A.Parser (V.Vector a)
+parseArrayVector = do
+  c <- A.anyWord8
+  case c of
+    _ | c .&. 0xF0 == 0x90 ->
+      flip V.replicateM get . fromIntegral $ c .&. 0x0F
+    0xDC ->
+      flip V.replicateM get . fromIntegral =<< parseUint16
+    0xDD ->
+      flip V.replicateM get . fromIntegral =<< parseUint32
+    _ ->
+      fail $ printf "invlid array tag: 0x%02X" c
+
+parseMap :: (ObjectGet k, ObjectGet v) => A.Parser [(k, v)]
+parseMap = do
+  c <- A.anyWord8
+  case c of
+    _ | c .&. 0xF0 == 0x80 ->
+      flip replicateM parsePair . fromIntegral $ c .&. 0x0F
+    0xDE ->
+      flip replicateM parsePair . fromIntegral =<< parseUint16
+    0xDF ->
+      flip replicateM parsePair . fromIntegral =<< parseUint32
+    _ ->
+      fail $ printf "invlid map tag: 0x%02X" c
+
+parseMapVector :: (ObjectGet k, ObjectGet v) => A.Parser (V.Vector (k, v))
+parseMapVector = do
+  c <- A.anyWord8
+  case c of
+    _ | c .&. 0xF0 == 0x80 ->
+      flip V.replicateM parsePair . fromIntegral $ c .&. 0x0F
+    0xDE ->
+      flip V.replicateM parsePair . fromIntegral =<< parseUint16
+    0xDF ->
+      flip V.replicateM parsePair . fromIntegral =<< parseUint32
+    _ ->
+      fail $ printf "invlid map tag: 0x%02X" c
+
+parseObject :: A.Parser Object
+parseObject =
+  A.choice
+  [ liftM ObjectInteger parseInt
+  , liftM (const ObjectNil) parseNil
+  , liftM ObjectBool parseBool
+  , liftM ObjectDouble parseDouble
+  , liftM ObjectRAW parseRAW
+  , liftM ObjectArray parseArray
+  , liftM ObjectMap parseMap
+  ]
+
+parsePair :: (ObjectGet k, ObjectGet v) => A.Parser (k, v)
+parsePair = do
+  a <- get
+  b <- get
+  return (a, b)
+
+parseUint16 :: A.Parser Word16
+parseUint16 = do
+  b0 <- A.anyWord8
+  b1 <- A.anyWord8
+  return $ (fromIntegral b0 `shiftL` 8) .|. fromIntegral b1
+
+parseUint32 :: A.Parser Word32
+parseUint32 = do
+  b0 <- A.anyWord8
+  b1 <- A.anyWord8
+  b2 <- A.anyWord8
+  b3 <- A.anyWord8
+  return $ (fromIntegral b0 `shiftL` 24) .|.
+           (fromIntegral b1 `shiftL` 16) .|.
+           (fromIntegral b2 `shiftL` 8) .|.
+           fromIntegral b3
+
+parseUint64 :: A.Parser Word64
+parseUint64 = do
+  b0 <- A.anyWord8
+  b1 <- A.anyWord8
+  b2 <- A.anyWord8
+  b3 <- A.anyWord8
+  b4 <- A.anyWord8
+  b5 <- A.anyWord8
+  b6 <- A.anyWord8
+  b7 <- A.anyWord8
+  return $ (fromIntegral b0 `shiftL` 56) .|.
+           (fromIntegral b1 `shiftL` 48) .|.
+           (fromIntegral b2 `shiftL` 40) .|.
+           (fromIntegral b3 `shiftL` 32) .|.
+           (fromIntegral b4 `shiftL` 24) .|.
+           (fromIntegral b5 `shiftL` 16) .|.
+           (fromIntegral b6 `shiftL` 8) .|.
+           fromIntegral b7
+
+parseInt8 :: A.Parser Int8
+parseInt8 = return . fromIntegral =<< A.anyWord8
+
+parseInt16 :: A.Parser Int16
+parseInt16 = return . fromIntegral =<< parseUint16
+
+parseInt32 :: A.Parser Int32
+parseInt32 = return . fromIntegral =<< parseUint32
+
+parseInt64 :: A.Parser Int64
+parseInt64 = return . fromIntegral =<< parseUint64
+
+toLBS :: B.ByteString -> L.ByteString
+toLBS bs = L.fromChunks [bs]

From 209d8d058c17b1ee92f13942a804eb2868191118 Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Mon, 6 Sep 2010 13:57:47 +0900
Subject: [PATCH 0663/1648] forgot to remove file

---
 haskell/src/Data/MessagePack/Packer.hs | 147 -------------------------
 1 file changed, 147 deletions(-)
 delete mode 100644 haskell/src/Data/MessagePack/Packer.hs

diff --git a/haskell/src/Data/MessagePack/Packer.hs b/haskell/src/Data/MessagePack/Packer.hs
deleted file mode 100644
index 9c10f5e..0000000
--- a/haskell/src/Data/MessagePack/Packer.hs
+++ /dev/null
@@ -1,147 +0,0 @@
-{-# Language FlexibleInstances #-}
-{-# Language OverlappingInstances #-}
-
-module Data.MessagePack.Packer(
-  ObjectPut(..),
-  ) where
-
-import Data.Binary.Put
-import Data.Binary.IEEE754
-import Data.Bits
-import qualified Data.ByteString as B
-
-import Data.MessagePack.Object
-
-class ObjectPut a where
-  put :: a -> Put
-
-instance ObjectPut Object where
-  put = putObject
-
-instance ObjectPut Int where
-  put = putInteger
-
-instance ObjectPut () where
-  put _ = putNil
-
-instance ObjectPut Bool where
-  put = putBool
-
-instance ObjectPut Double where
-  put = putDouble
-
-instance ObjectPut B.ByteString where
-  put = putRAW
-
-instance ObjectPut a => ObjectPut [a] where
-  put = putArray
-
-instance (ObjectPut k, ObjectPut v) => ObjectPut [(k, v)] where
-  put = putMap
-
-putObject :: Object -> Put
-putObject obj =
-  case obj of
-    ObjectInteger n ->
-      putInteger n
-    ObjectNil ->
-      putNil
-    ObjectBool b ->
-      putBool b
-    ObjectDouble d ->
-      putDouble d
-    ObjectRAW raw ->
-      putRAW raw
-    ObjectArray arr ->
-      putArray arr
-    ObjectMap m ->
-      putMap m
-
-putInteger :: Int -> Put      
-putInteger n =
-  case n of
-    _ | n >= 0 && n <= 127 ->
-      putWord8 $ fromIntegral n
-    _ | n >= -32 && n <= -1 ->
-      putWord8 $ fromIntegral n
-    _ | n >= 0 && n < 0x100 -> do
-      putWord8 0xCC
-      putWord8 $ fromIntegral n
-    _ | n >= 0 && n < 0x10000 -> do
-      putWord8 0xCD
-      putWord16be $ fromIntegral n
-    _ | n >= 0 && n < 0x100000000 -> do
-      putWord8 0xCE
-      putWord32be $ fromIntegral n
-    _ | n >= 0 -> do
-      putWord8 0xCF
-      putWord64be $ fromIntegral n
-    _ | n >= -0x100 -> do
-      putWord8 0xD0
-      putWord8 $ fromIntegral n
-    _ | n >= -0x10000 -> do
-      putWord8 0xD1
-      putWord16be $ fromIntegral n
-    _ | n >= -0x100000000 -> do
-      putWord8 0xD2
-      putWord32be $ fromIntegral n
-    _ -> do
-      putWord8 0xD3
-      putWord64be $ fromIntegral n
-
-putNil :: Put
-putNil = putWord8 0xC0
-
-putBool :: Bool -> Put
-putBool True = putWord8 0xC3
-putBool False = putWord8 0xC2
-
-putDouble :: Double -> Put
-putDouble d = do
-  putWord8 0xCB
-  putFloat64be d
-
-putRAW :: B.ByteString -> Put
-putRAW bs = do
-  case len of
-    _ | len <= 31 -> do
-      putWord8 $ 0xA0 .|. fromIntegral len
-    _ | len < 0x10000 -> do
-      putWord8 0xDA
-      putWord16be $ fromIntegral len
-    _ -> do
-      putWord8 0xDB
-      putWord32be $ fromIntegral len
-  putByteString bs
-  where
-    len = B.length bs
-
-putArray :: ObjectPut a => [a] -> Put
-putArray arr = do
-  case len of
-    _ | len <= 15 ->
-      putWord8 $ 0x90 .|. fromIntegral len
-    _ | len < 0x10000 -> do
-      putWord8 0xDC
-      putWord16be $ fromIntegral len
-    _ -> do
-      putWord8 0xDD
-      putWord32be $ fromIntegral len
-  mapM_ put arr
-  where
-    len = length arr
-
-putMap :: (ObjectPut k, ObjectPut v) => [(k, v)] -> Put
-putMap m = do
-  case len of
-    _ | len <= 15 ->
-      putWord8 $ 0x80 .|. fromIntegral len
-    _ | len < 0x10000 -> do
-      putWord8 0xDE
-      putWord16be $ fromIntegral len
-    _ -> do
-      putWord8 0xDF
-      putWord16be $ fromIntegral len
-  mapM_ (\(k, v) -> put k >> put v) m
-  where
-    len = length m

From 799935e44c6f27e81d780b324dd69bdbd71066d5 Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Mon, 6 Sep 2010 14:03:47 +0900
Subject: [PATCH 0664/1648] haskel: incr version and update infos.

---
 haskell/msgpack.cabal | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal
index 8346c1f..18ae3d8 100644
--- a/haskell/msgpack.cabal
+++ b/haskell/msgpack.cabal
@@ -1,15 +1,15 @@
 Name:               msgpack
-Version:            0.3.0
+Version:            0.3.1
 Synopsis:           A Haskell binding to MessagePack
 Description:
-  A Haskell binding to MessagePack 
+  A Haskell binding to MessagePack 
 
 License:            BSD3
 License-File:       LICENSE
 Category:           Data
 Author:             Hideyuki Tanaka
 Maintainer:         Hideyuki Tanaka 
-Homepage:           http://github.com/tanakh/hsmsgpack
+Homepage:           http://github.com/msgpack/msgpack
 Stability:          Experimental
 Tested-with:        GHC == 6.12.3
 Cabal-Version:      >= 1.2

From 802589516870df83cf209191e234266b09b1abee Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Sun, 5 Sep 2010 16:20:37 +0900
Subject: [PATCH 0665/1648] Checking in changes prior to tagging of version
 0.16_01.

Changelog diff is:
---
 perl/MANIFEST.SKIP           | 2 ++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP
index f634035..71a24e5 100644
--- a/perl/MANIFEST.SKIP
+++ b/perl/MANIFEST.SKIP
@@ -23,3 +23,5 @@
 \.o$
 \.bs$
 ^Data-MessagePack-[0-9.]+/
+^\.testenv/test_pp.pl
+^ppport.h$
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index d53ff22..4da67ff 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.16';
+our $VERSION = '0.16_01';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };

From e781831032c9091ab7e90bbbd9560828a8b69a30 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Mon, 6 Sep 2010 14:19:31 +0900
Subject: [PATCH 0666/1648] upgraded docs

---
 perl/README                  | 16 +++++++++++++---
 perl/lib/Data/MessagePack.pm | 15 ++++++++++++---
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/perl/README b/perl/README
index 31aae99..2ef686c 100644
--- a/perl/README
+++ b/perl/README
@@ -1,12 +1,16 @@
 NAME
-    Data::MessagePack - messagepack
+    Data::MessagePack - MessagePack serialising/deserialising
 
 SYNOPSIS
         my $packed = Data::MessagePack->pack($dat);
         my $unpacked = Data::MessagePack->unpack($dat);
 
 DESCRIPTION
-    Data::MessagePack is a binary packer for perl.
+    This module converts Perl data structures to MessagePack and vice versa.
+
+    MessagePack is a binary-based efficient object serialization format. It
+    enables to exchange structured objects between many languages like JSON.
+    But unlike JSON, it is very fast and small.
 
 METHODS
     my $packed = Data::MessagePack->pack($data);
@@ -22,13 +26,19 @@ Configuration Variables
 AUTHORS
     Tokuhiro Matsuno
 
+    Makamaka Hannyaharamitu
+
 THANKS TO
     Jun Kuriyama
 
+    Dan Kogai
+
+    FURUHASHI Sadayuki
+
 LICENSE
     This library is free software; you can redistribute it and/or modify it
     under the same terms as Perl itself.
 
 SEE ALSO
-    
+    
 
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 4da67ff..ecdc1e4 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -30,7 +30,7 @@ __END__
 
 =head1 NAME
 
-Data::MessagePack - messagepack
+Data::MessagePack - MessagePack serialising/deserialising
 
 =head1 SYNOPSIS
 
@@ -39,7 +39,10 @@ Data::MessagePack - messagepack
 
 =head1 DESCRIPTION
 
-Data::MessagePack is a binary packer for perl.
+This module converts Perl data structures to MessagePack and vice versa.
+
+MessagePack is a binary-based efficient object serialization format.
+It enables to exchange structured objects between many languages like JSON. But unlike JSON, it is very fast and small.
 
 =head1 METHODS
 
@@ -69,10 +72,16 @@ Pack the string as int when the value looks like int(EXPERIMENTAL).
 
 Tokuhiro Matsuno
 
+Makamaka Hannyaharamitu
+
 =head1 THANKS TO
 
 Jun Kuriyama
 
+Dan Kogai
+
+FURUHASHI Sadayuki
+
 =head1 LICENSE
 
 This library is free software; you can redistribute it and/or modify
@@ -81,5 +90,5 @@ it under the same terms as Perl itself.
 
 =head1 SEE ALSO
 
-L
+L
 

From c7555f1c3c471278b320db5ca71e5afdbcb52867 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Mon, 6 Sep 2010 14:31:53 +0900
Subject: [PATCH 0667/1648] Perl: added link to git repository.

---
 perl/Makefile.PL | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index 783e658..7958bc6 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -64,7 +64,7 @@ if($Module::Install::AUTHOR) {
     postamble qq{test :: test_pp\n\n};
 }
 
-auto_set_repository();
+repository('http://github.com/msgpack/msgpack');
 auto_include;
 WriteAll;
 

From 9281dba89672862ddf27384909264f4bd6ec12e8 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Mon, 6 Sep 2010 14:34:04 +0900
Subject: [PATCH 0668/1648] Checking in changes prior to tagging of version
 0.16_02.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index 9b061cf..68b58ba 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.16_02
+
+    - document enhancement(tokuhirom)
+    - M::I::XSUtil 0.26 is broken. use 0.27.
+
 0.16_01

     - added PP version (used in cases PERL_DATA_MESSAGEPACK=pp or fail to load XS).
---
 perl/Changes                 | 5 +++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index 9b061cf..68b58ba 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.16_02
+
+    - document enhancement(tokuhirom)
+    - M::I::XSUtil 0.26 is broken. use 0.27.
+
 0.16_01
 
     - added PP version (used in cases PERL_DATA_MESSAGEPACK=pp or fail to load XS).
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index ecdc1e4..79cc531 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.16_01';
+our $VERSION = '0.16_02';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };

From 8b90968cb111be903421083d8f3bebbef23e79c7 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Mon, 6 Sep 2010 14:34:48 +0900
Subject: [PATCH 0669/1648] Checking in changes prior to tagging of version
 0.16_03.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index 68b58ba..a4a3e36 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.16_03
+
+    - no feature changes
+
 0.16_02

     - document enhancement(tokuhirom)
---
 perl/Changes                 | 4 ++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index 68b58ba..a4a3e36 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.16_03
+
+    - no feature changes
+
 0.16_02
 
     - document enhancement(tokuhirom)
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 79cc531..b143e4a 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.16_02';
+our $VERSION = '0.16_03';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };

From c5afe7a5739fa48d207d85403771de4a526ff437 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Mon, 6 Sep 2010 14:35:41 +0900
Subject: [PATCH 0670/1648] Checking in changes prior to tagging of version
 0.16_04.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index a4a3e36..7910882 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,4 +1,4 @@
-0.16_03
+0.16_04

     - no feature changes
---
 perl/Changes                 | 2 +-
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/perl/Changes b/perl/Changes
index a4a3e36..7910882 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,4 +1,4 @@
-0.16_03
+0.16_04
 
     - no feature changes
 
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index b143e4a..b08bac2 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.16_03';
+our $VERSION = '0.16_04';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };

From aca2ba13c2f3ce3bc43897beb0a4a8529bab7a03 Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Mon, 6 Sep 2010 15:37:55 +0900
Subject: [PATCH 0671/1648] haskell: refactoring

---
 haskell/src/Data/MessagePack/Parser.hs | 246 +++++++++++--------------
 haskell/src/Data/MessagePack/Put.hs    | 232 +++++++++--------------
 2 files changed, 195 insertions(+), 283 deletions(-)

diff --git a/haskell/src/Data/MessagePack/Parser.hs b/haskell/src/Data/MessagePack/Parser.hs
index d0cd084..312e95f 100644
--- a/haskell/src/Data/MessagePack/Parser.hs
+++ b/haskell/src/Data/MessagePack/Parser.hs
@@ -40,166 +40,113 @@ class ObjectGet a where
   -- | Deserialize a value
   get :: A.Parser a
 
+instance ObjectGet Object where
+  get =
+    A.choice
+    [ liftM ObjectInteger get
+    , liftM (\() -> ObjectNil) get
+    , liftM ObjectBool get
+    , liftM ObjectDouble get
+    , liftM ObjectRAW get
+    , liftM ObjectArray get
+    , liftM ObjectMap get
+    ]
+
 instance ObjectGet Int where
-  get = parseInt
+  get = do
+    c <- A.anyWord8
+    case c of
+      _ | c .&. 0x80 == 0x00 ->
+        return $ fromIntegral c
+      _ | c .&. 0xE0 == 0xE0 ->
+        return $ fromIntegral (fromIntegral c :: Int8)
+      0xCC ->
+        return . fromIntegral =<< A.anyWord8
+      0xCD ->
+        return . fromIntegral =<< parseUint16
+      0xCE ->
+        return . fromIntegral =<< parseUint32
+      0xCF ->
+        return . fromIntegral =<< parseUint64
+      0xD0 ->
+        return . fromIntegral =<< parseInt8
+      0xD1 ->
+        return . fromIntegral =<< parseInt16
+      0xD2 ->
+        return . fromIntegral =<< parseInt32
+      0xD3 ->
+        return . fromIntegral =<< parseInt64
+      _ ->
+        fail $ printf "invlid integer tag: 0x%02X" c
 
 instance ObjectGet () where
-  get = parseNil
+  get = do
+    c <- A.anyWord8
+    case c of
+      0xC0 ->
+        return ()
+      _ ->
+        fail $ printf "invlid nil tag: 0x%02X" c
 
 instance ObjectGet Bool where
-  get = parseBool
+  get = do
+    c <- A.anyWord8
+    case c of
+      0xC3 ->
+        return True
+      0xC2 ->
+        return False
+      _ ->
+        fail $ printf "invlid bool tag: 0x%02X" c
 
 instance ObjectGet Double where
-  get = parseDouble
+  get = do
+    c <- A.anyWord8
+    case c of
+      0xCA ->
+        return . realToFrac . runGet getFloat32be . toLBS =<< A.take 4
+      0xCB ->
+        return . runGet getFloat64be . toLBS =<< A.take 8
+      _ ->
+        fail $ printf "invlid double tag: 0x%02X" c
 
 instance ObjectGet B.ByteString where
-  get = parseRAW
+  get = do
+    c <- A.anyWord8
+    case c of
+      _ | c .&. 0xE0 == 0xA0 ->
+        A.take . fromIntegral $ c .&. 0x1F
+      0xDA ->
+        A.take . fromIntegral =<< parseUint16
+      0xDB ->
+        A.take . fromIntegral =<< parseUint32
+      _ ->
+        fail $ printf "invlid raw tag: 0x%02X" c
 
 instance ObjectGet a => ObjectGet [a] where
-  get = parseArray
+  get = parseArray (flip replicateM get)
 
 instance ObjectGet a => ObjectGet (V.Vector a) where
-  get = parseArrayVector
+  get = parseArray (flip V.replicateM get)
+
+parseArray :: (Int -> A.Parser a) -> A.Parser a
+parseArray aget = do
+  c <- A.anyWord8
+  case c of
+    _ | c .&. 0xF0 == 0x90 ->
+      aget . fromIntegral $ c .&. 0x0F
+    0xDC ->
+      aget . fromIntegral =<< parseUint16
+    0xDD ->
+      aget . fromIntegral =<< parseUint32
+    _ ->
+      fail $ printf "invlid array tag: 0x%02X" c
 
 instance (ObjectGet k, ObjectGet v) => ObjectGet [(k, v)] where
-  get = parseMap
+  get = parseMap (flip replicateM parsePair)
 
 instance (ObjectGet k, ObjectGet v) => ObjectGet (V.Vector (k, v)) where
-  get = parseMapVector
-
-instance ObjectGet Object where
-  get = parseObject
-
-parseInt :: A.Parser Int
-parseInt = do
-  c <- A.anyWord8
-  case c of
-    _ | c .&. 0x80 == 0x00 ->
-      return $ fromIntegral c
-    _ | c .&. 0xE0 == 0xE0 ->
-      return $ fromIntegral (fromIntegral c :: Int8)      
-    0xCC ->
-      return . fromIntegral =<< A.anyWord8
-    0xCD ->
-      return . fromIntegral =<< parseUint16
-    0xCE ->
-      return . fromIntegral =<< parseUint32
-    0xCF ->
-      return . fromIntegral =<< parseUint64
-    0xD0 ->
-      return . fromIntegral =<< parseInt8
-    0xD1 ->
-      return . fromIntegral =<< parseInt16
-    0xD2 ->
-      return . fromIntegral =<< parseInt32
-    0xD3 ->
-      return . fromIntegral =<< parseInt64
-    _ ->
-      fail $ printf "invlid integer tag: 0x%02X" c
-
-parseNil :: A.Parser ()
-parseNil = do
-  _ <- A.word8 0xC0
-  return ()
-
-parseBool :: A.Parser Bool
-parseBool = do
-  c <- A.anyWord8
-  case c of
-    0xC3 ->
-      return True
-    0xC2 ->
-      return False
-    _ ->
-      fail $ printf "invlid bool tag: 0x%02X" c
-
-parseDouble :: A.Parser Double
-parseDouble = do
-  c <- A.anyWord8
-  case c of
-    0xCA ->
-      return . realToFrac . runGet getFloat32be . toLBS =<< A.take 4
-    0xCB ->
-      return . runGet getFloat64be . toLBS =<< A.take 8
-    _ ->
-      fail $ printf "invlid double tag: 0x%02X" c
-
-parseRAW :: A.Parser B.ByteString
-parseRAW = do
-  c <- A.anyWord8
-  case c of
-    _ | c .&. 0xE0 == 0xA0 ->
-      A.take . fromIntegral $ c .&. 0x1F
-    0xDA ->
-      A.take . fromIntegral =<< parseUint16
-    0xDB ->
-      A.take . fromIntegral =<< parseUint32
-    _ ->
-      fail $ printf "invlid raw tag: 0x%02X" c
-  
-parseArray :: ObjectGet a => A.Parser [a]
-parseArray = do
-  c <- A.anyWord8
-  case c of
-    _ | c .&. 0xF0 == 0x90 ->
-      flip replicateM get . fromIntegral $ c .&. 0x0F
-    0xDC ->
-      flip replicateM get . fromIntegral =<< parseUint16
-    0xDD ->
-      flip replicateM get . fromIntegral =<< parseUint32
-    _ ->
-      fail $ printf "invlid array tag: 0x%02X" c
-
-parseArrayVector :: ObjectGet a => A.Parser (V.Vector a)
-parseArrayVector = do
-  c <- A.anyWord8
-  case c of
-    _ | c .&. 0xF0 == 0x90 ->
-      flip V.replicateM get . fromIntegral $ c .&. 0x0F
-    0xDC ->
-      flip V.replicateM get . fromIntegral =<< parseUint16
-    0xDD ->
-      flip V.replicateM get . fromIntegral =<< parseUint32
-    _ ->
-      fail $ printf "invlid array tag: 0x%02X" c
-
-parseMap :: (ObjectGet k, ObjectGet v) => A.Parser [(k, v)]
-parseMap = do
-  c <- A.anyWord8
-  case c of
-    _ | c .&. 0xF0 == 0x80 ->
-      flip replicateM parsePair . fromIntegral $ c .&. 0x0F
-    0xDE ->
-      flip replicateM parsePair . fromIntegral =<< parseUint16
-    0xDF ->
-      flip replicateM parsePair . fromIntegral =<< parseUint32
-    _ ->
-      fail $ printf "invlid map tag: 0x%02X" c
-
-parseMapVector :: (ObjectGet k, ObjectGet v) => A.Parser (V.Vector (k, v))
-parseMapVector = do
-  c <- A.anyWord8
-  case c of
-    _ | c .&. 0xF0 == 0x80 ->
-      flip V.replicateM parsePair . fromIntegral $ c .&. 0x0F
-    0xDE ->
-      flip V.replicateM parsePair . fromIntegral =<< parseUint16
-    0xDF ->
-      flip V.replicateM parsePair . fromIntegral =<< parseUint32
-    _ ->
-      fail $ printf "invlid map tag: 0x%02X" c
-
-parseObject :: A.Parser Object
-parseObject =
-  A.choice
-  [ liftM ObjectInteger parseInt
-  , liftM (const ObjectNil) parseNil
-  , liftM ObjectBool parseBool
-  , liftM ObjectDouble parseDouble
-  , liftM ObjectRAW parseRAW
-  , liftM ObjectArray parseArray
-  , liftM ObjectMap parseMap
-  ]
+  get = parseMap (flip V.replicateM parsePair)
 
 parsePair :: (ObjectGet k, ObjectGet v) => A.Parser (k, v)
 parsePair = do
@@ -207,6 +154,19 @@ parsePair = do
   b <- get
   return (a, b)
 
+parseMap :: (Int -> A.Parser a) -> A.Parser a
+parseMap aget = do
+  c <- A.anyWord8
+  case c of
+    _ | c .&. 0xF0 == 0x80 ->
+      aget . fromIntegral $ c .&. 0x0F
+    0xDE ->
+      aget . fromIntegral =<< parseUint16
+    0xDF ->
+      aget . fromIntegral =<< parseUint32
+    _ ->
+      fail $ printf "invlid map tag: 0x%02X" c
+
 parseUint16 :: A.Parser Word16
 parseUint16 = do
   b0 <- A.anyWord8
diff --git a/haskell/src/Data/MessagePack/Put.hs b/haskell/src/Data/MessagePack/Put.hs
index 8d0af2b..95582dd 100644
--- a/haskell/src/Data/MessagePack/Put.hs
+++ b/haskell/src/Data/MessagePack/Put.hs
@@ -35,168 +35,120 @@ class ObjectPut a where
   put :: a -> Put
 
 instance ObjectPut Object where
-  put = putObject
+  put obj =
+    case obj of
+      ObjectInteger n ->
+        put n
+      ObjectNil ->
+        put ()
+      ObjectBool b ->
+        put b
+      ObjectDouble d ->
+        put d
+      ObjectRAW raw ->
+        put raw
+      ObjectArray arr ->
+        put arr
+      ObjectMap m ->
+        put m
 
 instance ObjectPut Int where
-  put = putInteger
+  put n =
+    case n of
+      _ | n >= 0 && n <= 127 ->
+        putWord8 $ fromIntegral n
+      _ | n >= -32 && n <= -1 ->
+        putWord8 $ fromIntegral n
+      _ | n >= 0 && n < 0x100 -> do
+        putWord8 0xCC
+        putWord8 $ fromIntegral n
+      _ | n >= 0 && n < 0x10000 -> do
+        putWord8 0xCD
+        putWord16be $ fromIntegral n
+      _ | n >= 0 && n < 0x100000000 -> do
+        putWord8 0xCE
+        putWord32be $ fromIntegral n
+      _ | n >= 0 -> do
+        putWord8 0xCF
+        putWord64be $ fromIntegral n
+      _ | n >= -0x80 -> do
+        putWord8 0xD0
+        putWord8 $ fromIntegral n
+      _ | n >= -0x8000 -> do
+        putWord8 0xD1
+        putWord16be $ fromIntegral n
+      _ | n >= -0x80000000 -> do
+        putWord8 0xD2
+        putWord32be $ fromIntegral n
+      _ -> do
+        putWord8 0xD3
+        putWord64be $ fromIntegral n
 
 instance ObjectPut () where
-  put _ = putNil
+  put _ = 
+    putWord8 0xC0
 
 instance ObjectPut Bool where
-  put = putBool
+  put True = putWord8 0xC3
+  put False = putWord8 0xC2
 
 instance ObjectPut Double where
-  put = putDouble
+  put d = do
+    putWord8 0xCB
+    putFloat64be d
 
 instance ObjectPut B.ByteString where
-  put = putRAW
+  put bs = do
+    case len of
+      _ | len <= 31 -> do
+        putWord8 $ 0xA0 .|. fromIntegral len
+      _ | len < 0x10000 -> do
+        putWord8 0xDA
+        putWord16be $ fromIntegral len
+      _ -> do
+        putWord8 0xDB
+        putWord32be $ fromIntegral len
+    putByteString bs
+    where
+      len = B.length bs
 
 instance ObjectPut a => ObjectPut [a] where
-  put = putArray
+  put = putArray length (mapM_ put)
 
 instance ObjectPut a => ObjectPut (V.Vector a) where
-  put = putArrayVector
+  put = putArray V.length (V.mapM_ put)
+
+putArray :: (a -> Int) -> (a -> Put) -> a -> Put
+putArray lf pf arr = do
+  case lf arr of
+    len | len <= 15 ->
+      putWord8 $ 0x90 .|. fromIntegral len
+    len | len < 0x10000 -> do
+      putWord8 0xDC
+      putWord16be $ fromIntegral len
+    len -> do
+      putWord8 0xDD
+      putWord32be $ fromIntegral len
+  pf arr
 
 instance (ObjectPut k, ObjectPut v) => ObjectPut [(k, v)] where
-  put = putMap
+  put = putMap length (mapM_ putPair)
 
 instance (ObjectPut k, ObjectPut v) => ObjectPut (V.Vector (k, v)) where
-  put = putMapVector
+  put = putMap V.length (V.mapM_ putPair)
 
-putObject :: Object -> Put
-putObject obj =
-  case obj of
-    ObjectInteger n ->
-      putInteger n
-    ObjectNil ->
-      putNil
-    ObjectBool b ->
-      putBool b
-    ObjectDouble d ->
-      putDouble d
-    ObjectRAW raw ->
-      putRAW raw
-    ObjectArray arr ->
-      putArray arr
-    ObjectMap m ->
-      putMap m
+putPair :: (ObjectPut a, ObjectPut b) => (a, b) -> Put
+putPair (a, b) = put a >> put b
 
-putInteger :: Int -> Put      
-putInteger n =
-  case n of
-    _ | n >= 0 && n <= 127 ->
-      putWord8 $ fromIntegral n
-    _ | n >= -32 && n <= -1 ->
-      putWord8 $ fromIntegral n
-    _ | n >= 0 && n < 0x100 -> do
-      putWord8 0xCC
-      putWord8 $ fromIntegral n
-    _ | n >= 0 && n < 0x10000 -> do
-      putWord8 0xCD
-      putWord16be $ fromIntegral n
-    _ | n >= 0 && n < 0x100000000 -> do
-      putWord8 0xCE
-      putWord32be $ fromIntegral n
-    _ | n >= 0 -> do
-      putWord8 0xCF
-      putWord64be $ fromIntegral n
-    _ | n >= -0x80 -> do
-      putWord8 0xD0
-      putWord8 $ fromIntegral n
-    _ | n >= -0x8000 -> do
-      putWord8 0xD1
-      putWord16be $ fromIntegral n
-    _ | n >= -0x80000000 -> do
-      putWord8 0xD2
-      putWord32be $ fromIntegral n
-    _ -> do
-      putWord8 0xD3
-      putWord64be $ fromIntegral n
-
-putNil :: Put
-putNil = putWord8 0xC0
-
-putBool :: Bool -> Put
-putBool True = putWord8 0xC3
-putBool False = putWord8 0xC2
-
-putDouble :: Double -> Put
-putDouble d = do
-  putWord8 0xCB
-  putFloat64be d
-
-putRAW :: B.ByteString -> Put
-putRAW bs = do
-  case len of
-    _ | len <= 31 -> do
-      putWord8 $ 0xA0 .|. fromIntegral len
-    _ | len < 0x10000 -> do
-      putWord8 0xDA
-      putWord16be $ fromIntegral len
-    _ -> do
-      putWord8 0xDB
-      putWord32be $ fromIntegral len
-  putByteString bs
-  where
-    len = B.length bs
-
-putArray :: ObjectPut a => [a] -> Put
-putArray arr = do
-  case len of
-    _ | len <= 15 ->
-      putWord8 $ 0x90 .|. fromIntegral len
-    _ | len < 0x10000 -> do
-      putWord8 0xDC
-      putWord16be $ fromIntegral len
-    _ -> do
-      putWord8 0xDD
-      putWord32be $ fromIntegral len
-  mapM_ put arr
-  where
-    len = length arr
-
-putArrayVector :: ObjectPut a => V.Vector a -> Put
-putArrayVector arr = do
-  case len of
-    _ | len <= 15 ->
-      putWord8 $ 0x90 .|. fromIntegral len
-    _ | len < 0x10000 -> do
-      putWord8 0xDC
-      putWord16be $ fromIntegral len
-    _ -> do
-      putWord8 0xDD
-      putWord32be $ fromIntegral len
-  V.mapM_ put arr
-  where
-    len = V.length arr
-
-putMap :: (ObjectPut k, ObjectPut v) => [(k, v)] -> Put
-putMap m = do
-  case len of
-    _ | len <= 15 ->
+putMap :: (a -> Int) -> (a -> Put) -> a -> Put
+putMap lf pf m = do
+  case lf m of
+    len | len <= 15 ->
       putWord8 $ 0x80 .|. fromIntegral len
-    _ | len < 0x10000 -> do
+    len | len < 0x10000 -> do
       putWord8 0xDE
       putWord16be $ fromIntegral len
-    _ -> do
+    len -> do
       putWord8 0xDF
       putWord32be $ fromIntegral len
-  mapM_ (\(k, v) -> put k >> put v) m
-  where
-    len = length m
-
-putMapVector :: (ObjectPut k, ObjectPut v) => V.Vector (k, v) -> Put
-putMapVector m = do
-  case len of
-    _ | len <= 15 ->
-      putWord8 $ 0x80 .|. fromIntegral len
-    _ | len < 0x10000 -> do
-      putWord8 0xDE
-      putWord16be $ fromIntegral len
-    _ -> do
-      putWord8 0xDF
-      putWord32be $ fromIntegral len
-  V.mapM_ (\(k, v) -> put k >> put v) m
-  where
-    len = V.length m
+  pf m

From 9e50ba6ec6f48071a5cc31b44864194446b9aa6f Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Mon, 6 Sep 2010 16:33:36 +0900
Subject: [PATCH 0672/1648] haskell: instance tupples and String and lazy
 ByteString

---
 haskell/src/Data/MessagePack/Parser.hs | 73 ++++++++++++++++++++++----
 haskell/src/Data/MessagePack/Put.hs    | 68 +++++++++++++++++++-----
 haskell/test/Test.hs                   | 25 +++++++--
 3 files changed, 139 insertions(+), 27 deletions(-)

diff --git a/haskell/src/Data/MessagePack/Parser.hs b/haskell/src/Data/MessagePack/Parser.hs
index 312e95f..200ad96 100644
--- a/haskell/src/Data/MessagePack/Parser.hs
+++ b/haskell/src/Data/MessagePack/Parser.hs
@@ -1,6 +1,7 @@
 {-# Language FlexibleInstances #-}
 {-# Language IncoherentInstances #-}
 {-# Language OverlappingInstances #-}
+{-# Language TypeSynonymInstances #-}
 
 --------------------------------------------------------------------
 -- |
@@ -27,6 +28,7 @@ import Data.Binary.Get
 import Data.Binary.IEEE754
 import Data.Bits
 import qualified Data.ByteString as B
+import qualified Data.ByteString.Char8 as B8
 import qualified Data.ByteString.Lazy as L
 import Data.Int
 import qualified Data.Vector as V
@@ -110,18 +112,27 @@ instance ObjectGet Double where
       _ ->
         fail $ printf "invlid double tag: 0x%02X" c
 
+instance ObjectGet String where
+  get = parseString (\n -> return . B8.unpack =<< A.take n)
+
 instance ObjectGet B.ByteString where
-  get = do
-    c <- A.anyWord8
-    case c of
-      _ | c .&. 0xE0 == 0xA0 ->
-        A.take . fromIntegral $ c .&. 0x1F
-      0xDA ->
-        A.take . fromIntegral =<< parseUint16
-      0xDB ->
-        A.take . fromIntegral =<< parseUint32
-      _ ->
-        fail $ printf "invlid raw tag: 0x%02X" c
+  get = parseString A.take
+
+instance ObjectGet L.ByteString where
+  get = parseString (\n -> do bs <- A.take n; return $ L.fromChunks [bs])
+
+parseString :: (Int -> A.Parser a) -> A.Parser a
+parseString aget = do
+  c <- A.anyWord8
+  case c of
+    _ | c .&. 0xE0 == 0xA0 ->
+      aget . fromIntegral $ c .&. 0x1F
+    0xDA ->
+      aget . fromIntegral =<< parseUint16
+    0xDB ->
+      aget . fromIntegral =<< parseUint32
+    _ ->
+      fail $ printf "invlid raw tag: 0x%02X" c
 
 instance ObjectGet a => ObjectGet [a] where
   get = parseArray (flip replicateM get)
@@ -129,6 +140,46 @@ instance ObjectGet a => ObjectGet [a] where
 instance ObjectGet a => ObjectGet (V.Vector a) where
   get = parseArray (flip V.replicateM get)
 
+instance (ObjectGet a1, ObjectGet a2) => ObjectGet (a1, a2) where
+  get = parseArray f where
+    f 2 = get >>= \a1 -> get >>= \a2 -> return (a1, a2)
+    f n = fail $ printf "wrong tupple size: expected 2 but got " n
+
+instance (ObjectGet a1, ObjectGet a2, ObjectGet a3) => ObjectGet (a1, a2, a3) where
+  get = parseArray f where
+    f 3 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> return (a1, a2, a3)
+    f n = fail $ printf "wrong tupple size: expected 3 but got " n
+
+instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4) => ObjectGet (a1, a2, a3, a4) where
+  get = parseArray f where
+    f 4 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> return (a1, a2, a3, a4)
+    f n = fail $ printf "wrong tupple size: expected 4 but got " n
+
+instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5) => ObjectGet (a1, a2, a3, a4, a5) where
+  get = parseArray f where
+    f 5 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> return (a1, a2, a3, a4, a5)
+    f n = fail $ printf "wrong tupple size: expected 5 but got " n
+
+instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6) => ObjectGet (a1, a2, a3, a4, a5, a6) where
+  get = parseArray f where
+    f 6 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> return (a1, a2, a3, a4, a5, a6)
+    f n = fail $ printf "wrong tupple size: expected 6 but got " n
+
+instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7) => ObjectGet (a1, a2, a3, a4, a5, a6, a7) where
+  get = parseArray f where
+    f 7 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> return (a1, a2, a3, a4, a5, a6, a7)
+    f n = fail $ printf "wrong tupple size: expected 7 but got " n
+
+instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7, ObjectGet a8) => ObjectGet (a1, a2, a3, a4, a5, a6, a7, a8) where
+  get = parseArray f where
+    f 8 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> get >>= \a8 -> return (a1, a2, a3, a4, a5, a6, a7, a8)
+    f n = fail $ printf "wrong tupple size: expected 8 but got " n
+
+instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7, ObjectGet a8, ObjectGet a9) => ObjectGet (a1, a2, a3, a4, a5, a6, a7, a8, a9) where
+  get = parseArray f where
+    f 9 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> get >>= \a8 -> get >>= \a9 -> return (a1, a2, a3, a4, a5, a6, a7, a8, a9)
+    f n = fail $ printf "wrong tupple size: expected 9 but got " n
+
 parseArray :: (Int -> A.Parser a) -> A.Parser a
 parseArray aget = do
   c <- A.anyWord8
diff --git a/haskell/src/Data/MessagePack/Put.hs b/haskell/src/Data/MessagePack/Put.hs
index 95582dd..24ec305 100644
--- a/haskell/src/Data/MessagePack/Put.hs
+++ b/haskell/src/Data/MessagePack/Put.hs
@@ -1,6 +1,7 @@
 {-# Language FlexibleInstances #-}
 {-# Language IncoherentInstances #-}
 {-# Language OverlappingInstances #-}
+{-# Language TypeSynonymInstances #-}
 
 --------------------------------------------------------------------
 -- |
@@ -25,6 +26,8 @@ import Data.Binary.Put
 import Data.Binary.IEEE754
 import Data.Bits
 import qualified Data.ByteString as B
+import qualified Data.ByteString.Char8 as B8
+import qualified Data.ByteString.Lazy as L
 import qualified Data.Vector as V
 
 import Data.MessagePack.Object
@@ -97,20 +100,27 @@ instance ObjectPut Double where
     putWord8 0xCB
     putFloat64be d
 
+instance ObjectPut String where
+  put = putString length (putByteString . B8.pack)
+
 instance ObjectPut B.ByteString where
-  put bs = do
-    case len of
-      _ | len <= 31 -> do
-        putWord8 $ 0xA0 .|. fromIntegral len
-      _ | len < 0x10000 -> do
-        putWord8 0xDA
-        putWord16be $ fromIntegral len
-      _ -> do
-        putWord8 0xDB
-        putWord32be $ fromIntegral len
-    putByteString bs
-    where
-      len = B.length bs
+  put = putString B.length putByteString
+
+instance ObjectPut L.ByteString where
+  put = putString (fromIntegral . L.length) putLazyByteString
+
+putString :: (s -> Int) -> (s -> Put) -> s -> Put
+putString lf pf str = do
+  case lf str of
+    len | len <= 31 -> do
+      putWord8 $ 0xA0 .|. fromIntegral len
+    len | len < 0x10000 -> do
+      putWord8 0xDA
+      putWord16be $ fromIntegral len
+    len -> do
+      putWord8 0xDB
+      putWord32be $ fromIntegral len
+  pf str
 
 instance ObjectPut a => ObjectPut [a] where
   put = putArray length (mapM_ put)
@@ -118,6 +128,38 @@ instance ObjectPut a => ObjectPut [a] where
 instance ObjectPut a => ObjectPut (V.Vector a) where
   put = putArray V.length (V.mapM_ put)
 
+instance (ObjectPut a1, ObjectPut a2) => ObjectPut (a1, a2) where
+  put = putArray (const 2) f where
+    f (a1, a2) = put a1 >> put a2
+
+instance (ObjectPut a1, ObjectPut a2, ObjectPut a3) => ObjectPut (a1, a2, a3) where
+  put = putArray (const 3) f where
+    f (a1, a2, a3) = put a1 >> put a2 >> put a3
+
+instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4) => ObjectPut (a1, a2, a3, a4) where
+  put = putArray (const 4) f where
+    f (a1, a2, a3, a4) = put a1 >> put a2 >> put a3 >> put a4
+
+instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5) => ObjectPut (a1, a2, a3, a4, a5) where
+  put = putArray (const 5) f where
+    f (a1, a2, a3, a4, a5) = put a1 >> put a2 >> put a3 >> put a4 >> put a5
+
+instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6) => ObjectPut (a1, a2, a3, a4, a5, a6) where
+  put = putArray (const 6) f where
+    f (a1, a2, a3, a4, a5, a6) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6
+
+instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7) => ObjectPut (a1, a2, a3, a4, a5, a6, a7) where
+  put = putArray (const 7) f where
+    f (a1, a2, a3, a4, a5, a6, a7) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7
+
+instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7, ObjectPut a8) => ObjectPut (a1, a2, a3, a4, a5, a6, a7, a8) where
+  put = putArray (const 8) f where
+    f (a1, a2, a3, a4, a5, a6, a7, a8) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7 >> put a8
+
+instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7, ObjectPut a8, ObjectPut a9) => ObjectPut (a1, a2, a3, a4, a5, a6, a7, a8, a9) where
+  put = putArray (const 9) f where
+    f (a1, a2, a3, a4, a5, a6, a7, a8, a9) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7 >> put a8 >> put a9
+
 putArray :: (a -> Int) -> (a -> Put) -> a -> Put
 putArray lf pf arr = do
   case lf arr of
diff --git a/haskell/test/Test.hs b/haskell/test/Test.hs
index 1bb551c..a73ac9a 100644
--- a/haskell/test/Test.hs
+++ b/haskell/test/Test.hs
@@ -4,6 +4,7 @@ import Test.QuickCheck
 
 import Control.Monad
 import qualified Data.ByteString.Char8 as B
+import qualified Data.ByteString.Lazy.Char8 as L
 import Data.MessagePack
 
 mid :: (ObjectGet a, ObjectPut a) => a -> a
@@ -17,15 +18,27 @@ prop_mid_bool a = a == mid a
   where types = a :: Bool
 prop_mid_double a = a == mid a
   where types = a :: Double
-prop_mid_string a = a == B.unpack (mid (B.pack a))
+prop_mid_string a = a == mid a
+  where types = a :: String
+prop_mid_bytestring a = B.pack a == mid (B.pack a)
+  where types = a :: String
+prop_mid_lazy_bytestring a = (L.pack a) == mid (L.pack a)
   where types = a :: String
 prop_mid_array_int a = a == mid a
   where types = a :: [Int]
-prop_mid_array_string a = a == map B.unpack (mid (map B.pack a))
+prop_mid_array_string a = a == mid a
   where types = a :: [String]
+prop_mid_pair2 a = a == mid a
+  where types = a :: (Int, Int)
+prop_mid_pair3 a = a == mid a
+  where types = a :: (Int, Int, Int)
+prop_mid_pair4 a = a == mid a
+  where types = a :: (Int, Int, Int, Int)
+prop_mid_pair5 a = a == mid a
+  where types = a :: (Int, Int, Int, Int, Int)
 prop_mid_map_int_double a = a == mid a
   where types = a :: [(Int, Double)]
-prop_mid_map_string_string a = a == map (\(x, y) -> (B.unpack x, B.unpack y)) (mid (map (\(x, y) -> (B.pack x, B.pack y)) a))
+prop_mid_map_string_string a = a == mid a
   where types = a :: [(String, String)]
 
 tests =
@@ -35,8 +48,14 @@ tests =
     , testProperty "bool" prop_mid_bool
     , testProperty "double" prop_mid_double
     , testProperty "string" prop_mid_string
+    , testProperty "bytestring" prop_mid_bytestring
+    , testProperty "lazy-bytestring" prop_mid_lazy_bytestring
     , testProperty "[int]" prop_mid_array_int
     , testProperty "[string]" prop_mid_array_string
+    , testProperty "(int, int)" prop_mid_pair2
+    , testProperty "(int, int, int)" prop_mid_pair3
+    , testProperty "(int, int, int, int)" prop_mid_pair4
+    , testProperty "(int, int, int, int, int)" prop_mid_pair5
     , testProperty "[(int, double)]" prop_mid_map_int_double
     , testProperty "[(string, string)]" prop_mid_map_string_string
     ]

From b75db110dceef9bf75c8410ca4b4fc031e1aad89 Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Mon, 6 Sep 2010 17:00:22 +0900
Subject: [PATCH 0673/1648] haskell: add Iteratee interface

---
 haskell/msgpack.cabal                    |  1 +
 haskell/src/Data/MessagePack.hs          | 45 ++++++++++++-----------
 haskell/src/Data/MessagePack/Iteratee.hs | 46 ++++++++++++++++++++++++
 3 files changed, 72 insertions(+), 20 deletions(-)
 create mode 100644 haskell/src/Data/MessagePack/Iteratee.hs

diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal
index 18ae3d8..3baff77 100644
--- a/haskell/msgpack.cabal
+++ b/haskell/msgpack.cabal
@@ -33,3 +33,4 @@ Library
     Data.MessagePack.Object
     Data.MessagePack.Put
     Data.MessagePack.Parser
+    Data.MessagePack.Iteratee
diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs
index 010eaab..92353c5 100644
--- a/haskell/src/Data/MessagePack.hs
+++ b/haskell/src/Data/MessagePack.hs
@@ -16,6 +16,7 @@ module Data.MessagePack(
   module Data.MessagePack.Object,
   module Data.MessagePack.Put,
   module Data.MessagePack.Parser,
+  module Data.MessagePack.Iteratee,
   
   -- * Simple functions of Pack and Unpack
   pack,
@@ -30,6 +31,9 @@ module Data.MessagePack(
   unpackFromString,
   unpackFromHandle,
   unpackFromFile,
+  unpackFromStringI,
+  unpackFromHandleI,
+  unpackFromFileI,
   
   ) where
 
@@ -47,6 +51,7 @@ import System.IO
 import Data.MessagePack.Object
 import Data.MessagePack.Put
 import Data.MessagePack.Parser
+import Data.MessagePack.Iteratee
 
 bufferSize :: Int
 bufferSize = 4 * 1024
@@ -67,7 +72,7 @@ pack = packToString . put
 -- | Unpack MessagePack string to Haskell data.
 unpack :: (ObjectGet a, IsByteString s) => s -> a
 unpack bs =
-  runIdentity $ I.run $ I.joinIM $ I.enumPure1Chunk (toBS bs) (parserToIteratee get)
+  runIdentity $ I.run $ I.joinIM $ I.enumPure1Chunk (toBS bs) getI
 
 -- TODO: tryUnpack
 
@@ -86,32 +91,32 @@ packToFile path = L.writeFile path . packToString
 -- | Unpack from ByteString
 unpackFromString :: (Monad m, IsByteString s) => s -> A.Parser a -> m a
 unpackFromString bs =
-  I.run . I.joinIM . I.enumPure1Chunk (toBS bs) . parserToIteratee
+  unpackFromStringI bs . parserToIteratee
 
 -- | Unpack from Handle
 unpackFromHandle :: CIO.MonadCatchIO m => Handle -> A.Parser a -> m a
 unpackFromHandle h =
-  I.run . I.joinIM . I.enumHandle bufferSize h . parserToIteratee
+  unpackFromHandleI h .parserToIteratee
 
 -- | Unpack from File
 unpackFromFile :: CIO.MonadCatchIO m => FilePath -> A.Parser a -> m a
-unpackFromFile path p =
+unpackFromFile path =
+  unpackFromFileI path . parserToIteratee
+
+-- | Iteratee interface to unpack from ByteString
+unpackFromStringI :: (Monad m, IsByteString s) => s -> I.Iteratee B.ByteString m a -> m a
+unpackFromStringI bs =
+  I.run . I.joinIM . I.enumPure1Chunk (toBS bs)
+
+-- | Iteratee interface to unpack from Handle
+unpackFromHandleI :: CIO.MonadCatchIO m => Handle -> I.Iteratee B.ByteString m a -> m a
+unpackFromHandleI h =
+  I.run . I.joinIM . I.enumHandle bufferSize h
+
+-- | Iteratee interface to unpack from File
+unpackFromFileI :: CIO.MonadCatchIO m => FilePath -> I.Iteratee B.ByteString m a -> m a
+unpackFromFileI path p =
   CIO.bracket
   (liftIO $ openBinaryFile path ReadMode)
   (liftIO . hClose)
-  (flip unpackFromHandle p)
-
-parserToIteratee :: Monad m => A.Parser a -> I.Iteratee B.ByteString m a
-parserToIteratee p = I.icont (itr (A.parse p)) Nothing
-  where
-    itr pcont s = case s of
-      I.EOF _ ->
-        I.throwErr (I.setEOF s)
-      I.Chunk bs ->
-        case pcont bs of
-          A.Fail _ _ msg ->
-            I.throwErr (I.iterStrExc msg)
-          A.Partial cont ->
-            I.icont (itr cont) Nothing
-          A.Done remain ret ->
-            I.idone ret (I.Chunk remain)
+  (flip unpackFromHandleI p)
diff --git a/haskell/src/Data/MessagePack/Iteratee.hs b/haskell/src/Data/MessagePack/Iteratee.hs
new file mode 100644
index 0000000..789b714
--- /dev/null
+++ b/haskell/src/Data/MessagePack/Iteratee.hs
@@ -0,0 +1,46 @@
+--------------------------------------------------------------------
+-- |
+-- Module    : Data.MessagePack.Iteratee
+-- Copyright : (c) Hideyuki Tanaka, 2009-2010
+-- License   : BSD3
+--
+-- Maintainer:  tanaka.hideyuki@gmail.com
+-- Stability :  experimental
+-- Portability: portable
+--
+-- MessagePack Deserializer interface to @Data.Iteratee@
+--
+--------------------------------------------------------------------
+
+module Data.MessagePack.Iteratee(
+  -- * Iteratee version of deserializer
+  getI,
+  -- * Convert Parser to Iteratee
+  parserToIteratee,
+  ) where
+
+import qualified Data.Attoparsec as A
+import qualified Data.ByteString as B
+import qualified Data.Iteratee as I
+
+import Data.MessagePack.Parser
+
+-- | Deserialize a value
+getI :: (Monad m, ObjectGet a) => I.Iteratee B.ByteString m a
+getI = parserToIteratee get
+
+-- | Convert Parser to Iteratee
+parserToIteratee :: Monad m => A.Parser a -> I.Iteratee B.ByteString m a
+parserToIteratee p = I.icont (itr (A.parse p)) Nothing
+  where
+    itr pcont s = case s of
+      I.EOF _ ->
+        I.throwErr (I.setEOF s)
+      I.Chunk bs ->
+        case pcont bs of
+          A.Fail _ _ msg ->
+            I.throwErr (I.iterStrExc msg)
+          A.Partial cont ->
+            I.icont (itr cont) Nothing
+          A.Done remain ret ->
+            I.idone ret (I.Chunk remain)

From dfe19d308caa43e8d763750faafc2baade7d013c Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Mon, 6 Sep 2010 18:14:47 +0900
Subject: [PATCH 0674/1648] haskell: add overlapping instances

---
 haskell/src/Data/MessagePack/Object.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/haskell/src/Data/MessagePack/Object.hs b/haskell/src/Data/MessagePack/Object.hs
index 19a3aeb..df0e89d 100644
--- a/haskell/src/Data/MessagePack/Object.hs
+++ b/haskell/src/Data/MessagePack/Object.hs
@@ -1,5 +1,6 @@
-{-# LANGUAGE TypeSynonymInstances #-}
-{-# LANGUAGE FlexibleInstances #-}
+{-# Language TypeSynonymInstances #-}
+{-# Language FlexibleInstances #-}
+{-# Language OverlappingInstances #-}
 
 --------------------------------------------------------------------
 -- |

From c6424c2ce71f3e79e8aacfe9c76846cf94e168de Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Mon, 6 Sep 2010 23:27:50 +0900
Subject: [PATCH 0675/1648] haskell: nonblocking enumerator

---
 haskell/src/Data/MessagePack.hs          |  3 +-
 haskell/src/Data/MessagePack/Iteratee.hs | 36 ++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs
index 92353c5..b53066b 100644
--- a/haskell/src/Data/MessagePack.hs
+++ b/haskell/src/Data/MessagePack.hs
@@ -45,7 +45,6 @@ import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as L
 import Data.Functor.Identity
 import qualified Data.Iteratee as I
-import qualified Data.Iteratee.IO as I
 import System.IO
 
 import Data.MessagePack.Object
@@ -111,7 +110,7 @@ unpackFromStringI bs =
 -- | Iteratee interface to unpack from Handle
 unpackFromHandleI :: CIO.MonadCatchIO m => Handle -> I.Iteratee B.ByteString m a -> m a
 unpackFromHandleI h =
-  I.run . I.joinIM . I.enumHandle bufferSize h
+  I.run . I.joinIM . enumHandleNonBlocking bufferSize h
 
 -- | Iteratee interface to unpack from File
 unpackFromFileI :: CIO.MonadCatchIO m => FilePath -> I.Iteratee B.ByteString m a -> m a
diff --git a/haskell/src/Data/MessagePack/Iteratee.hs b/haskell/src/Data/MessagePack/Iteratee.hs
index 789b714..4258cf6 100644
--- a/haskell/src/Data/MessagePack/Iteratee.hs
+++ b/haskell/src/Data/MessagePack/Iteratee.hs
@@ -15,13 +15,18 @@
 module Data.MessagePack.Iteratee(
   -- * Iteratee version of deserializer
   getI,
+  -- * Non Blocking Enumerator
+  enumHandleNonBlocking,
   -- * Convert Parser to Iteratee
   parserToIteratee,
   ) where
 
+import Control.Exception
+import Control.Monad.IO.Class
 import qualified Data.Attoparsec as A
 import qualified Data.ByteString as B
 import qualified Data.Iteratee as I
+import System.IO
 
 import Data.MessagePack.Parser
 
@@ -29,6 +34,37 @@ import Data.MessagePack.Parser
 getI :: (Monad m, ObjectGet a) => I.Iteratee B.ByteString m a
 getI = parserToIteratee get
 
+-- | Enumerator
+enumHandleNonBlocking :: MonadIO m => Int -> Handle -> I.Enumerator B.ByteString m a
+enumHandleNonBlocking bufSize h =
+  I.enumFromCallback $ readSome bufSize h
+
+readSome :: MonadIO m => Int -> Handle -> m (Either SomeException (Bool, B.ByteString))
+readSome bufSize h = liftIO $ do
+  ebs <- try $ hGetSome bufSize h
+  case ebs of
+    Left exc ->
+      return $ Left (exc :: SomeException)
+    Right bs | B.null bs ->
+      return $ Right (False, B.empty)
+    Right bs ->
+      return $ Right (True, bs)
+
+hGetSome :: Int -> Handle -> IO B.ByteString
+hGetSome bufSize h = do
+  bs <- B.hGetNonBlocking h bufSize
+  if B.null bs
+    then do
+    hd <- B.hGet h 1
+    if B.null hd
+      then do
+      return B.empty
+      else do
+      rest <- B.hGetNonBlocking h (bufSize - 1)
+      return $ B.cons (B.head hd) rest
+    else do
+    return bs
+
 -- | Convert Parser to Iteratee
 parserToIteratee :: Monad m => A.Parser a -> I.Iteratee B.ByteString m a
 parserToIteratee p = I.icont (itr (A.parse p)) Nothing

From c56926428c2b66fd3f112b9095c46f46e0527cd7 Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Tue, 7 Sep 2010 16:14:00 +0900
Subject: [PATCH 0676/1648] haskell: add packToHandle'

---
 haskell/src/Data/MessagePack.hs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs
index b53066b..7137589 100644
--- a/haskell/src/Data/MessagePack.hs
+++ b/haskell/src/Data/MessagePack.hs
@@ -25,6 +25,7 @@ module Data.MessagePack(
   -- * Pack functions
   packToString,
   packToHandle,
+  packToHandle',
   packToFile,
   
   -- * Unpack functions
@@ -83,6 +84,10 @@ packToString = runPut
 packToHandle :: Handle -> Put -> IO ()
 packToHandle h = L.hPutStr h . packToString
 
+-- | Pack to Handle and Flush Handle
+packToHandle' :: Handle -> Put -> IO ()
+packToHandle' h p = packToHandle h p >> hFlush h
+
 -- | Pack to File
 packToFile :: FilePath -> Put -> IO ()
 packToFile path = L.writeFile path . packToString

From 169f287970a68e52d766b485f9c870ef83916b59 Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Tue, 7 Sep 2010 16:14:29 +0900
Subject: [PATCH 0677/1648] haskell: Now, Object is an instance of NFData.

---
 haskell/msgpack.cabal                  |  4 +++-
 haskell/src/Data/MessagePack/Object.hs | 12 ++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal
index 3baff77..bd10c4a 100644
--- a/haskell/msgpack.cabal
+++ b/haskell/msgpack.cabal
@@ -24,7 +24,9 @@ Library
                     iteratee >= 0.4 && < 0.5,
                     attoparsec >= 0.8.1 && < 0.8.2,
                     binary >= 0.5.0 && < 0.5.1,
-                    data-binary-ieee754 >= 0.4 && < 0.5
+                    data-binary-ieee754 >= 0.4 && < 0.5,
+                    deepseq >= 1.1 && <1.2
+
   Ghc-options:      -Wall -O2
   Hs-source-dirs:   src
 
diff --git a/haskell/src/Data/MessagePack/Object.hs b/haskell/src/Data/MessagePack/Object.hs
index df0e89d..6806722 100644
--- a/haskell/src/Data/MessagePack/Object.hs
+++ b/haskell/src/Data/MessagePack/Object.hs
@@ -25,6 +25,7 @@ module Data.MessagePack.Object(
   Result,
   ) where
 
+import Control.DeepSeq
 import Control.Monad
 import Control.Monad.Trans.Error ()
 import qualified Data.ByteString as B
@@ -41,6 +42,17 @@ data Object =
   | ObjectMap [(Object, Object)]
   deriving (Show)
 
+instance NFData Object where
+  rnf obj =
+    case obj of
+      ObjectNil -> ()
+      ObjectBool b -> rnf b
+      ObjectInteger n -> rnf n
+      ObjectDouble d -> rnf d
+      ObjectRAW bs -> bs `seq` ()
+      ObjectArray a -> rnf a
+      ObjectMap m -> rnf m
+
 -- | The class of types serializable to and from MessagePack object
 class OBJECT a where
   -- | Encode a value to MessagePack object

From 5e19bc6f844500e729d498ee6275a6a2e6557ba2 Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Tue, 7 Sep 2010 17:35:24 +0900
Subject: [PATCH 0678/1648] haskell: Object is Eq, Ord, Typeable.

---
 haskell/src/Data/MessagePack/Object.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/haskell/src/Data/MessagePack/Object.hs b/haskell/src/Data/MessagePack/Object.hs
index 6806722..87f24bd 100644
--- a/haskell/src/Data/MessagePack/Object.hs
+++ b/haskell/src/Data/MessagePack/Object.hs
@@ -1,6 +1,7 @@
 {-# Language TypeSynonymInstances #-}
 {-# Language FlexibleInstances #-}
 {-# Language OverlappingInstances #-}
+{-# Language DeriveDataTypeable #-}
 
 --------------------------------------------------------------------
 -- |
@@ -30,6 +31,7 @@ import Control.Monad
 import Control.Monad.Trans.Error ()
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Char8 as C8
+import Data.Typeable
 
 -- | Object Representation of MessagePack data.
 data Object =
@@ -40,7 +42,7 @@ data Object =
   | ObjectRAW B.ByteString
   | ObjectArray [Object]
   | ObjectMap [(Object, Object)]
-  deriving (Show)
+  deriving (Show, Eq, Ord, Typeable)
 
 instance NFData Object where
   rnf obj =

From a99870645244f9073075a43c30dc20511de89097 Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Wed, 8 Sep 2010 13:36:45 +0900
Subject: [PATCH 0679/1648] haskell: update cabal file

---
 haskell/msgpack.cabal | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal
index bd10c4a..ccdb2f7 100644
--- a/haskell/msgpack.cabal
+++ b/haskell/msgpack.cabal
@@ -1,18 +1,18 @@
 Name:               msgpack
-Version:            0.3.1
+Version:            0.3.1.1
 Synopsis:           A Haskell binding to MessagePack
 Description:
   A Haskell binding to MessagePack 
 
 License:            BSD3
 License-File:       LICENSE
+Copyright:          Copyright (c) 2009-2010, Hideyuki Tanaka
 Category:           Data
 Author:             Hideyuki Tanaka
 Maintainer:         Hideyuki Tanaka 
 Homepage:           http://github.com/msgpack/msgpack
 Stability:          Experimental
-Tested-with:        GHC == 6.12.3
-Cabal-Version:      >= 1.2
+Cabal-Version:      >= 1.6
 Build-Type:         Simple
 
 Library
@@ -27,7 +27,7 @@ Library
                     data-binary-ieee754 >= 0.4 && < 0.5,
                     deepseq >= 1.1 && <1.2
 
-  Ghc-options:      -Wall -O2
+  Ghc-options:      -Wall
   Hs-source-dirs:   src
 
   Exposed-modules:
@@ -36,3 +36,7 @@ Library
     Data.MessagePack.Put
     Data.MessagePack.Parser
     Data.MessagePack.Iteratee
+
+Source-repository head
+  Type:     git
+  Location: git://github.com/msgpack/msgpack.git

From 9f684e7670877fe04d02afe8377e4a6191d74f31 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Fri, 10 Sep 2010 09:35:39 +0900
Subject: [PATCH 0680/1648] Checking in changes prior to tagging of version
 0.20.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index 7910882..dc3dd5c 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.20
+
+    - first production ready release with PP driver.
+
 0.16_04

     - no feature changes
---
 perl/Changes                 | 4 ++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index 7910882..dc3dd5c 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.20
+
+    - first production ready release with PP driver.
+
 0.16_04
 
     - no feature changes
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index b08bac2..eca24ec 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.16_04';
+our $VERSION = '0.20';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };

From f6f675d1e12b2c5994099df3d1af1513b1d83bf2 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Fri, 10 Sep 2010 20:27:11 +0900
Subject: [PATCH 0681/1648] updated benchmark script

---
 perl/benchmark/deserialize.pl | 9 ++++++---
 perl/benchmark/serialize.pl   | 5 ++++-
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl
index fd21f08..0ddcec9 100644
--- a/perl/benchmark/deserialize.pl
+++ b/perl/benchmark/deserialize.pl
@@ -3,18 +3,21 @@ use warnings;
 use Data::MessagePack;
 use JSON::XS;
 use Benchmark ':all';
+use Storable;
 
 my $a = [0..2**24];
 my $j = JSON::XS::encode_json($a);
 my $m = Data::MessagePack->pack($a);
+my $s = Storable::nfreeze($a);
 
 print "-- deserialize\n";
 print "JSON::XS: $JSON::XS::VERSION\n";
 print "Data::MessagePack: $Data::MessagePack::VERSION\n";
-cmpthese(
+timethese(
     -1 => {
-        json => sub { JSON::XS::decode_json($j)     },
-        mp   => sub { Data::MessagePack->unpack($m) },
+        json     => sub { JSON::XS::decode_json($j)     },
+        mp       => sub { Data::MessagePack->unpack($m) },
+        storable => sub { Storable::thaw($s) },
     }
 );
 
diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl
index 626ae03..b811bfe 100644
--- a/perl/benchmark/serialize.pl
+++ b/perl/benchmark/serialize.pl
@@ -2,6 +2,7 @@ use strict;
 use warnings;
 use Data::MessagePack;
 use JSON::XS;
+use Storable;
 use Benchmark ':all';
 
 my $a = [0..2**24];
@@ -9,9 +10,11 @@ my $a = [0..2**24];
 print "-- serialize\n";
 print "JSON::XS: $JSON::XS::VERSION\n";
 print "Data::MessagePack: $Data::MessagePack::VERSION\n";
-cmpthese(
+print "Storable: $Storable::VERSION\n";
+timethese(
     -1 => {
         json => sub { JSON::XS::encode_json($a)   },
+        storable => sub { Storable::nfreeze($a)   },
         mp   => sub { Data::MessagePack->pack($a) },
     }
 );

From 5bb8b6f16c426ebfeee0027c1026ab8f610bec05 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Fri, 10 Sep 2010 20:38:37 +0900
Subject: [PATCH 0682/1648] perl: ugpraded benchmarking script. and added
 result to docs.

---
 perl/benchmark/deserialize.pl | 12 ++++++---
 perl/benchmark/serialize.pl   | 15 +++++++----
 perl/lib/Data/MessagePack.pm  | 50 ++++++++++++++++++++++++++++++++++-
 3 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl
index 0ddcec9..750704e 100644
--- a/perl/benchmark/deserialize.pl
+++ b/perl/benchmark/deserialize.pl
@@ -5,16 +5,22 @@ use JSON::XS;
 use Benchmark ':all';
 use Storable;
 
-my $a = [0..2**24];
+my $a = {
+    "method" => "handleMessage",
+    "params" => [ "user1", "we were just talking" ],
+    "id"     => undef,
+    "array"  => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0 ]
+};
 my $j = JSON::XS::encode_json($a);
 my $m = Data::MessagePack->pack($a);
-my $s = Storable::nfreeze($a);
+my $s = Storable::freeze($a);
 
 print "-- deserialize\n";
 print "JSON::XS: $JSON::XS::VERSION\n";
 print "Data::MessagePack: $Data::MessagePack::VERSION\n";
+print "Storable: $Storable::VERSION\n";
 timethese(
-    -1 => {
+    1000000 => {
         json     => sub { JSON::XS::decode_json($j)     },
         mp       => sub { Data::MessagePack->unpack($m) },
         storable => sub { Storable::thaw($s) },
diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl
index b811bfe..c5ab15b 100644
--- a/perl/benchmark/serialize.pl
+++ b/perl/benchmark/serialize.pl
@@ -5,17 +5,22 @@ use JSON::XS;
 use Storable;
 use Benchmark ':all';
 
-my $a = [0..2**24];
+my $a = {
+    "method" => "handleMessage",
+    "params" => [ "user1", "we were just talking" ],
+    "id"     => undef,
+    "array"  => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0 ]
+};
 
 print "-- serialize\n";
 print "JSON::XS: $JSON::XS::VERSION\n";
 print "Data::MessagePack: $Data::MessagePack::VERSION\n";
 print "Storable: $Storable::VERSION\n";
 timethese(
-    -1 => {
-        json => sub { JSON::XS::encode_json($a)   },
-        storable => sub { Storable::nfreeze($a)   },
-        mp   => sub { Data::MessagePack->pack($a) },
+    1000000 => {
+        json     => sub { JSON::XS::encode_json($a) },
+        storable => sub { Storable::freeze($a) },
+        mp       => sub { Data::MessagePack->pack($a) },
     }
 );
 
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index eca24ec..fbf305a 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -41,9 +41,35 @@ Data::MessagePack - MessagePack serialising/deserialising
 
 This module converts Perl data structures to MessagePack and vice versa.
 
+=head1 ABOUT MESSAGEPACK FORMAT
+
 MessagePack is a binary-based efficient object serialization format.
 It enables to exchange structured objects between many languages like JSON. But unlike JSON, it is very fast and small.
 
+=head2 ADVANTAGES
+
+=over 4
+
+=item PORTABILITY
+
+Messagepack is language independent binary serialize format.
+
+=item SMALL SIZE
+
+    say length(JSON::XS::encode_json({a=>1, b=>2}));   # => 13
+    say length(Storable::nfreeze({a=>1, b=>2}));       # => 21
+    say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7
+
+MessagePack format saves memory than JSON and Storable format.
+
+=item STREAMING DESERIALIZER
+
+MessagePack supports streaming deserializer. It is useful for networking such as RPC.
+
+=back
+
+If you want to get more informations about messagepack format, please visit to L.
+
 =head1 METHODS
 
 =over 4
@@ -68,6 +94,28 @@ Pack the string as int when the value looks like int(EXPERIMENTAL).
 
 =back
 
+=head1 SPEED
+
+This is result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP).
+
+    -- serialize
+    JSON::XS: 2.3
+    Data::MessagePack: 0.20
+    Storable: 2.21
+    Benchmark: timing 1000000 iterations of json, mp, storable...
+          json:  5 wallclock secs ( 3.95 usr +  0.00 sys =  3.95 CPU) @ 253164.56/s (n=1000000)
+            mp:  3 wallclock secs ( 2.69 usr +  0.00 sys =  2.69 CPU) @ 371747.21/s (n=1000000)
+      storable: 26 wallclock secs (27.21 usr +  0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000)
+
+    -- deserialize
+    JSON::XS: 2.3
+    Data::MessagePack: 0.20
+    Storable: 2.21
+    Benchmark: timing 1000000 iterations of json, mp, storable...
+          json:  4 wallclock secs ( 4.45 usr +  0.00 sys =  4.45 CPU) @ 224719.10/s (n=1000000)
+            mp:  6 wallclock secs ( 5.45 usr +  0.00 sys =  5.45 CPU) @ 183486.24/s (n=1000000)
+      storable:  7 wallclock secs ( 7.77 usr +  0.00 sys =  7.77 CPU) @ 128700.13/s (n=1000000)
+
 =head1 AUTHORS
 
 Tokuhiro Matsuno
@@ -90,5 +138,5 @@ it under the same terms as Perl itself.
 
 =head1 SEE ALSO
 
-L
+L is official web site for MessagePack format.
 

From b79c1345b92d8cdb6427e0d83d7191262331fd5a Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Fri, 10 Sep 2010 20:42:40 +0900
Subject: [PATCH 0683/1648] use gfx's standard header.

---
 perl/README          | 43 ++++++++++++++++++++++++-
 perl/perlxs.h        | 76 ++++++++++++++++++++++++++++++++++++++++++++
 perl/xs-src/pack.c   | 11 +------
 perl/xs-src/unpack.c | 46 ++++++++++++---------------
 4 files changed, 140 insertions(+), 36 deletions(-)
 create mode 100644 perl/perlxs.h

diff --git a/perl/README b/perl/README
index 2ef686c..d5fc269 100644
--- a/perl/README
+++ b/perl/README
@@ -8,10 +8,29 @@ SYNOPSIS
 DESCRIPTION
     This module converts Perl data structures to MessagePack and vice versa.
 
+ABOUT MESSAGEPACK FORMAT
     MessagePack is a binary-based efficient object serialization format. It
     enables to exchange structured objects between many languages like JSON.
     But unlike JSON, it is very fast and small.
 
+  ADVANTAGES
+    PORTABILITY
+        Messagepack is language independent binary serialize format.
+
+    SMALL SIZE
+            say length(JSON::XS::encode_json({a=>1, b=>2}));   # => 13
+            say length(Storable::nfreeze({a=>1, b=>2}));       # => 21
+            say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7
+
+        MessagePack format saves memory than JSON and Storable format.
+
+    STREAMING DESERIALIZER
+        MessagePack supports streaming deserializer. It is useful for
+        networking such as RPC.
+
+    If you want to get more informations about messagepack format, please
+    visit to .
+
 METHODS
     my $packed = Data::MessagePack->pack($data);
         pack the $data to messagepack format string.
@@ -23,6 +42,28 @@ Configuration Variables
     $Data::MessagePack::PreferInteger
         Pack the string as int when the value looks like int(EXPERIMENTAL).
 
+SPEED
+    This is result of benchmark/serialize.pl and benchmark/deserialize.pl on
+    my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP).
+
+        -- serialize
+        JSON::XS: 2.3
+        Data::MessagePack: 0.20
+        Storable: 2.21
+        Benchmark: timing 1000000 iterations of json, mp, storable...
+              json:  5 wallclock secs ( 3.95 usr +  0.00 sys =  3.95 CPU) @ 253164.56/s (n=1000000)
+                mp:  3 wallclock secs ( 2.69 usr +  0.00 sys =  2.69 CPU) @ 371747.21/s (n=1000000)
+          storable: 26 wallclock secs (27.21 usr +  0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000)
+
+        -- deserialize
+        JSON::XS: 2.3
+        Data::MessagePack: 0.20
+        Storable: 2.21
+        Benchmark: timing 1000000 iterations of json, mp, storable...
+              json:  4 wallclock secs ( 4.45 usr +  0.00 sys =  4.45 CPU) @ 224719.10/s (n=1000000)
+                mp:  6 wallclock secs ( 5.45 usr +  0.00 sys =  5.45 CPU) @ 183486.24/s (n=1000000)
+          storable:  7 wallclock secs ( 7.77 usr +  0.00 sys =  7.77 CPU) @ 128700.13/s (n=1000000)
+
 AUTHORS
     Tokuhiro Matsuno
 
@@ -40,5 +81,5 @@ LICENSE
     under the same terms as Perl itself.
 
 SEE ALSO
-    
+     is official web site for MessagePack format.
 
diff --git a/perl/perlxs.h b/perl/perlxs.h
new file mode 100644
index 0000000..441682d
--- /dev/null
+++ b/perl/perlxs.h
@@ -0,0 +1,76 @@
+/*
+    perlxs.h - Standard XS header file
+    Copyright (c) Fuji, Goro (gfx)
+*/
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define PERL_NO_GET_CONTEXT /* we want efficiency */
+#include 
+
+#include 
+#define NO_XSLOCKS /* for exceptions */
+#include 
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#include "ppport.h"
+
+/* portability stuff not supported by ppport.h yet */
+
+#ifndef STATIC_INLINE /* from 5.13.4 */
+#   if defined(__GNUC__) || defined(__cplusplus__) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L))
+#       define STATIC_INLINE static inline
+#   else
+#       define STATIC_INLINE static
+#   endif
+#endif /* STATIC_INLINE */
+
+#ifndef __attribute__format__
+#define __attribute__format__(a,b,c) /* nothing */
+#endif
+
+#ifndef LIKELY /* they are just a compiler's hint */
+#define LIKELY(x)   (x)
+#define UNLIKELY(x) (x)
+#endif
+
+#ifndef newSVpvs_share
+#define newSVpvs_share(s) Perl_newSVpvn_share(aTHX_ STR_WITH_LEN(s), 0U)
+#endif
+
+#ifndef get_cvs
+#define get_cvs(name, flags) get_cv(name, flags)
+#endif
+
+#ifndef GvNAME_get
+#define GvNAME_get GvNAME
+#endif
+#ifndef GvNAMELEN_get
+#define GvNAMELEN_get GvNAMELEN
+#endif
+
+#ifndef CvGV_set
+#define CvGV_set(cv, gv) (CvGV(cv) = (gv))
+#endif
+
+/* general utility */
+
+#if PERL_BCDVERSION >= 0x5008005
+#define LooksLikeNumber(x) looks_like_number(x)
+#else
+#define LooksLikeNumber(x) (SvPOKp(x) ? looks_like_number(x) : (I32)SvNIOKp(x))
+#endif
+
+#define newAV_mortal() (AV*)sv_2mortal((SV*)newAV())
+#define newHV_mortal() (HV*)sv_2mortal((SV*)newHV())
+
+#define DECL_BOOT(name) EXTERN_C XS(CAT2(boot_, name))
+#define CALL_BOOT(name) STMT_START {            \
+        PUSHMARK(SP);                           \
+        CALL_FPTR(CAT2(boot_, name))(aTHX_ cv); \
+    } STMT_END
diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 93b2e2f..62eb002 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -2,16 +2,7 @@
  * code is written by tokuhirom.
  * buffer alocation technique is taken from JSON::XS. thanks to mlehmann.
  */
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "EXTERN.h"
-#include "perl.h"
-#include "XSUB.h"
-#include "ppport.h"
-#ifdef __cplusplus
-};
-#endif
+#include "perlxs.h"
 
 #include "msgpack/pack_define.h"
 
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index eb6e0dd..20a0737 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -2,13 +2,9 @@
 extern "C" {
 #endif
 
-#include "EXTERN.h"
-#include "perl.h"
-#include "XSUB.h"
-#include "util.h"
 #define NEED_newRV_noinc
 #define NEED_sv_2pv_flags
-#include "ppport.h"
+#include "perlxs.h"
 
 #ifdef __cplusplus
 };
@@ -38,7 +34,7 @@ typedef struct {
 /* ---------------------------------------------------------------------- */
 /* utility functions                                                      */
 
-static INLINE SV *
+STATIC_INLINE SV *
 get_bool (const char *name) {
     SV * sv = sv_mortalcopy(get_sv( name, 1 ));
 
@@ -60,19 +56,19 @@ static SV* template_data(msgpack_unpack_t* u);
 static int template_execute(msgpack_unpack_t* u,
     const char* data, size_t len, size_t* off);
 
-static INLINE SV* template_callback_root(unpack_user* u)
+STATIC_INLINE SV* template_callback_root(unpack_user* u)
 { return &PL_sv_undef; }
 
-static INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o)
+STATIC_INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o)
 { *o = sv_2mortal(newSVuv(d)); return 0; }
 
-static INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o)
+STATIC_INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o)
 { *o = sv_2mortal(newSVuv(d)); return 0; }
 
-static INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o)
+STATIC_INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o)
 { *o = sv_2mortal(newSVuv(d)); return 0; }
 
-static INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o)
+STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o)
 {
 #if IVSIZE==4
     *o = sv_2mortal(newSVnv(d));
@@ -82,47 +78,47 @@ static INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o)
     return 0;
 }
 
-static INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o)
+STATIC_INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o)
 { *o = sv_2mortal(newSViv((long)d)); return 0; }
 
-static INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o)
+STATIC_INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o)
 { *o = sv_2mortal(newSViv((long)d)); return 0; }
 
-static INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o)
+STATIC_INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o)
 { *o = sv_2mortal(newSViv((long)d)); return 0; }
 
-static INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o)
+STATIC_INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o)
 { *o = sv_2mortal(newSViv(d)); return 0; }
 
-static INLINE int template_callback_float(unpack_user* u, float d, SV** o)
+STATIC_INLINE int template_callback_float(unpack_user* u, float d, SV** o)
 { *o = sv_2mortal(newSVnv(d)); return 0; }
 
-static INLINE int template_callback_double(unpack_user* u, double d, SV** o)
+STATIC_INLINE int template_callback_double(unpack_user* u, double d, SV** o)
 { *o = sv_2mortal(newSVnv(d)); return 0; }
 
 /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */
-static INLINE int template_callback_nil(unpack_user* u, SV** o)
+STATIC_INLINE int template_callback_nil(unpack_user* u, SV** o)
 { *o = sv_newmortal(); return 0; }
 
-static INLINE int template_callback_true(unpack_user* u, SV** o)
+STATIC_INLINE int template_callback_true(unpack_user* u, SV** o)
 { *o = get_bool("Data::MessagePack::true") ; return 0; }
 
-static INLINE int template_callback_false(unpack_user* u, SV** o)
+STATIC_INLINE int template_callback_false(unpack_user* u, SV** o)
 { *o = get_bool("Data::MessagePack::false") ; return 0; }
 
-static INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o)
+STATIC_INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o)
 { AV* a = (AV*)sv_2mortal((SV*)newAV()); *o = sv_2mortal((SV*)newRV_inc((SV*)a)); av_extend(a, n); return 0; }
 
-static INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o)
+STATIC_INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o)
 { av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; }  /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */
 
-static INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o)
+STATIC_INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o)
 { HV * h = (HV*)sv_2mortal((SV*)newHV()); *o = sv_2mortal(newRV_inc((SV*)h)); return 0; }
 
-static INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v)
+STATIC_INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v)
 { hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; }
 
-static INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o)
+STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o)
 { *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l)); return 0; }
 /* { *o = newSVpvn_flags(p, l, SVs_TEMP); return 0; }  <= this does not works. */
 

From ef0a86e7ccc78bf632a3dea4b49fe8507d711151 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Fri, 10 Sep 2010 20:45:17 +0900
Subject: [PATCH 0684/1648] perl: more inline

---
 perl/xs-src/unpack.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 20a0737..16a52d7 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -131,7 +131,7 @@ STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const cha
 
 #include "msgpack/unpack_template.h"
 
-SV* _msgpack_unpack(SV* data, int limit) {
+STATIC_INLINE SV* _msgpack_unpack(SV* data, int limit) {
     msgpack_unpack_t mp;
     unpack_user u = {0, &PL_sv_undef};
 	int ret;
@@ -194,7 +194,7 @@ XS(xs_unpack) {
 /* ------------------------------ stream -- */
 /* http://twitter.com/frsyuki/status/13249304748 */
 
-static void _reset(SV* self) {
+STATIC_INLINE void _reset(SV* self) {
 	unpack_user u = {0, &PL_sv_undef, 0};
 
 	UNPACKER(self, mp);
@@ -220,7 +220,7 @@ XS(xs_unpacker_new) {
     XSRETURN(1);
 }
 
-static SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) {
+STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) {
     UNPACKER(self, mp);
 
     size_t from = off;

From 0cd31a4b96d1b8b4084083d3b7ed99b403338e2b Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Fri, 10 Sep 2010 21:00:27 +0900
Subject: [PATCH 0685/1648] perl: inlining utility functions

---
 perl/xs-src/unpack.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 16a52d7..f82fe07 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -22,7 +22,7 @@ typedef struct {
     struct template ## name
 
 #define msgpack_unpack_func(ret, name) \
-    ret template ## name
+    STATIC_INLINE ret template ## name
 
 #define msgpack_unpack_callback(name) \
     template_callback ## name

From 0c4f0de13dd9cfaa2f50b48177a0545e258c81b7 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Fri, 10 Sep 2010 21:18:45 +0900
Subject: [PATCH 0686/1648] perl: inlining the small functions

---
 perl/xs-src/pack.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 62eb002..e7a7c35 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -43,7 +43,7 @@ static void need(enc_t *enc, STRLEN len);
 #define ERR_NESTING_EXCEEDED "perl structure exceeds maximum nesting level (max_depth set too low?)"
 
 
-static void need(enc_t *enc, STRLEN len)
+STATIC_INLINE void need(enc_t *enc, STRLEN len)
 {
     if (enc->cur + len >= enc->end) {
         STRLEN cur = enc->cur - (char *)SvPVX (enc->sv);
@@ -56,7 +56,7 @@ static void need(enc_t *enc, STRLEN len)
 
 static int s_pref_int = 0;
 
-static int pref_int_set(pTHX_ SV* sv, MAGIC* mg) {
+STATIC_INLINE int pref_int_set(pTHX_ SV* sv, MAGIC* mg) {
     if (SvTRUE(sv)) {
         s_pref_int = 1;
     } else {
@@ -85,7 +85,7 @@ void boot_Data__MessagePack_pack(void) {
 }
 
 
-static int try_int(enc_t* enc, const char *p, size_t len) {
+STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) {
     int negative = 0;
     const char* pe = p + len;
     uint64_t num = 0;
@@ -141,7 +141,7 @@ static int try_int(enc_t* enc, const char *p, size_t len) {
 
 static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
 
-static void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) {
+STATIC_INLINE void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) {
     if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
     SvGETMAGIC(sv);
 
@@ -176,7 +176,7 @@ static void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) {
     }
 }
 
-static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
+STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
     svtype svt;
     if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
     SvGETMAGIC(sv);

From beb22844408b218a9ae7f494f3caa2aefe779a0e Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Fri, 10 Sep 2010 21:25:46 +0900
Subject: [PATCH 0687/1648] perl: added docs for circular reference and blessed
 object.

---
 perl/lib/Data/MessagePack.pm | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index fbf305a..0229bca 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -74,9 +74,13 @@ If you want to get more informations about messagepack format, please visit to L
 
 =over 4
 
-=item my $packed = Data::MessagePack->pack($data);
+=item my $packed = Data::MessagePack->pack($data[, $max_depth]);
 
-pack the $data to messagepack format string.
+Pack the $data to messagepack format string.
+
+This method throws exception when nesting perl structure more than $max_depth(default: 512) for detecting circular reference.
+
+Data::MessagePack->pack() throws exception when encountered blessed object. Because MessagePack is language independent format.
 
 =item my $unpacked = Data::MessagePack->unpack($msgpackstr);
 

From 1e6262f24f131b28b7c6d25d0c2464d49450ed08 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Fri, 10 Sep 2010 21:27:38 +0900
Subject: [PATCH 0688/1648] Checking in changes prior to tagging of version
 0.21.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index dc3dd5c..5d5a5e2 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.21
+
+    - doc enhancment
+    - micro performance tuning.
+
 0.20

     - first production ready release with PP driver.
---
 perl/Changes                 |  5 +++++
 perl/README                  | 10 ++++++++--
 perl/lib/Data/MessagePack.pm |  2 +-
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/perl/Changes b/perl/Changes
index dc3dd5c..5d5a5e2 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.21
+
+    - doc enhancment
+    - micro performance tuning.
+
 0.20
 
     - first production ready release with PP driver.
diff --git a/perl/README b/perl/README
index d5fc269..dec6125 100644
--- a/perl/README
+++ b/perl/README
@@ -32,8 +32,14 @@ ABOUT MESSAGEPACK FORMAT
     visit to .
 
 METHODS
-    my $packed = Data::MessagePack->pack($data);
-        pack the $data to messagepack format string.
+    my $packed = Data::MessagePack->pack($data[, $max_depth]);
+        Pack the $data to messagepack format string.
+
+        This method throws exception when nesting perl structure more than
+        $max_depth(default: 512) for detecting circular reference.
+
+        Data::MessagePack->pack() throws exception when encountered blessed
+        object. Because MessagePack is language independent format.
 
     my $unpacked = Data::MessagePack->unpack($msgpackstr);
         unpack the $msgpackstr to messagepack format string.
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 0229bca..197d744 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.20';
+our $VERSION = '0.21';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };

From a41f7ce3bd106937900576c00108fefd87559174 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Sun, 12 Sep 2010 00:09:44 +0900
Subject: [PATCH 0689/1648] oops. 0.21 breakes ithreads support!

---
 perl/.gitignore      |  1 +
 perl/Makefile.PL     |  2 +-
 perl/xs-src/pack.c   |  6 +++++-
 perl/xs-src/unpack.c | 43 ++++++++++++++++++++++++-------------------
 4 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/perl/.gitignore b/perl/.gitignore
index b806d2b..b64dcdf 100644
--- a/perl/.gitignore
+++ b/perl/.gitignore
@@ -12,3 +12,4 @@ unpack.o
 MANIFEST
 ppport.h
 .testenv/
+xshelper.h
diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index 7958bc6..7053cf9 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -18,7 +18,7 @@ if ( $] >= 5.008005 and want_xs() ) {
     my $has_c99 = c99_available(); # msgpack C library requires C99.
 
     if ( $has_c99 ) {
-        use_ppport 3.19;
+        use_xshelper();
         cc_src_paths('xs-src');
         if ($ENV{DEBUG}) {
             cc_append_to_ccflags '-g';
diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index e7a7c35..0aa3250 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -2,7 +2,7 @@
  * code is written by tokuhirom.
  * buffer alocation technique is taken from JSON::XS. thanks to mlehmann.
  */
-#include "perlxs.h"
+#include "xshelper.h"
 
 #include "msgpack/pack_define.h"
 
@@ -45,6 +45,7 @@ static void need(enc_t *enc, STRLEN len);
 
 STATIC_INLINE void need(enc_t *enc, STRLEN len)
 {
+    dTHX;
     if (enc->cur + len >= enc->end) {
         STRLEN cur = enc->cur - (char *)SvPVX (enc->sv);
         SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1);
@@ -79,6 +80,7 @@ MGVTBL pref_int_vtbl = {
 };
 
 void boot_Data__MessagePack_pack(void) {
+    dTHX;
     SV* var = get_sv("Data::MessagePack::PreferInteger", 0);
     sv_magicext(var, NULL, PERL_MAGIC_ext, &pref_int_vtbl, NULL, 0);
     SvSETMAGIC(var);
@@ -142,6 +144,7 @@ STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) {
 static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
 
 STATIC_INLINE void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) {
+    dTHX;
     if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
     SvGETMAGIC(sv);
 
@@ -178,6 +181,7 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) {
 
 STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
     svtype svt;
+    dTHX;
     if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
     SvGETMAGIC(sv);
     svt = SvTYPE(sv);
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index f82fe07..c329e99 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -4,7 +4,7 @@ extern "C" {
 
 #define NEED_newRV_noinc
 #define NEED_sv_2pv_flags
-#include "perlxs.h"
+#include "xshelper.h"
 
 #ifdef __cplusplus
 };
@@ -36,6 +36,7 @@ typedef struct {
 
 STATIC_INLINE SV *
 get_bool (const char *name) {
+    dTHX;
     SV * sv = sv_mortalcopy(get_sv( name, 1 ));
 
     SvREADONLY_on(sv);
@@ -57,19 +58,20 @@ static int template_execute(msgpack_unpack_t* u,
     const char* data, size_t len, size_t* off);
 
 STATIC_INLINE SV* template_callback_root(unpack_user* u)
-{ return &PL_sv_undef; }
+{ dTHX; return &PL_sv_undef; }
 
 STATIC_INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o)
-{ *o = sv_2mortal(newSVuv(d)); return 0; }
+{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; }
 
 STATIC_INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o)
-{ *o = sv_2mortal(newSVuv(d)); return 0; }
+{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; }
 
 STATIC_INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o)
-{ *o = sv_2mortal(newSVuv(d)); return 0; }
+{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; }
 
 STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o)
 {
+    dTHX;
 #if IVSIZE==4
     *o = sv_2mortal(newSVnv(d));
 #else
@@ -79,47 +81,47 @@ STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o)
 }
 
 STATIC_INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o)
-{ *o = sv_2mortal(newSViv((long)d)); return 0; }
+{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; }
 
 STATIC_INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o)
-{ *o = sv_2mortal(newSViv((long)d)); return 0; }
+{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; }
 
 STATIC_INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o)
-{ *o = sv_2mortal(newSViv((long)d)); return 0; }
+{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; }
 
 STATIC_INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o)
-{ *o = sv_2mortal(newSViv(d)); return 0; }
+{ dTHX; *o = sv_2mortal(newSViv(d)); return 0; }
 
 STATIC_INLINE int template_callback_float(unpack_user* u, float d, SV** o)
-{ *o = sv_2mortal(newSVnv(d)); return 0; }
+{ dTHX; *o = sv_2mortal(newSVnv(d)); return 0; }
 
 STATIC_INLINE int template_callback_double(unpack_user* u, double d, SV** o)
-{ *o = sv_2mortal(newSVnv(d)); return 0; }
+{ dTHX; *o = sv_2mortal(newSVnv(d)); return 0; }
 
 /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */
 STATIC_INLINE int template_callback_nil(unpack_user* u, SV** o)
-{ *o = sv_newmortal(); return 0; }
+{ dTHX; *o = sv_newmortal(); return 0; }
 
 STATIC_INLINE int template_callback_true(unpack_user* u, SV** o)
-{ *o = get_bool("Data::MessagePack::true") ; return 0; }
+{ dTHX; *o = get_bool("Data::MessagePack::true") ; return 0; }
 
 STATIC_INLINE int template_callback_false(unpack_user* u, SV** o)
-{ *o = get_bool("Data::MessagePack::false") ; return 0; }
+{ dTHX; *o = get_bool("Data::MessagePack::false") ; return 0; }
 
 STATIC_INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o)
-{ AV* a = (AV*)sv_2mortal((SV*)newAV()); *o = sv_2mortal((SV*)newRV_inc((SV*)a)); av_extend(a, n); return 0; }
+{ dTHX; AV* a = (AV*)sv_2mortal((SV*)newAV()); *o = sv_2mortal((SV*)newRV_inc((SV*)a)); av_extend(a, n); return 0; }
 
 STATIC_INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o)
-{ av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; }  /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */
+{ dTHX; av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; }  /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */
 
 STATIC_INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o)
-{ HV * h = (HV*)sv_2mortal((SV*)newHV()); *o = sv_2mortal(newRV_inc((SV*)h)); return 0; }
+{ dTHX; HV * h = (HV*)sv_2mortal((SV*)newHV()); *o = sv_2mortal(newRV_inc((SV*)h)); return 0; }
 
 STATIC_INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v)
-{ hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; }
+{ dTHX; hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; }
 
 STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o)
-{ *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l)); return 0; }
+{ dTHX; *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l)); return 0; }
 /* { *o = newSVpvn_flags(p, l, SVs_TEMP); return 0; }  <= this does not works. */
 
 #define UNPACKER(from, name) \
@@ -133,6 +135,7 @@ STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const cha
 
 STATIC_INLINE SV* _msgpack_unpack(SV* data, int limit) {
     msgpack_unpack_t mp;
+    dTHX;
     unpack_user u = {0, &PL_sv_undef};
 	int ret;
 	size_t from = 0;
@@ -195,6 +198,7 @@ XS(xs_unpack) {
 /* http://twitter.com/frsyuki/status/13249304748 */
 
 STATIC_INLINE void _reset(SV* self) {
+    dTHX;
 	unpack_user u = {0, &PL_sv_undef, 0};
 
 	UNPACKER(self, mp);
@@ -221,6 +225,7 @@ XS(xs_unpacker_new) {
 }
 
 STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) {
+    dTHX;
     UNPACKER(self, mp);
 
     size_t from = off;

From 65befb84a0d33d5dd4533cb5aea34cbbeffabfa0 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Sun, 12 Sep 2010 00:11:31 +0900
Subject: [PATCH 0690/1648] Checking in changes prior to tagging of version
 0.22.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index 5d5a5e2..dd47b98 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.22
+
+    - fixed issue on ithreads(broken from 0.21)
+
 0.21

     - doc enhancment
---
 perl/Changes                 | 4 ++++
 perl/MANIFEST.SKIP           | 1 +
 perl/lib/Data/MessagePack.pm | 2 +-
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index 5d5a5e2..dd47b98 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.22
+
+    - fixed issue on ithreads(broken from 0.21)
+
 0.21
 
     - doc enhancment
diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP
index 71a24e5..372742c 100644
--- a/perl/MANIFEST.SKIP
+++ b/perl/MANIFEST.SKIP
@@ -25,3 +25,4 @@
 ^Data-MessagePack-[0-9.]+/
 ^\.testenv/test_pp.pl
 ^ppport.h$
+^xshelper.h$
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 197d744..516e98b 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.21';
+our $VERSION = '0.22';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };

From 1242ffa4c6e640a9d649028f3f7659c881a61982 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Sun, 12 Sep 2010 05:38:15 +0900
Subject: [PATCH 0691/1648] Checking in changes prior to tagging of version
 0.23.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index dd47b98..4120376 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,10 +1,15 @@
+0.23
+
+    (NO FEATURE CHANGES)
+    - fixed english docs(hanekomu++)
+
 0.22

     - fixed issue on ithreads(broken from 0.21)

 0.21

-    - doc enhancment
+    - doc enhancments
     - micro performance tuning.

 0.20
---
 perl/Changes                           |  7 ++++++-
 perl/README                            | 22 +++++++++++++---------
 perl/lib/Data/MessagePack.pm           | 16 +++++++++-------
 perl/lib/Data/MessagePack/Unpacker.pod |  8 ++++----
 4 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/perl/Changes b/perl/Changes
index dd47b98..4120376 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,10 +1,15 @@
+0.23
+
+    (NO FEATURE CHANGES)
+    - fixed english docs(hanekomu++)
+
 0.22
 
     - fixed issue on ithreads(broken from 0.21)
 
 0.21
 
-    - doc enhancment
+    - doc enhancments
     - micro performance tuning.
 
 0.20
diff --git a/perl/README b/perl/README
index dec6125..e46323d 100644
--- a/perl/README
+++ b/perl/README
@@ -22,35 +22,37 @@ ABOUT MESSAGEPACK FORMAT
             say length(Storable::nfreeze({a=>1, b=>2}));       # => 21
             say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7
 
-        MessagePack format saves memory than JSON and Storable format.
+        The MessagePack format saves memory than JSON and Storable format.
 
     STREAMING DESERIALIZER
         MessagePack supports streaming deserializer. It is useful for
         networking such as RPC.
 
-    If you want to get more informations about messagepack format, please
+    If you want to get more information about the MessagePack format, please
     visit to .
 
 METHODS
     my $packed = Data::MessagePack->pack($data[, $max_depth]);
         Pack the $data to messagepack format string.
 
-        This method throws exception when nesting perl structure more than
-        $max_depth(default: 512) for detecting circular reference.
+        This method throws an exception when the perl structure is nested
+        more than $max_depth levels(default: 512) in order to detect
+        circular references.
 
-        Data::MessagePack->pack() throws exception when encountered blessed
-        object. Because MessagePack is language independent format.
+        Data::MessagePack->pack() throws an exception when encountering
+        blessed object, because MessagePack is language-independent format.
 
     my $unpacked = Data::MessagePack->unpack($msgpackstr);
-        unpack the $msgpackstr to messagepack format string.
+        unpack the $msgpackstr to a MessagePack format string.
 
 Configuration Variables
     $Data::MessagePack::PreferInteger
         Pack the string as int when the value looks like int(EXPERIMENTAL).
 
 SPEED
-    This is result of benchmark/serialize.pl and benchmark/deserialize.pl on
-    my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP).
+    This is the result of benchmark/serialize.pl and
+    benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu
+    SMP).
 
         -- serialize
         JSON::XS: 2.3
@@ -82,6 +84,8 @@ THANKS TO
 
     FURUHASHI Sadayuki
 
+    hanekomu
+
 LICENSE
     This library is free software; you can redistribute it and/or modify it
     under the same terms as Perl itself.
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 516e98b..3511628 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.22';
+our $VERSION = '0.23';
 our $PreferInteger = 0;
 
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };
@@ -60,7 +60,7 @@ Messagepack is language independent binary serialize format.
     say length(Storable::nfreeze({a=>1, b=>2}));       # => 21
     say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7
 
-MessagePack format saves memory than JSON and Storable format.
+The MessagePack format saves memory than JSON and Storable format.
 
 =item STREAMING DESERIALIZER
 
@@ -68,7 +68,7 @@ MessagePack supports streaming deserializer. It is useful for networking such as
 
 =back
 
-If you want to get more informations about messagepack format, please visit to L.
+If you want to get more information about the MessagePack format, please visit to L.
 
 =head1 METHODS
 
@@ -78,13 +78,13 @@ If you want to get more informations about messagepack format, please visit to L
 
 Pack the $data to messagepack format string.
 
-This method throws exception when nesting perl structure more than $max_depth(default: 512) for detecting circular reference.
+This method throws an exception when the perl structure is nested more than $max_depth levels(default: 512) in order to detect circular references.
 
-Data::MessagePack->pack() throws exception when encountered blessed object. Because MessagePack is language independent format.
+Data::MessagePack->pack() throws an exception when encountering blessed object, because MessagePack is language-independent format.
 
 =item my $unpacked = Data::MessagePack->unpack($msgpackstr);
 
-unpack the $msgpackstr to messagepack format string.
+unpack the $msgpackstr to a MessagePack format string.
 
 =back
 
@@ -100,7 +100,7 @@ Pack the string as int when the value looks like int(EXPERIMENTAL).
 
 =head1 SPEED
 
-This is result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP).
+This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP).
 
     -- serialize
     JSON::XS: 2.3
@@ -134,6 +134,8 @@ Dan Kogai
 
 FURUHASHI Sadayuki
 
+hanekomu
+
 =head1 LICENSE
 
 This library is free software; you can redistribute it and/or modify
diff --git a/perl/lib/Data/MessagePack/Unpacker.pod b/perl/lib/Data/MessagePack/Unpacker.pod
index c24eaf1..2bc4549 100644
--- a/perl/lib/Data/MessagePack/Unpacker.pod
+++ b/perl/lib/Data/MessagePack/Unpacker.pod
@@ -14,7 +14,7 @@ Data::MessagePack::Unpacker - messagepack streaming deserializer
 
 =head1 DESCRIPTION
 
-This is an streaming deserializer for messagepack.
+This is a streaming deserializer for messagepack.
 
 =head1 METHODS
 
@@ -22,7 +22,7 @@ This is an streaming deserializer for messagepack.
 
 =item my $up = Data::MessagePack::Unpacker->new()
 
-create new instance of stream deserializer.
+creates a new instance of stream deserializer.
 
 =item my $ret = $up->execute($data, $offset);
 
@@ -39,11 +39,11 @@ is this deserializer finished?
 
 =item my $data = $up->data();
 
-returns deserialized object.
+returns the deserialized object.
 
 =item $up->reset();
 
-reset the stream deserializer, without memory zone.
+resets the stream deserializer, without memory zone.
 
 =back
 

From a9566b31be76ebc8aaf9149c1a09d3720c3f2ec9 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sun, 12 Sep 2010 22:21:33 +0900
Subject: [PATCH 0692/1648] add annotation utilities for Java

---
 .../org/msgpack/tmp/DynamicCodeGenerator.java | 206 --------
 .../src/main/java/org/msgpack/tmp/Image1.java |  47 --
 .../src/main/java/org/msgpack/tmp/Image2.java |  85 ---
 .../src/main/java/org/msgpack/tmp/Image3.java |  38 --
 .../msgpack/tmp/ImagePackUnpackPerfTest.java  | 134 -----
 .../util/annotation/MessagePackOptional.java  |  12 +
 .../util/annotation/MessagePackRequired.java  |  12 +
 .../annotation/MessagePackUnpackable.java     |  11 +
 .../util/annotation/PackUnpackUtil.java       | 498 ++++++++++++++++++
 .../annotation/PackUnpackUtilException.java   |  12 +
 10 files changed, 545 insertions(+), 510 deletions(-)
 delete mode 100644 java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java
 delete mode 100644 java/src/main/java/org/msgpack/tmp/Image1.java
 delete mode 100644 java/src/main/java/org/msgpack/tmp/Image2.java
 delete mode 100644 java/src/main/java/org/msgpack/tmp/Image3.java
 delete mode 100644 java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java
 create mode 100644 java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java
 create mode 100644 java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java
 create mode 100644 java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java
 create mode 100644 java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
 create mode 100644 java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java

diff --git a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java b/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java
deleted file mode 100644
index 78238b4..0000000
--- a/java/src/main/java/org/msgpack/tmp/DynamicCodeGenerator.java
+++ /dev/null
@@ -1,206 +0,0 @@
-package org.msgpack.tmp;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.lang.reflect.Field;
-import java.util.HashMap;
-
-import javassist.ClassPool;
-import javassist.CtClass;
-import javassist.CtConstructor;
-import javassist.CtMethod;
-import javassist.CtNewConstructor;
-import javassist.CtNewMethod;
-
-import org.msgpack.Packer;
-import org.msgpack.Unpacker;
-
-public class DynamicCodeGenerator {
-
-	private static final String ENHANCED_CLASS_NAME_POSTFIX = "_$$_Enhanced";
-
-	private static final String SPACE = " ";
-
-	private static final String MODIFIER_PUBLIC = "public";
-
-	private static final String METHOD_NAME_MESSAGEPACK = "messagePack";
-
-	private static final String METHOD_NAME_MESSAGEUNPACK = "messageUnpack";
-
-	private static final String PACKER_CLASS_TYPE_NAME = Packer.class.getName();
-
-	private static final String VOID_TYPE_NAME = "void";
-
-	private static final String PACKER_OBJECT_NAME = "pk";
-
-	private static final String UNPACKER_CLASS_TYPE_NAME = Unpacker.class
-			.getName();
-
-	private static final String UNPACKER_OBJECT_NAME = "pk";
-
-	private HashMap> classMap;
-
-	private HashMap objectCacheMap;
-
-	private ClassPool pool;
-
-	public DynamicCodeGenerator() {
-		classMap = new HashMap>();
-		objectCacheMap = new HashMap();
-		pool = ClassPool.getDefault();
-	}
-
-	public Object newEnhancedInstance(Class targetClass) throws Exception {
-		String targetClassName = targetClass.getName();
-		// Class enhancedClass = classMap.get(targetClassName);
-		Object enhancedObject = objectCacheMap.get(targetClassName);
-		// if (enhancedClass == null) {
-		if (enhancedObject == null) {
-			CtClass enhancedCtClass = createEnhancedCtClass(targetClassName);
-			// System.out.println("enhanced class name: "
-			// + enhancedCtClass.getName());
-			addSuperclass(enhancedCtClass, targetClassName);
-			addConstructor(enhancedCtClass);
-			createMessagePackMethod(enhancedCtClass, targetClass);
-			createMessageUnpackMethod(enhancedCtClass, targetClass);
-			Class enhancedClass = loadEnhancedClass(enhancedCtClass);
-			// classMap.put(targetClassName, enhancedClass);
-			enhancedObject = enhancedClass.newInstance();
-			objectCacheMap.put(targetClassName, enhancedObject);
-		}
-		// return newEnhancedInstance0(enhancedClass);
-		return enhancedObject;
-	}
-
-	private CtClass createEnhancedCtClass(final String targetClassName)
-			throws Exception {
-		return pool.makeClass(targetClassName + ENHANCED_CLASS_NAME_POSTFIX);
-	}
-
-	private void addSuperclass(CtClass enhancedCtClass,
-			final String targetClassName) throws Exception {
-		CtClass targetCtClass = pool.get(targetClassName);
-		enhancedCtClass.setSuperclass(targetCtClass);
-	}
-
-	private void addConstructor(CtClass enhancedCtClass) throws Exception {
-		CtConstructor newCtConstructor = CtNewConstructor
-				.defaultConstructor(enhancedCtClass);
-		enhancedCtClass.addConstructor(newCtConstructor);
-	}
-
-	private void createMessagePackMethod(CtClass enhancedCtClass,
-			Class targetClass) throws Exception {
-		StringBuilder sb = new StringBuilder();
-		sb.append(MODIFIER_PUBLIC).append(SPACE).append(VOID_TYPE_NAME).append(
-				SPACE).append(METHOD_NAME_MESSAGEPACK).append("(").append(
-				PACKER_CLASS_TYPE_NAME).append(SPACE)
-				.append(PACKER_OBJECT_NAME).append(")").append(SPACE).append(
-						"throws").append(SPACE).append("java.io.IOException")
-				.append(SPACE).append("{");
-		Field[] fields = targetClass.getFields();
-		sb.append(PACKER_OBJECT_NAME).append(".").append("packArray").append(
-				"(").append(fields.length).append(")").append(";");
-		for (Field field : fields) {
-			insertCodeOfMessagePack(field, sb);
-		}
-		sb.append("}");
-		// System.out.println("messagePack method: " + sb.toString());
-		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass);
-		enhancedCtClass.addMethod(newCtMethod);
-	}
-
-	private void insertCodeOfMessagePack(Field field, StringBuilder sb) {
-		Class type = field.getType();
-		if (type.equals(int.class)) {
-			sb.append(PACKER_OBJECT_NAME).append(".").append("pack")
-					.append("(").append(field.getName()).append(")")
-					.append(";");
-		} else if (type.equals(String.class)) {
-			sb.append(PACKER_OBJECT_NAME).append(".").append("pack")
-					.append("(").append(field.getName()).append(")")
-					.append(";");
-		} else {
-			throw new UnsupportedOperationException();
-		}
-	}
-
-	private void createMessageUnpackMethod(CtClass enhancedCtClass,
-			Class targetClass) throws Exception {
-		StringBuilder sb = new StringBuilder();
-		sb
-				.append(MODIFIER_PUBLIC)
-				.append(SPACE)
-				.append(VOID_TYPE_NAME)
-				.append(SPACE)
-				.append(METHOD_NAME_MESSAGEUNPACK)
-				.append("(")
-				.append(UNPACKER_CLASS_TYPE_NAME)
-				.append(SPACE)
-				.append(UNPACKER_OBJECT_NAME)
-				.append(")")
-				.append(SPACE)
-				.append("throws")
-				.append(SPACE)
-				.append("org.msgpack.MessageTypeException, java.io.IOException")
-				.append(SPACE).append("{");
-		Field[] fields = targetClass.getFields();
-		sb.append(UNPACKER_OBJECT_NAME).append(".").append("unpackArray()")
-				.append(";");
-		// TODO
-		for (Field field : fields) {
-			insertCodeOfMessageUnpack(field, sb);
-		}
-		sb.append("}");
-		// System.out.println("messageUnpack method: " + sb.toString());
-		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhancedCtClass);
-		enhancedCtClass.addMethod(newCtMethod);
-	}
-
-	private void insertCodeOfMessageUnpack(Field field, StringBuilder sb) {
-		Class type = field.getType();
-		if (type.equals(int.class)) {
-			sb.append(field.getName()).append(SPACE).append("=").append(SPACE)
-					.append(PACKER_OBJECT_NAME).append(".").append("unpackInt")
-					.append("(").append(")").append(";");
-		} else if (type.equals(String.class)) {
-			sb.append(field.getName()).append(SPACE).append("=").append(SPACE)
-					.append(PACKER_OBJECT_NAME).append(".").append(
-							"unpackString").append("(").append(")").append(";");
-		} else {
-			throw new UnsupportedOperationException();
-		}
-	}
-
-	private Class loadEnhancedClass(CtClass enhancedCtClass)
-			throws Exception {
-		return enhancedCtClass.toClass(null, null);
-	}
-
-	private Object newEnhancedInstance0(Class enhancedClass) {
-		try {
-			return enhancedClass.newInstance();
-		} catch (Exception e) {
-			throw new UnsupportedOperationException();
-		}
-	}
-
-	public static void main(String[] args) throws Exception {
-		DynamicCodeGenerator gen = new DynamicCodeGenerator();
-		Image3 src = (Image3) gen.newEnhancedInstance(Image3.class);
-		src.title = "msgpack";
-		src.uri = "http://msgpack.org/";
-		src.width = 2560;
-		src.height = 1600;
-		src.size = 4096000;
-
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		src.messagePack(new Packer(out));
-		Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		dst.messageUnpack(pac);
-
-//		System.out.println(dst.equals(src));
-	}
-}
diff --git a/java/src/main/java/org/msgpack/tmp/Image1.java b/java/src/main/java/org/msgpack/tmp/Image1.java
deleted file mode 100644
index f1819ed..0000000
--- a/java/src/main/java/org/msgpack/tmp/Image1.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package org.msgpack.tmp;
-
-import org.msgpack.*;
-import java.io.*;
-
-public class Image1 implements MessagePackable, MessageUnpackable {
-	public String uri = "";
-	public String title = "";
-	public int width = 0;
-	public int height = 0;
-	public int size = 0;
-
-	public void messagePack(Packer pk) throws IOException {
-		pk.packArray(5);
-		pk.pack(uri);
-		pk.pack(title);
-		pk.pack(width);
-		pk.pack(height);
-		pk.pack(size);
-	}
-
-	public void messageUnpack(Unpacker pac) throws IOException,
-			MessageTypeException {
-		int length = pac.unpackArray();
-		if (length != 5) {
-			throw new MessageTypeException();
-		}
-		uri = pac.unpackString();
-		title = pac.unpackString();
-		width = pac.unpackInt();
-		height = pac.unpackInt();
-		size = pac.unpackInt();
-	}
-
-	public boolean equals(Image1 obj) {
-		return uri.equals(obj.uri) && title.equals(obj.title)
-				&& width == obj.width && height == obj.height
-				&& size == obj.size;
-	}
-
-	public boolean equals(Object obj) {
-		if (obj.getClass() != Image1.class) {
-			return false;
-		}
-		return equals((Image1) obj);
-	}
-}
diff --git a/java/src/main/java/org/msgpack/tmp/Image2.java b/java/src/main/java/org/msgpack/tmp/Image2.java
deleted file mode 100644
index 9c357bf..0000000
--- a/java/src/main/java/org/msgpack/tmp/Image2.java
+++ /dev/null
@@ -1,85 +0,0 @@
-package org.msgpack.tmp;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-
-import org.msgpack.MessageTypeException;
-import org.msgpack.Packer;
-import org.msgpack.Unpacker;
-
-public class Image2 {
-	public String uri = "";
-	public String title = "";
-	public int width = 0;
-	public int height = 0;
-	public int size = 0;
-
-	public void messagePack(Packer pk) throws IOException {
-		messagePackWithReflection(pk);
-	}
-
-	public void messagePackWithReflection(Packer pk) throws IOException {
-		Class cl = this.getClass();
-		Field[] fields = cl.getFields();
-		pk.packArray(fields.length);
-		for (Field field : fields) {
-			try {
-				Object obj = field.get(this);
-				pk.pack(obj);
-			} catch (IllegalArgumentException e) {
-				e.printStackTrace();
-			} catch (IllegalAccessException e) {
-				e.printStackTrace();
-			}
-		}
-	}
-
-	public void messageUnpack(Unpacker pac) throws IOException,
-			MessageTypeException {
-		messageUnpackWithReflection(pac);
-	}
-	
-	public void messageUnpackWithReflection(Unpacker pac) throws IOException,
-			MessageTypeException {
-		Class cl = this.getClass();
-		Field[] fields = cl.getFields();
-		int length = pac.unpackArray();
-		if (length != fields.length) {
-			throw new MessageTypeException();
-		}
-		for (Field field : fields) {
-			try {
-				field.set(this, unpack(pac, field.getType()));
-			} catch (IOException e) {
-				e.printStackTrace();
-			} catch (IllegalArgumentException e) {
-				e.printStackTrace();
-			} catch (IllegalAccessException e) {
-				e.printStackTrace();
-			}
-		}
-	}
-	
-	public static Object unpack(Unpacker pk, Class cl) throws IOException {
-		if (cl == int.class) {
-			return pk.unpackInt();
-		} else if (cl == String.class) {
-			return pk.unpackString();
-		} else {
-			throw new UnsupportedOperationException();
-		}
-	}
-
-	public boolean equals(Image2 obj) {
-		return uri.equals(obj.uri) && title.equals(obj.title)
-				&& width == obj.width && height == obj.height
-				&& size == obj.size;
-	}
-
-	public boolean equals(Object obj) {
-		if (obj.getClass() != Image2.class) {
-			return false;
-		}
-		return equals((Image2) obj);
-	}
-}
diff --git a/java/src/main/java/org/msgpack/tmp/Image3.java b/java/src/main/java/org/msgpack/tmp/Image3.java
deleted file mode 100644
index a28ff53..0000000
--- a/java/src/main/java/org/msgpack/tmp/Image3.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.msgpack.tmp;
-
-import java.io.IOException;
-import org.msgpack.MessagePackable;
-import org.msgpack.MessageTypeException;
-import org.msgpack.MessageUnpackable;
-import org.msgpack.Packer;
-import org.msgpack.Unpacker;
-
-public class Image3 implements MessagePackable, MessageUnpackable {
-	public String uri = "";
-	public String title = "";
-	public int width = 0;
-	public int height = 0;
-	public int size = 0;
-
-	public void messagePack(Packer pk) throws IOException {
-		// empty
-	}
-
-	public void messageUnpack(Unpacker pac) throws IOException,
-			MessageTypeException {
-		// empty
-	}
-
-	public boolean equals(Image3 obj) {
-		return uri.equals(obj.uri) && title.equals(obj.title)
-				&& width == obj.width && height == obj.height
-				&& size == obj.size;
-	}
-
-	public boolean equals(Object obj) {
-		if (obj.getClass() != Image3.class) {
-			return false;
-		}
-		return equals((Image3) obj);
-	}
-}
\ No newline at end of file
diff --git a/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java b/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java
deleted file mode 100644
index 1f27c1b..0000000
--- a/java/src/main/java/org/msgpack/tmp/ImagePackUnpackPerfTest.java
+++ /dev/null
@@ -1,134 +0,0 @@
-package org.msgpack.tmp;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-
-import org.msgpack.Packer;
-import org.msgpack.Unpacker;
-
-public class ImagePackUnpackPerfTest {
-
-	private static int BIG_LOOP = 5;
-
-	private static int SMALL_LOOP = 50000;
-
-	private static DynamicCodeGenerator gen;
-
-	private static void doGc() {
-		try {
-			Thread.sleep(50L);
-		} catch (InterruptedException ie) {
-			System.err
-					.println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()");
-		}
-		System.gc();
-		try { // longer sleep afterwards (not needed by GC, but may help with
-			// scheduling)
-			Thread.sleep(200L);
-		} catch (InterruptedException ie) {
-			System.err
-					.println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()");
-		}
-	}
-
-	public void doIt(int versionID) {
-		try {
-			doIt0(versionID);
-		} catch (Exception e) {
-			e.printStackTrace();
-		}
-		try {
-			for (int j = 0; j < BIG_LOOP; ++j) {
-				long t = System.currentTimeMillis();
-				doIt0(versionID);
-				t = System.currentTimeMillis() - t;
-				System.out.println("time: " + t);
-			}
-		} catch (Exception e) {
-			e.printStackTrace();
-		}
-	}
-
-	public void doIt0(int versionID) throws Exception {
-		for (int i = 0; i < SMALL_LOOP; ++i) {
-			switch (versionID) {
-			case 0:
-				doCurrentVersion();
-				break;
-			case 1:
-				doWithReflection();
-				break;
-			case 2:
-				doWithDynamicCodeGeneration();
-				break;
-			default:
-				throw new RuntimeException();
-			}
-		}
-	}
-
-	public void doCurrentVersion() throws Exception {
-		Image1 src = new Image1();
-		src.title = "msgpack";
-		src.uri = "http://msgpack.org/";
-		src.width = 2560;
-		src.height = 1600;
-		src.size = 4096000;
-
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		src.messagePack(new Packer(out));
-		Image1 dst = new Image1();
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		dst.messageUnpack(pac);
-	}
-
-	public void doWithReflection() throws Exception {
-		Image2 src = new Image2();
-		src.title = "msgpack";
-		src.uri = "http://msgpack.org/";
-		src.width = 2560;
-		src.height = 1600;
-		src.size = 4096000;
-
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		src.messagePack(new Packer(out));
-		Image2 dst = new Image2();
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		dst.messageUnpack(pac);
-	}
-
-	public void doWithDynamicCodeGeneration() throws Exception {
-		Image3 src = (Image3) gen.newEnhancedInstance(Image3.class);
-		src.title = "msgpack";
-		src.uri = "http://msgpack.org/";
-		src.width = 2560;
-		src.height = 1600;
-		src.size = 4096000;
-
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		src.messagePack(new Packer(out));
-		Image3 dst = (Image3) gen.newEnhancedInstance(Image3.class);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		dst.messageUnpack(pac);
-	}
-
-	public static void main(String[] args) {
-		ImagePackUnpackPerfTest test = new ImagePackUnpackPerfTest();
-
-		doGc();
-		System.out.println("test current version");
-		test.doIt(0);
-
-		doGc();
-		System.out.println("test with reflection");
-		test.doIt(1);
-
-		doGc();
-		System.out.println("test with dynamic codegen");
-		gen = new DynamicCodeGenerator();
-		test.doIt(2);
-	}
-}
diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java
new file mode 100644
index 0000000..a565292
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java
@@ -0,0 +1,12 @@
+package org.msgpack.util.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.FIELD)
+@Retention(RetentionPolicy.CLASS)
+public @interface MessagePackOptional {
+	int value() default -1;
+}
\ No newline at end of file
diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java
new file mode 100644
index 0000000..03e50cf
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java
@@ -0,0 +1,12 @@
+package org.msgpack.util.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.FIELD)
+@Retention(RetentionPolicy.CLASS)
+public @interface MessagePackRequired {
+	int value() default -1;
+}
\ No newline at end of file
diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java
new file mode 100644
index 0000000..473b541
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java
@@ -0,0 +1,11 @@
+package org.msgpack.util.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.CLASS)
+public @interface MessagePackUnpackable {
+}
diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
new file mode 100644
index 0000000..37b9e0d
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
@@ -0,0 +1,498 @@
+package org.msgpack.util.annotation;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javassist.CannotCompileException;
+import javassist.ClassPool;
+import javassist.CtClass;
+import javassist.CtConstructor;
+import javassist.CtField;
+import javassist.CtMethod;
+import javassist.CtNewConstructor;
+import javassist.CtNewMethod;
+import javassist.NotFoundException;
+
+import org.msgpack.MessageConvertable;
+import org.msgpack.MessagePackable;
+import org.msgpack.MessageTypeException;
+import org.msgpack.MessageUnpackable;
+import org.msgpack.Packer;
+import org.msgpack.Unpacker;
+
+public class PackUnpackUtil {
+	static class Constants {
+		static final String POSTFIX_TYPE_NAME_ENHANCER = "_$$_Enhanced";
+
+		static final String KEYWORD_MODIFIER_PUBLIC = "public";
+
+		static final String KEYWORD_THROWS = "throws";
+
+		static final String TYPE_NAME_VOID = void.class.getName();
+
+		static final String TYPE_NAME_BOOLEAN = boolean.class.getName();
+
+		static final String TYPE_NAME_BYTE = byte.class.getName();
+
+		static final String TYPE_NAME_DOUBLE = double.class.getName();
+
+		static final String TYPE_NAME_FLOAT = float.class.getName();
+
+		static final String TYPE_NAME_INT = int.class.getName();
+
+		static final String TYPE_NAME_LONG = long.class.getName();
+
+		static final String TYPE_NAME_SHORT = short.class.getName();
+
+		static final String TYPE_NAME_OBJECT = Object.class.getName();
+
+		static final String TYPE_NAME_BIGINTEGER = BigInteger.class.getName();
+
+		static final String TYPE_NAME_STRING = String.class.getName();
+
+		static final String TYPE_NAME_BOOLEAN2 = Boolean.class.getName();
+
+		static final String TYPE_NAME_BYTE2 = Byte.class.getName();
+
+		static final String TYPE_NAME_DOUBLE2 = Double.class.getName();
+
+		static final String TYPE_NAME_FLOAT2 = Float.class.getName();
+
+		static final String TYPE_NAME_INT2 = Integer.class.getName();
+
+		static final String TYPE_NAME_LONG2 = Long.class.getName();
+
+		static final String TYPE_NAME_SHORT2 = Short.class.getName();
+
+		static final String TYPE_NAME_BYTEARRAY = byte[].class.getName();
+
+		static final String TYPE_NAME_LIST = List.class.getName();
+
+		static final String TYPE_NAME_MAP = Map.class.getName();
+
+		static final String TYPE_NAME_IOEXCEPTION = IOException.class.getName();
+
+		static final String TYPE_NAME_PACKER = Packer.class.getName();
+
+		static final String TYPE_NAME_UNPACKER = Unpacker.class.getName();
+
+		static final String TYPE_NAME_MSGPACKABLE = MessagePackable.class
+				.getName();
+
+		static final String TYPE_NAME_MSGUNPACKABLE = MessageUnpackable.class
+				.getName();
+
+		static final String TYPE_NAME_MSGCONVERTABLE = MessageConvertable.class
+				.getName();
+
+		static final String TYPE_NAME_MSGTYPEEXCEPTION = MessageTypeException.class
+				.getName();
+
+		static final String TYPE_NAME_MSGPACKUNPACKABLE = MessagePackUnpackable.class
+				.getName();
+
+		static final String TYPE_NAME_MSGPACKOPTIONAL = MessagePackOptional.class
+				.getName();
+
+		static final String TYPE_NAME_MSGPACKOREQUIRED = MessagePackRequired.class
+				.getName();
+
+		static final String CHAR_NAME_SPACE = " ";
+
+		static final String CHAR_NAME_COMMA = ",";
+
+		static final String CHAR_NAME_EQUAL = "=";
+
+		static final String CHAR_NAME_RIGHT_PARENTHESIS = ")";
+
+		static final String CHAR_NAME_LEFT_PARENTHESIS = "(";
+
+		static final String CHAR_NAME_RIGHT_CURLY_BRACHET = "}";
+
+		static final String CHAR_NAME_LEFT_CURLY_BRACHET = "{";
+
+		static final String CHAR_NAME_DOT = ".";
+
+		static final String CHAR_NAME_SEMICOLON = ";";
+
+		static final String VARIABLE_NAME_PK = "pk";
+
+		static final String VARIABLE_NAME_OBJ = "obj";
+
+		static final String METHOD_NAME_MSGPACK = "messagePack";
+
+		static final String METHOD_NAME_MSGUNPACK = "messageUnpack";
+
+		static final String METHOD_NAME_MSGCONVERT = "messageConvert";
+
+		static final String METHOD_NAME_PACK = "pack";
+
+		static final String METHOD_NAME_PACKARRAY = "packArray";
+
+		static final String METHOD_NAME_UNPACK = "unpack";
+
+		static final String METHOD_NAME_UNPACKBOOLEAN = "unpackBoolean";
+
+		static final String METHOD_NAME_UNPACKBYTE = "unpackByte";
+
+		static final String METHOD_NAME_UNPACKDOUBLE = "unpackDouble";
+
+		static final String METHOD_NAME_UNPACKFLOAT = "unpackFloat";
+
+		static final String METHOD_NAME_UNPACKINT = "unpackInt";
+
+		static final String METHOD_NAME_UNPACKLONG = "unpackLong";
+
+		static final String METHOD_NAME_UNPACKSHORT = "unpackShort";
+
+		static final String METHOD_NAME_UNPACKSTRING = "unpackString";
+
+		static final String METHOD_NAME_UNPACKBIGINTEGER = "unpackBigInteger";
+
+		static final String METHOD_NAME_UNPACKOBJECT = "unpackObject";
+
+		static final String METHOD_NAME_UNPACKBYTEARRAY = "unpackByteArray";
+
+		static final String METHOD_NAME_UNPACKARRAY = "unpackArray";
+
+		static final String METHOD_NAME_UNPACKMAP = "unpackMap";
+	}
+
+	public static class Enhancer {
+
+		private ConcurrentHashMap> classCache;
+
+		private ClassPool pool;
+
+		protected Enhancer() {
+			classCache = new ConcurrentHashMap>();
+			pool = ClassPool.getDefault();
+		}
+
+		protected Class getCache(String origName) {
+			return classCache.get(origName);
+		}
+
+		protected void setCache(String origName, Class enhClass) {
+			classCache.putIfAbsent(origName, enhClass);
+		}
+
+		protected Class generate(Class origClass)
+				throws NotFoundException, CannotCompileException {
+			String origName = origClass.getName();
+			String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER;
+			CtClass origCtClass = pool.get(origName);
+			checkPackUnpackAnnotation(origCtClass);
+			CtClass enhCtClass = pool.makeClass(enhName);
+			setSuperclass(enhCtClass, origCtClass);
+			setInterfaces(enhCtClass);
+			addConstructor(enhCtClass);
+			addMessagePackMethod(enhCtClass, origCtClass);
+			addMessageUnpackMethod(enhCtClass, origCtClass);
+			addMessageConvertMethod(enhCtClass, origCtClass);
+			return createClass(enhCtClass);
+		}
+
+		private void checkPackUnpackAnnotation(CtClass origCtClass) {
+			try {
+				Object[] objs = origCtClass.getAnnotations();
+				for (Object obj : objs) {
+					if (obj instanceof MessagePackUnpackable) {
+						return;
+					}
+				}
+				throw new PackUnpackUtilException(
+						"Not annotated with this class: "
+								+ origCtClass.getName());
+			} catch (ClassNotFoundException e) {
+				throw new PackUnpackUtilException(e.getMessage(), e);
+			}
+		}
+
+		private void setSuperclass(CtClass enhCtClass, CtClass origCtClass)
+				throws CannotCompileException {
+			enhCtClass.setSuperclass(origCtClass);
+		}
+
+		private void setInterfaces(CtClass enhCtClass) throws NotFoundException {
+			CtClass pacCtClass = pool.get(Constants.TYPE_NAME_MSGPACKABLE);
+			enhCtClass.addInterface(pacCtClass);
+			CtClass unpacCtClass = pool.get(Constants.TYPE_NAME_MSGUNPACKABLE);
+			enhCtClass.addInterface(unpacCtClass);
+			CtClass convCtClass = pool.get(Constants.TYPE_NAME_MSGCONVERTABLE);
+			enhCtClass.addInterface(convCtClass);
+		}
+
+		private void addConstructor(CtClass enhCtClass)
+				throws CannotCompileException {
+			CtConstructor newCtCons = CtNewConstructor
+					.defaultConstructor(enhCtClass);
+			enhCtClass.addConstructor(newCtCons);
+		}
+
+		private void addMessagePackMethod(CtClass enhCtClass,
+				CtClass origCtClass) throws CannotCompileException,
+				NotFoundException {
+			StringBuilder sb = new StringBuilder();
+			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append(
+					Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID)
+					.append(Constants.CHAR_NAME_SPACE).append(
+							Constants.METHOD_NAME_MSGPACK).append(
+							Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
+							Constants.TYPE_NAME_PACKER).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.VARIABLE_NAME_PK).append(
+							Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.KEYWORD_THROWS).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.TYPE_NAME_IOEXCEPTION).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append(
+							Constants.CHAR_NAME_SPACE);
+			CtField[] fields = origCtClass.getDeclaredFields();
+			sb.append(Constants.VARIABLE_NAME_PK).append(
+					Constants.CHAR_NAME_DOT).append(
+					Constants.METHOD_NAME_PACKARRAY).append(
+					Constants.CHAR_NAME_LEFT_PARENTHESIS).append(fields.length)
+					.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
+							Constants.CHAR_NAME_SEMICOLON).append(
+							Constants.CHAR_NAME_SPACE);
+			for (CtField field : fields) {
+				insertCodeOfMessagePack(field, sb);
+			}
+			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
+			System.out.println("messagePack method: " + sb.toString());
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
+			enhCtClass.addMethod(newCtMethod);
+		}
+
+		private void insertCodeOfMessagePack(CtField field, StringBuilder sb)
+				throws NotFoundException {
+			sb.append(Constants.VARIABLE_NAME_PK).append(
+					Constants.CHAR_NAME_DOT).append(Constants.METHOD_NAME_PACK)
+					.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
+							field.getName()).append(
+							Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
+							Constants.CHAR_NAME_SEMICOLON).append(
+							Constants.CHAR_NAME_SPACE);
+		}
+
+		private void addMessageUnpackMethod(CtClass enhCtClass,
+				CtClass origCtClass) throws CannotCompileException,
+				NotFoundException {
+			StringBuilder sb = new StringBuilder();
+			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append(
+					Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID)
+					.append(Constants.CHAR_NAME_SPACE).append(
+							Constants.METHOD_NAME_MSGUNPACK).append(
+							Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
+							Constants.TYPE_NAME_UNPACKER).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.VARIABLE_NAME_PK).append(
+							Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.KEYWORD_THROWS).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.TYPE_NAME_MSGTYPEEXCEPTION).append(
+							Constants.CHAR_NAME_COMMA).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.TYPE_NAME_IOEXCEPTION).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append(
+							Constants.CHAR_NAME_SPACE);
+			CtField[] fields = origCtClass.getFields();
+			sb.append(Constants.VARIABLE_NAME_PK).append(
+					Constants.CHAR_NAME_DOT).append(
+					Constants.METHOD_NAME_UNPACKARRAY).append(
+					Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
+					Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
+					Constants.CHAR_NAME_SEMICOLON).append(
+					Constants.CHAR_NAME_SPACE);
+			for (CtField field : fields) {
+				insertCodeOfMessageUnpack(field, sb);
+			}
+			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
+			System.out.println("messageUnpack method: " + sb.toString());
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
+			enhCtClass.addMethod(newCtMethod);
+		}
+
+		private void insertCodeOfMessageUnpack(CtField field, StringBuilder sb)
+				throws NotFoundException {
+			CtClass type = field.getType();
+			sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE)
+					.append(Constants.CHAR_NAME_EQUAL).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.VARIABLE_NAME_PK).append(
+							Constants.CHAR_NAME_DOT);
+			if (type.equals(CtClass.booleanType)) { // boolean
+				sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN);
+			} else if (type.equals(CtClass.byteType)) { // byte
+				sb.append(Constants.METHOD_NAME_UNPACKBYTE);
+			} else if (type.equals(CtClass.doubleType)) { // double
+				sb.append(Constants.METHOD_NAME_UNPACKDOUBLE);
+			} else if (type.equals(CtClass.floatType)) { // float
+				sb.append(Constants.METHOD_NAME_UNPACKFLOAT);
+			} else if (type.equals(CtClass.intType)) { // int
+				sb.append(Constants.METHOD_NAME_UNPACKINT);
+			} else if (type.equals(CtClass.longType)) { // long
+				sb.append(Constants.METHOD_NAME_UNPACKLONG);
+			} else if (type.equals(CtClass.shortType)) { // short
+				sb.append(Constants.METHOD_NAME_UNPACKSHORT);
+			} else { // reference type
+				if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2))) { // Boolean
+					sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN);
+				} else if (type.equals(pool.get(Constants.TYPE_NAME_BYTE2))) { // Byte
+					sb.append(Constants.METHOD_NAME_UNPACKBYTE);
+				} else if (type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2))) { // Double
+					sb.append(Constants.METHOD_NAME_UNPACKDOUBLE);
+				} else if (type.equals(pool.get(Constants.TYPE_NAME_FLOAT2))) { // Float
+					sb.append(Constants.METHOD_NAME_UNPACKFLOAT);
+				} else if (type.equals(pool.get(Constants.TYPE_NAME_INT2))) { // Integer
+					sb.append(Constants.METHOD_NAME_UNPACKINT);
+				} else if (type.equals(pool.get(Constants.TYPE_NAME_LONG2))) { // Long
+					sb.append(Constants.METHOD_NAME_UNPACKLONG);
+				} else if (type.equals(pool.get(Constants.TYPE_NAME_SHORT2))) { // Short
+					sb.append(Constants.METHOD_NAME_UNPACKSHORT);
+				} else if (type
+						.equals(pool.get(Constants.TYPE_NAME_BIGINTEGER))) { // BigInteger
+					sb.append(Constants.METHOD_NAME_UNPACKBIGINTEGER);
+				} else if (type.equals(pool.get(Constants.TYPE_NAME_STRING))) { // String
+					sb.append(Constants.METHOD_NAME_UNPACKSTRING);
+				} else if (type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY))) { // byte[]
+					sb.append(Constants.METHOD_NAME_UNPACKBYTEARRAY);
+				} else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST))) { // List
+					sb.append(Constants.METHOD_NAME_UNPACKARRAY);
+				} else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP))) { // Map
+					sb.append(Constants.METHOD_NAME_UNPACKMAP);
+				} else if (type.subtypeOf(pool
+						.get(Constants.TYPE_NAME_MSGUNPACKABLE))) { // MessageUnpackable
+					sb.append(Constants.METHOD_NAME_UNPACKOBJECT);
+				} else {
+					throw new NotFoundException("unknown type:  "
+							+ type.getName());
+				}
+			}
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
+					Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
+					Constants.CHAR_NAME_SEMICOLON).append(
+					Constants.CHAR_NAME_SPACE);
+		}
+
+		private void addMessageConvertMethod(CtClass enhCtClass,
+				CtClass origCtClass) throws CannotCompileException {
+			StringBuilder sb = new StringBuilder();
+			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append(
+					Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID)
+					.append(Constants.CHAR_NAME_SPACE).append(
+							Constants.METHOD_NAME_MSGCONVERT).append(
+							Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
+							Constants.TYPE_NAME_OBJECT).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.VARIABLE_NAME_OBJ).append(
+							Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.KEYWORD_THROWS).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.TYPE_NAME_MSGTYPEEXCEPTION).append(
+							Constants.CHAR_NAME_SPACE).append(
+							Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append(
+							Constants.CHAR_NAME_SPACE);
+			// TODO
+			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
+			System.out.println("messageConvert method: " + sb.toString());
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
+			enhCtClass.addMethod(newCtMethod);
+		}
+
+		private Class createClass(CtClass enhCtClass)
+				throws CannotCompileException {
+			return enhCtClass.toClass(null, null);
+		}
+	}
+
+	private static Enhancer enhancer;
+
+	public static Class getEnhancedClass(Class origClass) {
+		if (enhancer == null) {
+			enhancer = new Enhancer();
+		}
+
+		String origName = origClass.getName();
+		Class enhClass = enhancer.getCache(origName);
+		if (enhClass == null) {
+			// generate a class object related to the original class
+			try {
+				enhClass = enhancer.generate(origClass);
+			} catch (NotFoundException e) {
+				throw new PackUnpackUtilException(e.getMessage(), e);
+			} catch (CannotCompileException e) {
+				throw new PackUnpackUtilException(e.getMessage(), e);
+			}
+			// set the generated class to the cache
+			enhancer.setCache(origName, enhClass);
+		}
+		return enhClass;
+	}
+
+	public static Object newEnhancedInstance(Class origClass) {
+		try {
+			Class enhClass = getEnhancedClass(origClass);
+			// create a new object of the generated class
+			return enhClass.newInstance();
+		} catch (InstantiationException e) {
+			throw new PackUnpackUtilException(e.getMessage(), e);
+		} catch (IllegalAccessException e) {
+			throw new PackUnpackUtilException(e.getMessage(), e);
+		}
+	}
+
+	public static void main(final String[] args) throws Exception {
+		@MessagePackUnpackable
+		class Image {
+			public String uri = "";
+
+			public String title = "";
+
+			public int width = 0;
+
+			public int height = 0;
+
+			public int size = 0;
+
+			public boolean equals(Image obj) {
+				return uri.equals(obj.uri) && title.equals(obj.title)
+						&& width == obj.width && height == obj.height
+						&& size == obj.size;
+			}
+
+			public boolean equals(Object obj) {
+				if (obj.getClass() != Image.class) {
+					return false;
+				}
+				return equals((Image) obj);
+			}
+		}
+
+		PackUnpackUtil.getEnhancedClass(Image.class);
+		Image src = (Image) PackUnpackUtil.newEnhancedInstance(Image.class);
+		src.title = "msgpack";
+		src.uri = "http://msgpack.org/";
+		src.width = 2560;
+		src.height = 1600;
+		src.size = 4096000;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		Image dst = (Image) PackUnpackUtil.newEnhancedInstance(Image.class);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		pac.unpack((MessageUnpackable) dst);
+		System.out.println(src.equals(dst));
+	}
+}
diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java
new file mode 100644
index 0000000..df3af05
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java
@@ -0,0 +1,12 @@
+package org.msgpack.util.annotation;
+
+public class PackUnpackUtilException extends RuntimeException {
+
+	public PackUnpackUtilException(String reason) {
+		super(reason);
+	}
+	
+	public PackUnpackUtilException(String reason, Throwable t) {
+		super(reason, t);
+	}
+}

From 599b200ca5dda7957b47fb5156199a75a50a57ae Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sun, 12 Sep 2010 23:04:23 +0900
Subject: [PATCH 0693/1648] change the version of javassist 3.12.1.GA

---
 java/pom.xml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/java/pom.xml b/java/pom.xml
index 6591a72..70da6a6 100755
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -29,11 +29,11 @@
       4.8.1
       test
     
-
     
-      jboss
+      javassist
       javassist
-      3.7.ga
+      3.12.1.GA
+      compile
     
   
 
@@ -107,11 +107,12 @@
       MessagePack Maven2 Repository
       http://msgpack.org/maven2
     
-
     
-      repo2.maven.org
-      repo2.maven.org Maven2 Repository
-      http://repo2.maven.org/maven2
+      repository.jboss.org
+      https://repository.jboss.org/nexus/content/groups/public/
+      
+        false
+      
     
   
 

From 95b820305ae55823ee5b64476acfa5c2a7f44beb Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Tue, 14 Sep 2010 13:34:11 +0900
Subject: [PATCH 0694/1648] add a new test program for annotation-utilities

---
 .../util/annotation/PackUnpackUtil.java       | 116 +++++++-
 .../annotation/TestMessagePackUnpackable.java | 280 ++++++++++++++++++
 2 files changed, 385 insertions(+), 11 deletions(-)
 create mode 100644 java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java

diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
index 37b9e0d..5efef79 100644
--- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
+++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
@@ -3,6 +3,7 @@ package org.msgpack.util.annotation;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.lang.reflect.Modifier;
 import java.math.BigInteger;
 import java.util.List;
 import java.util.Map;
@@ -124,6 +125,8 @@ public class PackUnpackUtil {
 
 		static final String VARIABLE_NAME_OBJ = "obj";
 
+		static final String METHOD_NAME_VALUEOF = "valueOf";
+
 		static final String METHOD_NAME_MSGPACK = "messagePack";
 
 		static final String METHOD_NAME_MSGUNPACK = "messageUnpack";
@@ -187,7 +190,8 @@ public class PackUnpackUtil {
 			String origName = origClass.getName();
 			String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER;
 			CtClass origCtClass = pool.get(origName);
-			checkPackUnpackAnnotation(origCtClass);
+			checkClassValidation(origCtClass);
+			checkDefaultConstructorValidation(origCtClass);
 			CtClass enhCtClass = pool.makeClass(enhName);
 			setSuperclass(enhCtClass, origCtClass);
 			setInterfaces(enhCtClass);
@@ -198,6 +202,28 @@ public class PackUnpackUtil {
 			return createClass(enhCtClass);
 		}
 
+		private void checkClassValidation(CtClass origCtClass) {
+			// not public, abstract, final
+			int mod = origCtClass.getModifiers();
+			if ((!Modifier.isPublic(mod)) || Modifier.isAbstract(mod)
+					|| Modifier.isFinal(mod)) {
+				throwClassValidationException(origCtClass);
+			}
+			// interface, enum
+			if (origCtClass.isInterface() || origCtClass.isEnum()) {
+				throwClassValidationException(origCtClass);
+			}
+			// annotation
+			checkPackUnpackAnnotation(origCtClass);
+		}
+
+		private static void throwClassValidationException(CtClass origCtClass) {
+			throw new PackUnpackUtilException(
+					"it must be a public class and have @"
+							+ MessagePackUnpackable.class.getName() + ": "
+							+ origCtClass.getName());
+		}
+
 		private void checkPackUnpackAnnotation(CtClass origCtClass) {
 			try {
 				Object[] objs = origCtClass.getAnnotations();
@@ -206,14 +232,32 @@ public class PackUnpackUtil {
 						return;
 					}
 				}
-				throw new PackUnpackUtilException(
-						"Not annotated with this class: "
-								+ origCtClass.getName());
+				throwClassValidationException(origCtClass);
 			} catch (ClassNotFoundException e) {
 				throw new PackUnpackUtilException(e.getMessage(), e);
 			}
 		}
 
+		private void checkDefaultConstructorValidation(CtClass origCtClass) {
+			CtConstructor cons = null;
+			try {
+				cons = origCtClass.getDeclaredConstructor(new CtClass[0]);
+			} catch (NotFoundException e) {
+				throwConstructoValidationException(origCtClass);
+			}
+			int mod = cons.getModifiers();
+			if (!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) {
+				throwConstructoValidationException(origCtClass);
+			}
+		}
+
+		private static void throwConstructoValidationException(
+				CtClass origCtClass) {
+			throw new PackUnpackUtilException(
+					"it must have a public zero-argument constructor: "
+							+ origCtClass.getName());
+		}
+
 		private void setSuperclass(CtClass enhCtClass, CtClass origCtClass)
 				throws CannotCompileException {
 			enhCtClass.setSuperclass(origCtClass);
@@ -328,9 +372,43 @@ public class PackUnpackUtil {
 			CtClass type = field.getType();
 			sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE)
 					.append(Constants.CHAR_NAME_EQUAL).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.VARIABLE_NAME_PK).append(
-							Constants.CHAR_NAME_DOT);
+							Constants.CHAR_NAME_SPACE);
+			insertValueOfMethodAndLeftParenthesis(sb, type);
+			sb.append(Constants.VARIABLE_NAME_PK).append(
+					Constants.CHAR_NAME_DOT);
+			insertUnpackMethod(sb, type);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
+					Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			insertValueOfMethodAndRightParenthesis(sb, type);
+			sb.append(Constants.CHAR_NAME_SEMICOLON).append(
+					Constants.CHAR_NAME_SPACE);
+
+		}
+
+		private void insertValueOfMethodAndLeftParenthesis(StringBuilder sb,
+				CtClass type) throws NotFoundException {
+			if (type.isPrimitive()) { // primitive type
+				return;
+			} else { // reference type
+				if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean
+						|| type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte
+						|| type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double
+						|| type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float
+						|| type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer
+						|| type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long
+						|| type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short
+				) {
+					sb.append(type.getName()).append(Constants.CHAR_NAME_DOT)
+							.append(Constants.METHOD_NAME_VALUEOF).append(
+									Constants.CHAR_NAME_LEFT_PARENTHESIS);
+				} else {
+					return;
+				}
+			}
+		}
+
+		private void insertUnpackMethod(StringBuilder sb, CtClass type)
+				throws NotFoundException {
 			if (type.equals(CtClass.booleanType)) { // boolean
 				sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN);
 			} else if (type.equals(CtClass.byteType)) { // byte
@@ -379,10 +457,26 @@ public class PackUnpackUtil {
 							+ type.getName());
 				}
 			}
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
-					Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
-					Constants.CHAR_NAME_SEMICOLON).append(
-					Constants.CHAR_NAME_SPACE);
+		}
+
+		private void insertValueOfMethodAndRightParenthesis(StringBuilder sb,
+				CtClass type) throws NotFoundException {
+			if (type.isPrimitive()) { // primitive type
+				return;
+			} else { // reference type
+				if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean
+						|| type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte
+						|| type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double
+						|| type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float
+						|| type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer
+						|| type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long
+						|| type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short
+				) {
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+				} else {
+					return;
+				}
+			}
 		}
 
 		private void addMessageConvertMethod(CtClass enhCtClass,
diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
new file mode 100644
index 0000000..1ec7d64
--- /dev/null
+++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
@@ -0,0 +1,280 @@
+package org.msgpack.util.annotation;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+import org.msgpack.MessageUnpackable;
+import org.msgpack.Packer;
+import org.msgpack.Unpacker;
+
+public class TestMessagePackUnpackable extends TestCase {
+
+	@Test
+	public void testGeneralPrimitiveTypeFieldsClass() throws Exception {
+		GeneralPrimitiveTypeFieldsClass src = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil
+				.newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class);
+		src.f0 = (byte) 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f6 = false;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		GeneralPrimitiveTypeFieldsClass dst = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil
+				.newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		pac.unpack((MessageUnpackable) dst);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+	}
+
+	@MessagePackUnpackable
+	public static class GeneralPrimitiveTypeFieldsClass {
+		public byte f0;
+		public short f1;
+		public int f2;
+		public long f3;
+		public float f4;
+		public double f5;
+		public boolean f6;
+
+		public GeneralPrimitiveTypeFieldsClass() {
+		}
+	}
+
+	@Test
+	public void testGeneralReferenceTypeFieldsClass() throws Exception {
+		GeneralReferenceTypeFieldsClass src = (GeneralReferenceTypeFieldsClass) PackUnpackUtil
+				.newEnhancedInstance(GeneralReferenceTypeFieldsClass.class);
+		src.f0 = 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = (long) 3;
+		src.f4 = (float) 4;
+		src.f5 = (double) 5;
+		src.f6 = false;
+		src.f7 = new BigInteger("7");
+		src.f8 = "8";
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) PackUnpackUtil
+				.newEnhancedInstance(GeneralReferenceTypeFieldsClass.class);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		pac.unpack((MessageUnpackable) dst);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+	}
+
+	@MessagePackUnpackable
+	public static class GeneralReferenceTypeFieldsClass {
+		public Byte f0;
+		public Short f1;
+		public Integer f2;
+		public Long f3;
+		public Float f4;
+		public Double f5;
+		public Boolean f6;
+		public BigInteger f7;
+		public String f8;
+
+		public GeneralReferenceTypeFieldsClass() {
+		}
+	}
+
+	@Test
+	public void testPublicDefaultConstructorClass() throws Exception {
+		try {
+			PackUnpackUtil.newEnhancedInstance(NoDefaultConstructorClass.class);
+			fail();
+		} catch (PackUnpackUtilException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			PackUnpackUtil.newEnhancedInstance(PrivateDefaultConstructorClass.class);
+			fail();
+		} catch (PackUnpackUtilException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			PackUnpackUtil.newEnhancedInstance(ProtectedDefaultConstructorClass.class);
+			assertTrue(true);
+		} catch (PackUnpackUtilException e) {
+			fail();
+		}
+		assertTrue(true);
+		try {
+			PackUnpackUtil.newEnhancedInstance(PackageDefaultConstructorClass.class);
+			fail();
+		} catch (PackUnpackUtilException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@MessagePackUnpackable
+	public static class NoDefaultConstructorClass {
+		public NoDefaultConstructorClass(int i) {
+		}
+	}
+
+	@MessagePackUnpackable
+	public static class PrivateDefaultConstructorClass {
+		private PrivateDefaultConstructorClass() {
+		}
+	}
+
+	@MessagePackUnpackable
+	public static class ProtectedDefaultConstructorClass {
+		protected ProtectedDefaultConstructorClass() {
+		}
+	}
+
+	@MessagePackUnpackable
+	public static class PackageDefaultConstructorClass {
+		PackageDefaultConstructorClass() {
+		}
+	}
+
+	@Test
+	public void testPublicModifierClass() throws Exception {
+		try {
+			PackUnpackUtil.newEnhancedInstance(PrivateModifierClass.class);
+			fail();
+		} catch (PackUnpackUtilException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			PackUnpackUtil.newEnhancedInstance(ProtectedModifierClass.class);
+			assertTrue(true);
+		} catch (PackUnpackUtilException e) {
+			fail();
+		}
+		assertTrue(true);
+		try {
+			PackUnpackUtil.newEnhancedInstance(PackageModifierClass.class);
+			fail();
+		} catch (PackUnpackUtilException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@MessagePackUnpackable
+	private static class PrivateModifierClass {
+	}
+
+	@MessagePackUnpackable
+	protected static class ProtectedModifierClass {
+		protected ProtectedModifierClass() {
+		}
+	}
+
+	@MessagePackUnpackable
+	static class PackageModifierClass {
+	}
+
+	@Test
+	public void testFinalAndAbstractModifierClass() throws Exception {
+		try {
+			PackUnpackUtil.newEnhancedInstance(FinalModifierClass.class);
+			fail();
+		} catch (PackUnpackUtilException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			PackUnpackUtil.newEnhancedInstance(AbstractModifierClass.class);
+			fail();
+		} catch (PackUnpackUtilException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@MessagePackUnpackable
+	public final static class FinalModifierClass {
+	}
+
+	@MessagePackUnpackable
+	public abstract static class AbstractModifierClass {
+	}
+
+	@Test
+	public void testInterfaceAndEnum() throws Exception {
+		try {
+			PackUnpackUtil.newEnhancedInstance(SampleInterface.class);
+			fail();
+		} catch (PackUnpackUtilException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			PackUnpackUtil.newEnhancedInstance(SampleEnum.class);
+			fail();
+		} catch (PackUnpackUtilException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@MessagePackUnpackable
+	public interface SampleInterface {
+	}
+
+	@MessagePackUnpackable
+	public enum SampleEnum {
+	}
+
+	@Test
+	public void testFinalFieldClass() throws Exception {
+
+	}
+
+	@Test
+	public void testPrivateFieldClass() throws Exception {
+
+	}
+
+	@Test
+	public void testProtectedFieldClass() throws Exception {
+
+	}
+
+	@Test
+	public void testNonModifierFieldClass() throws Exception {
+
+	}
+
+	@Test
+	public void testNestedAnnotatedFieldClass() throws Exception {
+
+	}
+
+	@Test
+	public void testSuperClass() throws Exception {
+
+	}
+}

From c7f8b94ccdde64512aced14930ebdc0e14825e66 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Tue, 14 Sep 2010 22:26:04 +0900
Subject: [PATCH 0695/1648] fixed several bugs of a verify error within
 annotation-utilities

---
 .../util/annotation/PackUnpackUtil.java       | 170 ++++++++++++++----
 .../annotation/TestMessagePackUnpackable.java | 160 ++++++++++++++---
 2 files changed, 270 insertions(+), 60 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
index 5efef79..a58b4bd 100644
--- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
+++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
@@ -5,6 +5,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.lang.reflect.Modifier;
 import java.math.BigInteger;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -196,9 +197,10 @@ public class PackUnpackUtil {
 			setSuperclass(enhCtClass, origCtClass);
 			setInterfaces(enhCtClass);
 			addConstructor(enhCtClass);
-			addMessagePackMethod(enhCtClass, origCtClass);
-			addMessageUnpackMethod(enhCtClass, origCtClass);
-			addMessageConvertMethod(enhCtClass, origCtClass);
+			CtField[] fields = getDeclaredFields(origCtClass);
+			addMessagePackMethod(enhCtClass, origCtClass, fields);
+			addMessageUnpackMethod(enhCtClass, origCtClass, fields);
+			addMessageConvertMethod(enhCtClass, origCtClass, fields);
 			return createClass(enhCtClass);
 		}
 
@@ -279,9 +281,54 @@ public class PackUnpackUtil {
 			enhCtClass.addConstructor(newCtCons);
 		}
 
+		private CtField[] getDeclaredFields(CtClass origCtClass) {
+			ArrayList allFields = new ArrayList();
+			try {
+				CtClass nextCtClass = origCtClass;
+				while (!nextCtClass
+						.equals(pool.get(Constants.TYPE_NAME_OBJECT))) {
+					CtField[] fields = nextCtClass.getDeclaredFields();
+					for (CtField field : fields) {
+						try {
+							checkFieldValidation(field, allFields);
+							allFields.add(field);
+						} catch (PackUnpackUtilException e) { // ignore
+						}
+					}
+					nextCtClass = nextCtClass.getSuperclass();
+				}
+
+			} catch (NotFoundException e) {
+				throw new PackUnpackUtilException(e.getMessage(), e);
+			}
+			return allFields.toArray(new CtField[0]);
+		}
+
+		private void checkFieldValidation(CtField field,
+				ArrayList allFields) {
+			// check modifiers (public or protected)
+			int mod = field.getModifiers();
+			if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
+					|| Modifier.isStatic(mod) || Modifier.isFinal(mod)
+					|| Modifier.isTransient(mod)) {
+				throwFieldValidationException(field);
+			}
+			// check same name
+			for (CtField f : allFields) {
+				if (f.getName().equals(field.getName())) {
+					throwFieldValidationException(field);
+				}
+			}
+		}
+
+		private static void throwFieldValidationException(CtField field) {
+			throw new PackUnpackUtilException("it must be a public field: "
+					+ field.getName());
+		}
+
 		private void addMessagePackMethod(CtClass enhCtClass,
-				CtClass origCtClass) throws CannotCompileException,
-				NotFoundException {
+				CtClass origCtClass, CtField[] fields)
+				throws CannotCompileException, NotFoundException {
 			StringBuilder sb = new StringBuilder();
 			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append(
 					Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID)
@@ -299,7 +346,6 @@ public class PackUnpackUtil {
 							Constants.CHAR_NAME_SPACE).append(
 							Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append(
 							Constants.CHAR_NAME_SPACE);
-			CtField[] fields = origCtClass.getDeclaredFields();
 			sb.append(Constants.VARIABLE_NAME_PK).append(
 					Constants.CHAR_NAME_DOT).append(
 					Constants.METHOD_NAME_PACKARRAY).append(
@@ -328,8 +374,8 @@ public class PackUnpackUtil {
 		}
 
 		private void addMessageUnpackMethod(CtClass enhCtClass,
-				CtClass origCtClass) throws CannotCompileException,
-				NotFoundException {
+				CtClass origCtClass, CtField[] fields)
+				throws CannotCompileException, NotFoundException {
 			StringBuilder sb = new StringBuilder();
 			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append(
 					Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID)
@@ -350,7 +396,6 @@ public class PackUnpackUtil {
 							Constants.CHAR_NAME_SPACE).append(
 							Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append(
 							Constants.CHAR_NAME_SPACE);
-			CtField[] fields = origCtClass.getFields();
 			sb.append(Constants.VARIABLE_NAME_PK).append(
 					Constants.CHAR_NAME_DOT).append(
 					Constants.METHOD_NAME_UNPACKARRAY).append(
@@ -370,21 +415,53 @@ public class PackUnpackUtil {
 		private void insertCodeOfMessageUnpack(CtField field, StringBuilder sb)
 				throws NotFoundException {
 			CtClass type = field.getType();
-			sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE)
-					.append(Constants.CHAR_NAME_EQUAL).append(
-							Constants.CHAR_NAME_SPACE);
+			insertRightVariable(sb, field, type);
 			insertValueOfMethodAndLeftParenthesis(sb, type);
 			sb.append(Constants.VARIABLE_NAME_PK).append(
 					Constants.CHAR_NAME_DOT);
 			insertUnpackMethod(sb, type);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
-					Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			insertUnpackMethodArgumenet(sb, field, type);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 			insertValueOfMethodAndRightParenthesis(sb, type);
 			sb.append(Constants.CHAR_NAME_SEMICOLON).append(
 					Constants.CHAR_NAME_SPACE);
 
 		}
 
+		private void insertRightVariable(StringBuilder sb, CtField field,
+				CtClass type) throws NotFoundException {
+			if (type.isPrimitive()) { // primitive type
+				sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE)
+						.append(Constants.CHAR_NAME_EQUAL).append(
+								Constants.CHAR_NAME_SPACE);
+			} else { // reference type
+				if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean
+						|| type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte
+						|| type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double
+						|| type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float
+						|| type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer
+						|| type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long
+						|| type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short
+						|| type
+								.equals(pool
+										.get(Constants.TYPE_NAME_BIGINTEGER)) // BigInteger
+						|| type.equals(pool.get(Constants.TYPE_NAME_STRING)) // String
+						|| type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY)) // byte[]
+						|| type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST)) // List
+						|| type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP)) // Map
+				) {
+					sb.append(field.getName())
+							.append(Constants.CHAR_NAME_SPACE).append(
+									Constants.CHAR_NAME_EQUAL).append(
+									Constants.CHAR_NAME_SPACE);
+				} else { // MessageUnpackable
+					return;
+				}
+			}
+
+		}
+
 		private void insertValueOfMethodAndLeftParenthesis(StringBuilder sb,
 				CtClass type) throws NotFoundException {
 			if (type.isPrimitive()) { // primitive type
@@ -424,6 +501,7 @@ public class PackUnpackUtil {
 			} else if (type.equals(CtClass.shortType)) { // short
 				sb.append(Constants.METHOD_NAME_UNPACKSHORT);
 			} else { // reference type
+				Class c = null;
 				if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2))) { // Boolean
 					sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN);
 				} else if (type.equals(pool.get(Constants.TYPE_NAME_BYTE2))) { // Byte
@@ -450,8 +528,9 @@ public class PackUnpackUtil {
 				} else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP))) { // Map
 					sb.append(Constants.METHOD_NAME_UNPACKMAP);
 				} else if (type.subtypeOf(pool
-						.get(Constants.TYPE_NAME_MSGUNPACKABLE))) { // MessageUnpackable
-					sb.append(Constants.METHOD_NAME_UNPACKOBJECT);
+						.get(Constants.TYPE_NAME_MSGUNPACKABLE))
+						|| ((c = getCache(type.getName())) != null)) { // MessageUnpackable
+					sb.append(Constants.METHOD_NAME_UNPACK);
 				} else {
 					throw new NotFoundException("unknown type:  "
 							+ type.getName());
@@ -459,6 +538,22 @@ public class PackUnpackUtil {
 			}
 		}
 
+		private void insertUnpackMethodArgumenet(StringBuilder sb,
+				CtField field, CtClass type) throws NotFoundException {
+			if (type.isPrimitive()) { // primitive type
+				return;
+			} else { // reference type
+				Class c = null;
+				if (type.equals(pool.get(Constants.TYPE_NAME_MSGUNPACKABLE))
+						|| ((c = getCache(type.getName())) != null)) {
+					sb.append("(org.msgpack.MessageUnpackable)");
+					sb.append(field.getName());
+				} else {
+					return;
+				}
+			}
+		}
+
 		private void insertValueOfMethodAndRightParenthesis(StringBuilder sb,
 				CtClass type) throws NotFoundException {
 			if (type.isPrimitive()) { // primitive type
@@ -480,7 +575,8 @@ public class PackUnpackUtil {
 		}
 
 		private void addMessageConvertMethod(CtClass enhCtClass,
-				CtClass origCtClass) throws CannotCompileException {
+				CtClass origCtClass, CtField[] fields)
+				throws CannotCompileException {
 			StringBuilder sb = new StringBuilder();
 			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append(
 					Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID)
@@ -547,33 +643,33 @@ public class PackUnpackUtil {
 		}
 	}
 
-	public static void main(final String[] args) throws Exception {
-		@MessagePackUnpackable
-		class Image {
-			public String uri = "";
+	@MessagePackUnpackable
+	public static class Image {
+		public String uri = "";
 
-			public String title = "";
+		public String title = "";
 
-			public int width = 0;
+		public int width = 0;
 
-			public int height = 0;
+		public int height = 0;
 
-			public int size = 0;
+		public int size = 0;
 
-			public boolean equals(Image obj) {
-				return uri.equals(obj.uri) && title.equals(obj.title)
-						&& width == obj.width && height == obj.height
-						&& size == obj.size;
-			}
-
-			public boolean equals(Object obj) {
-				if (obj.getClass() != Image.class) {
-					return false;
-				}
-				return equals((Image) obj);
-			}
+		public boolean equals(Image obj) {
+			return uri.equals(obj.uri) && title.equals(obj.title)
+					&& width == obj.width && height == obj.height
+					&& size == obj.size;
 		}
 
+		public boolean equals(Object obj) {
+			if (obj.getClass() != Image.class) {
+				return false;
+			}
+			return equals((Image) obj);
+		}
+	}
+
+	public static void main(final String[] args) throws Exception {
 		PackUnpackUtil.getEnhancedClass(Image.class);
 		Image src = (Image) PackUnpackUtil.newEnhancedInstance(Image.class);
 		src.title = "msgpack";
diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
index 1ec7d64..4f6e74e 100644
--- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
+++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
@@ -3,6 +3,8 @@ package org.msgpack.util.annotation;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.math.BigInteger;
+import java.util.List;
+import java.util.Map;
 
 import junit.framework.TestCase;
 
@@ -14,7 +16,7 @@ import org.msgpack.Unpacker;
 public class TestMessagePackUnpackable extends TestCase {
 
 	@Test
-	public void testGeneralPrimitiveTypeFieldsClass() throws Exception {
+	public void testGeneralPrimitiveTypeFields() throws Exception {
 		GeneralPrimitiveTypeFieldsClass src = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil
 				.newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class);
 		src.f0 = (byte) 0;
@@ -101,8 +103,21 @@ public class TestMessagePackUnpackable extends TestCase {
 		}
 	}
 
+	public void testListAndMap() throws Exception {
+		// TODO
+	}
+
+	@MessagePackUnpackable
+	public static class ListAndMapClass {
+		public List f0;
+		public Map f1;
+
+		public ListAndMapClass() {
+		}
+	}
+
 	@Test
-	public void testPublicDefaultConstructorClass() throws Exception {
+	public void testDefaultConstructorModifiers() throws Exception {
 		try {
 			PackUnpackUtil.newEnhancedInstance(NoDefaultConstructorClass.class);
 			fail();
@@ -111,21 +126,24 @@ public class TestMessagePackUnpackable extends TestCase {
 		}
 		assertTrue(true);
 		try {
-			PackUnpackUtil.newEnhancedInstance(PrivateDefaultConstructorClass.class);
+			PackUnpackUtil
+					.newEnhancedInstance(PrivateDefaultConstructorClass.class);
 			fail();
 		} catch (PackUnpackUtilException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			PackUnpackUtil.newEnhancedInstance(ProtectedDefaultConstructorClass.class);
+			PackUnpackUtil
+					.newEnhancedInstance(ProtectedDefaultConstructorClass.class);
 			assertTrue(true);
 		} catch (PackUnpackUtilException e) {
 			fail();
 		}
 		assertTrue(true);
 		try {
-			PackUnpackUtil.newEnhancedInstance(PackageDefaultConstructorClass.class);
+			PackUnpackUtil
+					.newEnhancedInstance(PackageDefaultConstructorClass.class);
 			fail();
 		} catch (PackUnpackUtilException e) {
 			assertTrue(true);
@@ -158,7 +176,7 @@ public class TestMessagePackUnpackable extends TestCase {
 	}
 
 	@Test
-	public void testPublicModifierClass() throws Exception {
+	public void testClassModifiers() throws Exception {
 		try {
 			PackUnpackUtil.newEnhancedInstance(PrivateModifierClass.class);
 			fail();
@@ -197,7 +215,7 @@ public class TestMessagePackUnpackable extends TestCase {
 	}
 
 	@Test
-	public void testFinalAndAbstractModifierClass() throws Exception {
+	public void testFinalClassAndAbstractClass() throws Exception {
 		try {
 			PackUnpackUtil.newEnhancedInstance(FinalModifierClass.class);
 			fail();
@@ -223,7 +241,7 @@ public class TestMessagePackUnpackable extends TestCase {
 	}
 
 	@Test
-	public void testInterfaceAndEnum() throws Exception {
+	public void testInterfaceAndEnumType() throws Exception {
 		try {
 			PackUnpackUtil.newEnhancedInstance(SampleInterface.class);
 			fail();
@@ -249,32 +267,128 @@ public class TestMessagePackUnpackable extends TestCase {
 	}
 
 	@Test
-	public void testFinalFieldClass() throws Exception {
-
+	public void testFieldModifiers() throws Exception {
+		FieldModifiersClass src = (FieldModifiersClass) PackUnpackUtil
+				.newEnhancedInstance(FieldModifiersClass.class);
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		FieldModifiersClass dst = (FieldModifiersClass) PackUnpackUtil
+				.newEnhancedInstance(FieldModifiersClass.class);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		pac.unpack((MessageUnpackable) dst);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f4 != dst.f4);
 	}
 
-	@Test
-	public void testPrivateFieldClass() throws Exception {
-
-	}
-
-	@Test
-	public void testProtectedFieldClass() throws Exception {
-
-	}
-
-	@Test
-	public void testNonModifierFieldClass() throws Exception {
+	@MessagePackUnpackable
+	public static class FieldModifiersClass {
+		public int f0;
+		public final int f1 = 1;
+		private int f2;
+		protected int f3;
+		int f4;
 
+		public FieldModifiersClass() {
+		}
 	}
 
 	@Test
 	public void testNestedAnnotatedFieldClass() throws Exception {
+		NestedClass src2 = (NestedClass) PackUnpackUtil
+				.newEnhancedInstance(NestedClass.class);
+		BaseClass src = (BaseClass) PackUnpackUtil
+				.newEnhancedInstance(BaseClass.class);
+		src.f0 = 0;
+		src2.f2 = 2;
+		src.f1 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		NestedClass dst2 = (NestedClass) PackUnpackUtil
+				.newEnhancedInstance(NestedClass.class);
+		BaseClass dst = (BaseClass) PackUnpackUtil
+				.newEnhancedInstance(BaseClass.class);
+		dst.f1 = dst2;
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		pac.unpack((MessageUnpackable) dst);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src2.f2 == dst.f1.f2);
+	}
 
+	@MessagePackUnpackable
+	public static class BaseClass {
+		public int f0;
+		public NestedClass f1;
+
+		public BaseClass() {
+		}
+	}
+
+	@MessagePackUnpackable
+	public static class NestedClass {
+		public int f2;
+
+		public NestedClass() {
+		}
 	}
 
 	@Test
-	public void testSuperClass() throws Exception {
+	public void testExtendedClass() throws Exception {
+		SampleSubClass src = (SampleSubClass) PackUnpackUtil
+				.newEnhancedInstance(SampleSubClass.class);
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f8 = 8;
+		src.f9 = 9;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		SampleSubClass dst = (SampleSubClass) PackUnpackUtil
+				.newEnhancedInstance(SampleSubClass.class);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		pac.unpack((MessageUnpackable) dst);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f4 != dst.f4);
+		assertTrue(src.f5 == dst.f5);
+		assertTrue(src.f6 == dst.f6);
+		assertTrue(src.f8 == dst.f8);
+		assertTrue(src.f9 != dst.f9);
+	}
 
+	@MessagePackUnpackable
+	public static class SampleSubClass extends SampleSuperClass {
+		public int f0;
+		public final int f1 = 1;
+		private int f2;
+		protected int f3;
+		int f4;
+
+		public SampleSubClass() {
+		}
+	}
+
+	public static class SampleSuperClass {
+		public int f5;
+		public final int f6 = 2;
+		private int f7;
+		protected int f8;
+		int f9;
+
+		public SampleSuperClass() {
+		}
 	}
 }

From 56ece4db0f03f086c2d4608472de4b20e6a83a19 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Tue, 14 Sep 2010 23:23:09 +0900
Subject: [PATCH 0696/1648] fixed a bug the program for packing and unpacking
 byte[] in annotation-utility

---
 .../java/org/msgpack/util/annotation/PackUnpackUtil.java     | 3 ++-
 .../msgpack/util/annotation/TestMessagePackUnpackable.java   | 5 ++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
index a58b4bd..f92c51a 100644
--- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
+++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
@@ -71,7 +71,8 @@ public class PackUnpackUtil {
 
 		static final String TYPE_NAME_SHORT2 = Short.class.getName();
 
-		static final String TYPE_NAME_BYTEARRAY = byte[].class.getName();
+		//static final String TYPE_NAME_BYTEARRAY = byte[].class.getName();
+		static final String TYPE_NAME_BYTEARRAY = "byte[]";
 
 		static final String TYPE_NAME_LIST = List.class.getName();
 
diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
index 4f6e74e..1cde65f 100644
--- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
+++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
@@ -69,6 +69,7 @@ public class TestMessagePackUnpackable extends TestCase {
 		src.f6 = false;
 		src.f7 = new BigInteger("7");
 		src.f8 = "8";
+		src.f9 = new byte[] { 0x01, 0x02 };
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		new Packer(out).pack(src);
 		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) PackUnpackUtil
@@ -85,6 +86,8 @@ public class TestMessagePackUnpackable extends TestCase {
 		assertEquals(src.f6, dst.f6);
 		assertEquals(src.f7, dst.f7);
 		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9[0], dst.f9[0]);
+		assertEquals(src.f9[1], dst.f9[1]);
 	}
 
 	@MessagePackUnpackable
@@ -98,13 +101,13 @@ public class TestMessagePackUnpackable extends TestCase {
 		public Boolean f6;
 		public BigInteger f7;
 		public String f8;
+		public byte[] f9;
 
 		public GeneralReferenceTypeFieldsClass() {
 		}
 	}
 
 	public void testListAndMap() throws Exception {
-		// TODO
 	}
 
 	@MessagePackUnpackable

From 987248ccbb4d18d532f2d07c28f47d37934b9c59 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 12:31:01 +0900
Subject: [PATCH 0697/1648] Use xshelper.h in all the C files

---
 perl/MANIFEST.SKIP        |  1 -
 perl/perlxs.h             | 76 ---------------------------------------
 perl/util.h               | 11 ------
 perl/xs-src/MessagePack.c | 14 +-------
 perl/xs-src/unpack.c      |  8 -----
 5 files changed, 1 insertion(+), 109 deletions(-)
 delete mode 100644 perl/perlxs.h
 delete mode 100644 perl/util.h

diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP
index 372742c..71a24e5 100644
--- a/perl/MANIFEST.SKIP
+++ b/perl/MANIFEST.SKIP
@@ -25,4 +25,3 @@
 ^Data-MessagePack-[0-9.]+/
 ^\.testenv/test_pp.pl
 ^ppport.h$
-^xshelper.h$
diff --git a/perl/perlxs.h b/perl/perlxs.h
deleted file mode 100644
index 441682d..0000000
--- a/perl/perlxs.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-    perlxs.h - Standard XS header file
-    Copyright (c) Fuji, Goro (gfx)
-*/
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define PERL_NO_GET_CONTEXT /* we want efficiency */
-#include 
-
-#include 
-#define NO_XSLOCKS /* for exceptions */
-#include 
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
-#include "ppport.h"
-
-/* portability stuff not supported by ppport.h yet */
-
-#ifndef STATIC_INLINE /* from 5.13.4 */
-#   if defined(__GNUC__) || defined(__cplusplus__) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L))
-#       define STATIC_INLINE static inline
-#   else
-#       define STATIC_INLINE static
-#   endif
-#endif /* STATIC_INLINE */
-
-#ifndef __attribute__format__
-#define __attribute__format__(a,b,c) /* nothing */
-#endif
-
-#ifndef LIKELY /* they are just a compiler's hint */
-#define LIKELY(x)   (x)
-#define UNLIKELY(x) (x)
-#endif
-
-#ifndef newSVpvs_share
-#define newSVpvs_share(s) Perl_newSVpvn_share(aTHX_ STR_WITH_LEN(s), 0U)
-#endif
-
-#ifndef get_cvs
-#define get_cvs(name, flags) get_cv(name, flags)
-#endif
-
-#ifndef GvNAME_get
-#define GvNAME_get GvNAME
-#endif
-#ifndef GvNAMELEN_get
-#define GvNAMELEN_get GvNAMELEN
-#endif
-
-#ifndef CvGV_set
-#define CvGV_set(cv, gv) (CvGV(cv) = (gv))
-#endif
-
-/* general utility */
-
-#if PERL_BCDVERSION >= 0x5008005
-#define LooksLikeNumber(x) looks_like_number(x)
-#else
-#define LooksLikeNumber(x) (SvPOKp(x) ? looks_like_number(x) : (I32)SvNIOKp(x))
-#endif
-
-#define newAV_mortal() (AV*)sv_2mortal((SV*)newAV())
-#define newHV_mortal() (HV*)sv_2mortal((SV*)newHV())
-
-#define DECL_BOOT(name) EXTERN_C XS(CAT2(boot_, name))
-#define CALL_BOOT(name) STMT_START {            \
-        PUSHMARK(SP);                           \
-        CALL_FPTR(CAT2(boot_, name))(aTHX_ cv); \
-    } STMT_END
diff --git a/perl/util.h b/perl/util.h
deleted file mode 100644
index 2b4ed07..0000000
--- a/perl/util.h
+++ /dev/null
@@ -1,11 +0,0 @@
-#ifndef __PERL_MSGPACK_UTIL_H__
-#define __PERL_MSGPACK_UTIL_H__
-
-#if __GNUC__ >= 3
-# define INLINE                     inline
-#else
-# define INLINE                     
-#endif
-
-#endif // __PERL_MSGPACK_UTIL_H__
-
diff --git a/perl/xs-src/MessagePack.c b/perl/xs-src/MessagePack.c
index fd1b344..259fd15 100644
--- a/perl/xs-src/MessagePack.c
+++ b/perl/xs-src/MessagePack.c
@@ -1,14 +1,4 @@
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "EXTERN.h"
-#include "perl.h"
-#include "XSUB.h"
-#define NEED_newCONSTSUB
-#include "ppport.h"
-#ifdef __cplusplus
-};
-#endif
+#include "xshelper.h"
 
 XS(xs_pack);
 XS(xs_unpack);
@@ -24,13 +14,11 @@ void boot_Data__MessagePack_pack(void);
 
 XS(boot_Data__MessagePack) {
     dXSARGS;
-    HV * stash;
 
     boot_Data__MessagePack_pack();
 
     newXS("Data::MessagePack::pack", xs_pack, __FILE__);
     newXS("Data::MessagePack::unpack", xs_unpack, __FILE__);
-    stash = gv_stashpvn("Data::MessagePack", strlen("Data::MessagePack"), TRUE);
 
     newXS("Data::MessagePack::Unpacker::new",           xs_unpacker_new, __FILE__);
     newXS("Data::MessagePack::Unpacker::execute",       xs_unpacker_execute, __FILE__);
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index c329e99..2edfdf7 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -1,15 +1,7 @@
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 #define NEED_newRV_noinc
 #define NEED_sv_2pv_flags
 #include "xshelper.h"
 
-#ifdef __cplusplus
-};
-#endif
-
 typedef struct {
     int finished;
     SV* source;

From bebcc24ab8908a07449c8ff6992a5dea2371e136 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 12:32:01 +0900
Subject: [PATCH 0698/1648] Depends on XSUtil 0.32

---
 perl/Makefile.PL | 1 +
 1 file changed, 1 insertion(+)

diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index 7053cf9..e7b8c47 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -1,4 +1,5 @@
 use inc::Module::Install;
+use Module::Install::XSUtil 0.32;
 use Config;
 
 name 'Data-MessagePack';

From 197205853fb552234250e1616af85b60552c9869 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 12:35:10 +0900
Subject: [PATCH 0699/1648] Use newSV(). NEWSV() is deprecated.

---
 perl/xs-src/pack.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 0aa3250..4378b05 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -252,7 +252,7 @@ XS(xs_pack) {
     if (items >= 3) depth = SvIV(ST(2));
 
     enc_t enc;
-    enc.sv        = sv_2mortal(NEWSV(0, INIT_SIZE));
+    enc.sv        = sv_2mortal(newSV(INIT_SIZE));
     enc.cur       = SvPVX(enc.sv);
     enc.end       = SvEND(enc.sv);
     SvPOK_only(enc.sv);

From 0768cf17b61465ebc2fe5e0132e0494399eb94d1 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 12:36:43 +0900
Subject: [PATCH 0700/1648] Taking NULL is a bug

---
 perl/xs-src/pack.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 4378b05..7bad614 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -143,14 +143,13 @@ STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) {
 
 static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
 
-STATIC_INLINE void _msgpack_pack_sv(enc_t *enc, SV* sv, int depth) {
+STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
     dTHX;
+    assert(sv);
     if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
     SvGETMAGIC(sv);
 
-    if (sv==NULL) {
-        msgpack_pack_nil(enc);
-    } else if (SvPOKp(sv)) {
+    if (SvPOKp(sv)) {
         STRLEN len;
         char * csv = SvPV(sv, len);
 

From 0f02ef20a9ff77e830d5f199a4deac1c9a4d7d87 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 12:46:11 +0900
Subject: [PATCH 0701/1648] Improve benchmarks

---
 perl/benchmark/deserialize.pl | 6 +++---
 perl/benchmark/serialize.pl   | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl
index 750704e..9658c0c 100644
--- a/perl/benchmark/deserialize.pl
+++ b/perl/benchmark/deserialize.pl
@@ -9,7 +9,7 @@ my $a = {
     "method" => "handleMessage",
     "params" => [ "user1", "we were just talking" ],
     "id"     => undef,
-    "array"  => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0 ]
+    "array"  => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ]
 };
 my $j = JSON::XS::encode_json($a);
 my $m = Data::MessagePack->pack($a);
@@ -19,8 +19,8 @@ print "-- deserialize\n";
 print "JSON::XS: $JSON::XS::VERSION\n";
 print "Data::MessagePack: $Data::MessagePack::VERSION\n";
 print "Storable: $Storable::VERSION\n";
-timethese(
-    1000000 => {
+cmpthese timethese(
+    -1 => {
         json     => sub { JSON::XS::decode_json($j)     },
         mp       => sub { Data::MessagePack->unpack($m) },
         storable => sub { Storable::thaw($s) },
diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl
index c5ab15b..ee9e7a4 100644
--- a/perl/benchmark/serialize.pl
+++ b/perl/benchmark/serialize.pl
@@ -9,15 +9,15 @@ my $a = {
     "method" => "handleMessage",
     "params" => [ "user1", "we were just talking" ],
     "id"     => undef,
-    "array"  => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0 ]
+    "array"  => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ]
 };
 
 print "-- serialize\n";
 print "JSON::XS: $JSON::XS::VERSION\n";
 print "Data::MessagePack: $Data::MessagePack::VERSION\n";
 print "Storable: $Storable::VERSION\n";
-timethese(
-    1000000 => {
+cmpthese timethese(
+    -1 => {
         json     => sub { JSON::XS::encode_json($a) },
         storable => sub { Storable::freeze($a) },
         mp       => sub { Data::MessagePack->pack($a) },

From d86104ed5dc3b2a99542a8e6c4468b778200f5fc Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 12:48:23 +0900
Subject: [PATCH 0702/1648] Tweaks

---
 perl/xs-src/pack.c | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 7bad614..67483dc 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -48,7 +48,7 @@ STATIC_INLINE void need(enc_t *enc, STRLEN len)
     dTHX;
     if (enc->cur + len >= enc->end) {
         STRLEN cur = enc->cur - (char *)SvPVX (enc->sv);
-        SvGROW (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1);
+        sv_grow (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1);
         enc->cur = SvPVX (enc->sv) + cur;
         enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1;
     }
@@ -159,13 +159,17 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const de
             msgpack_pack_raw(enc, len);
             msgpack_pack_raw_body(enc, csv, len);
         }
-    } else if (SvNOKp(sv)) {
-        /* XXX long double is not supported yet. */
-        msgpack_pack_double(enc, (double)SvNVX(sv));
-    } else if (SvIOK_UV(sv)) {
-        msgpack_pack_uint32(enc, SvUV(sv));
-    } else if (SvIOKp(sv)) {
-        PACK_IV(enc, SvIV(sv));
+    } else if (SvNIOKp(sv)) {
+        if(SvUOK(sv)) {
+            msgpack_pack_uint32(enc, SvUV(sv));
+        }
+        else if(SvIOKp(sv)) {
+            PACK_IV(enc, SvIV(sv));
+        }
+        else {
+            /* XXX long double is not supported yet. */
+            msgpack_pack_double(enc, (double)SvNVX(sv));
+        }
     } else if (SvROK(sv)) {
         _msgpack_pack_rv(enc, SvRV(sv), depth-1);
     } else if (!SvOK(sv)) {

From 60b36ffaa38ba32c1a97b4143138bb2e511fb1ae Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 12:50:11 +0900
Subject: [PATCH 0703/1648] Micro optimizations

---
 perl/xs-src/pack.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 67483dc..30bc032 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -32,10 +32,13 @@ static void need(enc_t *enc, STRLEN len);
 
 #if   IVSIZE == 8
 #  define PACK_IV msgpack_pack_int64
+#  define PACK_UV msgpack_pack_uint64
 #elif IVSIZE == 4
 #  define PACK_IV msgpack_pack_int32
+#  define PACK_UV msgpack_pack_uint32
 #elif IVSIZE == 2
 #  define PACK_IV msgpack_pack_int16
+#  define PACK_UV msgpack_pack_uint16
 #else
 #  error  "msgpack only supports IVSIZE = 8,4,2 environment."
 #endif
@@ -150,21 +153,21 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const de
     SvGETMAGIC(sv);
 
     if (SvPOKp(sv)) {
-        STRLEN len;
-        char * csv = SvPV(sv, len);
+        STRLEN const len     = SvCUR(sv);
+        const char* const pv = SvPVX_const(sv);
 
-        if (s_pref_int && try_int(enc, csv, len)) {
+        if (s_pref_int && try_int(enc, pv, len)) {
             return;
         } else {
             msgpack_pack_raw(enc, len);
-            msgpack_pack_raw_body(enc, csv, len);
+            msgpack_pack_raw_body(enc, pv, len);
         }
     } else if (SvNIOKp(sv)) {
         if(SvUOK(sv)) {
-            msgpack_pack_uint32(enc, SvUV(sv));
+            PACK_UV(enc, SvUVX(sv));
         }
         else if(SvIOKp(sv)) {
-            PACK_IV(enc, SvIV(sv));
+            PACK_IV(enc, SvIVX(sv));
         }
         else {
             /* XXX long double is not supported yet. */

From 4adcdb5ba81544517d65953497f0b0ac17b95a0a Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 12:51:33 +0900
Subject: [PATCH 0704/1648] Remove a duplicated depth check

---
 perl/xs-src/pack.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 30bc032..c01ba44 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -149,7 +149,7 @@ static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
 STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
     dTHX;
     assert(sv);
-    if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
+    if (UNLIKELY(depth <= 0)) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
     SvGETMAGIC(sv);
 
     if (SvPOKp(sv)) {
@@ -188,7 +188,6 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const de
 STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
     svtype svt;
     dTHX;
-    if (depth <= 0) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
     SvGETMAGIC(sv);
     svt = SvTYPE(sv);
 

From c5e15123fd14fe448222689d460b0b02a7659e14 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 12:52:07 +0900
Subject: [PATCH 0705/1648] Add an assertion

---
 perl/xs-src/pack.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index c01ba44..5feba4a 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -188,6 +188,7 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const de
 STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
     svtype svt;
     dTHX;
+    assert(sv);
     SvGETMAGIC(sv);
     svt = SvTYPE(sv);
 

From 50c74103aaecaff6f72ada55a1c9dc683c3017ef Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 12:56:13 +0900
Subject: [PATCH 0706/1648] Avoid compiler's warnings

---
 perl/xs-src/pack.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 5feba4a..c0c610e 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -60,7 +60,7 @@ STATIC_INLINE void need(enc_t *enc, STRLEN len)
 
 static int s_pref_int = 0;
 
-STATIC_INLINE int pref_int_set(pTHX_ SV* sv, MAGIC* mg) {
+STATIC_INLINE int pref_int_set(pTHX_ SV* sv, MAGIC* mg PERL_UNUSED_DECL) {
     if (SvTRUE(sv)) {
         s_pref_int = 1;
     } else {
@@ -211,7 +211,7 @@ STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
 
         msgpack_pack_map(enc, count);
 
-        while (he = hv_iternext(hval)) {
+        while ((he = hv_iternext(hval))) {
             _msgpack_pack_sv(enc, hv_iterkeysv(he), depth);
             _msgpack_pack_sv(enc, HeVAL(he), depth);
         }

From 9953218de18a87173869e8ffe97aa274046ddae1 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 13:03:47 +0900
Subject: [PATCH 0707/1648] Tidy

---
 perl/xs-src/unpack.c | 117 +++++++++++++++++++++++++++++++++++--------
 1 file changed, 96 insertions(+), 21 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 2edfdf7..05d953a 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -50,16 +50,31 @@ static int template_execute(msgpack_unpack_t* u,
     const char* data, size_t len, size_t* off);
 
 STATIC_INLINE SV* template_callback_root(unpack_user* u)
-{ dTHX; return &PL_sv_undef; }
+{
+    dTHX;
+    return &PL_sv_undef;
+}
 
 STATIC_INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o)
-{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; }
+{
+    dTHX;
+    *o = sv_2mortal(newSVuv(d));
+    return 0;
+}
 
 STATIC_INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o)
-{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; }
+{
+    dTHX;
+    *o = sv_2mortal(newSVuv(d));
+    return 0;
+}
 
 STATIC_INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o)
-{ dTHX; *o = sv_2mortal(newSVuv(d)); return 0; }
+{
+    dTHX;
+    *o = sv_2mortal(newSVuv(d));
+    return 0;
+}
 
 STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o)
 {
@@ -72,49 +87,109 @@ STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o)
     return 0;
 }
 
-STATIC_INLINE int template_callback_int8(unpack_user* u, int8_t d, SV** o)
-{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; }
+STATIC_INLINE int template_callback_int8(unpack_user* u PERL_UNUSED_DECL, int8_t d, SV** o)
+{
+    dTHX;
+    *o = sv_2mortal(newSViv(d));
+    return 0;
+}
 
-STATIC_INLINE int template_callback_int16(unpack_user* u, int16_t d, SV** o)
-{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; }
+STATIC_INLINE int template_callback_int16(unpack_user* u PERL_UNUSED_DECL, int16_t d, SV** o)
+{
+    dTHX;
+    *o = sv_2mortal(newSViv(d));
+    return 0;
+}
 
 STATIC_INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o)
-{ dTHX; *o = sv_2mortal(newSViv((long)d)); return 0; }
+{
+    dTHX;
+    *o = sv_2mortal(newSViv(d));
+    return 0;
+}
 
 STATIC_INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o)
-{ dTHX; *o = sv_2mortal(newSViv(d)); return 0; }
+{
+    dTHX;
+    *o = sv_2mortal(newSViv(d));
+    return 0;
+}
 
 STATIC_INLINE int template_callback_float(unpack_user* u, float d, SV** o)
-{ dTHX; *o = sv_2mortal(newSVnv(d)); return 0; }
+{
+    dTHX;
+    *o = sv_2mortal(newSVnv(d));
+    return 0;
+}
 
 STATIC_INLINE int template_callback_double(unpack_user* u, double d, SV** o)
-{ dTHX; *o = sv_2mortal(newSVnv(d)); return 0; }
+{
+    dTHX;
+    *o = sv_2mortal(newSVnv(d));
+    return 0;
+}
 
 /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */
 STATIC_INLINE int template_callback_nil(unpack_user* u, SV** o)
-{ dTHX; *o = sv_newmortal(); return 0; }
+{
+    dTHX;
+    *o = sv_newmortal();
+    return 0;
+}
 
 STATIC_INLINE int template_callback_true(unpack_user* u, SV** o)
-{ dTHX; *o = get_bool("Data::MessagePack::true") ; return 0; }
+{
+    dTHX;
+    *o = get_bool("Data::MessagePack::true");
+    return 0;
+}
 
 STATIC_INLINE int template_callback_false(unpack_user* u, SV** o)
-{ dTHX; *o = get_bool("Data::MessagePack::false") ; return 0; }
+{
+    dTHX; *o = get_bool("Data::MessagePack::false");
+    return 0;
+}
 
 STATIC_INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o)
-{ dTHX; AV* a = (AV*)sv_2mortal((SV*)newAV()); *o = sv_2mortal((SV*)newRV_inc((SV*)a)); av_extend(a, n); return 0; }
+{
+    dTHX;
+    AV* a = (AV*)sv_2mortal((SV*)newAV());
+    *o = sv_2mortal((SV*)newRV_inc((SV*)a));
+    av_extend(a, n);
+    return 0;
+}
 
 STATIC_INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o)
-{ dTHX; av_push((AV*)SvRV(*c), o); SvREFCNT_inc(o); return 0; }  /* FIXME set value directry RARRAY_PTR(obj)[RARRAY_LEN(obj)++] */
+{
+    dTHX;
+    av_push((AV*)SvRV(*c), o);
+    SvREFCNT_inc(o);
+    return 0;
+}
 
 STATIC_INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o)
-{ dTHX; HV * h = (HV*)sv_2mortal((SV*)newHV()); *o = sv_2mortal(newRV_inc((SV*)h)); return 0; }
+{
+    dTHX;
+    HV* h = (HV*)sv_2mortal((SV*)newHV());
+    *o = sv_2mortal(newRV_inc((SV*)h));
+    return 0;
+}
 
 STATIC_INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v)
-{ dTHX; hv_store_ent((HV*)SvRV(*c), k, v, 0); SvREFCNT_inc(v); return 0; }
+{
+    dTHX;
+    hv_store_ent((HV*)SvRV(*c), k, v, 0);
+    SvREFCNT_inc(v);
+    return 0;
+}
 
 STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o)
-{ dTHX; *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l)); return 0; }
-/* { *o = newSVpvn_flags(p, l, SVs_TEMP); return 0; }  <= this does not works. */
+{
+    dTHX;
+    /*  *o = newSVpvn_flags(p, l, SVs_TEMP); <= this does not work. */
+    *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l));
+    return 0;
+}
 
 #define UNPACKER(from, name) \
 	msgpack_unpack_t *name; \

From 10bf3ee9dec23337f30074785d7c21b81ffe2c40 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 13:07:44 +0900
Subject: [PATCH 0708/1648] Avoid compiler's warnings

---
 perl/xs-src/unpack.c | 43 +++++++++++++++++++++----------------------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 05d953a..108f9fd 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -46,37 +46,37 @@ static void template_init(msgpack_unpack_t* u);
 
 static SV* template_data(msgpack_unpack_t* u);
 
-static int template_execute(msgpack_unpack_t* u,
+static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL,
     const char* data, size_t len, size_t* off);
 
-STATIC_INLINE SV* template_callback_root(unpack_user* u)
+STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
 {
     dTHX;
     return &PL_sv_undef;
 }
 
-STATIC_INLINE int template_callback_uint8(unpack_user* u, uint8_t d, SV** o)
+STATIC_INLINE int template_callback_uint8(unpack_user* u PERL_UNUSED_DECL, uint8_t d, SV** o)
 {
     dTHX;
     *o = sv_2mortal(newSVuv(d));
     return 0;
 }
 
-STATIC_INLINE int template_callback_uint16(unpack_user* u, uint16_t d, SV** o)
+STATIC_INLINE int template_callback_uint16(unpack_user* u PERL_UNUSED_DECL, uint16_t d, SV** o)
 {
     dTHX;
     *o = sv_2mortal(newSVuv(d));
     return 0;
 }
 
-STATIC_INLINE int template_callback_uint32(unpack_user* u, uint32_t d, SV** o)
+STATIC_INLINE int template_callback_uint32(unpack_user* u PERL_UNUSED_DECL, uint32_t d, SV** o)
 {
     dTHX;
     *o = sv_2mortal(newSVuv(d));
     return 0;
 }
 
-STATIC_INLINE int template_callback_uint64(unpack_user* u, uint64_t d, SV** o)
+STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t d, SV** o)
 {
     dTHX;
 #if IVSIZE==4
@@ -101,28 +101,28 @@ STATIC_INLINE int template_callback_int16(unpack_user* u PERL_UNUSED_DECL, int16
     return 0;
 }
 
-STATIC_INLINE int template_callback_int32(unpack_user* u, int32_t d, SV** o)
+STATIC_INLINE int template_callback_int32(unpack_user* u PERL_UNUSED_DECL, int32_t d, SV** o)
 {
     dTHX;
     *o = sv_2mortal(newSViv(d));
     return 0;
 }
 
-STATIC_INLINE int template_callback_int64(unpack_user* u, int64_t d, SV** o)
+STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t d, SV** o)
 {
     dTHX;
     *o = sv_2mortal(newSViv(d));
     return 0;
 }
 
-STATIC_INLINE int template_callback_float(unpack_user* u, float d, SV** o)
+STATIC_INLINE int template_callback_float(unpack_user* u PERL_UNUSED_DECL, float d, SV** o)
 {
     dTHX;
     *o = sv_2mortal(newSVnv(d));
     return 0;
 }
 
-STATIC_INLINE int template_callback_double(unpack_user* u, double d, SV** o)
+STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
 {
     dTHX;
     *o = sv_2mortal(newSVnv(d));
@@ -130,27 +130,27 @@ STATIC_INLINE int template_callback_double(unpack_user* u, double d, SV** o)
 }
 
 /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */
-STATIC_INLINE int template_callback_nil(unpack_user* u, SV** o)
+STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     dTHX;
     *o = sv_newmortal();
     return 0;
 }
 
-STATIC_INLINE int template_callback_true(unpack_user* u, SV** o)
+STATIC_INLINE int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     dTHX;
     *o = get_bool("Data::MessagePack::true");
     return 0;
 }
 
-STATIC_INLINE int template_callback_false(unpack_user* u, SV** o)
+STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     dTHX; *o = get_bool("Data::MessagePack::false");
     return 0;
 }
 
-STATIC_INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o)
+STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
 {
     dTHX;
     AV* a = (AV*)sv_2mortal((SV*)newAV());
@@ -159,7 +159,7 @@ STATIC_INLINE int template_callback_array(unpack_user* u, unsigned int n, SV** o
     return 0;
 }
 
-STATIC_INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o)
+STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
 {
     dTHX;
     av_push((AV*)SvRV(*c), o);
@@ -167,7 +167,7 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u, SV** c, SV* o)
     return 0;
 }
 
-STATIC_INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o)
+STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n PERL_UNUSED_DECL, SV** o)
 {
     dTHX;
     HV* h = (HV*)sv_2mortal((SV*)newHV());
@@ -175,15 +175,15 @@ STATIC_INLINE int template_callback_map(unpack_user* u, unsigned int n, SV** o)
     return 0;
 }
 
-STATIC_INLINE int template_callback_map_item(unpack_user* u, SV** c, SV* k, SV* v)
+STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v)
 {
     dTHX;
-    hv_store_ent((HV*)SvRV(*c), k, v, 0);
+    (void)hv_store_ent((HV*)SvRV(*c), k, v, 0);
     SvREFCNT_inc(v);
     return 0;
 }
 
-STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, SV** o)
+STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o)
 {
     dTHX;
     /*  *o = newSVpvn_flags(p, l, SVs_TEMP); <= this does not work. */
@@ -203,7 +203,7 @@ STATIC_INLINE int template_callback_raw(unpack_user* u, const char* b, const cha
 STATIC_INLINE SV* _msgpack_unpack(SV* data, int limit) {
     msgpack_unpack_t mp;
     dTHX;
-    unpack_user u = {0, &PL_sv_undef};
+    unpack_user u = {0, &PL_sv_undef, false};
 	int ret;
 	size_t from = 0;
     STRLEN dlen;
@@ -248,7 +248,6 @@ XS(xs_unpack_limit) {
 
 XS(xs_unpack) {
     dXSARGS;
-    msgpack_unpack_t mp;
 
     if (items != 2) {
         Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata')");
@@ -300,7 +299,7 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) {
 	long dlen = limit;
 	int ret;
 
-	if(from >= dlen) {
+	if(from >= (size_t)dlen) {
         Perl_croak(aTHX_ "offset is bigger than data buffer size.");
 	}
 

From 6981234736b10631fc79ffe59933ee3a009c0cc2 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 13:09:14 +0900
Subject: [PATCH 0709/1648] Fix a possible mis-unpack on int64

---
 perl/xs-src/unpack.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 108f9fd..4d4e965 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -111,7 +111,11 @@ STATIC_INLINE int template_callback_int32(unpack_user* u PERL_UNUSED_DECL, int32
 STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t d, SV** o)
 {
     dTHX;
+#if IVSIZE==4
+    *o = sv_2mortal(newSVnv(d));
+#else
     *o = sv_2mortal(newSViv(d));
+#endif
     return 0;
 }
 

From d36543b20419fa87ec71c42ee5f80ca1e03f71a1 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 13:12:17 +0900
Subject: [PATCH 0710/1648] Micro optimizations

---
 perl/xs-src/unpack.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 4d4e965..0a79ef5 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -157,9 +157,9 @@ STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV**
 STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
 {
     dTHX;
-    AV* a = (AV*)sv_2mortal((SV*)newAV());
-    *o = sv_2mortal((SV*)newRV_inc((SV*)a));
-    av_extend(a, n);
+    AV* const a = newAV();
+    *o = sv_2mortal(newRV_noinc((SV*)a));
+    av_extend(a, n + 1);
     return 0;
 }
 
@@ -167,15 +167,15 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL,
 {
     dTHX;
     av_push((AV*)SvRV(*c), o);
-    SvREFCNT_inc(o);
+    SvREFCNT_inc_simple_void_NN(o);
     return 0;
 }
 
 STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n PERL_UNUSED_DECL, SV** o)
 {
     dTHX;
-    HV* h = (HV*)sv_2mortal((SV*)newHV());
-    *o = sv_2mortal(newRV_inc((SV*)h));
+    HV* const h = newHV();
+    *o = sv_2mortal(newRV_noinc((SV*)h));
     return 0;
 }
 
@@ -183,7 +183,7 @@ STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV
 {
     dTHX;
     (void)hv_store_ent((HV*)SvRV(*c), k, v, 0);
-    SvREFCNT_inc(v);
+    SvREFCNT_inc_simple_void_NN(v);
     return 0;
 }
 

From c694f1a4a9b5df801419d23265d6519c59f9e1ae Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 13:16:13 +0900
Subject: [PATCH 0711/1648] Tweaks

---
 perl/xs-src/unpack.c | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 0a79ef5..e3bb901 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -204,7 +204,7 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c
 
 #include "msgpack/unpack_template.h"
 
-STATIC_INLINE SV* _msgpack_unpack(SV* data, int limit) {
+STATIC_INLINE SV* _msgpack_unpack(SV* data, size_t limit PERL_UNUSED_DECL) {
     msgpack_unpack_t mp;
     dTHX;
     unpack_user u = {0, &PL_sv_undef, false};
@@ -252,14 +252,20 @@ XS(xs_unpack_limit) {
 
 XS(xs_unpack) {
     dXSARGS;
+    SV* const data = ST(1);
+    size_t limit;
 
-    if (items != 2) {
-        Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata')");
+    if (items == 2) {
+        limit = sv_len(data);
+    }
+    else if(items == 3) {
+        limit = SvUVx(ST(2));
+    }
+    else {
+        Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('data' [, $limit])");
     }
 
-    {
-        ST(0) = _msgpack_unpack(ST(1), sv_len(ST(1)));
-    }
+    ST(0) = _msgpack_unpack(data, limit);
 
     XSRETURN(1);
 }

From 6852a8ca9d8585fd42ec761900d334ae964732c9 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 13:16:55 +0900
Subject: [PATCH 0712/1648] Remove an unused function: xs_unpack_limit

---
 perl/xs-src/unpack.c | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index e3bb901..5e75717 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -235,21 +235,6 @@ STATIC_INLINE SV* _msgpack_unpack(SV* data, size_t limit PERL_UNUSED_DECL) {
 	}
 }
 
-XS(xs_unpack_limit) {
-    dXSARGS;
-
-    if (items != 3) {
-        Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('datadata', $limit)");
-    }
-
-    {
-        int limit = SvIV(ST(2));
-        ST(0) = _msgpack_unpack(ST(1), limit);
-    }
-    XSRETURN(1);
-}
-
-
 XS(xs_unpack) {
     dXSARGS;
     SV* const data = ST(1);

From 859969241a8695e306789ed30659d05571db8fe2 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 13:20:20 +0900
Subject: [PATCH 0713/1648] Tweaks

---
 perl/xs-src/unpack.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 5e75717..7894223 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -190,8 +190,8 @@ STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV
 STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o)
 {
     dTHX;
-    /*  *o = newSVpvn_flags(p, l, SVs_TEMP); <= this does not work. */
-    *o = sv_2mortal((l==0) ? newSVpv("", 0) : newSVpv(p, l));
+    /*  newSVpvn_flags(p, l, SVs_TEMP) returns an undef if l == 0 */
+    *o = ((l==0) ? newSVpvs_flags("", SVs_TEMP) : newSVpvn_flags(p, l, SVs_TEMP));
     return 0;
 }
 

From af73b9d11b0a8e0a0d1b6a02ee0b8213cca3867c Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 13:22:39 +0900
Subject: [PATCH 0714/1648] Shortcut av_push()

---
 perl/xs-src/unpack.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 7894223..080c4bd 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -166,7 +166,8 @@ STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsig
 STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
 {
     dTHX;
-    av_push((AV*)SvRV(*c), o);
+    AV* const a = (AV*)SvRV(*c);
+    (void)av_store(a, AvFILLp(a) + 1, o); // the same as av_push(a, o)
     SvREFCNT_inc_simple_void_NN(o);
     return 0;
 }

From 0e0a2aa9810c22e3744694965e3d45cab6c135e6 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 13:34:18 +0900
Subject: [PATCH 0715/1648] Add various integers to benchmarks

---
 perl/benchmark/deserialize.pl | 4 +++-
 perl/benchmark/serialize.pl   | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl
index 9658c0c..634a79e 100644
--- a/perl/benchmark/deserialize.pl
+++ b/perl/benchmark/deserialize.pl
@@ -5,11 +5,13 @@ use JSON::XS;
 use Benchmark ':all';
 use Storable;
 
+#$Data::MessagePack::PreferInteger = 1;
+
 my $a = {
     "method" => "handleMessage",
     "params" => [ "user1", "we were just talking" ],
     "id"     => undef,
-    "array"  => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ]
+    "array"  => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ],
 };
 my $j = JSON::XS::encode_json($a);
 my $m = Data::MessagePack->pack($a);
diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl
index ee9e7a4..e0509ff 100644
--- a/perl/benchmark/serialize.pl
+++ b/perl/benchmark/serialize.pl
@@ -9,7 +9,7 @@ my $a = {
     "method" => "handleMessage",
     "params" => [ "user1", "we were just talking" ],
     "id"     => undef,
-    "array"  => [ 1, 11, 234, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ]
+    "array"  => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ],
 };
 
 print "-- serialize\n";

From 1de03fbe180e9abed96aec275cf67b6f7d46a232 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 13:41:10 +0900
Subject: [PATCH 0716/1648] Tweaks for unpacker

---
 perl/xs-src/unpack.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 080c4bd..320eb45 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -196,12 +196,15 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c
     return 0;
 }
 
-#define UNPACKER(from, name) \
-	msgpack_unpack_t *name; \
-    name = INT2PTR(msgpack_unpack_t*, SvROK((from)) ? SvIV(SvRV((from))) : SvIV((from))); \
-	if(name == NULL) { \
-		Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \
-	}
+#define UNPACKER(from, name)                                              \
+    msgpack_unpack_t *name;                                               \
+    if(!(SvROK(from) && SvIOK(SvRV(from)))) {                             \
+        Perl_croak(aTHX_ "Invalid unpacker instance for " #name);         \
+    }                                                                     \
+    name = INT2PTR(msgpack_unpack_t*, SvIVX(SvRV((from))));               \
+    if(name == NULL) {                                                    \
+        Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \
+    }
 
 #include "msgpack/unpack_template.h"
 

From 6a60cb4dc0d5c2e36a4aa8fd8dbe5b3c826a935d Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 13:54:18 +0900
Subject: [PATCH 0717/1648] Add const

---
 perl/xs-src/pack.c | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index c0c610e..6926839 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -13,17 +13,18 @@
     static inline void msgpack_pack ## name
 
 typedef struct {
-    char *cur; /* SvPVX (sv) + current output position */
-    char *end; /* SvEND (sv) */
-    SV *sv;    /* result scalar */
+    char *cur;       /* SvPVX (sv) + current output position */
+    const char *end; /* SvEND (sv) */
+    SV *sv;          /* result scalar */
 } enc_t;
-static void need(enc_t *enc, STRLEN len);
+
+STATIC_INLINE void need(enc_t* const enc, STRLEN const len);
 
 #define msgpack_pack_user enc_t*
 
 #define msgpack_pack_append_buffer(enc, buf, len) \
-    need(enc, len); \
-    memcpy(enc->cur, buf, len); \
+    need(enc, len);                               \
+    memcpy(enc->cur, buf, len);                   \
     enc->cur += len;
 
 #include "msgpack/pack_template.h"
@@ -46,14 +47,14 @@ static void need(enc_t *enc, STRLEN len);
 #define ERR_NESTING_EXCEEDED "perl structure exceeds maximum nesting level (max_depth set too low?)"
 
 
-STATIC_INLINE void need(enc_t *enc, STRLEN len)
+STATIC_INLINE void need(enc_t* const enc, STRLEN const len)
 {
-    dTHX;
     if (enc->cur + len >= enc->end) {
-        STRLEN cur = enc->cur - (char *)SvPVX (enc->sv);
+        dTHX;
+        STRLEN const cur = enc->cur - SvPVX_const(enc->sv);
         sv_grow (enc->sv, cur + (len < (cur >> 2) ? cur >> 2 : len) + 1);
-        enc->cur = SvPVX (enc->sv) + cur;
-        enc->end = SvPVX (enc->sv) + SvLEN (enc->sv) - 1;
+        enc->cur = SvPVX_mutable(enc->sv) + cur;
+        enc->end = SvPVX_const(enc->sv) + SvLEN (enc->sv) - 1;
     }
 }
 

From 83acd6529fe4902624117cab62c2377aa3fe2b27 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 14:06:10 +0900
Subject: [PATCH 0718/1648] Remove an unused user data: source (sv)

---
 perl/xs-src/unpack.c | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 320eb45..90cfa7b 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -3,9 +3,8 @@
 #include "xshelper.h"
 
 typedef struct {
-    int finished;
-    SV* source;
-    int incremented;
+    bool finished;
+    bool incremented;
 } unpack_user;
 
 #include "msgpack/unpack_define.h"
@@ -211,7 +210,7 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c
 STATIC_INLINE SV* _msgpack_unpack(SV* data, size_t limit PERL_UNUSED_DECL) {
     msgpack_unpack_t mp;
     dTHX;
-    unpack_user u = {0, &PL_sv_undef, false};
+    unpack_user u = {false, false};
 	int ret;
 	size_t from = 0;
     STRLEN dlen;
@@ -221,10 +220,7 @@ STATIC_INLINE SV* _msgpack_unpack(SV* data, size_t limit PERL_UNUSED_DECL) {
 	template_init(&mp);
     mp.user = u;
 
-	mp.user.source = data;
 	ret = template_execute(&mp, dptr, (size_t)dlen, &from);
-	mp.user.source = &PL_sv_undef;
-
 	obj = template_data(&mp);
 
 	if(ret < 0) {
@@ -264,7 +260,7 @@ XS(xs_unpack) {
 
 STATIC_INLINE void _reset(SV* self) {
     dTHX;
-	unpack_user u = {0, &PL_sv_undef, 0};
+	unpack_user u = {false, false};
 
 	UNPACKER(self, mp);
 	template_init(mp);
@@ -302,17 +298,15 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) {
         Perl_croak(aTHX_ "offset is bigger than data buffer size.");
 	}
 
-	mp->user.source = data;
 	ret = template_execute(mp, dptr, (size_t)dlen, &from);
-	mp->user.source = &PL_sv_undef;
 
 	if(ret < 0) {
 		Perl_croak(aTHX_ "parse error.");
 	} else if(ret > 0) {
-		mp->user.finished = 1;
+		mp->user.finished = true;
 		return sv_2mortal(newSVuv(from));
 	} else {
-		mp->user.finished = 0;
+		mp->user.finished = false;
 		return sv_2mortal(newSVuv(from));
 	}
 }
@@ -335,7 +329,7 @@ XS(xs_unpacker_execute) {
             SV * d2 = template_data(mp);
             if (!mp->user.incremented && d2) {
                 SvREFCNT_inc(d2);
-                mp->user.incremented = 1;
+                mp->user.incremented = true;
             }
         }
     }
@@ -366,7 +360,7 @@ XS(xs_unpacker_is_finished) {
     }
 
 	UNPACKER(ST(0), mp);
-    ST(0) = (mp->user.finished) ? &PL_sv_yes : &PL_sv_no;
+    ST(0) = boolSV(mp->user.finished);
 
     XSRETURN(1);
 }

From f32234291e26301d4159b109b7fed78fb6e3ee56 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 14:07:33 +0900
Subject: [PATCH 0719/1648] Remove an useless local variable

---
 perl/xs-src/unpack.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 90cfa7b..85136d9 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -285,20 +285,19 @@ XS(xs_unpacker_new) {
     XSRETURN(1);
 }
 
-STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, I32 limit) {
+STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) {
     dTHX;
     UNPACKER(self, mp);
 
     size_t from = off;
 	const char* dptr = SvPV_nolen_const(data);
-	long dlen = limit;
 	int ret;
 
-	if(from >= (size_t)dlen) {
+	if(from >= limit) {
         Perl_croak(aTHX_ "offset is bigger than data buffer size.");
 	}
 
-	ret = template_execute(mp, dptr, (size_t)dlen, &from);
+	ret = template_execute(mp, dptr, limit, &from);
 
 	if(ret < 0) {
 		Perl_croak(aTHX_ "parse error.");
@@ -323,7 +322,7 @@ XS(xs_unpacker_execute) {
         SV* data = ST(1);
         IV  off  = SvIV(ST(2)); /* offset of $data. normaly, 0. */
 
-        ST(0) = _execute_impl(self, data, off, sv_len(data));
+        ST(0) = _execute_impl(self, data, off, (size_t)sv_len(data));
 
         {
             SV * d2 = template_data(mp);
@@ -348,7 +347,7 @@ XS(xs_unpacker_execute_limit) {
     IV off   = SvIV(ST(2));
     IV limit = SvIV(ST(3));
 
-    ST(0) = _execute_impl(self, data, off, limit);
+    ST(0) = _execute_impl(self, data, off, (size_t)limit);
 
     XSRETURN(1);
 }

From f0e044ecd8550a14a9fbda9448bd74a3dace4d1a Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 14:09:03 +0900
Subject: [PATCH 0720/1648] Cleanup

---
 perl/xs-src/unpack.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 85136d9..9500212 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -301,11 +301,8 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) {
 
 	if(ret < 0) {
 		Perl_croak(aTHX_ "parse error.");
-	} else if(ret > 0) {
-		mp->user.finished = true;
-		return sv_2mortal(newSVuv(from));
 	} else {
-		mp->user.finished = false;
+		mp->user.finished = (ret > 0) ? true : false;
 		return sv_2mortal(newSVuv(from));
 	}
 }

From 5bdac96375f0cc5b75313cc3a9a7bbeb21594a0b Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 14:18:38 +0900
Subject: [PATCH 0721/1648] The object root can be NULL

---
 perl/xs-src/unpack.c | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 9500212..dce6782 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -50,8 +50,7 @@ static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL,
 
 STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
 {
-    dTHX;
-    return &PL_sv_undef;
+    return NULL;
 }
 
 STATIC_INLINE int template_callback_uint8(unpack_user* u PERL_UNUSED_DECL, uint8_t d, SV** o)
@@ -310,7 +309,7 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) {
 XS(xs_unpacker_execute) {
     dXSARGS;
     if (items != 3) {
-        Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)");
+        Perl_croak(aTHX_ "Usage: $unpacker->execute(data, off)");
     }
 
 	UNPACKER(ST(0), mp);
@@ -398,8 +397,8 @@ XS(xs_unpacker_destroy) {
     }
 
 	UNPACKER(ST(0), mp);
-    SV * data = template_data(mp);
-    if (SvOK(data)) {
+    SV* const data = template_data(mp);
+    if (data) {
         SvREFCNT_dec(data);
     }
     Safefree(mp);

From 0ae206b1bb5233b7f4538dcf5fc40ff8080588bf Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 14:19:22 +0900
Subject: [PATCH 0722/1648] Revert "The object root can be NULL"

This reverts commit 5bdac96375f0cc5b75313cc3a9a7bbeb21594a0b.
---
 perl/xs-src/unpack.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index dce6782..9500212 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -50,7 +50,8 @@ static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL,
 
 STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
 {
-    return NULL;
+    dTHX;
+    return &PL_sv_undef;
 }
 
 STATIC_INLINE int template_callback_uint8(unpack_user* u PERL_UNUSED_DECL, uint8_t d, SV** o)
@@ -309,7 +310,7 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) {
 XS(xs_unpacker_execute) {
     dXSARGS;
     if (items != 3) {
-        Perl_croak(aTHX_ "Usage: $unpacker->execute(data, off)");
+        Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)");
     }
 
 	UNPACKER(ST(0), mp);
@@ -397,8 +398,8 @@ XS(xs_unpacker_destroy) {
     }
 
 	UNPACKER(ST(0), mp);
-    SV* const data = template_data(mp);
-    if (data) {
+    SV * data = template_data(mp);
+    if (SvOK(data)) {
         SvREFCNT_dec(data);
     }
     Safefree(mp);

From 07e68aa6945e4be0820f9c53a3cc1bbbb041a556 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 14:20:32 +0900
Subject: [PATCH 0723/1648] Fix an usage message

---
 perl/xs-src/unpack.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 9500212..c4ac22f 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -310,7 +310,7 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) {
 XS(xs_unpacker_execute) {
     dXSARGS;
     if (items != 3) {
-        Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off)");
+        Perl_croak(aTHX_ "Usage: $unpacker->execute(data, off)");
     }
 
 	UNPACKER(ST(0), mp);

From 7644555d6bd0f17472f02c18dd65bd24aa14edd4 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 14:21:44 +0900
Subject: [PATCH 0724/1648] Use sv_mortalcopy()

---
 perl/xs-src/unpack.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index c4ac22f..2b6eafe 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -368,7 +368,7 @@ XS(xs_unpacker_data) {
     }
 
 	UNPACKER(ST(0), mp);
-	ST(0) = sv_2mortal(newSVsv(template_data(mp)));
+	ST(0) = sv_mortalcopy(template_data(mp));
 
     XSRETURN(1);
 }

From cd862409cc6028bac9fd5825cfdc0386804091de Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 14:25:50 +0900
Subject: [PATCH 0725/1648] Clean up

---
 perl/xs-src/unpack.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 2b6eafe..2d659b4 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -382,9 +382,7 @@ XS(xs_unpacker_reset) {
 	UNPACKER(ST(0), mp);
     {
         SV * data = template_data(mp);
-        if (data) {
-            SvREFCNT_dec(data);
-        }
+        SvREFCNT_dec(data);
     }
     _reset(ST(0));
 

From 11cde61eab8e15f9936c3234897cef90fef05cd6 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 14:38:26 +0900
Subject: [PATCH 0726/1648] No debug output

---
 perl/xs-src/pack.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 6926839..5eaf17e 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -238,7 +238,7 @@ STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
         else if (len == 1 && *pv == '0')
             msgpack_pack_false(enc); 
         else {
-            sv_dump(sv);
+            //sv_dump(sv);
             croak("cannot encode reference to scalar '%s' unless the scalar is 0 or 1",
                     SvPV_nolen (sv_2mortal (newRV_inc (sv))));
         }

From a11165830bc6075c39978adcb72af07d5d7c6234 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 15:06:03 +0900
Subject: [PATCH 0727/1648] More useful error messages

---
 perl/xs-src/unpack.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 2d659b4..9ccc44a 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -294,7 +294,7 @@ STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) {
 	int ret;
 
 	if(from >= limit) {
-        Perl_croak(aTHX_ "offset is bigger than data buffer size.");
+        Perl_croak(aTHX_ "offset (%lu) is bigger than data buffer size (%lu)", (unsigned long)off, (unsigned long)limit);
 	}
 
 	ret = template_execute(mp, dptr, limit, &from);

From f8ee79ab72034223b0ae698b00e7e369d217cb47 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 15:25:48 +0900
Subject: [PATCH 0728/1648] Add failing tests

---
 perl/t/06_stream_unpack2.t | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t
index eaf2cb4..bc158cd 100644
--- a/perl/t/06_stream_unpack2.t
+++ b/perl/t/06_stream_unpack2.t
@@ -3,7 +3,7 @@ use warnings;
 use Data::MessagePack;
 use Test::More tests => 6;
 
-my $input = [(undef)x16];
+my $input = [42, "foo", { x => [ (undef)x16 ] }, 3.14 ];
 my $packed = Data::MessagePack->pack($input);
 is_deeply(Data::MessagePack->unpack($packed), $input);
 
@@ -17,9 +17,33 @@ is_deeply(Data::MessagePack->unpack($packed), $input);
 {
     my $up = Data::MessagePack::Unpacker->new();
     is $up->execute(substr($packed, 0, 3), 0), 3;
+    ok !$up->is_finished;
     $up->execute($packed, 3);
     ok $up->is_finished;
     is_deeply $up->data, $input;
 }
 
 
+{
+    my $up = Data::MessagePack::Unpacker->new();
+    my $offset = 0;
+    my $size   = 5;
+
+    note "length: ", length($packed);
+    while(not $up->is_finished) {
+        note "offset: ", $offset;
+        my $bytes = substr($packed, $offset, $size);
+        note join " ", map { unpack 'H2', $_ } split //, $bytes;
+        my $x = $up->execute($bytes, 0);
+        if($x <= 0) {
+            diag "Something's wrong: $x";
+            last;
+         }
+         else {
+            $offset += $x;
+         }
+    }
+    ok $up->is_finished;
+    is_deeply $up->data, $input;
+}
+

From fe7e7a8d077de47f3c92dad3ec6eb39e8c1ae077 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 15:26:02 +0900
Subject: [PATCH 0729/1648] Add leaktrace tests

---
 perl/xt/leaks/leaktrace.t | 40 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100755 perl/xt/leaks/leaktrace.t

diff --git a/perl/xt/leaks/leaktrace.t b/perl/xt/leaks/leaktrace.t
new file mode 100755
index 0000000..ff64f6f
--- /dev/null
+++ b/perl/xt/leaks/leaktrace.t
@@ -0,0 +1,40 @@
+#!perl -w
+use strict;
+use Test::Requires 'Test::LeakTrace';
+use Test::More;
+
+use Data::MessagePack;
+
+my $data = {
+    a => 'foo',
+    b => 42,
+    c => undef,
+    d => [qw(bar baz)],
+    e => 3.14,
+};
+
+no_leaks_ok {
+    my $s = Data::MessagePack->pack($data);
+};
+
+no_leaks_ok {
+    eval { Data::MessagePack->pack([\*STDIN]) };
+    #note $@;
+    $@ or die "it must die";
+};
+
+my $s = Data::MessagePack->pack($data);
+
+no_leaks_ok {
+    my $data = Data::MessagePack->unpack($s);
+};
+
+no_leaks_ok {
+    my $ss = $s;
+    chop $ss;
+    eval { Data::MessagePack->unpack($ss) };
+    #note $@;
+    $@ or die "it must die";
+};
+
+done_testing;

From 4cb6d6995f4b07971078f684c035fa343b0cc567 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 15 Sep 2010 15:27:26 +0900
Subject: [PATCH 0730/1648] Make the code clearer

---
 perl/xs-src/unpack.c | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 9ccc44a..9219ed2 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -37,7 +37,6 @@ get_bool (const char *name) {
 }
 
 /* ---------------------------------------------------------------------- */
-
 struct template_context;
 typedef struct template_context msgpack_unpack_t;
 
@@ -166,6 +165,7 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL,
 {
     dTHX;
     AV* const a = (AV*)SvRV(*c);
+    assert(SvTYPE(a) == SVt_PVAV);
     (void)av_store(a, AvFILLp(a) + 1, o); // the same as av_push(a, o)
     SvREFCNT_inc_simple_void_NN(o);
     return 0;
@@ -182,7 +182,9 @@ STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigne
 STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v)
 {
     dTHX;
-    (void)hv_store_ent((HV*)SvRV(*c), k, v, 0);
+    HV* const h = (HV*)SvRV(*c);
+    assert(SvTYPE(h) == SVt_PVHV);
+    (void)hv_store_ent(h, k, v, 0);
     SvREFCNT_inc_simple_void_NN(v);
     return 0;
 }
@@ -321,12 +323,10 @@ XS(xs_unpacker_execute) {
 
         ST(0) = _execute_impl(self, data, off, (size_t)sv_len(data));
 
-        {
-            SV * d2 = template_data(mp);
-            if (!mp->user.incremented && d2) {
-                SvREFCNT_inc(d2);
-                mp->user.incremented = true;
-            }
+        if (!mp->user.incremented) {
+            SV* tmp_obj = template_data(mp);
+            SvREFCNT_inc_simple_void_NN(tmp_obj);
+            mp->user.incremented = true;
         }
     }
 
@@ -368,7 +368,7 @@ XS(xs_unpacker_data) {
     }
 
 	UNPACKER(ST(0), mp);
-	ST(0) = sv_mortalcopy(template_data(mp));
+	ST(0) = template_data(mp);
 
     XSRETURN(1);
 }
@@ -380,10 +380,9 @@ XS(xs_unpacker_reset) {
     }
 
 	UNPACKER(ST(0), mp);
-    {
-        SV * data = template_data(mp);
-        SvREFCNT_dec(data);
-    }
+
+    SV* const data = template_data(mp);
+    SvREFCNT_dec(data);
     _reset(ST(0));
 
     XSRETURN(0);
@@ -396,10 +395,9 @@ XS(xs_unpacker_destroy) {
     }
 
 	UNPACKER(ST(0), mp);
-    SV * data = template_data(mp);
-    if (SvOK(data)) {
-        SvREFCNT_dec(data);
-    }
+
+    SV* const data = template_data(mp);
+    SvREFCNT_dec(data);
     Safefree(mp);
 
     XSRETURN(0);

From 9eeb702ca536f0e97e53202b177d9db5ac9e322f Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Wed, 15 Sep 2010 22:28:46 +0900
Subject: [PATCH 0731/1648] change an annotation-utility in Java.  it allows
 users to pack and unpack a List object.

---
 .../annotation/MessagePackUnpackable.java     |   2 +-
 .../util/annotation/PackUnpackUtil.java       | 705 ++++++++++--------
 .../annotation/TestMessagePackUnpackable.java | 105 ++-
 3 files changed, 499 insertions(+), 313 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java
index 473b541..77f6e6d 100644
--- a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java
+++ b/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java
@@ -6,6 +6,6 @@ import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 
 @Target(ElementType.TYPE)
-@Retention(RetentionPolicy.CLASS)
+@Retention(RetentionPolicy.RUNTIME)
 public @interface MessagePackUnpackable {
 }
diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
index f92c51a..5238444 100644
--- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
+++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
@@ -3,9 +3,14 @@ package org.msgpack.util.annotation;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
+import java.lang.reflect.ParameterizedType;
 import java.math.BigInteger;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -14,7 +19,6 @@ import javassist.CannotCompileException;
 import javassist.ClassPool;
 import javassist.CtClass;
 import javassist.CtConstructor;
-import javassist.CtField;
 import javassist.CtMethod;
 import javassist.CtNewConstructor;
 import javassist.CtNewMethod;
@@ -33,51 +37,16 @@ public class PackUnpackUtil {
 
 		static final String KEYWORD_MODIFIER_PUBLIC = "public";
 
+		static final String KEYWORD_FOR = "for";
+
 		static final String KEYWORD_THROWS = "throws";
 
+		static final String KEYWORD_NEW = "new";
+
 		static final String TYPE_NAME_VOID = void.class.getName();
 
-		static final String TYPE_NAME_BOOLEAN = boolean.class.getName();
-
-		static final String TYPE_NAME_BYTE = byte.class.getName();
-
-		static final String TYPE_NAME_DOUBLE = double.class.getName();
-
-		static final String TYPE_NAME_FLOAT = float.class.getName();
-
-		static final String TYPE_NAME_INT = int.class.getName();
-
-		static final String TYPE_NAME_LONG = long.class.getName();
-
-		static final String TYPE_NAME_SHORT = short.class.getName();
-
 		static final String TYPE_NAME_OBJECT = Object.class.getName();
 
-		static final String TYPE_NAME_BIGINTEGER = BigInteger.class.getName();
-
-		static final String TYPE_NAME_STRING = String.class.getName();
-
-		static final String TYPE_NAME_BOOLEAN2 = Boolean.class.getName();
-
-		static final String TYPE_NAME_BYTE2 = Byte.class.getName();
-
-		static final String TYPE_NAME_DOUBLE2 = Double.class.getName();
-
-		static final String TYPE_NAME_FLOAT2 = Float.class.getName();
-
-		static final String TYPE_NAME_INT2 = Integer.class.getName();
-
-		static final String TYPE_NAME_LONG2 = Long.class.getName();
-
-		static final String TYPE_NAME_SHORT2 = Short.class.getName();
-
-		//static final String TYPE_NAME_BYTEARRAY = byte[].class.getName();
-		static final String TYPE_NAME_BYTEARRAY = "byte[]";
-
-		static final String TYPE_NAME_LIST = List.class.getName();
-
-		static final String TYPE_NAME_MAP = Map.class.getName();
-
 		static final String TYPE_NAME_IOEXCEPTION = IOException.class.getName();
 
 		static final String TYPE_NAME_PACKER = Packer.class.getName();
@@ -111,6 +80,10 @@ public class PackUnpackUtil {
 
 		static final String CHAR_NAME_EQUAL = "=";
 
+		static final String CHAR_NAME_PLUS = "+";
+
+		static final String CHAR_NAME_LESSTHAN = "<";
+
 		static final String CHAR_NAME_RIGHT_PARENTHESIS = ")";
 
 		static final String CHAR_NAME_LEFT_PARENTHESIS = "(";
@@ -127,8 +100,16 @@ public class PackUnpackUtil {
 
 		static final String VARIABLE_NAME_OBJ = "obj";
 
+		static final String VARIABLE_NAME_SIZE = "len";
+
+		static final String VARIABLE_NAME_I = "i";
+
 		static final String METHOD_NAME_VALUEOF = "valueOf";
 
+		static final String METHOD_NAME_ADD = "add";
+
+		static final String METHOD_NAME_PUT = "put";
+
 		static final String METHOD_NAME_MSGPACK = "messagePack";
 
 		static final String METHOD_NAME_MSGUNPACK = "messageUnpack";
@@ -192,73 +173,68 @@ public class PackUnpackUtil {
 			String origName = origClass.getName();
 			String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER;
 			CtClass origCtClass = pool.get(origName);
-			checkClassValidation(origCtClass);
-			checkDefaultConstructorValidation(origCtClass);
+			checkClassValidation(origClass);
+			checkDefaultConstructorValidation(origClass);
 			CtClass enhCtClass = pool.makeClass(enhName);
 			setSuperclass(enhCtClass, origCtClass);
 			setInterfaces(enhCtClass);
 			addConstructor(enhCtClass);
-			CtField[] fields = getDeclaredFields(origCtClass);
+			Field[] fields = getDeclaredFields(origClass);
 			addMessagePackMethod(enhCtClass, origCtClass, fields);
 			addMessageUnpackMethod(enhCtClass, origCtClass, fields);
 			addMessageConvertMethod(enhCtClass, origCtClass, fields);
 			return createClass(enhCtClass);
 		}
 
-		private void checkClassValidation(CtClass origCtClass) {
+		private void checkClassValidation(Class origClass) {
 			// not public, abstract, final
-			int mod = origCtClass.getModifiers();
-			if ((!Modifier.isPublic(mod)) || Modifier.isAbstract(mod)
-					|| Modifier.isFinal(mod)) {
-				throwClassValidationException(origCtClass);
+			int mod = origClass.getModifiers();
+			if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
+					|| Modifier.isAbstract(mod) || Modifier.isFinal(mod)) {
+				throwClassValidationException(origClass);
 			}
 			// interface, enum
-			if (origCtClass.isInterface() || origCtClass.isEnum()) {
-				throwClassValidationException(origCtClass);
+			if (origClass.isInterface() || origClass.isEnum()) {
+				throwClassValidationException(origClass);
 			}
 			// annotation
-			checkPackUnpackAnnotation(origCtClass);
+			checkPackUnpackAnnotation(origClass);
 		}
 
-		private static void throwClassValidationException(CtClass origCtClass) {
+		private static void throwClassValidationException(Class origClass) {
 			throw new PackUnpackUtilException(
 					"it must be a public class and have @"
 							+ MessagePackUnpackable.class.getName() + ": "
-							+ origCtClass.getName());
+							+ origClass.getName());
 		}
 
-		private void checkPackUnpackAnnotation(CtClass origCtClass) {
-			try {
-				Object[] objs = origCtClass.getAnnotations();
-				for (Object obj : objs) {
-					if (obj instanceof MessagePackUnpackable) {
-						return;
-					}
-				}
-				throwClassValidationException(origCtClass);
-			} catch (ClassNotFoundException e) {
-				throw new PackUnpackUtilException(e.getMessage(), e);
+		private void checkPackUnpackAnnotation(Class origClass) {
+			Annotation anno = origClass
+					.getAnnotation(MessagePackUnpackable.class);
+			if (anno == null) {
+				throwClassValidationException(origClass);
 			}
 		}
 
-		private void checkDefaultConstructorValidation(CtClass origCtClass) {
-			CtConstructor cons = null;
+		private void checkDefaultConstructorValidation(Class origClass) {
+			Constructor cons = null;
 			try {
-				cons = origCtClass.getDeclaredConstructor(new CtClass[0]);
-			} catch (NotFoundException e) {
-				throwConstructoValidationException(origCtClass);
+				cons = origClass.getDeclaredConstructor(new Class[0]);
+			} catch (Exception e1) {
+				throwConstructoValidationException(origClass);
 			}
+
 			int mod = cons.getModifiers();
 			if (!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) {
-				throwConstructoValidationException(origCtClass);
+				throwConstructoValidationException(origClass);
 			}
 		}
 
 		private static void throwConstructoValidationException(
-				CtClass origCtClass) {
+				Class origClass) {
 			throw new PackUnpackUtilException(
 					"it must have a public zero-argument constructor: "
-							+ origCtClass.getName());
+							+ origClass.getName());
 		}
 
 		private void setSuperclass(CtClass enhCtClass, CtClass origCtClass)
@@ -282,301 +258,430 @@ public class PackUnpackUtil {
 			enhCtClass.addConstructor(newCtCons);
 		}
 
-		private CtField[] getDeclaredFields(CtClass origCtClass) {
-			ArrayList allFields = new ArrayList();
-			try {
-				CtClass nextCtClass = origCtClass;
-				while (!nextCtClass
-						.equals(pool.get(Constants.TYPE_NAME_OBJECT))) {
-					CtField[] fields = nextCtClass.getDeclaredFields();
-					for (CtField field : fields) {
-						try {
-							checkFieldValidation(field, allFields);
-							allFields.add(field);
-						} catch (PackUnpackUtilException e) { // ignore
-						}
+		private Field[] getDeclaredFields(Class origClass) {
+			ArrayList allFields = new ArrayList();
+			Class nextClass = origClass;
+			while (nextClass != Object.class) {
+				Field[] fields = nextClass.getDeclaredFields();
+				for (Field field : fields) {
+					try {
+						checkFieldValidation(field, allFields);
+						allFields.add(field);
+					} catch (PackUnpackUtilException e) { // ignore
 					}
-					nextCtClass = nextCtClass.getSuperclass();
 				}
-
-			} catch (NotFoundException e) {
-				throw new PackUnpackUtilException(e.getMessage(), e);
+				nextClass = nextClass.getSuperclass();
 			}
-			return allFields.toArray(new CtField[0]);
+			return allFields.toArray(new Field[0]);
 		}
 
-		private void checkFieldValidation(CtField field,
-				ArrayList allFields) {
+		private void checkFieldValidation(Field field, List fields) {
 			// check modifiers (public or protected)
 			int mod = field.getModifiers();
 			if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
 					|| Modifier.isStatic(mod) || Modifier.isFinal(mod)
-					|| Modifier.isTransient(mod)) {
+					|| Modifier.isTransient(mod) || field.isSynthetic()) {
 				throwFieldValidationException(field);
 			}
 			// check same name
-			for (CtField f : allFields) {
+			for (Field f : fields) {
 				if (f.getName().equals(field.getName())) {
 					throwFieldValidationException(field);
 				}
 			}
 		}
 
-		private static void throwFieldValidationException(CtField field) {
+		private static void throwFieldValidationException(Field field) {
 			throw new PackUnpackUtilException("it must be a public field: "
 					+ field.getName());
 		}
 
 		private void addMessagePackMethod(CtClass enhCtClass,
-				CtClass origCtClass, CtField[] fields)
+				CtClass origCtClass, Field[] fields)
 				throws CannotCompileException, NotFoundException {
 			StringBuilder sb = new StringBuilder();
-			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append(
-					Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID)
-					.append(Constants.CHAR_NAME_SPACE).append(
-							Constants.METHOD_NAME_MSGPACK).append(
-							Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
-							Constants.TYPE_NAME_PACKER).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.VARIABLE_NAME_PK).append(
-							Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.KEYWORD_THROWS).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.TYPE_NAME_IOEXCEPTION).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append(
-							Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_PK).append(
-					Constants.CHAR_NAME_DOT).append(
-					Constants.METHOD_NAME_PACKARRAY).append(
-					Constants.CHAR_NAME_LEFT_PARENTHESIS).append(fields.length)
-					.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
-							Constants.CHAR_NAME_SEMICOLON).append(
-							Constants.CHAR_NAME_SPACE);
-			for (CtField field : fields) {
-				insertCodeOfMessagePack(field, sb);
+			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.TYPE_NAME_VOID);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.METHOD_NAME_MSGPACK);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.TYPE_NAME_PACKER);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_PK);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.KEYWORD_THROWS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.TYPE_NAME_IOEXCEPTION);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_PK);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_PACKARRAY);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(fields.length);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			for (Field field : fields) {
+				insertCodeOfMessagePack(sb, field);
 			}
 			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			System.out.println("messagePack method: " + sb.toString());
+			//System.out.println("messagePack method: " + sb.toString());
 			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
 			enhCtClass.addMethod(newCtMethod);
 		}
 
-		private void insertCodeOfMessagePack(CtField field, StringBuilder sb)
-				throws NotFoundException {
-			sb.append(Constants.VARIABLE_NAME_PK).append(
-					Constants.CHAR_NAME_DOT).append(Constants.METHOD_NAME_PACK)
-					.append(Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
-							field.getName()).append(
-							Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
-							Constants.CHAR_NAME_SEMICOLON).append(
-							Constants.CHAR_NAME_SPACE);
+		private void insertCodeOfMessagePack(StringBuilder sb, Field field) {
+			sb.append(Constants.VARIABLE_NAME_PK);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_PACK);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(field.getName());
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
 		}
 
 		private void addMessageUnpackMethod(CtClass enhCtClass,
-				CtClass origCtClass, CtField[] fields)
+				CtClass origCtClass, Field[] fields)
 				throws CannotCompileException, NotFoundException {
 			StringBuilder sb = new StringBuilder();
-			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append(
-					Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID)
-					.append(Constants.CHAR_NAME_SPACE).append(
-							Constants.METHOD_NAME_MSGUNPACK).append(
-							Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
-							Constants.TYPE_NAME_UNPACKER).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.VARIABLE_NAME_PK).append(
-							Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.KEYWORD_THROWS).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.TYPE_NAME_MSGTYPEEXCEPTION).append(
-							Constants.CHAR_NAME_COMMA).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.TYPE_NAME_IOEXCEPTION).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append(
-							Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_PK).append(
-					Constants.CHAR_NAME_DOT).append(
-					Constants.METHOD_NAME_UNPACKARRAY).append(
-					Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
-					Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
-					Constants.CHAR_NAME_SEMICOLON).append(
-					Constants.CHAR_NAME_SPACE);
-			for (CtField field : fields) {
-				insertCodeOfMessageUnpack(field, sb);
+			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.TYPE_NAME_VOID);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.METHOD_NAME_MSGUNPACK);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.TYPE_NAME_UNPACKER);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_PK);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.KEYWORD_THROWS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.TYPE_NAME_MSGTYPEEXCEPTION);
+			sb.append(Constants.CHAR_NAME_COMMA);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.TYPE_NAME_IOEXCEPTION);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_PK);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_UNPACKARRAY);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			for (Field field : fields) {
+				insertCodeOfMessageUnpack(sb, field, field.getType());
 			}
 			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			System.out.println("messageUnpack method: " + sb.toString());
+			//System.out.println("messageUnpack method: " + sb.toString());
 			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
 			enhCtClass.addMethod(newCtMethod);
 		}
 
-		private void insertCodeOfMessageUnpack(CtField field, StringBuilder sb)
-				throws NotFoundException {
-			CtClass type = field.getType();
-			insertRightVariable(sb, field, type);
-			insertValueOfMethodAndLeftParenthesis(sb, type);
-			sb.append(Constants.VARIABLE_NAME_PK).append(
-					Constants.CHAR_NAME_DOT);
-			insertUnpackMethod(sb, type);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			insertUnpackMethodArgumenet(sb, field, type);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			insertValueOfMethodAndRightParenthesis(sb, type);
-			sb.append(Constants.CHAR_NAME_SEMICOLON).append(
-					Constants.CHAR_NAME_SPACE);
-
-		}
-
-		private void insertRightVariable(StringBuilder sb, CtField field,
-				CtClass type) throws NotFoundException {
-			if (type.isPrimitive()) { // primitive type
-				sb.append(field.getName()).append(Constants.CHAR_NAME_SPACE)
-						.append(Constants.CHAR_NAME_EQUAL).append(
-								Constants.CHAR_NAME_SPACE);
-			} else { // reference type
-				if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean
-						|| type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte
-						|| type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double
-						|| type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float
-						|| type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer
-						|| type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long
-						|| type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short
-						|| type
-								.equals(pool
-										.get(Constants.TYPE_NAME_BIGINTEGER)) // BigInteger
-						|| type.equals(pool.get(Constants.TYPE_NAME_STRING)) // String
-						|| type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY)) // byte[]
-						|| type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST)) // List
-						|| type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP)) // Map
-				) {
-					sb.append(field.getName())
-							.append(Constants.CHAR_NAME_SPACE).append(
-									Constants.CHAR_NAME_EQUAL).append(
-									Constants.CHAR_NAME_SPACE);
-				} else { // MessageUnpackable
-					return;
-				}
-			}
-
-		}
-
-		private void insertValueOfMethodAndLeftParenthesis(StringBuilder sb,
-				CtClass type) throws NotFoundException {
-			if (type.isPrimitive()) { // primitive type
-				return;
-			} else { // reference type
-				if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean
-						|| type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte
-						|| type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double
-						|| type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float
-						|| type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer
-						|| type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long
-						|| type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short
-				) {
-					sb.append(type.getName()).append(Constants.CHAR_NAME_DOT)
-							.append(Constants.METHOD_NAME_VALUEOF).append(
-									Constants.CHAR_NAME_LEFT_PARENTHESIS);
-				} else {
-					return;
-				}
+		private void insertCodeOfMessageUnpack(StringBuilder sb, Field field,
+				Class type) throws NotFoundException {
+			if (type.isPrimitive()) {
+				// primitive type
+				insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type);
+			} else if (type.equals(Boolean.class) || // Boolean
+					type.equals(Byte.class) || // Byte
+					type.equals(Double.class) || // Double
+					type.equals(Float.class) || // Float
+					type.equals(Integer.class) || // Integer
+					type.equals(Long.class) || // Long
+					type.equals(Short.class)) { // Short
+				// reference type (wrapper type)
+				insertCodeOfMessageUnpackForWrapperTypes(sb, field, type);
+			} else if (type.equals(BigInteger.class) || // BigInteger
+					type.equals(String.class) || // String
+					type.equals(byte[].class)) { // byte[]
+				// reference type (other type)
+				insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type);
+			} else if (List.class.isAssignableFrom(type)) {
+				// List
+				insertCodeOfMessageUnpackForListType(sb, field, type);
+			} else if (Map.class.isAssignableFrom(type)) {
+				// Map
+				insertCodeOfMessageUnpackForMapType(sb, field, type);
+			} else if (MessageUnpackable.class.isAssignableFrom(type)
+					|| (getCache(type.getName()) != null)) {
+				// MessageUnpackable
+				insertCodeOfMessageUnpackForMsgUnpackableType(sb, field, type);
+			} else {
+				throw new NotFoundException("unknown type:  " + type.getName());
 			}
 		}
 
-		private void insertUnpackMethod(StringBuilder sb, CtClass type)
+		private void insertCodeOfMessageUnpackForPrimitiveTypes(
+				StringBuilder sb, Field field, Class type)
 				throws NotFoundException {
-			if (type.equals(CtClass.booleanType)) { // boolean
+			// insert a right variable
+			if (field != null) {
+				sb.append(field.getName());
+				sb.append(Constants.CHAR_NAME_SPACE);
+				sb.append(Constants.CHAR_NAME_EQUAL);
+				sb.append(Constants.CHAR_NAME_SPACE);
+			}
+			sb.append(Constants.VARIABLE_NAME_PK);
+			sb.append(Constants.CHAR_NAME_DOT);
+			// insert unpack method
+			if (type.equals(boolean.class)) { // boolean
 				sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN);
-			} else if (type.equals(CtClass.byteType)) { // byte
+			} else if (type.equals(byte.class)) { // byte
 				sb.append(Constants.METHOD_NAME_UNPACKBYTE);
-			} else if (type.equals(CtClass.doubleType)) { // double
+			} else if (type.equals(double.class)) { // double
 				sb.append(Constants.METHOD_NAME_UNPACKDOUBLE);
-			} else if (type.equals(CtClass.floatType)) { // float
+			} else if (type.equals(float.class)) { // float
 				sb.append(Constants.METHOD_NAME_UNPACKFLOAT);
-			} else if (type.equals(CtClass.intType)) { // int
+			} else if (type.equals(int.class)) { // int
 				sb.append(Constants.METHOD_NAME_UNPACKINT);
-			} else if (type.equals(CtClass.longType)) { // long
+			} else if (type.equals(long.class)) { // long
 				sb.append(Constants.METHOD_NAME_UNPACKLONG);
-			} else if (type.equals(CtClass.shortType)) { // short
+			} else if (type.equals(short.class)) { // short
 				sb.append(Constants.METHOD_NAME_UNPACKSHORT);
 			} else { // reference type
-				Class c = null;
-				if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2))) { // Boolean
-					sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN);
-				} else if (type.equals(pool.get(Constants.TYPE_NAME_BYTE2))) { // Byte
-					sb.append(Constants.METHOD_NAME_UNPACKBYTE);
-				} else if (type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2))) { // Double
-					sb.append(Constants.METHOD_NAME_UNPACKDOUBLE);
-				} else if (type.equals(pool.get(Constants.TYPE_NAME_FLOAT2))) { // Float
-					sb.append(Constants.METHOD_NAME_UNPACKFLOAT);
-				} else if (type.equals(pool.get(Constants.TYPE_NAME_INT2))) { // Integer
-					sb.append(Constants.METHOD_NAME_UNPACKINT);
-				} else if (type.equals(pool.get(Constants.TYPE_NAME_LONG2))) { // Long
-					sb.append(Constants.METHOD_NAME_UNPACKLONG);
-				} else if (type.equals(pool.get(Constants.TYPE_NAME_SHORT2))) { // Short
-					sb.append(Constants.METHOD_NAME_UNPACKSHORT);
-				} else if (type
-						.equals(pool.get(Constants.TYPE_NAME_BIGINTEGER))) { // BigInteger
+				if (type.equals(BigInteger.class)) { // BigInteger
 					sb.append(Constants.METHOD_NAME_UNPACKBIGINTEGER);
-				} else if (type.equals(pool.get(Constants.TYPE_NAME_STRING))) { // String
+				} else if (type.equals(String.class)) { // String
 					sb.append(Constants.METHOD_NAME_UNPACKSTRING);
-				} else if (type.equals(pool.get(Constants.TYPE_NAME_BYTEARRAY))) { // byte[]
+				} else if (type.equals(byte[].class)) { // byte[]
 					sb.append(Constants.METHOD_NAME_UNPACKBYTEARRAY);
-				} else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_LIST))) { // List
-					sb.append(Constants.METHOD_NAME_UNPACKARRAY);
-				} else if (type.subtypeOf(pool.get(Constants.TYPE_NAME_MAP))) { // Map
-					sb.append(Constants.METHOD_NAME_UNPACKMAP);
-				} else if (type.subtypeOf(pool
-						.get(Constants.TYPE_NAME_MSGUNPACKABLE))
-						|| ((c = getCache(type.getName())) != null)) { // MessageUnpackable
-					sb.append(Constants.METHOD_NAME_UNPACK);
 				} else {
 					throw new NotFoundException("unknown type:  "
 							+ type.getName());
 				}
 			}
-		}
-
-		private void insertUnpackMethodArgumenet(StringBuilder sb,
-				CtField field, CtClass type) throws NotFoundException {
-			if (type.isPrimitive()) { // primitive type
-				return;
-			} else { // reference type
-				Class c = null;
-				if (type.equals(pool.get(Constants.TYPE_NAME_MSGUNPACKABLE))
-						|| ((c = getCache(type.getName())) != null)) {
-					sb.append("(org.msgpack.MessageUnpackable)");
-					sb.append(field.getName());
-				} else {
-					return;
-				}
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			if (field != null) {
+				sb.append(Constants.CHAR_NAME_SEMICOLON);
+				sb.append(Constants.CHAR_NAME_SPACE);
 			}
 		}
 
-		private void insertValueOfMethodAndRightParenthesis(StringBuilder sb,
-				CtClass type) throws NotFoundException {
-			if (type.isPrimitive()) { // primitive type
-				return;
-			} else { // reference type
-				if (type.equals(pool.get(Constants.TYPE_NAME_BOOLEAN2)) // Boolean
-						|| type.equals(pool.get(Constants.TYPE_NAME_BYTE2)) // Byte
-						|| type.equals(pool.get(Constants.TYPE_NAME_DOUBLE2)) // Double
-						|| type.equals(pool.get(Constants.TYPE_NAME_FLOAT2)) // Float
-						|| type.equals(pool.get(Constants.TYPE_NAME_INT2)) // Integer
-						|| type.equals(pool.get(Constants.TYPE_NAME_LONG2)) // Long
-						|| type.equals(pool.get(Constants.TYPE_NAME_SHORT2)) // Short
-				) {
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-				} else {
-					return;
-				}
+		private void insertCodeOfMessageUnpackForWrapperTypes(StringBuilder sb,
+				Field field, Class type) throws NotFoundException {
+			// insert a right variable
+			if (field != null) {
+				sb.append(field.getName());
+				sb.append(Constants.CHAR_NAME_SPACE);
+				sb.append(Constants.CHAR_NAME_EQUAL);
+				sb.append(Constants.CHAR_NAME_SPACE);
 			}
+			sb.append(type.getName());
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_VALUEOF);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.VARIABLE_NAME_PK);
+			sb.append(Constants.CHAR_NAME_DOT);
+			// insert the name of a unpack method
+			if (type.equals(Boolean.class)) { // Boolean
+				sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN);
+			} else if (type.equals(Byte.class)) { // Byte
+				sb.append(Constants.METHOD_NAME_UNPACKBYTE);
+			} else if (type.equals(Double.class)) { // Double
+				sb.append(Constants.METHOD_NAME_UNPACKDOUBLE);
+			} else if (type.equals(Float.class)) { // Float
+				sb.append(Constants.METHOD_NAME_UNPACKFLOAT);
+			} else if (type.equals(Integer.class)) { // Integer
+				sb.append(Constants.METHOD_NAME_UNPACKINT);
+			} else if (type.equals(Long.class)) { // Long
+				sb.append(Constants.METHOD_NAME_UNPACKLONG);
+			} else if (type.equals(Short.class)) { // Short
+				sb.append(Constants.METHOD_NAME_UNPACKSHORT);
+			} else {
+				throw new NotFoundException("unknown type:  " + type.getName());
+			}
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			if (field != null) {
+				sb.append(Constants.CHAR_NAME_SEMICOLON);
+				sb.append(Constants.CHAR_NAME_SPACE);
+			}
+		}
+
+		private void insertCodeOfMessageUnpackForListType(StringBuilder sb,
+				Field field, Class type) throws NotFoundException {
+			ParameterizedType generic = (ParameterizedType) field
+					.getGenericType();
+			Class genericType = (Class) generic.getActualTypeArguments()[0];
+
+			// len
+			sb.append(int.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_SIZE);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_PK);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_UNPACKARRAY);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// field initializer
+			sb.append(field.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.KEYWORD_NEW);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(ArrayList.class.getName());
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// for loop
+			sb.append(Constants.KEYWORD_FOR);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(int.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_I);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(0);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_I);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LESSTHAN);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_SIZE);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_PLUS);
+			sb.append(Constants.CHAR_NAME_PLUS);
+			sb.append(Constants.VARIABLE_NAME_I);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// block
+			sb.append(field.getName());
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_ADD);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			insertCodeOfMessageUnpack(sb, null, genericType);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+		}
+
+		private void insertCodeOfMessageUnpackForMapType(StringBuilder sb,
+				Field field, Class type) throws NotFoundException {
+			ParameterizedType generic = (ParameterizedType) field
+					.getGenericType();
+			Class genericType0 = (Class) generic.getActualTypeArguments()[0];
+			Class genericType1 = (Class) generic.getActualTypeArguments()[1];
+
+			// len
+			sb.append(int.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_SIZE);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_PK);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_UNPACKMAP);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// field initializer
+			sb.append(field.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.KEYWORD_NEW);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(HashMap.class.getName());
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// for loop
+			sb.append(Constants.KEYWORD_FOR);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(int.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_I);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(0);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_I);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LESSTHAN);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_SIZE);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_PLUS);
+			sb.append(Constants.CHAR_NAME_PLUS);
+			sb.append(Constants.VARIABLE_NAME_I);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// block map.
+			sb.append(field.getName());
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_PUT);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			insertCodeOfMessageUnpack(sb, null, genericType0);
+			sb.append(Constants.CHAR_NAME_COMMA);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			insertCodeOfMessageUnpack(sb, null, genericType1);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+		}
+
+		private void insertCodeOfMessageUnpackForMsgUnpackableType(
+				StringBuilder sb, Field field, Class type) {
+			// insert a right variable // ignore
+			sb.append(Constants.VARIABLE_NAME_PK);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_UNPACK);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(MessageUnpackable.class.getName());
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(field.getName());
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
 		}
 
 		private void addMessageConvertMethod(CtClass enhCtClass,
-				CtClass origCtClass, CtField[] fields)
+				CtClass origCtClass, Field[] fields)
 				throws CannotCompileException {
 			StringBuilder sb = new StringBuilder();
 			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append(
@@ -597,7 +702,7 @@ public class PackUnpackUtil {
 							Constants.CHAR_NAME_SPACE);
 			// TODO
 			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			System.out.println("messageConvert method: " + sb.toString());
+			//System.out.println("messageConvert method: " + sb.toString());
 			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
 			enhCtClass.addMethod(newCtMethod);
 		}
@@ -684,6 +789,6 @@ public class PackUnpackUtil {
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Unpacker pac = new Unpacker(in);
 		pac.unpack((MessageUnpackable) dst);
-		System.out.println(src.equals(dst));
+		//System.out.println(src.equals(dst));
 	}
 }
diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
index 1cde65f..dbea7cb 100644
--- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
+++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
@@ -3,6 +3,9 @@ package org.msgpack.util.annotation;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -16,9 +19,9 @@ import org.msgpack.Unpacker;
 public class TestMessagePackUnpackable extends TestCase {
 
 	@Test
-	public void testGeneralPrimitiveTypeFields() throws Exception {
-		GeneralPrimitiveTypeFieldsClass src = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil
-				.newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class);
+	public void testPrimitiveTypeFields() throws Exception {
+		PrimitiveTypeFieldsClass src = (PrimitiveTypeFieldsClass) PackUnpackUtil
+				.newEnhancedInstance(PrimitiveTypeFieldsClass.class);
 		src.f0 = (byte) 0;
 		src.f1 = 1;
 		src.f2 = 2;
@@ -28,8 +31,8 @@ public class TestMessagePackUnpackable extends TestCase {
 		src.f6 = false;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		new Packer(out).pack(src);
-		GeneralPrimitiveTypeFieldsClass dst = (GeneralPrimitiveTypeFieldsClass) PackUnpackUtil
-				.newEnhancedInstance(GeneralPrimitiveTypeFieldsClass.class);
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) PackUnpackUtil
+				.newEnhancedInstance(PrimitiveTypeFieldsClass.class);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Unpacker pac = new Unpacker(in);
 		pac.unpack((MessageUnpackable) dst);
@@ -43,7 +46,7 @@ public class TestMessagePackUnpackable extends TestCase {
 	}
 
 	@MessagePackUnpackable
-	public static class GeneralPrimitiveTypeFieldsClass {
+	public static class PrimitiveTypeFieldsClass {
 		public byte f0;
 		public short f1;
 		public int f2;
@@ -52,7 +55,7 @@ public class TestMessagePackUnpackable extends TestCase {
 		public double f5;
 		public boolean f6;
 
-		public GeneralPrimitiveTypeFieldsClass() {
+		public PrimitiveTypeFieldsClass() {
 		}
 	}
 
@@ -107,15 +110,93 @@ public class TestMessagePackUnpackable extends TestCase {
 		}
 	}
 
-	public void testListAndMap() throws Exception {
+	public void testListTypes() throws Exception {
+		SampleListTypes src = (SampleListTypes) PackUnpackUtil
+				.newEnhancedInstance(SampleListTypes.class);
+		src.f0 = new ArrayList();
+		src.f1 = new ArrayList();
+		src.f1.add(1);
+		src.f1.add(2);
+		src.f1.add(3);
+		src.f2 = new ArrayList();
+		src.f2.add("e1");
+		src.f2.add("e2");
+		src.f2.add("e3");
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		SampleListTypes dst = (SampleListTypes) PackUnpackUtil
+				.newEnhancedInstance(SampleListTypes.class);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		pac.unpack((MessageUnpackable) dst);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		for (int i = 0; i < src.f1.size(); ++i) {
+			assertEquals(src.f1.get(i), dst.f1.get(i));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		for (int i = 0; i < src.f2.size(); ++i) {
+			assertEquals(src.f2.get(i), dst.f2.get(i));
+		}
 	}
 
 	@MessagePackUnpackable
-	public static class ListAndMapClass {
-		public List f0;
-		public Map f1;
+	public static class SampleListTypes {
+		public List f0;
+		public List f1;
+		public List f2;
 
-		public ListAndMapClass() {
+		public SampleListTypes() {
+		}
+	}
+
+	public void testMapTypes() throws Exception {
+		SampleMapTypes src = (SampleMapTypes) PackUnpackUtil
+				.newEnhancedInstance(SampleMapTypes.class);
+		src.f0 = new HashMap();
+		src.f1 = new HashMap();
+		src.f1.put(1, 1);
+		src.f1.put(2, 2);
+		src.f1.put(3, 3);
+		src.f2 = new HashMap();
+		src.f2.put("k1", 1);
+		src.f2.put("k2", 2);
+		src.f2.put("k3", 3);
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		SampleMapTypes dst = (SampleMapTypes) PackUnpackUtil
+				.newEnhancedInstance(SampleMapTypes.class);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		pac.unpack((MessageUnpackable) dst);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		Iterator srcf1 = src.f1.keySet().iterator();
+		Iterator dstf1 = dst.f1.keySet().iterator();
+		while (srcf1.hasNext()) {
+			Integer s1 = srcf1.next();
+			Integer d1 = dstf1.next();
+			assertEquals(s1, d1);
+			assertEquals(src.f1.get(s1), dst.f1.get(d1));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		Iterator srcf2 = src.f2.keySet().iterator();
+		Iterator dstf2 = dst.f2.keySet().iterator();
+		while (srcf2.hasNext()) {
+			String s2 = srcf2.next();
+			String d2 = dstf2.next();
+			assertEquals(s2, d2);
+			assertEquals(src.f2.get(s2), dst.f2.get(d2));
+		}
+	}
+
+	@MessagePackUnpackable
+	public static class SampleMapTypes {
+		public Map f0;
+		public Map f1;
+		public Map f2;
+
+		public SampleMapTypes() {
 		}
 	}
 

From afbddbfcda1ee0acde3c5343a77c4c575f73ad95 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Thu, 16 Sep 2010 20:24:01 +0900
Subject: [PATCH 0732/1648] Fix the stream unpacker

---
 perl/t/06_stream_unpack2.t |  33 ++---
 perl/xs-src/MessagePack.c  |   1 +
 perl/xs-src/pack.c         |   2 +-
 perl/xs-src/unpack.c       | 254 +++++++++++++++++--------------------
 perl/xt/leaks/leaktrace.t  |  22 +++-
 5 files changed, 148 insertions(+), 164 deletions(-)

diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t
index bc158cd..68a2873 100644
--- a/perl/t/06_stream_unpack2.t
+++ b/perl/t/06_stream_unpack2.t
@@ -1,9 +1,9 @@
 use strict;
 use warnings;
 use Data::MessagePack;
-use Test::More tests => 6;
+use Test::More tests => 9;
 
-my $input = [42, "foo", { x => [ (undef)x16 ] }, 3.14 ];
+my $input = [ 42, "foo", { x => [ (1) x 16 ] }, undef, 1 ];
 my $packed = Data::MessagePack->pack($input);
 is_deeply(Data::MessagePack->unpack($packed), $input);
 
@@ -26,24 +26,19 @@ is_deeply(Data::MessagePack->unpack($packed), $input);
 
 {
     my $up = Data::MessagePack::Unpacker->new();
-    my $offset = 0;
-    my $size   = 5;
+    my $size = 8;
 
-    note "length: ", length($packed);
-    while(not $up->is_finished) {
-        note "offset: ", $offset;
-        my $bytes = substr($packed, $offset, $size);
-        note join " ", map { unpack 'H2', $_ } split //, $bytes;
-        my $x = $up->execute($bytes, 0);
-        if($x <= 0) {
-            diag "Something's wrong: $x";
-            last;
-         }
-         else {
-            $offset += $x;
-         }
+    note "packed size: ", length($packed);
+    open my $stream, '<:bytes :scalar', \$packed;
+    my $buff;
+    while( read($stream, $buff, $size) ) {
+        note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff;
+
+        $up->execute($buff);
     }
-    ok $up->is_finished;
-    is_deeply $up->data, $input;
+    ok $up->is_finished, 'is_finished';
+    my $data = $up->data;
+    note explain($data);
+    is_deeply $data, $input;
 }
 
diff --git a/perl/xs-src/MessagePack.c b/perl/xs-src/MessagePack.c
index 259fd15..aba8ef9 100644
--- a/perl/xs-src/MessagePack.c
+++ b/perl/xs-src/MessagePack.c
@@ -14,6 +14,7 @@ void boot_Data__MessagePack_pack(void);
 
 XS(boot_Data__MessagePack) {
     dXSARGS;
+    PERL_UNUSED_VAR(items);
 
     boot_Data__MessagePack_pack();
 
diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 5eaf17e..9a58ed0 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -145,7 +145,7 @@ STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) {
 }
 
 
-static void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
+STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
 
 STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
     dTHX;
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 9219ed2..a429ecd 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -26,14 +26,9 @@ typedef struct {
 /* utility functions                                                      */
 
 STATIC_INLINE SV *
-get_bool (const char *name) {
+get_bool (const char* const name) {
     dTHX;
-    SV * sv = sv_mortalcopy(get_sv( name, 1 ));
-
-    SvREADONLY_on(sv);
-    SvREADONLY_on( SvRV(sv) );
-
-    return sv;
+    return newSVsv(get_sv( name, GV_ADD ));
 }
 
 /* ---------------------------------------------------------------------- */
@@ -49,85 +44,76 @@ static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL,
 
 STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
 {
-    dTHX;
-    return &PL_sv_undef;
+    return NULL;
 }
 
-STATIC_INLINE int template_callback_uint8(unpack_user* u PERL_UNUSED_DECL, uint8_t d, SV** o)
+#if IVSIZE == 4
+
+STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
 {
     dTHX;
-    *o = sv_2mortal(newSVuv(d));
+    *o = newSVuv(d);
     return 0;
 }
 
-STATIC_INLINE int template_callback_uint16(unpack_user* u PERL_UNUSED_DECL, uint16_t d, SV** o)
+STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
 {
     dTHX;
-    *o = sv_2mortal(newSVuv(d));
+    *o = newSVnv((NV)d);
     return 0;
 }
 
-STATIC_INLINE int template_callback_uint32(unpack_user* u PERL_UNUSED_DECL, uint32_t d, SV** o)
+STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
 {
     dTHX;
-    *o = sv_2mortal(newSVuv(d));
+    *o = newSViv(d);
     return 0;
 }
 
-STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t d, SV** o)
+STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
 {
     dTHX;
-#if IVSIZE==4
-    *o = sv_2mortal(newSVnv(d));
-#else
-    *o = sv_2mortal(newSVuv(d));
-#endif
+    *o = newSVnv((NV)d);
     return 0;
 }
 
-STATIC_INLINE int template_callback_int8(unpack_user* u PERL_UNUSED_DECL, int8_t d, SV** o)
+#else /* IVSIZE == 8 */
+
+
+STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
 {
     dTHX;
-    *o = sv_2mortal(newSViv(d));
+    *o = newSVuv(d);
     return 0;
 }
 
-STATIC_INLINE int template_callback_int16(unpack_user* u PERL_UNUSED_DECL, int16_t d, SV** o)
+#define template_callback_uint64 template_callback_UV
+
+STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
 {
     dTHX;
-    *o = sv_2mortal(newSViv(d));
+    *o = newSViv(d);
     return 0;
 }
 
-STATIC_INLINE int template_callback_int32(unpack_user* u PERL_UNUSED_DECL, int32_t d, SV** o)
-{
-    dTHX;
-    *o = sv_2mortal(newSViv(d));
-    return 0;
-}
+#define template_callback_uint64 template_callback_IV
 
-STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t d, SV** o)
-{
-    dTHX;
-#if IVSIZE==4
-    *o = sv_2mortal(newSVnv(d));
-#else
-    *o = sv_2mortal(newSViv(d));
-#endif
-    return 0;
-}
+#endif /* IVSIZE */
 
-STATIC_INLINE int template_callback_float(unpack_user* u PERL_UNUSED_DECL, float d, SV** o)
-{
-    dTHX;
-    *o = sv_2mortal(newSVnv(d));
-    return 0;
-}
+#define template_callback_uint8  template_callback_UV
+#define template_callback_uint16 template_callback_UV
+#define template_callback_uint32 template_callback_UV
+
+#define template_callback_int8  template_callback_IV
+#define template_callback_int16 template_callback_IV
+#define template_callback_int32 template_callback_IV
+
+#define template_callback_float template_callback_double
 
 STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
 {
     dTHX;
-    *o = sv_2mortal(newSVnv(d));
+    *o = newSVnv(d);
     return 0;
 }
 
@@ -135,7 +121,7 @@ STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, doub
 STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     dTHX;
-    *o = sv_newmortal();
+    *o = newSV(0);
     return 0;
 }
 
@@ -148,7 +134,8 @@ STATIC_INLINE int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o
 
 STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
-    dTHX; *o = get_bool("Data::MessagePack::false");
+    dTHX;
+    *o = get_bool("Data::MessagePack::false");
     return 0;
 }
 
@@ -156,7 +143,7 @@ STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsig
 {
     dTHX;
     AV* const a = newAV();
-    *o = sv_2mortal(newRV_noinc((SV*)a));
+    *o = newRV_noinc((SV*)a);
     av_extend(a, n + 1);
     return 0;
 }
@@ -167,7 +154,6 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL,
     AV* const a = (AV*)SvRV(*c);
     assert(SvTYPE(a) == SVt_PVAV);
     (void)av_store(a, AvFILLp(a) + 1, o); // the same as av_push(a, o)
-    SvREFCNT_inc_simple_void_NN(o);
     return 0;
 }
 
@@ -175,7 +161,7 @@ STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigne
 {
     dTHX;
     HV* const h = newHV();
-    *o = sv_2mortal(newRV_noinc((SV*)h));
+    *o = newRV_noinc((SV*)h);
     return 0;
 }
 
@@ -185,7 +171,7 @@ STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV
     HV* const h = (HV*)SvRV(*c);
     assert(SvTYPE(h) == SVt_PVHV);
     (void)hv_store_ent(h, k, v, 0);
-    SvREFCNT_inc_simple_void_NN(v);
+    SvREFCNT_dec(k);
     return 0;
 }
 
@@ -193,10 +179,12 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c
 {
     dTHX;
     /*  newSVpvn_flags(p, l, SVs_TEMP) returns an undef if l == 0 */
-    *o = ((l==0) ? newSVpvs_flags("", SVs_TEMP) : newSVpvn_flags(p, l, SVs_TEMP));
+    *o = ((l==0) ? newSVpvs("") : newSVpvn(p, l));
     return 0;
 }
 
+#include "msgpack/unpack_template.h"
+
 #define UNPACKER(from, name)                                              \
     msgpack_unpack_t *name;                                               \
     if(!(SvROK(from) && SvIOK(SvRV(from)))) {                             \
@@ -207,36 +195,6 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c
         Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \
     }
 
-#include "msgpack/unpack_template.h"
-
-STATIC_INLINE SV* _msgpack_unpack(SV* data, size_t limit PERL_UNUSED_DECL) {
-    msgpack_unpack_t mp;
-    dTHX;
-    unpack_user u = {false, false};
-	int ret;
-	size_t from = 0;
-    STRLEN dlen;
-    const char * dptr = SvPV_const(data, dlen);
-    SV* obj;
-
-	template_init(&mp);
-    mp.user = u;
-
-	ret = template_execute(&mp, dptr, (size_t)dlen, &from);
-	obj = template_data(&mp);
-
-	if(ret < 0) {
-        Perl_croak(aTHX_ "parse error.");
-	} else if(ret == 0) {
-        Perl_croak(aTHX_ "insufficient bytes.");
-	} else {
-		if(from < dlen) {
-            Perl_croak(aTHX_ "extra bytes.");
-		}
-        return obj;
-	}
-}
-
 XS(xs_unpack) {
     dXSARGS;
     SV* const data = ST(1);
@@ -252,17 +210,40 @@ XS(xs_unpack) {
         Perl_croak(aTHX_ "Usage: Data::MessagePack->unpack('data' [, $limit])");
     }
 
-    ST(0) = _msgpack_unpack(data, limit);
+    STRLEN dlen;
+    const char* const dptr = SvPV_const(data, dlen);
 
+    msgpack_unpack_t mp;
+    template_init(&mp);
+
+    unpack_user const u = {false, false};
+    mp.user = u;
+
+    size_t from = 0;
+    int const ret = template_execute(&mp, dptr, (size_t)dlen, &from);
+    SV* const obj = template_data(&mp);
+    sv_2mortal(obj);
+
+    if(ret < 0) {
+        Perl_croak(aTHX_ "Data::MessagePack->unpack: parse error");
+    } else if(ret == 0) {
+        Perl_croak(aTHX_ "Data::MessagePack->unpack: insufficient bytes");
+    } else {
+        if(from < dlen) {
+            Perl_croak(aTHX_ "Data::MessagePack->unpack: extra bytes");
+        }
+    }
+
+    ST(0) = obj;
     XSRETURN(1);
 }
 
 /* ------------------------------ stream -- */
 /* http://twitter.com/frsyuki/status/13249304748 */
 
-STATIC_INLINE void _reset(SV* self) {
+STATIC_INLINE void _reset(SV* const self) {
     dTHX;
-	unpack_user u = {false, false};
+	unpack_user const u = {false, false};
 
 	UNPACKER(self, mp);
 	template_init(mp);
@@ -275,10 +256,10 @@ XS(xs_unpacker_new) {
         Perl_croak(aTHX_ "Usage: Data::MessagePack::Unpacker->new()");
     }
 
-    SV* self = sv_newmortal();
-	msgpack_unpack_t *mp;
+    SV* const self = sv_newmortal();
+    msgpack_unpack_t *mp;
 
-    Newx(mp, 1, msgpack_unpack_t);
+    Newxz(mp, 1, msgpack_unpack_t);
 
     sv_setref_pv(self, "Data::MessagePack::Unpacker", mp);
     _reset(self);
@@ -287,65 +268,64 @@ XS(xs_unpacker_new) {
     XSRETURN(1);
 }
 
-STATIC_INLINE SV* _execute_impl(SV* self, SV* data, UV off, size_t limit) {
+STATIC_INLINE SV*
+_execute_impl(SV* const self, SV* const data, UV const offset, UV const limit) {
     dTHX;
+
+    if(offset >= limit) {
+        Perl_croak(aTHX_ "offset (%"UVuf") is bigger than data buffer size (%"UVuf")",
+            offset, limit);
+    }
+
     UNPACKER(self, mp);
 
-    size_t from = off;
-	const char* dptr = SvPV_nolen_const(data);
-	int ret;
+    size_t from = offset;
+    const char* const dptr = SvPV_nolen_const(data);
 
-	if(from >= limit) {
-        Perl_croak(aTHX_ "offset (%lu) is bigger than data buffer size (%lu)", (unsigned long)off, (unsigned long)limit);
-	}
+    int const ret = template_execute(mp, dptr, limit, &from);
 
-	ret = template_execute(mp, dptr, limit, &from);
-
-	if(ret < 0) {
-		Perl_croak(aTHX_ "parse error.");
-	} else {
-		mp->user.finished = (ret > 0) ? true : false;
-		return sv_2mortal(newSVuv(from));
-	}
+    if(ret < 0) {
+        Perl_croak(aTHX_ "Data::MessagePack::Unpacker: parse error while executing");
+    } else {
+        mp->user.finished = (ret > 0) ? true : false;
+        return sv_2mortal(newSVuv(from));
+    }
 }
 
 XS(xs_unpacker_execute) {
     dXSARGS;
-    if (items != 3) {
-        Perl_croak(aTHX_ "Usage: $unpacker->execute(data, off)");
+    SV* const self = ST(0);
+    SV* const data = ST(1);
+    UV offset;
+
+    if (items == 2) {
+        offset = 0;
+    }
+    else if (items == 3) {
+        offset = SvUVx(ST(2));
+    }
+    else {
+        Perl_croak(aTHX_ "Usage: $unpacker->execute(data, offset = 0)");
     }
 
-	UNPACKER(ST(0), mp);
-    {
-        SV* self = ST(0);
-        SV* data = ST(1);
-        IV  off  = SvIV(ST(2)); /* offset of $data. normaly, 0. */
-
-        ST(0) = _execute_impl(self, data, off, (size_t)sv_len(data));
-
-        if (!mp->user.incremented) {
-            SV* tmp_obj = template_data(mp);
-            SvREFCNT_inc_simple_void_NN(tmp_obj);
-            mp->user.incremented = true;
-        }
-    }
+    UNPACKER(self, mp);
 
+    ST(0) = _execute_impl(self, data, offset, sv_len(data));
     XSRETURN(1);
 }
 
 XS(xs_unpacker_execute_limit) {
     dXSARGS;
     if (items != 4) {
-        Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, off, limit)");
+        Perl_croak(aTHX_ "Usage: $unpacker->execute_limit(data, offset, limit)");
     }
 
-    SV* self = ST(0);
-    SV* data = ST(1);
-    IV off   = SvIV(ST(2));
-    IV limit = SvIV(ST(3));
-
-    ST(0) = _execute_impl(self, data, off, (size_t)limit);
+    SV* const self   = ST(0);
+    SV* const data   = ST(1);
+    UV  const offset = SvUVx(ST(2));
+    UV  const limit  = SvUVx(ST(3));
 
+    ST(0) = _execute_impl(self, data, offset, limit);
     XSRETURN(1);
 }
 
@@ -355,9 +335,8 @@ XS(xs_unpacker_is_finished) {
         Perl_croak(aTHX_ "Usage: $unpacker->is_finished()");
     }
 
-	UNPACKER(ST(0), mp);
+    UNPACKER(ST(0), mp);
     ST(0) = boolSV(mp->user.finished);
-
     XSRETURN(1);
 }
 
@@ -367,9 +346,8 @@ XS(xs_unpacker_data) {
         Perl_croak(aTHX_ "Usage: $unpacker->data()");
     }
 
-	UNPACKER(ST(0), mp);
-	ST(0) = template_data(mp);
-
+    UNPACKER(ST(0), mp);
+    ST(0) = template_data(mp);
     XSRETURN(1);
 }
 
@@ -379,10 +357,10 @@ XS(xs_unpacker_reset) {
         Perl_croak(aTHX_ "Usage: $unpacker->reset()");
     }
 
-	UNPACKER(ST(0), mp);
+    UNPACKER(ST(0), mp);
 
     SV* const data = template_data(mp);
-    SvREFCNT_dec(data);
+    sv_2mortal(data);
     _reset(ST(0));
 
     XSRETURN(0);
@@ -394,10 +372,10 @@ XS(xs_unpacker_destroy) {
         Perl_croak(aTHX_ "Usage: $unpacker->DESTROY()");
     }
 
-	UNPACKER(ST(0), mp);
+    UNPACKER(ST(0), mp);
 
     SV* const data = template_data(mp);
-    SvREFCNT_dec(data);
+    sv_2mortal(data);
     Safefree(mp);
 
     XSRETURN(0);
diff --git a/perl/xt/leaks/leaktrace.t b/perl/xt/leaks/leaktrace.t
index ff64f6f..1836ad6 100755
--- a/perl/xt/leaks/leaktrace.t
+++ b/perl/xt/leaks/leaktrace.t
@@ -5,7 +5,8 @@ use Test::More;
 
 use Data::MessagePack;
 
-my $data = {
+my $simple_data  = "xyz";
+my $complex_data = {
     a => 'foo',
     b => 42,
     c => undef,
@@ -13,8 +14,10 @@ my $data = {
     e => 3.14,
 };
 
+note 'pack';
+
 no_leaks_ok {
-    my $s = Data::MessagePack->pack($data);
+    my $s = Data::MessagePack->pack($complex_data);
 };
 
 no_leaks_ok {
@@ -23,16 +26,23 @@ no_leaks_ok {
     $@ or die "it must die";
 };
 
-my $s = Data::MessagePack->pack($data);
+note 'unpack';
+
+my $s = Data::MessagePack->pack($simple_data);
+my $c = Data::MessagePack->pack($complex_data);
 
 no_leaks_ok {
     my $data = Data::MessagePack->unpack($s);
 };
 
 no_leaks_ok {
-    my $ss = $s;
-    chop $ss;
-    eval { Data::MessagePack->unpack($ss) };
+    my $data = Data::MessagePack->unpack($c);
+};
+
+no_leaks_ok {
+    my $broken = $s;
+    chop $broken;
+    eval { Data::MessagePack->unpack($broken) };
     #note $@;
     $@ or die "it must die";
 };

From 7c1e0ea95d1aed954ee97bc3a72c7779e085f9ec Mon Sep 17 00:00:00 2001
From: gfx 
Date: Thu, 16 Sep 2010 20:27:25 +0900
Subject: [PATCH 0733/1648] Add binmode() for stream unpacking

---
 perl/t/06_stream_unpack2.t | 1 +
 1 file changed, 1 insertion(+)

diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t
index 68a2873..78ca8f7 100644
--- a/perl/t/06_stream_unpack2.t
+++ b/perl/t/06_stream_unpack2.t
@@ -30,6 +30,7 @@ is_deeply(Data::MessagePack->unpack($packed), $input);
 
     note "packed size: ", length($packed);
     open my $stream, '<:bytes :scalar', \$packed;
+    binmode $stream;
     my $buff;
     while( read($stream, $buff, $size) ) {
         note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff;

From bd887b660d9bcbdb6dd3458223331eea8dcdc654 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Thu, 16 Sep 2010 20:31:34 +0900
Subject: [PATCH 0734/1648] Preallocate hv keys

---
 perl/xs-src/unpack.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index a429ecd..1d7de71 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -157,10 +157,11 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL,
     return 0;
 }
 
-STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
 {
     dTHX;
     HV* const h = newHV();
+    hv_ksplit(h, n);
     *o = newRV_noinc((SV*)h);
     return 0;
 }

From e239bfda8add9d7fd42dcd5f2e2bfc01eae9e824 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Thu, 16 Sep 2010 20:36:07 +0900
Subject: [PATCH 0735/1648] Make leaktrace.t as a regular test

---
 .../{xt/leaks/leaktrace.t => t/50_leaktrace.t} | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)
 rename perl/{xt/leaks/leaktrace.t => t/50_leaktrace.t} (72%)

diff --git a/perl/xt/leaks/leaktrace.t b/perl/t/50_leaktrace.t
similarity index 72%
rename from perl/xt/leaks/leaktrace.t
rename to perl/t/50_leaktrace.t
index 1836ad6..2948527 100755
--- a/perl/xt/leaks/leaktrace.t
+++ b/perl/t/50_leaktrace.t
@@ -1,6 +1,6 @@
 #!perl -w
 use strict;
-use Test::Requires 'Test::LeakTrace';
+use Test::Requires { 'Test::LeakTrace' => 0.13 };
 use Test::More;
 
 use Data::MessagePack;
@@ -22,8 +22,8 @@ no_leaks_ok {
 
 no_leaks_ok {
     eval { Data::MessagePack->pack([\*STDIN]) };
-    #note $@;
-    $@ or die "it must die";
+    note $@;
+    $@ or warn "# it must die";
 };
 
 note 'unpack';
@@ -43,8 +43,16 @@ no_leaks_ok {
     my $broken = $s;
     chop $broken;
     eval { Data::MessagePack->unpack($broken) };
-    #note $@;
-    $@ or die "it must die";
+    note $@;
+    $@ or warn "# it must die";
+};
+
+note 'stream';
+
+no_leaks_ok {
+    my $up = Data::MessagePack::Unpacker->new();
+    $up->execute($c);
+    my $data = $up->data();
 };
 
 done_testing;

From 3cffd46008c8a25dcc77818dbc4fbfd675923ea2 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Thu, 16 Sep 2010 20:41:52 +0900
Subject: [PATCH 0736/1648] Fix a comment

---
 perl/xs-src/unpack.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 1d7de71..da985e3 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -179,7 +179,7 @@ STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV
 STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o)
 {
     dTHX;
-    /*  newSVpvn_flags(p, l, SVs_TEMP) returns an undef if l == 0 */
+    /*  newSVpvn(p, l) returns an undef if p == NULL */
     *o = ((l==0) ? newSVpvs("") : newSVpvn(p, l));
     return 0;
 }

From 8eaed95e027bcf66b61611871bc12e0cb110d859 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Thu, 16 Sep 2010 20:44:51 +0900
Subject: [PATCH 0737/1648] Fix an use of execute()

---
 perl/t/06_stream_unpack2.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t
index 78ca8f7..25af21d 100644
--- a/perl/t/06_stream_unpack2.t
+++ b/perl/t/06_stream_unpack2.t
@@ -35,7 +35,7 @@ is_deeply(Data::MessagePack->unpack($packed), $input);
     while( read($stream, $buff, $size) ) {
         note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff;
 
-        $up->execute($buff);
+        $up->execute($buff, 0);
     }
     ok $up->is_finished, 'is_finished';
     my $data = $up->data;

From 562de7926b87a027842e624fe385468145d09ed9 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Thu, 16 Sep 2010 21:37:49 +0900
Subject: [PATCH 0738/1648] More tests; some fails now :(

---
 perl/.gitignore               |  1 +
 perl/Makefile.PL              |  5 +++++
 perl/lib/Data/MessagePack.pm  | 12 ++++++++++++
 perl/t/06_stream_unpack2.t    |  2 +-
 perl/t/09_stddata.t           | 35 +++++++++++++++++++++++++++++++++++
 ruby/test/test_pack_unpack.rb |  4 ++--
 6 files changed, 56 insertions(+), 3 deletions(-)
 create mode 100644 perl/t/09_stddata.t

diff --git a/perl/.gitignore b/perl/.gitignore
index b64dcdf..3e0e73e 100644
--- a/perl/.gitignore
+++ b/perl/.gitignore
@@ -6,6 +6,7 @@ MessagePack.o
 blib/
 inc/
 msgpack/
+t/std/
 pack.o
 pm_to_blib
 unpack.o
diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index e7b8c47..fafc387 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -54,6 +54,11 @@ if ($Module::Install::AUTHOR && -d File::Spec->catfile('..', 'msgpack')) {
     for my $src (<../msgpack/*.h>) {
         File::Copy::copy($src, 'msgpack/') or die "copy failed: $!";
     }
+
+    mkdir 't/std';
+    for my $data(<../test/*.{json,mpac}>) {
+        File::Copy::copy($data, 't/std') or die "copy failed: $!";
+    }
 }
 
 requires 'Test::More' => 0.94; # done_testing
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 3511628..ece0050 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -6,6 +6,18 @@ use 5.008001;
 our $VERSION = '0.23';
 our $PreferInteger = 0;
 
+{
+    package
+        Data::MessagePack::Boolean;
+    use overload
+        'bool' => sub { ${ $_[0] } },
+        '0+'   => sub { ${ $_[0] } },
+        '""'   => sub { ${ $_[0] } ? 'true' : 'false' },
+
+        fallback => 1,
+    ;
+}
+
 our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };
 our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" };
 sub true  () { $true  }
diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t
index 25af21d..78ca8f7 100644
--- a/perl/t/06_stream_unpack2.t
+++ b/perl/t/06_stream_unpack2.t
@@ -35,7 +35,7 @@ is_deeply(Data::MessagePack->unpack($packed), $input);
     while( read($stream, $buff, $size) ) {
         note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff;
 
-        $up->execute($buff, 0);
+        $up->execute($buff);
     }
     ok $up->is_finished, 'is_finished';
     my $data = $up->data;
diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t
new file mode 100644
index 0000000..d035b4b
--- /dev/null
+++ b/perl/t/09_stddata.t
@@ -0,0 +1,35 @@
+use strict;
+use Test::More;
+use Test::Requires qw(JSON);
+use t::Util;
+
+use Data::MessagePack;
+
+sub slurp {
+    open my $fh, '<:raw', $_[0] or die "failed to open '$_[0]': $!";
+    local $/;
+    return scalar <$fh>;
+}
+
+my @data = @{ JSON::decode_json(slurp("t/std/cases.json")) };
+
+my $mpac1  = slurp("t/std/cases.mpac");
+my $mpac2  = slurp("t/std/cases_compact.mpac");
+
+my $mps = Data::MessagePack::Unpacker->new();
+
+my $t = 1;
+for my $mpac($mpac1, $mpac2) {
+    note "mpac", $t++;
+
+    my $offset = 0;
+    my $i = 0;
+    while($offset < length($mpac)) {
+        $offset += $mps->execute($mpac, $offset);
+        is_deeply $mps->data, $data[$i], "data[$i]";
+        $mps->reset;
+        $i++;
+    }
+}
+
+done_testing;
diff --git a/ruby/test/test_pack_unpack.rb b/ruby/test/test_pack_unpack.rb
index 545e593..f378c3c 100644
--- a/ruby/test/test_pack_unpack.rb
+++ b/ruby/test/test_pack_unpack.rb
@@ -239,7 +239,7 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase
 	end
 
 	it "gc mark" do
-		obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"]
+		obj = [1024, {["a","b"]=>["c","d"]}, ["e","f"], "d", 70000, 4.12, 1.5, 1.5, 1.5]
 		num = 4
 		raw = obj.to_msgpack * num
 		pac = MessagePack::Unpacker.new
@@ -257,7 +257,7 @@ class MessagePackTestPackUnpack < Test::Unit::TestCase
 	end
 
 	it "streaming backward compatibility" do
-		obj = [{["a","b"]=>["c","d"]}, ["e","f"], "d"]
+		obj = [1024, {["a","b"]=>["c","d"]}, ["e","f"], "d", 70000, 4.12, 1.5, 1.5, 1.5]
 		num = 4
 		raw = obj.to_msgpack * num
 		pac = MessagePack::Unpacker.new

From 599964ea5f1a064163ddda8940bb8c5b12ef022a Mon Sep 17 00:00:00 2001
From: gfx 
Date: Thu, 16 Sep 2010 21:45:06 +0900
Subject: [PATCH 0739/1648] Comments

---
 perl/t/09_stddata.t | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t
index d035b4b..b6a612e 100644
--- a/perl/t/09_stddata.t
+++ b/perl/t/09_stddata.t
@@ -1,3 +1,6 @@
+#!perl -w
+# Testing standard dataset in msgpack/test/*.{json,mpac}.
+# Don't edit msgpack/perl/t/std/*, which are just copies.
 use strict;
 use Test::More;
 use Test::Requires qw(JSON);

From 5e602fb575b0941194be42504475208ca7d8d6ad Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 13:10:54 +0900
Subject: [PATCH 0740/1648] Fix tests

---
 perl/t/06_stream_unpack2.t | 44 ++++++++++++++++++--------------------
 perl/t/09_stddata.t        |  2 +-
 perl/t/10_splitted_bytes.t | 40 ++++++++++++++++++++++++++++++++++
 perl/t/Util.pm             |  2 ++
 4 files changed, 64 insertions(+), 24 deletions(-)
 create mode 100755 perl/t/10_splitted_bytes.t

diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t
index 78ca8f7..bb6fe93 100644
--- a/perl/t/06_stream_unpack2.t
+++ b/perl/t/06_stream_unpack2.t
@@ -2,8 +2,16 @@ use strict;
 use warnings;
 use Data::MessagePack;
 use Test::More tests => 9;
+use t::Util;
+
+my $input = [
+    false,true,null,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,
+    127,127,255,65535,4294967295,-32,-32,-128,-32768,
+    -2147483648,0.0,-0.0,1.0,-1.0,"a","a","a","","","",
+    [0],[0],[0],[],[],[],{},{},{},
+    {"a" => 97},{"a" => 97},{"a" => 97},[[]],[["a"]]
+];
 
-my $input = [ 42, "foo", { x => [ (1) x 16 ] }, undef, 1 ];
 my $packed = Data::MessagePack->pack($input);
 is_deeply(Data::MessagePack->unpack($packed), $input);
 
@@ -16,30 +24,20 @@ is_deeply(Data::MessagePack->unpack($packed), $input);
 
 {
     my $up = Data::MessagePack::Unpacker->new();
-    is $up->execute(substr($packed, 0, 3), 0), 3;
-    ok !$up->is_finished;
-    $up->execute($packed, 3);
-    ok $up->is_finished;
-    is_deeply $up->data, $input;
-}
+    $packed x= 3;
 
+    my $offset = 0;
+    for my $i(1 .. 3) {
+        note "block $i (offset: $offset/".length($packed).")";
+        note "starting 3 bytes: ", join " ", map { unpack 'H2', $_ }
+            split //, substr($packed, $offset, 3);
 
-{
-    my $up = Data::MessagePack::Unpacker->new();
-    my $size = 8;
-
-    note "packed size: ", length($packed);
-    open my $stream, '<:bytes :scalar', \$packed;
-    binmode $stream;
-    my $buff;
-    while( read($stream, $buff, $size) ) {
-        note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff;
-
-        $up->execute($buff);
+        $offset = $up->execute($packed, $offset);
+        ok $up->is_finished, 'finished';
+        my $data = $up->data;
+        is_deeply $data, $input;
+        $up->reset();
     }
-    ok $up->is_finished, 'is_finished';
-    my $data = $up->data;
-    note explain($data);
-    is_deeply $data, $input;
 }
 
+
diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t
index b6a612e..976fc5d 100644
--- a/perl/t/09_stddata.t
+++ b/perl/t/09_stddata.t
@@ -28,7 +28,7 @@ for my $mpac($mpac1, $mpac2) {
     my $offset = 0;
     my $i = 0;
     while($offset < length($mpac)) {
-        $offset += $mps->execute($mpac, $offset);
+        $offset = $mps->execute($mpac, $offset);
         is_deeply $mps->data, $data[$i], "data[$i]";
         $mps->reset;
         $i++;
diff --git a/perl/t/10_splitted_bytes.t b/perl/t/10_splitted_bytes.t
new file mode 100755
index 0000000..232d870
--- /dev/null
+++ b/perl/t/10_splitted_bytes.t
@@ -0,0 +1,40 @@
+#!perl
+
+# This feature is not yet supported, but 0.23 (or former) caused SEGV in this code,
+# so we put it here.
+
+use strict;
+use warnings;
+use Data::MessagePack;
+use Test::More;
+use t::Util;
+
+my $input = [
+    false,true,null,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,
+    127,127,255,65535,4294967295,-32,-32,-128,-32768,
+    -2147483648,0.0,-0.0,1.0,-1.0,"a","a","a","","","",
+    [0],[0],[0],[],[],[],{},{},{},
+    {"a" => 97},{"a" => 97},{"a" => 97},[[]],[["a"]]
+];
+
+my $packed = Data::MessagePack->pack($input);
+
+foreach my $size(1 .. 16) {
+    local $TODO = "Splitted byte streaming is not yet supported (bufer size: $size)";
+
+    my $up = Data::MessagePack::Unpacker->new();
+
+    open my $stream, '<:bytes :scalar', \$packed;
+    binmode $stream;
+    my $buff;
+    while( read($stream, $buff, $size) ) {
+        #note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff;
+
+        $up->execute($buff);
+    }
+    ok $up->is_finished, 'is_finished';
+    my $data = $up->data;
+    is_deeply $data, $input;
+}
+
+done_testing;
diff --git a/perl/t/Util.pm b/perl/t/Util.pm
index c8debef..ad69c4d 100644
--- a/perl/t/Util.pm
+++ b/perl/t/Util.pm
@@ -1,6 +1,7 @@
 package t::Util;
 use strict;
 use warnings;
+use Data::MessagePack;
 
 sub import {
     my $pkg = caller(0);
@@ -15,6 +16,7 @@ sub import {
     *{"$pkg\::false"} = sub () {
         Data::MessagePack::false()
     };
+    *{"$pkg\::null"} = sub() { undef };
 }
 
 1;

From d2962d8676060c8fdd5d474db7497910293d4f8a Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 13:25:23 +0900
Subject: [PATCH 0741/1648] Split the boolean class into an outer module

---
 perl/lib/Data/MessagePack.pm         | 27 +++++++------
 perl/lib/Data/MessagePack/Boolean.pm | 14 +++++++
 perl/xs-src/MessagePack.c            | 10 ++++-
 perl/xs-src/pack.c                   |  3 +-
 perl/xs-src/unpack.c                 | 58 ++++++++++++++++++++++++----
 5 files changed, 87 insertions(+), 25 deletions(-)
 create mode 100755 perl/lib/Data/MessagePack/Boolean.pm

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index ece0050..b1e0174 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -6,22 +6,21 @@ use 5.008001;
 our $VERSION = '0.23';
 our $PreferInteger = 0;
 
-{
-    package
-        Data::MessagePack::Boolean;
-    use overload
-        'bool' => sub { ${ $_[0] } },
-        '0+'   => sub { ${ $_[0] } },
-        '""'   => sub { ${ $_[0] } ? 'true' : 'false' },
-
-        fallback => 1,
-    ;
+sub true () {
+    require Data::MessagePack::Boolean;
+    no warnings 'once', 'redefine';
+    my $t = $Data::MessagePack::Boolean::true;
+    *true = sub (){ $t };
+    return $t;
 }
 
-our $true  = do { bless \(my $dummy = 1), "Data::MessagePack::Boolean" };
-our $false = do { bless \(my $dummy = 0), "Data::MessagePack::Boolean" };
-sub true  () { $true  }
-sub false () { $false }
+sub false () {
+    require Data::MessagePack::Boolean;
+    no warnings 'once', 'redefine';
+    my $f = $Data::MessagePack::Boolean::false;
+    *false = sub (){ $f };
+    return $f;
+}
 
 if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate
     my $backend = $ENV{ PERL_DATA_MESSAGEPACK } || '';
diff --git a/perl/lib/Data/MessagePack/Boolean.pm b/perl/lib/Data/MessagePack/Boolean.pm
new file mode 100755
index 0000000..2bb3eca
--- /dev/null
+++ b/perl/lib/Data/MessagePack/Boolean.pm
@@ -0,0 +1,14 @@
+package Data::MessagePack::Boolean;
+use strict;
+use overload
+    'bool' => sub { ${ $_[0] } },
+    '0+'   => sub { ${ $_[0] } },
+    '""'   => sub { ${ $_[0] } ? 'true' : 'false' },
+
+    fallback => 1,
+;
+
+our $true  = do { bless \(my $dummy = 1) };
+our $false = do { bless \(my $dummy = 0) };
+
+1;
diff --git a/perl/xs-src/MessagePack.c b/perl/xs-src/MessagePack.c
index aba8ef9..69337f4 100644
--- a/perl/xs-src/MessagePack.c
+++ b/perl/xs-src/MessagePack.c
@@ -1,5 +1,9 @@
 #include "xshelper.h"
 
+#ifndef __cplusplus
+#include 
+#endif
+
 XS(xs_pack);
 XS(xs_unpack);
 XS(xs_unpacker_new);
@@ -10,13 +14,15 @@ XS(xs_unpacker_data);
 XS(xs_unpacker_reset);
 XS(xs_unpacker_destroy);
 
-void boot_Data__MessagePack_pack(void);
+void init_Data__MessagePack_pack(pTHX_ bool const cloning);
+void init_Data__MessagePack_unpack(pTHX_ bool const cloning);
 
 XS(boot_Data__MessagePack) {
     dXSARGS;
     PERL_UNUSED_VAR(items);
 
-    boot_Data__MessagePack_pack();
+    init_Data__MessagePack_pack(aTHX_   false);
+    init_Data__MessagePack_unpack(aTHX_ false);
 
     newXS("Data::MessagePack::pack", xs_pack, __FILE__);
     newXS("Data::MessagePack::unpack", xs_unpack, __FILE__);
diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 9a58ed0..862808e 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -83,8 +83,7 @@ MGVTBL pref_int_vtbl = {
 #endif
 };
 
-void boot_Data__MessagePack_pack(void) {
-    dTHX;
+void init_Data__MessagePack_pack(pTHX_ bool const cloning) {
     SV* var = get_sv("Data::MessagePack::PreferInteger", 0);
     sv_magicext(var, NULL, PERL_MAGIC_ext, &pref_int_vtbl, NULL, 0);
     SvSETMAGIC(var);
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index da985e3..6ebb48c 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -2,6 +2,13 @@
 #define NEED_sv_2pv_flags
 #include "xshelper.h"
 
+#define MY_CXT_KEY "Data::MessagePack::_unpack_guts" XS_VERSION
+typedef struct {
+    SV* msgpack_true;
+    SV* msgpack_false;
+} my_cxt_t;
+START_MY_CXT
+
 typedef struct {
     bool finished;
     bool incremented;
@@ -22,13 +29,52 @@ typedef struct {
 
 #define msgpack_unpack_user unpack_user
 
+void init_Data__MessagePack_unpack(pTHX_ bool const cloning) {
+    if(!cloning) {
+        MY_CXT_INIT;
+        MY_CXT.msgpack_true  = NULL;
+        MY_CXT.msgpack_false = NULL;
+    }
+    else {
+        MY_CXT_CLONE;
+        MY_CXT.msgpack_true  = NULL;
+        MY_CXT.msgpack_false = NULL;
+    }
+}
+
+
+
 /* ---------------------------------------------------------------------- */
 /* utility functions                                                      */
 
-STATIC_INLINE SV *
-get_bool (const char* const name) {
+static SV*
+load_bool(pTHX_ const char* const name) {
+    CV* const cv = get_cv(name, GV_ADD);
+    dSP;
+    PUSHMARK(SP);
+    call_sv((SV*)cv, G_SCALAR);
+    SPAGAIN;
+    SV* const sv = newSVsv(POPs);
+    PUTBACK;
+    return sv;
+}
+
+static SV*
+get_bool(bool const value) {
     dTHX;
-    return newSVsv(get_sv( name, GV_ADD ));
+    dMY_CXT;
+    if(value) {
+        if(!MY_CXT.msgpack_true) {
+            MY_CXT.msgpack_true = load_bool(aTHX_ "Data::MessagePack::true");
+        }
+        return newSVsv(MY_CXT.msgpack_true);
+    }
+    else {
+        if(!MY_CXT.msgpack_false) {
+            MY_CXT.msgpack_false = load_bool(aTHX_ "Data::MessagePack::false");
+        }
+        return newSVsv(MY_CXT.msgpack_false);
+    }
 }
 
 /* ---------------------------------------------------------------------- */
@@ -127,15 +173,13 @@ STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
 
 STATIC_INLINE int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
-    dTHX;
-    *o = get_bool("Data::MessagePack::true");
+    *o = get_bool(true);
     return 0;
 }
 
 STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
-    dTHX;
-    *o = get_bool("Data::MessagePack::false");
+    *o = get_bool(false);
     return 0;
 }
 

From eab7c877813e7c1dd7820307260221444773d116 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 13:37:17 +0900
Subject: [PATCH 0742/1648] Tidy PP

---
 perl/lib/Data/MessagePack/PP.pm | 119 ++++++++++++++++++--------------
 1 file changed, 68 insertions(+), 51 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 9e32299..c7eaadf 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -1,11 +1,8 @@
 package Data::MessagePack::PP;
-
-use 5.008000;
+use 5.008001;
 use strict;
 use Carp ();
 
-our $VERSION = '0.15';
-
 # See also
 # http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec
 # http://cpansearch.perl.org/src/YAPPO/Data-Model-0.00006/lib/Data/Model/Driver/Memcached.pm
@@ -25,49 +22,74 @@ BEGIN {
         # require $Config{byteorder}; my $bo_is_le = ( $Config{byteorder} =~ /^1234/ );
         # which better?
         my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE
-        # In really, since 5.9.2 '>' is introduced. but 'n!' and 'N!'?
-        *pack_uint64 = $bo_is_le ? sub {
-            my @v = unpack( 'V2', pack( 'Q', $_[0] ) );
-            return pack 'CN2', 0xcf, @v[1,0];
-        } : sub { pack 'CQ', 0xcf, $_[0]; };
-        *pack_int64 = $bo_is_le ? sub {
-            my @v = unpack( 'V2', pack( 'q', $_[0] ) );
-            return pack 'CN2', 0xd3, @v[1,0];
-        } : sub { pack 'Cq', 0xd3, $_[0]; };
-        *pack_double = $bo_is_le ? sub {
-            my @v = unpack( 'V2', pack( 'd', $_[0] ) );
-            return pack 'CN2', 0xcb, @v[1,0];
-        } : sub { pack 'Cd', 0xcb, $_[0]; };
-        *unpack_float = $bo_is_le ? sub {
-            my @v = unpack( 'v2', substr( $_[0], $_[1], 4 ) );
-            return unpack( 'f', pack( 'n2', @v[1,0] ) );
-        } : sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); };
-        *unpack_double = $bo_is_le ? sub {
-            my @v = unpack( 'V2', substr( $_[0], $_[1], 8 ) );
-            return unpack( 'd', pack( 'N2', @v[1,0] ) );
-        } : sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); };
-        *unpack_int16  = sub {
-            my $v = unpack 'n', substr( $_[0], $_[1], 2 );
-            return $v ? $v - 0x10000 : 0;
-        };
-        *unpack_int32  = sub {
-            no warnings; # avoid for warning about Hexadecimal number
-            my $v = unpack 'N', substr( $_[0], $_[1], 4 );
-            return $v ? $v - 0x100000000 : 0;
-        };
-        *unpack_int64 = $bo_is_le ? sub {
-            my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) );
-            return unpack( 'q', pack( 'N2', @v[1,0] ) );
-        } : sub { pack 'q', substr( $_[0], $_[1], 8 ); };
-        *unpack_uint64 = $bo_is_le ? sub {
-            my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) );
-            return unpack( 'Q', pack( 'N2', @v[1,0] ) );
-        } : sub { pack 'Q', substr( $_[0], $_[1], 8 ); };
+
+        # In reality, since 5.9.2 '>' is introduced. but 'n!' and 'N!'?
+        if($bo_is_le) {
+            *pack_uint64 = sub {
+                my @v = unpack( 'V2', pack( 'Q', $_[0] ) );
+                return pack 'CN2', 0xcf, @v[1,0];
+            };
+            *pack_int64 = sub {
+                my @v = unpack( 'V2', pack( 'q', $_[0] ) );
+                return pack 'CN2', 0xd3, @v[1,0];
+            };
+            *pack_double = sub {
+                my @v = unpack( 'V2', pack( 'd', $_[0] ) );
+                return pack 'CN2', 0xcb, @v[1,0];
+            };
+
+            *unpack_float = sub {
+                my @v = unpack( 'v2', substr( $_[0], $_[1], 4 ) );
+                return unpack( 'f', pack( 'n2', @v[1,0] ) );
+            };
+            *unpack_double = sub {
+                my @v = unpack( 'V2', substr( $_[0], $_[1], 8 ) );
+                return unpack( 'd', pack( 'N2', @v[1,0] ) );
+            };
+
+            *unpack_int16  = sub {
+                my $v = unpack 'n', substr( $_[0], $_[1], 2 );
+                return $v ? $v - 0x10000 : 0;
+            };
+            *unpack_int32  = sub {
+                no warnings; # avoid for warning about Hexadecimal number
+                my $v = unpack 'N', substr( $_[0], $_[1], 4 );
+                return $v ? $v - 0x100000000 : 0;
+            };
+            *unpack_int64 = sub {
+                my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) );
+                return unpack( 'q', pack( 'N2', @v[1,0] ) );
+            };
+            *unpack_uint64 = sub {
+                my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) );
+                return unpack( 'Q', pack( 'N2', @v[1,0] ) );
+            };
+        }
+        else { # big endian
+            *pack_uint64   = sub { return pack 'CQ', 0xcf, $_[0]; };
+            *pack_int64    = sub { return pack 'Cq', 0xd3, $_[0]; };
+            *pack_double   = sub { return pack 'Cd', 0xcb, $_[0]; };
+
+            *unpack_float  = sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); };
+            *unpack_double = sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); };
+            *unpack_int16  = sub {
+                my $v = unpack 'n', substr( $_[0], $_[1], 2 );
+                return $v ? $v - 0x10000 : 0;
+            };
+            *unpack_int32  = sub {
+                no warnings; # avoid for warning about Hexadecimal number
+                my $v = unpack 'N', substr( $_[0], $_[1], 4 );
+                return $v ? $v - 0x100000000 : 0;
+            };
+            *unpack_int64  = sub { pack 'q', substr( $_[0], $_[1], 8 ); };
+            *unpack_uint64 = sub { pack 'Q', substr( $_[0], $_[1], 8 ); };
+        }
     }
     else {
         *pack_uint64   = sub { return pack 'CQ>', 0xcf, $_[0]; };
         *pack_int64    = sub { return pack 'Cq>', 0xd3, $_[0]; };
         *pack_double   = sub { return pack 'Cd>', 0xcb, $_[0]; };
+
         *unpack_float  = sub { return unpack( 'f>', substr( $_[0], $_[1], 4 ) ); };
         *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); };
         *unpack_int16  = sub { return unpack( 'n!', substr( $_[0], $_[1], 2 ) ); };
@@ -75,11 +97,6 @@ BEGIN {
         *unpack_int64  = sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); };
         *unpack_uint64 = sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); };
     }
-    # for 5.8 etc.
-    unless ( defined &utf8::is_utf8 ) {
-       require Encode;
-       *utf8::is_utf8 = *Encode::is_utf8;
-    }
 }
 
 
@@ -92,7 +109,7 @@ BEGIN {
 
     my $max_depth;
 
-sub pack {
+sub pack :method {
     Carp::croak('Usage: Data::MessagePack->pack($dat [,$max_depth])') if @_ < 2;
     $max_depth = defined $_[2] ? $_[2] : 512; # init
     return _pack( $_[1] );
@@ -209,7 +226,7 @@ sub _pack {
 
     my $p; # position variables for speed.
 
-sub unpack {
+sub unpack :method {
     $p = 0; # init
     _unpack( $_[1] );
 }
@@ -370,7 +387,7 @@ sub execute_limit {
 
 sub execute {
     my ( $self, $data, $offset, $limit ) = @_;
-    my $value = substr( $data, $offset, $limit ? $limit : length $data );
+    my $value = substr( $data, $offset || 0, $limit ? $limit : length $data );
     my $len   = length $value;
 
     $p = 0;
@@ -509,7 +526,7 @@ sub is_finished {
 }
 
 
-sub reset {
+sub reset :method {
     $_[0]->{ stack }  = [];
     $_[0]->{ data }   = undef;
     $_[0]->{ remain } = undef;

From d5a17a3c25068573b20a15ccaa22f6b8926e8d0b Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 13:43:42 +0900
Subject: [PATCH 0743/1648] Fix stddata.t

---
 perl/t/09_stddata.t | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t
index 976fc5d..a618787 100644
--- a/perl/t/09_stddata.t
+++ b/perl/t/09_stddata.t
@@ -3,7 +3,6 @@
 # Don't edit msgpack/perl/t/std/*, which are just copies.
 use strict;
 use Test::More;
-use Test::Requires qw(JSON);
 use t::Util;
 
 use Data::MessagePack;
@@ -14,7 +13,11 @@ sub slurp {
     return scalar <$fh>;
 }
 
-my @data = @{ JSON::decode_json(slurp("t/std/cases.json")) };
+my @data = do {
+    my $json = slurp("t/std/cases.json");
+    $json =~ s/:/=>/g;
+    @{ eval $json };
+};
 
 my $mpac1  = slurp("t/std/cases.mpac");
 my $mpac2  = slurp("t/std/cases_compact.mpac");

From 80058083b86919c02165e91719d7eca662b81db5 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 13:49:08 +0900
Subject: [PATCH 0744/1648] Tweaks

---
 perl/lib/Data/MessagePack/PP.pm | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index c7eaadf..6a06c3c 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -121,9 +121,7 @@ sub _pack {
 
     return CORE::pack( 'C', 0xc0 ) if ( not defined $value );
 
-    my $b_obj = B::svref_2object( ref $value ? $value : \$value );
-
-    if ( $b_obj->isa('B::AV') ) {
+    if ( ref($value) eq 'ARRAY' ) {
         my $num = @$value;
         my $header = 
               $num < 16          ? CORE::pack( 'C',  0x90 + $num )
@@ -137,7 +135,7 @@ sub _pack {
         return join( '', $header, map { _pack( $_ ) } @$value );
     }
 
-    elsif ( $b_obj->isa('B::HV') ) {
+    elsif ( ref($value) eq 'HASH' ) {
         my $num = keys %$value;
         my $header = 
               $num < 16          ? CORE::pack( 'C',  0x80 + $num )
@@ -151,10 +149,12 @@ sub _pack {
         return join( '', $header, map { _pack( $_ ) } %$value );
     }
 
-    elsif ( blessed( $value ) and blessed( $value ) eq 'Data::MessagePack::Boolean' ) {
-        return  CORE::pack( 'C', $$value ? 0xc3 : 0xc2 );
+    elsif ( ref( $value ) eq 'Data::MessagePack::Boolean' ) {
+        return  CORE::pack( 'C', ${$value} ? 0xc3 : 0xc2 );
     }
 
+
+    my $b_obj = B::svref_2object( \$value );
     my $flags = $b_obj->FLAGS;
 
     if ( $flags & ( B::SVf_IOK | B::SVp_IOK ) ) {
@@ -175,7 +175,6 @@ sub _pack {
         }
 
     }
-
     elsif ( $flags & B::SVf_POK ) { # raw / check needs before dboule
 
         if ( $Data::MessagePack::PreferInteger ) {
@@ -204,11 +203,9 @@ sub _pack {
         return $header . $value;
 
     }
-
     elsif ( $flags & ( B::SVf_NOK | B::SVp_NOK ) ) { # double only
         return pack_double( $value );
     }
-
     else {
         die "???";
     }

From 8512f9eda168b37b8d362484a13480ef3d06f24b Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 13:49:55 +0900
Subject: [PATCH 0745/1648] Add .gitignore

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 .gitignore

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..d740b18
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+*.o
+*.so
+ruby/Makefile

From b71cc5d7ee99259950602bdcddce68a0fb1dc52b Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 13:59:52 +0900
Subject: [PATCH 0746/1648] chmod -x

---
 perl/lib/Data/MessagePack/Boolean.pm | 0
 perl/t/10_splitted_bytes.t           | 0
 perl/t/50_leaktrace.t                | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 perl/lib/Data/MessagePack/Boolean.pm
 mode change 100755 => 100644 perl/t/10_splitted_bytes.t
 mode change 100755 => 100644 perl/t/50_leaktrace.t

diff --git a/perl/lib/Data/MessagePack/Boolean.pm b/perl/lib/Data/MessagePack/Boolean.pm
old mode 100755
new mode 100644
diff --git a/perl/t/10_splitted_bytes.t b/perl/t/10_splitted_bytes.t
old mode 100755
new mode 100644
diff --git a/perl/t/50_leaktrace.t b/perl/t/50_leaktrace.t
old mode 100755
new mode 100644

From a10eb2a0d760f0a2f3aaa16d539d621c0ec41de4 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 14:02:12 +0900
Subject: [PATCH 0747/1648] Changelogging

---
 perl/Changes | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/perl/Changes b/perl/Changes
index 4120376..ce52581 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+
+0.24
+    - Fixed a possible SEGV on streaming unpacking (gfx)
+    - Improve performance, esp. in unpacking (gfx)
+
 0.23
 
     (NO FEATURE CHANGES)

From 130d2064d583065c185e7d6d868c6c51b3731fa2 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Fri, 17 Sep 2010 15:28:24 +0900
Subject: [PATCH 0748/1648] perl: updated benchmark result! gfx++ # performance
 tuning

---
 perl/lib/Data/MessagePack.pm | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index b1e0174..7456c3a 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -113,23 +113,32 @@ Pack the string as int when the value looks like int(EXPERIMENTAL).
 
 This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP).
 
+
     -- serialize
     JSON::XS: 2.3
-    Data::MessagePack: 0.20
+    Data::MessagePack: 0.24
     Storable: 2.21
-    Benchmark: timing 1000000 iterations of json, mp, storable...
-          json:  5 wallclock secs ( 3.95 usr +  0.00 sys =  3.95 CPU) @ 253164.56/s (n=1000000)
-            mp:  3 wallclock secs ( 2.69 usr +  0.00 sys =  2.69 CPU) @ 371747.21/s (n=1000000)
-      storable: 26 wallclock secs (27.21 usr +  0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000)
-
+    Benchmark: running json, mp, storable for at least 1 CPU seconds...
+          json:  1 wallclock secs ( 1.00 usr +  0.01 sys =  1.01 CPU) @ 141939.60/s (n=143359)
+            mp:  1 wallclock secs ( 1.06 usr +  0.00 sys =  1.06 CPU) @ 355500.94/s (n=376831)
+      storable:  1 wallclock secs ( 1.12 usr +  0.00 sys =  1.12 CPU) @ 38399.11/s (n=43007)
+                 Rate storable     json       mp
+    storable  38399/s       --     -73%     -89%
+    json     141940/s     270%       --     -60%
+    mp       355501/s     826%     150%       --
+    
     -- deserialize
     JSON::XS: 2.3
-    Data::MessagePack: 0.20
+    Data::MessagePack: 0.24
     Storable: 2.21
-    Benchmark: timing 1000000 iterations of json, mp, storable...
-          json:  4 wallclock secs ( 4.45 usr +  0.00 sys =  4.45 CPU) @ 224719.10/s (n=1000000)
-            mp:  6 wallclock secs ( 5.45 usr +  0.00 sys =  5.45 CPU) @ 183486.24/s (n=1000000)
-      storable:  7 wallclock secs ( 7.77 usr +  0.00 sys =  7.77 CPU) @ 128700.13/s (n=1000000)
+    Benchmark: running json, mp, storable for at least 1 CPU seconds...
+          json:  0 wallclock secs ( 1.05 usr +  0.00 sys =  1.05 CPU) @ 179442.86/s (n=188415)
+            mp:  0 wallclock secs ( 1.01 usr +  0.00 sys =  1.01 CPU) @ 212909.90/s (n=215039)
+      storable:  2 wallclock secs ( 1.14 usr +  0.00 sys =  1.14 CPU) @ 114974.56/s (n=131071)
+                 Rate storable     json       mp
+    storable 114975/s       --     -36%     -46%
+    json     179443/s      56%       --     -16%
+    mp       212910/s      85%      19%       --
 
 =head1 AUTHORS
 

From 2c9d90d463429902226eca593fccebec440a57ef Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 18:08:15 +0900
Subject: [PATCH 0749/1648] perl: regen README

---
 perl/README                  | 28 ++++++++++++++++++----------
 perl/lib/Data/MessagePack.pm |  2 +-
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/perl/README b/perl/README
index e46323d..6272a3b 100644
--- a/perl/README
+++ b/perl/README
@@ -56,21 +56,29 @@ SPEED
 
         -- serialize
         JSON::XS: 2.3
-        Data::MessagePack: 0.20
+        Data::MessagePack: 0.24
         Storable: 2.21
-        Benchmark: timing 1000000 iterations of json, mp, storable...
-              json:  5 wallclock secs ( 3.95 usr +  0.00 sys =  3.95 CPU) @ 253164.56/s (n=1000000)
-                mp:  3 wallclock secs ( 2.69 usr +  0.00 sys =  2.69 CPU) @ 371747.21/s (n=1000000)
-          storable: 26 wallclock secs (27.21 usr +  0.00 sys = 27.21 CPU) @ 36751.19/s (n=1000000)
+        Benchmark: running json, mp, storable for at least 1 CPU seconds...
+              json:  1 wallclock secs ( 1.00 usr +  0.01 sys =  1.01 CPU) @ 141939.60/s (n=143359)
+                mp:  1 wallclock secs ( 1.06 usr +  0.00 sys =  1.06 CPU) @ 355500.94/s (n=376831)
+          storable:  1 wallclock secs ( 1.12 usr +  0.00 sys =  1.12 CPU) @ 38399.11/s (n=43007)
+                     Rate storable     json       mp
+        storable  38399/s       --     -73%     -89%
+        json     141940/s     270%       --     -60%
+        mp       355501/s     826%     150%       --
 
         -- deserialize
         JSON::XS: 2.3
-        Data::MessagePack: 0.20
+        Data::MessagePack: 0.24
         Storable: 2.21
-        Benchmark: timing 1000000 iterations of json, mp, storable...
-              json:  4 wallclock secs ( 4.45 usr +  0.00 sys =  4.45 CPU) @ 224719.10/s (n=1000000)
-                mp:  6 wallclock secs ( 5.45 usr +  0.00 sys =  5.45 CPU) @ 183486.24/s (n=1000000)
-          storable:  7 wallclock secs ( 7.77 usr +  0.00 sys =  7.77 CPU) @ 128700.13/s (n=1000000)
+        Benchmark: running json, mp, storable for at least 1 CPU seconds...
+              json:  0 wallclock secs ( 1.05 usr +  0.00 sys =  1.05 CPU) @ 179442.86/s (n=188415)
+                mp:  0 wallclock secs ( 1.01 usr +  0.00 sys =  1.01 CPU) @ 212909.90/s (n=215039)
+          storable:  2 wallclock secs ( 1.14 usr +  0.00 sys =  1.14 CPU) @ 114974.56/s (n=131071)
+                     Rate storable     json       mp
+        storable 114975/s       --     -36%     -46%
+        json     179443/s      56%       --     -16%
+        mp       212910/s      85%      19%       --
 
 AUTHORS
     Tokuhiro Matsuno
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 7456c3a..609ea3c 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -126,7 +126,7 @@ This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my
     storable  38399/s       --     -73%     -89%
     json     141940/s     270%       --     -60%
     mp       355501/s     826%     150%       --
-    
+
     -- deserialize
     JSON::XS: 2.3
     Data::MessagePack: 0.24

From e8d8099563ced184991b188bfe7ed4189e24f334 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 18:08:34 +0900
Subject: [PATCH 0750/1648] Fix a macro redefinition

---
 perl/xs-src/unpack.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 6ebb48c..e89b22c 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -142,7 +142,7 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const
     return 0;
 }
 
-#define template_callback_uint64 template_callback_IV
+#define template_callback_int64 template_callback_IV
 
 #endif /* IVSIZE */
 

From a0c18e4380a6c96688101e508acab43982dddfe1 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 18:16:33 +0900
Subject: [PATCH 0751/1648] Docs

---
 perl/lib/Data/MessagePack.pm | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 609ea3c..5202109 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -45,7 +45,7 @@ Data::MessagePack - MessagePack serialising/deserialising
 
 =head1 SYNOPSIS
 
-    my $packed = Data::MessagePack->pack($dat);
+    my $packed   = Data::MessagePack->pack($dat);
     my $unpacked = Data::MessagePack->unpack($dat);
 
 =head1 DESCRIPTION
@@ -61,11 +61,11 @@ It enables to exchange structured objects between many languages like JSON. But
 
 =over 4
 
-=item PORTABILITY
+=item PORTABLE
 
-Messagepack is language independent binary serialize format.
+The MessagePack format does not depend on language nor byte order.
 
-=item SMALL SIZE
+=item SMALL IN SIZE
 
     say length(JSON::XS::encode_json({a=>1, b=>2}));   # => 13
     say length(Storable::nfreeze({a=>1, b=>2}));       # => 21
@@ -76,6 +76,7 @@ The MessagePack format saves memory than JSON and Storable format.
 =item STREAMING DESERIALIZER
 
 MessagePack supports streaming deserializer. It is useful for networking such as RPC.
+See L for details.
 
 =back
 
@@ -105,13 +106,13 @@ unpack the $msgpackstr to a MessagePack format string.
 
 =item $Data::MessagePack::PreferInteger
 
-Pack the string as int when the value looks like int(EXPERIMENTAL).
+Packs a string as an integer, when it looks like an integer.
 
 =back
 
 =head1 SPEED
 
-This is the result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP).
+This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP).
 
 
     -- serialize
@@ -156,13 +157,15 @@ FURUHASHI Sadayuki
 
 hanekomu
 
+gfx
+
 =head1 LICENSE
 
 This library is free software; you can redistribute it and/or modify
 it under the same terms as Perl itself.
 
-
 =head1 SEE ALSO
 
-L is official web site for MessagePack format.
+L is the official web site for the  MessagePack format.
 
+=cut

From 7c8f8703a19d77e23e07b777054d888eff49774a Mon Sep 17 00:00:00 2001
From: gfx 
Date: Fri, 17 Sep 2010 18:26:16 +0900
Subject: [PATCH 0752/1648] Add TODOs

---
 perl/lib/Data/MessagePack.pm | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 5202109..8c5cfac 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -141,6 +141,25 @@ This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC
     json     179443/s      56%       --     -16%
     mp       212910/s      85%      19%       --
 
+=head1 TODO
+
+=over
+
+=item Error handling
+
+MessagePack cannot deal with complex scalars such as object references,
+filehandles, and code references. We should report the errors more kindly.
+
+=item Streaming deserializer
+
+The current implementation of the streaming deserializer does not have internal
+buffers while some other bindings (such as Ruby binding) does. This limitation
+will astonish those who try to unpack byte streams with an arbitrary buffer size
+(e.g. C<< while(read($socket, $buffer, $arbitrary_buffer_size)) { ... } >>).
+We should implement the internal buffer for the unpacker.
+
+=back
+
 =head1 AUTHORS
 
 Tokuhiro Matsuno

From 845af014dce3c54c4294abefa8eb33b6bb39dd61 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Sat, 18 Sep 2010 06:15:51 +0900
Subject: [PATCH 0753/1648] perl: gfx is a author.

---
 perl/lib/Data/MessagePack.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 8c5cfac..953bdf8 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -166,6 +166,8 @@ Tokuhiro Matsuno
 
 Makamaka Hannyaharamitu
 
+gfx
+
 =head1 THANKS TO
 
 Jun Kuriyama
@@ -176,8 +178,6 @@ FURUHASHI Sadayuki
 
 hanekomu
 
-gfx
-
 =head1 LICENSE
 
 This library is free software; you can redistribute it and/or modify

From 446266776eb1a36fd562307eb6de4c64e6cc36d1 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Sat, 18 Sep 2010 06:16:17 +0900
Subject: [PATCH 0754/1648] perl: regenerate README file

---
 perl/README | 36 ++++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/perl/README b/perl/README
index 6272a3b..3105278 100644
--- a/perl/README
+++ b/perl/README
@@ -2,7 +2,7 @@ NAME
     Data::MessagePack - MessagePack serialising/deserialising
 
 SYNOPSIS
-        my $packed = Data::MessagePack->pack($dat);
+        my $packed   = Data::MessagePack->pack($dat);
         my $unpacked = Data::MessagePack->unpack($dat);
 
 DESCRIPTION
@@ -14,10 +14,10 @@ ABOUT MESSAGEPACK FORMAT
     But unlike JSON, it is very fast and small.
 
   ADVANTAGES
-    PORTABILITY
-        Messagepack is language independent binary serialize format.
+    PORTABLE
+        The MessagePack format does not depend on language nor byte order.
 
-    SMALL SIZE
+    SMALL IN SIZE
             say length(JSON::XS::encode_json({a=>1, b=>2}));   # => 13
             say length(Storable::nfreeze({a=>1, b=>2}));       # => 21
             say length(Data::MessagePack->pack({a=>1, b=>2})); # => 7
@@ -26,7 +26,7 @@ ABOUT MESSAGEPACK FORMAT
 
     STREAMING DESERIALIZER
         MessagePack supports streaming deserializer. It is useful for
-        networking such as RPC.
+        networking such as RPC. See Data::MessagePack::Unpacker for details.
 
     If you want to get more information about the MessagePack format, please
     visit to .
@@ -47,12 +47,11 @@ METHODS
 
 Configuration Variables
     $Data::MessagePack::PreferInteger
-        Pack the string as int when the value looks like int(EXPERIMENTAL).
+        Packs a string as an integer, when it looks like an integer.
 
 SPEED
-    This is the result of benchmark/serialize.pl and
-    benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu
-    SMP).
+    This is a result of benchmark/serialize.pl and benchmark/deserialize.pl
+    on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP).
 
         -- serialize
         JSON::XS: 2.3
@@ -80,11 +79,27 @@ SPEED
         json     179443/s      56%       --     -16%
         mp       212910/s      85%      19%       --
 
+TODO
+    Error handling
+        MessagePack cannot deal with complex scalars such as object
+        references, filehandles, and code references. We should report the
+        errors more kindly.
+
+    Streaming deserializer
+        The current implementation of the streaming deserializer does not
+        have internal buffers while some other bindings (such as Ruby
+        binding) does. This limitation will astonish those who try to unpack
+        byte streams with an arbitrary buffer size (e.g.
+        "while(read($socket, $buffer, $arbitrary_buffer_size)) { ... }"). We
+        should implement the internal buffer for the unpacker.
+
 AUTHORS
     Tokuhiro Matsuno
 
     Makamaka Hannyaharamitu
 
+    gfx
+
 THANKS TO
     Jun Kuriyama
 
@@ -99,5 +114,6 @@ LICENSE
     under the same terms as Perl itself.
 
 SEE ALSO
-     is official web site for MessagePack format.
+     is the official web site for the MessagePack
+    format.
 

From 953aa95c648fef85f9c11c5ed251f3ddab988a83 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Sat, 18 Sep 2010 06:16:26 +0900
Subject: [PATCH 0755/1648] perl: added failing test case for streaming
 unpacker with PP.

---
 perl/t/11_stream_unpack3.t | 39 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 perl/t/11_stream_unpack3.t

diff --git a/perl/t/11_stream_unpack3.t b/perl/t/11_stream_unpack3.t
new file mode 100644
index 0000000..0eb8bff
--- /dev/null
+++ b/perl/t/11_stream_unpack3.t
@@ -0,0 +1,39 @@
+use strict;
+use warnings;
+use Test::More;
+use Data::MessagePack;
+
+my @data = ( [ 1, 2, 3 ], [ 4, 5, 6 ] );
+
+# serialize
+my $buffer = '';
+for my $d (@data) {
+    $buffer .= Data::MessagePack->pack($d);
+}
+
+# deserialize
+my $cb = sub {
+    my ($data) = @_;
+
+    my $d = shift @data;
+    is_deeply $data, $d;
+};
+my $unpacker = Data::MessagePack::Unpacker->new();
+my $nread = 0;
+while (1) {
+    $nread = $unpacker->execute( $buffer, $nread );
+    if ( $unpacker->is_finished ) {
+        my $ret = $unpacker->data;
+        $cb->( $ret );
+        $unpacker->reset;
+
+        $buffer = substr( $buffer, $nread );
+        $nread = 0;
+        next if length($buffer) != 0;
+    }
+    last;
+}
+is scalar(@data), 0;
+
+done_testing;
+

From 2c9966a0a304dc4bdb5fc003100ec37b2ec3d70a Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Sat, 18 Sep 2010 09:44:32 +0900
Subject: [PATCH 0756/1648] perl: fixed stream deserializer in pp.

---
 perl/lib/Data/MessagePack/PP.pm | 48 ++++++++++++++++++---------------
 perl/t/03_stream_unpack.t       |  4 +--
 perl/t/09_stddata.t             |  1 +
 perl/t/10_splitted_bytes.t      |  6 +++--
 perl/t/12_stream_unpack3.t      | 23 ++++++++++++++++
 5 files changed, 57 insertions(+), 25 deletions(-)
 create mode 100644 perl/t/12_stream_unpack3.t

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 6a06c3c..0dd6427 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -370,7 +370,7 @@ package
 use strict;
 
 sub new {
-    bless { stack => [] }, shift;
+    bless { pos => 0 }, shift;
 }
 
 
@@ -384,25 +384,30 @@ sub execute_limit {
 
 sub execute {
     my ( $self, $data, $offset, $limit ) = @_;
-    my $value = substr( $data, $offset || 0, $limit ? $limit : length $data );
+    $offset ||= 0;
+    my $value = substr( $data, $offset, $limit ? $limit : length $data );
     my $len   = length $value;
 
+    $self->{data} .= $value;
+    local $self->{stack} = [];
+
     $p = 0;
 
-    while ( $len > $p ) {
-        _count( $self, $value ) or last;
+    LOOP: while ( length($self->{data}) > $p ) {
+        _count( $self, $self->{data} ) or last;
 
-        if ( @{ $self->{stack} } > 0 ) {
-            pop @{ $self->{stack} } if --$self->{stack}->[-1] == 0;
+        while ( @{ $self->{stack} } > 0 && --$self->{stack}->[-1] == 0) {
+            pop @{ $self->{stack} };
+        }
+
+        if (@{$self->{stack}} == 0) {
+            $self->{is_finished}++;
+            last LOOP;
         }
     }
+    $self->{pos} = $p;
 
-    if ( $len == $p ) {
-        $self->{ data } .= substr( $value, 0, $p );
-        $self->{ remain } = undef;
-    }
-
-    return $p;
+    return $p + $offset;
 }
 
 
@@ -424,7 +429,9 @@ sub _count {
             $num = $byte & ~0x90;
         }
 
-        push @{ $self->{stack} }, $num + 1;
+        if (defined($num) && $num > 0) {
+            push @{ $self->{stack} }, $num + 1;
+        }
 
         return 1;
     }
@@ -443,7 +450,9 @@ sub _count {
             $num = $byte & ~0x80;
         }
 
-        push @{ $self->{stack} }, $num * 2 + 1; # a pair
+        if ($num > 0) {
+            push @{ $self->{stack} }, $num * 2 + 1; # a pair
+        }
 
         return 1;
     }
@@ -511,22 +520,19 @@ sub _count {
 
 
 sub data {
-    my $data = Data::MessagePack->unpack( $_[0]->{ data } );
-    $_[0]->reset;
-    return $data;
+    return Data::MessagePack->unpack( substr($_[0]->{ data }, 0, $_[0]->{pos}) );
 }
 
 
 sub is_finished {
     my ( $self ) = @_;
-    ( scalar( @{ $self->{stack} } ) or defined $self->{ remain } ) ? 0 : 1;
+    return $self->{is_finished};
 }
 
-
 sub reset :method {
-    $_[0]->{ stack }  = [];
     $_[0]->{ data }   = undef;
-    $_[0]->{ remain } = undef;
+    $_[0]->{ pos }    = 0;
+    $_[0]->{ is_finished } = 0;
 }
 
 1;
diff --git a/perl/t/03_stream_unpack.t b/perl/t/03_stream_unpack.t
index a4ab4eb..646fc24 100644
--- a/perl/t/03_stream_unpack.t
+++ b/perl/t/03_stream_unpack.t
@@ -37,7 +37,7 @@ for (my $i=0; $iexecute("\xc0", 0); # nil
     }
-    ok $up->is_finished;
-    is_deeply $up->data, [undef, undef, undef, undef, undef];
+    ok $up->is_finished, 'finished';
+    is_deeply $up->data, [undef, undef, undef, undef, undef], 'array, is_deeply';
 }
 
diff --git a/perl/t/09_stddata.t b/perl/t/09_stddata.t
index a618787..f98d696 100644
--- a/perl/t/09_stddata.t
+++ b/perl/t/09_stddata.t
@@ -32,6 +32,7 @@ for my $mpac($mpac1, $mpac2) {
     my $i = 0;
     while($offset < length($mpac)) {
         $offset = $mps->execute($mpac, $offset);
+        ok $mps->is_finished, "data[$i] : is_finished";
         is_deeply $mps->data, $data[$i], "data[$i]";
         $mps->reset;
         $i++;
diff --git a/perl/t/10_splitted_bytes.t b/perl/t/10_splitted_bytes.t
index 232d870..15598f4 100644
--- a/perl/t/10_splitted_bytes.t
+++ b/perl/t/10_splitted_bytes.t
@@ -27,12 +27,14 @@ foreach my $size(1 .. 16) {
     open my $stream, '<:bytes :scalar', \$packed;
     binmode $stream;
     my $buff;
+    my $done = 0;
     while( read($stream, $buff, $size) ) {
         #note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff;
 
-        $up->execute($buff);
+        $done = $up->execute($buff);
     }
-    ok $up->is_finished, 'is_finished';
+    is $done, length($packed);
+    ok $up->is_finished, "is_finished: $size";
     my $data = $up->data;
     is_deeply $data, $input;
 }
diff --git a/perl/t/12_stream_unpack3.t b/perl/t/12_stream_unpack3.t
new file mode 100644
index 0000000..118acc3
--- /dev/null
+++ b/perl/t/12_stream_unpack3.t
@@ -0,0 +1,23 @@
+use strict;
+use warnings;
+use Data::MessagePack;
+use Test::More;
+use t::Util;
+
+my @input = (
+    +[[]],
+    [[],[]],
+    [{"a" => 97},{"a" => 97}],
+    [{"a" => 97},{"a" => 97},{"a" => 97}],
+);
+
+plan tests => @input * 2;
+
+for my $input (@input) {
+    my $packed = Data::MessagePack->pack($input);
+    my $up = Data::MessagePack::Unpacker->new();
+    $up->execute($packed, 0);
+    ok $up->is_finished, 'finished';
+    is_deeply($up->data, $input);
+}
+

From 4767e450351dfbc807e5346ceaa4252bd99ae866 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 13:18:25 +0900
Subject: [PATCH 0757/1648] perl: fix a test name

---
 perl/t/{12_stream_unpack3.t => 12_stream_unpack4.t} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename perl/t/{12_stream_unpack3.t => 12_stream_unpack4.t} (100%)

diff --git a/perl/t/12_stream_unpack3.t b/perl/t/12_stream_unpack4.t
similarity index 100%
rename from perl/t/12_stream_unpack3.t
rename to perl/t/12_stream_unpack4.t

From 1f07721ec41147e02fa49aea19a3f6aa7b1eb723 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 13:33:27 +0900
Subject: [PATCH 0758/1648] perl: Scalar::Util is no longer used

---
 perl/lib/Data/MessagePack/PP.pm | 1 -
 1 file changed, 1 deletion(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 0dd6427..abb6e9a 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -12,7 +12,6 @@ use Carp ();
 package
     Data::MessagePack;
 
-use Scalar::Util qw( blessed );
 use strict;
 use B ();
 

From c707392a5a9307504595f6fb9f11930a6a514531 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 14:30:08 +0900
Subject: [PATCH 0759/1648] perl: fix int64_t unpacking in both XS and PP

---
 perl/lib/Data/MessagePack/PP.pm | 67 ++++++++++++++++++++-------------
 perl/t/data.pl                  | 30 ++++++++++++---
 perl/xs-src/unpack.c            | 32 +++++++++++-----
 3 files changed, 89 insertions(+), 40 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index abb6e9a..c3ce230 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -16,12 +16,44 @@ use strict;
 use B ();
 
 BEGIN {
+    my $unpack_int64_slow;
+    my $unpack_uint64_slow;
+
+    if(!eval { pack 'Q', 1 }) { # don't have quad types
+        $unpack_int64_slow = sub {
+            require Math::BigInt;
+            my $high = Math::BigInt->new( unpack_int32(  $_[0], $_[1]) );
+            my $low  = Math::BigInt->new( unpack_uint32( $_[0], $_[1] + 4) );
+
+            return +($high << 32 | $low)->bstr;
+        };
+        $unpack_uint64_slow = sub {
+            require Math::BigInt;
+            my $high = Math::BigInt->new( unpack_uint32(  $_[0], $_[1]) );
+            my $low  = Math::BigInt->new( unpack_uint32( $_[0], $_[1] + 4) );
+            return +($high << 32 | $low)->bstr;
+        };
+    }
+
+    *unpack_uint16 = sub { return unpack 'n', substr( $_[0], $_[1], 2 ) };
+    *unpack_uint32 = sub { return unpack 'N', substr( $_[0], $_[1], 4 ) };
+
     # for pack and unpack compatibility
     if ( $] < 5.010 ) {
         # require $Config{byteorder}; my $bo_is_le = ( $Config{byteorder} =~ /^1234/ );
         # which better?
         my $bo_is_le = unpack ( 'd', "\x00\x00\x00\x00\x00\x00\xf0\x3f") == 1; # 1.0LE
 
+        *unpack_int16  = sub {
+            my $v = unpack 'n', substr( $_[0], $_[1], 2 );
+            return $v ? $v - 0x10000 : 0;
+        };
+        *unpack_int32  = sub {
+            no warnings; # avoid for warning about Hexadecimal number
+            my $v = unpack 'N', substr( $_[0], $_[1], 4 );
+            return $v ? $v - 0x100000000 : 0;
+        };
+
         # In reality, since 5.9.2 '>' is introduced. but 'n!' and 'N!'?
         if($bo_is_le) {
             *pack_uint64 = sub {
@@ -46,20 +78,11 @@ BEGIN {
                 return unpack( 'd', pack( 'N2', @v[1,0] ) );
             };
 
-            *unpack_int16  = sub {
-                my $v = unpack 'n', substr( $_[0], $_[1], 2 );
-                return $v ? $v - 0x10000 : 0;
-            };
-            *unpack_int32  = sub {
-                no warnings; # avoid for warning about Hexadecimal number
-                my $v = unpack 'N', substr( $_[0], $_[1], 4 );
-                return $v ? $v - 0x100000000 : 0;
-            };
-            *unpack_int64 = sub {
+            *unpack_int64 = $unpack_int64_slow ||_sub {
                 my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) );
                 return unpack( 'q', pack( 'N2', @v[1,0] ) );
             };
-            *unpack_uint64 = sub {
+            *unpack_uint64 = $unpack_uint64_slow || sub {
                 my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) );
                 return unpack( 'Q', pack( 'N2', @v[1,0] ) );
             };
@@ -71,17 +94,8 @@ BEGIN {
 
             *unpack_float  = sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); };
             *unpack_double = sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); };
-            *unpack_int16  = sub {
-                my $v = unpack 'n', substr( $_[0], $_[1], 2 );
-                return $v ? $v - 0x10000 : 0;
-            };
-            *unpack_int32  = sub {
-                no warnings; # avoid for warning about Hexadecimal number
-                my $v = unpack 'N', substr( $_[0], $_[1], 4 );
-                return $v ? $v - 0x100000000 : 0;
-            };
-            *unpack_int64  = sub { pack 'q', substr( $_[0], $_[1], 8 ); };
-            *unpack_uint64 = sub { pack 'Q', substr( $_[0], $_[1], 8 ); };
+            *unpack_int64  = $unpack_int64_slow  || sub { pack 'q', substr( $_[0], $_[1], 8 ); };
+            *unpack_uint64 = $unpack_uint64_slow || sub { pack 'Q', substr( $_[0], $_[1], 8 ); };
         }
     }
     else {
@@ -93,8 +107,9 @@ BEGIN {
         *unpack_double = sub { return unpack( 'd>', substr( $_[0], $_[1], 8 ) ); };
         *unpack_int16  = sub { return unpack( 'n!', substr( $_[0], $_[1], 2 ) ); };
         *unpack_int32  = sub { return unpack( 'N!', substr( $_[0], $_[1], 4 ) ); };
-        *unpack_int64  = sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); };
-        *unpack_uint64 = sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); };
+
+        *unpack_int64  = $unpack_int64_slow  || sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); };
+        *unpack_uint64 = $unpack_uint64_slow || sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); };
     }
 }
 
@@ -283,11 +298,11 @@ sub _unpack {
     }
     elsif ( $byte == 0xcd ) { # uint16
         $p += 2;
-        return CORE::unpack 'n', substr( $value, $p - 2, 2 );
+        return unpack_uint16( $value, $p - 2 );
     }
     elsif ( $byte == 0xce ) { # unit32
         $p += 4;
-        return CORE::unpack 'N', substr( $value, $p - 4, 4 );
+        return unpack_uint32( $value, $p - 4 );
     }
     elsif ( $byte == 0xcf ) { # unit64
         $p += 8;
diff --git a/perl/t/data.pl b/perl/t/data.pl
index 2f58d38..8ffd25a 100644
--- a/perl/t/data.pl
+++ b/perl/t/data.pl
@@ -5,14 +5,34 @@ no warnings; # i need this, i need this.
     '92 90 91 91 c0', [[], [[undef]]],
     '93 c0 c2 c3', [undef, false, true],
     'ce 80 00 00 00', 2147483648,
-    '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff', [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295],
+    '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff',
+        [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295],
     '92 93 00 40 7f 93 e0 f0 ff', [[0, 64, 127], [-32, -16, -1]],
-    '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3', [[], [undef], [false, true], [], [undef], [false, true]],
-    '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62', ["", "a", "ab", "", "a", "ab"],
-    '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff', [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1],
+    '96 dc 00 00 dc 00 01 c0 dc 00 02 c2 c3 dd 00 00 00 00 dd 00 00 00 01 c0 dd 00 00 00 02 c2 c3',
+        [[], [undef], [false, true], [], [undef], [false, true]],
+    '96 da 00 00 da 00 01 61 da 00 02 61 62 db 00 00 00 00 db 00 00 00 01 61 db 00 00 00 02 61 62',
+        ["", "a", "ab", "", "a", "ab"],
+    '99 d0 00 d0 80 d0 ff d1 00 00 d1 80 00 d1 ff ff d2 00 00 00 00 d2 80 00 00 00 d2 ff ff ff ff',
+        [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1],
     '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}},
-    '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2', [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}],
+    '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2',
+        [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}],
     'ce 00 ff ff ff' => ''.0xFFFFFF,
     'aa 34 32 39 34 39 36 37 32 39 35' => ''.0xFFFFFFFF,
     'ab 36 38 37 31 39 34 37 36 37 33 35' => ''.0xFFFFFFFFF,
+
+    'd2 80 00 00 01' => '-2147483647', # int32_t
+    'ce 80 00 00 01' => '2147483649',  # uint32_t
+
+    'd2 ff ff ff ff' => '-1',          # int32_t
+    'ce ff ff ff ff' => '4294967295',  # uint32_t
+
+    'd3 00 00 00 00 80 00 00 01' => '2147483649', # int64_t
+    'cf 00 00 00 00 80 00 00 01' => '2147483649',  # uint64_t
+
+    'd3 ff 00 ff ff ff ff ff ff' => '-71776119061217281',    # int64_t
+    'cf ff 00 ff ff ff ff ff ff' => '18374967954648334335',  # uint64_t
+
+    'd3 ff ff ff ff ff ff ff ff' => '-1',                    # int64_t
+    'cf ff ff ff ff ff ff ff ff' => '18446744073709551615',  # uint64_t
 )
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index e89b22c..fefb52e 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -102,13 +102,6 @@ STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const
     return 0;
 }
 
-STATIC_INLINE int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
-{
-    dTHX;
-    *o = newSVnv((NV)d);
-    return 0;
-}
-
 STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
 {
     dTHX;
@@ -116,10 +109,31 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const
     return 0;
 }
 
-STATIC_INLINE int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
+static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
 {
     dTHX;
-    *o = newSVnv((NV)d);
+    if((uint64_t)(NV)d == d) {
+        *o = newSVnv((NV)d);
+    }
+    else {
+        char tbuf[64];
+        STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d);
+        *o = newSVpvn(tbuf, len);
+    }
+    return 0;
+}
+
+static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
+{
+    dTHX;
+    if((uint64_t)(NV)d == (uint64_t)d) {
+        *o = newSVnv((NV)d);
+    }
+    else {
+        char tbuf[64];
+        STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d);
+        *o = newSVpvn(tbuf, len);
+    }
     return 0;
 }
 

From a86c1624a70b8f8a5012065019d9f1ba4b44595b Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 14:38:35 +0900
Subject: [PATCH 0760/1648] perl: More kind error messages in PP

---
 perl/lib/Data/MessagePack/PP.pm | 19 +++++++++++--------
 perl/t/50_leaktrace.t           |  6 +++++-
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index c3ce230..31c0833 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -113,6 +113,9 @@ BEGIN {
     }
 }
 
+sub _unexpected {
+    Carp::confess("Unexpected " . sprintf(shift, @_) . " found");
+}
 
 #
 # PACK
@@ -141,7 +144,7 @@ sub _pack {
               $num < 16          ? CORE::pack( 'C',  0x90 + $num )
             : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xdc,  $num )
             : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdd,  $num )
-            : die "" # don't arrivie here
+            : _unexpected("number %d", $num)
         ;
         if ( --$max_depth <= 0 ) {
             Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)");
@@ -155,7 +158,7 @@ sub _pack {
               $num < 16          ? CORE::pack( 'C',  0x80 + $num )
             : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xde,  $num )
             : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdf,  $num )
-            : die "" # don't arrivie here
+            : _unexpected("number %d", $num)
         ;
         if ( --$max_depth <= 0 ) {
             Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)");
@@ -211,7 +214,7 @@ sub _pack {
               $num < 32          ? CORE::pack( 'C',  0xa0 + $num )
             : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xda, $num )
             : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdb, $num )
-            : die "" # don't arrivie here
+            : _unexpected_number($num)
         ;
 
         return $header . $value;
@@ -221,7 +224,7 @@ sub _pack {
         return pack_double( $value );
     }
     else {
-        die "???";
+        _unexpected("data type %s", $b_obj);
     }
 
 }
@@ -365,7 +368,7 @@ sub _unpack {
     }
 
     else {
-        die "???";
+        _unexpected("byte 0x%02x", $byte);
     }
 
 }
@@ -484,7 +487,7 @@ sub _count {
             : $byte == 0xcd ? 2
             : $byte == 0xce ? 4
             : $byte == 0xcf ? 8
-            : die;
+            : _unexpected("byte 0x%02x", $byte);
         return 1;
     }
 
@@ -493,7 +496,7 @@ sub _count {
             : $byte == 0xd1 ? 2
             : $byte == 0xd2 ? 4
             : $byte == 0xd3 ? 8
-            : die;
+            : _unexpected("byte 0x%02x", $byte);
         return 1;
     }
 
@@ -524,7 +527,7 @@ sub _count {
     }
 
     else {
-        die "???";
+        _unexpected("byte 0x%02x", $byte);
     }
 
     return 0;
diff --git a/perl/t/50_leaktrace.t b/perl/t/50_leaktrace.t
index 2948527..440ac90 100644
--- a/perl/t/50_leaktrace.t
+++ b/perl/t/50_leaktrace.t
@@ -2,8 +2,12 @@
 use strict;
 use Test::Requires { 'Test::LeakTrace' => 0.13 };
 use Test::More;
-
 use Data::MessagePack;
+BEGIN {
+    if($INC{'Data/MessagePack/PP.pm'}) {
+        plan skip_all => 'disabled in PP';
+     }
+}
 
 my $simple_data  = "xyz";
 my $complex_data = {

From bab622de25042d11dc8c149da26e178c22398c56 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 14:46:10 +0900
Subject: [PATCH 0761/1648] perl: fix max depth checks in PP

---
 perl/lib/Data/MessagePack/PP.pm | 16 ++++++++--------
 perl/t/12_stream_unpack4.t      |  1 +
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 31c0833..8a904a9 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -124,11 +124,11 @@ sub _unexpected {
 {
     no warnings 'recursion';
 
-    my $max_depth;
+    our $_max_depth;
 
 sub pack :method {
     Carp::croak('Usage: Data::MessagePack->pack($dat [,$max_depth])') if @_ < 2;
-    $max_depth = defined $_[2] ? $_[2] : 512; # init
+    $_max_depth = defined $_[2] ? $_[2] : 512; # init
     return _pack( $_[1] );
 }
 
@@ -136,6 +136,12 @@ sub pack :method {
 sub _pack {
     my ( $value ) = @_;
 
+    local $_max_depth = $_max_depth - 1;
+
+    if ( $_max_depth < 0 ) {
+        Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)");
+    }
+
     return CORE::pack( 'C', 0xc0 ) if ( not defined $value );
 
     if ( ref($value) eq 'ARRAY' ) {
@@ -146,9 +152,6 @@ sub _pack {
             : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdd,  $num )
             : _unexpected("number %d", $num)
         ;
-        if ( --$max_depth <= 0 ) {
-            Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)");
-        }
         return join( '', $header, map { _pack( $_ ) } @$value );
     }
 
@@ -160,9 +163,6 @@ sub _pack {
             : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdf,  $num )
             : _unexpected("number %d", $num)
         ;
-        if ( --$max_depth <= 0 ) {
-            Carp::croak("perl structure exceeds maximum nesting level (max_depth set too low?)");
-        }
         return join( '', $header, map { _pack( $_ ) } %$value );
     }
 
diff --git a/perl/t/12_stream_unpack4.t b/perl/t/12_stream_unpack4.t
index 118acc3..de76e81 100644
--- a/perl/t/12_stream_unpack4.t
+++ b/perl/t/12_stream_unpack4.t
@@ -9,6 +9,7 @@ my @input = (
     [[],[]],
     [{"a" => 97},{"a" => 97}],
     [{"a" => 97},{"a" => 97},{"a" => 97}],
+    [ map { +{ "foo $_" => "bar $_" } } 'aa' .. 'zz' ],
 );
 
 plan tests => @input * 2;

From e6f6aba2071c0a55b67dc935fa876af6b23e6c11 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 14:50:52 +0900
Subject: [PATCH 0762/1648] perl: add portability stuff

---
 perl/xs-src/unpack.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index fefb52e..a4929c0 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -1,5 +1,6 @@
 #define NEED_newRV_noinc
 #define NEED_sv_2pv_flags
+#define NEED_my_snprintf
 #include "xshelper.h"
 
 #define MY_CXT_KEY "Data::MessagePack::_unpack_guts" XS_VERSION

From 4902bed4098a4c98fc02555584150b1822c90b6b Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 14:51:17 +0900
Subject: [PATCH 0763/1648] perl: more kind testing messages

---
 perl/t/00_compile.t | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/perl/t/00_compile.t b/perl/t/00_compile.t
index f91b29e..d465f8d 100644
--- a/perl/t/00_compile.t
+++ b/perl/t/00_compile.t
@@ -3,4 +3,5 @@ use warnings;
 use Test::More tests => 1;
 
 use_ok 'Data::MessagePack';
-diag ( $INC{'Data/MessagePack/PP.pm'} ? 'PP' : 'XS' );
+diag ( "Testing Data::MessagePack/$Data::MessagePack::VERSION (",
+    $INC{'Data/MessagePack/PP.pm'} ? 'PP' : 'XS', ")" );

From 8935ecfdb86ad7a44c4bbf34acdd99cf80457cf0 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 14:56:33 +0900
Subject: [PATCH 0764/1648] perl: requires the latest version of Math::BigInt
 for PP

---
 perl/Makefile.PL | 1 +
 1 file changed, 1 insertion(+)

diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index fafc387..586a052 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -37,6 +37,7 @@ NOT_SUPPORT_C99
 }
 else {
     print "configure PP version\n\n";
+    requires 'Math::BigInt' => 1.95; # old versions of BigInt were broken
 }
 
 clean_files qw{

From 1865898cd42c2745850e3fd502f32c4584968263 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 14:58:32 +0900
Subject: [PATCH 0765/1648] perl: Fix Makefile.PL

---
 perl/Makefile.PL | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index 586a052..b5d2701 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -1,3 +1,5 @@
+# Usage: Makefile.PL --pp # disable XS
+#        Makefile.PL -g   # add -g to the compiler and disable optimization flags
 use inc::Module::Install;
 use Module::Install::XSUtil 0.32;
 use Config;
@@ -21,8 +23,9 @@ if ( $] >= 5.008005 and want_xs() ) {
     if ( $has_c99 ) {
         use_xshelper();
         cc_src_paths('xs-src');
-        if ($ENV{DEBUG}) {
-            cc_append_to_ccflags '-g';
+
+        if($Module::Install::AUTHOR) {
+            postamble qq{test :: test_pp\n\n};
         }
     }
     else {
@@ -67,10 +70,6 @@ test_requires('Test::Requires');
 
 test_with_env( test_pp => PERL_DATA_MESSAGEPACK => 'pp' );
 
-if($Module::Install::AUTHOR) {
-    postamble qq{test :: test_pp\n\n};
-}
-
 repository('http://github.com/msgpack/msgpack');
 auto_include;
 WriteAll;

From 63f6c86b46464cd5ff0c48c691bbfe2cee0a484d Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 15:05:22 +0900
Subject: [PATCH 0766/1648] perl: add a note about 64 bit integers

---
 perl/README                  | 6 ++++++
 perl/lib/Data/MessagePack.pm | 8 ++++++++
 2 files changed, 14 insertions(+)

diff --git a/perl/README b/perl/README
index 3105278..3476960 100644
--- a/perl/README
+++ b/perl/README
@@ -79,6 +79,12 @@ SPEED
         json     179443/s      56%       --     -16%
         mp       212910/s      85%      19%       --
 
+CAVEAT
+  Unpacking 64 bit integers
+    This module can unpack 64 bit integers even if your perl does not
+    support them (i.e. where "perl -V:ivsize" is 4), but you cannot
+    calculate these values unless you use "Math::BigInt".
+
 TODO
     Error handling
         MessagePack cannot deal with complex scalars such as object
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 953bdf8..2998178 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -141,6 +141,14 @@ This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC
     json     179443/s      56%       --     -16%
     mp       212910/s      85%      19%       --
 
+=head1 CAVEAT
+
+=head2 Unpacking 64 bit integers
+
+This module can unpack 64 bit integers even if your perl does not support them
+(i.e. where C<< perl -V:ivsize >> is 4), but you cannot calculate these values
+unless you use C.
+
 =head1 TODO
 
 =over

From 49379140c7c431c3a27f11dc7565da118dd38f16 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 15:14:29 +0900
Subject: [PATCH 0767/1648] perl: PERL_ONLY=1 disables XS

---
 perl/lib/Data/MessagePack.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 2998178..0f389b1 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -24,7 +24,7 @@ sub false () {
 
 if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate
     my $backend = $ENV{ PERL_DATA_MESSAGEPACK } || '';
-    if ( $backend !~ /\b pp \b/xms ) {
+    if ( $backend !~ /\b pp \b/xms  or $ENV{PERL_ONLY} ) {
         eval {
             require XSLoader;
             XSLoader::load(__PACKAGE__, $VERSION);

From cb85dcfcb8bdaf706ab515bce4bc1991c990716c Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 15:49:25 +0900
Subject: [PATCH 0768/1648] perl: tweaks for benchmarks

---
 perl/benchmark/data.pl        | 6 ++++++
 perl/benchmark/deserialize.pl | 8 ++------
 perl/benchmark/serialize.pl   | 7 +------
 3 files changed, 9 insertions(+), 12 deletions(-)
 create mode 100755 perl/benchmark/data.pl

diff --git a/perl/benchmark/data.pl b/perl/benchmark/data.pl
new file mode 100755
index 0000000..6908d1c
--- /dev/null
+++ b/perl/benchmark/data.pl
@@ -0,0 +1,6 @@
++{
+    "method" => "handleMessage",
+    "params" => [ "user1", "we were just talking", "foo\nbar\nbaz\nqux"  ],
+    "id"     => undef,
+    "array"  => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2), 1 .. 100 ],
+};
diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl
index 634a79e..b1d7fdf 100644
--- a/perl/benchmark/deserialize.pl
+++ b/perl/benchmark/deserialize.pl
@@ -7,12 +7,8 @@ use Storable;
 
 #$Data::MessagePack::PreferInteger = 1;
 
-my $a = {
-    "method" => "handleMessage",
-    "params" => [ "user1", "we were just talking" ],
-    "id"     => undef,
-    "array"  => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ],
-};
+my $a = do 'benchmark/data.pl';
+
 my $j = JSON::XS::encode_json($a);
 my $m = Data::MessagePack->pack($a);
 my $s = Storable::freeze($a);
diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl
index e0509ff..3374684 100644
--- a/perl/benchmark/serialize.pl
+++ b/perl/benchmark/serialize.pl
@@ -5,12 +5,7 @@ use JSON::XS;
 use Storable;
 use Benchmark ':all';
 
-my $a = {
-    "method" => "handleMessage",
-    "params" => [ "user1", "we were just talking" ],
-    "id"     => undef,
-    "array"  => [ 1, 1024, 70000, -5, 1e5, 1e7, 1, 0, 3.14, sqrt(2) ],
-};
+my $a = do 'benchmark/data.pl';
 
 print "-- serialize\n";
 print "JSON::XS: $JSON::XS::VERSION\n";

From b40284955781a152221e6fe6e0d2568ef4fdc27a Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 15:54:22 +0900
Subject: [PATCH 0769/1648] perl: docs

---
 perl/lib/Data/MessagePack.pm | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 0f389b1..4e4064b 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -45,6 +45,8 @@ Data::MessagePack - MessagePack serialising/deserialising
 
 =head1 SYNOPSIS
 
+    use Data::MessagePack;
+
     my $packed   = Data::MessagePack->pack($dat);
     my $unpacked = Data::MessagePack->unpack($dat);
 
@@ -55,7 +57,8 @@ This module converts Perl data structures to MessagePack and vice versa.
 =head1 ABOUT MESSAGEPACK FORMAT
 
 MessagePack is a binary-based efficient object serialization format.
-It enables to exchange structured objects between many languages like JSON. But unlike JSON, it is very fast and small.
+It enables to exchange structured objects between many languages like JSON.
+But unlike JSON, it is very fast and small.
 
 =head2 ADVANTAGES
 
@@ -113,7 +116,7 @@ Packs a string as an integer, when it looks like an integer.
 =head1 SPEED
 
 This is a result of benchmark/serialize.pl and benchmark/deserialize.pl on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP).
-
+(You should benchmark them with B data if the speed matters, of course.)
 
     -- serialize
     JSON::XS: 2.3
@@ -195,4 +198,8 @@ it under the same terms as Perl itself.
 
 L is the official web site for the  MessagePack format.
 
+L
+
+L
+
 =cut

From c2bf2a817410b013aecc30d6a5abc0cad0423dff Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 16:16:51 +0900
Subject: [PATCH 0770/1648] perl: make pp benchmarks available

---
 perl/benchmark/deserialize.pl | 10 +++++-----
 perl/benchmark/serialize.pl   |  6 +++---
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/perl/benchmark/deserialize.pl b/perl/benchmark/deserialize.pl
index b1d7fdf..faa2582 100644
--- a/perl/benchmark/deserialize.pl
+++ b/perl/benchmark/deserialize.pl
@@ -1,25 +1,25 @@
 use strict;
 use warnings;
 use Data::MessagePack;
-use JSON::XS;
-use Benchmark ':all';
+use JSON;
 use Storable;
+use Benchmark ':all';
 
 #$Data::MessagePack::PreferInteger = 1;
 
 my $a = do 'benchmark/data.pl';
 
-my $j = JSON::XS::encode_json($a);
+my $j = JSON::encode_json($a);
 my $m = Data::MessagePack->pack($a);
 my $s = Storable::freeze($a);
 
 print "-- deserialize\n";
-print "JSON::XS: $JSON::XS::VERSION\n";
+print "$JSON::Backend: ", $JSON::Backend->VERSION, "\n";
 print "Data::MessagePack: $Data::MessagePack::VERSION\n";
 print "Storable: $Storable::VERSION\n";
 cmpthese timethese(
     -1 => {
-        json     => sub { JSON::XS::decode_json($j)     },
+        json     => sub { JSON::decode_json($j)     },
         mp       => sub { Data::MessagePack->unpack($m) },
         storable => sub { Storable::thaw($s) },
     }
diff --git a/perl/benchmark/serialize.pl b/perl/benchmark/serialize.pl
index 3374684..4982ff6 100644
--- a/perl/benchmark/serialize.pl
+++ b/perl/benchmark/serialize.pl
@@ -1,19 +1,19 @@
 use strict;
 use warnings;
 use Data::MessagePack;
-use JSON::XS;
+use JSON;
 use Storable;
 use Benchmark ':all';
 
 my $a = do 'benchmark/data.pl';
 
 print "-- serialize\n";
-print "JSON::XS: $JSON::XS::VERSION\n";
+print "$JSON::Backend: ", $JSON::Backend->VERSION, "\n";
 print "Data::MessagePack: $Data::MessagePack::VERSION\n";
 print "Storable: $Storable::VERSION\n";
 cmpthese timethese(
     -1 => {
-        json     => sub { JSON::XS::encode_json($a) },
+        json     => sub { JSON::encode_json($a) },
         storable => sub { Storable::freeze($a) },
         mp       => sub { Data::MessagePack->pack($a) },
     }

From 29707bd2ead5fa41dd0908fd7039124be1f6c5d8 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 16:19:16 +0900
Subject: [PATCH 0771/1648] perl: fix bootstrap

---
 perl/lib/Data/MessagePack.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 4e4064b..154c5b1 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -23,8 +23,8 @@ sub false () {
 }
 
 if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate
-    my $backend = $ENV{ PERL_DATA_MESSAGEPACK } || '';
-    if ( $backend !~ /\b pp \b/xms  or $ENV{PERL_ONLY} ) {
+    my $backend = $ENV{PERL_DATA_MESSAGEPACK} || ($ENV{PERL_ONLY} ? 'pp' : '');
+    if ( $backend !~ /\b pp \b/xms ) {
         eval {
             require XSLoader;
             XSLoader::load(__PACKAGE__, $VERSION);

From 8d182f1d79808812bd095125eee1a43c7f7904f5 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sat, 18 Sep 2010 16:30:07 +0900
Subject: [PATCH 0772/1648] perl: Changelogging

---
 perl/Changes | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/perl/Changes b/perl/Changes
index ce52581..d338cf8 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,7 +1,8 @@
 
 0.24
-    - Fixed a possible SEGV on streaming unpacking (gfx)
-    - Improve performance, esp. in unpacking (gfx)
+    - Fixed a lot of streaming unpacking issues (tokuhirom, gfx)
+    - Fixed unpacking issues for 64 bit integers on 32 bit perls (gfx)
+    - Improved performance, esp. in unpacking (gfx)
 
 0.23
 

From a1c01c6722a50db0aedcc73f2bb6cdd6f16c8f3a Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sun, 19 Sep 2010 15:15:31 +0900
Subject: [PATCH 0773/1648] perl: regen README

---
 perl/README | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/perl/README b/perl/README
index 3476960..224ff08 100644
--- a/perl/README
+++ b/perl/README
@@ -2,6 +2,8 @@ NAME
     Data::MessagePack - MessagePack serialising/deserialising
 
 SYNOPSIS
+        use Data::MessagePack;
+
         my $packed   = Data::MessagePack->pack($dat);
         my $unpacked = Data::MessagePack->unpack($dat);
 
@@ -51,7 +53,8 @@ Configuration Variables
 
 SPEED
     This is a result of benchmark/serialize.pl and benchmark/deserialize.pl
-    on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP).
+    on my SC440(Linux 2.6.32-23-server #37-Ubuntu SMP). (You should
+    benchmark them with your data if the speed matters, of course.)
 
         -- serialize
         JSON::XS: 2.3
@@ -123,3 +126,7 @@ SEE ALSO
      is the official web site for the MessagePack
     format.
 
+    Data::MessagePack::Unpacker
+
+    AnyEvent::MPRPC
+

From d6a825981d14079bf4fa74a0605ecec8e873f94b Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sun, 19 Sep 2010 15:16:08 +0900
Subject: [PATCH 0774/1648] perl: fix unpacking int64_t in PP (based on
 makamaka's patch)

---
 perl/lib/Data/MessagePack/PP.pm | 15 ++++++++++++---
 perl/t/data.pl                  |  5 +++++
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 8a904a9..44940de 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -22,10 +22,19 @@ BEGIN {
     if(!eval { pack 'Q', 1 }) { # don't have quad types
         $unpack_int64_slow = sub {
             require Math::BigInt;
-            my $high = Math::BigInt->new( unpack_int32(  $_[0], $_[1]) );
-            my $low  = Math::BigInt->new( unpack_uint32( $_[0], $_[1] + 4) );
+            my $high = unpack_uint32( $_[0], $_[1] );
+            my $low  = unpack_uint32( $_[0], $_[1] + 4);
 
-            return +($high << 32 | $low)->bstr;
+            if($high < 0xF0000000) { # positive
+                $high = Math::BigInt->new( $high );
+                $low  = Math::BigInt->new( $low  );
+                return +($high << 32 | $low)->bstr;
+            }
+            else { # negative
+                $high = Math::BigInt->new( ~$high );
+                $low  = Math::BigInt->new( ~$low  );
+                return +( -($high << 32 | $low + 1) )->bstr;
+            }
         };
         $unpack_uint64_slow = sub {
             require Math::BigInt;
diff --git a/perl/t/data.pl b/perl/t/data.pl
index 8ffd25a..95eac41 100644
--- a/perl/t/data.pl
+++ b/perl/t/data.pl
@@ -35,4 +35,9 @@ no warnings; # i need this, i need this.
 
     'd3 ff ff ff ff ff ff ff ff' => '-1',                    # int64_t
     'cf ff ff ff ff ff ff ff ff' => '18446744073709551615',  # uint64_t
+
+    # int64_t
+    'd3 00 00 00 10 00 00 00 00' => '68719476736',
+    'd3 00 00 00 10 00 00 00 01' => '68719476737',
+    'd3 10 00 00 00 00 00 00 00' => '1152921504606846976',
 )

From 5cd37e550533a4d12b3390ff161af83db9b21246 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sun, 19 Sep 2010 15:20:03 +0900
Subject: [PATCH 0775/1648] perl: always unpacking 64 bit ints as a string on
 32 bit perls

---
 perl/xs-src/unpack.c | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index a4929c0..20b345f 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -113,28 +113,18 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const
 static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
 {
     dTHX;
-    if((uint64_t)(NV)d == d) {
-        *o = newSVnv((NV)d);
-    }
-    else {
-        char tbuf[64];
-        STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d);
-        *o = newSVpvn(tbuf, len);
-    }
+    char tbuf[64];
+    STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d);
+    *o = newSVpvn(tbuf, len);
     return 0;
 }
 
 static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
 {
     dTHX;
-    if((uint64_t)(NV)d == (uint64_t)d) {
-        *o = newSVnv((NV)d);
-    }
-    else {
-        char tbuf[64];
-        STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d);
-        *o = newSVpvn(tbuf, len);
-    }
+    char tbuf[64];
+    STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d);
+    *o = newSVpvn(tbuf, len);
     return 0;
 }
 

From afefbe4e564686ac3b9fac9e1d9782f0114c3383 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sun, 19 Sep 2010 15:21:25 +0900
Subject: [PATCH 0776/1648] perl: update .gitignore and MANIFEST.SKIP

---
 perl/.gitignore    | 1 +
 perl/MANIFEST.SKIP | 1 +
 2 files changed, 2 insertions(+)

diff --git a/perl/.gitignore b/perl/.gitignore
index 3e0e73e..1656847 100644
--- a/perl/.gitignore
+++ b/perl/.gitignore
@@ -1,4 +1,5 @@
 META.yml
+MYMETA.yml
 Makefile
 Makefile.old
 MessagePack.bs
diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP
index 71a24e5..1d2192f 100644
--- a/perl/MANIFEST.SKIP
+++ b/perl/MANIFEST.SKIP
@@ -2,6 +2,7 @@
 \bCVS\b
 ^MANIFEST\.
 ^Makefile$
+^MYMETA\.yml$
 ~$
 ^#
 \.old$

From 7f42ed86f29ad9eeab64c6816644bfb83c6eea11 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sun, 19 Sep 2010 15:47:32 +0900
Subject: [PATCH 0777/1648] More tests

---
 perl/t/02_unpack.t | 5 +++--
 perl/t/data.pl     | 8 ++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/perl/t/02_unpack.t b/perl/t/02_unpack.t
index 4da69e9..657387a 100644
--- a/perl/t/02_unpack.t
+++ b/perl/t/02_unpack.t
@@ -5,13 +5,14 @@ no warnings 'uninitialized'; # i need this. i need this.
 
 sub unpackit {
     my $v = $_[0];
-    $v =~ s/ //g;
+    $v =~ s/ +//g;
     $v = pack 'H*', $v;
     return Data::MessagePack->unpack($v);
 }
 
 sub pis ($$) {
-    is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0];
+    is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]
+        or diag( explain(unpackit($_[0])) );
 }
 
 my @dat = do 't/data.pl';
diff --git a/perl/t/data.pl b/perl/t/data.pl
index 95eac41..9bf07b7 100644
--- a/perl/t/data.pl
+++ b/perl/t/data.pl
@@ -4,6 +4,11 @@ no warnings; # i need this, i need this.
     '94 a0 a1 61 a2 62 63 a3 64 65 66', ["", "a", "bc", "def"],
     '92 90 91 91 c0', [[], [[undef]]],
     '93 c0 c2 c3', [undef, false, true],
+
+    '82             d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # fix map
+    'de       00 02 d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # map 16
+    'df 00 00 00 02 d0 2a c2 d0 2b c3', { 42 => false, 43 => true }, # map 32
+
     'ce 80 00 00 00', 2147483648,
     '99 cc 00 cc 80 cc ff cd 00 00 cd 80 00 cd ff ff ce 00 00 00 00 ce 80 00 00 00 ce ff ff ff ff',
         [0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295],
@@ -39,5 +44,8 @@ no warnings; # i need this, i need this.
     # int64_t
     'd3 00 00 00 10 00 00 00 00' => '68719476736',
     'd3 00 00 00 10 00 00 00 01' => '68719476737',
+    'd3 00 00 10 00 00 00 00 00' => '17592186044416',
+    'd3 00 10 00 00 00 00 00 00' => '4503599627370496',
     'd3 10 00 00 00 00 00 00 00' => '1152921504606846976',
+    'd3 11 00 00 00 00 00 00 00' => '1224979098644774912',
 )

From 6379d0fe0f1bc2d3b76480edbe9bd99947aae54e Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sun, 19 Sep 2010 15:53:54 +0900
Subject: [PATCH 0778/1648] perl: more tests for nil/true/false entities

---
 perl/t/07_break.t | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/perl/t/07_break.t b/perl/t/07_break.t
index dd27ad2..cf3f107 100644
--- a/perl/t/07_break.t
+++ b/perl/t/07_break.t
@@ -1,12 +1,23 @@
 use Test::More;
 use Data::MessagePack;
 use t::Util;
-no warnings 'uninitialized'; # i need this. i need this.
 
-plan tests => 1;
+plan tests => 4;
 
-my $d = Data::MessagePack->unpack(Data::MessagePack->pack([{x => undef}]));
-$d->[0]->{x} = 1;
-ok delete $d->[0]->{x};
-$d->[0] = 4;
+my $d = Data::MessagePack->unpack(Data::MessagePack->pack({
+    nil   => undef,
+    true  => true,
+    false => false,
+    foo   => [undef, true, false],
+}));
 
+$d->{nil} = 42;
+is $d->{nil}, 42;
+
+$d->{true} = 43;
+is $d->{true}, 43;
+
+$d->{false} = 44;
+is $d->{false}, 44;
+
+is_deeply $d->{foo}, [undef, true, false];

From 53899cc492fc8ae4bc4095506d25e4d0eff48309 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Sun, 19 Sep 2010 22:12:28 +0900
Subject: [PATCH 0779/1648] perl: tweaks for Makefile.PL

---
 perl/Makefile.PL | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index b5d2701..219400f 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -16,12 +16,12 @@ recursive_author_tests('xt');
 
 
 if ( $] >= 5.008005 and want_xs() ) {
-    can_cc or die "This module requires a C compiler. Please retry with --pp";
-
     my $has_c99 = c99_available(); # msgpack C library requires C99.
 
     if ( $has_c99 ) {
+        requires_c99();
         use_xshelper();
+        cc_warnings;
         cc_src_paths('xs-src');
 
         if($Module::Install::AUTHOR) {
@@ -40,7 +40,7 @@ NOT_SUPPORT_C99
 }
 else {
     print "configure PP version\n\n";
-    requires 'Math::BigInt' => 1.95; # old versions of BigInt were broken
+    requires 'Math::BigInt' => 1.89; # old versions of BigInt were broken
 }
 
 clean_files qw{

From f59178bc3369f43c3c63f02823bd01c1d4f8314a Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Mon, 20 Sep 2010 09:30:26 +0900
Subject: [PATCH 0780/1648] Checking in changes prior to tagging of version
 0.24.

Changelog diff is:
---
 perl/lib/Data/MessagePack.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 154c5b1..5990bdb 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.23';
+our $VERSION = '0.24';
 our $PreferInteger = 0;
 
 sub true () {

From 978bb5059f544b44cb277e0408103ca4a0c27d47 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Mon, 20 Sep 2010 09:54:25 +0900
Subject: [PATCH 0781/1648] Checking in changes prior to tagging of version
 0.25.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index d338cf8..3d455cb 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.25
+
+    (NO FEATURE CHANGES)
+    - oops. I failed releng.

 0.24
     - Fixed a lot of streaming unpacking issues (tokuhirom, gfx)
---
 perl/Changes                 | 4 ++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index d338cf8..3d455cb 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.25
+
+    (NO FEATURE CHANGES)
+    - oops. I failed releng.
 
 0.24
     - Fixed a lot of streaming unpacking issues (tokuhirom, gfx)
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 5990bdb..2cbbe79 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.24';
+our $VERSION = '0.25';
 our $PreferInteger = 0;
 
 sub true () {

From d973192b5e44ade6be1b290bc5020c749b43e585 Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Mon, 20 Sep 2010 11:49:52 +0900
Subject: [PATCH 0782/1648] perl: modified a serious code typo in PP

---
 perl/lib/Data/MessagePack/PP.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 44940de..2394222 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -87,7 +87,7 @@ BEGIN {
                 return unpack( 'd', pack( 'N2', @v[1,0] ) );
             };
 
-            *unpack_int64 = $unpack_int64_slow ||_sub {
+            *unpack_int64 = $unpack_int64_slow || sub {
                 my @v = unpack( 'V*', substr( $_[0], $_[1], 8 ) );
                 return unpack( 'q', pack( 'N2', @v[1,0] ) );
             };

From 664eefdddb6633badeecbbd4dc654eb113b3814d Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Mon, 20 Sep 2010 23:40:12 +0900
Subject: [PATCH 0783/1648] Checking in changes prior to tagging of version
 0.26.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index 3d455cb..b717a82 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.26
+
+    - fixed a serious code typo in PP(makamaka)
+
 0.25

     (NO FEATURE CHANGES)
---
 perl/Changes                 | 4 ++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index 3d455cb..b717a82 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.26
+
+    - fixed a serious code typo in PP(makamaka)
+
 0.25
 
     (NO FEATURE CHANGES)
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 2cbbe79..f6e68eb 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.25';
+our $VERSION = '0.26';
 our $PreferInteger = 0;
 
 sub true () {

From ead8edc7cd3e797e5457c8ba63a2374fb8a87067 Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Wed, 22 Sep 2010 14:12:19 +0900
Subject: [PATCH 0784/1648] modified be unpack_(u)int64 in PP

---
 perl/lib/Data/MessagePack/PP.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 2394222..679dc00 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -103,8 +103,8 @@ BEGIN {
 
             *unpack_float  = sub { return unpack( 'f', substr( $_[0], $_[1], 4 ) ); };
             *unpack_double = sub { return unpack( 'd', substr( $_[0], $_[1], 8 ) ); };
-            *unpack_int64  = $unpack_int64_slow  || sub { pack 'q', substr( $_[0], $_[1], 8 ); };
-            *unpack_uint64 = $unpack_uint64_slow || sub { pack 'Q', substr( $_[0], $_[1], 8 ); };
+            *unpack_int64  = $unpack_int64_slow  || sub { unpack 'q', substr( $_[0], $_[1], 8 ); };
+            *unpack_uint64 = $unpack_uint64_slow || sub { unpack 'Q', substr( $_[0], $_[1], 8 ); };
         }
     }
     else {

From 6d9a629b155c7a6f814a6973c6c24da308b295ce Mon Sep 17 00:00:00 2001
From: makamaka 
Date: Wed, 22 Sep 2010 14:19:09 +0900
Subject: [PATCH 0785/1648] perl: modified trivial codes in PP::Unpacker

---
 perl/lib/Data/MessagePack/PP.pm | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 679dc00..8234b49 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -419,7 +419,7 @@ sub execute {
 
     $p = 0;
 
-    LOOP: while ( length($self->{data}) > $p ) {
+    while ( length($self->{data}) > $p ) {
         _count( $self, $self->{data} ) or last;
 
         while ( @{ $self->{stack} } > 0 && --$self->{stack}->[-1] == 0) {
@@ -428,7 +428,7 @@ sub execute {
 
         if (@{$self->{stack}} == 0) {
             $self->{is_finished}++;
-            last LOOP;
+            last;
         }
     }
     $self->{pos} = $p;
@@ -455,7 +455,7 @@ sub _count {
             $num = $byte & ~0x90;
         }
 
-        if (defined($num) && $num > 0) {
+        if ( $num ) {
             push @{ $self->{stack} }, $num + 1;
         }
 
@@ -476,7 +476,7 @@ sub _count {
             $num = $byte & ~0x80;
         }
 
-        if ($num > 0) {
+        if ( $num ) {
             push @{ $self->{stack} }, $num * 2 + 1; # a pair
         }
 

From c0e2041006d26f47bcba5ec272d19f3e606c4c4c Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Wed, 22 Sep 2010 15:12:51 +0900
Subject: [PATCH 0786/1648] Checking in changes prior to tagging of version
 0.27.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index b717a82..82174fe 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.27
+
+    - * 6d9a629 perl: modified trivial codes in PP::Unpacker(makamaka)
+    - * ead8edc modified be unpack_(u)int64 in PP(makamaka)
+
 0.26

     - fixed a serious code typo in PP(makamaka)
---
 perl/Changes                 | 5 +++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index b717a82..82174fe 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.27
+
+    - * 6d9a629 perl: modified trivial codes in PP::Unpacker(makamaka)
+    - * ead8edc modified be unpack_(u)int64 in PP(makamaka)
+
 0.26
 
     - fixed a serious code typo in PP(makamaka)
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index f6e68eb..5e01b08 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.26';
+our $VERSION = '0.27';
 our $PreferInteger = 0;
 
 sub true () {

From 80f7c54e4d14e4762714c6b26e1dcd994f2f0727 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 22 Sep 2010 15:25:08 +0900
Subject: [PATCH 0787/1648] perl: make scopes

---
 perl/xs-src/unpack.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 20b345f..95b5910 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -31,6 +31,7 @@ typedef struct {
 #define msgpack_unpack_user unpack_user
 
 void init_Data__MessagePack_unpack(pTHX_ bool const cloning) {
+    // booleans are load on demand (lazy load).
     if(!cloning) {
         MY_CXT_INIT;
         MY_CXT.msgpack_true  = NULL;
@@ -52,11 +53,17 @@ static SV*
 load_bool(pTHX_ const char* const name) {
     CV* const cv = get_cv(name, GV_ADD);
     dSP;
+    ENTER;
+    SAVETMPS;
     PUSHMARK(SP);
     call_sv((SV*)cv, G_SCALAR);
     SPAGAIN;
     SV* const sv = newSVsv(POPs);
     PUTBACK;
+    FREETMPS;
+    LEAVE;
+    assert(sv);
+    assert(sv_isobject(sv));
     return sv;
 }
 

From 68b6fa46e61490c3f5fdca71b3e1dc90ebb4a02d Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 22 Sep 2010 15:28:14 +0900
Subject: [PATCH 0788/1648] perl: add tests for boolean values

---
 perl/t/13_booleans.t | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100755 perl/t/13_booleans.t

diff --git a/perl/t/13_booleans.t b/perl/t/13_booleans.t
new file mode 100755
index 0000000..1ecbe64
--- /dev/null
+++ b/perl/t/13_booleans.t
@@ -0,0 +1,12 @@
+#!perl -w
+use strict;
+use Test::More tests => 6;
+use Data::MessagePack;
+
+ok defined(Data::MessagePack::true()), 'true (1)';
+ok defined(Data::MessagePack::true()), 'true (2)';
+ok Data::MessagePack::true(),          'true is true';
+
+ok defined(Data::MessagePack::false()), 'false (1)';
+ok defined(Data::MessagePack::false()), 'false (2)';
+ok !Data::MessagePack::false(),         'false is false';

From 0a8a6ed168dfd9338c2e97fd6b87b88ef39644ef Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 22 Sep 2010 15:59:21 +0900
Subject: [PATCH 0789/1648] perl: cleanup PP

---
 perl/lib/Data/MessagePack/PP.pm | 54 +++++++++++++++------------------
 1 file changed, 24 insertions(+), 30 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 8234b49..5dccc0b 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -1,20 +1,17 @@
 package Data::MessagePack::PP;
 use 5.008001;
 use strict;
+use warnings;
+no warnings 'recursion';
+
 use Carp ();
+use B ();
 
 # See also
 # http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec
 # http://cpansearch.perl.org/src/YAPPO/Data-Model-0.00006/lib/Data/Model/Driver/Memcached.pm
 # http://frox25.no-ip.org/~mtve/wiki/MessagePack.html : reference to using CORE::pack, CORE::unpack
 
-
-package
-    Data::MessagePack;
-
-use strict;
-use B ();
-
 BEGIN {
     my $unpack_int64_slow;
     my $unpack_uint64_slow;
@@ -120,6 +117,18 @@ BEGIN {
         *unpack_int64  = $unpack_int64_slow  || sub { return unpack( 'q>', substr( $_[0], $_[1], 8 ) ); };
         *unpack_uint64 = $unpack_uint64_slow || sub { return unpack( 'Q>', substr( $_[0], $_[1], 8 ) ); };
     }
+
+    # fixin package symbols
+    no warnings 'once';
+    sub pack   :method;
+    sub unpack :method;
+    *Data::MessagePack::pack   = \&pack;
+    *Data::MessagePack::unpack = \&unpack;
+
+    @Data::MessagePack::Unpacker::ISA = qw(Data::MessagePack::PP::Unpacker);
+
+    *true  = \&Data::MessagePack::true;
+    *false = \&Data::MessagePack::false;
 }
 
 sub _unexpected {
@@ -130,10 +139,7 @@ sub _unexpected {
 # PACK
 #
 
-{
-    no warnings 'recursion';
-
-    our $_max_depth;
+our $_max_depth;
 
 sub pack :method {
     Carp::croak('Usage: Data::MessagePack->pack($dat [,$max_depth])') if @_ < 2;
@@ -238,20 +244,19 @@ sub _pack {
 
 }
 
-} # PACK
-
-
 #
 # UNPACK
 #
 
-{
-
-    my $p; # position variables for speed.
+my $p; # position variables for speed.
 
 sub unpack :method {
     $p = 0; # init
-    _unpack( $_[1] );
+    my $data = _unpack( $_[1] );
+    if($p < length($_[1])) {
+        Carp::croak("Data::MessagePack->unpack: extra bytes");
+    }
+    return $data;
 }
 
 
@@ -383,17 +388,12 @@ sub _unpack {
 }
 
 
-} # UNPACK
-
-
 #
 # Data::MessagePack::Unpacker
 #
 
 package
-    Data::MessagePack::Unpacker;
-
-use strict;
+    Data::MessagePack::PP::Unpacker;
 
 sub new {
     bless { pos => 0 }, shift;
@@ -404,10 +404,6 @@ sub execute_limit {
     execute( @_ );
 }
 
-
-{
-    my $p;
-
 sub execute {
     my ( $self, $data, $offset, $limit ) = @_;
     $offset ||= 0;
@@ -542,8 +538,6 @@ sub _count {
     return 0;
 }
 
-} # execute
-
 
 sub data {
     return Data::MessagePack->unpack( substr($_[0]->{ data }, 0, $_[0]->{pos}) );

From 3d905a7a4fe437433dd0746d9864928ec8f464b9 Mon Sep 17 00:00:00 2001
From: gfx 
Date: Wed, 22 Sep 2010 16:14:55 +0900
Subject: [PATCH 0790/1648] perl: add tests for 'extra bytes' exceptions

---
 perl/t/14_invalid_data.t | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100755 perl/t/14_invalid_data.t

diff --git a/perl/t/14_invalid_data.t b/perl/t/14_invalid_data.t
new file mode 100755
index 0000000..f534485
--- /dev/null
+++ b/perl/t/14_invalid_data.t
@@ -0,0 +1,18 @@
+use strict;
+use warnings;
+use Data::MessagePack;
+use Test::More;
+use t::Util;
+
+my $nil = Data::MessagePack->pack(undef);
+
+my @data = do 't/data.pl';
+while(my($dump, $data) = splice @data, 0, 2) {
+    my $s = Data::MessagePack->pack($data);
+    eval {
+        Data::MessagePack->unpack($s . $nil);
+    };
+    like $@, qr/extra bytes/, "dump $dump";
+}
+
+done_testing;

From 142493076a7a268bdb38f02b06ac5bc34a0f3004 Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Thu, 23 Sep 2010 00:04:34 +0900
Subject: [PATCH 0791/1648] haskell: TH support and refactoring

---
 haskell/msgpack.cabal                         |  10 +-
 haskell/src/Data/MessagePack.hs               |  35 +--
 haskell/src/Data/MessagePack/Derive.hs        |  74 ++++++
 haskell/src/Data/MessagePack/Iteratee.hs      |   4 +-
 haskell/src/Data/MessagePack/Object.hs        | 235 +++++++++++++++---
 .../src/Data/MessagePack/{Put.hs => Pack.hs}  |  78 +++---
 .../Data/MessagePack/{Parser.hs => Unpack.hs} | 112 ++++++---
 haskell/test/UserData.hs                      |  19 ++
 8 files changed, 422 insertions(+), 145 deletions(-)
 create mode 100644 haskell/src/Data/MessagePack/Derive.hs
 rename haskell/src/Data/MessagePack/{Put.hs => Pack.hs} (66%)
 rename haskell/src/Data/MessagePack/{Parser.hs => Unpack.hs} (70%)
 create mode 100644 haskell/test/UserData.hs

diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal
index ccdb2f7..9c67bdc 100644
--- a/haskell/msgpack.cabal
+++ b/haskell/msgpack.cabal
@@ -1,5 +1,5 @@
 Name:               msgpack
-Version:            0.3.1.1
+Version:            0.4.0
 Synopsis:           A Haskell binding to MessagePack
 Description:
   A Haskell binding to MessagePack 
@@ -25,17 +25,19 @@ Library
                     attoparsec >= 0.8.1 && < 0.8.2,
                     binary >= 0.5.0 && < 0.5.1,
                     data-binary-ieee754 >= 0.4 && < 0.5,
-                    deepseq >= 1.1 && <1.2
+                    deepseq >= 1.1 && <1.2,
+                    template-haskell >= 2.4 && < 2.5
 
   Ghc-options:      -Wall
   Hs-source-dirs:   src
 
   Exposed-modules:
     Data.MessagePack
+    Data.MessagePack.Pack
+    Data.MessagePack.Unpack
     Data.MessagePack.Object
-    Data.MessagePack.Put
-    Data.MessagePack.Parser
     Data.MessagePack.Iteratee
+    Data.MessagePack.Derive
 
 Source-repository head
   Type:     git
diff --git a/haskell/src/Data/MessagePack.hs b/haskell/src/Data/MessagePack.hs
index 7137589..b71190d 100644
--- a/haskell/src/Data/MessagePack.hs
+++ b/haskell/src/Data/MessagePack.hs
@@ -13,14 +13,11 @@
 --------------------------------------------------------------------
 
 module Data.MessagePack(
+  module Data.MessagePack.Pack,
+  module Data.MessagePack.Unpack,
   module Data.MessagePack.Object,
-  module Data.MessagePack.Put,
-  module Data.MessagePack.Parser,
   module Data.MessagePack.Iteratee,
-  
-  -- * Simple functions of Pack and Unpack
-  pack,
-  unpack,
+  module Data.MessagePack.Derive,
   
   -- * Pack functions
   packToString,
@@ -44,38 +41,18 @@ import qualified Data.Attoparsec as A
 import Data.Binary.Put
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as L
-import Data.Functor.Identity
 import qualified Data.Iteratee as I
 import System.IO
 
+import Data.MessagePack.Pack
+import Data.MessagePack.Unpack
 import Data.MessagePack.Object
-import Data.MessagePack.Put
-import Data.MessagePack.Parser
 import Data.MessagePack.Iteratee
+import Data.MessagePack.Derive
 
 bufferSize :: Int
 bufferSize = 4 * 1024
 
-class IsByteString s where
-  toBS :: s -> B.ByteString
-
-instance IsByteString B.ByteString where
-  toBS = id
-
-instance IsByteString L.ByteString where
-  toBS = B.concat . L.toChunks
-
--- | Pack Haskell data to MessagePack string.
-pack :: ObjectPut a => a -> L.ByteString
-pack = packToString . put
-
--- | Unpack MessagePack string to Haskell data.
-unpack :: (ObjectGet a, IsByteString s) => s -> a
-unpack bs =
-  runIdentity $ I.run $ I.joinIM $ I.enumPure1Chunk (toBS bs) getI
-
--- TODO: tryUnpack
-
 -- | Pack to ByteString.
 packToString :: Put -> L.ByteString
 packToString = runPut
diff --git a/haskell/src/Data/MessagePack/Derive.hs b/haskell/src/Data/MessagePack/Derive.hs
new file mode 100644
index 0000000..cfdb658
--- /dev/null
+++ b/haskell/src/Data/MessagePack/Derive.hs
@@ -0,0 +1,74 @@
+{-# Language TemplateHaskell #-}
+
+module Data.MessagePack.Derive (
+  derivePack,
+  deriveUnpack,
+  deriveObject,
+  ) where
+
+import Control.Applicative
+import Language.Haskell.TH
+
+import Data.MessagePack.Pack
+import Data.MessagePack.Unpack
+
+deriveUnpack :: Name -> Q [Dec]
+deriveUnpack typName = do
+  TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName
+  
+  return
+    [ InstanceD [] (AppT (ConT ''Unpackable) (ConT name))
+      [ FunD 'get [Clause [] (NormalB $ ch $ map body cons) []]
+        ]]
+        
+  where
+    body (NormalC conName elms) =
+      DoE
+      [ BindS (tupOrList $ map VarP names) (VarE 'get)
+      , NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ]
+        where
+          names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms
+
+    tupOrList ls
+      | length ls <= 1 = ListP ls
+      | otherwise = TupP ls
+
+    ch = foldl1 (\e f -> AppE (AppE (VarE '(<|>)) e) f)
+
+derivePack :: Name -> Q [Dec]
+derivePack typName = do
+  TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName
+  
+  return
+    [ InstanceD [] (AppT (ConT ''Packable) (ConT name))
+      [ FunD 'put (map body cons)
+        ]]
+        
+  where
+    body (NormalC conName elms) =
+      Clause
+        [ ConP conName $ map VarP names ]
+        (NormalB $ AppE (VarE 'put) $ tupOrList $ map VarE names) []
+      where
+        names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms
+
+    tupOrList ls
+      | length ls <= 1 = ListE ls
+      | otherwise = TupE ls
+
+deriveObject :: Name -> Q [Dec]
+deriveObject typName = do
+  g <- derivePack typName
+  p <- deriveUnpack typName
+  {-
+  TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName
+  let o = InstanceD [] (AppT (ConT ''OBJECT) (ConT name))
+          [ FunD 'toObject (map toObjectBody cons) ]
+  -}
+  return $ g ++ p -- ++ [o]
+{-
+  where
+    toObjectBody (NormalC conName elms) =
+      Clause
+        [ ConP conP
+-}
diff --git a/haskell/src/Data/MessagePack/Iteratee.hs b/haskell/src/Data/MessagePack/Iteratee.hs
index 4258cf6..6bc0898 100644
--- a/haskell/src/Data/MessagePack/Iteratee.hs
+++ b/haskell/src/Data/MessagePack/Iteratee.hs
@@ -28,10 +28,10 @@ import qualified Data.ByteString as B
 import qualified Data.Iteratee as I
 import System.IO
 
-import Data.MessagePack.Parser
+import Data.MessagePack.Unpack
 
 -- | Deserialize a value
-getI :: (Monad m, ObjectGet a) => I.Iteratee B.ByteString m a
+getI :: (Monad m, Unpackable a) => I.Iteratee B.ByteString m a
 getI = parserToIteratee get
 
 -- | Enumerator
diff --git a/haskell/src/Data/MessagePack/Object.hs b/haskell/src/Data/MessagePack/Object.hs
index 87f24bd..5111ebb 100644
--- a/haskell/src/Data/MessagePack/Object.hs
+++ b/haskell/src/Data/MessagePack/Object.hs
@@ -1,6 +1,7 @@
 {-# Language TypeSynonymInstances #-}
 {-# Language FlexibleInstances #-}
 {-# Language OverlappingInstances #-}
+{-# Language IncoherentInstances #-}
 {-# Language DeriveDataTypeable #-}
 
 --------------------------------------------------------------------
@@ -23,16 +24,21 @@ module Data.MessagePack.Object(
   
   -- * Serialization to and from Object
   OBJECT(..),
-  Result,
+  -- Result,
   ) where
 
 import Control.DeepSeq
+import Control.Exception
 import Control.Monad
 import Control.Monad.Trans.Error ()
+import qualified Data.Attoparsec as A
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Char8 as C8
 import Data.Typeable
 
+import Data.MessagePack.Pack
+import Data.MessagePack.Unpack
+
 -- | Object Representation of MessagePack data.
 data Object =
   ObjectNil
@@ -55,70 +61,241 @@ instance NFData Object where
       ObjectArray a -> rnf a
       ObjectMap m -> rnf m
 
+instance Unpackable Object where
+  get =
+    A.choice
+    [ liftM ObjectInteger get
+    , liftM (\() -> ObjectNil) get
+    , liftM ObjectBool get
+    , liftM ObjectDouble get
+    , liftM ObjectRAW get
+    , liftM ObjectArray get
+    , liftM ObjectMap get
+    ]
+
+instance Packable Object where
+  put obj =
+    case obj of
+      ObjectInteger n ->
+        put n
+      ObjectNil ->
+        put ()
+      ObjectBool b ->
+        put b
+      ObjectDouble d ->
+        put d
+      ObjectRAW raw ->
+        put raw
+      ObjectArray arr ->
+        put arr
+      ObjectMap m ->
+        put m
+
 -- | The class of types serializable to and from MessagePack object
-class OBJECT a where
+class (Unpackable a, Packable a) => OBJECT a where
   -- | Encode a value to MessagePack object
   toObject :: a -> Object
+  toObject = unpack . pack
+  
   -- | Decode a value from MessagePack object
-  fromObject :: Object -> Result a
+  fromObject :: Object -> a
+  fromObject a =
+    case tryFromObject a of
+      Left err ->
+        throw $ UnpackError err
+      Right ret ->
+        ret
 
--- | A type for parser results
-type Result a = Either String a
+  -- | Decode a value from MessagePack object
+  tryFromObject :: Object -> Either String a
+  tryFromObject = tryUnpack . pack
 
 instance OBJECT Object where
   toObject = id
-  fromObject = Right
+  tryFromObject = Right
 
-fromObjectError :: String
-fromObjectError = "fromObject: cannot cast"
+tryFromObjectError :: Either String a
+tryFromObjectError = Left "tryFromObject: cannot cast"
 
 instance OBJECT () where
   toObject = const ObjectNil
-  fromObject ObjectNil = Right ()
-  fromObject _ = Left fromObjectError
+  tryFromObject ObjectNil = Right ()
+  tryFromObject _ = tryFromObjectError
 
 instance OBJECT Int where
   toObject = ObjectInteger
-  fromObject (ObjectInteger n) = Right n
-  fromObject _ = Left fromObjectError
+  tryFromObject (ObjectInteger n) = Right n
+  tryFromObject _ = tryFromObjectError
 
 instance OBJECT Bool where
   toObject = ObjectBool
-  fromObject (ObjectBool b) = Right b
-  fromObject _ = Left fromObjectError
+  tryFromObject (ObjectBool b) = Right b
+  tryFromObject _ = tryFromObjectError
 
 instance OBJECT Double where
   toObject = ObjectDouble
-  fromObject (ObjectDouble d) = Right d
-  fromObject _ = Left fromObjectError
+  tryFromObject (ObjectDouble d) = Right d
+  tryFromObject _ = tryFromObjectError
 
 instance OBJECT B.ByteString where
   toObject = ObjectRAW
-  fromObject (ObjectRAW bs) = Right bs
-  fromObject _ = Left fromObjectError
+  tryFromObject (ObjectRAW bs) = Right bs
+  tryFromObject _ = tryFromObjectError
 
 instance OBJECT String where
   toObject = toObject . C8.pack
-  fromObject obj = liftM C8.unpack $ fromObject obj
+  tryFromObject obj = liftM C8.unpack $ tryFromObject obj
 
 instance OBJECT a => OBJECT [a] where
   toObject = ObjectArray . map toObject
-  fromObject (ObjectArray arr) =
-    mapM fromObject arr
-  fromObject _ =
-    Left fromObjectError
+  tryFromObject (ObjectArray arr) =
+    mapM tryFromObject arr
+  tryFromObject _ =
+    tryFromObjectError
+
+instance (OBJECT a1, OBJECT a2) => OBJECT (a1, a2) where
+  toObject (a1, a2) = ObjectArray [toObject a1, toObject a2]
+  tryFromObject (ObjectArray arr) =
+    case arr of
+      [o1, o2] -> do
+        v1 <- tryFromObject o1
+        v2 <- tryFromObject o2
+        return (v1, v2)
+      _ ->
+        tryFromObjectError
+  tryFromObject _ =
+    tryFromObjectError
+
+instance (OBJECT a1, OBJECT a2, OBJECT a3) => OBJECT (a1, a2, a3) where
+  toObject (a1, a2, a3) = ObjectArray [toObject a1, toObject a2, toObject a3]
+  tryFromObject (ObjectArray arr) =
+    case arr of
+      [o1, o2, o3] -> do
+        v1 <- tryFromObject o1
+        v2 <- tryFromObject o2
+        v3 <- tryFromObject o3
+        return (v1, v2, v3)
+      _ ->
+        tryFromObjectError
+  tryFromObject _ =
+    tryFromObjectError
+
+instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4) => OBJECT (a1, a2, a3, a4) where
+  toObject (a1, a2, a3, a4) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4]
+  tryFromObject (ObjectArray arr) =
+    case arr of
+      [o1, o2, o3, o4] -> do
+        v1 <- tryFromObject o1
+        v2 <- tryFromObject o2
+        v3 <- tryFromObject o3
+        v4 <- tryFromObject o4
+        return (v1, v2, v3, v4)
+      _ ->
+        tryFromObjectError
+  tryFromObject _ =
+    tryFromObjectError
+
+instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4, OBJECT a5) => OBJECT (a1, a2, a3, a4, a5) where
+  toObject (a1, a2, a3, a4, a5) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4, toObject a5]
+  tryFromObject (ObjectArray arr) =
+    case arr of
+      [o1, o2, o3, o4, o5] -> do
+        v1 <- tryFromObject o1
+        v2 <- tryFromObject o2
+        v3 <- tryFromObject o3
+        v4 <- tryFromObject o4
+        v5 <- tryFromObject o5
+        return (v1, v2, v3, v4, v5)
+      _ ->
+        tryFromObjectError
+  tryFromObject _ =
+    tryFromObjectError
+
+instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4, OBJECT a5, OBJECT a6) => OBJECT (a1, a2, a3, a4, a5, a6) where
+  toObject (a1, a2, a3, a4, a5, a6) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4, toObject a5, toObject a6]
+  tryFromObject (ObjectArray arr) =
+    case arr of
+      [o1, o2, o3, o4, o5, o6] -> do
+        v1 <- tryFromObject o1
+        v2 <- tryFromObject o2
+        v3 <- tryFromObject o3
+        v4 <- tryFromObject o4
+        v5 <- tryFromObject o5
+        v6 <- tryFromObject o6
+        return (v1, v2, v3, v4, v5, v6)
+      _ ->
+        tryFromObjectError
+  tryFromObject _ =
+    tryFromObjectError
+
+instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4, OBJECT a5, OBJECT a6, OBJECT a7) => OBJECT (a1, a2, a3, a4, a5, a6, a7) where
+  toObject (a1, a2, a3, a4, a5, a6, a7) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4, toObject a5, toObject a6, toObject a7]
+  tryFromObject (ObjectArray arr) =
+    case arr of
+      [o1, o2, o3, o4, o5, o6, o7] -> do
+        v1 <- tryFromObject o1
+        v2 <- tryFromObject o2
+        v3 <- tryFromObject o3
+        v4 <- tryFromObject o4
+        v5 <- tryFromObject o5
+        v6 <- tryFromObject o6
+        v7 <- tryFromObject o7
+        return (v1, v2, v3, v4, v5, v6, v7)
+      _ ->
+        tryFromObjectError
+  tryFromObject _ =
+    tryFromObjectError
+
+instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4, OBJECT a5, OBJECT a6, OBJECT a7, OBJECT a8) => OBJECT (a1, a2, a3, a4, a5, a6, a7, a8) where
+  toObject (a1, a2, a3, a4, a5, a6, a7, a8) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4, toObject a5, toObject a6, toObject a7, toObject a8]
+  tryFromObject (ObjectArray arr) =
+    case arr of
+      [o1, o2, o3, o4, o5, o6, o7, o8] -> do
+        v1 <- tryFromObject o1
+        v2 <- tryFromObject o2
+        v3 <- tryFromObject o3
+        v4 <- tryFromObject o4
+        v5 <- tryFromObject o5
+        v6 <- tryFromObject o6
+        v7 <- tryFromObject o7
+        v8 <- tryFromObject o8
+        return (v1, v2, v3, v4, v5, v6, v7, v8)
+      _ ->
+        tryFromObjectError
+  tryFromObject _ =
+    tryFromObjectError
+
+instance (OBJECT a1, OBJECT a2, OBJECT a3, OBJECT a4, OBJECT a5, OBJECT a6, OBJECT a7, OBJECT a8, OBJECT a9) => OBJECT (a1, a2, a3, a4, a5, a6, a7, a8, a9) where
+  toObject (a1, a2, a3, a4, a5, a6, a7, a8, a9) = ObjectArray [toObject a1, toObject a2, toObject a3, toObject a4, toObject a5, toObject a6, toObject a7, toObject a8, toObject a9]
+  tryFromObject (ObjectArray arr) =
+    case arr of
+      [o1, o2, o3, o4, o5, o6, o7, o8, o9] -> do
+        v1 <- tryFromObject o1
+        v2 <- tryFromObject o2
+        v3 <- tryFromObject o3
+        v4 <- tryFromObject o4
+        v5 <- tryFromObject o5
+        v6 <- tryFromObject o6
+        v7 <- tryFromObject o7
+        v8 <- tryFromObject o8
+        v9 <- tryFromObject o9
+        return (v1, v2, v3, v4, v5, v6, v7, v8, v9)
+      _ ->
+        tryFromObjectError
+  tryFromObject _ =
+    tryFromObjectError
 
 instance (OBJECT a, OBJECT b) => OBJECT [(a, b)] where
   toObject =
     ObjectMap . map (\(a, b) -> (toObject a, toObject b))
-  fromObject (ObjectMap mem) = do
-    mapM (\(a, b) -> liftM2 (,) (fromObject a) (fromObject b)) mem
-  fromObject _ =
-    Left fromObjectError
+  tryFromObject (ObjectMap mem) = do
+    mapM (\(a, b) -> liftM2 (,) (tryFromObject a) (tryFromObject b)) mem
+  tryFromObject _ =
+    tryFromObjectError
 
 instance OBJECT a => OBJECT (Maybe a) where
   toObject (Just a) = toObject a
   toObject Nothing = ObjectNil
   
-  fromObject ObjectNil = return Nothing
-  fromObject obj = liftM Just $ fromObject obj
+  tryFromObject ObjectNil = return Nothing
+  tryFromObject obj = liftM Just $ tryFromObject obj
diff --git a/haskell/src/Data/MessagePack/Put.hs b/haskell/src/Data/MessagePack/Pack.hs
similarity index 66%
rename from haskell/src/Data/MessagePack/Put.hs
rename to haskell/src/Data/MessagePack/Pack.hs
index 24ec305..16243ad 100644
--- a/haskell/src/Data/MessagePack/Put.hs
+++ b/haskell/src/Data/MessagePack/Pack.hs
@@ -5,7 +5,7 @@
 
 --------------------------------------------------------------------
 -- |
--- Module    : Data.MessagePack.Put
+-- Module    : Data.MessagePack.Pack
 -- Copyright : (c) Hideyuki Tanaka, 2009-2010
 -- License   : BSD3
 --
@@ -13,13 +13,15 @@
 -- Stability :  experimental
 -- Portability: portable
 --
--- MessagePack Serializer using @Data.Binary.Put@
+-- MessagePack Serializer using @Data.Binary.Pack@
 --
 --------------------------------------------------------------------
 
-module Data.MessagePack.Put(
+module Data.MessagePack.Pack (
   -- * Serializable class
-  ObjectPut(..),
+  Packable(..),
+  -- * Simple function to pack a Haskell value
+  pack,
   ) where
 
 import Data.Binary.Put
@@ -30,32 +32,16 @@ import qualified Data.ByteString.Char8 as B8
 import qualified Data.ByteString.Lazy as L
 import qualified Data.Vector as V
 
-import Data.MessagePack.Object
-
 -- | Serializable class
-class ObjectPut a where
+class Packable a where
   -- | Serialize a value
   put :: a -> Put
 
-instance ObjectPut Object where
-  put obj =
-    case obj of
-      ObjectInteger n ->
-        put n
-      ObjectNil ->
-        put ()
-      ObjectBool b ->
-        put b
-      ObjectDouble d ->
-        put d
-      ObjectRAW raw ->
-        put raw
-      ObjectArray arr ->
-        put arr
-      ObjectMap m ->
-        put m
+-- | Pack Haskell data to MessagePack string.
+pack :: Packable a => a -> L.ByteString
+pack = runPut . put
 
-instance ObjectPut Int where
+instance Packable Int where
   put n =
     case n of
       _ | n >= 0 && n <= 127 ->
@@ -87,26 +73,26 @@ instance ObjectPut Int where
         putWord8 0xD3
         putWord64be $ fromIntegral n
 
-instance ObjectPut () where
+instance Packable () where
   put _ = 
     putWord8 0xC0
 
-instance ObjectPut Bool where
+instance Packable Bool where
   put True = putWord8 0xC3
   put False = putWord8 0xC2
 
-instance ObjectPut Double where
+instance Packable Double where
   put d = do
     putWord8 0xCB
     putFloat64be d
 
-instance ObjectPut String where
+instance Packable String where
   put = putString length (putByteString . B8.pack)
 
-instance ObjectPut B.ByteString where
+instance Packable B.ByteString where
   put = putString B.length putByteString
 
-instance ObjectPut L.ByteString where
+instance Packable L.ByteString where
   put = putString (fromIntegral . L.length) putLazyByteString
 
 putString :: (s -> Int) -> (s -> Put) -> s -> Put
@@ -122,41 +108,41 @@ putString lf pf str = do
       putWord32be $ fromIntegral len
   pf str
 
-instance ObjectPut a => ObjectPut [a] where
+instance Packable a => Packable [a] where
   put = putArray length (mapM_ put)
 
-instance ObjectPut a => ObjectPut (V.Vector a) where
+instance Packable a => Packable (V.Vector a) where
   put = putArray V.length (V.mapM_ put)
 
-instance (ObjectPut a1, ObjectPut a2) => ObjectPut (a1, a2) where
+instance (Packable a1, Packable a2) => Packable (a1, a2) where
   put = putArray (const 2) f where
     f (a1, a2) = put a1 >> put a2
 
-instance (ObjectPut a1, ObjectPut a2, ObjectPut a3) => ObjectPut (a1, a2, a3) where
+instance (Packable a1, Packable a2, Packable a3) => Packable (a1, a2, a3) where
   put = putArray (const 3) f where
     f (a1, a2, a3) = put a1 >> put a2 >> put a3
 
-instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4) => ObjectPut (a1, a2, a3, a4) where
+instance (Packable a1, Packable a2, Packable a3, Packable a4) => Packable (a1, a2, a3, a4) where
   put = putArray (const 4) f where
     f (a1, a2, a3, a4) = put a1 >> put a2 >> put a3 >> put a4
 
-instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5) => ObjectPut (a1, a2, a3, a4, a5) where
+instance (Packable a1, Packable a2, Packable a3, Packable a4, Packable a5) => Packable (a1, a2, a3, a4, a5) where
   put = putArray (const 5) f where
     f (a1, a2, a3, a4, a5) = put a1 >> put a2 >> put a3 >> put a4 >> put a5
 
-instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6) => ObjectPut (a1, a2, a3, a4, a5, a6) where
+instance (Packable a1, Packable a2, Packable a3, Packable a4, Packable a5, Packable a6) => Packable (a1, a2, a3, a4, a5, a6) where
   put = putArray (const 6) f where
     f (a1, a2, a3, a4, a5, a6) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6
 
-instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7) => ObjectPut (a1, a2, a3, a4, a5, a6, a7) where
+instance (Packable a1, Packable a2, Packable a3, Packable a4, Packable a5, Packable a6, Packable a7) => Packable (a1, a2, a3, a4, a5, a6, a7) where
   put = putArray (const 7) f where
     f (a1, a2, a3, a4, a5, a6, a7) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7
 
-instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7, ObjectPut a8) => ObjectPut (a1, a2, a3, a4, a5, a6, a7, a8) where
+instance (Packable a1, Packable a2, Packable a3, Packable a4, Packable a5, Packable a6, Packable a7, Packable a8) => Packable (a1, a2, a3, a4, a5, a6, a7, a8) where
   put = putArray (const 8) f where
     f (a1, a2, a3, a4, a5, a6, a7, a8) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7 >> put a8
 
-instance (ObjectPut a1, ObjectPut a2, ObjectPut a3, ObjectPut a4, ObjectPut a5, ObjectPut a6, ObjectPut a7, ObjectPut a8, ObjectPut a9) => ObjectPut (a1, a2, a3, a4, a5, a6, a7, a8, a9) where
+instance (Packable a1, Packable a2, Packable a3, Packable a4, Packable a5, Packable a6, Packable a7, Packable a8, Packable a9) => Packable (a1, a2, a3, a4, a5, a6, a7, a8, a9) where
   put = putArray (const 9) f where
     f (a1, a2, a3, a4, a5, a6, a7, a8, a9) = put a1 >> put a2 >> put a3 >> put a4 >> put a5 >> put a6 >> put a7 >> put a8 >> put a9
 
@@ -173,13 +159,13 @@ putArray lf pf arr = do
       putWord32be $ fromIntegral len
   pf arr
 
-instance (ObjectPut k, ObjectPut v) => ObjectPut [(k, v)] where
+instance (Packable k, Packable v) => Packable [(k, v)] where
   put = putMap length (mapM_ putPair)
 
-instance (ObjectPut k, ObjectPut v) => ObjectPut (V.Vector (k, v)) where
+instance (Packable k, Packable v) => Packable (V.Vector (k, v)) where
   put = putMap V.length (V.mapM_ putPair)
 
-putPair :: (ObjectPut a, ObjectPut b) => (a, b) -> Put
+putPair :: (Packable a, Packable b) => (a, b) -> Put
 putPair (a, b) = put a >> put b
 
 putMap :: (a -> Int) -> (a -> Put) -> a -> Put
@@ -194,3 +180,7 @@ putMap lf pf m = do
       putWord8 0xDF
       putWord32be $ fromIntegral len
   pf m
+
+instance Packable a => Packable (Maybe a) where
+  put Nothing = put ()
+  put (Just a) = put a
diff --git a/haskell/src/Data/MessagePack/Parser.hs b/haskell/src/Data/MessagePack/Unpack.hs
similarity index 70%
rename from haskell/src/Data/MessagePack/Parser.hs
rename to haskell/src/Data/MessagePack/Unpack.hs
index 200ad96..a0d618e 100644
--- a/haskell/src/Data/MessagePack/Parser.hs
+++ b/haskell/src/Data/MessagePack/Unpack.hs
@@ -2,10 +2,11 @@
 {-# Language IncoherentInstances #-}
 {-# Language OverlappingInstances #-}
 {-# Language TypeSynonymInstances #-}
+{-# Language DeriveDataTypeable #-}
 
 --------------------------------------------------------------------
 -- |
--- Module    : Data.MessagePack.Parser
+-- Module    : Data.MessagePack.Unpack
 -- Copyright : (c) Hideyuki Tanaka, 2009-2010
 -- License   : BSD3
 --
@@ -17,11 +18,19 @@
 --
 --------------------------------------------------------------------
 
-module Data.MessagePack.Parser(
+module Data.MessagePack.Unpack(
   -- * MessagePack deserializer
-  ObjectGet(..),
+  Unpackable(..),
+  -- * Simple function to unpack a Haskell value
+  unpack,
+  tryUnpack,
+  -- * Unpack exception
+  UnpackError(..),
+  -- * ByteString utils
+  IsByteString(..),
   ) where
 
+import Control.Exception
 import Control.Monad
 import qualified Data.Attoparsec as A
 import Data.Binary.Get
@@ -31,30 +40,53 @@ import qualified Data.ByteString as B
 import qualified Data.ByteString.Char8 as B8
 import qualified Data.ByteString.Lazy as L
 import Data.Int
+import Data.Typeable
 import qualified Data.Vector as V
 import Data.Word
 import Text.Printf
 
-import Data.MessagePack.Object
-
 -- | Deserializable class
-class ObjectGet a where
+class Unpackable a where
   -- | Deserialize a value
   get :: A.Parser a
 
-instance ObjectGet Object where
-  get =
-    A.choice
-    [ liftM ObjectInteger get
-    , liftM (\() -> ObjectNil) get
-    , liftM ObjectBool get
-    , liftM ObjectDouble get
-    , liftM ObjectRAW get
-    , liftM ObjectArray get
-    , liftM ObjectMap get
-    ]
+class IsByteString s where
+  toBS :: s -> B.ByteString
 
-instance ObjectGet Int where
+instance IsByteString B.ByteString where
+  toBS = id
+
+instance IsByteString L.ByteString where
+  toBS = B.concat . L.toChunks
+
+-- | The exception of unpack
+data UnpackError =
+  UnpackError String
+  deriving (Show, Typeable)
+
+instance Exception UnpackError
+
+-- | Unpack MessagePack string to Haskell data.
+unpack :: (Unpackable a, IsByteString s) => s -> a
+unpack bs =
+  case tryUnpack bs of
+    Left err ->
+      throw $ UnpackError err
+    Right ret ->
+      ret
+
+-- | Unpack MessagePack string to Haskell data.
+tryUnpack :: (Unpackable a, IsByteString s) => s -> Either String a
+tryUnpack bs =
+  case A.parse get (toBS bs) of
+    A.Fail _ _ err ->
+      Left err
+    A.Partial _ ->
+      Left "not enough input"
+    A.Done _ ret ->
+      Right ret
+
+instance Unpackable Int where
   get = do
     c <- A.anyWord8
     case c of
@@ -81,7 +113,7 @@ instance ObjectGet Int where
       _ ->
         fail $ printf "invlid integer tag: 0x%02X" c
 
-instance ObjectGet () where
+instance Unpackable () where
   get = do
     c <- A.anyWord8
     case c of
@@ -90,7 +122,7 @@ instance ObjectGet () where
       _ ->
         fail $ printf "invlid nil tag: 0x%02X" c
 
-instance ObjectGet Bool where
+instance Unpackable Bool where
   get = do
     c <- A.anyWord8
     case c of
@@ -101,7 +133,7 @@ instance ObjectGet Bool where
       _ ->
         fail $ printf "invlid bool tag: 0x%02X" c
 
-instance ObjectGet Double where
+instance Unpackable Double where
   get = do
     c <- A.anyWord8
     case c of
@@ -112,13 +144,13 @@ instance ObjectGet Double where
       _ ->
         fail $ printf "invlid double tag: 0x%02X" c
 
-instance ObjectGet String where
+instance Unpackable String where
   get = parseString (\n -> return . B8.unpack =<< A.take n)
 
-instance ObjectGet B.ByteString where
+instance Unpackable B.ByteString where
   get = parseString A.take
 
-instance ObjectGet L.ByteString where
+instance Unpackable L.ByteString where
   get = parseString (\n -> do bs <- A.take n; return $ L.fromChunks [bs])
 
 parseString :: (Int -> A.Parser a) -> A.Parser a
@@ -134,48 +166,48 @@ parseString aget = do
     _ ->
       fail $ printf "invlid raw tag: 0x%02X" c
 
-instance ObjectGet a => ObjectGet [a] where
+instance Unpackable a => Unpackable [a] where
   get = parseArray (flip replicateM get)
 
-instance ObjectGet a => ObjectGet (V.Vector a) where
+instance Unpackable a => Unpackable (V.Vector a) where
   get = parseArray (flip V.replicateM get)
 
-instance (ObjectGet a1, ObjectGet a2) => ObjectGet (a1, a2) where
+instance (Unpackable a1, Unpackable a2) => Unpackable (a1, a2) where
   get = parseArray f where
     f 2 = get >>= \a1 -> get >>= \a2 -> return (a1, a2)
     f n = fail $ printf "wrong tupple size: expected 2 but got " n
 
-instance (ObjectGet a1, ObjectGet a2, ObjectGet a3) => ObjectGet (a1, a2, a3) where
+instance (Unpackable a1, Unpackable a2, Unpackable a3) => Unpackable (a1, a2, a3) where
   get = parseArray f where
     f 3 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> return (a1, a2, a3)
     f n = fail $ printf "wrong tupple size: expected 3 but got " n
 
-instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4) => ObjectGet (a1, a2, a3, a4) where
+instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4) => Unpackable (a1, a2, a3, a4) where
   get = parseArray f where
     f 4 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> return (a1, a2, a3, a4)
     f n = fail $ printf "wrong tupple size: expected 4 but got " n
 
-instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5) => ObjectGet (a1, a2, a3, a4, a5) where
+instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4, Unpackable a5) => Unpackable (a1, a2, a3, a4, a5) where
   get = parseArray f where
     f 5 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> return (a1, a2, a3, a4, a5)
     f n = fail $ printf "wrong tupple size: expected 5 but got " n
 
-instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6) => ObjectGet (a1, a2, a3, a4, a5, a6) where
+instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4, Unpackable a5, Unpackable a6) => Unpackable (a1, a2, a3, a4, a5, a6) where
   get = parseArray f where
     f 6 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> return (a1, a2, a3, a4, a5, a6)
     f n = fail $ printf "wrong tupple size: expected 6 but got " n
 
-instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7) => ObjectGet (a1, a2, a3, a4, a5, a6, a7) where
+instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4, Unpackable a5, Unpackable a6, Unpackable a7) => Unpackable (a1, a2, a3, a4, a5, a6, a7) where
   get = parseArray f where
     f 7 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> return (a1, a2, a3, a4, a5, a6, a7)
     f n = fail $ printf "wrong tupple size: expected 7 but got " n
 
-instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7, ObjectGet a8) => ObjectGet (a1, a2, a3, a4, a5, a6, a7, a8) where
+instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4, Unpackable a5, Unpackable a6, Unpackable a7, Unpackable a8) => Unpackable (a1, a2, a3, a4, a5, a6, a7, a8) where
   get = parseArray f where
     f 8 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> get >>= \a8 -> return (a1, a2, a3, a4, a5, a6, a7, a8)
     f n = fail $ printf "wrong tupple size: expected 8 but got " n
 
-instance (ObjectGet a1, ObjectGet a2, ObjectGet a3, ObjectGet a4, ObjectGet a5, ObjectGet a6, ObjectGet a7, ObjectGet a8, ObjectGet a9) => ObjectGet (a1, a2, a3, a4, a5, a6, a7, a8, a9) where
+instance (Unpackable a1, Unpackable a2, Unpackable a3, Unpackable a4, Unpackable a5, Unpackable a6, Unpackable a7, Unpackable a8, Unpackable a9) => Unpackable (a1, a2, a3, a4, a5, a6, a7, a8, a9) where
   get = parseArray f where
     f 9 = get >>= \a1 -> get >>= \a2 -> get >>= \a3 -> get >>= \a4 -> get >>= \a5 -> get >>= \a6 -> get >>= \a7 -> get >>= \a8 -> get >>= \a9 -> return (a1, a2, a3, a4, a5, a6, a7, a8, a9)
     f n = fail $ printf "wrong tupple size: expected 9 but got " n
@@ -193,13 +225,13 @@ parseArray aget = do
     _ ->
       fail $ printf "invlid array tag: 0x%02X" c
 
-instance (ObjectGet k, ObjectGet v) => ObjectGet [(k, v)] where
+instance (Unpackable k, Unpackable v) => Unpackable [(k, v)] where
   get = parseMap (flip replicateM parsePair)
 
-instance (ObjectGet k, ObjectGet v) => ObjectGet (V.Vector (k, v)) where
+instance (Unpackable k, Unpackable v) => Unpackable (V.Vector (k, v)) where
   get = parseMap (flip V.replicateM parsePair)
 
-parsePair :: (ObjectGet k, ObjectGet v) => A.Parser (k, v)
+parsePair :: (Unpackable k, Unpackable v) => A.Parser (k, v)
 parsePair = do
   a <- get
   b <- get
@@ -218,6 +250,12 @@ parseMap aget = do
     _ ->
       fail $ printf "invlid map tag: 0x%02X" c
 
+instance Unpackable a => Unpackable (Maybe a) where
+  get = 
+    A.choice
+    [ liftM Just get
+    , liftM (\() -> Nothing) get ]
+
 parseUint16 :: A.Parser Word16
 parseUint16 = do
   b0 <- A.anyWord8
diff --git a/haskell/test/UserData.hs b/haskell/test/UserData.hs
new file mode 100644
index 0000000..8aced13
--- /dev/null
+++ b/haskell/test/UserData.hs
@@ -0,0 +1,19 @@
+{-# Language TemplateHaskell #-}
+
+import Data.MessagePack
+import Data.MessagePack.Derive
+
+data T
+  = A Int String
+  | B Double
+  deriving (Show)
+
+$(deriveObject ''T)
+
+main = do
+  let bs = pack $ A 123 "hoge"
+  print bs
+  print (unpack bs :: T)
+  let cs = pack $ B 3.14
+  print cs
+  print (unpack cs :: T)

From 13b6708a099c1e3a6f498f9f4899b6b139c112d4 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 23 Sep 2010 15:24:12 +0900
Subject: [PATCH 0792/1648] java: append a code for generating a messageConvert
 method to annotation-utilities

---
 .../util/annotation/PackUnpackUtil.java       | 882 ++++++++++++++++--
 .../annotation/TestMessagePackUnpackable.java | 243 ++++-
 2 files changed, 1045 insertions(+), 80 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
index 5238444..a337ff9 100644
--- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
+++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
@@ -11,6 +11,7 @@ import java.lang.reflect.ParameterizedType;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -25,6 +26,7 @@ import javassist.CtNewMethod;
 import javassist.NotFoundException;
 
 import org.msgpack.MessageConvertable;
+import org.msgpack.MessagePackObject;
 import org.msgpack.MessagePackable;
 import org.msgpack.MessageTypeException;
 import org.msgpack.MessageUnpackable;
@@ -43,37 +45,6 @@ public class PackUnpackUtil {
 
 		static final String KEYWORD_NEW = "new";
 
-		static final String TYPE_NAME_VOID = void.class.getName();
-
-		static final String TYPE_NAME_OBJECT = Object.class.getName();
-
-		static final String TYPE_NAME_IOEXCEPTION = IOException.class.getName();
-
-		static final String TYPE_NAME_PACKER = Packer.class.getName();
-
-		static final String TYPE_NAME_UNPACKER = Unpacker.class.getName();
-
-		static final String TYPE_NAME_MSGPACKABLE = MessagePackable.class
-				.getName();
-
-		static final String TYPE_NAME_MSGUNPACKABLE = MessageUnpackable.class
-				.getName();
-
-		static final String TYPE_NAME_MSGCONVERTABLE = MessageConvertable.class
-				.getName();
-
-		static final String TYPE_NAME_MSGTYPEEXCEPTION = MessageTypeException.class
-				.getName();
-
-		static final String TYPE_NAME_MSGPACKUNPACKABLE = MessagePackUnpackable.class
-				.getName();
-
-		static final String TYPE_NAME_MSGPACKOPTIONAL = MessagePackOptional.class
-				.getName();
-
-		static final String TYPE_NAME_MSGPACKOREQUIRED = MessagePackRequired.class
-				.getName();
-
 		static final String CHAR_NAME_SPACE = " ";
 
 		static final String CHAR_NAME_COMMA = ",";
@@ -92,15 +63,27 @@ public class PackUnpackUtil {
 
 		static final String CHAR_NAME_LEFT_CURLY_BRACHET = "{";
 
+		static final String CHAR_NAME_RIGHT_SQUARE_BRACKET = "]";
+
+		static final String CHAR_NAME_LEFT_SQUARE_BRACKET = "[";
+
 		static final String CHAR_NAME_DOT = ".";
 
 		static final String CHAR_NAME_SEMICOLON = ";";
 
-		static final String VARIABLE_NAME_PK = "pk";
+		static final String VARIABLE_NAME_PK = "_$$_pk";
 
-		static final String VARIABLE_NAME_OBJ = "obj";
+		static final String VARIABLE_NAME_SIZE = "_$$_len";
 
-		static final String VARIABLE_NAME_SIZE = "len";
+		static final String VARIABLE_NAME_ARRAY = "_$$_ary";
+
+		static final String VARIABLE_NAME_LIST = "_$$_list";
+
+		static final String VARIABLE_NAME_MAP = "_$$_map";
+
+		static final String VARIABLE_NAME_ITER = "_$$_iter";
+
+		static final String VARIABLE_NAME_MPO1 = "_$$_mpo1";
 
 		static final String VARIABLE_NAME_I = "i";
 
@@ -110,6 +93,18 @@ public class PackUnpackUtil {
 
 		static final String METHOD_NAME_PUT = "put";
 
+		static final String METHOD_NAME_GET = "get";
+
+		static final String METHOD_NAME_SIZE = "size";
+
+		static final String METHOD_NAME_KEYSET = "keySet";
+
+		static final String METHOD_NAME_ITERATOR = "iterator";
+
+		static final String METHOD_NAME_HASNEXT = "hasNext";
+
+		static final String METHOD_NAME_NEXT = "next";
+
 		static final String METHOD_NAME_MSGPACK = "messagePack";
 
 		static final String METHOD_NAME_MSGUNPACK = "messageUnpack";
@@ -147,6 +142,8 @@ public class PackUnpackUtil {
 		static final String METHOD_NAME_UNPACKARRAY = "unpackArray";
 
 		static final String METHOD_NAME_UNPACKMAP = "unpackMap";
+
+		static final String METHOD_NAME_ASARRAY = "asArray";
 	}
 
 	public static class Enhancer {
@@ -180,9 +177,9 @@ public class PackUnpackUtil {
 			setInterfaces(enhCtClass);
 			addConstructor(enhCtClass);
 			Field[] fields = getDeclaredFields(origClass);
-			addMessagePackMethod(enhCtClass, origCtClass, fields);
-			addMessageUnpackMethod(enhCtClass, origCtClass, fields);
-			addMessageConvertMethod(enhCtClass, origCtClass, fields);
+			addMessagePackMethod(enhCtClass, fields);
+			addMessageUnpackMethod(enhCtClass, fields);
+			addMessageConvertMethod(enhCtClass, fields);
 			return createClass(enhCtClass);
 		}
 
@@ -243,11 +240,11 @@ public class PackUnpackUtil {
 		}
 
 		private void setInterfaces(CtClass enhCtClass) throws NotFoundException {
-			CtClass pacCtClass = pool.get(Constants.TYPE_NAME_MSGPACKABLE);
+			CtClass pacCtClass = pool.get(MessagePackable.class.getName());
 			enhCtClass.addInterface(pacCtClass);
-			CtClass unpacCtClass = pool.get(Constants.TYPE_NAME_MSGUNPACKABLE);
+			CtClass unpacCtClass = pool.get(MessageUnpackable.class.getName());
 			enhCtClass.addInterface(unpacCtClass);
-			CtClass convCtClass = pool.get(Constants.TYPE_NAME_MSGCONVERTABLE);
+			CtClass convCtClass = pool.get(MessageConvertable.class.getName());
 			enhCtClass.addInterface(convCtClass);
 		}
 
@@ -296,24 +293,23 @@ public class PackUnpackUtil {
 					+ field.getName());
 		}
 
-		private void addMessagePackMethod(CtClass enhCtClass,
-				CtClass origCtClass, Field[] fields)
+		private void addMessagePackMethod(CtClass enhCtClass, Field[] fields)
 				throws CannotCompileException, NotFoundException {
 			StringBuilder sb = new StringBuilder();
 			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC);
 			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.TYPE_NAME_VOID);
+			sb.append(void.class.getName());
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.METHOD_NAME_MSGPACK);
 			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.TYPE_NAME_PACKER);
+			sb.append(Packer.class.getName());
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.VARIABLE_NAME_PK);
 			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.KEYWORD_THROWS);
 			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.TYPE_NAME_IOEXCEPTION);
+			sb.append(IOException.class.getName());
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
 			sb.append(Constants.CHAR_NAME_SPACE);
@@ -329,7 +325,7 @@ public class PackUnpackUtil {
 				insertCodeOfMessagePack(sb, field);
 			}
 			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			//System.out.println("messagePack method: " + sb.toString());
+			// System.out.println("messagePack method: " + sb.toString());
 			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
 			enhCtClass.addMethod(newCtMethod);
 		}
@@ -345,27 +341,26 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_SPACE);
 		}
 
-		private void addMessageUnpackMethod(CtClass enhCtClass,
-				CtClass origCtClass, Field[] fields)
+		private void addMessageUnpackMethod(CtClass enhCtClass, Field[] fields)
 				throws CannotCompileException, NotFoundException {
 			StringBuilder sb = new StringBuilder();
 			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC);
 			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.TYPE_NAME_VOID);
+			sb.append(void.class.getName());
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.METHOD_NAME_MSGUNPACK);
 			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.TYPE_NAME_UNPACKER);
+			sb.append(Unpacker.class.getName());
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.VARIABLE_NAME_PK);
 			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.KEYWORD_THROWS);
 			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.TYPE_NAME_MSGTYPEEXCEPTION);
+			sb.append(MessageTypeException.class.getName());
 			sb.append(Constants.CHAR_NAME_COMMA);
 			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.TYPE_NAME_IOEXCEPTION);
+			sb.append(IOException.class.getName());
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
 			sb.append(Constants.CHAR_NAME_SPACE);
@@ -380,7 +375,7 @@ public class PackUnpackUtil {
 				insertCodeOfMessageUnpack(sb, field, field.getType());
 			}
 			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			//System.out.println("messageUnpack method: " + sb.toString());
+			// System.out.println("messageUnpack method: " + sb.toString());
 			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
 			enhCtClass.addMethod(newCtMethod);
 		}
@@ -680,33 +675,761 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_SPACE);
 		}
 
-		private void addMessageConvertMethod(CtClass enhCtClass,
-				CtClass origCtClass, Field[] fields)
+		private void addMessageConvertMethod(CtClass enhCtClass, Field[] fields)
 				throws CannotCompileException {
+			// messageConvert(MessagePackObject obj) throws MessageTypeException
 			StringBuilder sb = new StringBuilder();
-			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC).append(
-					Constants.CHAR_NAME_SPACE).append(Constants.TYPE_NAME_VOID)
-					.append(Constants.CHAR_NAME_SPACE).append(
-							Constants.METHOD_NAME_MSGCONVERT).append(
-							Constants.CHAR_NAME_LEFT_PARENTHESIS).append(
-							Constants.TYPE_NAME_OBJECT).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.VARIABLE_NAME_OBJ).append(
-							Constants.CHAR_NAME_RIGHT_PARENTHESIS).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.KEYWORD_THROWS).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.TYPE_NAME_MSGTYPEEXCEPTION).append(
-							Constants.CHAR_NAME_SPACE).append(
-							Constants.CHAR_NAME_LEFT_CURLY_BRACHET).append(
-							Constants.CHAR_NAME_SPACE);
-			// TODO
+			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(void.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.METHOD_NAME_MSGCONVERT);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(MessagePackObject.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_MPO1);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.KEYWORD_THROWS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(MessageTypeException.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			insertCodeOfMessagePackObjectArrayGet(sb);
+			insertCodeOfMesageConvertCalls(sb, fields);
 			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
 			//System.out.println("messageConvert method: " + sb.toString());
 			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
 			enhCtClass.addMethod(newCtMethod);
 		}
 
+		private void insertCodeOfMessagePackObjectArrayGet(StringBuilder sb) {
+			// MessagePackObject[] ary = obj.asArray();
+			sb.append(MessagePackObject.class.getName());
+			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_ARRAY);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_MPO1);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_ASARRAY);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+		}
+
+		private void insertCodeOfMesageConvertCalls(StringBuilder sb,
+				Field[] fields) {
+			for (int i = 0; i < fields.length; ++i) {
+				insertCodeOfMessageConvertCall(sb, fields[i], fields[i]
+						.getType(), i, null);
+			}
+		}
+
+		private void insertCodeOfMessageConvertCall(StringBuilder sb, Field f,
+				Class c, int i, String name) {
+			if (c.isPrimitive()) { // primitive type
+				// f0 = objs[0].intValue();
+				if (c.equals(boolean.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asBoolean");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(byte.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asByte");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(short.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asShort");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(int.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asInt");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(long.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asLong");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(float.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asFloat");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(double.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asDouble");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else {
+					throw new PackUnpackUtilException("fatal error: "
+							+ c.getName());
+				}
+			} else { // reference type
+				if (c.equals(Boolean.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+					sb.append(Constants.KEYWORD_NEW);
+					sb.append(Constants.CHAR_NAME_SPACE);
+					sb.append(Boolean.class.getName());
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asBoolean");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(Byte.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+					sb.append(Constants.KEYWORD_NEW);
+					sb.append(Constants.CHAR_NAME_SPACE);
+					sb.append(Byte.class.getName());
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asByte");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(Short.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+					sb.append(Constants.KEYWORD_NEW);
+					sb.append(Constants.CHAR_NAME_SPACE);
+					sb.append(Short.class.getName());
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asShort");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(Integer.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+					sb.append(Constants.KEYWORD_NEW);
+					sb.append(Constants.CHAR_NAME_SPACE);
+					sb.append(Integer.class.getName());
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asInt");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(Long.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+					sb.append(Constants.KEYWORD_NEW);
+					sb.append(Constants.CHAR_NAME_SPACE);
+					sb.append(Long.class.getName());
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asLong");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(Float.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+					sb.append(Constants.KEYWORD_NEW);
+					sb.append(Constants.CHAR_NAME_SPACE);
+					sb.append(Float.class.getName());
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asFloat");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else if (c.equals(Double.class)) {
+					if (f != null) {
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_SPACE);
+						sb.append(Constants.CHAR_NAME_EQUAL);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+					sb.append(Constants.KEYWORD_NEW);
+					sb.append(Constants.CHAR_NAME_SPACE);
+					sb.append(Double.class.getName());
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+					} else {
+						sb.append(name);
+					}
+					sb.append(Constants.CHAR_NAME_DOT);
+					sb.append("asDouble");
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					if (f != null) {
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					}
+				} else {
+					if (c.equals(String.class)) {
+						if (f != null) {
+							sb.append(f.getName());
+							sb.append(Constants.CHAR_NAME_SPACE);
+							sb.append(Constants.CHAR_NAME_EQUAL);
+							sb.append(Constants.CHAR_NAME_SPACE);
+							sb.append(Constants.VARIABLE_NAME_ARRAY);
+							sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+							sb.append(i);
+							sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+						} else {
+							sb.append(name);
+						}
+						sb.append(Constants.CHAR_NAME_DOT);
+						sb.append("asString");
+						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+						if (f != null) {
+							sb.append(Constants.CHAR_NAME_SEMICOLON);
+							sb.append(Constants.CHAR_NAME_SPACE);
+						}
+					} else if (c.equals(byte[].class)) {
+						if (f != null) {
+							sb.append(f.getName());
+							sb.append(Constants.CHAR_NAME_SPACE);
+							sb.append(Constants.CHAR_NAME_EQUAL);
+							sb.append(Constants.CHAR_NAME_SPACE);
+							sb.append(Constants.VARIABLE_NAME_ARRAY);
+							sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+							sb.append(i);
+							sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+						} else {
+							sb.append(name);
+						}
+						sb.append(Constants.CHAR_NAME_DOT);
+						sb.append("asByteArray");
+						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+						if (f != null) {
+							sb.append(Constants.CHAR_NAME_SEMICOLON);
+							sb.append(Constants.CHAR_NAME_SPACE);
+						}
+					} else if (c.equals(BigInteger.class)) {
+						if (f != null) {
+							sb.append(f.getName());
+							sb.append(Constants.CHAR_NAME_SPACE);
+							sb.append(Constants.CHAR_NAME_EQUAL);
+							sb.append(Constants.CHAR_NAME_SPACE);
+							sb.append(Constants.VARIABLE_NAME_ARRAY);
+							sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+							sb.append(i);
+							sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+						} else {
+							sb.append(name);
+						}
+						sb.append(Constants.CHAR_NAME_DOT);
+						sb.append("asBigInteger");
+						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+						if (f != null) {
+							sb.append(Constants.CHAR_NAME_SEMICOLON);
+							sb.append(Constants.CHAR_NAME_SPACE);
+						}
+					} else if (List.class.isAssignableFrom(c)) {
+						insertCodeOfMessageConvertCallForList(sb, f, c, i);
+					} else if (Map.class.isAssignableFrom(c)) {
+						insertCodeOfMessageConveretCallForMap(sb, f, c, i);
+					} else if (MessageConvertable.class.isAssignableFrom(c)
+							|| (getCache(c.getName()) != null)) {
+						// TODO
+						// TODO
+						// TODO
+						// ((MessageConvertable)f_i).messageConvert(ary[i]);
+						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+						sb.append(MessageConvertable.class.getName());
+						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+						sb.append(f.getName());
+						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+						sb.append(Constants.CHAR_NAME_DOT);
+						sb.append(Constants.METHOD_NAME_MSGCONVERT);
+						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+						sb.append(Constants.VARIABLE_NAME_ARRAY);
+						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+						sb.append(i);
+						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+						sb.append(Constants.CHAR_NAME_SEMICOLON);
+						sb.append(Constants.CHAR_NAME_SPACE);
+					} else {
+						throw new MessageTypeException("Type error: "
+								+ c.getName());
+					}
+				}
+			}
+		}
+
+		private void insertCodeOfMessageConvertCallForList(StringBuilder sb,
+				Field field, Class type, int i) {
+			ParameterizedType generic = (ParameterizedType) field
+					.getGenericType();
+			Class genericType = (Class) generic.getActualTypeArguments()[0];
+
+			// List list = ary[i].asList();
+			sb.append(List.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_LIST);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_ARRAY);
+			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(i);
+			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append("asList");
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// int size = list.size();
+			sb.append(int.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_SIZE);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_LIST);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_SIZE);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// field initializer
+			sb.append(field.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.KEYWORD_NEW);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(ArrayList.class.getName());
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// for loop
+			sb.append(Constants.KEYWORD_FOR);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(int.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_I);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(0);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_I);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LESSTHAN);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_SIZE);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_PLUS);
+			sb.append(Constants.CHAR_NAME_PLUS);
+			sb.append(Constants.VARIABLE_NAME_I);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// block begin
+			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(MessagePackObject.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append("_$$_val");
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(MessagePackObject.class.getName());
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.VARIABLE_NAME_LIST);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_GET);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.VARIABLE_NAME_I);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			sb.append(field.getName());
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_ADD);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			insertCodeOfMessageConvertCall(sb, null, genericType, -1, "_$$_val");
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			// block end
+			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+		}
+
+		private void insertCodeOfMessageConveretCallForMap(StringBuilder sb,
+				Field f, Class c, int i) {
+			ParameterizedType generic = (ParameterizedType) f.getGenericType();
+			Class genericType0 = (Class) generic.getActualTypeArguments()[0];
+			Class genericType1 = (Class) generic.getActualTypeArguments()[1];
+
+			// Map map = ary[i].asMap();
+			sb.append(Map.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_MAP);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_ARRAY);
+			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(i);
+			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append("asMap");
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// int size = list.size();
+			sb.append(int.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_SIZE);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_MAP);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_SIZE);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// field initializer
+			sb.append(f.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.KEYWORD_NEW);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(HashMap.class.getName());
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// for loop
+			sb.append(Constants.KEYWORD_FOR);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Iterator.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_ITER);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_MAP);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_KEYSET);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_ITERATOR);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_ITER);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_HASNEXT);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// block map.
+			sb.append(MessagePackObject.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append("_$$_key");
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(MessagePackObject.class.getName());
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.VARIABLE_NAME_ITER);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_NEXT);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(MessagePackObject.class.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append("_$$_val");
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(MessagePackObject.class.getName());
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.VARIABLE_NAME_MAP);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_GET);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append("_$$_key");
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			sb.append(f.getName());
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_PUT);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			insertCodeOfMessageConvertCall(sb, null, genericType0, -1,
+					"_$$_key");
+			sb.append(Constants.CHAR_NAME_COMMA);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			insertCodeOfMessageConvertCall(sb, null, genericType1, -1,
+					"_$$_val");
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+		}
+
 		private Class createClass(CtClass enhCtClass)
 				throws CannotCompileException {
 			return enhCtClass.toClass(null, null);
@@ -749,6 +1472,19 @@ public class PackUnpackUtil {
 		}
 	}
 
+	public static Object initEnhancedInstance(MessagePackObject obj,
+			Class origClass) {
+		Object ret = newEnhancedInstance(origClass);
+		((MessageConvertable) ret).messageConvert(obj);
+		return ret;
+	}
+
+	public static Object initEnhancedInstance(MessagePackObject obj,
+			Object origObj) {
+		((MessageConvertable) origObj).messageConvert(obj);
+		return origObj;
+	}
+
 	@MessagePackUnpackable
 	public static class Image {
 		public String uri = "";
@@ -789,6 +1525,6 @@ public class PackUnpackUtil {
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Unpacker pac = new Unpacker(in);
 		pac.unpack((MessageUnpackable) dst);
-		//System.out.println(src.equals(dst));
+		// System.out.println(src.equals(dst));
 	}
 }
diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
index dbea7cb..4b1e666 100644
--- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
+++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
@@ -12,6 +12,7 @@ import java.util.Map;
 import junit.framework.TestCase;
 
 import org.junit.Test;
+import org.msgpack.MessagePackObject;
 import org.msgpack.MessageUnpackable;
 import org.msgpack.Packer;
 import org.msgpack.Unpacker;
@@ -19,7 +20,7 @@ import org.msgpack.Unpacker;
 public class TestMessagePackUnpackable extends TestCase {
 
 	@Test
-	public void testPrimitiveTypeFields() throws Exception {
+	public void testPrimitiveTypeFields01() throws Exception {
 		PrimitiveTypeFieldsClass src = (PrimitiveTypeFieldsClass) PackUnpackUtil
 				.newEnhancedInstance(PrimitiveTypeFieldsClass.class);
 		src.f0 = (byte) 0;
@@ -45,6 +46,36 @@ public class TestMessagePackUnpackable extends TestCase {
 		assertEquals(src.f6, dst.f6);
 	}
 
+	@Test
+	public void testPrimitiveTypeFields02() throws Exception {
+		PrimitiveTypeFieldsClass src = (PrimitiveTypeFieldsClass) PackUnpackUtil
+				.newEnhancedInstance(PrimitiveTypeFieldsClass.class);
+		src.f0 = (byte) 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f6 = false;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) PackUnpackUtil
+				.initEnhancedInstance(mpo, PrimitiveTypeFieldsClass.class);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertFalse(it.hasNext());
+	}
+
 	@MessagePackUnpackable
 	public static class PrimitiveTypeFieldsClass {
 		public byte f0;
@@ -60,7 +91,7 @@ public class TestMessagePackUnpackable extends TestCase {
 	}
 
 	@Test
-	public void testGeneralReferenceTypeFieldsClass() throws Exception {
+	public void testGeneralReferenceTypeFieldsClass01() throws Exception {
 		GeneralReferenceTypeFieldsClass src = (GeneralReferenceTypeFieldsClass) PackUnpackUtil
 				.newEnhancedInstance(GeneralReferenceTypeFieldsClass.class);
 		src.f0 = 0;
@@ -93,6 +124,44 @@ public class TestMessagePackUnpackable extends TestCase {
 		assertEquals(src.f9[1], dst.f9[1]);
 	}
 
+	@Test
+	public void testGeneralReferenceTypeFieldsClass02() throws Exception {
+		GeneralReferenceTypeFieldsClass src = (GeneralReferenceTypeFieldsClass) PackUnpackUtil
+				.newEnhancedInstance(GeneralReferenceTypeFieldsClass.class);
+		src.f0 = 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = (long) 3;
+		src.f4 = (float) 4;
+		src.f5 = (double) 5;
+		src.f6 = false;
+		src.f7 = new BigInteger("7");
+		src.f8 = "8";
+		src.f9 = new byte[] { 0x01, 0x02 };
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) PackUnpackUtil
+				.initEnhancedInstance(mpo,
+						GeneralReferenceTypeFieldsClass.class);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9[0], dst.f9[0]);
+		assertEquals(src.f9[1], dst.f9[1]);
+		assertFalse(it.hasNext());
+	}
+
 	@MessagePackUnpackable
 	public static class GeneralReferenceTypeFieldsClass {
 		public Byte f0;
@@ -110,7 +179,7 @@ public class TestMessagePackUnpackable extends TestCase {
 		}
 	}
 
-	public void testListTypes() throws Exception {
+	public void testListTypes01() throws Exception {
 		SampleListTypes src = (SampleListTypes) PackUnpackUtil
 				.newEnhancedInstance(SampleListTypes.class);
 		src.f0 = new ArrayList();
@@ -139,6 +208,39 @@ public class TestMessagePackUnpackable extends TestCase {
 			assertEquals(src.f2.get(i), dst.f2.get(i));
 		}
 	}
+	
+	public void testListTypes02() throws Exception {
+		SampleListTypes src = (SampleListTypes) PackUnpackUtil
+				.newEnhancedInstance(SampleListTypes.class);
+		src.f0 = new ArrayList();
+		src.f1 = new ArrayList();
+		src.f1.add(1);
+		src.f1.add(2);
+		src.f1.add(3);
+		src.f2 = new ArrayList();
+		src.f2.add("e1");
+		src.f2.add("e2");
+		src.f2.add("e3");
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleListTypes dst = (SampleListTypes) PackUnpackUtil
+				.initEnhancedInstance(mpo, SampleListTypes.class);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		for (int i = 0; i < src.f1.size(); ++i) {
+			assertEquals(src.f1.get(i), dst.f1.get(i));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		for (int i = 0; i < src.f2.size(); ++i) {
+			assertEquals(src.f2.get(i), dst.f2.get(i));
+		}
+		assertFalse(it.hasNext());
+	}
 
 	@MessagePackUnpackable
 	public static class SampleListTypes {
@@ -150,7 +252,7 @@ public class TestMessagePackUnpackable extends TestCase {
 		}
 	}
 
-	public void testMapTypes() throws Exception {
+	public void testMapTypes01() throws Exception {
 		SampleMapTypes src = (SampleMapTypes) PackUnpackUtil
 				.newEnhancedInstance(SampleMapTypes.class);
 		src.f0 = new HashMap();
@@ -189,6 +291,49 @@ public class TestMessagePackUnpackable extends TestCase {
 			assertEquals(src.f2.get(s2), dst.f2.get(d2));
 		}
 	}
+	
+	public void testMapTypes02() throws Exception {
+		SampleMapTypes src = (SampleMapTypes) PackUnpackUtil
+				.newEnhancedInstance(SampleMapTypes.class);
+		src.f0 = new HashMap();
+		src.f1 = new HashMap();
+		src.f1.put(1, 1);
+		src.f1.put(2, 2);
+		src.f1.put(3, 3);
+		src.f2 = new HashMap();
+		src.f2.put("k1", 1);
+		src.f2.put("k2", 2);
+		src.f2.put("k3", 3);
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleMapTypes dst = (SampleMapTypes) PackUnpackUtil
+				.initEnhancedInstance(mpo, SampleMapTypes.class);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		Iterator srcf1 = src.f1.keySet().iterator();
+		Iterator dstf1 = dst.f1.keySet().iterator();
+		while (srcf1.hasNext()) {
+			Integer s1 = srcf1.next();
+			Integer d1 = dstf1.next();
+			assertEquals(s1, d1);
+			assertEquals(src.f1.get(s1), dst.f1.get(d1));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		Iterator srcf2 = src.f2.keySet().iterator();
+		Iterator dstf2 = dst.f2.keySet().iterator();
+		while (srcf2.hasNext()) {
+			String s2 = srcf2.next();
+			String d2 = dstf2.next();
+			assertEquals(s2, d2);
+			assertEquals(src.f2.get(s2), dst.f2.get(d2));
+		}
+		assertFalse(it.hasNext());
+	}
 
 	@MessagePackUnpackable
 	public static class SampleMapTypes {
@@ -351,7 +496,7 @@ public class TestMessagePackUnpackable extends TestCase {
 	}
 
 	@Test
-	public void testFieldModifiers() throws Exception {
+	public void testFieldModifiers01() throws Exception {
 		FieldModifiersClass src = (FieldModifiersClass) PackUnpackUtil
 				.newEnhancedInstance(FieldModifiersClass.class);
 		src.f0 = 0;
@@ -372,6 +517,31 @@ public class TestMessagePackUnpackable extends TestCase {
 		assertTrue(src.f4 != dst.f4);
 	}
 
+	@Test
+	public void testFieldModifiers02() throws Exception {
+		FieldModifiersClass src = (FieldModifiersClass) PackUnpackUtil
+				.newEnhancedInstance(FieldModifiersClass.class);
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		FieldModifiersClass dst = (FieldModifiersClass) PackUnpackUtil
+				.initEnhancedInstance(mpo, FieldModifiersClass.class);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f4 != dst.f4);
+		assertFalse(it.hasNext());
+	}
+
 	@MessagePackUnpackable
 	public static class FieldModifiersClass {
 		public int f0;
@@ -385,7 +555,7 @@ public class TestMessagePackUnpackable extends TestCase {
 	}
 
 	@Test
-	public void testNestedAnnotatedFieldClass() throws Exception {
+	public void testNestedAnnotatedFieldClass01() throws Exception {
 		NestedClass src2 = (NestedClass) PackUnpackUtil
 				.newEnhancedInstance(NestedClass.class);
 		BaseClass src = (BaseClass) PackUnpackUtil
@@ -407,6 +577,33 @@ public class TestMessagePackUnpackable extends TestCase {
 		assertTrue(src2.f2 == dst.f1.f2);
 	}
 
+	@Test
+	public void testNestedAnnotatedFieldClass02() throws Exception {
+		NestedClass src2 = (NestedClass) PackUnpackUtil
+				.newEnhancedInstance(NestedClass.class);
+		BaseClass src = (BaseClass) PackUnpackUtil
+				.newEnhancedInstance(BaseClass.class);
+		src.f0 = 0;
+		src2.f2 = 2;
+		src.f1 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		NestedClass dst2 = (NestedClass) PackUnpackUtil
+				.newEnhancedInstance(NestedClass.class);
+		BaseClass dst = (BaseClass) PackUnpackUtil
+				.newEnhancedInstance(BaseClass.class);
+		dst.f1 = dst2;
+		dst = (BaseClass) PackUnpackUtil.initEnhancedInstance(mpo, dst);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src2.f2 == dst.f1.f2);
+		assertFalse(it.hasNext());
+	}
+
 	@MessagePackUnpackable
 	public static class BaseClass {
 		public int f0;
@@ -425,7 +622,7 @@ public class TestMessagePackUnpackable extends TestCase {
 	}
 
 	@Test
-	public void testExtendedClass() throws Exception {
+	public void testExtendedClass01() throws Exception {
 		SampleSubClass src = (SampleSubClass) PackUnpackUtil
 				.newEnhancedInstance(SampleSubClass.class);
 		src.f0 = 0;
@@ -452,6 +649,38 @@ public class TestMessagePackUnpackable extends TestCase {
 		assertTrue(src.f8 == dst.f8);
 		assertTrue(src.f9 != dst.f9);
 	}
+	
+	@Test
+	public void testExtendedClass02() throws Exception {
+		SampleSubClass src = (SampleSubClass) PackUnpackUtil
+				.newEnhancedInstance(SampleSubClass.class);
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f8 = 8;
+		src.f9 = 9;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleSubClass dst = (SampleSubClass) PackUnpackUtil
+				.initEnhancedInstance(mpo, SampleSubClass.class);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f4 != dst.f4);
+		assertTrue(src.f5 == dst.f5);
+		assertTrue(src.f6 == dst.f6);
+		assertTrue(src.f8 == dst.f8);
+		assertTrue(src.f9 != dst.f9);
+		assertFalse(it.hasNext());
+	}
 
 	@MessagePackUnpackable
 	public static class SampleSubClass extends SampleSuperClass {

From df8a3e870ae9c350394ded3e2e73f9c0bca023f7 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 23 Sep 2010 16:18:23 +0900
Subject: [PATCH 0793/1648] java: refactor annotation-utilities

---
 .../util/annotation/PackUnpackUtil.java       | 678 ++++++------------
 1 file changed, 208 insertions(+), 470 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
index a337ff9..ddf30b4 100644
--- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
+++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
@@ -81,9 +81,13 @@ public class PackUnpackUtil {
 
 		static final String VARIABLE_NAME_MAP = "_$$_map";
 
+		static final String VARIABLE_NAME_KEY = "_$$_key";
+
+		static final String VARIABLE_NAME_VAL = "_$$_val";
+
 		static final String VARIABLE_NAME_ITER = "_$$_iter";
 
-		static final String VARIABLE_NAME_MPO1 = "_$$_mpo1";
+		static final String VARIABLE_NAME_MPO = "_$$_mpo";
 
 		static final String VARIABLE_NAME_I = "i";
 
@@ -144,6 +148,26 @@ public class PackUnpackUtil {
 		static final String METHOD_NAME_UNPACKMAP = "unpackMap";
 
 		static final String METHOD_NAME_ASARRAY = "asArray";
+
+		static final String METHOD_NAME_ASBOOLEAN = "asBoolean";
+
+		static final String METHOD_NAME_ASBYTE = "asByte";
+
+		static final String METHOD_NAME_ASSHORT = "asShort";
+
+		static final String METHOD_NAME_ASINT = "asInt";
+
+		static final String METHOD_NAME_ASFLOAT = "asFloat";
+
+		static final String METHOD_NAME_ASLONG = "asLong";
+
+		static final String METHOD_NAME_ASDOUBLE = "asDouble";
+
+		static final String METHOD_NAME_ASSTRING = "asString";
+
+		static final String METHOD_NAME_ASBYTEARRAY = "asByteArray";
+
+		static final String METHOD_NAME_ASBIGINTEGER = "asBigInteger";
 	}
 
 	public static class Enhancer {
@@ -322,7 +346,7 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_SEMICOLON);
 			sb.append(Constants.CHAR_NAME_SPACE);
 			for (Field field : fields) {
-				insertCodeOfMessagePack(sb, field);
+				insertCodeOfMessagePackCall(sb, field);
 			}
 			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
 			// System.out.println("messagePack method: " + sb.toString());
@@ -330,7 +354,7 @@ public class PackUnpackUtil {
 			enhCtClass.addMethod(newCtMethod);
 		}
 
-		private void insertCodeOfMessagePack(StringBuilder sb, Field field) {
+		private void insertCodeOfMessagePackCall(StringBuilder sb, Field field) {
 			sb.append(Constants.VARIABLE_NAME_PK);
 			sb.append(Constants.CHAR_NAME_DOT);
 			sb.append(Constants.METHOD_NAME_PACK);
@@ -371,20 +395,25 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 			sb.append(Constants.CHAR_NAME_SEMICOLON);
 			sb.append(Constants.CHAR_NAME_SPACE);
-			for (Field field : fields) {
-				insertCodeOfMessageUnpack(sb, field, field.getType());
-			}
+			insertCodeOfMessageUnpackCalls(sb, fields);
 			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
 			// System.out.println("messageUnpack method: " + sb.toString());
 			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
 			enhCtClass.addMethod(newCtMethod);
 		}
 
-		private void insertCodeOfMessageUnpack(StringBuilder sb, Field field,
-				Class type) throws NotFoundException {
+		private void insertCodeOfMessageUnpackCalls(StringBuilder sb,
+				Field[] fields) throws NotFoundException {
+			for (Field field : fields) {
+				insertCodeOfMessageUnpackCall(sb, field, field.getType());
+			}
+		}
+
+		private void insertCodeOfMessageUnpackCall(StringBuilder sb,
+				Field field, Class type) throws NotFoundException {
 			if (type.isPrimitive()) {
 				// primitive type
-				insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type);
+				insertCodeOfMessageUnpackCallForPrimitiveTypes(sb, field, type);
 			} else if (type.equals(Boolean.class) || // Boolean
 					type.equals(Byte.class) || // Byte
 					type.equals(Double.class) || // Double
@@ -393,28 +422,29 @@ public class PackUnpackUtil {
 					type.equals(Long.class) || // Long
 					type.equals(Short.class)) { // Short
 				// reference type (wrapper type)
-				insertCodeOfMessageUnpackForWrapperTypes(sb, field, type);
+				insertCodeOfMessageUnpackCallForWrapperTypes(sb, field, type);
 			} else if (type.equals(BigInteger.class) || // BigInteger
 					type.equals(String.class) || // String
 					type.equals(byte[].class)) { // byte[]
 				// reference type (other type)
-				insertCodeOfMessageUnpackForPrimitiveTypes(sb, field, type);
+				insertCodeOfMessageUnpackCallForPrimitiveTypes(sb, field, type);
 			} else if (List.class.isAssignableFrom(type)) {
 				// List
-				insertCodeOfMessageUnpackForListType(sb, field, type);
+				insertCodeOfMessageUnpackCallForListType(sb, field, type);
 			} else if (Map.class.isAssignableFrom(type)) {
 				// Map
-				insertCodeOfMessageUnpackForMapType(sb, field, type);
+				insertCodeOfMessageUnpackCallForMapType(sb, field, type);
 			} else if (MessageUnpackable.class.isAssignableFrom(type)
 					|| (getCache(type.getName()) != null)) {
 				// MessageUnpackable
-				insertCodeOfMessageUnpackForMsgUnpackableType(sb, field, type);
+				insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, field,
+						type);
 			} else {
 				throw new NotFoundException("unknown type:  " + type.getName());
 			}
 		}
 
-		private void insertCodeOfMessageUnpackForPrimitiveTypes(
+		private void insertCodeOfMessageUnpackCallForPrimitiveTypes(
 				StringBuilder sb, Field field, Class type)
 				throws NotFoundException {
 			// insert a right variable
@@ -461,8 +491,9 @@ public class PackUnpackUtil {
 			}
 		}
 
-		private void insertCodeOfMessageUnpackForWrapperTypes(StringBuilder sb,
-				Field field, Class type) throws NotFoundException {
+		private void insertCodeOfMessageUnpackCallForWrapperTypes(
+				StringBuilder sb, Field field, Class type)
+				throws NotFoundException {
 			// insert a right variable
 			if (field != null) {
 				sb.append(field.getName());
@@ -503,7 +534,7 @@ public class PackUnpackUtil {
 			}
 		}
 
-		private void insertCodeOfMessageUnpackForListType(StringBuilder sb,
+		private void insertCodeOfMessageUnpackCallForListType(StringBuilder sb,
 				Field field, Class type) throws NotFoundException {
 			ParameterizedType generic = (ParameterizedType) field
 					.getGenericType();
@@ -570,7 +601,7 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_DOT);
 			sb.append(Constants.METHOD_NAME_ADD);
 			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			insertCodeOfMessageUnpack(sb, null, genericType);
+			insertCodeOfMessageUnpackCall(sb, null, genericType);
 			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 			sb.append(Constants.CHAR_NAME_SEMICOLON);
 			sb.append(Constants.CHAR_NAME_SPACE);
@@ -579,7 +610,7 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_SPACE);
 		}
 
-		private void insertCodeOfMessageUnpackForMapType(StringBuilder sb,
+		private void insertCodeOfMessageUnpackCallForMapType(StringBuilder sb,
 				Field field, Class type) throws NotFoundException {
 			ParameterizedType generic = (ParameterizedType) field
 					.getGenericType();
@@ -647,10 +678,10 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_DOT);
 			sb.append(Constants.METHOD_NAME_PUT);
 			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			insertCodeOfMessageUnpack(sb, null, genericType0);
+			insertCodeOfMessageUnpackCall(sb, null, genericType0);
 			sb.append(Constants.CHAR_NAME_COMMA);
 			sb.append(Constants.CHAR_NAME_SPACE);
-			insertCodeOfMessageUnpack(sb, null, genericType1);
+			insertCodeOfMessageUnpackCall(sb, null, genericType1);
 			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 			sb.append(Constants.CHAR_NAME_SEMICOLON);
 			sb.append(Constants.CHAR_NAME_SPACE);
@@ -659,7 +690,7 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_SPACE);
 		}
 
-		private void insertCodeOfMessageUnpackForMsgUnpackableType(
+		private void insertCodeOfMessageUnpackCallForMsgUnpackableType(
 				StringBuilder sb, Field field, Class type) {
 			// insert a right variable // ignore
 			sb.append(Constants.VARIABLE_NAME_PK);
@@ -687,7 +718,7 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
 			sb.append(MessagePackObject.class.getName());
 			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_MPO1);
+			sb.append(Constants.VARIABLE_NAME_MPO);
 			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.KEYWORD_THROWS);
@@ -699,7 +730,7 @@ public class PackUnpackUtil {
 			insertCodeOfMessagePackObjectArrayGet(sb);
 			insertCodeOfMesageConvertCalls(sb, fields);
 			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			//System.out.println("messageConvert method: " + sb.toString());
+			// System.out.println("messageConvert method: " + sb.toString());
 			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
 			enhCtClass.addMethod(newCtMethod);
 		}
@@ -714,7 +745,7 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.CHAR_NAME_EQUAL);
 			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_MPO1);
+			sb.append(Constants.VARIABLE_NAME_MPO);
 			sb.append(Constants.CHAR_NAME_DOT);
 			sb.append(Constants.METHOD_NAME_ASARRAY);
 			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
@@ -732,455 +763,161 @@ public class PackUnpackUtil {
 		}
 
 		private void insertCodeOfMessageConvertCall(StringBuilder sb, Field f,
-				Class c, int i, String name) {
+				Class c, int i, String v) {
 			if (c.isPrimitive()) { // primitive type
-				// f0 = objs[0].intValue();
-				if (c.equals(boolean.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asBoolean");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(byte.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asByte");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(short.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asShort");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(int.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asInt");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(long.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asLong");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(float.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asFloat");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(double.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asDouble");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else {
-					throw new PackUnpackUtilException("fatal error: "
-							+ c.getName());
-				}
+				insertCodeOfMessageConvertCallForPrimTypes(sb, f, c, i, v);
 			} else { // reference type
-				if (c.equals(Boolean.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-					sb.append(Constants.KEYWORD_NEW);
-					sb.append(Constants.CHAR_NAME_SPACE);
-					sb.append(Boolean.class.getName());
+				if (c.equals(Boolean.class) || c.equals(Byte.class)
+						|| c.equals(Short.class) || c.equals(Integer.class)
+						|| c.equals(Float.class) || c.equals(Long.class)
+						|| c.equals(Double.class)) {
+					// wrapper type
+					insertCodeOfMessageConvertCallForWrapTypes(sb, f, c, i, v);
+				} else if (c.equals(String.class) || c.equals(byte[].class)
+						|| c.equals(BigInteger.class)) {
+					insertCodeOfMessageConvertCallForPrimTypes(sb, f, c, i, v);
+				} else if (List.class.isAssignableFrom(c)) {
+					insertCodeOfMessageConvertCallForList(sb, f, c, i);
+				} else if (Map.class.isAssignableFrom(c)) {
+					insertCodeOfMessageConveretCallForMap(sb, f, c, i);
+				} else if (MessageConvertable.class.isAssignableFrom(c)
+						|| (getCache(c.getName()) != null)) {
+					// TODO
+					// TODO
+					// TODO
+					// ((MessageConvertable)f_i).messageConvert(ary[i]);
 					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
+					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(MessageConvertable.class.getName());
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+					sb.append(f.getName());
+					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asBoolean");
+					sb.append(Constants.METHOD_NAME_MSGCONVERT);
 					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+					sb.append(Constants.VARIABLE_NAME_ARRAY);
+					sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+					sb.append(i);
+					sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
 					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(Byte.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-					sb.append(Constants.KEYWORD_NEW);
+					sb.append(Constants.CHAR_NAME_SEMICOLON);
 					sb.append(Constants.CHAR_NAME_SPACE);
-					sb.append(Byte.class.getName());
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asByte");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(Short.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-					sb.append(Constants.KEYWORD_NEW);
-					sb.append(Constants.CHAR_NAME_SPACE);
-					sb.append(Short.class.getName());
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asShort");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(Integer.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-					sb.append(Constants.KEYWORD_NEW);
-					sb.append(Constants.CHAR_NAME_SPACE);
-					sb.append(Integer.class.getName());
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asInt");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(Long.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-					sb.append(Constants.KEYWORD_NEW);
-					sb.append(Constants.CHAR_NAME_SPACE);
-					sb.append(Long.class.getName());
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asLong");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(Float.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-					sb.append(Constants.KEYWORD_NEW);
-					sb.append(Constants.CHAR_NAME_SPACE);
-					sb.append(Float.class.getName());
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asFloat");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-				} else if (c.equals(Double.class)) {
-					if (f != null) {
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_SPACE);
-						sb.append(Constants.CHAR_NAME_EQUAL);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
-					sb.append(Constants.KEYWORD_NEW);
-					sb.append(Constants.CHAR_NAME_SPACE);
-					sb.append(Double.class.getName());
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					} else {
-						sb.append(name);
-					}
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append("asDouble");
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					if (f != null) {
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					}
 				} else {
-					if (c.equals(String.class)) {
-						if (f != null) {
-							sb.append(f.getName());
-							sb.append(Constants.CHAR_NAME_SPACE);
-							sb.append(Constants.CHAR_NAME_EQUAL);
-							sb.append(Constants.CHAR_NAME_SPACE);
-							sb.append(Constants.VARIABLE_NAME_ARRAY);
-							sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-							sb.append(i);
-							sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-						} else {
-							sb.append(name);
-						}
-						sb.append(Constants.CHAR_NAME_DOT);
-						sb.append("asString");
-						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-						if (f != null) {
-							sb.append(Constants.CHAR_NAME_SEMICOLON);
-							sb.append(Constants.CHAR_NAME_SPACE);
-						}
-					} else if (c.equals(byte[].class)) {
-						if (f != null) {
-							sb.append(f.getName());
-							sb.append(Constants.CHAR_NAME_SPACE);
-							sb.append(Constants.CHAR_NAME_EQUAL);
-							sb.append(Constants.CHAR_NAME_SPACE);
-							sb.append(Constants.VARIABLE_NAME_ARRAY);
-							sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-							sb.append(i);
-							sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-						} else {
-							sb.append(name);
-						}
-						sb.append(Constants.CHAR_NAME_DOT);
-						sb.append("asByteArray");
-						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-						if (f != null) {
-							sb.append(Constants.CHAR_NAME_SEMICOLON);
-							sb.append(Constants.CHAR_NAME_SPACE);
-						}
-					} else if (c.equals(BigInteger.class)) {
-						if (f != null) {
-							sb.append(f.getName());
-							sb.append(Constants.CHAR_NAME_SPACE);
-							sb.append(Constants.CHAR_NAME_EQUAL);
-							sb.append(Constants.CHAR_NAME_SPACE);
-							sb.append(Constants.VARIABLE_NAME_ARRAY);
-							sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-							sb.append(i);
-							sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-						} else {
-							sb.append(name);
-						}
-						sb.append(Constants.CHAR_NAME_DOT);
-						sb.append("asBigInteger");
-						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-						if (f != null) {
-							sb.append(Constants.CHAR_NAME_SEMICOLON);
-							sb.append(Constants.CHAR_NAME_SPACE);
-						}
-					} else if (List.class.isAssignableFrom(c)) {
-						insertCodeOfMessageConvertCallForList(sb, f, c, i);
-					} else if (Map.class.isAssignableFrom(c)) {
-						insertCodeOfMessageConveretCallForMap(sb, f, c, i);
-					} else if (MessageConvertable.class.isAssignableFrom(c)
-							|| (getCache(c.getName()) != null)) {
-						// TODO
-						// TODO
-						// TODO
-						// ((MessageConvertable)f_i).messageConvert(ary[i]);
-						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-						sb.append(MessageConvertable.class.getName());
-						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-						sb.append(f.getName());
-						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-						sb.append(Constants.CHAR_NAME_DOT);
-						sb.append(Constants.METHOD_NAME_MSGCONVERT);
-						sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-						sb.append(Constants.VARIABLE_NAME_ARRAY);
-						sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-						sb.append(i);
-						sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-						sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-						sb.append(Constants.CHAR_NAME_SEMICOLON);
-						sb.append(Constants.CHAR_NAME_SPACE);
-					} else {
-						throw new MessageTypeException("Type error: "
-								+ c.getName());
-					}
+					throw new MessageTypeException("Type error: " + c.getName());
 				}
 			}
 		}
 
+		private void insertCodeOfMessageConvertCallForPrimTypes(
+				StringBuilder sb, Field f, Class c, int i, String name) {
+			// f0 = objs[0].intValue();
+			if (f != null) {
+				sb.append(f.getName());
+				sb.append(Constants.CHAR_NAME_SPACE);
+				sb.append(Constants.CHAR_NAME_EQUAL);
+				sb.append(Constants.CHAR_NAME_SPACE);
+				sb.append(Constants.VARIABLE_NAME_ARRAY);
+				sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+				sb.append(i);
+				sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+			} else {
+				sb.append(name);
+			}
+			sb.append(Constants.CHAR_NAME_DOT);
+			if (c.equals(boolean.class)) {
+				sb.append(Constants.METHOD_NAME_ASBOOLEAN);
+			} else if (c.equals(byte.class)) {
+				sb.append(Constants.METHOD_NAME_ASBYTE);
+			} else if (c.equals(short.class)) {
+				sb.append(Constants.METHOD_NAME_ASSHORT);
+			} else if (c.equals(int.class)) {
+				sb.append(Constants.METHOD_NAME_ASINT);
+			} else if (c.equals(float.class)) {
+				sb.append(Constants.METHOD_NAME_ASFLOAT);
+			} else if (c.equals(long.class)) {
+				sb.append(Constants.METHOD_NAME_ASLONG);
+			} else if (c.equals(double.class)) {
+				sb.append(Constants.METHOD_NAME_ASDOUBLE);
+			} else if (c.equals(String.class)) {
+				sb.append(Constants.METHOD_NAME_ASSTRING);
+			} else if (c.equals(byte[].class)) {
+				sb.append(Constants.METHOD_NAME_ASBYTEARRAY);
+			} else if (c.equals(BigInteger.class)) {
+				sb.append(Constants.METHOD_NAME_ASBIGINTEGER);
+			} else {
+				throw new MessageTypeException("Type error: " + c.getName());
+			}
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			if (f != null) {
+				sb.append(Constants.CHAR_NAME_SEMICOLON);
+				sb.append(Constants.CHAR_NAME_SPACE);
+			}
+		}
+
+		private void insertCodeOfMessageConvertCallForWrapTypes(
+				StringBuilder sb, Field f, Class c, int i, String v) {
+			if (f != null) {
+				sb.append(f.getName());
+				sb.append(Constants.CHAR_NAME_SPACE);
+				sb.append(Constants.CHAR_NAME_EQUAL);
+				sb.append(Constants.CHAR_NAME_SPACE);
+			}
+			sb.append(Constants.KEYWORD_NEW);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			if (c.equals(Boolean.class)) {
+				sb.append(Boolean.class.getName());
+			} else if (c.equals(Byte.class)) {
+				sb.append(Byte.class.getName());
+			} else if (c.equals(Short.class)) {
+				sb.append(Short.class.getName());
+			} else if (c.equals(Integer.class)) {
+				sb.append(Integer.class.getName());
+			} else if (c.equals(Float.class)) {
+				sb.append(Float.class.getName());
+			} else if (c.equals(Long.class)) {
+				sb.append(Long.class.getName());
+			} else if (c.equals(Double.class)) {
+				sb.append(Double.class.getName());
+			} else {
+				throw new MessageTypeException("Type error: " + c.getName());
+			}
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			if (f != null) {
+				sb.append(Constants.VARIABLE_NAME_ARRAY);
+				sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+				sb.append(i);
+				sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+			} else {
+				sb.append(v);
+			}
+			sb.append(Constants.CHAR_NAME_DOT);
+			if (c.equals(Boolean.class)) {
+				sb.append(Constants.METHOD_NAME_ASBOOLEAN);
+			} else if (c.equals(Byte.class)) {
+				sb.append(Constants.METHOD_NAME_ASBYTE);
+			} else if (c.equals(Short.class)) {
+				sb.append(Constants.METHOD_NAME_ASSHORT);
+			} else if (c.equals(Integer.class)) {
+				sb.append(Constants.METHOD_NAME_ASINT);
+			} else if (c.equals(Float.class)) {
+				sb.append(Constants.METHOD_NAME_ASFLOAT);
+			} else if (c.equals(Long.class)) {
+				sb.append(Constants.METHOD_NAME_ASLONG);
+			} else if (c.equals(Double.class)) {
+				sb.append(Constants.METHOD_NAME_ASDOUBLE);
+			} else {
+				throw new MessageTypeException("Type error: " + c.getName());
+			}
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			if (f != null) {
+				sb.append(Constants.CHAR_NAME_SEMICOLON);
+				sb.append(Constants.CHAR_NAME_SPACE);
+			}
+		}
+
 		private void insertCodeOfMessageConvertCallForList(StringBuilder sb,
 				Field field, Class type, int i) {
 			ParameterizedType generic = (ParameterizedType) field
@@ -1264,7 +1001,7 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(MessagePackObject.class.getName());
 			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append("_$$_val");
+			sb.append(Constants.VARIABLE_NAME_VAL);
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.CHAR_NAME_EQUAL);
 			sb.append(Constants.CHAR_NAME_SPACE);
@@ -1284,7 +1021,8 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_DOT);
 			sb.append(Constants.METHOD_NAME_ADD);
 			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			insertCodeOfMessageConvertCall(sb, null, genericType, -1, "_$$_val");
+			insertCodeOfMessageConvertCall(sb, null, genericType, -1,
+					Constants.VARIABLE_NAME_VAL);
 			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 			sb.append(Constants.CHAR_NAME_SEMICOLON);
 			sb.append(Constants.CHAR_NAME_SPACE);
@@ -1380,7 +1118,7 @@ public class PackUnpackUtil {
 			// block map.
 			sb.append(MessagePackObject.class.getName());
 			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append("_$$_key");
+			sb.append(Constants.VARIABLE_NAME_KEY);
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.CHAR_NAME_EQUAL);
 			sb.append(Constants.CHAR_NAME_SPACE);
@@ -1396,7 +1134,7 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(MessagePackObject.class.getName());
 			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append("_$$_val");
+			sb.append(Constants.VARIABLE_NAME_VAL);
 			sb.append(Constants.CHAR_NAME_SPACE);
 			sb.append(Constants.CHAR_NAME_EQUAL);
 			sb.append(Constants.CHAR_NAME_SPACE);
@@ -1407,7 +1145,7 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_DOT);
 			sb.append(Constants.METHOD_NAME_GET);
 			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append("_$$_key");
+			sb.append(Constants.VARIABLE_NAME_KEY);
 			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 			sb.append(Constants.CHAR_NAME_SEMICOLON);
 			sb.append(Constants.CHAR_NAME_SPACE);
@@ -1417,11 +1155,11 @@ public class PackUnpackUtil {
 			sb.append(Constants.METHOD_NAME_PUT);
 			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
 			insertCodeOfMessageConvertCall(sb, null, genericType0, -1,
-					"_$$_key");
+					Constants.VARIABLE_NAME_KEY);
 			sb.append(Constants.CHAR_NAME_COMMA);
 			sb.append(Constants.CHAR_NAME_SPACE);
 			insertCodeOfMessageConvertCall(sb, null, genericType1, -1,
-					"_$$_val");
+					Constants.VARIABLE_NAME_VAL);
 			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 			sb.append(Constants.CHAR_NAME_SEMICOLON);
 			sb.append(Constants.CHAR_NAME_SPACE);

From e121f344077bad630fbe4c249c5c80f6a3932a35 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 23 Sep 2010 17:26:31 +0900
Subject: [PATCH 0794/1648] java: refactor annotation-utilities and edit those
 test programs

---
 .../util/annotation/PackUnpackUtil.java       | 182 +++++++++++++-----
 .../annotation/TestMessagePackUnpackable.java |  17 +-
 2 files changed, 134 insertions(+), 65 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
index ddf30b4..6e6a665 100644
--- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
+++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
@@ -41,10 +41,14 @@ public class PackUnpackUtil {
 
 		static final String KEYWORD_FOR = "for";
 
+		static final String KEYWORD_IF = "if";
+
 		static final String KEYWORD_THROWS = "throws";
 
 		static final String KEYWORD_NEW = "new";
 
+		static final String KEYWORD_NULL = "null";
+
 		static final String CHAR_NAME_SPACE = " ";
 
 		static final String CHAR_NAME_COMMA = ",";
@@ -409,42 +413,43 @@ public class PackUnpackUtil {
 			}
 		}
 
-		private void insertCodeOfMessageUnpackCall(StringBuilder sb,
-				Field field, Class type) throws NotFoundException {
-			if (type.isPrimitive()) {
+		private void insertCodeOfMessageUnpackCall(StringBuilder sb, Field f,
+				Class c) throws NotFoundException {
+			if (c.isPrimitive()) {
 				// primitive type
-				insertCodeOfMessageUnpackCallForPrimitiveTypes(sb, field, type);
-			} else if (type.equals(Boolean.class) || // Boolean
-					type.equals(Byte.class) || // Byte
-					type.equals(Double.class) || // Double
-					type.equals(Float.class) || // Float
-					type.equals(Integer.class) || // Integer
-					type.equals(Long.class) || // Long
-					type.equals(Short.class)) { // Short
+				insertCodeOfMessageUnpackCallForPrimTypes(sb, f, c);
+			} else if (c.equals(Boolean.class) || // Boolean
+					c.equals(Byte.class) || // Byte
+					c.equals(Double.class) || // Double
+					c.equals(Float.class) || // Float
+					c.equals(Integer.class) || // Integer
+					c.equals(Long.class) || // Long
+					c.equals(Short.class)) { // Short
 				// reference type (wrapper type)
-				insertCodeOfMessageUnpackCallForWrapperTypes(sb, field, type);
-			} else if (type.equals(BigInteger.class) || // BigInteger
-					type.equals(String.class) || // String
-					type.equals(byte[].class)) { // byte[]
+				insertCodeOfMessageUnpackCallForWrapTypes(sb, f, c);
+			} else if (c.equals(BigInteger.class) || // BigInteger
+					c.equals(String.class) || // String
+					c.equals(byte[].class)) { // byte[]
 				// reference type (other type)
-				insertCodeOfMessageUnpackCallForPrimitiveTypes(sb, field, type);
-			} else if (List.class.isAssignableFrom(type)) {
+				insertCodeOfMessageUnpackCallForPrimTypes(sb, f, c);
+			} else if (List.class.isAssignableFrom(c)) {
 				// List
-				insertCodeOfMessageUnpackCallForListType(sb, field, type);
-			} else if (Map.class.isAssignableFrom(type)) {
+				insertCodeOfMessageUnpackCallForListType(sb, f, c);
+			} else if (Map.class.isAssignableFrom(c)) {
 				// Map
-				insertCodeOfMessageUnpackCallForMapType(sb, field, type);
-			} else if (MessageUnpackable.class.isAssignableFrom(type)
-					|| (getCache(type.getName()) != null)) {
+				insertCodeOfMessageUnpackCallForMapType(sb, f, c);
+			} else if (getCache(c.getName()) != null) {
+				// cached
+				insertCodeOfMessageUnpackCallForEnhancedType(sb, f, c);
+			} else if (MessageUnpackable.class.isAssignableFrom(c)) {
 				// MessageUnpackable
-				insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, field,
-						type);
+				insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, f, c);
 			} else {
-				throw new NotFoundException("unknown type:  " + type.getName());
+				throw new NotFoundException("unknown type:  " + c.getName());
 			}
 		}
 
-		private void insertCodeOfMessageUnpackCallForPrimitiveTypes(
+		private void insertCodeOfMessageUnpackCallForPrimTypes(
 				StringBuilder sb, Field field, Class type)
 				throws NotFoundException {
 			// insert a right variable
@@ -491,7 +496,7 @@ public class PackUnpackUtil {
 			}
 		}
 
-		private void insertCodeOfMessageUnpackCallForWrapperTypes(
+		private void insertCodeOfMessageUnpackCallForWrapTypes(
 				StringBuilder sb, Field field, Class type)
 				throws NotFoundException {
 			// insert a right variable
@@ -690,8 +695,42 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_SPACE);
 		}
 
+		private void insertCodeOfMessageUnpackCallForEnhancedType(
+				StringBuilder sb, Field f, Class c) {
+			c = this.getCache(c.getName());
+			insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, f, c);
+		}
+
 		private void insertCodeOfMessageUnpackCallForMsgUnpackableType(
-				StringBuilder sb, Field field, Class type) {
+				StringBuilder sb, Field f, Class c) {
+			// if (fi == null) { fi = new Foo_$$_Enhanced(); }
+			sb.append(Constants.KEYWORD_IF);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(f.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.KEYWORD_NULL);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(f.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.KEYWORD_NEW);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(c.getName());
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
 			// insert a right variable // ignore
 			sb.append(Constants.VARIABLE_NAME_PK);
 			sb.append(Constants.CHAR_NAME_DOT);
@@ -700,7 +739,7 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
 			sb.append(MessageUnpackable.class.getName());
 			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(field.getName());
+			sb.append(f.getName());
 			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
 			sb.append(Constants.CHAR_NAME_SEMICOLON);
 			sb.append(Constants.CHAR_NAME_SPACE);
@@ -779,35 +818,72 @@ public class PackUnpackUtil {
 				} else if (List.class.isAssignableFrom(c)) {
 					insertCodeOfMessageConvertCallForList(sb, f, c, i);
 				} else if (Map.class.isAssignableFrom(c)) {
-					insertCodeOfMessageConveretCallForMap(sb, f, c, i);
-				} else if (MessageConvertable.class.isAssignableFrom(c)
-						|| (getCache(c.getName()) != null)) {
-					// TODO
-					// TODO
-					// TODO
-					// ((MessageConvertable)f_i).messageConvert(ary[i]);
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(MessageConvertable.class.getName());
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					sb.append(f.getName());
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_DOT);
-					sb.append(Constants.METHOD_NAME_MSGCONVERT);
-					sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-					sb.append(Constants.VARIABLE_NAME_ARRAY);
-					sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-					sb.append(i);
-					sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-					sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-					sb.append(Constants.CHAR_NAME_SEMICOLON);
-					sb.append(Constants.CHAR_NAME_SPACE);
+					insertCodeOfMessageConvertCallForMap(sb, f, c, i);
+				} else if ((getCache(c.getName()) != null)) {
+					insertCodeOfMessageConvertCallForEnhancedType(sb, f, c, i);
+				} else if (MessageConvertable.class.isAssignableFrom(c)) {
+					insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
 				} else {
 					throw new MessageTypeException("Type error: " + c.getName());
 				}
 			}
 		}
 
+		private void insertCodeOfMessageConvertCallForEnhancedType(
+				StringBuilder sb, Field f, Class c, int i) {
+			c = getCache(c.getName());
+			insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
+		}
+
+		private void insertCodeOfMessageConvertCallForMsgConvtblType(
+				StringBuilder sb, Field f, Class c, int i) {
+			// if (fi == null) { fi = new Foo_$$_Enhanced(); }
+			sb.append(Constants.KEYWORD_IF);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(f.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.KEYWORD_NULL);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(f.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.KEYWORD_NEW);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(c.getName());
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
+			sb.append(Constants.CHAR_NAME_SPACE);
+
+			// ((MessageConvertable)f_i).messageConvert(ary[i]);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(MessageConvertable.class.getName());
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(f.getName());
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(Constants.METHOD_NAME_MSGCONVERT);
+			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(Constants.VARIABLE_NAME_ARRAY);
+			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(i);
+			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+		}
+
 		private void insertCodeOfMessageConvertCallForPrimTypes(
 				StringBuilder sb, Field f, Class c, int i, String name) {
 			// f0 = objs[0].intValue();
@@ -1031,7 +1107,7 @@ public class PackUnpackUtil {
 			sb.append(Constants.CHAR_NAME_SPACE);
 		}
 
-		private void insertCodeOfMessageConveretCallForMap(StringBuilder sb,
+		private void insertCodeOfMessageConvertCallForMap(StringBuilder sb,
 				Field f, Class c, int i) {
 			ParameterizedType generic = (ParameterizedType) f.getGenericType();
 			Class genericType0 = (Class) generic.getActualTypeArguments()[0];
diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
index 4b1e666..08c4fa8 100644
--- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
+++ b/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
@@ -208,7 +208,7 @@ public class TestMessagePackUnpackable extends TestCase {
 			assertEquals(src.f2.get(i), dst.f2.get(i));
 		}
 	}
-	
+
 	public void testListTypes02() throws Exception {
 		SampleListTypes src = (SampleListTypes) PackUnpackUtil
 				.newEnhancedInstance(SampleListTypes.class);
@@ -291,7 +291,7 @@ public class TestMessagePackUnpackable extends TestCase {
 			assertEquals(src.f2.get(s2), dst.f2.get(d2));
 		}
 	}
-	
+
 	public void testMapTypes02() throws Exception {
 		SampleMapTypes src = (SampleMapTypes) PackUnpackUtil
 				.newEnhancedInstance(SampleMapTypes.class);
@@ -565,11 +565,8 @@ public class TestMessagePackUnpackable extends TestCase {
 		src.f1 = src2;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		new Packer(out).pack(src);
-		NestedClass dst2 = (NestedClass) PackUnpackUtil
-				.newEnhancedInstance(NestedClass.class);
 		BaseClass dst = (BaseClass) PackUnpackUtil
 				.newEnhancedInstance(BaseClass.class);
-		dst.f1 = dst2;
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Unpacker pac = new Unpacker(in);
 		pac.unpack((MessageUnpackable) dst);
@@ -593,12 +590,8 @@ public class TestMessagePackUnpackable extends TestCase {
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
 		MessagePackObject mpo = it.next();
-		NestedClass dst2 = (NestedClass) PackUnpackUtil
-				.newEnhancedInstance(NestedClass.class);
-		BaseClass dst = (BaseClass) PackUnpackUtil
-				.newEnhancedInstance(BaseClass.class);
-		dst.f1 = dst2;
-		dst = (BaseClass) PackUnpackUtil.initEnhancedInstance(mpo, dst);
+		BaseClass dst = (BaseClass) PackUnpackUtil.initEnhancedInstance(mpo,
+				BaseClass.class);
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src2.f2 == dst.f1.f2);
 		assertFalse(it.hasNext());
@@ -649,7 +642,7 @@ public class TestMessagePackUnpackable extends TestCase {
 		assertTrue(src.f8 == dst.f8);
 		assertTrue(src.f9 != dst.f9);
 	}
-	
+
 	@Test
 	public void testExtendedClass02() throws Exception {
 		SampleSubClass src = (SampleSubClass) PackUnpackUtil

From 22ddd91b1f73adc8b0cc402e5346504df4f4a935 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 23 Sep 2010 20:38:54 +0900
Subject: [PATCH 0795/1648] java: add several API to annotation-utilities

---
 .../util/annotation/PackUnpackUtil.java       | 62 +++++++++++++------
 1 file changed, 43 insertions(+), 19 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
index 6e6a665..e509cd8 100644
--- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
+++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
@@ -193,12 +193,12 @@ public class PackUnpackUtil {
 			classCache.putIfAbsent(origName, enhClass);
 		}
 
-		protected Class generate(Class origClass)
+		protected Class generate(Class origClass, boolean packUnpackable)
 				throws NotFoundException, CannotCompileException {
 			String origName = origClass.getName();
 			String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER;
 			CtClass origCtClass = pool.get(origName);
-			checkClassValidation(origClass);
+			checkClassValidation(origClass, packUnpackable);
 			checkDefaultConstructorValidation(origClass);
 			CtClass enhCtClass = pool.makeClass(enhName);
 			setSuperclass(enhCtClass, origCtClass);
@@ -211,7 +211,7 @@ public class PackUnpackUtil {
 			return createClass(enhCtClass);
 		}
 
-		private void checkClassValidation(Class origClass) {
+		private void checkClassValidation(Class origClass, boolean packUnpackable) {
 			// not public, abstract, final
 			int mod = origClass.getModifiers();
 			if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
@@ -223,7 +223,9 @@ public class PackUnpackUtil {
 				throwClassValidationException(origClass);
 			}
 			// annotation
-			checkPackUnpackAnnotation(origClass);
+			if (!packUnpackable) {
+				checkPackUnpackAnnotation(origClass);
+			}
 		}
 
 		private static void throwClassValidationException(Class origClass) {
@@ -1251,18 +1253,17 @@ public class PackUnpackUtil {
 	}
 
 	private static Enhancer enhancer;
-
-	public static Class getEnhancedClass(Class origClass) {
+	
+	public static void registerEnhancedClass(Class origClass, boolean packUnpackable) {
 		if (enhancer == null) {
 			enhancer = new Enhancer();
 		}
-
 		String origName = origClass.getName();
 		Class enhClass = enhancer.getCache(origName);
 		if (enhClass == null) {
 			// generate a class object related to the original class
 			try {
-				enhClass = enhancer.generate(origClass);
+				enhClass = enhancer.generate(origClass, packUnpackable);
 			} catch (NotFoundException e) {
 				throw new PackUnpackUtilException(e.getMessage(), e);
 			} catch (CannotCompileException e) {
@@ -1270,33 +1271,57 @@ public class PackUnpackUtil {
 			}
 			// set the generated class to the cache
 			enhancer.setCache(origName, enhClass);
+		}			
+	}
+	
+	public static void registerEnhancedClass(Class origClass) {
+		registerEnhancedClass(origClass, false);
+	}
+	
+	public static boolean isRegistered(Class origClass) {
+		if (enhancer == null) {
+			enhancer = new Enhancer();
 		}
-		return enhClass;
+		return enhancer.getCache(origClass.getName()) != null;
 	}
 
-	public static Object newEnhancedInstance(Class origClass) {
+	public static Class getEnhancedClass(Class origClass, boolean packUnpackable) {
+		if (!isRegistered(origClass)) {
+			registerEnhancedClass(origClass, packUnpackable);
+		}
+		return enhancer.getCache(origClass.getName());
+	}
+	
+	public static Class getEnhancedClass(Class origClass) {
+		return getEnhancedClass(origClass, false);
+	}
+
+	public static Object newEnhancedInstance(Class origClass, boolean packUnpackable) {
 		try {
-			Class enhClass = getEnhancedClass(origClass);
+			Class enhClass = getEnhancedClass(origClass, packUnpackable);
 			// create a new object of the generated class
 			return enhClass.newInstance();
 		} catch (InstantiationException e) {
 			throw new PackUnpackUtilException(e.getMessage(), e);
 		} catch (IllegalAccessException e) {
 			throw new PackUnpackUtilException(e.getMessage(), e);
-		}
+		}		
 	}
-
+	
+	public static Object newEnhancedInstance(Class origClass) {
+		return newEnhancedInstance(origClass, false);
+	}
+	
 	public static Object initEnhancedInstance(MessagePackObject obj,
-			Class origClass) {
-		Object ret = newEnhancedInstance(origClass);
+			Class origClass, boolean packUnpackable) {
+		Object ret = newEnhancedInstance(origClass, packUnpackable);
 		((MessageConvertable) ret).messageConvert(obj);
 		return ret;
 	}
 
 	public static Object initEnhancedInstance(MessagePackObject obj,
-			Object origObj) {
-		((MessageConvertable) origObj).messageConvert(obj);
-		return origObj;
+			Class origClass) {
+		return initEnhancedInstance(obj, origClass, false);
 	}
 
 	@MessagePackUnpackable
@@ -1326,7 +1351,6 @@ public class PackUnpackUtil {
 	}
 
 	public static void main(final String[] args) throws Exception {
-		PackUnpackUtil.getEnhancedClass(Image.class);
 		Image src = (Image) PackUnpackUtil.newEnhancedInstance(Image.class);
 		src.title = "msgpack";
 		src.uri = "http://msgpack.org/";

From 34c008adce9ffd716dfcac11b35490b6b1244f3f Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 23 Sep 2010 20:40:50 +0900
Subject: [PATCH 0796/1648] java: refactor annotation-utilities

---
 .../util/annotation/PackUnpackUtil.java       | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
index e509cd8..0aeb6d8 100644
--- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
+++ b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
@@ -211,7 +211,8 @@ public class PackUnpackUtil {
 			return createClass(enhCtClass);
 		}
 
-		private void checkClassValidation(Class origClass, boolean packUnpackable) {
+		private void checkClassValidation(Class origClass,
+				boolean packUnpackable) {
 			// not public, abstract, final
 			int mod = origClass.getModifiers();
 			if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
@@ -1253,8 +1254,9 @@ public class PackUnpackUtil {
 	}
 
 	private static Enhancer enhancer;
-	
-	public static void registerEnhancedClass(Class origClass, boolean packUnpackable) {
+
+	public static void registerEnhancedClass(Class origClass,
+			boolean packUnpackable) {
 		if (enhancer == null) {
 			enhancer = new Enhancer();
 		}
@@ -1271,13 +1273,13 @@ public class PackUnpackUtil {
 			}
 			// set the generated class to the cache
 			enhancer.setCache(origName, enhClass);
-		}			
+		}
 	}
-	
+
 	public static void registerEnhancedClass(Class origClass) {
 		registerEnhancedClass(origClass, false);
 	}
-	
+
 	public static boolean isRegistered(Class origClass) {
 		if (enhancer == null) {
 			enhancer = new Enhancer();
@@ -1285,18 +1287,20 @@ public class PackUnpackUtil {
 		return enhancer.getCache(origClass.getName()) != null;
 	}
 
-	public static Class getEnhancedClass(Class origClass, boolean packUnpackable) {
+	public static Class getEnhancedClass(Class origClass,
+			boolean packUnpackable) {
 		if (!isRegistered(origClass)) {
 			registerEnhancedClass(origClass, packUnpackable);
 		}
 		return enhancer.getCache(origClass.getName());
 	}
-	
+
 	public static Class getEnhancedClass(Class origClass) {
 		return getEnhancedClass(origClass, false);
 	}
 
-	public static Object newEnhancedInstance(Class origClass, boolean packUnpackable) {
+	public static Object newEnhancedInstance(Class origClass,
+			boolean packUnpackable) {
 		try {
 			Class enhClass = getEnhancedClass(origClass, packUnpackable);
 			// create a new object of the generated class
@@ -1305,13 +1309,13 @@ public class PackUnpackUtil {
 			throw new PackUnpackUtilException(e.getMessage(), e);
 		} catch (IllegalAccessException e) {
 			throw new PackUnpackUtilException(e.getMessage(), e);
-		}		
+		}
 	}
-	
+
 	public static Object newEnhancedInstance(Class origClass) {
 		return newEnhancedInstance(origClass, false);
 	}
-	
+
 	public static Object initEnhancedInstance(MessagePackObject obj,
 			Class origClass, boolean packUnpackable) {
 		Object ret = newEnhancedInstance(origClass, packUnpackable);

From 93bed9c5df6d4fe7a0defdaeb2f158e27d4feb1d Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Fri, 24 Sep 2010 01:24:13 +0900
Subject: [PATCH 0797/1648] haskell: finish template-haskell deriving implement

---
 haskell/msgpack.cabal                  |  4 ++
 haskell/src/Data/MessagePack/Derive.hs | 62 ++++++++++++++++++--------
 haskell/test/Test.hs                   |  2 +-
 haskell/test/UserData.hs               | 26 +++++++++++
 4 files changed, 75 insertions(+), 19 deletions(-)

diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal
index 9c67bdc..9950273 100644
--- a/haskell/msgpack.cabal
+++ b/haskell/msgpack.cabal
@@ -15,6 +15,10 @@ Stability:          Experimental
 Cabal-Version:      >= 1.6
 Build-Type:         Simple
 
+Extra-source-files:
+  test/Test.hs
+  test/UserData.hs
+
 Library
   Build-depends:    base >=4 && <5,
                     transformers >= 0.2.1 && < 0.2.2,
diff --git a/haskell/src/Data/MessagePack/Derive.hs b/haskell/src/Data/MessagePack/Derive.hs
index cfdb658..e998473 100644
--- a/haskell/src/Data/MessagePack/Derive.hs
+++ b/haskell/src/Data/MessagePack/Derive.hs
@@ -11,10 +11,11 @@ import Language.Haskell.TH
 
 import Data.MessagePack.Pack
 import Data.MessagePack.Unpack
+import Data.MessagePack.Object
 
 deriveUnpack :: Name -> Q [Dec]
 deriveUnpack typName = do
-  TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName
+  TyConI (DataD _ name _ cons _) <- reify typName
   
   return
     [ InstanceD [] (AppT (ConT ''Unpackable) (ConT name))
@@ -24,20 +25,19 @@ deriveUnpack typName = do
   where
     body (NormalC conName elms) =
       DoE
-      [ BindS (tupOrList $ map VarP names) (VarE 'get)
+      [ BindS (tupOrListP $ map VarP names) (VarE 'get)
       , NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ]
         where
           names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms
 
-    tupOrList ls
-      | length ls <= 1 = ListP ls
-      | otherwise = TupP ls
+    body (RecC conName elms) =
+      body (NormalC conName $ map (\(_, b, c) -> (b, c)) elms)
 
     ch = foldl1 (\e f -> AppE (AppE (VarE '(<|>)) e) f)
 
 derivePack :: Name -> Q [Dec]
 derivePack typName = do
-  TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName
+  TyConI (DataD _ name _ cons _) <- reify typName
   
   return
     [ InstanceD [] (AppT (ConT ''Packable) (ConT name))
@@ -48,27 +48,53 @@ derivePack typName = do
     body (NormalC conName elms) =
       Clause
         [ ConP conName $ map VarP names ]
-        (NormalB $ AppE (VarE 'put) $ tupOrList $ map VarE names) []
+        (NormalB $ AppE (VarE 'put) $ tupOrListE $ map VarE names) []
       where
         names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms
 
-    tupOrList ls
-      | length ls <= 1 = ListE ls
-      | otherwise = TupE ls
+    body (RecC conName elms) =
+      body (NormalC conName $ map (\(_, b, c) -> (b, c)) elms)
 
 deriveObject :: Name -> Q [Dec]
 deriveObject typName = do
   g <- derivePack typName
   p <- deriveUnpack typName
-  {-
-  TyConI (DataD cxt name tyVarBndrs cons names) <- reify typName
+
+  TyConI (DataD _ name _ cons _) <- reify typName
   let o = InstanceD [] (AppT (ConT ''OBJECT) (ConT name))
-          [ FunD 'toObject (map toObjectBody cons) ]
-  -}
-  return $ g ++ p -- ++ [o]
-{-
+          [ FunD 'toObject (map toObjectBody cons),
+            FunD 'tryFromObject [Clause [ VarP oname ]
+                                 (NormalB $ ch $ map tryFromObjectBody cons) []]]
+
+  return $ g ++ p ++ [o]
   where
     toObjectBody (NormalC conName elms) =
       Clause
-        [ ConP conP
--}
+        [ ConP conName $ map VarP names ]
+        (NormalB $ AppE (VarE 'toObject) $ tupOrListE $ map VarE names) []
+      where
+        names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms
+    toObjectBody (RecC conName elms) =
+      toObjectBody (NormalC conName $ map (\(_, b, c) -> (b, c)) elms)
+
+    tryFromObjectBody (NormalC conName elms) =
+      DoE
+      [ BindS (tupOrListP $ map VarP names) (AppE (VarE 'tryFromObject) (VarE oname))
+      , NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ]
+        where
+          names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms
+    tryFromObjectBody (RecC conName elms) =
+      tryFromObjectBody (NormalC conName $ map (\(_, b, c) -> (b, c)) elms)
+
+    oname = mkName "o"
+    ch = foldl1 (\e f -> AppE (AppE (VarE '(<|>)) e) f)
+
+tupOrListP :: [Pat] -> Pat
+tupOrListP ls
+  | length ls <= 1 = ListP ls
+  | otherwise = TupP ls
+
+tupOrListE :: [Exp] -> Exp
+tupOrListE ls
+  | length ls <= 1 = ListE ls
+  | otherwise = TupE ls
diff --git a/haskell/test/Test.hs b/haskell/test/Test.hs
index a73ac9a..43af2ef 100644
--- a/haskell/test/Test.hs
+++ b/haskell/test/Test.hs
@@ -7,7 +7,7 @@ import qualified Data.ByteString.Char8 as B
 import qualified Data.ByteString.Lazy.Char8 as L
 import Data.MessagePack
 
-mid :: (ObjectGet a, ObjectPut a) => a -> a
+mid :: (Packable a, Unpackable a) => a -> a
 mid = unpack . pack
 
 prop_mid_int a = a == mid a
diff --git a/haskell/test/UserData.hs b/haskell/test/UserData.hs
index 8aced13..73647ff 100644
--- a/haskell/test/UserData.hs
+++ b/haskell/test/UserData.hs
@@ -10,6 +10,13 @@ data T
 
 $(deriveObject ''T)
 
+data U
+  = C { c1 :: Int, c2 :: String }
+  | D { d1 :: Double }
+  deriving (Show)
+
+$(deriveObject ''U)
+
 main = do
   let bs = pack $ A 123 "hoge"
   print bs
@@ -17,3 +24,22 @@ main = do
   let cs = pack $ B 3.14
   print cs
   print (unpack cs :: T)
+  let oa = toObject $ A 123 "hoge"
+  print oa
+  print (fromObject oa :: T)
+  let ob = toObject $ B 3.14
+  print ob
+  print (fromObject ob :: T)
+
+  let ds = pack $ C 123 "hoge"
+  print ds
+  print (unpack ds :: U)
+  let es = pack $ D 3.14
+  print es
+  print (unpack es :: U)
+  let oc = toObject $ C 123 "hoge"
+  print oc
+  print (fromObject oc :: U)
+  let od = toObject $ D 3.14
+  print od
+  print (fromObject od :: U)

From 894ff716647eeb63b8a04e279faa09092ac9c1c7 Mon Sep 17 00:00:00 2001
From: tanakh 
Date: Fri, 24 Sep 2010 03:49:31 +0900
Subject: [PATCH 0798/1648] haskell: fix for empty constructor

---
 haskell/msgpack.cabal                  |  2 +-
 haskell/src/Data/MessagePack/Derive.hs | 28 ++++++++------
 haskell/test/UserData.hs               | 52 +++++++++++++-------------
 3 files changed, 43 insertions(+), 39 deletions(-)

diff --git a/haskell/msgpack.cabal b/haskell/msgpack.cabal
index 9950273..98133a9 100644
--- a/haskell/msgpack.cabal
+++ b/haskell/msgpack.cabal
@@ -1,5 +1,5 @@
 Name:               msgpack
-Version:            0.4.0
+Version:            0.4.0.1
 Synopsis:           A Haskell binding to MessagePack
 Description:
   A Haskell binding to MessagePack 
diff --git a/haskell/src/Data/MessagePack/Derive.hs b/haskell/src/Data/MessagePack/Derive.hs
index e998473..74943e9 100644
--- a/haskell/src/Data/MessagePack/Derive.hs
+++ b/haskell/src/Data/MessagePack/Derive.hs
@@ -7,6 +7,7 @@ module Data.MessagePack.Derive (
   ) where
 
 import Control.Applicative
+import Control.Monad
 import Language.Haskell.TH
 
 import Data.MessagePack.Pack
@@ -24,9 +25,9 @@ deriveUnpack typName = do
         
   where
     body (NormalC conName elms) =
-      DoE
-      [ BindS (tupOrListP $ map VarP names) (VarE 'get)
-      , NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ]
+      DoE $
+      tupOrListP (map VarP names) (VarE 'get) ++
+      [ NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ]
         where
           names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms
 
@@ -78,9 +79,9 @@ deriveObject typName = do
       toObjectBody (NormalC conName $ map (\(_, b, c) -> (b, c)) elms)
 
     tryFromObjectBody (NormalC conName elms) =
-      DoE
-      [ BindS (tupOrListP $ map VarP names) (AppE (VarE 'tryFromObject) (VarE oname))
-      , NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ]
+      DoE $
+      tupOrListP (map VarP names) (AppE (VarE 'tryFromObject) (VarE oname)) ++
+      [ NoBindS $ AppE (VarE 'return) $ foldl AppE (ConE conName) $ map VarE names ]
         where
           names = zipWith (\ix _ -> mkName $ "a" ++ show (ix :: Int)) [1..] elms
     tryFromObjectBody (RecC conName elms) =
@@ -89,12 +90,17 @@ deriveObject typName = do
     oname = mkName "o"
     ch = foldl1 (\e f -> AppE (AppE (VarE '(<|>)) e) f)
 
-tupOrListP :: [Pat] -> Pat
-tupOrListP ls
-  | length ls <= 1 = ListP ls
-  | otherwise = TupP ls
+tupOrListP :: [Pat] -> Exp -> [Stmt]
+tupOrListP ls e
+  | length ls == 0 =
+    let lsname = mkName "ls" in
+    [ BindS (VarP lsname) e
+    , NoBindS $ AppE (VarE 'guard) $ AppE (VarE 'null) $ SigE (VarE lsname) (AppT ListT (ConT ''())) ]
+  | length ls == 1 = [ BindS (ListP ls) e ]
+  | otherwise = [ BindS (TupP ls) e ]
 
 tupOrListE :: [Exp] -> Exp
 tupOrListE ls
-  | length ls <= 1 = ListE ls
+  | length ls == 0 = SigE (ListE []) (AppT ListT (ConT ''()))
+  | length ls == 1 = ListE ls
   | otherwise = TupE ls
diff --git a/haskell/test/UserData.hs b/haskell/test/UserData.hs
index 73647ff..5e5d0ea 100644
--- a/haskell/test/UserData.hs
+++ b/haskell/test/UserData.hs
@@ -6,40 +6,38 @@ import Data.MessagePack.Derive
 data T
   = A Int String
   | B Double
-  deriving (Show)
+  deriving (Show, Eq)
 
 $(deriveObject ''T)
 
 data U
   = C { c1 :: Int, c2 :: String }
   | D { d1 :: Double }
-  deriving (Show)
+  deriving (Show, Eq)
 
 $(deriveObject ''U)
 
-main = do
-  let bs = pack $ A 123 "hoge"
-  print bs
-  print (unpack bs :: T)
-  let cs = pack $ B 3.14
-  print cs
-  print (unpack cs :: T)
-  let oa = toObject $ A 123 "hoge"
-  print oa
-  print (fromObject oa :: T)
-  let ob = toObject $ B 3.14
-  print ob
-  print (fromObject ob :: T)
+data V
+  = E String | F
+  deriving (Show, Eq)
 
-  let ds = pack $ C 123 "hoge"
-  print ds
-  print (unpack ds :: U)
-  let es = pack $ D 3.14
-  print es
-  print (unpack es :: U)
-  let oc = toObject $ C 123 "hoge"
-  print oc
-  print (fromObject oc :: U)
-  let od = toObject $ D 3.14
-  print od
-  print (fromObject od :: U)
+$(deriveObject ''V)
+
+test :: (OBJECT a, Show a, Eq a) => a -> IO ()
+test v = do
+  let bs = pack v
+  print bs
+  print (unpack bs == v)
+
+  let oa = toObject v
+  print oa
+  print (fromObject oa == v)
+
+main = do
+  test $ A 123 "hoge"
+  test $ B 3.14
+  test $ C 123 "hoge"
+  test $ D 3.14
+  test $ E "hello"
+  test $ F
+  return ()
\ No newline at end of file

From e8abcc1765da98885d03cb0ca0784cf332bf0ee0 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 26 Sep 2010 11:36:57 +0900
Subject: [PATCH 0799/1648] cpp: sbuffer: check initial buffer size != 0

---
 cpp/bootstrap               |  2 +-
 cpp/src/msgpack/sbuffer.hpp | 12 ++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/cpp/bootstrap b/cpp/bootstrap
index a95c304..7f3a182 100755
--- a/cpp/bootstrap
+++ b/cpp/bootstrap
@@ -38,7 +38,7 @@ test -f ChangeLog || touch ChangeLog
 test -f NEWS      || touch NEWS
 test -f README    || cp -f README.md README
 
-if ! ./preprocess; then
+if test ! ./preprocess; then
 	exit 1
 fi
 
diff --git a/cpp/src/msgpack/sbuffer.hpp b/cpp/src/msgpack/sbuffer.hpp
index a9efc6d..14c5d2a 100644
--- a/cpp/src/msgpack/sbuffer.hpp
+++ b/cpp/src/msgpack/sbuffer.hpp
@@ -28,9 +28,13 @@ class sbuffer : public msgpack_sbuffer {
 public:
 	sbuffer(size_t initsz = MSGPACK_SBUFFER_INIT_SIZE)
 	{
-		base::data = (char*)::malloc(initsz);
-		if(!base::data) {
-			throw std::bad_alloc();
+		if(initsz == 0) {
+			base::data = NULL;
+		} else {
+			base::data = (char*)::malloc(initsz);
+			if(!base::data) {
+				throw std::bad_alloc();
+			}
 		}
 
 		base::size = 0;
@@ -80,7 +84,7 @@ public:
 private:
 	void expand_buffer(size_t len)
 	{
-		size_t nsize = (base::alloc) ?
+		size_t nsize = (base::alloc > 0) ?
 				base::alloc * 2 : MSGPACK_SBUFFER_INIT_SIZE;
 	
 		while(nsize < base::size + len) { nsize *= 2; }

From 7974060a4010c91f978fe91e74b0e1967dc30a90 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 26 Sep 2010 11:37:37 +0900
Subject: [PATCH 0800/1648] cpp: zone: adds msgpack_zone_swap and
 msgpack::zone::swap

---
 cpp/src/msgpack/zone.h       | 11 ++++++++++-
 cpp/src/msgpack/zone.hpp.erb |  9 ++++++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/cpp/src/msgpack/zone.h b/cpp/src/msgpack/zone.h
index d8c60b6..0f5817f 100644
--- a/cpp/src/msgpack/zone.h
+++ b/cpp/src/msgpack/zone.h
@@ -1,7 +1,7 @@
 /*
  * MessagePack for C memory pool implementation
  *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
  *
  *    Licensed under the Apache License, Version 2.0 (the "License");
  *    you may not use this file except in compliance with the License.
@@ -73,6 +73,8 @@ static inline void* msgpack_zone_malloc_no_align(msgpack_zone* zone, size_t size
 static inline bool msgpack_zone_push_finalizer(msgpack_zone* zone,
 		void (*func)(void* data), void* data);
 
+static inline void msgpack_zone_swap(msgpack_zone* a, msgpack_zone* b);
+
 bool msgpack_zone_is_empty(msgpack_zone* zone);
 
 void msgpack_zone_clear(msgpack_zone* zone);
@@ -129,6 +131,13 @@ bool msgpack_zone_push_finalizer(msgpack_zone* zone,
 	return true;
 }
 
+void msgpack_zone_swap(msgpack_zone* a, msgpack_zone* b)
+{
+	msgpack_zone tmp = *a;
+	*a = *b;
+	*b = tmp;
+}
+
 
 #ifdef __cplusplus
 }
diff --git a/cpp/src/msgpack/zone.hpp.erb b/cpp/src/msgpack/zone.hpp.erb
index 1cef05e..c6f5481 100644
--- a/cpp/src/msgpack/zone.hpp.erb
+++ b/cpp/src/msgpack/zone.hpp.erb
@@ -1,7 +1,7 @@
 //
 // MessagePack for C++ memory pool
 //
-// Copyright (C) 2008-2009 FURUHASHI Sadayuki
+// Copyright (C) 2008-2010 FURUHASHI Sadayuki
 //
 //    Licensed under the Apache License, Version 2.0 (the "License");
 //    you may not use this file except in compliance with the License.
@@ -43,6 +43,8 @@ public:
 
 	void clear();
 
+	void swap(zone& o);
+
 	<%0.upto(GENERATION_LIMIT) {|i|%>
 	template , typename A<%=j%><%}%>>
 	T* allocate(<%=(1..i).map{|j|"A#{j} a#{j}"}.join(', ')%>);
@@ -111,6 +113,11 @@ inline void zone::clear()
 	msgpack_zone_clear(this);
 }
 
+inline void zone::swap(zone& o)
+{
+	msgpack_zone_swap(this, &o);
+}
+
 template 
 void zone::object_destructor(void* obj)
 {

From 446a7fbd679fffd6b2b5e3c8b5673ed824aa133f Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Mon, 27 Sep 2010 03:05:32 +0900
Subject: [PATCH 0801/1648] java: adds CustomMessage class (currently not
 implemented on Packer, Unpacker and ClassTemplate)

---
 .../java/org/msgpack/CustomConverter.java     | 39 +++++++++++++++++++
 .../main/java/org/msgpack/CustomMessage.java  | 30 ++++++++++++++
 .../main/java/org/msgpack/CustomPacker.java   | 39 +++++++++++++++++++
 .../main/java/org/msgpack/CustomUnpacker.java | 39 +++++++++++++++++++
 4 files changed, 147 insertions(+)
 create mode 100644 java/src/main/java/org/msgpack/CustomConverter.java
 create mode 100644 java/src/main/java/org/msgpack/CustomMessage.java
 create mode 100644 java/src/main/java/org/msgpack/CustomPacker.java
 create mode 100644 java/src/main/java/org/msgpack/CustomUnpacker.java

diff --git a/java/src/main/java/org/msgpack/CustomConverter.java b/java/src/main/java/org/msgpack/CustomConverter.java
new file mode 100644
index 0000000..5905419
--- /dev/null
+++ b/java/src/main/java/org/msgpack/CustomConverter.java
@@ -0,0 +1,39 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.util.Map;
+import java.util.HashMap;
+
+// FIXME public?
+class CustomConverter {
+	public static void register(Class target, MessageConverter converter) {
+		map.put(target, converter);
+	}
+
+	public static MessageConverter get(Class target) {
+		return map.get(target);
+	}
+
+	public static boolean isRegistered(Class target) {
+		return map.containsKey(target);
+	}
+
+	private static Map map = new HashMap();
+}
+
diff --git a/java/src/main/java/org/msgpack/CustomMessage.java b/java/src/main/java/org/msgpack/CustomMessage.java
new file mode 100644
index 0000000..f87898c
--- /dev/null
+++ b/java/src/main/java/org/msgpack/CustomMessage.java
@@ -0,0 +1,30 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+public class CustomMessage {
+	public static void registerPacker(Class target, MessagePacker packer) {
+		CustomPacker.register(target, packer);
+	}
+
+	public static void registerTemplate(Class target, Template tmpl) {
+		CustomUnpacker.register(target, tmpl);
+		CustomConverter.register(target, tmpl);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/CustomPacker.java b/java/src/main/java/org/msgpack/CustomPacker.java
new file mode 100644
index 0000000..f828c31
--- /dev/null
+++ b/java/src/main/java/org/msgpack/CustomPacker.java
@@ -0,0 +1,39 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.util.Map;
+import java.util.HashMap;
+
+// FIXME public?
+class CustomPacker {
+	public static void register(Class target, MessagePacker converter) {
+		map.put(target, converter);
+	}
+
+	public static MessagePacker get(Class target) {
+		return map.get(target);
+	}
+
+	public static boolean isRegistered(Class target) {
+		return map.containsKey(target);
+	}
+
+	private static Map map = new HashMap();
+}
+
diff --git a/java/src/main/java/org/msgpack/CustomUnpacker.java b/java/src/main/java/org/msgpack/CustomUnpacker.java
new file mode 100644
index 0000000..255368d
--- /dev/null
+++ b/java/src/main/java/org/msgpack/CustomUnpacker.java
@@ -0,0 +1,39 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.util.Map;
+import java.util.HashMap;
+
+// FIXME public?
+class CustomUnpacker {
+	public static void register(Class target, MessageUnpacker converter) {
+		map.put(target, converter);
+	}
+
+	public static MessageUnpacker get(Class target) {
+		return map.get(target);
+	}
+
+	public static boolean isRegistered(Class target) {
+		return map.containsKey(target);
+	}
+
+	private static Map map = new HashMap();
+}
+

From 391034a7859dac3e1c77164c46da03026ec4d743 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Mon, 27 Sep 2010 03:06:06 +0900
Subject: [PATCH 0802/1648] java: adds type-conversion mechanisms

---
 .../java/org/msgpack/AbstractTemplate.java    | 27 +++++++++
 .../java/org/msgpack/MessageConverter.java    | 25 +++++++++
 .../java/org/msgpack/MessagePackObject.java   |  4 ++
 .../main/java/org/msgpack/MessagePacker.java  | 25 +++++++++
 .../java/org/msgpack/MessageUnpacker.java     | 25 +++++++++
 java/src/main/java/org/msgpack/Packer.java    |  2 +
 java/src/main/java/org/msgpack/Template.java  | 22 ++++++++
 java/src/main/java/org/msgpack/Templates.java | 45 +++++++++++++++
 java/src/main/java/org/msgpack/Unpacker.java  | 15 ++++-
 .../msgpack/template/ByteArrayTemplate.java   | 44 +++++++++++++++
 .../org/msgpack/template/ClassTemplate.java   | 45 +++++++++++++++
 .../org/msgpack/template/ListTemplate.java    | 50 +++++++++++++++++
 .../org/msgpack/template/MapTemplate.java     | 56 +++++++++++++++++++
 .../org/msgpack/template/StringTemplate.java  | 44 +++++++++++++++
 14 files changed, 427 insertions(+), 2 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/AbstractTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/MessageConverter.java
 create mode 100644 java/src/main/java/org/msgpack/MessagePacker.java
 create mode 100644 java/src/main/java/org/msgpack/MessageUnpacker.java
 create mode 100644 java/src/main/java/org/msgpack/Template.java
 create mode 100644 java/src/main/java/org/msgpack/Templates.java
 create mode 100644 java/src/main/java/org/msgpack/template/ByteArrayTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/template/ClassTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/template/ListTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/template/MapTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/template/StringTemplate.java

diff --git a/java/src/main/java/org/msgpack/AbstractTemplate.java b/java/src/main/java/org/msgpack/AbstractTemplate.java
new file mode 100644
index 0000000..5b4442e
--- /dev/null
+++ b/java/src/main/java/org/msgpack/AbstractTemplate.java
@@ -0,0 +1,27 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.io.IOException;
+
+public abstract class AbstractTemplate implements Template {
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return convert(pac.unpackObject());
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/MessageConverter.java b/java/src/main/java/org/msgpack/MessageConverter.java
new file mode 100644
index 0000000..8388ddc
--- /dev/null
+++ b/java/src/main/java/org/msgpack/MessageConverter.java
@@ -0,0 +1,25 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.io.IOException;
+
+public interface MessageConverter {
+	public Object convert(MessagePackObject obj) throws MessageTypeException;
+}
+
diff --git a/java/src/main/java/org/msgpack/MessagePackObject.java b/java/src/main/java/org/msgpack/MessagePackObject.java
index 2424446..f7e9e0e 100644
--- a/java/src/main/java/org/msgpack/MessagePackObject.java
+++ b/java/src/main/java/org/msgpack/MessagePackObject.java
@@ -132,5 +132,9 @@ public abstract class MessagePackObject implements Cloneable, MessagePackable {
 	}
 
 	abstract public Object clone();
+
+	public Object convert(Template tmpl) throws MessageTypeException {
+		return tmpl.convert(this);
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/MessagePacker.java b/java/src/main/java/org/msgpack/MessagePacker.java
new file mode 100644
index 0000000..05d4de5
--- /dev/null
+++ b/java/src/main/java/org/msgpack/MessagePacker.java
@@ -0,0 +1,25 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.io.IOException;
+
+public interface MessagePacker {
+	public void pack(Packer pk, Object target) throws IOException;
+}
+
diff --git a/java/src/main/java/org/msgpack/MessageUnpacker.java b/java/src/main/java/org/msgpack/MessageUnpacker.java
new file mode 100644
index 0000000..1817269
--- /dev/null
+++ b/java/src/main/java/org/msgpack/MessageUnpacker.java
@@ -0,0 +1,25 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.io.IOException;
+
+public interface MessageUnpacker {
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException;
+}
+
diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index 139b3b1..687f09c 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -474,6 +474,8 @@ public class Packer {
 		} else if(o instanceof BigInteger) {
 			return packBigInteger((BigInteger)o);
 		} else {
+			// FIXME check CustomPacker.get(o.getClass());
+			// FIXME check annotations
 			throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")");
 		}
 	}
diff --git a/java/src/main/java/org/msgpack/Template.java b/java/src/main/java/org/msgpack/Template.java
new file mode 100644
index 0000000..71e64e0
--- /dev/null
+++ b/java/src/main/java/org/msgpack/Template.java
@@ -0,0 +1,22 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+public interface Template extends MessageUnpacker, MessageConverter {
+}
+
diff --git a/java/src/main/java/org/msgpack/Templates.java b/java/src/main/java/org/msgpack/Templates.java
new file mode 100644
index 0000000..4c3fdae
--- /dev/null
+++ b/java/src/main/java/org/msgpack/Templates.java
@@ -0,0 +1,45 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import org.msgpack.template.*;
+
+public class Templates {
+	public static Template tList(Template elementTemplate) {
+		return new ListTemplate(elementTemplate);
+	}
+
+	public static Template tMap(Template keyTemplate, Template valueTemplate) {
+		return new MapTemplate(keyTemplate, valueTemplate);
+	}
+
+
+	public static final Template TString = StringTemplate.getInstance();
+
+	public static Template tString() {
+		return TString;
+	}
+
+
+	public static final Template TByteArray = ByteArrayTemplate.getInstance();
+
+	public static Template tByteArray() {
+		return TByteArray;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index 3cae502..1b5621f 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -561,12 +561,23 @@ public class Unpacker implements Iterable {
 		return impl.unpackObject();
 	}
 
+	final public boolean tryUnpackNull() throws IOException {
+		return impl.tryUnpackNull();
+	}
+
+	final public Object unpack(MessageUnpacker unpacker) throws IOException, MessageTypeException {
+		return unpacker.unpack(this);
+	}
+
 	final public void unpack(MessageUnpackable obj) throws IOException, MessageTypeException {
 		obj.messageUnpack(this);
 	}
 
-	final public boolean tryUnpackNull() throws IOException {
-		return impl.tryUnpackNull();
+	final public Object unpack(Class klass) throws MessageTypeException {
+		// FIXME check MessageUnpackable
+		// FIXME check CustomPacker
+		// FIXME check annotations
+		throw new MessageTypeException();
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/ByteArrayTemplate.java b/java/src/main/java/org/msgpack/template/ByteArrayTemplate.java
new file mode 100644
index 0000000..fe26369
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/ByteArrayTemplate.java
@@ -0,0 +1,44 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class ByteArrayTemplate implements Template {
+	private ByteArrayTemplate() { }
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpackByteArray();
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		return from.asByteArray();
+	}
+
+	static public ByteArrayTemplate getInstance() {
+		return instance;
+	}
+
+	static final ByteArrayTemplate instance = new ByteArrayTemplate();
+
+	static {
+		CustomMessage.registerTemplate(byte[].class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/ClassTemplate.java b/java/src/main/java/org/msgpack/template/ClassTemplate.java
new file mode 100644
index 0000000..b5ed854
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/ClassTemplate.java
@@ -0,0 +1,45 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class ClassTemplate implements Template {
+	private Class klass;
+
+	public ClassTemplate(Class klass) {
+		this.klass = klass;
+	}
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpack(klass);
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		MessageConverter converter = CustomConverter.get(klass);
+		if(converter != null) {
+			return converter.convert(from);
+		}
+
+		// FIXME check annotations
+
+		throw new MessageTypeException();
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/ListTemplate.java b/java/src/main/java/org/msgpack/template/ListTemplate.java
new file mode 100644
index 0000000..54975f8
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/ListTemplate.java
@@ -0,0 +1,50 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.io.IOException;
+import org.msgpack.*;
+
+public class ListTemplate implements Template {
+	private Template elementTemplate;
+
+	public ListTemplate(Template elementTemplate) {
+		this.elementTemplate = elementTemplate;
+	}
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		int length = pac.unpackArray();
+		List list = new ArrayList(length);
+		for(; length > 0; length--) {
+			list.add( elementTemplate.unpack(pac) );
+		}
+		return list;
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		MessagePackObject[] array = from.asArray();
+		List list = new ArrayList(array.length);
+		for(MessagePackObject element : array) {
+			list.add( elementTemplate.convert(element) );
+		}
+		return list;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/MapTemplate.java b/java/src/main/java/org/msgpack/template/MapTemplate.java
new file mode 100644
index 0000000..d2b4eff
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/MapTemplate.java
@@ -0,0 +1,56 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.io.IOException;
+import org.msgpack.*;
+
+public class MapTemplate implements Template {
+	private Template keyTemplate;
+	private Template valueTemplate;
+
+	public MapTemplate(Template keyTemplate, Template valueTemplate) {
+		this.keyTemplate = keyTemplate;
+		this.valueTemplate = valueTemplate;
+	}
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		int length = pac.unpackMap();
+		Map map = new HashMap(length);
+		for(; length > 0; length--) {
+			Object key = keyTemplate.unpack(pac);
+			Object value = valueTemplate.unpack(pac);
+			map.put(key, value);
+		}
+		return map;
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		Map src = from.asMap();
+		Map map = new HashMap();
+		for(Map.Entry pair : src.entrySet()) {
+			Object key = keyTemplate.convert(pair.getKey());
+			Object value = valueTemplate.convert(pair.getValue());
+			map.put(key, value);
+		}
+		return map;
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/StringTemplate.java b/java/src/main/java/org/msgpack/template/StringTemplate.java
new file mode 100644
index 0000000..563112d
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/StringTemplate.java
@@ -0,0 +1,44 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class StringTemplate implements Template {
+	private StringTemplate() { }
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpackString();
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		return from.asString();
+	}
+
+	static public StringTemplate getInstance() {
+		return instance;
+	}
+
+	static final StringTemplate instance = new StringTemplate();
+
+	static {
+		CustomMessage.registerTemplate(String.class, instance);
+	}
+}
+

From 7161a235f180fcb23e6451c6a10bfa5a55065eff Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Mon, 27 Sep 2010 03:33:21 +0900
Subject: [PATCH 0803/1648] java: uses CustomMessage class on Packer, Unpacker
 and ClassTemplate

---
 .../java/org/msgpack/CustomConverter.java     |  4 ++--
 .../main/java/org/msgpack/CustomPacker.java   |  4 ++--
 .../main/java/org/msgpack/CustomUnpacker.java |  4 ++--
 java/src/main/java/org/msgpack/Packer.java    | 16 +++++++++----
 java/src/main/java/org/msgpack/Templates.java |  4 ++++
 java/src/main/java/org/msgpack/Unpacker.java  | 23 +++++++++++++++----
 .../org/msgpack/template/ClassTemplate.java   | 23 +++++++++++++++++--
 7 files changed, 62 insertions(+), 16 deletions(-)

diff --git a/java/src/main/java/org/msgpack/CustomConverter.java b/java/src/main/java/org/msgpack/CustomConverter.java
index 5905419..4351464 100644
--- a/java/src/main/java/org/msgpack/CustomConverter.java
+++ b/java/src/main/java/org/msgpack/CustomConverter.java
@@ -20,8 +20,8 @@ package org.msgpack;
 import java.util.Map;
 import java.util.HashMap;
 
-// FIXME public?
-class CustomConverter {
+// FIXME package private?
+public class CustomConverter {
 	public static void register(Class target, MessageConverter converter) {
 		map.put(target, converter);
 	}
diff --git a/java/src/main/java/org/msgpack/CustomPacker.java b/java/src/main/java/org/msgpack/CustomPacker.java
index f828c31..f0c6b62 100644
--- a/java/src/main/java/org/msgpack/CustomPacker.java
+++ b/java/src/main/java/org/msgpack/CustomPacker.java
@@ -20,8 +20,8 @@ package org.msgpack;
 import java.util.Map;
 import java.util.HashMap;
 
-// FIXME public?
-class CustomPacker {
+// FIXME package private?
+public class CustomPacker {
 	public static void register(Class target, MessagePacker converter) {
 		map.put(target, converter);
 	}
diff --git a/java/src/main/java/org/msgpack/CustomUnpacker.java b/java/src/main/java/org/msgpack/CustomUnpacker.java
index 255368d..b6047b8 100644
--- a/java/src/main/java/org/msgpack/CustomUnpacker.java
+++ b/java/src/main/java/org/msgpack/CustomUnpacker.java
@@ -20,8 +20,8 @@ package org.msgpack;
 import java.util.Map;
 import java.util.HashMap;
 
-// FIXME public?
-class CustomUnpacker {
+// FIXME package private?
+public class CustomUnpacker {
 	public static void register(Class target, MessageUnpacker converter) {
 		map.put(target, converter);
 	}
diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index 687f09c..dd8cdee 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -473,11 +473,19 @@ public class Packer {
 			return packDouble((Double)o);
 		} else if(o instanceof BigInteger) {
 			return packBigInteger((BigInteger)o);
-		} else {
-			// FIXME check CustomPacker.get(o.getClass());
-			// FIXME check annotations
-			throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")");
 		}
+
+		Class klass = o.getClass();
+
+		MessagePacker packer = CustomPacker.get(klass);
+		if(packer != null) {
+			packer.pack(this, o);
+			return this;
+		}
+
+		// FIXME check annotations -> code generation -> CustomMessage.registerPacker
+
+		throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")");
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/Templates.java b/java/src/main/java/org/msgpack/Templates.java
index 4c3fdae..a31dd91 100644
--- a/java/src/main/java/org/msgpack/Templates.java
+++ b/java/src/main/java/org/msgpack/Templates.java
@@ -28,6 +28,10 @@ public class Templates {
 		return new MapTemplate(keyTemplate, valueTemplate);
 	}
 
+	public static Template tClass(Class target) {
+		return new ClassTemplate(target);
+	}
+
 
 	public static final Template TString = StringTemplate.getInstance();
 
diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index 1b5621f..4e39748 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -573,10 +573,25 @@ public class Unpacker implements Iterable {
 		obj.messageUnpack(this);
 	}
 
-	final public Object unpack(Class klass) throws MessageTypeException {
-		// FIXME check MessageUnpackable
-		// FIXME check CustomPacker
-		// FIXME check annotations
+	final public Object unpack(Class klass) throws IOException, MessageTypeException, InstantiationException, IllegalAccessException {
+		if(MessageUnpackable.class.isAssignableFrom(klass)) {
+			Object obj = klass.newInstance();
+			((MessageUnpackable)obj).messageUnpack(this);
+			return obj;
+		}
+
+		MessageUnpacker unpacker = CustomUnpacker.get(klass);
+		if(unpacker != null) {
+			return unpacker.unpack(this);
+		}
+
+		// FIXME check annotations -> code generation -> CustomMessage.registerTemplate
+
+		MessageConverter converter = CustomConverter.get(klass);
+		if(converter != null) {
+			return converter.convert(unpackObject());
+		}
+
 		throw new MessageTypeException();
 	}
 }
diff --git a/java/src/main/java/org/msgpack/template/ClassTemplate.java b/java/src/main/java/org/msgpack/template/ClassTemplate.java
index b5ed854..c529edd 100644
--- a/java/src/main/java/org/msgpack/template/ClassTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ClassTemplate.java
@@ -28,16 +28,35 @@ public class ClassTemplate implements Template {
 	}
 
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
-		return pac.unpack(klass);
+		try {
+			return pac.unpack(klass);
+		} catch (IllegalAccessException e) {
+			throw new MessageTypeException(e.getMessage());  // FIXME
+		} catch (InstantiationException e) {
+			throw new MessageTypeException(e.getMessage());  // FIXME
+		}
 	}
 
 	public Object convert(MessagePackObject from) throws MessageTypeException {
+		if(MessageConvertable.class.isAssignableFrom(klass)) {
+			Object obj;
+			try {
+				obj = klass.newInstance();
+			} catch (IllegalAccessException e) {
+				throw new MessageTypeException(e.getMessage());  // FIXME
+			} catch (InstantiationException e) {
+				throw new MessageTypeException(e.getMessage());  // FIXME
+			}
+			((MessageConvertable)obj).messageConvert(from);
+			return obj;
+		}
+
 		MessageConverter converter = CustomConverter.get(klass);
 		if(converter != null) {
 			return converter.convert(from);
 		}
 
-		// FIXME check annotations
+		// FIXME check annotations -> code generation -> CustomMessage.registerTemplate
 
 		throw new MessageTypeException();
 	}

From 002b86198c2a4db8aa6372f12fdaa055b9157e89 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Mon, 27 Sep 2010 04:17:20 +0900
Subject: [PATCH 0804/1648] java: adds ReflectionPacker and ReflectionTemplate

---
 .../java/org/msgpack/MessageConverter.java    |  2 +-
 .../main/java/org/msgpack/ReflectionBase.java | 26 +++++++
 .../java/org/msgpack/ReflectionPacker.java    | 49 ++++++++++++
 .../java/org/msgpack/ReflectionTemplate.java  | 74 +++++++++++++++++++
 .../msgpack/TestReflectionPackerTemplate.java | 46 ++++++++++++
 5 files changed, 196 insertions(+), 1 deletion(-)
 create mode 100644 java/src/main/java/org/msgpack/ReflectionBase.java
 create mode 100644 java/src/main/java/org/msgpack/ReflectionPacker.java
 create mode 100644 java/src/main/java/org/msgpack/ReflectionTemplate.java
 create mode 100644 java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java

diff --git a/java/src/main/java/org/msgpack/MessageConverter.java b/java/src/main/java/org/msgpack/MessageConverter.java
index 8388ddc..8ad60ae 100644
--- a/java/src/main/java/org/msgpack/MessageConverter.java
+++ b/java/src/main/java/org/msgpack/MessageConverter.java
@@ -20,6 +20,6 @@ package org.msgpack;
 import java.io.IOException;
 
 public interface MessageConverter {
-	public Object convert(MessagePackObject obj) throws MessageTypeException;
+	public Object convert(MessagePackObject from) throws MessageTypeException;
 }
 
diff --git a/java/src/main/java/org/msgpack/ReflectionBase.java b/java/src/main/java/org/msgpack/ReflectionBase.java
new file mode 100644
index 0000000..66ec12a
--- /dev/null
+++ b/java/src/main/java/org/msgpack/ReflectionBase.java
@@ -0,0 +1,26 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.io.IOException;
+import java.lang.reflect.*;
+
+// FIXME mock-up
+abstract class ReflectionBase {
+}
+
diff --git a/java/src/main/java/org/msgpack/ReflectionPacker.java b/java/src/main/java/org/msgpack/ReflectionPacker.java
new file mode 100644
index 0000000..72406aa
--- /dev/null
+++ b/java/src/main/java/org/msgpack/ReflectionPacker.java
@@ -0,0 +1,49 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.io.IOException;
+import java.lang.reflect.*;
+
+// FIXME mock-up
+public class ReflectionPacker extends ReflectionBase implements MessagePacker {
+	private Class klass;
+
+	private ReflectionPacker(Class klass) {
+		this.klass = klass;
+	}
+
+	static public ReflectionPacker create(Class klass) {
+		// FIXME code generation: generates subclass of ReflectionPacker
+		//       returned instance will be cached by Packer into CustomPacker
+		return new ReflectionPacker(klass);
+	}
+
+	public void pack(Packer pk, Object target) throws IOException {
+		Field[] fields = klass.getDeclaredFields();
+		pk.packArray(fields.length);
+		try {
+			for(int i=0; i < fields.length; i++) {
+				pk.pack(fields[i].get(target));
+			}
+		} catch(IllegalAccessException e) {
+			throw new MessageTypeException(e.getMessage());  // FIXME
+		}
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/ReflectionTemplate.java b/java/src/main/java/org/msgpack/ReflectionTemplate.java
new file mode 100644
index 0000000..5b49078
--- /dev/null
+++ b/java/src/main/java/org/msgpack/ReflectionTemplate.java
@@ -0,0 +1,74 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.io.IOException;
+import java.lang.reflect.*;
+import org.msgpack.template.ClassTemplate;
+
+// FIXME mock-up
+public class ReflectionTemplate extends ReflectionBase implements Template {
+	private Class klass;
+
+	private ReflectionTemplate(Class klass) {
+		this.klass = klass;
+	}
+
+	static public ReflectionTemplate create(Class klass) {
+		// FIXME code generation: generates subclass of ReflectionPacker
+		//       returned instance will be cached by ClassTemplate into CustomUnpacker/CustomConverter
+		return new ReflectionTemplate(klass);
+	}
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		// FIXME optimize it
+		return convert(pac.unpackObject());
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		Object obj;
+		try {
+			obj = klass.newInstance();
+		} catch (IllegalAccessException e) {
+			throw new MessageTypeException(e.getMessage());  // FIXME
+		} catch (InstantiationException e) {
+			throw new MessageTypeException(e.getMessage());  // FIXME
+		}
+
+		// FIXME check Requred/Optional
+
+		Field[] fields = klass.getDeclaredFields();
+		MessagePackObject[] array = from.asArray();
+		if(fields.length < array.length) {
+			throw new MessageTypeException();
+		}
+
+		try {
+			for(int i=0; i < fields.length; i++) {
+				// FIXME generics getDeclaringClass
+				Object value = new ClassTemplate(fields[i].getType()).convert(array[i]);
+				fields[i].set(obj, value);
+			}
+		} catch(IllegalAccessException e) {
+			throw new MessageTypeException(e.getMessage());  // FIXME
+		}
+
+		return obj;
+	}
+}
+
diff --git a/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java b/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
new file mode 100644
index 0000000..ddf88a0
--- /dev/null
+++ b/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
@@ -0,0 +1,46 @@
+package org.msgpack;
+
+import static org.msgpack.Templates.*;
+
+import java.util.*;
+import java.io.*;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestReflectionPackerTemplate {
+
+	public static class StringFieldClass {
+		public String s1;
+		public String s2;
+		public StringFieldClass() { }
+	}
+
+	@Test
+	public void testPackConvert() throws Exception {
+		tString();
+
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+		MessagePacker packer = ReflectionPacker.create(StringFieldClass.class);
+
+		StringFieldClass src = new StringFieldClass();
+
+		src.s1 = "kumofs";
+		src.s2 = "frsyuki";
+
+		packer.pack(new Packer(out), src);
+
+		Template tmpl = ReflectionTemplate.create(StringFieldClass.class);
+
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+
+		Object obj = tmpl.unpack(new Unpacker(in));
+		assertEquals(obj.getClass(), StringFieldClass.class);
+
+		StringFieldClass dst = (StringFieldClass)obj;
+		assertEquals(src.s1, dst.s1);
+		assertEquals(src.s2, dst.s2);
+	}
+}
+

From 0a41b253f3a688cbdd11c2cfcb574f1333b32a0e Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Mon, 27 Sep 2010 04:27:44 +0900
Subject: [PATCH 0805/1648] java: adds templates for primitive types

---
 java/src/main/java/org/msgpack/Templates.java | 43 ++++++++++++++++--
 .../msgpack/template/BigIntegerTemplate.java  | 45 +++++++++++++++++++
 .../org/msgpack/template/BooleanTemplate.java | 44 ++++++++++++++++++
 .../org/msgpack/template/ByteTemplate.java    | 44 ++++++++++++++++++
 .../org/msgpack/template/DoubleTemplate.java  | 44 ++++++++++++++++++
 .../org/msgpack/template/FloatTemplate.java   | 44 ++++++++++++++++++
 .../org/msgpack/template/IntegerTemplate.java | 44 ++++++++++++++++++
 .../org/msgpack/template/LongTemplate.java    | 44 ++++++++++++++++++
 .../org/msgpack/template/ShortTemplate.java   | 44 ++++++++++++++++++
 .../msgpack/TestReflectionPackerTemplate.java |  2 +-
 10 files changed, 394 insertions(+), 4 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/template/BigIntegerTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/template/BooleanTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/template/ByteTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/template/DoubleTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/template/FloatTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/template/IntegerTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/template/LongTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/template/ShortTemplate.java

diff --git a/java/src/main/java/org/msgpack/Templates.java b/java/src/main/java/org/msgpack/Templates.java
index a31dd91..222f625 100644
--- a/java/src/main/java/org/msgpack/Templates.java
+++ b/java/src/main/java/org/msgpack/Templates.java
@@ -33,15 +33,52 @@ public class Templates {
 	}
 
 
-	public static final Template TString = StringTemplate.getInstance();
+	public static final Template TByte = ByteTemplate.getInstance();
+	public static Template tByte() {
+		return TByte;
+	}
 
+	public static final Template TShort = ShortTemplate.getInstance();
+	public static Template tShort() {
+		return TShort;
+	}
+
+	public static final Template TInteger = IntegerTemplate.getInstance();
+	public static Template tInteger() {
+		return TInteger;
+	}
+
+	public static final Template TLong = LongTemplate.getInstance();
+	public static Template tLong() {
+		return TLong;
+	}
+
+	public static final Template TBigInteger = BigIntegerTemplate.getInstance();
+	public static Template tBigInteger() {
+		return TBigInteger;
+	}
+
+	public static final Template TFloat = FloatTemplate.getInstance();
+	public static Template tFloat() {
+		return TFloat;
+	}
+
+	public static final Template TDouble = DoubleTemplate.getInstance();
+	public static Template tDouble() {
+		return TDouble;
+	}
+
+	public static final Template TBoolean = BooleanTemplate.getInstance();
+	public static Template tBoolean() {
+		return TBoolean;
+	}
+
+	public static final Template TString = StringTemplate.getInstance();
 	public static Template tString() {
 		return TString;
 	}
 
-
 	public static final Template TByteArray = ByteArrayTemplate.getInstance();
-
 	public static Template tByteArray() {
 		return TByteArray;
 	}
diff --git a/java/src/main/java/org/msgpack/template/BigIntegerTemplate.java b/java/src/main/java/org/msgpack/template/BigIntegerTemplate.java
new file mode 100644
index 0000000..e8a2993
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/BigIntegerTemplate.java
@@ -0,0 +1,45 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import org.msgpack.*;
+
+public class BigIntegerTemplate implements Template {
+	private BigIntegerTemplate() { }
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpackBigInteger();
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		return from.asBigInteger();
+	}
+
+	static public BigIntegerTemplate getInstance() {
+		return instance;
+	}
+
+	static final BigIntegerTemplate instance = new BigIntegerTemplate();
+
+	static {
+		CustomMessage.registerTemplate(BigInteger.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/BooleanTemplate.java b/java/src/main/java/org/msgpack/template/BooleanTemplate.java
new file mode 100644
index 0000000..0d64ecc
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/BooleanTemplate.java
@@ -0,0 +1,44 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class BooleanTemplate implements Template {
+	private BooleanTemplate() { }
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpackBoolean();
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		return from.asBoolean();
+	}
+
+	static public BooleanTemplate getInstance() {
+		return instance;
+	}
+
+	static final BooleanTemplate instance = new BooleanTemplate();
+
+	static {
+		CustomMessage.registerTemplate(Boolean.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/ByteTemplate.java b/java/src/main/java/org/msgpack/template/ByteTemplate.java
new file mode 100644
index 0000000..8d0e6e6
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/ByteTemplate.java
@@ -0,0 +1,44 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class ByteTemplate implements Template {
+	private ByteTemplate() { }
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpackByte();
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		return from.asByte();
+	}
+
+	static public ByteTemplate getInstance() {
+		return instance;
+	}
+
+	static final ByteTemplate instance = new ByteTemplate();
+
+	static {
+		CustomMessage.registerTemplate(Byte.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/DoubleTemplate.java b/java/src/main/java/org/msgpack/template/DoubleTemplate.java
new file mode 100644
index 0000000..2e26f50
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/DoubleTemplate.java
@@ -0,0 +1,44 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class DoubleTemplate implements Template {
+	private DoubleTemplate() { }
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpackDouble();
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		return from.asDouble();
+	}
+
+	static public DoubleTemplate getInstance() {
+		return instance;
+	}
+
+	static final DoubleTemplate instance = new DoubleTemplate();
+
+	static {
+		CustomMessage.registerTemplate(Double.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/FloatTemplate.java b/java/src/main/java/org/msgpack/template/FloatTemplate.java
new file mode 100644
index 0000000..8730172
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/FloatTemplate.java
@@ -0,0 +1,44 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class FloatTemplate implements Template {
+	private FloatTemplate() { }
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpackFloat();
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		return from.asFloat();
+	}
+
+	static public FloatTemplate getInstance() {
+		return instance;
+	}
+
+	static final FloatTemplate instance = new FloatTemplate();
+
+	static {
+		CustomMessage.registerTemplate(Float.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/IntegerTemplate.java b/java/src/main/java/org/msgpack/template/IntegerTemplate.java
new file mode 100644
index 0000000..c56c044
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/IntegerTemplate.java
@@ -0,0 +1,44 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class IntegerTemplate implements Template {
+	private IntegerTemplate() { }
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpackInt();
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		return from.asInt();
+	}
+
+	static public IntegerTemplate getInstance() {
+		return instance;
+	}
+
+	static final IntegerTemplate instance = new IntegerTemplate();
+
+	static {
+		CustomMessage.registerTemplate(Integer.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/LongTemplate.java b/java/src/main/java/org/msgpack/template/LongTemplate.java
new file mode 100644
index 0000000..a0c8210
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/LongTemplate.java
@@ -0,0 +1,44 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class LongTemplate implements Template {
+	private LongTemplate() { }
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpackLong();
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		return from.asLong();
+	}
+
+	static public LongTemplate getInstance() {
+		return instance;
+	}
+
+	static final LongTemplate instance = new LongTemplate();
+
+	static {
+		CustomMessage.registerTemplate(Long.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/ShortTemplate.java b/java/src/main/java/org/msgpack/template/ShortTemplate.java
new file mode 100644
index 0000000..b3bf43b
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/ShortTemplate.java
@@ -0,0 +1,44 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class ShortTemplate implements Template {
+	private ShortTemplate() { }
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpackShort();
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		return from.asShort();
+	}
+
+	static public ShortTemplate getInstance() {
+		return instance;
+	}
+
+	static final ShortTemplate instance = new ShortTemplate();
+
+	static {
+		CustomMessage.registerTemplate(Short.class, instance);
+	}
+}
+
diff --git a/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java b/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
index ddf88a0..f361eb4 100644
--- a/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
+++ b/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
@@ -18,7 +18,7 @@ public class TestReflectionPackerTemplate {
 
 	@Test
 	public void testPackConvert() throws Exception {
-		tString();
+		tString();  // FIXME link StringTemplate
 
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 

From 54e03a62bd2fa20054835939907bdc03aa212cd6 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Mon, 27 Sep 2010 08:35:26 +0900
Subject: [PATCH 0806/1648] Checking in changes prior to tagging of version
 0.28.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index 82174fe..b506234 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.28
+
+    - added more tests(gfx)
+    - refactor the PP code(gfx)
+
 0.27

     - * 6d9a629 perl: modified trivial codes in PP::Unpacker(makamaka)
---
 perl/Changes                 | 5 +++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index 82174fe..b506234 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.28
+
+    - added more tests(gfx)
+    - refactor the PP code(gfx)
+
 0.27
 
     - * 6d9a629 perl: modified trivial codes in PP::Unpacker(makamaka)
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 5e01b08..1fccbf7 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.27';
+our $VERSION = '0.28';
 our $PreferInteger = 0;
 
 sub true () {

From dfb97e7961e7947d09240f18234a5aaced6e2d6f Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Mon, 27 Sep 2010 10:00:47 +0900
Subject: [PATCH 0807/1648] java: adds several annotations in an
 org.msgpack.annotation package and edits Packer.java and its test program

---
 .../main/java/org/msgpack/CustomPacker.java   | 17 ++++-----
 java/src/main/java/org/msgpack/Packer.java    | 38 +++++++++++++++----
 .../annotation/MessagePackDelegate.java       | 12 ++++++
 .../annotation/MessagePackMessage.java        | 12 ++++++
 .../annotation/MessagePackOrdinalEnum.java    | 12 ++++++
 .../msgpack/TestReflectionPackerTemplate.java | 28 ++++++++++++++
 6 files changed, 101 insertions(+), 18 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/annotation/MessagePackDelegate.java
 create mode 100644 java/src/main/java/org/msgpack/annotation/MessagePackMessage.java
 create mode 100644 java/src/main/java/org/msgpack/annotation/MessagePackOrdinalEnum.java

diff --git a/java/src/main/java/org/msgpack/CustomPacker.java b/java/src/main/java/org/msgpack/CustomPacker.java
index f0c6b62..743a52b 100644
--- a/java/src/main/java/org/msgpack/CustomPacker.java
+++ b/java/src/main/java/org/msgpack/CustomPacker.java
@@ -17,23 +17,20 @@
 //
 package org.msgpack;
 
-import java.util.Map;
-import java.util.HashMap;
+import java.util.concurrent.ConcurrentHashMap;
 
-// FIXME package private?
 public class CustomPacker {
-	public static void register(Class target, MessagePacker converter) {
-		map.put(target, converter);
+	private static ConcurrentHashMap, MessagePacker> map = new ConcurrentHashMap, MessagePacker>();
+
+	public static void register(Class target, MessagePacker packer) {
+		map.putIfAbsent(target, packer);
 	}
 
-	public static MessagePacker get(Class target) {
+	public static MessagePacker get(Class target) {
 		return map.get(target);
 	}
 
-	public static boolean isRegistered(Class target) {
+	public static boolean isRegistered(Class target) {
 		return map.containsKey(target);
 	}
-
-	private static Map map = new HashMap();
 }
-
diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index dd8cdee..2f34bde 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -22,8 +22,13 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.List;
 import java.util.Map;
+import java.lang.annotation.Annotation;
 import java.math.BigInteger;
 
+import org.msgpack.annotation.MessagePackDelegate;
+import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.annotation.MessagePackOrdinalEnum;
+
 /**
  * Packer enables you to serialize objects into OutputStream.
  *
@@ -473,19 +478,36 @@ public class Packer {
 			return packDouble((Double)o);
 		} else if(o instanceof BigInteger) {
 			return packBigInteger((BigInteger)o);
-		}
+		} 
 
-		Class klass = o.getClass();
-
-		MessagePacker packer = CustomPacker.get(klass);
-		if(packer != null) {
+		Class klass = o.getClass();
+		if (CustomPacker.isRegistered(klass)) {
+			MessagePacker packer = CustomPacker.get(klass);
 			packer.pack(this, o);
 			return this;
+		} else if (isAnnotated(klass, MessagePackMessage.class)) {
+			MessagePacker packer = ReflectionPacker.create(klass);
+			CustomPacker.register(klass, packer);
+			packer.pack(this, o);
+			return this;
+		} else if (isAnnotated(klass, MessagePackDelegate.class)) {
+			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
+		} else if (isAnnotated(klass, MessagePackOrdinalEnum.class)) {
+			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
 		}
-
-		// FIXME check annotations -> code generation -> CustomMessage.registerPacker
+//      Class klass = o.getClass();
+//		MessagePacker packer = CustomPacker.get(klass);
+//		if(packer != null) {
+//			packer.pack(this, o);
+//			return this;
+//		}
+//
+//		// FIXME check annotations -> code generation -> CustomMessage.registerPacker
 
 		throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")");
 	}
+	
+	static boolean isAnnotated(Class target, Class with) {
+		return target.getAnnotation(with) != null;
+	}
 }
-
diff --git a/java/src/main/java/org/msgpack/annotation/MessagePackDelegate.java b/java/src/main/java/org/msgpack/annotation/MessagePackDelegate.java
new file mode 100644
index 0000000..2a72d73
--- /dev/null
+++ b/java/src/main/java/org/msgpack/annotation/MessagePackDelegate.java
@@ -0,0 +1,12 @@
+package org.msgpack.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface MessagePackDelegate {
+
+}
diff --git a/java/src/main/java/org/msgpack/annotation/MessagePackMessage.java b/java/src/main/java/org/msgpack/annotation/MessagePackMessage.java
new file mode 100644
index 0000000..6efeb9d
--- /dev/null
+++ b/java/src/main/java/org/msgpack/annotation/MessagePackMessage.java
@@ -0,0 +1,12 @@
+package org.msgpack.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface MessagePackMessage {
+
+}
diff --git a/java/src/main/java/org/msgpack/annotation/MessagePackOrdinalEnum.java b/java/src/main/java/org/msgpack/annotation/MessagePackOrdinalEnum.java
new file mode 100644
index 0000000..4b0d9bb
--- /dev/null
+++ b/java/src/main/java/org/msgpack/annotation/MessagePackOrdinalEnum.java
@@ -0,0 +1,12 @@
+package org.msgpack.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface MessagePackOrdinalEnum {
+
+}
diff --git a/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java b/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
index f361eb4..27edde1 100644
--- a/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
+++ b/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
@@ -42,5 +42,33 @@ public class TestReflectionPackerTemplate {
 		assertEquals(src.s1, dst.s1);
 		assertEquals(src.s2, dst.s2);
 	}
+
+	@Test
+	public void testPackConvert02() throws Exception {
+		tString();  // FIXME link StringTemplate
+
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+		CustomPacker.register(StringFieldClass.class, ReflectionPacker.create(StringFieldClass.class));
+				
+
+		StringFieldClass src = new StringFieldClass();
+
+		src.s1 = "kumofs";
+		src.s2 = "frsyuki";
+
+		new Packer(out).pack(src);
+
+		Template tmpl = ReflectionTemplate.create(StringFieldClass.class);
+
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+
+		Object obj = tmpl.unpack(new Unpacker(in));
+		assertEquals(obj.getClass(), StringFieldClass.class);
+
+		StringFieldClass dst = (StringFieldClass)obj;
+		assertEquals(src.s1, dst.s1);
+		assertEquals(src.s2, dst.s2);
+	}
 }
 

From 02342ba540d9b1a30917c08c8892e2347f508798 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Mon, 27 Sep 2010 10:10:10 +0900
Subject: [PATCH 0808/1648] java: refactor a Packer class

---
 java/src/main/java/org/msgpack/Packer.java | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index 2f34bde..00a3aae 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -481,12 +481,12 @@ public class Packer {
 		} 
 
 		Class klass = o.getClass();
-		if (CustomPacker.isRegistered(klass)) {
-			MessagePacker packer = CustomPacker.get(klass);
+		MessagePacker packer = CustomPacker.get(klass);
+		if(packer != null) {
 			packer.pack(this, o);
 			return this;
 		} else if (isAnnotated(klass, MessagePackMessage.class)) {
-			MessagePacker packer = ReflectionPacker.create(klass);
+			ReflectionPacker.create(klass);
 			CustomPacker.register(klass, packer);
 			packer.pack(this, o);
 			return this;
@@ -495,14 +495,8 @@ public class Packer {
 		} else if (isAnnotated(klass, MessagePackOrdinalEnum.class)) {
 			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
 		}
-//      Class klass = o.getClass();
-//		MessagePacker packer = CustomPacker.get(klass);
-//		if(packer != null) {
-//			packer.pack(this, o);
-//			return this;
-//		}
-//
-//		// FIXME check annotations -> code generation -> CustomMessage.registerPacker
+
+		// FIXME check annotations -> code generation -> CustomMessage.registerPacker
 
 		throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")");
 	}

From ee1ba5c0f22968b580200b001fc2b94a705b5521 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Mon, 27 Sep 2010 10:16:32 +0900
Subject: [PATCH 0809/1648] java: fixed a bug within a Packer class

---
 java/src/main/java/org/msgpack/Packer.java | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index 00a3aae..b4526cd 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -486,8 +486,7 @@ public class Packer {
 			packer.pack(this, o);
 			return this;
 		} else if (isAnnotated(klass, MessagePackMessage.class)) {
-			ReflectionPacker.create(klass);
-			CustomPacker.register(klass, packer);
+			packer = ReflectionPacker.create(klass);
 			packer.pack(this, o);
 			return this;
 		} else if (isAnnotated(klass, MessagePackDelegate.class)) {
@@ -495,7 +494,7 @@ public class Packer {
 		} else if (isAnnotated(klass, MessagePackOrdinalEnum.class)) {
 			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
 		}
-
+		CustomPacker.register(klass, packer);
 		// FIXME check annotations -> code generation -> CustomMessage.registerPacker
 
 		throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")");

From 1c0afbc5c5abba1bddff6743581d4e1e48b895f8 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Mon, 27 Sep 2010 17:42:00 +0900
Subject: [PATCH 0810/1648] java: loads template classes when Unpacker,
 MessagePackObject or ClassTemplate is loaded

---
 java/src/main/java/org/msgpack/MessagePackObject.java         | 4 ++++
 java/src/main/java/org/msgpack/Templates.java                 | 2 ++
 java/src/main/java/org/msgpack/Unpacker.java                  | 3 +++
 .../test/java/org/msgpack/TestReflectionPackerTemplate.java   | 4 ----
 4 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/java/src/main/java/org/msgpack/MessagePackObject.java b/java/src/main/java/org/msgpack/MessagePackObject.java
index f7e9e0e..8dd9d8b 100644
--- a/java/src/main/java/org/msgpack/MessagePackObject.java
+++ b/java/src/main/java/org/msgpack/MessagePackObject.java
@@ -23,6 +23,10 @@ import java.util.Map;
 import java.math.BigInteger;
 
 public abstract class MessagePackObject implements Cloneable, MessagePackable {
+	static {
+		Templates.load();
+	}
+
 	public boolean isNil() {
 		return false;
 	}
diff --git a/java/src/main/java/org/msgpack/Templates.java b/java/src/main/java/org/msgpack/Templates.java
index 222f625..b693840 100644
--- a/java/src/main/java/org/msgpack/Templates.java
+++ b/java/src/main/java/org/msgpack/Templates.java
@@ -20,6 +20,8 @@ package org.msgpack;
 import org.msgpack.template.*;
 
 public class Templates {
+	public static void load() { }
+
 	public static Template tList(Template elementTemplate) {
 		return new ListTemplate(elementTemplate);
 	}
diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index 4e39748..9966fca 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -103,6 +103,9 @@ import java.math.BigInteger;
  * 
  */
 public class Unpacker implements Iterable {
+	static {
+		Templates.load();
+	}
 
 	// buffer:
 	// +---------------------------------------------+
diff --git a/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java b/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
index 27edde1..1f0016d 100644
--- a/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
+++ b/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
@@ -18,8 +18,6 @@ public class TestReflectionPackerTemplate {
 
 	@Test
 	public void testPackConvert() throws Exception {
-		tString();  // FIXME link StringTemplate
-
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 
 		MessagePacker packer = ReflectionPacker.create(StringFieldClass.class);
@@ -45,8 +43,6 @@ public class TestReflectionPackerTemplate {
 
 	@Test
 	public void testPackConvert02() throws Exception {
-		tString();  // FIXME link StringTemplate
-
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 
 		CustomPacker.register(StringFieldClass.class, ReflectionPacker.create(StringFieldClass.class));

From e739c60e9f37c9d9903df7f11c866e15f2c12404 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Mon, 27 Sep 2010 17:42:22 +0900
Subject: [PATCH 0811/1648] java: pom.xml: v0.4

---
 java/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/pom.xml b/java/pom.xml
index 70da6a6..eb7930d 100755
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -3,7 +3,7 @@
   4.0.0
   org.msgpack
   msgpack
-  0.3
+  0.4
   MessagePack for Java
 
   MessagePack for Java

From 2736b88dd569cc0e11c9ea18a917ba4dfc1a095e Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Mon, 27 Sep 2010 17:55:48 +0900
Subject: [PATCH 0812/1648] edit Packer and Unpacker classes, and move
 org.msgpack.util.annotation.*.java to org.msgpack.util.codegen.*.java

---
 .../java/org/msgpack/CustomConverter.java     |  15 +-
 .../main/java/org/msgpack/CustomMessage.java  |  11 +-
 .../main/java/org/msgpack/CustomUnpacker.java |  16 +-
 .../java/org/msgpack/MessageConverter.java    |   4 +-
 java/src/main/java/org/msgpack/Packer.java    |  19 +-
 java/src/main/java/org/msgpack/Unpacker.java  |  26 +-
 .../annotation/PackUnpackUtilException.java   |  12 -
 .../msgpack/util/codegen/BasicConstants.java  | 160 ++++++++++++
 .../org/msgpack/util/codegen/Constants.java   |   7 +
 .../msgpack/util/codegen/DynamicCodeGen.java  | 232 ++++++++++++++++++
 .../util/codegen/DynamicCodeGenBase.java      | 189 ++++++++++++++
 .../util/codegen/DynamicCodeGenException.java |  12 +
 .../util/codegen/DynamicCodeGenPacker.java    |  18 ++
 .../util/codegen/DynamicCodeGenTemplate.java  |  17 ++
 .../MessagePackOptional.java                  |   2 +-
 .../MessagePackRequired.java                  |   2 +-
 .../MessagePackUnpackable.java                |   2 +-
 .../PackUnpackUtil.java                       |  18 +-
 .../codegen/TestDynamicCodeGenPacker.java     |  51 ++++
 .../TestMessagePackUnpackable.java            |  27 +-
 20 files changed, 766 insertions(+), 74 deletions(-)
 delete mode 100644 java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/Constants.java
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenPacker.java
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java
 rename java/src/main/java/org/msgpack/util/{annotation => codegen}/MessagePackOptional.java (88%)
 rename java/src/main/java/org/msgpack/util/{annotation => codegen}/MessagePackRequired.java (88%)
 rename java/src/main/java/org/msgpack/util/{annotation => codegen}/MessagePackUnpackable.java (87%)
 rename java/src/main/java/org/msgpack/util/{annotation => codegen}/PackUnpackUtil.java (99%)
 create mode 100644 java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java
 rename java/src/test/java/org/msgpack/util/{annotation => codegen}/TestMessagePackUnpackable.java (96%)

diff --git a/java/src/main/java/org/msgpack/CustomConverter.java b/java/src/main/java/org/msgpack/CustomConverter.java
index 4351464..abbc88a 100644
--- a/java/src/main/java/org/msgpack/CustomConverter.java
+++ b/java/src/main/java/org/msgpack/CustomConverter.java
@@ -17,23 +17,22 @@
 //
 package org.msgpack;
 
-import java.util.Map;
-import java.util.HashMap;
+import java.util.concurrent.ConcurrentHashMap;
 
 // FIXME package private?
 public class CustomConverter {
-	public static void register(Class target, MessageConverter converter) {
-		map.put(target, converter);
+	private static ConcurrentHashMap, MessageConverter> map = new ConcurrentHashMap, MessageConverter>();
+	
+	public static void register(Class target, MessageConverter converter) {
+		map.putIfAbsent(target, converter);
 	}
 
-	public static MessageConverter get(Class target) {
+	public static MessageConverter get(Class target) {
 		return map.get(target);
 	}
 
-	public static boolean isRegistered(Class target) {
+	public static boolean isRegistered(Class target) {
 		return map.containsKey(target);
 	}
-
-	private static Map map = new HashMap();
 }
 
diff --git a/java/src/main/java/org/msgpack/CustomMessage.java b/java/src/main/java/org/msgpack/CustomMessage.java
index f87898c..53f8323 100644
--- a/java/src/main/java/org/msgpack/CustomMessage.java
+++ b/java/src/main/java/org/msgpack/CustomMessage.java
@@ -17,14 +17,19 @@
 //
 package org.msgpack;
 
+import java.lang.annotation.Annotation;
+
 public class CustomMessage {
-	public static void registerPacker(Class target, MessagePacker packer) {
+	public static void registerPacker(Class target, MessagePacker packer) {
 		CustomPacker.register(target, packer);
 	}
 
-	public static void registerTemplate(Class target, Template tmpl) {
+	public static void registerTemplate(Class target, Template tmpl) {
 		CustomUnpacker.register(target, tmpl);
 		CustomConverter.register(target, tmpl);
 	}
-}
 
+	static boolean isAnnotated(Class target, Class with) {
+		return target.getAnnotation(with) != null;
+	}
+}
diff --git a/java/src/main/java/org/msgpack/CustomUnpacker.java b/java/src/main/java/org/msgpack/CustomUnpacker.java
index b6047b8..b45292b 100644
--- a/java/src/main/java/org/msgpack/CustomUnpacker.java
+++ b/java/src/main/java/org/msgpack/CustomUnpacker.java
@@ -17,23 +17,21 @@
 //
 package org.msgpack;
 
-import java.util.Map;
-import java.util.HashMap;
+import java.util.concurrent.ConcurrentHashMap;
 
 // FIXME package private?
 public class CustomUnpacker {
-	public static void register(Class target, MessageUnpacker converter) {
-		map.put(target, converter);
+	private static ConcurrentHashMap, MessageUnpacker> map = new ConcurrentHashMap, MessageUnpacker>();
+
+	public static void register(Class target, MessageUnpacker converter) {
+		map.putIfAbsent(target, converter);
 	}
 
-	public static MessageUnpacker get(Class target) {
+	public static MessageUnpacker get(Class target) {
 		return map.get(target);
 	}
 
-	public static boolean isRegistered(Class target) {
+	public static boolean isRegistered(Class target) {
 		return map.containsKey(target);
 	}
-
-	private static Map map = new HashMap();
 }
-
diff --git a/java/src/main/java/org/msgpack/MessageConverter.java b/java/src/main/java/org/msgpack/MessageConverter.java
index 8ad60ae..5e5f437 100644
--- a/java/src/main/java/org/msgpack/MessageConverter.java
+++ b/java/src/main/java/org/msgpack/MessageConverter.java
@@ -17,9 +17,7 @@
 //
 package org.msgpack;
 
-import java.io.IOException;
-
 public interface MessageConverter {
-	public Object convert(MessagePackObject from) throws MessageTypeException;
+	Object convert(MessagePackObject from) throws MessageTypeException;
 }
 
diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index b4526cd..9ea4ea2 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -485,22 +485,21 @@ public class Packer {
 		if(packer != null) {
 			packer.pack(this, o);
 			return this;
-		} else if (isAnnotated(klass, MessagePackMessage.class)) {
+		} else if (CustomMessage.isAnnotated(klass, MessagePackMessage.class)) {
 			packer = ReflectionPacker.create(klass);
 			packer.pack(this, o);
 			return this;
-		} else if (isAnnotated(klass, MessagePackDelegate.class)) {
+		} else if (CustomMessage.isAnnotated(klass, MessagePackDelegate.class)) {
+			// FIXME DelegatePacker
 			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
-		} else if (isAnnotated(klass, MessagePackOrdinalEnum.class)) {
+		} else if (CustomMessage.isAnnotated(klass, MessagePackOrdinalEnum.class)) {
+			// FIXME OrdinalEnumPacker
 			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
 		}
-		CustomPacker.register(klass, packer);
-		// FIXME check annotations -> code generation -> CustomMessage.registerPacker
-
+		if (packer != null) {
+			CustomMessage.registerPacker(klass, packer);
+		}
 		throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")");
 	}
-	
-	static boolean isAnnotated(Class target, Class with) {
-		return target.getAnnotation(with) != null;
-	}
+
 }
diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index 4e39748..add3312 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -18,12 +18,17 @@
 package org.msgpack;
 
 import java.lang.Iterable;
+import java.lang.annotation.Annotation;
 import java.io.InputStream;
 import java.io.IOException;
 import java.util.Iterator;
 import java.nio.ByteBuffer;
 import java.math.BigInteger;
 
+import org.msgpack.annotation.MessagePackDelegate;
+import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.annotation.MessagePackOrdinalEnum;
+
 /**
  * Unpacker enables you to deserialize objects from stream.
  *
@@ -573,7 +578,7 @@ public class Unpacker implements Iterable {
 		obj.messageUnpack(this);
 	}
 
-	final public Object unpack(Class klass) throws IOException, MessageTypeException, InstantiationException, IllegalAccessException {
+	final public Object unpack(Class klass) throws IOException, MessageTypeException, InstantiationException, IllegalAccessException {
 		if(MessageUnpackable.class.isAssignableFrom(klass)) {
 			Object obj = klass.newInstance();
 			((MessageUnpackable)obj).messageUnpack(this);
@@ -584,14 +589,25 @@ public class Unpacker implements Iterable {
 		if(unpacker != null) {
 			return unpacker.unpack(this);
 		}
-
-		// FIXME check annotations -> code generation -> CustomMessage.registerTemplate
-
+		
+		Template tmpl = null;
+		if (CustomMessage.isAnnotated(klass, MessagePackMessage.class)) {
+			tmpl = ReflectionTemplate.create(klass);
+			return tmpl.unpack(this);
+		} else if (CustomMessage.isAnnotated(klass, MessagePackDelegate.class)) {
+			// FIXME DelegateTemplate
+			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
+		} else if (CustomMessage.isAnnotated(klass, MessagePackOrdinalEnum.class)) {
+			// FIXME OrdinalEnumTemplate
+			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
+		}
+		if (tmpl != null) {
+			CustomMessage.registerTemplate(klass, tmpl);
+		}
 		MessageConverter converter = CustomConverter.get(klass);
 		if(converter != null) {
 			return converter.convert(unpackObject());
 		}
-
 		throw new MessageTypeException();
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java b/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java
deleted file mode 100644
index df3af05..0000000
--- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtilException.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.msgpack.util.annotation;
-
-public class PackUnpackUtilException extends RuntimeException {
-
-	public PackUnpackUtilException(String reason) {
-		super(reason);
-	}
-	
-	public PackUnpackUtilException(String reason, Throwable t) {
-		super(reason, t);
-	}
-}
diff --git a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
new file mode 100644
index 0000000..53a8a4c
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
@@ -0,0 +1,160 @@
+package org.msgpack.util.codegen;
+
+public interface BasicConstants {
+
+    String KEYWORD_MODIFIER_PUBLIC = "public";
+
+    String KEYWORD_CATCH = "catch";
+
+    String KEYWORD_ELSE = "else";
+
+    String KEYWORD_ELSEIF = "else if";
+
+    String KEYWORD_FOR = "for";
+
+    String KEYWORD_IF = "if";
+
+    String KEYWORD_INSTANCEOF = "instanceof";
+
+    String KEYWORD_NEW = "new";
+
+    String KEYWORD_NULL = "null";
+
+    String KEYWORD_THROW = "throw";
+
+    String KEYWORD_THROWS = "throws";
+
+    String KEYWORD_TRY = "try";
+
+	String CHAR_NAME_SPACE = " ";
+
+	String CHAR_NAME_COMMA = ",";
+
+	String CHAR_NAME_EQUAL = "=";
+
+	String CHAR_NAME_PLUS = "+";
+
+	String CHAR_NAME_LESSTHAN = "<";
+
+    String CHAR_NAME_RIGHT_PARENTHESIS = ")";
+
+    String CHAR_NAME_LEFT_PARENTHESIS = "(";
+
+    String CHAR_NAME_RIGHT_CURLY_BRACKET = "}";
+
+    String CHAR_NAME_LEFT_CURLY_BRACKET = "{";
+
+    String CHAR_NAME_RIGHT_SQUARE_BRACKET = "]";
+
+    String CHAR_NAME_LEFT_SQUARE_BRACKET = "[";
+
+	String CHAR_NAME_DOT = ".";
+
+	String CHAR_NAME_SEMICOLON = ";";
+
+	String VARIABLE_NAME_PK = "_$$_pk";
+	
+	String VARIABLE_NAME_OBJ = "_$$_obj";
+	
+	String VARIABLE_NAME_TARGET = "_$$_target";
+
+	String VARIABLE_NAME_SIZE = "_$$_len";
+
+	String VARIABLE_NAME_ARRAY = "_$$_ary";
+
+	String VARIABLE_NAME_LIST = "_$$_list";
+
+	String VARIABLE_NAME_MAP = "_$$_map";
+
+	String VARIABLE_NAME_KEY = "_$$_key";
+
+	String VARIABLE_NAME_VAL = "_$$_val";
+
+	String VARIABLE_NAME_ITER = "_$$_iter";
+
+	String VARIABLE_NAME_MPO = "_$$_mpo";
+
+	String VARIABLE_NAME_I = "i";
+
+	String METHOD_NAME_VALUEOF = "valueOf";
+
+	String METHOD_NAME_ADD = "add";
+
+	String METHOD_NAME_PUT = "put";
+
+	String METHOD_NAME_GET = "get";
+
+	String METHOD_NAME_SIZE = "size";
+
+	String METHOD_NAME_KEYSET = "keySet";
+
+	String METHOD_NAME_ITERATOR = "iterator";
+
+	String METHOD_NAME_HASNEXT = "hasNext";
+
+	String METHOD_NAME_NEXT = "next";
+
+	String METHOD_NAME_MSGPACK = "messagePack";
+
+	String METHOD_NAME_MSGUNPACK = "messageUnpack";
+
+	String METHOD_NAME_MSGCONVERT = "messageConvert";
+
+	String METHOD_NAME_PACK = "pack";
+
+	String METHOD_NAME_PACKARRAY = "packArray";
+
+	String METHOD_NAME_UNPACK = "unpack";
+
+	String METHOD_NAME_UNPACKBOOLEAN = "unpackBoolean";
+
+	String METHOD_NAME_UNPACKBYTE = "unpackByte";
+
+	String METHOD_NAME_UNPACKDOUBLE = "unpackDouble";
+
+	String METHOD_NAME_UNPACKFLOAT = "unpackFloat";
+
+	String METHOD_NAME_UNPACKINT = "unpackInt";
+
+	String METHOD_NAME_UNPACKLONG = "unpackLong";
+
+	String METHOD_NAME_UNPACKSHORT = "unpackShort";
+
+	String METHOD_NAME_UNPACKSTRING = "unpackString";
+
+	String METHOD_NAME_UNPACKBIGINTEGER = "unpackBigInteger";
+
+	String METHOD_NAME_UNPACKOBJECT = "unpackObject";
+
+	String METHOD_NAME_UNPACKBYTEARRAY = "unpackByteArray";
+
+	String METHOD_NAME_UNPACKARRAY = "unpackArray";
+
+	String METHOD_NAME_UNPACKMAP = "unpackMap";
+
+	String METHOD_NAME_ASARRAY = "asArray";
+
+	String METHOD_NAME_ASBOOLEAN = "asBoolean";
+
+	String METHOD_NAME_ASBYTE = "asByte";
+
+	String METHOD_NAME_ASSHORT = "asShort";
+
+	String METHOD_NAME_ASINT = "asInt";
+
+	String METHOD_NAME_ASFLOAT = "asFloat";
+
+	String METHOD_NAME_ASLONG = "asLong";
+
+	String METHOD_NAME_ASDOUBLE = "asDouble";
+
+	String METHOD_NAME_ASSTRING = "asString";
+
+	String METHOD_NAME_ASBYTEARRAY = "asByteArray";
+
+	String METHOD_NAME_ASBIGINTEGER = "asBigInteger";
+	
+	String METHOD_NAME_ASLIST = "asList";
+	
+	String METHOD_NAME_ASMAP = "asMap";
+}
diff --git a/java/src/main/java/org/msgpack/util/codegen/Constants.java b/java/src/main/java/org/msgpack/util/codegen/Constants.java
new file mode 100644
index 0000000..ddeef31
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/Constants.java
@@ -0,0 +1,7 @@
+package org.msgpack.util.codegen;
+
+public interface Constants extends BasicConstants {
+	String POSTFIX_TYPE_NAME_PACKER = "_$$_Packer";
+	
+	String POSTFIX_TYPE_NAME_ENHANCER = "_$$_Enhanced";
+}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
new file mode 100644
index 0000000..9f63242
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -0,0 +1,232 @@
+package org.msgpack.util.codegen;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javassist.CannotCompileException;
+import javassist.ClassPool;
+import javassist.CtClass;
+import javassist.CtConstructor;
+import javassist.CtMethod;
+import javassist.CtNewConstructor;
+import javassist.CtNewMethod;
+import javassist.NotFoundException;
+
+import org.msgpack.MessagePacker;
+import org.msgpack.Packer;
+import org.msgpack.Template;
+
+public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
+
+	private static DynamicCodeGen INSTANCE;
+
+	private static AtomicInteger COUNTER;
+
+	public static DynamicCodeGen getInstance() {
+		if (INSTANCE == null) {
+			INSTANCE = new DynamicCodeGen();
+		}
+		return INSTANCE;
+	}
+
+	private ClassPool pool;
+
+	private DynamicCodeGen() {
+		this.pool = ClassPool.getDefault();
+	}
+
+	public Class generateMessagePackerClass(Class origClass) {
+		try {
+			String origName = origClass.getName();
+			String packerName = origName + POSTFIX_TYPE_NAME_PACKER;
+			checkClassValidation(origClass);
+			checkDefaultConstructorValidation(origClass);
+			CtClass packerCtClass = pool.makeClass(packerName);
+			setInterface(packerCtClass, MessagePacker.class);
+			addDefaultConstructor(packerCtClass);
+			Field[] fields = getDeclaredFields(origClass);
+			addPackMethod(packerCtClass, origClass, fields);
+			return createClass(packerCtClass);
+		} catch (NotFoundException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (CannotCompileException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+
+	public Class generateTemplateClass(Class origClass) {
+		try {
+			String origName = origClass.getName();
+			String packerName = origName + POSTFIX_TYPE_NAME_PACKER;
+			checkClassValidation(origClass);
+			checkDefaultConstructorValidation(origClass);
+			CtClass packerCtClass = pool.makeClass(packerName);
+			setInterface(packerCtClass, Template.class);
+			addDefaultConstructor(packerCtClass);
+			Field[] fields = getDeclaredFields(origClass);
+			addPackMethod(packerCtClass, origClass, fields);
+			return createClass(packerCtClass);
+		} catch (NotFoundException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (CannotCompileException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+
+	private void checkClassValidation(Class origClass) {
+		// not public, abstract, final
+		int mod = origClass.getModifiers();
+		if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
+				|| Modifier.isAbstract(mod) || Modifier.isFinal(mod)) {
+			throwClassValidationException(origClass,
+					"it must be a public class");
+		}
+		// interface, enum
+		if (origClass.isInterface() || origClass.isEnum()) {
+			throwClassValidationException(origClass,
+					"it must not be an interface or enum");
+		}
+	}
+
+	private static void throwClassValidationException(Class origClass,
+			String msg) {
+		throw new DynamicCodeGenException(msg + ": " + origClass.getName());
+	}
+
+	private void checkDefaultConstructorValidation(Class origClass) {
+		Constructor cons = null;
+		try {
+			cons = origClass.getDeclaredConstructor(new Class[0]);
+		} catch (Exception e1) {
+			throwConstructoValidationException(origClass);
+		}
+
+		int mod = cons.getModifiers();
+		if (!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) {
+			throwConstructoValidationException(origClass);
+		}
+	}
+
+	private static void throwConstructoValidationException(Class origClass) {
+		throw new DynamicCodeGenException(
+				"it must have a public zero-argument constructor: "
+						+ origClass.getName());
+	}
+
+	private void setInterface(CtClass packerCtClass, Class infClass)
+			throws NotFoundException {
+		CtClass infCtClass = pool.get(infClass.getName());
+		packerCtClass.addInterface(infCtClass);
+	}
+
+	private void addDefaultConstructor(CtClass enhCtClass)
+			throws CannotCompileException {
+		CtConstructor newCtCons = CtNewConstructor
+				.defaultConstructor(enhCtClass);
+		enhCtClass.addConstructor(newCtCons);
+	}
+
+	private Field[] getDeclaredFields(Class origClass) {
+		ArrayList allFields = new ArrayList();
+		Class nextClass = origClass;
+		while (nextClass != Object.class) {
+			Field[] fields = nextClass.getDeclaredFields();
+			for (Field field : fields) {
+				try {
+					checkFieldValidation(field, allFields);
+					allFields.add(field);
+				} catch (DynamicCodeGenException e) { // ignore
+				}
+			}
+			nextClass = nextClass.getSuperclass();
+		}
+		return allFields.toArray(new Field[0]);
+	}
+
+	private void checkFieldValidation(Field field, List fields) {
+		// check modifiers (public or protected)
+		int mod = field.getModifiers();
+		if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
+				|| Modifier.isStatic(mod) || Modifier.isFinal(mod)
+				|| Modifier.isTransient(mod) || field.isSynthetic()) {
+			throwFieldValidationException(field);
+		}
+		// check same name
+		for (Field f : fields) {
+			if (f.getName().equals(field.getName())) {
+				throwFieldValidationException(field);
+			}
+		}
+	}
+
+	private static void throwFieldValidationException(Field field) {
+		throw new DynamicCodeGenException("it must be a public field: "
+				+ field.getName());
+	}
+
+	private void addPackMethod(CtClass packerCtClass, Class c, Field[] fs)
+			throws CannotCompileException, NotFoundException {
+		StringBuilder sb = new StringBuilder();
+		sb.append(KEYWORD_MODIFIER_PUBLIC);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(void.class.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(METHOD_NAME_PACK);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Packer.class.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_PK);
+		sb.append(CHAR_NAME_COMMA);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(Object.class.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_OBJ);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(KEYWORD_THROWS);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(IOException.class.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		insertPackMethodBody(sb, c, fs);
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		// System.out.println("pack method: " + sb.toString());
+		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), packerCtClass);
+		packerCtClass.addMethod(newCtMethod);
+	}
+
+	private void insertPackMethodBody(StringBuilder sb, Class c, Field[] fs) {
+		insertLocalVariableDecl(sb, c, VARIABLE_NAME_TARGET);
+		StringBuilder mc = new StringBuilder();
+		insertTypeCast(mc, c, VARIABLE_NAME_OBJ);
+		insertValueInsertion(sb, mc.toString());
+		insertSemicolon(sb);
+		insertMethodCall(sb, VARIABLE_NAME_PK, METHOD_NAME_PACKARRAY,
+				new String[] { new Integer(fs.length).toString() });
+		insertSemicolon(sb);
+		for (Field f : fs) {
+			insertCodeOfPackCall(sb, f);
+		}
+	}
+
+	private void insertCodeOfPackCall(StringBuilder sb, Field field) {
+		StringBuilder aname = new StringBuilder();
+		aname.append(VARIABLE_NAME_TARGET);
+		aname.append(CHAR_NAME_DOT);
+		aname.append(field.getName());
+		insertMethodCall(sb, VARIABLE_NAME_PK, METHOD_NAME_PACK,
+				new String[] { aname.toString() });
+		insertSemicolon(sb);
+	}
+
+	private Class createClass(CtClass packerCtClass)
+			throws CannotCompileException {
+		return packerCtClass.toClass(null, null);
+	}
+}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
new file mode 100644
index 0000000..24ae579
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -0,0 +1,189 @@
+package org.msgpack.util.codegen;
+
+import java.math.BigInteger;
+import java.util.List;
+import java.util.Map;
+
+public class DynamicCodeGenBase implements BasicConstants {
+    public DynamicCodeGenBase() {
+    }
+
+    public void insertSemicolon(StringBuilder sb) {
+        sb.append(CHAR_NAME_SEMICOLON);
+        sb.append(CHAR_NAME_SPACE);
+    }
+
+    public void insertPublicFieldDecl(StringBuilder sb, Class type,
+            String name) {
+        sb.append(KEYWORD_MODIFIER_PUBLIC);
+        sb.append(CHAR_NAME_SPACE);
+        sb.append(type.getName());
+        sb.append(CHAR_NAME_SPACE);
+        sb.append(name);
+    }
+
+    public void insertLocalVariableDecl(StringBuilder sb, Class type,
+            String name) {
+        sb.append(type.getName());
+        sb.append(CHAR_NAME_SPACE);
+        sb.append(name);
+    }
+    
+    public void insertValueInsertion(StringBuilder sb, String expr) {
+        //  = expr
+        sb.append(CHAR_NAME_SPACE);
+        sb.append(CHAR_NAME_EQUAL);
+        sb.append(CHAR_NAME_SPACE);
+        sb.append(expr);
+    }
+
+    public void insertDefaultConsCall(StringBuilder sb, Class type) {
+        // new tname()
+        insertConsCall(sb, type, null);
+    }
+
+    public void insertConsCall(StringBuilder sb, Class type, String expr) {
+        // new tname(expr)
+        sb.append(KEYWORD_NEW);
+        sb.append(CHAR_NAME_SPACE);
+        sb.append(type.getName());
+        sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+        if (expr != null) {
+            sb.append(expr);
+        }
+        sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+    }
+
+    public void insertMethodCall(StringBuilder sb, String tname, String mname,
+            String[] anames) {
+        // tname.mname(anames[0], anames[1], ...)
+        int len = anames.length;
+        sb.append(tname);
+        sb.append(CHAR_NAME_DOT);
+        sb.append(mname);
+        sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+        for (int i = 0; i < len; ++i) {
+            sb.append(anames[i]);
+            if (i + 1 != len) {
+                sb.append(CHAR_NAME_COMMA);
+                sb.append(CHAR_NAME_SPACE);
+            }
+        }
+        sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+    }
+    
+    public void insertTypeCast(StringBuilder sb, Class type) {
+        // (type)
+        sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+        sb.append(type.getName());
+        sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+    }
+    
+    public void insertTypeCast(StringBuilder sb, Class type, String varName) {
+        // ((type)var)
+        sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+        sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+        sb.append(type.getName());
+        sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+        sb.append(varName);
+        sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+    }
+    
+    public void insertTypeConvToObjectType(StringBuilder sb, Class type,
+            String expr) throws DynamicCodeGenException {
+        if (type.isPrimitive()) { // primitive type
+            if (type.equals(boolean.class)) {
+                // new Boolean(expr)
+                insertConsCall(sb, Boolean.class, expr);
+            } else if (type.equals(byte.class)) {
+                insertConsCall(sb, Byte.class, expr);
+            } else if (type.equals(short.class)) {
+                insertConsCall(sb, Short.class, expr);
+            } else if (type.equals(int.class)) {
+                insertConsCall(sb, Integer.class, expr);
+            } else if (type.equals(long.class)) {
+                insertConsCall(sb, Long.class, expr);
+            } else if (type.equals(float.class)) {
+                insertConsCall(sb, Float.class, expr);
+            } else if (type.equals(double.class)) {
+                insertConsCall(sb, Double.class, expr);
+            } else {
+                throw new DynamicCodeGenException("Type error: "
+                        + type.getName());
+            }
+        } else { // reference type
+            sb.append(expr);
+        }
+    }
+
+    public void insertTryCatchBlocks(StringBuilder sb, String tryBody,
+            List> types, List names, List catchBodies) {
+        int len = types.size();
+        sb.append(KEYWORD_TRY);
+        sb.append(CHAR_NAME_SPACE);
+        sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+        sb.append(CHAR_NAME_SPACE);
+        sb.append(tryBody);
+        sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+        sb.append(CHAR_NAME_SPACE);
+        for (int i = 0; i < len; ++i) {
+            sb.append(KEYWORD_CATCH);
+            sb.append(CHAR_NAME_SPACE);
+            sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+            sb.append(types.get(i).getName());
+            sb.append(CHAR_NAME_SPACE);
+            sb.append(names.get(i));
+            sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+            sb.append(CHAR_NAME_SPACE);
+            sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+            sb.append(CHAR_NAME_SPACE);
+            sb.append(catchBodies.get(i));
+            sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+            sb.append(CHAR_NAME_SPACE);
+        }
+    }
+
+    public String getAsMethod(Class c) throws DynamicCodeGenException {
+        if (c.equals(boolean.class)) {
+            return METHOD_NAME_ASBOOLEAN;
+        } else if (c.equals(byte.class)) {
+            return METHOD_NAME_ASBYTE;
+        } else if (c.equals(short.class)) {
+            return METHOD_NAME_ASSHORT;
+        } else if (c.equals(int.class)) {
+            return METHOD_NAME_ASINT;
+        } else if (c.equals(float.class)) {
+            return METHOD_NAME_ASFLOAT;
+        } else if (c.equals(long.class)) {
+            return METHOD_NAME_ASLONG;
+        } else if (c.equals(double.class)) {
+            return METHOD_NAME_ASDOUBLE;
+        } else if (c.equals(Boolean.class)) {
+            return METHOD_NAME_ASBOOLEAN;
+        } else if (c.equals(Byte.class)) {
+            return METHOD_NAME_ASBYTE;
+        } else if (c.equals(Short.class)) {
+            return METHOD_NAME_ASSHORT;
+        } else if (c.equals(Integer.class)) {
+            return METHOD_NAME_ASINT;
+        } else if (c.equals(Float.class)) {
+            return METHOD_NAME_ASFLOAT;
+        } else if (c.equals(Long.class)) {
+            return METHOD_NAME_ASLONG;
+        } else if (c.equals(Double.class)) {
+            return METHOD_NAME_ASDOUBLE;
+        } else if (c.equals(String.class)) {
+            return METHOD_NAME_ASSTRING;
+        } else if (c.equals(byte[].class)) {
+            return METHOD_NAME_ASBYTEARRAY;
+        } else if (c.equals(BigInteger.class)) {
+            return METHOD_NAME_ASBIGINTEGER;
+        } else if (List.class.isAssignableFrom(c)) {
+            return METHOD_NAME_ASLIST;
+        } else if (Map.class.isAssignableFrom(c)) {
+            return METHOD_NAME_ASMAP;
+        } else {
+            throw new DynamicCodeGenException("Type error: " + c.getName());
+        }
+    }
+}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
new file mode 100644
index 0000000..1877ed5
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
@@ -0,0 +1,12 @@
+package org.msgpack.util.codegen;
+
+public class DynamicCodeGenException extends RuntimeException {
+
+	public DynamicCodeGenException(String reason) {
+		super(reason);
+	}
+	
+	public DynamicCodeGenException(String reason, Throwable t) {
+		super(reason, t);
+	}
+}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenPacker.java
new file mode 100644
index 0000000..2a014b8
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenPacker.java
@@ -0,0 +1,18 @@
+package org.msgpack.util.codegen;
+
+import org.msgpack.MessagePacker;
+
+public class DynamicCodeGenPacker {
+
+	public static MessagePacker create(Class c) {
+		try {
+			DynamicCodeGen gen = DynamicCodeGen.getInstance();
+			Class packerClass = gen.generateMessagePackerClass(c);
+			return (MessagePacker)packerClass.newInstance();
+		} catch (InstantiationException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalAccessException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java
new file mode 100644
index 0000000..1e12f2e
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java
@@ -0,0 +1,17 @@
+package org.msgpack.util.codegen;
+
+import org.msgpack.Template;
+
+public class DynamicCodeGenTemplate {
+	public static Template create(Class c) {
+		try {
+			DynamicCodeGen gen = DynamicCodeGen.getInstance();
+			Class tmplClass = gen.generateTemplateClass(c);
+			return (Template) tmplClass.newInstance();
+		} catch (InstantiationException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalAccessException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+}
diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java b/java/src/main/java/org/msgpack/util/codegen/MessagePackOptional.java
similarity index 88%
rename from java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java
rename to java/src/main/java/org/msgpack/util/codegen/MessagePackOptional.java
index a565292..736c2ed 100644
--- a/java/src/main/java/org/msgpack/util/annotation/MessagePackOptional.java
+++ b/java/src/main/java/org/msgpack/util/codegen/MessagePackOptional.java
@@ -1,4 +1,4 @@
-package org.msgpack.util.annotation;
+package org.msgpack.util.codegen;
 
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java b/java/src/main/java/org/msgpack/util/codegen/MessagePackRequired.java
similarity index 88%
rename from java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java
rename to java/src/main/java/org/msgpack/util/codegen/MessagePackRequired.java
index 03e50cf..a0c956f 100644
--- a/java/src/main/java/org/msgpack/util/annotation/MessagePackRequired.java
+++ b/java/src/main/java/org/msgpack/util/codegen/MessagePackRequired.java
@@ -1,4 +1,4 @@
-package org.msgpack.util.annotation;
+package org.msgpack.util.codegen;
 
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
diff --git a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java b/java/src/main/java/org/msgpack/util/codegen/MessagePackUnpackable.java
similarity index 87%
rename from java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java
rename to java/src/main/java/org/msgpack/util/codegen/MessagePackUnpackable.java
index 77f6e6d..e5aff40 100644
--- a/java/src/main/java/org/msgpack/util/annotation/MessagePackUnpackable.java
+++ b/java/src/main/java/org/msgpack/util/codegen/MessagePackUnpackable.java
@@ -1,4 +1,4 @@
-package org.msgpack.util.annotation;
+package org.msgpack.util.codegen;
 
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
diff --git a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/codegen/PackUnpackUtil.java
similarity index 99%
rename from java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
rename to java/src/main/java/org/msgpack/util/codegen/PackUnpackUtil.java
index 0aeb6d8..2062ee4 100644
--- a/java/src/main/java/org/msgpack/util/annotation/PackUnpackUtil.java
+++ b/java/src/main/java/org/msgpack/util/codegen/PackUnpackUtil.java
@@ -1,4 +1,4 @@
-package org.msgpack.util.annotation;
+package org.msgpack.util.codegen;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -230,7 +230,7 @@ public class PackUnpackUtil {
 		}
 
 		private static void throwClassValidationException(Class origClass) {
-			throw new PackUnpackUtilException(
+			throw new DynamicCodeGenException(
 					"it must be a public class and have @"
 							+ MessagePackUnpackable.class.getName() + ": "
 							+ origClass.getName());
@@ -260,7 +260,7 @@ public class PackUnpackUtil {
 
 		private static void throwConstructoValidationException(
 				Class origClass) {
-			throw new PackUnpackUtilException(
+			throw new DynamicCodeGenException(
 					"it must have a public zero-argument constructor: "
 							+ origClass.getName());
 		}
@@ -295,7 +295,7 @@ public class PackUnpackUtil {
 					try {
 						checkFieldValidation(field, allFields);
 						allFields.add(field);
-					} catch (PackUnpackUtilException e) { // ignore
+					} catch (DynamicCodeGenException e) { // ignore
 					}
 				}
 				nextClass = nextClass.getSuperclass();
@@ -320,7 +320,7 @@ public class PackUnpackUtil {
 		}
 
 		private static void throwFieldValidationException(Field field) {
-			throw new PackUnpackUtilException("it must be a public field: "
+			throw new DynamicCodeGenException("it must be a public field: "
 					+ field.getName());
 		}
 
@@ -1267,9 +1267,9 @@ public class PackUnpackUtil {
 			try {
 				enhClass = enhancer.generate(origClass, packUnpackable);
 			} catch (NotFoundException e) {
-				throw new PackUnpackUtilException(e.getMessage(), e);
+				throw new DynamicCodeGenException(e.getMessage(), e);
 			} catch (CannotCompileException e) {
-				throw new PackUnpackUtilException(e.getMessage(), e);
+				throw new DynamicCodeGenException(e.getMessage(), e);
 			}
 			// set the generated class to the cache
 			enhancer.setCache(origName, enhClass);
@@ -1306,9 +1306,9 @@ public class PackUnpackUtil {
 			// create a new object of the generated class
 			return enhClass.newInstance();
 		} catch (InstantiationException e) {
-			throw new PackUnpackUtilException(e.getMessage(), e);
+			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (IllegalAccessException e) {
-			throw new PackUnpackUtilException(e.getMessage(), e);
+			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java
new file mode 100644
index 0000000..d4e9c16
--- /dev/null
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java
@@ -0,0 +1,51 @@
+package org.msgpack.util.codegen;
+
+import static org.msgpack.Templates.tString;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+
+import org.junit.Test;
+import org.msgpack.MessagePacker;
+import org.msgpack.Packer;
+import org.msgpack.ReflectionPacker;
+import org.msgpack.ReflectionTemplate;
+import org.msgpack.Template;
+import org.msgpack.Unpacker;
+
+import junit.framework.TestCase;
+
+
+public class TestDynamicCodeGenPacker extends TestCase {
+	public static class StringFieldClass {
+		public String s1;
+		public String s2;
+		public StringFieldClass() { }
+	}
+
+	@Test
+	public void testPackConvert() throws Exception {
+		tString();  // FIXME link StringTemplate
+
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker.create(StringFieldClass.class);
+
+		StringFieldClass src = new StringFieldClass();
+
+		src.s1 = "kumofs";
+		src.s2 = "frsyuki";
+
+		packer.pack(new Packer(out), src);
+
+		Template tmpl = ReflectionTemplate.create(StringFieldClass.class);
+
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+
+		Object obj = tmpl.unpack(new Unpacker(in));
+		assertEquals(obj.getClass(), StringFieldClass.class);
+
+		StringFieldClass dst = (StringFieldClass)obj;
+		assertEquals(src.s1, dst.s1);
+		assertEquals(src.s2, dst.s2);
+	}
+}
diff --git a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/codegen/TestMessagePackUnpackable.java
similarity index 96%
rename from java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
rename to java/src/test/java/org/msgpack/util/codegen/TestMessagePackUnpackable.java
index 08c4fa8..7e0bacc 100644
--- a/java/src/test/java/org/msgpack/util/annotation/TestMessagePackUnpackable.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestMessagePackUnpackable.java
@@ -1,4 +1,4 @@
-package org.msgpack.util.annotation;
+package org.msgpack.util.codegen;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -16,6 +16,9 @@ import org.msgpack.MessagePackObject;
 import org.msgpack.MessageUnpackable;
 import org.msgpack.Packer;
 import org.msgpack.Unpacker;
+import org.msgpack.util.codegen.MessagePackUnpackable;
+import org.msgpack.util.codegen.PackUnpackUtil;
+import org.msgpack.util.codegen.DynamicCodeGenException;
 
 public class TestMessagePackUnpackable extends TestCase {
 
@@ -350,7 +353,7 @@ public class TestMessagePackUnpackable extends TestCase {
 		try {
 			PackUnpackUtil.newEnhancedInstance(NoDefaultConstructorClass.class);
 			fail();
-		} catch (PackUnpackUtilException e) {
+		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
@@ -358,7 +361,7 @@ public class TestMessagePackUnpackable extends TestCase {
 			PackUnpackUtil
 					.newEnhancedInstance(PrivateDefaultConstructorClass.class);
 			fail();
-		} catch (PackUnpackUtilException e) {
+		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
@@ -366,7 +369,7 @@ public class TestMessagePackUnpackable extends TestCase {
 			PackUnpackUtil
 					.newEnhancedInstance(ProtectedDefaultConstructorClass.class);
 			assertTrue(true);
-		} catch (PackUnpackUtilException e) {
+		} catch (DynamicCodeGenException e) {
 			fail();
 		}
 		assertTrue(true);
@@ -374,7 +377,7 @@ public class TestMessagePackUnpackable extends TestCase {
 			PackUnpackUtil
 					.newEnhancedInstance(PackageDefaultConstructorClass.class);
 			fail();
-		} catch (PackUnpackUtilException e) {
+		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
@@ -409,21 +412,21 @@ public class TestMessagePackUnpackable extends TestCase {
 		try {
 			PackUnpackUtil.newEnhancedInstance(PrivateModifierClass.class);
 			fail();
-		} catch (PackUnpackUtilException e) {
+		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
 			PackUnpackUtil.newEnhancedInstance(ProtectedModifierClass.class);
 			assertTrue(true);
-		} catch (PackUnpackUtilException e) {
+		} catch (DynamicCodeGenException e) {
 			fail();
 		}
 		assertTrue(true);
 		try {
 			PackUnpackUtil.newEnhancedInstance(PackageModifierClass.class);
 			fail();
-		} catch (PackUnpackUtilException e) {
+		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
@@ -448,14 +451,14 @@ public class TestMessagePackUnpackable extends TestCase {
 		try {
 			PackUnpackUtil.newEnhancedInstance(FinalModifierClass.class);
 			fail();
-		} catch (PackUnpackUtilException e) {
+		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
 			PackUnpackUtil.newEnhancedInstance(AbstractModifierClass.class);
 			fail();
-		} catch (PackUnpackUtilException e) {
+		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
@@ -474,14 +477,14 @@ public class TestMessagePackUnpackable extends TestCase {
 		try {
 			PackUnpackUtil.newEnhancedInstance(SampleInterface.class);
 			fail();
-		} catch (PackUnpackUtilException e) {
+		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
 			PackUnpackUtil.newEnhancedInstance(SampleEnum.class);
 			fail();
-		} catch (PackUnpackUtilException e) {
+		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);

From bffe0443f9e20fb9bf55e58ab95510ed19e74ab7 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Tue, 28 Sep 2010 12:17:32 +0900
Subject: [PATCH 0813/1648] edit DynamicCodeGenPacker and
 DynamicCodeGenUnpacker class

---
 .../java/org/msgpack/MessageUnpacker.java     |   2 +-
 java/src/main/java/org/msgpack/Packer.java    |   1 -
 java/src/main/java/org/msgpack/Unpacker.java  |   1 -
 .../msgpack/util/codegen/BasicConstants.java  |  50 ++-
 .../org/msgpack/util/codegen/Constants.java   |   6 +-
 .../msgpack/util/codegen/DynamicCodeGen.java  | 396 +++++++++++++++--
 .../util/codegen/DynamicCodeGenBase.java      | 406 ++++++++++--------
 .../util/codegen/DynamicCodeGenUnpacker.java  |  17 +
 .../codegen/TestDynamicCodeGenPacker.java     |  50 ++-
 9 files changed, 674 insertions(+), 255 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenUnpacker.java

diff --git a/java/src/main/java/org/msgpack/MessageUnpacker.java b/java/src/main/java/org/msgpack/MessageUnpacker.java
index 1817269..2a89e45 100644
--- a/java/src/main/java/org/msgpack/MessageUnpacker.java
+++ b/java/src/main/java/org/msgpack/MessageUnpacker.java
@@ -20,6 +20,6 @@ package org.msgpack;
 import java.io.IOException;
 
 public interface MessageUnpacker {
-	public Object unpack(Unpacker pac) throws IOException, MessageTypeException;
+	Object unpack(Unpacker pac) throws IOException, MessageTypeException;
 }
 
diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index 9ea4ea2..52bd29d 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -22,7 +22,6 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.List;
 import java.util.Map;
-import java.lang.annotation.Annotation;
 import java.math.BigInteger;
 
 import org.msgpack.annotation.MessagePackDelegate;
diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index 33c7d4c..d91fc0e 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -18,7 +18,6 @@
 package org.msgpack;
 
 import java.lang.Iterable;
-import java.lang.annotation.Annotation;
 import java.io.InputStream;
 import java.io.IOException;
 import java.util.Iterator;
diff --git a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
index 53a8a4c..f675f81 100644
--- a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
@@ -2,29 +2,31 @@ package org.msgpack.util.codegen;
 
 public interface BasicConstants {
 
-    String KEYWORD_MODIFIER_PUBLIC = "public";
+	String KEYWORD_MODIFIER_PUBLIC = "public";
 
-    String KEYWORD_CATCH = "catch";
+	String KEYWORD_CATCH = "catch";
 
-    String KEYWORD_ELSE = "else";
+	String KEYWORD_ELSE = "else";
 
-    String KEYWORD_ELSEIF = "else if";
+	String KEYWORD_ELSEIF = "else if";
 
-    String KEYWORD_FOR = "for";
+	String KEYWORD_FOR = "for";
 
-    String KEYWORD_IF = "if";
+	String KEYWORD_IF = "if";
 
-    String KEYWORD_INSTANCEOF = "instanceof";
+	String KEYWORD_INSTANCEOF = "instanceof";
 
-    String KEYWORD_NEW = "new";
+	String KEYWORD_NEW = "new";
 
-    String KEYWORD_NULL = "null";
+	String KEYWORD_NULL = "null";
+	
+	String KEYWORD_RETURN = "return";
 
-    String KEYWORD_THROW = "throw";
+	String KEYWORD_THROW = "throw";
 
-    String KEYWORD_THROWS = "throws";
+	String KEYWORD_THROWS = "throws";
 
-    String KEYWORD_TRY = "try";
+	String KEYWORD_TRY = "try";
 
 	String CHAR_NAME_SPACE = " ";
 
@@ -34,28 +36,30 @@ public interface BasicConstants {
 
 	String CHAR_NAME_PLUS = "+";
 
+	String CHAR_NAME_MINUS = "-";
+
 	String CHAR_NAME_LESSTHAN = "<";
 
-    String CHAR_NAME_RIGHT_PARENTHESIS = ")";
+	String CHAR_NAME_RIGHT_PARENTHESIS = ")";
 
-    String CHAR_NAME_LEFT_PARENTHESIS = "(";
+	String CHAR_NAME_LEFT_PARENTHESIS = "(";
 
-    String CHAR_NAME_RIGHT_CURLY_BRACKET = "}";
+	String CHAR_NAME_RIGHT_CURLY_BRACKET = "}";
 
-    String CHAR_NAME_LEFT_CURLY_BRACKET = "{";
+	String CHAR_NAME_LEFT_CURLY_BRACKET = "{";
 
-    String CHAR_NAME_RIGHT_SQUARE_BRACKET = "]";
+	String CHAR_NAME_RIGHT_SQUARE_BRACKET = "]";
 
-    String CHAR_NAME_LEFT_SQUARE_BRACKET = "[";
+	String CHAR_NAME_LEFT_SQUARE_BRACKET = "[";
 
 	String CHAR_NAME_DOT = ".";
 
 	String CHAR_NAME_SEMICOLON = ";";
 
 	String VARIABLE_NAME_PK = "_$$_pk";
-	
-	String VARIABLE_NAME_OBJ = "_$$_obj";
-	
+
+	String VARIABLE_NAME_OBJECT = "_$$_obj";
+
 	String VARIABLE_NAME_TARGET = "_$$_target";
 
 	String VARIABLE_NAME_SIZE = "_$$_len";
@@ -153,8 +157,8 @@ public interface BasicConstants {
 	String METHOD_NAME_ASBYTEARRAY = "asByteArray";
 
 	String METHOD_NAME_ASBIGINTEGER = "asBigInteger";
-	
+
 	String METHOD_NAME_ASLIST = "asList";
-	
+
 	String METHOD_NAME_ASMAP = "asMap";
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/Constants.java b/java/src/main/java/org/msgpack/util/codegen/Constants.java
index ddeef31..2226594 100644
--- a/java/src/main/java/org/msgpack/util/codegen/Constants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/Constants.java
@@ -2,6 +2,8 @@ package org.msgpack.util.codegen;
 
 public interface Constants extends BasicConstants {
 	String POSTFIX_TYPE_NAME_PACKER = "_$$_Packer";
-	
-	String POSTFIX_TYPE_NAME_ENHANCER = "_$$_Enhanced";
+
+	String POSTFIX_TYPE_NAME_UNPACKER = "_$$_Unpacker";
+
+	String POSTFIX_TYPE_NAME_TEMPLATE = "_$$_Template";
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 9f63242..bd010ea 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -4,8 +4,12 @@ import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
+import java.lang.reflect.ParameterizedType;
+import java.math.BigInteger;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import javassist.CannotCompileException;
@@ -17,15 +21,24 @@ import javassist.CtNewConstructor;
 import javassist.CtNewMethod;
 import javassist.NotFoundException;
 
+import org.msgpack.MessagePackObject;
 import org.msgpack.MessagePacker;
+import org.msgpack.MessageTypeException;
+import org.msgpack.MessageUnpackable;
+import org.msgpack.MessageUnpacker;
 import org.msgpack.Packer;
 import org.msgpack.Template;
+import org.msgpack.Unpacker;
 
 public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private static DynamicCodeGen INSTANCE;
 
-	private static AtomicInteger COUNTER;
+	private static AtomicInteger COUNTER = new AtomicInteger(0);
+
+	private static int inc() {
+		return COUNTER.addAndGet(1);
+	}
 
 	public static DynamicCodeGen getInstance() {
 		if (INSTANCE == null) {
@@ -43,7 +56,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	public Class generateMessagePackerClass(Class origClass) {
 		try {
 			String origName = origClass.getName();
-			String packerName = origName + POSTFIX_TYPE_NAME_PACKER;
+			String packerName = origName + POSTFIX_TYPE_NAME_PACKER + inc();
 			checkClassValidation(origClass);
 			checkDefaultConstructorValidation(origClass);
 			CtClass packerCtClass = pool.makeClass(packerName);
@@ -59,18 +72,38 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
+	public Class generateMessageUnpackerClass(Class origClass) {
+		try {
+			String origName = origClass.getName();
+			String unpackerName = origName + POSTFIX_TYPE_NAME_UNPACKER + inc();
+			checkClassValidation(origClass);
+			checkDefaultConstructorValidation(origClass);
+			CtClass unpackerCtClass = pool.makeClass(unpackerName);
+			setInterface(unpackerCtClass, MessageUnpacker.class);
+			addDefaultConstructor(unpackerCtClass);
+			Field[] fields = getDeclaredFields(origClass);
+			addUnpackMethod(unpackerCtClass, origClass, fields);
+			return createClass(unpackerCtClass);
+		} catch (NotFoundException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (CannotCompileException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+
 	public Class generateTemplateClass(Class origClass) {
 		try {
 			String origName = origClass.getName();
-			String packerName = origName + POSTFIX_TYPE_NAME_PACKER;
+			String tmplName = origName + POSTFIX_TYPE_NAME_TEMPLATE + inc();
 			checkClassValidation(origClass);
 			checkDefaultConstructorValidation(origClass);
-			CtClass packerCtClass = pool.makeClass(packerName);
-			setInterface(packerCtClass, Template.class);
-			addDefaultConstructor(packerCtClass);
+			CtClass tmplCtClass = pool.makeClass(tmplName);
+			setInterface(tmplCtClass, Template.class);
+			addDefaultConstructor(tmplCtClass);
 			Field[] fields = getDeclaredFields(origClass);
-			addPackMethod(packerCtClass, origClass, fields);
-			return createClass(packerCtClass);
+			addUnpackMethod(tmplCtClass, origClass, fields);
+			addConvertMethod(tmplCtClass, origClass, fields);
+			return createClass(tmplCtClass);
 		} catch (NotFoundException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (CannotCompileException e) {
@@ -172,31 +205,13 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	private void addPackMethod(CtClass packerCtClass, Class c, Field[] fs)
 			throws CannotCompileException, NotFoundException {
 		StringBuilder sb = new StringBuilder();
-		sb.append(KEYWORD_MODIFIER_PUBLIC);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(void.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(METHOD_NAME_PACK);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Packer.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_PK);
-		sb.append(CHAR_NAME_COMMA);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(Object.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_OBJ);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(KEYWORD_THROWS);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(IOException.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-		insertPackMethodBody(sb, c, fs);
-		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-		// System.out.println("pack method: " + sb.toString());
+		StringBuilder bsb = new StringBuilder();
+		insertPackMethodBody(bsb, c, fs);
+		addPublicMethodDecl(sb, METHOD_NAME_PACK, void.class, new Class[] {
+				Packer.class, Object.class }, new String[] { VARIABLE_NAME_PK,
+				VARIABLE_NAME_OBJECT }, new Class[] { IOException.class }, bsb
+				.toString());
+		System.out.println("pack method: " + sb.toString());
 		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), packerCtClass);
 		packerCtClass.addMethod(newCtMethod);
 	}
@@ -204,7 +219,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	private void insertPackMethodBody(StringBuilder sb, Class c, Field[] fs) {
 		insertLocalVariableDecl(sb, c, VARIABLE_NAME_TARGET);
 		StringBuilder mc = new StringBuilder();
-		insertTypeCast(mc, c, VARIABLE_NAME_OBJ);
+		insertTypeCast(mc, c, VARIABLE_NAME_OBJECT);
 		insertValueInsertion(sb, mc.toString());
 		insertSemicolon(sb);
 		insertMethodCall(sb, VARIABLE_NAME_PK, METHOD_NAME_PACKARRAY,
@@ -225,6 +240,319 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		insertSemicolon(sb);
 	}
 
+	private void addUnpackMethod(CtClass unpackerCtClass, Class c, Field[] fs)
+			throws CannotCompileException, NotFoundException {
+		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
+		StringBuilder sb = new StringBuilder();
+		StringBuilder bsb = new StringBuilder();
+		insertUnpackMethodBody(bsb, c, fs);
+		addPublicMethodDecl(sb, METHOD_NAME_UNPACK, Object.class,
+				new Class[] { Unpacker.class },
+				new String[] { VARIABLE_NAME_PK }, new Class[] {
+						MessageTypeException.class, IOException.class }, bsb
+						.toString());
+		System.out.println("unpack method: " + sb.toString());
+		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), unpackerCtClass);
+		unpackerCtClass.addMethod(newCtMethod);
+	}
+
+	private void insertUnpackMethodBody(StringBuilder sb, Class c, Field[] fs) {
+		insertLocalVariableDecl(sb, c, VARIABLE_NAME_TARGET);
+		StringBuilder mc = new StringBuilder();
+		insertDefaultConsCall(mc, c);
+		insertValueInsertion(sb, mc.toString());
+		insertSemicolon(sb);
+		insertMethodCall(sb, VARIABLE_NAME_PK, METHOD_NAME_UNPACKARRAY,
+				new String[0]);
+		insertSemicolon(sb);
+		for (Field f : fs) {
+			insertCodeOfUnpackMethodCall(sb, f, f.getType());
+		}
+		insertReturnStat(sb, VARIABLE_NAME_TARGET);
+		insertSemicolon(sb);
+	}
+
+	private void insertCodeOfUnpackMethodCall(StringBuilder sb, Field f,
+			Class c) {
+		if (c.isPrimitive()) {
+			// primitive type
+			insertCodeOfUnpackMethodCallForPrimTypes(sb, f, c);
+		} else if (c.equals(Boolean.class) || // Boolean
+				c.equals(Byte.class) || // Byte
+				c.equals(Double.class) || // Double
+				c.equals(Float.class) || // Float
+				c.equals(Integer.class) || // Integer
+				c.equals(Long.class) || // Long
+				c.equals(Short.class)) { // Short
+			// reference type (wrapper type)
+			insertCodeOfUnpackMethodCallForWrapTypes(sb, f, c);
+		} else if (c.equals(BigInteger.class) || // BigInteger
+				c.equals(String.class) || // String
+				c.equals(byte[].class)) { // byte[]
+			// reference type (other type)
+			insertCodeOfUnpackMethodCallForPrimTypes(sb, f, c);
+		} else if (List.class.isAssignableFrom(c)) {
+			// List
+			insertCodeOfUnpackMethodCallForListType(sb, f, c);
+		} else if (Map.class.isAssignableFrom(c)) {
+			// Map
+			insertCodeOfUnpackMethodCallForMapType(sb, f, c);
+		} else if (MessageUnpackable.class.isAssignableFrom(c)) {
+			// MessageUnpackable
+			insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, f, c);
+		} else {
+			throw new MessageTypeException("unknown type:  " + c.getName());
+		}
+	}
+
+	private void insertCodeOfUnpackMethodCallForPrimTypes(StringBuilder sb,
+			Field f, Class c) {
+		if (f != null) {
+			sb.append(VARIABLE_NAME_TARGET);
+			sb.append(CHAR_NAME_DOT);
+			sb.append(f.getName());
+			sb.append(CHAR_NAME_SPACE);
+			sb.append(CHAR_NAME_EQUAL);
+			sb.append(CHAR_NAME_SPACE);
+		}
+		insertMethodCall(sb, VARIABLE_NAME_PK, getUnpackMethodName(c),
+				new String[0]);
+		if (f != null) {
+			insertSemicolon(sb);
+		}
+	}
+
+	private void insertCodeOfUnpackMethodCallForWrapTypes(StringBuilder sb,
+			Field f, Class c) {
+		if (f != null) {
+			sb.append(VARIABLE_NAME_TARGET);
+			sb.append(f.getName());
+			sb.append(CHAR_NAME_SPACE);
+			sb.append(CHAR_NAME_EQUAL);
+			sb.append(CHAR_NAME_SPACE);
+		}
+		StringBuilder mc = new StringBuilder();
+		insertMethodCall(mc, VARIABLE_NAME_PK, getUnpackMethodName(c),
+				new String[0]);
+		insertMethodCall(sb, c.getName(), METHOD_NAME_VALUEOF,
+				new String[] { mc.toString() });
+		if (f != null) {
+			sb.append(CHAR_NAME_SEMICOLON);
+			sb.append(CHAR_NAME_SPACE);
+		}
+	}
+
+	private void insertCodeOfUnpackMethodCallForListType(StringBuilder sb,
+			Field field, Class type) {
+		ParameterizedType generic = (ParameterizedType) field.getGenericType();
+		Class genericType = (Class) generic.getActualTypeArguments()[0];
+
+		// len
+		sb.append(int.class.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_SIZE);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_PK);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_UNPACKARRAY);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+
+		// field initializer
+		sb.append(field.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(KEYWORD_NEW);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(ArrayList.class.getName());
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+
+		// for loop
+		sb.append(KEYWORD_FOR);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(int.class.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_I);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(0);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_I);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LESSTHAN);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_SIZE);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_PLUS);
+		sb.append(CHAR_NAME_PLUS);
+		sb.append(VARIABLE_NAME_I);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+
+		// block
+		sb.append(field.getName());
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_ADD);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		insertCodeOfUnpackMethodCall(sb, null, genericType);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+	}
+
+	private void insertCodeOfUnpackMethodCallForMapType(StringBuilder sb,
+			Field field, Class type) {
+		ParameterizedType generic = (ParameterizedType) field.getGenericType();
+		Class genericType0 = (Class) generic.getActualTypeArguments()[0];
+		Class genericType1 = (Class) generic.getActualTypeArguments()[1];
+
+		// len
+		sb.append(int.class.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_SIZE);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_PK);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_UNPACKMAP);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+
+		// field initializer
+		sb.append(field.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(KEYWORD_NEW);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(HashMap.class.getName());
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+
+		// for loop
+		sb.append(KEYWORD_FOR);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(int.class.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_I);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(0);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_I);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LESSTHAN);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_SIZE);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_PLUS);
+		sb.append(CHAR_NAME_PLUS);
+		sb.append(VARIABLE_NAME_I);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+
+		// block map.
+		sb.append(field.getName());
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_PUT);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		insertCodeOfUnpackMethodCall(sb, null, genericType0);
+		sb.append(CHAR_NAME_COMMA);
+		sb.append(CHAR_NAME_SPACE);
+		insertCodeOfUnpackMethodCall(sb, null, genericType1);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+	}
+
+	// private void insertCodeOfMessageUnpackCallForEnhancedType(StringBuilder
+	// sb,
+	// Field f, Class c) {
+	// c = this.getCache(c.getName());
+	// insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, f, c);
+	// }
+
+	private void insertCodeOfMessageUnpackCallForMsgUnpackableType(
+			StringBuilder sb, Field f, Class c) {
+		// if (fi == null) { fi = new Foo_$$_Enhanced(); }
+		sb.append(KEYWORD_IF);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(f.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(KEYWORD_NULL);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(f.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(KEYWORD_NEW);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(c.getName());
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+
+		// insert a right variable // ignore
+		sb.append(VARIABLE_NAME_PK);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_UNPACK);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(MessageUnpackable.class.getName());
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(f.getName());
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+	}
+
+	private void addConvertMethod(CtClass tmplCtClass, Class c, Field[] fs)
+			throws CannotCompileException, NotFoundException {
+		// Object convert(MessagePackObject from) throws MessageTypeException;
+		// FIXME
+	}
+
 	private Class createClass(CtClass packerCtClass)
 			throws CannotCompileException {
 		return packerCtClass.toClass(null, null);
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 24ae579..1c907a2 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -5,185 +5,245 @@ import java.util.List;
 import java.util.Map;
 
 public class DynamicCodeGenBase implements BasicConstants {
-    public DynamicCodeGenBase() {
-    }
+	public DynamicCodeGenBase() {
+	}
 
-    public void insertSemicolon(StringBuilder sb) {
-        sb.append(CHAR_NAME_SEMICOLON);
-        sb.append(CHAR_NAME_SPACE);
-    }
+	public void addPublicFieldDecl(StringBuilder sb, Class type, String name) {
+		sb.append(KEYWORD_MODIFIER_PUBLIC);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(type.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(name);
+	}
+	
+	public void addPublicMethodDecl(StringBuilder sb, String mname, 
+			Class returnType, Class[] paramTypes, String[] anames,
+			Class[] exceptTypes, String methodBody) {
+		sb.append(KEYWORD_MODIFIER_PUBLIC);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(returnType.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(mname);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		for (int i = 0; i < paramTypes.length; ++i) {
+			sb.append(paramTypes[i].getName());
+			sb.append(CHAR_NAME_SPACE);
+			sb.append(anames[i]);
+			if (i + 1 != paramTypes.length) {
+				sb.append(CHAR_NAME_COMMA);
+				sb.append(CHAR_NAME_SPACE);
+			}
+		}
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(KEYWORD_THROWS);
+		sb.append(CHAR_NAME_SPACE);
+		for (int i = 0; i < exceptTypes.length; ++i) {
+			sb.append(exceptTypes[i].getName());
+			if (i + 1 != exceptTypes.length) {
+				sb.append(CHAR_NAME_COMMA);
+				sb.append(CHAR_NAME_SPACE);
+			}
+		}
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(methodBody);
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+	}
 
-    public void insertPublicFieldDecl(StringBuilder sb, Class type,
-            String name) {
-        sb.append(KEYWORD_MODIFIER_PUBLIC);
-        sb.append(CHAR_NAME_SPACE);
-        sb.append(type.getName());
-        sb.append(CHAR_NAME_SPACE);
-        sb.append(name);
-    }
+	public void insertSemicolon(StringBuilder sb) {
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+	}
 
-    public void insertLocalVariableDecl(StringBuilder sb, Class type,
-            String name) {
-        sb.append(type.getName());
-        sb.append(CHAR_NAME_SPACE);
-        sb.append(name);
-    }
-    
-    public void insertValueInsertion(StringBuilder sb, String expr) {
-        //  = expr
-        sb.append(CHAR_NAME_SPACE);
-        sb.append(CHAR_NAME_EQUAL);
-        sb.append(CHAR_NAME_SPACE);
-        sb.append(expr);
-    }
+	public void insertLocalVariableDecl(StringBuilder sb, Class type,
+			String name) {
+		sb.append(type.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(name);
+	}
 
-    public void insertDefaultConsCall(StringBuilder sb, Class type) {
-        // new tname()
-        insertConsCall(sb, type, null);
-    }
+	public void insertValueInsertion(StringBuilder sb, String expr) {
+		// = expr
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(expr);
+	}
 
-    public void insertConsCall(StringBuilder sb, Class type, String expr) {
-        // new tname(expr)
-        sb.append(KEYWORD_NEW);
-        sb.append(CHAR_NAME_SPACE);
-        sb.append(type.getName());
-        sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-        if (expr != null) {
-            sb.append(expr);
-        }
-        sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-    }
+	public void insertDefaultConsCall(StringBuilder sb, Class type) {
+		// new tname()
+		insertConsCall(sb, type, null);
+	}
 
-    public void insertMethodCall(StringBuilder sb, String tname, String mname,
-            String[] anames) {
-        // tname.mname(anames[0], anames[1], ...)
-        int len = anames.length;
-        sb.append(tname);
-        sb.append(CHAR_NAME_DOT);
-        sb.append(mname);
-        sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-        for (int i = 0; i < len; ++i) {
-            sb.append(anames[i]);
-            if (i + 1 != len) {
-                sb.append(CHAR_NAME_COMMA);
-                sb.append(CHAR_NAME_SPACE);
-            }
-        }
-        sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-    }
-    
-    public void insertTypeCast(StringBuilder sb, Class type) {
-        // (type)
-        sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-        sb.append(type.getName());
-        sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-    }
-    
-    public void insertTypeCast(StringBuilder sb, Class type, String varName) {
-        // ((type)var)
-        sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-        sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-        sb.append(type.getName());
-        sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-        sb.append(varName);
-        sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-    }
-    
-    public void insertTypeConvToObjectType(StringBuilder sb, Class type,
-            String expr) throws DynamicCodeGenException {
-        if (type.isPrimitive()) { // primitive type
-            if (type.equals(boolean.class)) {
-                // new Boolean(expr)
-                insertConsCall(sb, Boolean.class, expr);
-            } else if (type.equals(byte.class)) {
-                insertConsCall(sb, Byte.class, expr);
-            } else if (type.equals(short.class)) {
-                insertConsCall(sb, Short.class, expr);
-            } else if (type.equals(int.class)) {
-                insertConsCall(sb, Integer.class, expr);
-            } else if (type.equals(long.class)) {
-                insertConsCall(sb, Long.class, expr);
-            } else if (type.equals(float.class)) {
-                insertConsCall(sb, Float.class, expr);
-            } else if (type.equals(double.class)) {
-                insertConsCall(sb, Double.class, expr);
-            } else {
-                throw new DynamicCodeGenException("Type error: "
-                        + type.getName());
-            }
-        } else { // reference type
-            sb.append(expr);
-        }
-    }
+	public void insertConsCall(StringBuilder sb, Class type, String expr) {
+		// new tname(expr)
+		sb.append(KEYWORD_NEW);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(type.getName());
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		if (expr != null) {
+			sb.append(expr);
+		}
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+	}
 
-    public void insertTryCatchBlocks(StringBuilder sb, String tryBody,
-            List> types, List names, List catchBodies) {
-        int len = types.size();
-        sb.append(KEYWORD_TRY);
-        sb.append(CHAR_NAME_SPACE);
-        sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-        sb.append(CHAR_NAME_SPACE);
-        sb.append(tryBody);
-        sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-        sb.append(CHAR_NAME_SPACE);
-        for (int i = 0; i < len; ++i) {
-            sb.append(KEYWORD_CATCH);
-            sb.append(CHAR_NAME_SPACE);
-            sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-            sb.append(types.get(i).getName());
-            sb.append(CHAR_NAME_SPACE);
-            sb.append(names.get(i));
-            sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-            sb.append(CHAR_NAME_SPACE);
-            sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-            sb.append(CHAR_NAME_SPACE);
-            sb.append(catchBodies.get(i));
-            sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-            sb.append(CHAR_NAME_SPACE);
-        }
-    }
+	public void insertMethodCall(StringBuilder sb, String tname, String mname,
+			String[] anames) {
+		// tname.mname(anames[0], anames[1], ...)
+		int len = anames.length;
+		sb.append(tname);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(mname);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		for (int i = 0; i < len; ++i) {
+			sb.append(anames[i]);
+			if (i + 1 != len) {
+				sb.append(CHAR_NAME_COMMA);
+				sb.append(CHAR_NAME_SPACE);
+			}
+		}
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+	}
 
-    public String getAsMethod(Class c) throws DynamicCodeGenException {
-        if (c.equals(boolean.class)) {
-            return METHOD_NAME_ASBOOLEAN;
-        } else if (c.equals(byte.class)) {
-            return METHOD_NAME_ASBYTE;
-        } else if (c.equals(short.class)) {
-            return METHOD_NAME_ASSHORT;
-        } else if (c.equals(int.class)) {
-            return METHOD_NAME_ASINT;
-        } else if (c.equals(float.class)) {
-            return METHOD_NAME_ASFLOAT;
-        } else if (c.equals(long.class)) {
-            return METHOD_NAME_ASLONG;
-        } else if (c.equals(double.class)) {
-            return METHOD_NAME_ASDOUBLE;
-        } else if (c.equals(Boolean.class)) {
-            return METHOD_NAME_ASBOOLEAN;
-        } else if (c.equals(Byte.class)) {
-            return METHOD_NAME_ASBYTE;
-        } else if (c.equals(Short.class)) {
-            return METHOD_NAME_ASSHORT;
-        } else if (c.equals(Integer.class)) {
-            return METHOD_NAME_ASINT;
-        } else if (c.equals(Float.class)) {
-            return METHOD_NAME_ASFLOAT;
-        } else if (c.equals(Long.class)) {
-            return METHOD_NAME_ASLONG;
-        } else if (c.equals(Double.class)) {
-            return METHOD_NAME_ASDOUBLE;
-        } else if (c.equals(String.class)) {
-            return METHOD_NAME_ASSTRING;
-        } else if (c.equals(byte[].class)) {
-            return METHOD_NAME_ASBYTEARRAY;
-        } else if (c.equals(BigInteger.class)) {
-            return METHOD_NAME_ASBIGINTEGER;
-        } else if (List.class.isAssignableFrom(c)) {
-            return METHOD_NAME_ASLIST;
-        } else if (Map.class.isAssignableFrom(c)) {
-            return METHOD_NAME_ASMAP;
-        } else {
-            throw new DynamicCodeGenException("Type error: " + c.getName());
-        }
-    }
+	public void insertTypeCast(StringBuilder sb, Class type) {
+		// (type)
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(type.getName());
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+	}
+
+	public void insertTypeCast(StringBuilder sb, Class type, String varName) {
+		// ((type)var)
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(type.getName());
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(varName);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+	}
+
+	public void insertReturnStat(StringBuilder sb, String expr) {
+		// return expr
+		sb.append(KEYWORD_RETURN);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(expr);
+	}
+
+	public void insertTypeConvToObjectType(StringBuilder sb, Class type,
+			String expr) throws DynamicCodeGenException {
+		if (type.isPrimitive()) { // primitive type
+			if (type.equals(boolean.class)) {
+				// new Boolean(expr)
+				insertConsCall(sb, Boolean.class, expr);
+			} else if (type.equals(byte.class)) {
+				insertConsCall(sb, Byte.class, expr);
+			} else if (type.equals(short.class)) {
+				insertConsCall(sb, Short.class, expr);
+			} else if (type.equals(int.class)) {
+				insertConsCall(sb, Integer.class, expr);
+			} else if (type.equals(long.class)) {
+				insertConsCall(sb, Long.class, expr);
+			} else if (type.equals(float.class)) {
+				insertConsCall(sb, Float.class, expr);
+			} else if (type.equals(double.class)) {
+				insertConsCall(sb, Double.class, expr);
+			} else {
+				throw new DynamicCodeGenException("Type error: "
+						+ type.getName());
+			}
+		} else { // reference type
+			sb.append(expr);
+		}
+	}
+
+	public void insertTryCatchBlocks(StringBuilder sb, String tryBody,
+			List> types, List names, List catchBodies) {
+		int len = types.size();
+		sb.append(KEYWORD_TRY);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(tryBody);
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		for (int i = 0; i < len; ++i) {
+			sb.append(KEYWORD_CATCH);
+			sb.append(CHAR_NAME_SPACE);
+			sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+			sb.append(types.get(i).getName());
+			sb.append(CHAR_NAME_SPACE);
+			sb.append(names.get(i));
+			sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+			sb.append(CHAR_NAME_SPACE);
+			sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+			sb.append(CHAR_NAME_SPACE);
+			sb.append(catchBodies.get(i));
+			sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+			sb.append(CHAR_NAME_SPACE);
+		}
+	}
+
+	public String getUnpackMethodName(Class c)
+			throws DynamicCodeGenException {
+		if (c.equals(boolean.class) || c.equals(Boolean.class)) {
+			return METHOD_NAME_UNPACKBOOLEAN;
+		} else if (c.equals(byte.class) || c.equals(Byte.class)) {
+			return METHOD_NAME_UNPACKBYTE;
+		} else if (c.equals(short.class) || c.equals(Short.class)) {
+			return METHOD_NAME_UNPACKSHORT;
+		} else if (c.equals(int.class) || c.equals(Integer.class)) {
+			return METHOD_NAME_UNPACKINT;
+		} else if (c.equals(float.class) || c.equals(Float.class)) {
+			return METHOD_NAME_UNPACKFLOAT;
+		} else if (c.equals(long.class) || c.equals(Long.class)) {
+			return METHOD_NAME_UNPACKLONG;
+		} else if (c.equals(double.class) || c.equals(Double.class)) {
+			return METHOD_NAME_UNPACKDOUBLE;
+		} else if (c.equals(String.class)) {
+			return METHOD_NAME_UNPACKSTRING;
+		} else if (c.equals(byte[].class)) {
+			return METHOD_NAME_UNPACKBYTEARRAY;
+		} else if (c.equals(BigInteger.class)) {
+			return METHOD_NAME_UNPACKBIGINTEGER;
+		} else if (List.class.isAssignableFrom(c)) {
+			return METHOD_NAME_UNPACK;
+		} else if (Map.class.isAssignableFrom(c)) {
+			return METHOD_NAME_UNPACK;
+		} else {
+			throw new DynamicCodeGenException("Type error: " + c.getName());
+		}
+	}
+
+	public String getAsMethodName(Class c) throws DynamicCodeGenException {
+		if (c.equals(boolean.class) || c.equals(Boolean.class)) {
+			return METHOD_NAME_ASBOOLEAN;
+		} else if (c.equals(byte.class) || c.equals(Byte.class)) {
+			return METHOD_NAME_ASBYTE;
+		} else if (c.equals(short.class) || c.equals(Short.class)) {
+			return METHOD_NAME_ASSHORT;
+		} else if (c.equals(int.class) || c.equals(Integer.class)) {
+			return METHOD_NAME_ASINT;
+		} else if (c.equals(float.class) || c.equals(Float.class)) {
+			return METHOD_NAME_ASFLOAT;
+		} else if (c.equals(long.class) || c.equals(Long.class)) {
+			return METHOD_NAME_ASLONG;
+		} else if (c.equals(double.class) || c.equals(Double.class)) {
+			return METHOD_NAME_ASDOUBLE;
+		} else if (c.equals(String.class)) {
+			return METHOD_NAME_ASSTRING;
+		} else if (c.equals(byte[].class)) {
+			return METHOD_NAME_ASBYTEARRAY;
+		} else if (c.equals(BigInteger.class)) {
+			return METHOD_NAME_ASBIGINTEGER;
+		} else if (List.class.isAssignableFrom(c)) {
+			return METHOD_NAME_ASLIST;
+		} else if (Map.class.isAssignableFrom(c)) {
+			return METHOD_NAME_ASMAP;
+		} else {
+			throw new DynamicCodeGenException("Type error: " + c.getName());
+		}
+	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenUnpacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenUnpacker.java
new file mode 100644
index 0000000..e4fe76b
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenUnpacker.java
@@ -0,0 +1,17 @@
+package org.msgpack.util.codegen;
+
+import org.msgpack.MessageUnpacker;
+
+public class DynamicCodeGenUnpacker {
+	public static MessageUnpacker create(Class c) {
+		try {
+			DynamicCodeGen gen = DynamicCodeGen.getInstance();
+			Class unpackerClass = gen.generateMessageUnpackerClass(c);
+			return (MessageUnpacker) unpackerClass.newInstance();
+		} catch (InstantiationException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalAccessException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+}
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java
index d4e9c16..8851537 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java
@@ -1,50 +1,60 @@
 package org.msgpack.util.codegen;
 
-import static org.msgpack.Templates.tString;
-
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 
 import org.junit.Test;
 import org.msgpack.MessagePacker;
+import org.msgpack.MessageUnpacker;
 import org.msgpack.Packer;
-import org.msgpack.ReflectionPacker;
-import org.msgpack.ReflectionTemplate;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
 
 import junit.framework.TestCase;
 
-
 public class TestDynamicCodeGenPacker extends TestCase {
 	public static class StringFieldClass {
 		public String s1;
 		public String s2;
-		public StringFieldClass() { }
+
+		public StringFieldClass() {
+		}
 	}
 
 	@Test
-	public void testPackConvert() throws Exception {
-		tString();  // FIXME link StringTemplate
-
+	public void testPackConvert01() throws Exception {
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker.create(StringFieldClass.class);
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(StringFieldClass.class);
+		MessageUnpacker unpacker = DynamicCodeGenUnpacker
+				.create(StringFieldClass.class);
 
 		StringFieldClass src = new StringFieldClass();
-
-		src.s1 = "kumofs";
-		src.s2 = "frsyuki";
-
+		src.s1 = "muga";
+		src.s2 = "nishizawa";
 		packer.pack(new Packer(out), src);
 
-		Template tmpl = ReflectionTemplate.create(StringFieldClass.class);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		StringFieldClass dst = (StringFieldClass) new Unpacker(in)
+				.unpack(unpacker);
+		assertEquals(src.s1, dst.s1);
+		assertEquals(src.s2, dst.s2);
+	}
+
+	@Test
+	public void testPackConvert02() throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(StringFieldClass.class);
+		Template tmpl = DynamicCodeGenTemplate.create(StringFieldClass.class);
+
+		StringFieldClass src = new StringFieldClass();
+		src.s1 = "muga";
+		src.s2 = "nishizawa";
+		packer.pack(new Packer(out), src);
 
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-
-		Object obj = tmpl.unpack(new Unpacker(in));
-		assertEquals(obj.getClass(), StringFieldClass.class);
-
-		StringFieldClass dst = (StringFieldClass)obj;
+		StringFieldClass dst = (StringFieldClass) tmpl.unpack(new Unpacker(in));
 		assertEquals(src.s1, dst.s1);
 		assertEquals(src.s2, dst.s2);
 	}

From 190af1d32b0ddec84525e9e0a6d90b19c6b72768 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Tue, 28 Sep 2010 15:24:47 +0900
Subject: [PATCH 0814/1648] delete old programs:
 org.msgpack.util.codegen.MessagePackOptional, MessagePackRequired,
 MessagePackUnpackable, and PackUnpackUtil classes

---
 .../util/codegen/MessagePackOptional.java     |   12 -
 .../util/codegen/MessagePackRequired.java     |   12 -
 .../util/codegen/MessagePackUnpackable.java   |   11 -
 .../msgpack/util/codegen/PackUnpackUtil.java  | 1372 -----------------
 4 files changed, 1407 deletions(-)
 delete mode 100644 java/src/main/java/org/msgpack/util/codegen/MessagePackOptional.java
 delete mode 100644 java/src/main/java/org/msgpack/util/codegen/MessagePackRequired.java
 delete mode 100644 java/src/main/java/org/msgpack/util/codegen/MessagePackUnpackable.java
 delete mode 100644 java/src/main/java/org/msgpack/util/codegen/PackUnpackUtil.java

diff --git a/java/src/main/java/org/msgpack/util/codegen/MessagePackOptional.java b/java/src/main/java/org/msgpack/util/codegen/MessagePackOptional.java
deleted file mode 100644
index 736c2ed..0000000
--- a/java/src/main/java/org/msgpack/util/codegen/MessagePackOptional.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.msgpack.util.codegen;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target(ElementType.FIELD)
-@Retention(RetentionPolicy.CLASS)
-public @interface MessagePackOptional {
-	int value() default -1;
-}
\ No newline at end of file
diff --git a/java/src/main/java/org/msgpack/util/codegen/MessagePackRequired.java b/java/src/main/java/org/msgpack/util/codegen/MessagePackRequired.java
deleted file mode 100644
index a0c956f..0000000
--- a/java/src/main/java/org/msgpack/util/codegen/MessagePackRequired.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.msgpack.util.codegen;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target(ElementType.FIELD)
-@Retention(RetentionPolicy.CLASS)
-public @interface MessagePackRequired {
-	int value() default -1;
-}
\ No newline at end of file
diff --git a/java/src/main/java/org/msgpack/util/codegen/MessagePackUnpackable.java b/java/src/main/java/org/msgpack/util/codegen/MessagePackUnpackable.java
deleted file mode 100644
index e5aff40..0000000
--- a/java/src/main/java/org/msgpack/util/codegen/MessagePackUnpackable.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.msgpack.util.codegen;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target(ElementType.TYPE)
-@Retention(RetentionPolicy.RUNTIME)
-public @interface MessagePackUnpackable {
-}
diff --git a/java/src/main/java/org/msgpack/util/codegen/PackUnpackUtil.java b/java/src/main/java/org/msgpack/util/codegen/PackUnpackUtil.java
deleted file mode 100644
index 2062ee4..0000000
--- a/java/src/main/java/org/msgpack/util/codegen/PackUnpackUtil.java
+++ /dev/null
@@ -1,1372 +0,0 @@
-package org.msgpack.util.codegen;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.lang.reflect.ParameterizedType;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import javassist.CannotCompileException;
-import javassist.ClassPool;
-import javassist.CtClass;
-import javassist.CtConstructor;
-import javassist.CtMethod;
-import javassist.CtNewConstructor;
-import javassist.CtNewMethod;
-import javassist.NotFoundException;
-
-import org.msgpack.MessageConvertable;
-import org.msgpack.MessagePackObject;
-import org.msgpack.MessagePackable;
-import org.msgpack.MessageTypeException;
-import org.msgpack.MessageUnpackable;
-import org.msgpack.Packer;
-import org.msgpack.Unpacker;
-
-public class PackUnpackUtil {
-	static class Constants {
-		static final String POSTFIX_TYPE_NAME_ENHANCER = "_$$_Enhanced";
-
-		static final String KEYWORD_MODIFIER_PUBLIC = "public";
-
-		static final String KEYWORD_FOR = "for";
-
-		static final String KEYWORD_IF = "if";
-
-		static final String KEYWORD_THROWS = "throws";
-
-		static final String KEYWORD_NEW = "new";
-
-		static final String KEYWORD_NULL = "null";
-
-		static final String CHAR_NAME_SPACE = " ";
-
-		static final String CHAR_NAME_COMMA = ",";
-
-		static final String CHAR_NAME_EQUAL = "=";
-
-		static final String CHAR_NAME_PLUS = "+";
-
-		static final String CHAR_NAME_LESSTHAN = "<";
-
-		static final String CHAR_NAME_RIGHT_PARENTHESIS = ")";
-
-		static final String CHAR_NAME_LEFT_PARENTHESIS = "(";
-
-		static final String CHAR_NAME_RIGHT_CURLY_BRACHET = "}";
-
-		static final String CHAR_NAME_LEFT_CURLY_BRACHET = "{";
-
-		static final String CHAR_NAME_RIGHT_SQUARE_BRACKET = "]";
-
-		static final String CHAR_NAME_LEFT_SQUARE_BRACKET = "[";
-
-		static final String CHAR_NAME_DOT = ".";
-
-		static final String CHAR_NAME_SEMICOLON = ";";
-
-		static final String VARIABLE_NAME_PK = "_$$_pk";
-
-		static final String VARIABLE_NAME_SIZE = "_$$_len";
-
-		static final String VARIABLE_NAME_ARRAY = "_$$_ary";
-
-		static final String VARIABLE_NAME_LIST = "_$$_list";
-
-		static final String VARIABLE_NAME_MAP = "_$$_map";
-
-		static final String VARIABLE_NAME_KEY = "_$$_key";
-
-		static final String VARIABLE_NAME_VAL = "_$$_val";
-
-		static final String VARIABLE_NAME_ITER = "_$$_iter";
-
-		static final String VARIABLE_NAME_MPO = "_$$_mpo";
-
-		static final String VARIABLE_NAME_I = "i";
-
-		static final String METHOD_NAME_VALUEOF = "valueOf";
-
-		static final String METHOD_NAME_ADD = "add";
-
-		static final String METHOD_NAME_PUT = "put";
-
-		static final String METHOD_NAME_GET = "get";
-
-		static final String METHOD_NAME_SIZE = "size";
-
-		static final String METHOD_NAME_KEYSET = "keySet";
-
-		static final String METHOD_NAME_ITERATOR = "iterator";
-
-		static final String METHOD_NAME_HASNEXT = "hasNext";
-
-		static final String METHOD_NAME_NEXT = "next";
-
-		static final String METHOD_NAME_MSGPACK = "messagePack";
-
-		static final String METHOD_NAME_MSGUNPACK = "messageUnpack";
-
-		static final String METHOD_NAME_MSGCONVERT = "messageConvert";
-
-		static final String METHOD_NAME_PACK = "pack";
-
-		static final String METHOD_NAME_PACKARRAY = "packArray";
-
-		static final String METHOD_NAME_UNPACK = "unpack";
-
-		static final String METHOD_NAME_UNPACKBOOLEAN = "unpackBoolean";
-
-		static final String METHOD_NAME_UNPACKBYTE = "unpackByte";
-
-		static final String METHOD_NAME_UNPACKDOUBLE = "unpackDouble";
-
-		static final String METHOD_NAME_UNPACKFLOAT = "unpackFloat";
-
-		static final String METHOD_NAME_UNPACKINT = "unpackInt";
-
-		static final String METHOD_NAME_UNPACKLONG = "unpackLong";
-
-		static final String METHOD_NAME_UNPACKSHORT = "unpackShort";
-
-		static final String METHOD_NAME_UNPACKSTRING = "unpackString";
-
-		static final String METHOD_NAME_UNPACKBIGINTEGER = "unpackBigInteger";
-
-		static final String METHOD_NAME_UNPACKOBJECT = "unpackObject";
-
-		static final String METHOD_NAME_UNPACKBYTEARRAY = "unpackByteArray";
-
-		static final String METHOD_NAME_UNPACKARRAY = "unpackArray";
-
-		static final String METHOD_NAME_UNPACKMAP = "unpackMap";
-
-		static final String METHOD_NAME_ASARRAY = "asArray";
-
-		static final String METHOD_NAME_ASBOOLEAN = "asBoolean";
-
-		static final String METHOD_NAME_ASBYTE = "asByte";
-
-		static final String METHOD_NAME_ASSHORT = "asShort";
-
-		static final String METHOD_NAME_ASINT = "asInt";
-
-		static final String METHOD_NAME_ASFLOAT = "asFloat";
-
-		static final String METHOD_NAME_ASLONG = "asLong";
-
-		static final String METHOD_NAME_ASDOUBLE = "asDouble";
-
-		static final String METHOD_NAME_ASSTRING = "asString";
-
-		static final String METHOD_NAME_ASBYTEARRAY = "asByteArray";
-
-		static final String METHOD_NAME_ASBIGINTEGER = "asBigInteger";
-	}
-
-	public static class Enhancer {
-
-		private ConcurrentHashMap> classCache;
-
-		private ClassPool pool;
-
-		protected Enhancer() {
-			classCache = new ConcurrentHashMap>();
-			pool = ClassPool.getDefault();
-		}
-
-		protected Class getCache(String origName) {
-			return classCache.get(origName);
-		}
-
-		protected void setCache(String origName, Class enhClass) {
-			classCache.putIfAbsent(origName, enhClass);
-		}
-
-		protected Class generate(Class origClass, boolean packUnpackable)
-				throws NotFoundException, CannotCompileException {
-			String origName = origClass.getName();
-			String enhName = origName + Constants.POSTFIX_TYPE_NAME_ENHANCER;
-			CtClass origCtClass = pool.get(origName);
-			checkClassValidation(origClass, packUnpackable);
-			checkDefaultConstructorValidation(origClass);
-			CtClass enhCtClass = pool.makeClass(enhName);
-			setSuperclass(enhCtClass, origCtClass);
-			setInterfaces(enhCtClass);
-			addConstructor(enhCtClass);
-			Field[] fields = getDeclaredFields(origClass);
-			addMessagePackMethod(enhCtClass, fields);
-			addMessageUnpackMethod(enhCtClass, fields);
-			addMessageConvertMethod(enhCtClass, fields);
-			return createClass(enhCtClass);
-		}
-
-		private void checkClassValidation(Class origClass,
-				boolean packUnpackable) {
-			// not public, abstract, final
-			int mod = origClass.getModifiers();
-			if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
-					|| Modifier.isAbstract(mod) || Modifier.isFinal(mod)) {
-				throwClassValidationException(origClass);
-			}
-			// interface, enum
-			if (origClass.isInterface() || origClass.isEnum()) {
-				throwClassValidationException(origClass);
-			}
-			// annotation
-			if (!packUnpackable) {
-				checkPackUnpackAnnotation(origClass);
-			}
-		}
-
-		private static void throwClassValidationException(Class origClass) {
-			throw new DynamicCodeGenException(
-					"it must be a public class and have @"
-							+ MessagePackUnpackable.class.getName() + ": "
-							+ origClass.getName());
-		}
-
-		private void checkPackUnpackAnnotation(Class origClass) {
-			Annotation anno = origClass
-					.getAnnotation(MessagePackUnpackable.class);
-			if (anno == null) {
-				throwClassValidationException(origClass);
-			}
-		}
-
-		private void checkDefaultConstructorValidation(Class origClass) {
-			Constructor cons = null;
-			try {
-				cons = origClass.getDeclaredConstructor(new Class[0]);
-			} catch (Exception e1) {
-				throwConstructoValidationException(origClass);
-			}
-
-			int mod = cons.getModifiers();
-			if (!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) {
-				throwConstructoValidationException(origClass);
-			}
-		}
-
-		private static void throwConstructoValidationException(
-				Class origClass) {
-			throw new DynamicCodeGenException(
-					"it must have a public zero-argument constructor: "
-							+ origClass.getName());
-		}
-
-		private void setSuperclass(CtClass enhCtClass, CtClass origCtClass)
-				throws CannotCompileException {
-			enhCtClass.setSuperclass(origCtClass);
-		}
-
-		private void setInterfaces(CtClass enhCtClass) throws NotFoundException {
-			CtClass pacCtClass = pool.get(MessagePackable.class.getName());
-			enhCtClass.addInterface(pacCtClass);
-			CtClass unpacCtClass = pool.get(MessageUnpackable.class.getName());
-			enhCtClass.addInterface(unpacCtClass);
-			CtClass convCtClass = pool.get(MessageConvertable.class.getName());
-			enhCtClass.addInterface(convCtClass);
-		}
-
-		private void addConstructor(CtClass enhCtClass)
-				throws CannotCompileException {
-			CtConstructor newCtCons = CtNewConstructor
-					.defaultConstructor(enhCtClass);
-			enhCtClass.addConstructor(newCtCons);
-		}
-
-		private Field[] getDeclaredFields(Class origClass) {
-			ArrayList allFields = new ArrayList();
-			Class nextClass = origClass;
-			while (nextClass != Object.class) {
-				Field[] fields = nextClass.getDeclaredFields();
-				for (Field field : fields) {
-					try {
-						checkFieldValidation(field, allFields);
-						allFields.add(field);
-					} catch (DynamicCodeGenException e) { // ignore
-					}
-				}
-				nextClass = nextClass.getSuperclass();
-			}
-			return allFields.toArray(new Field[0]);
-		}
-
-		private void checkFieldValidation(Field field, List fields) {
-			// check modifiers (public or protected)
-			int mod = field.getModifiers();
-			if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
-					|| Modifier.isStatic(mod) || Modifier.isFinal(mod)
-					|| Modifier.isTransient(mod) || field.isSynthetic()) {
-				throwFieldValidationException(field);
-			}
-			// check same name
-			for (Field f : fields) {
-				if (f.getName().equals(field.getName())) {
-					throwFieldValidationException(field);
-				}
-			}
-		}
-
-		private static void throwFieldValidationException(Field field) {
-			throw new DynamicCodeGenException("it must be a public field: "
-					+ field.getName());
-		}
-
-		private void addMessagePackMethod(CtClass enhCtClass, Field[] fields)
-				throws CannotCompileException, NotFoundException {
-			StringBuilder sb = new StringBuilder();
-			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(void.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.METHOD_NAME_MSGPACK);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Packer.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_PK);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.KEYWORD_THROWS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(IOException.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_PK);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_PACKARRAY);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(fields.length);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			for (Field field : fields) {
-				insertCodeOfMessagePackCall(sb, field);
-			}
-			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			// System.out.println("messagePack method: " + sb.toString());
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
-			enhCtClass.addMethod(newCtMethod);
-		}
-
-		private void insertCodeOfMessagePackCall(StringBuilder sb, Field field) {
-			sb.append(Constants.VARIABLE_NAME_PK);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_PACK);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(field.getName());
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-		}
-
-		private void addMessageUnpackMethod(CtClass enhCtClass, Field[] fields)
-				throws CannotCompileException, NotFoundException {
-			StringBuilder sb = new StringBuilder();
-			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(void.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.METHOD_NAME_MSGUNPACK);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Unpacker.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_PK);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.KEYWORD_THROWS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(MessageTypeException.class.getName());
-			sb.append(Constants.CHAR_NAME_COMMA);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(IOException.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_PK);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_UNPACKARRAY);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			insertCodeOfMessageUnpackCalls(sb, fields);
-			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			// System.out.println("messageUnpack method: " + sb.toString());
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
-			enhCtClass.addMethod(newCtMethod);
-		}
-
-		private void insertCodeOfMessageUnpackCalls(StringBuilder sb,
-				Field[] fields) throws NotFoundException {
-			for (Field field : fields) {
-				insertCodeOfMessageUnpackCall(sb, field, field.getType());
-			}
-		}
-
-		private void insertCodeOfMessageUnpackCall(StringBuilder sb, Field f,
-				Class c) throws NotFoundException {
-			if (c.isPrimitive()) {
-				// primitive type
-				insertCodeOfMessageUnpackCallForPrimTypes(sb, f, c);
-			} else if (c.equals(Boolean.class) || // Boolean
-					c.equals(Byte.class) || // Byte
-					c.equals(Double.class) || // Double
-					c.equals(Float.class) || // Float
-					c.equals(Integer.class) || // Integer
-					c.equals(Long.class) || // Long
-					c.equals(Short.class)) { // Short
-				// reference type (wrapper type)
-				insertCodeOfMessageUnpackCallForWrapTypes(sb, f, c);
-			} else if (c.equals(BigInteger.class) || // BigInteger
-					c.equals(String.class) || // String
-					c.equals(byte[].class)) { // byte[]
-				// reference type (other type)
-				insertCodeOfMessageUnpackCallForPrimTypes(sb, f, c);
-			} else if (List.class.isAssignableFrom(c)) {
-				// List
-				insertCodeOfMessageUnpackCallForListType(sb, f, c);
-			} else if (Map.class.isAssignableFrom(c)) {
-				// Map
-				insertCodeOfMessageUnpackCallForMapType(sb, f, c);
-			} else if (getCache(c.getName()) != null) {
-				// cached
-				insertCodeOfMessageUnpackCallForEnhancedType(sb, f, c);
-			} else if (MessageUnpackable.class.isAssignableFrom(c)) {
-				// MessageUnpackable
-				insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, f, c);
-			} else {
-				throw new NotFoundException("unknown type:  " + c.getName());
-			}
-		}
-
-		private void insertCodeOfMessageUnpackCallForPrimTypes(
-				StringBuilder sb, Field field, Class type)
-				throws NotFoundException {
-			// insert a right variable
-			if (field != null) {
-				sb.append(field.getName());
-				sb.append(Constants.CHAR_NAME_SPACE);
-				sb.append(Constants.CHAR_NAME_EQUAL);
-				sb.append(Constants.CHAR_NAME_SPACE);
-			}
-			sb.append(Constants.VARIABLE_NAME_PK);
-			sb.append(Constants.CHAR_NAME_DOT);
-			// insert unpack method
-			if (type.equals(boolean.class)) { // boolean
-				sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN);
-			} else if (type.equals(byte.class)) { // byte
-				sb.append(Constants.METHOD_NAME_UNPACKBYTE);
-			} else if (type.equals(double.class)) { // double
-				sb.append(Constants.METHOD_NAME_UNPACKDOUBLE);
-			} else if (type.equals(float.class)) { // float
-				sb.append(Constants.METHOD_NAME_UNPACKFLOAT);
-			} else if (type.equals(int.class)) { // int
-				sb.append(Constants.METHOD_NAME_UNPACKINT);
-			} else if (type.equals(long.class)) { // long
-				sb.append(Constants.METHOD_NAME_UNPACKLONG);
-			} else if (type.equals(short.class)) { // short
-				sb.append(Constants.METHOD_NAME_UNPACKSHORT);
-			} else { // reference type
-				if (type.equals(BigInteger.class)) { // BigInteger
-					sb.append(Constants.METHOD_NAME_UNPACKBIGINTEGER);
-				} else if (type.equals(String.class)) { // String
-					sb.append(Constants.METHOD_NAME_UNPACKSTRING);
-				} else if (type.equals(byte[].class)) { // byte[]
-					sb.append(Constants.METHOD_NAME_UNPACKBYTEARRAY);
-				} else {
-					throw new NotFoundException("unknown type:  "
-							+ type.getName());
-				}
-			}
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			if (field != null) {
-				sb.append(Constants.CHAR_NAME_SEMICOLON);
-				sb.append(Constants.CHAR_NAME_SPACE);
-			}
-		}
-
-		private void insertCodeOfMessageUnpackCallForWrapTypes(
-				StringBuilder sb, Field field, Class type)
-				throws NotFoundException {
-			// insert a right variable
-			if (field != null) {
-				sb.append(field.getName());
-				sb.append(Constants.CHAR_NAME_SPACE);
-				sb.append(Constants.CHAR_NAME_EQUAL);
-				sb.append(Constants.CHAR_NAME_SPACE);
-			}
-			sb.append(type.getName());
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_VALUEOF);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.VARIABLE_NAME_PK);
-			sb.append(Constants.CHAR_NAME_DOT);
-			// insert the name of a unpack method
-			if (type.equals(Boolean.class)) { // Boolean
-				sb.append(Constants.METHOD_NAME_UNPACKBOOLEAN);
-			} else if (type.equals(Byte.class)) { // Byte
-				sb.append(Constants.METHOD_NAME_UNPACKBYTE);
-			} else if (type.equals(Double.class)) { // Double
-				sb.append(Constants.METHOD_NAME_UNPACKDOUBLE);
-			} else if (type.equals(Float.class)) { // Float
-				sb.append(Constants.METHOD_NAME_UNPACKFLOAT);
-			} else if (type.equals(Integer.class)) { // Integer
-				sb.append(Constants.METHOD_NAME_UNPACKINT);
-			} else if (type.equals(Long.class)) { // Long
-				sb.append(Constants.METHOD_NAME_UNPACKLONG);
-			} else if (type.equals(Short.class)) { // Short
-				sb.append(Constants.METHOD_NAME_UNPACKSHORT);
-			} else {
-				throw new NotFoundException("unknown type:  " + type.getName());
-			}
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			if (field != null) {
-				sb.append(Constants.CHAR_NAME_SEMICOLON);
-				sb.append(Constants.CHAR_NAME_SPACE);
-			}
-		}
-
-		private void insertCodeOfMessageUnpackCallForListType(StringBuilder sb,
-				Field field, Class type) throws NotFoundException {
-			ParameterizedType generic = (ParameterizedType) field
-					.getGenericType();
-			Class genericType = (Class) generic.getActualTypeArguments()[0];
-
-			// len
-			sb.append(int.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_SIZE);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_PK);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_UNPACKARRAY);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// field initializer
-			sb.append(field.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.KEYWORD_NEW);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(ArrayList.class.getName());
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// for loop
-			sb.append(Constants.KEYWORD_FOR);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(int.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_I);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(0);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_I);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LESSTHAN);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_SIZE);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_PLUS);
-			sb.append(Constants.CHAR_NAME_PLUS);
-			sb.append(Constants.VARIABLE_NAME_I);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// block
-			sb.append(field.getName());
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_ADD);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			insertCodeOfMessageUnpackCall(sb, null, genericType);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-		}
-
-		private void insertCodeOfMessageUnpackCallForMapType(StringBuilder sb,
-				Field field, Class type) throws NotFoundException {
-			ParameterizedType generic = (ParameterizedType) field
-					.getGenericType();
-			Class genericType0 = (Class) generic.getActualTypeArguments()[0];
-			Class genericType1 = (Class) generic.getActualTypeArguments()[1];
-
-			// len
-			sb.append(int.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_SIZE);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_PK);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_UNPACKMAP);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// field initializer
-			sb.append(field.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.KEYWORD_NEW);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(HashMap.class.getName());
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// for loop
-			sb.append(Constants.KEYWORD_FOR);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(int.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_I);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(0);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_I);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LESSTHAN);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_SIZE);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_PLUS);
-			sb.append(Constants.CHAR_NAME_PLUS);
-			sb.append(Constants.VARIABLE_NAME_I);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// block map.
-			sb.append(field.getName());
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_PUT);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			insertCodeOfMessageUnpackCall(sb, null, genericType0);
-			sb.append(Constants.CHAR_NAME_COMMA);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			insertCodeOfMessageUnpackCall(sb, null, genericType1);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-		}
-
-		private void insertCodeOfMessageUnpackCallForEnhancedType(
-				StringBuilder sb, Field f, Class c) {
-			c = this.getCache(c.getName());
-			insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, f, c);
-		}
-
-		private void insertCodeOfMessageUnpackCallForMsgUnpackableType(
-				StringBuilder sb, Field f, Class c) {
-			// if (fi == null) { fi = new Foo_$$_Enhanced(); }
-			sb.append(Constants.KEYWORD_IF);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(f.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.KEYWORD_NULL);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(f.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.KEYWORD_NEW);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(c.getName());
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// insert a right variable // ignore
-			sb.append(Constants.VARIABLE_NAME_PK);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_UNPACK);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(MessageUnpackable.class.getName());
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(f.getName());
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-		}
-
-		private void addMessageConvertMethod(CtClass enhCtClass, Field[] fields)
-				throws CannotCompileException {
-			// messageConvert(MessagePackObject obj) throws MessageTypeException
-			StringBuilder sb = new StringBuilder();
-			sb.append(Constants.KEYWORD_MODIFIER_PUBLIC);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(void.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.METHOD_NAME_MSGCONVERT);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(MessagePackObject.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_MPO);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.KEYWORD_THROWS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(MessageTypeException.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			insertCodeOfMessagePackObjectArrayGet(sb);
-			insertCodeOfMesageConvertCalls(sb, fields);
-			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			// System.out.println("messageConvert method: " + sb.toString());
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), enhCtClass);
-			enhCtClass.addMethod(newCtMethod);
-		}
-
-		private void insertCodeOfMessagePackObjectArrayGet(StringBuilder sb) {
-			// MessagePackObject[] ary = obj.asArray();
-			sb.append(MessagePackObject.class.getName());
-			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_ARRAY);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_MPO);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_ASARRAY);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-		}
-
-		private void insertCodeOfMesageConvertCalls(StringBuilder sb,
-				Field[] fields) {
-			for (int i = 0; i < fields.length; ++i) {
-				insertCodeOfMessageConvertCall(sb, fields[i], fields[i]
-						.getType(), i, null);
-			}
-		}
-
-		private void insertCodeOfMessageConvertCall(StringBuilder sb, Field f,
-				Class c, int i, String v) {
-			if (c.isPrimitive()) { // primitive type
-				insertCodeOfMessageConvertCallForPrimTypes(sb, f, c, i, v);
-			} else { // reference type
-				if (c.equals(Boolean.class) || c.equals(Byte.class)
-						|| c.equals(Short.class) || c.equals(Integer.class)
-						|| c.equals(Float.class) || c.equals(Long.class)
-						|| c.equals(Double.class)) {
-					// wrapper type
-					insertCodeOfMessageConvertCallForWrapTypes(sb, f, c, i, v);
-				} else if (c.equals(String.class) || c.equals(byte[].class)
-						|| c.equals(BigInteger.class)) {
-					insertCodeOfMessageConvertCallForPrimTypes(sb, f, c, i, v);
-				} else if (List.class.isAssignableFrom(c)) {
-					insertCodeOfMessageConvertCallForList(sb, f, c, i);
-				} else if (Map.class.isAssignableFrom(c)) {
-					insertCodeOfMessageConvertCallForMap(sb, f, c, i);
-				} else if ((getCache(c.getName()) != null)) {
-					insertCodeOfMessageConvertCallForEnhancedType(sb, f, c, i);
-				} else if (MessageConvertable.class.isAssignableFrom(c)) {
-					insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
-				} else {
-					throw new MessageTypeException("Type error: " + c.getName());
-				}
-			}
-		}
-
-		private void insertCodeOfMessageConvertCallForEnhancedType(
-				StringBuilder sb, Field f, Class c, int i) {
-			c = getCache(c.getName());
-			insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
-		}
-
-		private void insertCodeOfMessageConvertCallForMsgConvtblType(
-				StringBuilder sb, Field f, Class c, int i) {
-			// if (fi == null) { fi = new Foo_$$_Enhanced(); }
-			sb.append(Constants.KEYWORD_IF);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(f.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.KEYWORD_NULL);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(f.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.KEYWORD_NEW);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(c.getName());
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// ((MessageConvertable)f_i).messageConvert(ary[i]);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(MessageConvertable.class.getName());
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(f.getName());
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_MSGCONVERT);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.VARIABLE_NAME_ARRAY);
-			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-			sb.append(i);
-			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-		}
-
-		private void insertCodeOfMessageConvertCallForPrimTypes(
-				StringBuilder sb, Field f, Class c, int i, String name) {
-			// f0 = objs[0].intValue();
-			if (f != null) {
-				sb.append(f.getName());
-				sb.append(Constants.CHAR_NAME_SPACE);
-				sb.append(Constants.CHAR_NAME_EQUAL);
-				sb.append(Constants.CHAR_NAME_SPACE);
-				sb.append(Constants.VARIABLE_NAME_ARRAY);
-				sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-				sb.append(i);
-				sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-			} else {
-				sb.append(name);
-			}
-			sb.append(Constants.CHAR_NAME_DOT);
-			if (c.equals(boolean.class)) {
-				sb.append(Constants.METHOD_NAME_ASBOOLEAN);
-			} else if (c.equals(byte.class)) {
-				sb.append(Constants.METHOD_NAME_ASBYTE);
-			} else if (c.equals(short.class)) {
-				sb.append(Constants.METHOD_NAME_ASSHORT);
-			} else if (c.equals(int.class)) {
-				sb.append(Constants.METHOD_NAME_ASINT);
-			} else if (c.equals(float.class)) {
-				sb.append(Constants.METHOD_NAME_ASFLOAT);
-			} else if (c.equals(long.class)) {
-				sb.append(Constants.METHOD_NAME_ASLONG);
-			} else if (c.equals(double.class)) {
-				sb.append(Constants.METHOD_NAME_ASDOUBLE);
-			} else if (c.equals(String.class)) {
-				sb.append(Constants.METHOD_NAME_ASSTRING);
-			} else if (c.equals(byte[].class)) {
-				sb.append(Constants.METHOD_NAME_ASBYTEARRAY);
-			} else if (c.equals(BigInteger.class)) {
-				sb.append(Constants.METHOD_NAME_ASBIGINTEGER);
-			} else {
-				throw new MessageTypeException("Type error: " + c.getName());
-			}
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			if (f != null) {
-				sb.append(Constants.CHAR_NAME_SEMICOLON);
-				sb.append(Constants.CHAR_NAME_SPACE);
-			}
-		}
-
-		private void insertCodeOfMessageConvertCallForWrapTypes(
-				StringBuilder sb, Field f, Class c, int i, String v) {
-			if (f != null) {
-				sb.append(f.getName());
-				sb.append(Constants.CHAR_NAME_SPACE);
-				sb.append(Constants.CHAR_NAME_EQUAL);
-				sb.append(Constants.CHAR_NAME_SPACE);
-			}
-			sb.append(Constants.KEYWORD_NEW);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			if (c.equals(Boolean.class)) {
-				sb.append(Boolean.class.getName());
-			} else if (c.equals(Byte.class)) {
-				sb.append(Byte.class.getName());
-			} else if (c.equals(Short.class)) {
-				sb.append(Short.class.getName());
-			} else if (c.equals(Integer.class)) {
-				sb.append(Integer.class.getName());
-			} else if (c.equals(Float.class)) {
-				sb.append(Float.class.getName());
-			} else if (c.equals(Long.class)) {
-				sb.append(Long.class.getName());
-			} else if (c.equals(Double.class)) {
-				sb.append(Double.class.getName());
-			} else {
-				throw new MessageTypeException("Type error: " + c.getName());
-			}
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			if (f != null) {
-				sb.append(Constants.VARIABLE_NAME_ARRAY);
-				sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-				sb.append(i);
-				sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-			} else {
-				sb.append(v);
-			}
-			sb.append(Constants.CHAR_NAME_DOT);
-			if (c.equals(Boolean.class)) {
-				sb.append(Constants.METHOD_NAME_ASBOOLEAN);
-			} else if (c.equals(Byte.class)) {
-				sb.append(Constants.METHOD_NAME_ASBYTE);
-			} else if (c.equals(Short.class)) {
-				sb.append(Constants.METHOD_NAME_ASSHORT);
-			} else if (c.equals(Integer.class)) {
-				sb.append(Constants.METHOD_NAME_ASINT);
-			} else if (c.equals(Float.class)) {
-				sb.append(Constants.METHOD_NAME_ASFLOAT);
-			} else if (c.equals(Long.class)) {
-				sb.append(Constants.METHOD_NAME_ASLONG);
-			} else if (c.equals(Double.class)) {
-				sb.append(Constants.METHOD_NAME_ASDOUBLE);
-			} else {
-				throw new MessageTypeException("Type error: " + c.getName());
-			}
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			if (f != null) {
-				sb.append(Constants.CHAR_NAME_SEMICOLON);
-				sb.append(Constants.CHAR_NAME_SPACE);
-			}
-		}
-
-		private void insertCodeOfMessageConvertCallForList(StringBuilder sb,
-				Field field, Class type, int i) {
-			ParameterizedType generic = (ParameterizedType) field
-					.getGenericType();
-			Class genericType = (Class) generic.getActualTypeArguments()[0];
-
-			// List list = ary[i].asList();
-			sb.append(List.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_LIST);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_ARRAY);
-			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-			sb.append(i);
-			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append("asList");
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// int size = list.size();
-			sb.append(int.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_SIZE);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_LIST);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_SIZE);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// field initializer
-			sb.append(field.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.KEYWORD_NEW);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(ArrayList.class.getName());
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// for loop
-			sb.append(Constants.KEYWORD_FOR);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(int.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_I);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(0);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_I);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LESSTHAN);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_SIZE);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_PLUS);
-			sb.append(Constants.CHAR_NAME_PLUS);
-			sb.append(Constants.VARIABLE_NAME_I);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// block begin
-			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(MessagePackObject.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_VAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(MessagePackObject.class.getName());
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.VARIABLE_NAME_LIST);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_GET);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.VARIABLE_NAME_I);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			sb.append(field.getName());
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_ADD);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			insertCodeOfMessageConvertCall(sb, null, genericType, -1,
-					Constants.VARIABLE_NAME_VAL);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			// block end
-			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-		}
-
-		private void insertCodeOfMessageConvertCallForMap(StringBuilder sb,
-				Field f, Class c, int i) {
-			ParameterizedType generic = (ParameterizedType) f.getGenericType();
-			Class genericType0 = (Class) generic.getActualTypeArguments()[0];
-			Class genericType1 = (Class) generic.getActualTypeArguments()[1];
-
-			// Map map = ary[i].asMap();
-			sb.append(Map.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_MAP);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_ARRAY);
-			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-			sb.append(i);
-			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append("asMap");
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// int size = list.size();
-			sb.append(int.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_SIZE);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_MAP);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_SIZE);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// field initializer
-			sb.append(f.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.KEYWORD_NEW);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(HashMap.class.getName());
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// for loop
-			sb.append(Constants.KEYWORD_FOR);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Iterator.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_ITER);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_MAP);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_KEYSET);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_ITERATOR);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_ITER);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_HASNEXT);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			// block map.
-			sb.append(MessagePackObject.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_KEY);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(MessagePackObject.class.getName());
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.VARIABLE_NAME_ITER);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_NEXT);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(MessagePackObject.class.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_VAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(MessagePackObject.class.getName());
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.VARIABLE_NAME_MAP);
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_GET);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(Constants.VARIABLE_NAME_KEY);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			sb.append(f.getName());
-			sb.append(Constants.CHAR_NAME_DOT);
-			sb.append(Constants.METHOD_NAME_PUT);
-			sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			insertCodeOfMessageConvertCall(sb, null, genericType0, -1,
-					Constants.VARIABLE_NAME_KEY);
-			sb.append(Constants.CHAR_NAME_COMMA);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			insertCodeOfMessageConvertCall(sb, null, genericType1, -1,
-					Constants.VARIABLE_NAME_VAL);
-			sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
-
-			sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACHET);
-			sb.append(Constants.CHAR_NAME_SPACE);
-		}
-
-		private Class createClass(CtClass enhCtClass)
-				throws CannotCompileException {
-			return enhCtClass.toClass(null, null);
-		}
-	}
-
-	private static Enhancer enhancer;
-
-	public static void registerEnhancedClass(Class origClass,
-			boolean packUnpackable) {
-		if (enhancer == null) {
-			enhancer = new Enhancer();
-		}
-		String origName = origClass.getName();
-		Class enhClass = enhancer.getCache(origName);
-		if (enhClass == null) {
-			// generate a class object related to the original class
-			try {
-				enhClass = enhancer.generate(origClass, packUnpackable);
-			} catch (NotFoundException e) {
-				throw new DynamicCodeGenException(e.getMessage(), e);
-			} catch (CannotCompileException e) {
-				throw new DynamicCodeGenException(e.getMessage(), e);
-			}
-			// set the generated class to the cache
-			enhancer.setCache(origName, enhClass);
-		}
-	}
-
-	public static void registerEnhancedClass(Class origClass) {
-		registerEnhancedClass(origClass, false);
-	}
-
-	public static boolean isRegistered(Class origClass) {
-		if (enhancer == null) {
-			enhancer = new Enhancer();
-		}
-		return enhancer.getCache(origClass.getName()) != null;
-	}
-
-	public static Class getEnhancedClass(Class origClass,
-			boolean packUnpackable) {
-		if (!isRegistered(origClass)) {
-			registerEnhancedClass(origClass, packUnpackable);
-		}
-		return enhancer.getCache(origClass.getName());
-	}
-
-	public static Class getEnhancedClass(Class origClass) {
-		return getEnhancedClass(origClass, false);
-	}
-
-	public static Object newEnhancedInstance(Class origClass,
-			boolean packUnpackable) {
-		try {
-			Class enhClass = getEnhancedClass(origClass, packUnpackable);
-			// create a new object of the generated class
-			return enhClass.newInstance();
-		} catch (InstantiationException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (IllegalAccessException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
-	}
-
-	public static Object newEnhancedInstance(Class origClass) {
-		return newEnhancedInstance(origClass, false);
-	}
-
-	public static Object initEnhancedInstance(MessagePackObject obj,
-			Class origClass, boolean packUnpackable) {
-		Object ret = newEnhancedInstance(origClass, packUnpackable);
-		((MessageConvertable) ret).messageConvert(obj);
-		return ret;
-	}
-
-	public static Object initEnhancedInstance(MessagePackObject obj,
-			Class origClass) {
-		return initEnhancedInstance(obj, origClass, false);
-	}
-
-	@MessagePackUnpackable
-	public static class Image {
-		public String uri = "";
-
-		public String title = "";
-
-		public int width = 0;
-
-		public int height = 0;
-
-		public int size = 0;
-
-		public boolean equals(Image obj) {
-			return uri.equals(obj.uri) && title.equals(obj.title)
-					&& width == obj.width && height == obj.height
-					&& size == obj.size;
-		}
-
-		public boolean equals(Object obj) {
-			if (obj.getClass() != Image.class) {
-				return false;
-			}
-			return equals((Image) obj);
-		}
-	}
-
-	public static void main(final String[] args) throws Exception {
-		Image src = (Image) PackUnpackUtil.newEnhancedInstance(Image.class);
-		src.title = "msgpack";
-		src.uri = "http://msgpack.org/";
-		src.width = 2560;
-		src.height = 1600;
-		src.size = 4096000;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		Image dst = (Image) PackUnpackUtil.newEnhancedInstance(Image.class);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		pac.unpack((MessageUnpackable) dst);
-		// System.out.println(src.equals(dst));
-	}
-}

From 923580d2cd2356e40b1d542df3625381b139f8cc Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Tue, 28 Sep 2010 15:31:55 +0900
Subject: [PATCH 0815/1648] write a simple test program for a
 DynamicCodeGenPacker class

---
 .../main/java/org/msgpack/MessagePacker.java  |   2 +-
 .../annotation/MessagePackOptional.java       |  12 +
 .../annotation/MessagePackRequired.java       |  12 +
 .../msgpack/util/codegen/BasicConstants.java  |   6 +-
 .../msgpack/util/codegen/DynamicCodeGen.java  | 515 ++++++++++++-
 .../util/codegen/DynamicCodeGenBase.java      |  13 +-
 .../TestDynamicCodeGenPackerTemplate.java     |  77 ++
 .../codegen/TestDynamicCodeGenPacker.java     |  61 --
 .../codegen/TestMessagePackUnpackable.java    | 703 ------------------
 9 files changed, 619 insertions(+), 782 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/annotation/MessagePackOptional.java
 create mode 100644 java/src/main/java/org/msgpack/annotation/MessagePackRequired.java
 create mode 100644 java/src/test/java/org/msgpack/util/TestDynamicCodeGenPackerTemplate.java
 delete mode 100644 java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java
 delete mode 100644 java/src/test/java/org/msgpack/util/codegen/TestMessagePackUnpackable.java

diff --git a/java/src/main/java/org/msgpack/MessagePacker.java b/java/src/main/java/org/msgpack/MessagePacker.java
index 05d4de5..e5d387e 100644
--- a/java/src/main/java/org/msgpack/MessagePacker.java
+++ b/java/src/main/java/org/msgpack/MessagePacker.java
@@ -20,6 +20,6 @@ package org.msgpack;
 import java.io.IOException;
 
 public interface MessagePacker {
-	public void pack(Packer pk, Object target) throws IOException;
+	void pack(Packer pk, Object target) throws IOException;
 }
 
diff --git a/java/src/main/java/org/msgpack/annotation/MessagePackOptional.java b/java/src/main/java/org/msgpack/annotation/MessagePackOptional.java
new file mode 100644
index 0000000..081b28c
--- /dev/null
+++ b/java/src/main/java/org/msgpack/annotation/MessagePackOptional.java
@@ -0,0 +1,12 @@
+package org.msgpack.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.FIELD)
+@Retention(RetentionPolicy.CLASS)
+public @interface MessagePackOptional {
+	int value() default -1;
+}
\ No newline at end of file
diff --git a/java/src/main/java/org/msgpack/annotation/MessagePackRequired.java b/java/src/main/java/org/msgpack/annotation/MessagePackRequired.java
new file mode 100644
index 0000000..e268c4f
--- /dev/null
+++ b/java/src/main/java/org/msgpack/annotation/MessagePackRequired.java
@@ -0,0 +1,12 @@
+package org.msgpack.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target(ElementType.FIELD)
+@Retention(RetentionPolicy.CLASS)
+public @interface MessagePackRequired {
+	int value() default -1;
+}
\ No newline at end of file
diff --git a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
index f675f81..e568a76 100644
--- a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
@@ -98,10 +98,8 @@ public interface BasicConstants {
 
 	String METHOD_NAME_NEXT = "next";
 
-	String METHOD_NAME_MSGPACK = "messagePack";
-
-	String METHOD_NAME_MSGUNPACK = "messageUnpack";
-
+	String METHOD_NAME_CONVERT = "convert";
+	
 	String METHOD_NAME_MSGCONVERT = "messageConvert";
 
 	String METHOD_NAME_PACK = "pack";
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index bd010ea..9193b98 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -8,6 +8,7 @@ import java.lang.reflect.ParameterizedType;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -21,6 +22,7 @@ import javassist.CtNewConstructor;
 import javassist.CtNewMethod;
 import javassist.NotFoundException;
 
+import org.msgpack.MessageConvertable;
 import org.msgpack.MessagePackObject;
 import org.msgpack.MessagePacker;
 import org.msgpack.MessageTypeException;
@@ -204,6 +206,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private void addPackMethod(CtClass packerCtClass, Class c, Field[] fs)
 			throws CannotCompileException, NotFoundException {
+		// void pack(Packer pk, Object target) throws IOException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
 		insertPackMethodBody(bsb, c, fs);
@@ -211,7 +214,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				Packer.class, Object.class }, new String[] { VARIABLE_NAME_PK,
 				VARIABLE_NAME_OBJECT }, new Class[] { IOException.class }, bsb
 				.toString());
-		System.out.println("pack method: " + sb.toString());
+		// System.out.println("pack method: " + sb.toString());
 		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), packerCtClass);
 		packerCtClass.addMethod(newCtMethod);
 	}
@@ -226,17 +229,15 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				new String[] { new Integer(fs.length).toString() });
 		insertSemicolon(sb);
 		for (Field f : fs) {
-			insertCodeOfPackCall(sb, f);
+			insertCodeOfPackMethodCall(sb, f);
 		}
 	}
 
-	private void insertCodeOfPackCall(StringBuilder sb, Field field) {
-		StringBuilder aname = new StringBuilder();
-		aname.append(VARIABLE_NAME_TARGET);
-		aname.append(CHAR_NAME_DOT);
-		aname.append(field.getName());
+	private void insertCodeOfPackMethodCall(StringBuilder sb, Field field) {
+		StringBuilder fa = new StringBuilder();
+		insertFieldAccess(fa, VARIABLE_NAME_TARGET, field.getName());
 		insertMethodCall(sb, VARIABLE_NAME_PK, METHOD_NAME_PACK,
-				new String[] { aname.toString() });
+				new String[] { fa.toString() });
 		insertSemicolon(sb);
 	}
 
@@ -251,7 +252,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				new String[] { VARIABLE_NAME_PK }, new Class[] {
 						MessageTypeException.class, IOException.class }, bsb
 						.toString());
-		System.out.println("unpack method: " + sb.toString());
+		// System.out.println("unpack method: " + sb.toString());
 		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), unpackerCtClass);
 		unpackerCtClass.addMethod(newCtMethod);
 	}
@@ -326,6 +327,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			Field f, Class c) {
 		if (f != null) {
 			sb.append(VARIABLE_NAME_TARGET);
+			sb.append(CHAR_NAME_DOT);
 			sb.append(f.getName());
 			sb.append(CHAR_NAME_SPACE);
 			sb.append(CHAR_NAME_EQUAL);
@@ -547,10 +549,501 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 	}
 
-	private void addConvertMethod(CtClass tmplCtClass, Class c, Field[] fs)
+	public void addConvertMethod(CtClass tmplCtClass, Class c, Field[] fs)
 			throws CannotCompileException, NotFoundException {
 		// Object convert(MessagePackObject from) throws MessageTypeException;
-		// FIXME
+		StringBuilder sb = new StringBuilder();
+		StringBuilder bsb = new StringBuilder();
+		insertConvertMethodBody(bsb, c, fs);
+		addPublicMethodDecl(sb, METHOD_NAME_CONVERT, Object.class,
+				new Class[] { MessagePackObject.class },
+				new String[] { VARIABLE_NAME_MPO },
+				new Class[] { MessageTypeException.class }, bsb.toString());
+		// System.out.println("convert method: " + sb.toString());
+		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
+		tmplCtClass.addMethod(newCtMethod);
+	}
+
+	private void insertConvertMethodBody(StringBuilder sb, Class c,
+			Field[] fields) throws CannotCompileException {
+		insertLocalVariableDecl(sb, c, VARIABLE_NAME_TARGET);
+		StringBuilder mc = new StringBuilder();
+		insertDefaultConsCall(mc, c);
+		insertValueInsertion(sb, mc.toString());
+		insertSemicolon(sb);
+		insertCodeOfMessagePackObjectArrayGet(sb);
+		insertCodeOfConvertMethodCalls(sb, fields);
+		insertReturnStat(sb, VARIABLE_NAME_TARGET);
+		insertSemicolon(sb);
+	}
+
+	private void insertCodeOfMessagePackObjectArrayGet(StringBuilder sb) {
+		// MessagePackObject[] ary = obj.asArray();
+		sb.append(MessagePackObject.class.getName());
+		sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+		sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_ARRAY);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_MPO);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_ASARRAY);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+	}
+
+	private void insertCodeOfConvertMethodCalls(StringBuilder sb, Field[] fields) {
+		for (int i = 0; i < fields.length; ++i) {
+			insertCodeOfMessageConvertCall(sb, fields[i], fields[i].getType(),
+					i, null);
+		}
+	}
+
+	private void insertCodeOfMessageConvertCall(StringBuilder sb, Field f,
+			Class c, int i, String v) {
+		if (c.isPrimitive()) { // primitive type
+			insertCodeOfConvertMethodCallForPrimTypes(sb, f, c, i, v);
+		} else { // reference type
+			if (c.equals(Boolean.class) || c.equals(Byte.class)
+					|| c.equals(Short.class) || c.equals(Integer.class)
+					|| c.equals(Float.class) || c.equals(Long.class)
+					|| c.equals(Double.class)) {
+				// wrapper type
+				insertCodeOfMessageConvertCallForWrapTypes(sb, f, c, i, v);
+			} else if (c.equals(String.class) || c.equals(byte[].class)
+					|| c.equals(BigInteger.class)) {
+				insertCodeOfConvertMethodCallForPrimTypes(sb, f, c, i, v);
+			} else if (List.class.isAssignableFrom(c)) {
+				insertCodeOfMessageConvertCallForList(sb, f, c, i);
+			} else if (Map.class.isAssignableFrom(c)) {
+				insertCodeOfMessageConvertCallForMap(sb, f, c, i);
+			} else if (MessageConvertable.class.isAssignableFrom(c)) {
+				insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
+			} else {
+				throw new MessageTypeException("Type error: " + c.getName());
+			}
+		}
+	}
+
+	//
+	// private void insertCodeOfMessageConvertCallForEnhancedType(
+	// StringBuilder sb, Field f, Class c, int i) {
+	// c = getCache(c.getName());
+	// insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
+	// }
+
+	private void insertCodeOfMessageConvertCallForMsgConvtblType(
+			StringBuilder sb, Field f, Class c, int i) {
+		// if (fi == null) { fi = new Foo_$$_Enhanced(); }
+		sb.append(Constants.KEYWORD_IF);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(f.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.KEYWORD_NULL);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(f.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.KEYWORD_NEW);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(c.getName());
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACKET);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		// ((MessageConvertable)f_i).messageConvert(ary[i]);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(MessageConvertable.class.getName());
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(f.getName());
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_MSGCONVERT);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.VARIABLE_NAME_ARRAY);
+		sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+		sb.append(i);
+		sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+	}
+
+	private void insertCodeOfConvertMethodCallForPrimTypes(StringBuilder sb,
+			Field f, Class c, int i, String name) {
+		// target.f0 = objs[0].intValue();
+		if (f != null) {
+			sb.append(Constants.VARIABLE_NAME_TARGET);
+			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(f.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.VARIABLE_NAME_ARRAY);
+			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(i);
+			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		} else {
+			sb.append(name);
+		}
+		sb.append(Constants.CHAR_NAME_DOT);
+		if (c.equals(boolean.class)) {
+			sb.append(Constants.METHOD_NAME_ASBOOLEAN);
+		} else if (c.equals(byte.class)) {
+			sb.append(Constants.METHOD_NAME_ASBYTE);
+		} else if (c.equals(short.class)) {
+			sb.append(Constants.METHOD_NAME_ASSHORT);
+		} else if (c.equals(int.class)) {
+			sb.append(Constants.METHOD_NAME_ASINT);
+		} else if (c.equals(float.class)) {
+			sb.append(Constants.METHOD_NAME_ASFLOAT);
+		} else if (c.equals(long.class)) {
+			sb.append(Constants.METHOD_NAME_ASLONG);
+		} else if (c.equals(double.class)) {
+			sb.append(Constants.METHOD_NAME_ASDOUBLE);
+		} else if (c.equals(String.class)) {
+			sb.append(Constants.METHOD_NAME_ASSTRING);
+		} else if (c.equals(byte[].class)) {
+			sb.append(Constants.METHOD_NAME_ASBYTEARRAY);
+		} else if (c.equals(BigInteger.class)) {
+			sb.append(Constants.METHOD_NAME_ASBIGINTEGER);
+		} else {
+			throw new MessageTypeException("Type error: " + c.getName());
+		}
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		if (f != null) {
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+		}
+	}
+
+	private void insertCodeOfMessageConvertCallForWrapTypes(StringBuilder sb,
+			Field f, Class c, int i, String v) {
+		if (f != null) {
+			sb.append(f.getName());
+			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(Constants.CHAR_NAME_EQUAL);
+			sb.append(Constants.CHAR_NAME_SPACE);
+		}
+		sb.append(Constants.KEYWORD_NEW);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		if (c.equals(Boolean.class)) {
+			sb.append(Boolean.class.getName());
+		} else if (c.equals(Byte.class)) {
+			sb.append(Byte.class.getName());
+		} else if (c.equals(Short.class)) {
+			sb.append(Short.class.getName());
+		} else if (c.equals(Integer.class)) {
+			sb.append(Integer.class.getName());
+		} else if (c.equals(Float.class)) {
+			sb.append(Float.class.getName());
+		} else if (c.equals(Long.class)) {
+			sb.append(Long.class.getName());
+		} else if (c.equals(Double.class)) {
+			sb.append(Double.class.getName());
+		} else {
+			throw new MessageTypeException("Type error: " + c.getName());
+		}
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		if (f != null) {
+			sb.append(Constants.VARIABLE_NAME_ARRAY);
+			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(i);
+			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		} else {
+			sb.append(v);
+		}
+		sb.append(Constants.CHAR_NAME_DOT);
+		if (c.equals(Boolean.class)) {
+			sb.append(Constants.METHOD_NAME_ASBOOLEAN);
+		} else if (c.equals(Byte.class)) {
+			sb.append(Constants.METHOD_NAME_ASBYTE);
+		} else if (c.equals(Short.class)) {
+			sb.append(Constants.METHOD_NAME_ASSHORT);
+		} else if (c.equals(Integer.class)) {
+			sb.append(Constants.METHOD_NAME_ASINT);
+		} else if (c.equals(Float.class)) {
+			sb.append(Constants.METHOD_NAME_ASFLOAT);
+		} else if (c.equals(Long.class)) {
+			sb.append(Constants.METHOD_NAME_ASLONG);
+		} else if (c.equals(Double.class)) {
+			sb.append(Constants.METHOD_NAME_ASDOUBLE);
+		} else {
+			throw new MessageTypeException("Type error: " + c.getName());
+		}
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		if (f != null) {
+			sb.append(Constants.CHAR_NAME_SEMICOLON);
+			sb.append(Constants.CHAR_NAME_SPACE);
+		}
+	}
+
+	private void insertCodeOfMessageConvertCallForList(StringBuilder sb,
+			Field field, Class type, int i) {
+		ParameterizedType generic = (ParameterizedType) field.getGenericType();
+		Class genericType = (Class) generic.getActualTypeArguments()[0];
+
+		// List list = ary[i].asList();
+		sb.append(List.class.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_LIST);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_ARRAY);
+		sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+		sb.append(i);
+		sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append("asList");
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		// int size = list.size();
+		sb.append(int.class.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_SIZE);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_LIST);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_SIZE);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		// field initializer
+		sb.append(field.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.KEYWORD_NEW);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(ArrayList.class.getName());
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		// for loop
+		sb.append(Constants.KEYWORD_FOR);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(int.class.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_I);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(0);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_I);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_LESSTHAN);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_SIZE);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_PLUS);
+		sb.append(Constants.CHAR_NAME_PLUS);
+		sb.append(Constants.VARIABLE_NAME_I);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		// block begin
+		sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(MessagePackObject.class.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_VAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(MessagePackObject.class.getName());
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.VARIABLE_NAME_LIST);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_GET);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.VARIABLE_NAME_I);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		sb.append(field.getName());
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_ADD);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		insertCodeOfMessageConvertCall(sb, null, genericType, -1,
+				Constants.VARIABLE_NAME_VAL);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		// block end
+		sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACKET);
+		sb.append(Constants.CHAR_NAME_SPACE);
+	}
+
+	private void insertCodeOfMessageConvertCallForMap(StringBuilder sb,
+			Field f, Class c, int i) {
+		ParameterizedType generic = (ParameterizedType) f.getGenericType();
+		Class genericType0 = (Class) generic.getActualTypeArguments()[0];
+		Class genericType1 = (Class) generic.getActualTypeArguments()[1];
+
+		// Map map = ary[i].asMap();
+		sb.append(Map.class.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_MAP);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_ARRAY);
+		sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+		sb.append(i);
+		sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append("asMap");
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		// int size = list.size();
+		sb.append(int.class.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_SIZE);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_MAP);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_SIZE);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		// field initializer
+		sb.append(f.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.KEYWORD_NEW);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(HashMap.class.getName());
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		// for loop
+		sb.append(Constants.KEYWORD_FOR);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Iterator.class.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_ITER);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_MAP);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_KEYSET);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_ITERATOR);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_ITER);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_HASNEXT);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		// block map.
+		sb.append(MessagePackObject.class.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_KEY);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(MessagePackObject.class.getName());
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.VARIABLE_NAME_ITER);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_NEXT);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(MessagePackObject.class.getName());
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.VARIABLE_NAME_VAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_EQUAL);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(MessagePackObject.class.getName());
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.VARIABLE_NAME_MAP);
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_GET);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(Constants.VARIABLE_NAME_KEY);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		sb.append(f.getName());
+		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(Constants.METHOD_NAME_PUT);
+		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		insertCodeOfMessageConvertCall(sb, null, genericType0, -1,
+				Constants.VARIABLE_NAME_KEY);
+		sb.append(Constants.CHAR_NAME_COMMA);
+		sb.append(Constants.CHAR_NAME_SPACE);
+		insertCodeOfMessageConvertCall(sb, null, genericType1, -1,
+				Constants.VARIABLE_NAME_VAL);
+		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(Constants.CHAR_NAME_SEMICOLON);
+		sb.append(Constants.CHAR_NAME_SPACE);
+
+		sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACKET);
+		sb.append(Constants.CHAR_NAME_SPACE);
 	}
 
 	private Class createClass(CtClass packerCtClass)
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 1c907a2..9e10cf7 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -15,8 +15,8 @@ public class DynamicCodeGenBase implements BasicConstants {
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(name);
 	}
-	
-	public void addPublicMethodDecl(StringBuilder sb, String mname, 
+
+	public void addPublicMethodDecl(StringBuilder sb, String mname,
 			Class returnType, Class[] paramTypes, String[] anames,
 			Class[] exceptTypes, String methodBody) {
 		sb.append(KEYWORD_MODIFIER_PUBLIC);
@@ -54,12 +54,14 @@ public class DynamicCodeGenBase implements BasicConstants {
 	}
 
 	public void insertSemicolon(StringBuilder sb) {
+		// ;
 		sb.append(CHAR_NAME_SEMICOLON);
 		sb.append(CHAR_NAME_SPACE);
 	}
 
 	public void insertLocalVariableDecl(StringBuilder sb, Class type,
 			String name) {
+		// int lv
 		sb.append(type.getName());
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(name);
@@ -73,6 +75,13 @@ public class DynamicCodeGenBase implements BasicConstants {
 		sb.append(expr);
 	}
 
+	public void insertFieldAccess(StringBuilder sb, String target, String field) {
+		// target.field
+		sb.append(target);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(field);
+	}
+
 	public void insertDefaultConsCall(StringBuilder sb, Class type) {
 		// new tname()
 		insertConsCall(sb, type, null);
diff --git a/java/src/test/java/org/msgpack/util/TestDynamicCodeGenPackerTemplate.java b/java/src/test/java/org/msgpack/util/TestDynamicCodeGenPackerTemplate.java
new file mode 100644
index 0000000..5b52c8a
--- /dev/null
+++ b/java/src/test/java/org/msgpack/util/TestDynamicCodeGenPackerTemplate.java
@@ -0,0 +1,77 @@
+package org.msgpack.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+
+import org.junit.Test;
+import org.msgpack.CustomPacker;
+import org.msgpack.MessagePacker;
+import org.msgpack.Packer;
+import org.msgpack.ReflectionTemplate;
+import org.msgpack.Template;
+import org.msgpack.Unpacker;
+import org.msgpack.util.codegen.DynamicCodeGenPacker;
+import org.msgpack.util.codegen.DynamicCodeGenTemplate;
+
+import static org.junit.Assert.*;
+
+public class TestDynamicCodeGenPackerTemplate {
+
+	public static class StringFieldClass {
+		public String s1;
+		public String s2;
+		public StringFieldClass() { }
+	}
+
+	@Test
+	public void testPackConvert() throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+		MessagePacker packer = DynamicCodeGenPacker.create(StringFieldClass.class);
+
+		StringFieldClass src = new StringFieldClass();
+
+		src.s1 = "kumofs";
+		src.s2 = "frsyuki";
+
+		packer.pack(new Packer(out), src);
+
+		Template tmpl = DynamicCodeGenTemplate.create(StringFieldClass.class);
+
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+
+		Object obj = tmpl.unpack(new Unpacker(in));
+		assertEquals(obj.getClass(), StringFieldClass.class);
+
+		StringFieldClass dst = (StringFieldClass)obj;
+		assertEquals(src.s1, dst.s1);
+		assertEquals(src.s2, dst.s2);
+	}
+
+	@Test
+	public void testPackConvert02() throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+		CustomPacker.register(StringFieldClass.class, DynamicCodeGenPacker.create(StringFieldClass.class));
+				
+
+		StringFieldClass src = new StringFieldClass();
+
+		src.s1 = "kumofs";
+		src.s2 = "frsyuki";
+
+		new Packer(out).pack(src);
+
+		Template tmpl = ReflectionTemplate.create(StringFieldClass.class);
+
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+
+		Object obj = tmpl.unpack(new Unpacker(in));
+		assertEquals(obj.getClass(), StringFieldClass.class);
+
+		StringFieldClass dst = (StringFieldClass)obj;
+		assertEquals(src.s1, dst.s1);
+		assertEquals(src.s2, dst.s2);
+	}
+}
+
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java
deleted file mode 100644
index 8851537..0000000
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPacker.java
+++ /dev/null
@@ -1,61 +0,0 @@
-package org.msgpack.util.codegen;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-
-import org.junit.Test;
-import org.msgpack.MessagePacker;
-import org.msgpack.MessageUnpacker;
-import org.msgpack.Packer;
-import org.msgpack.Template;
-import org.msgpack.Unpacker;
-
-import junit.framework.TestCase;
-
-public class TestDynamicCodeGenPacker extends TestCase {
-	public static class StringFieldClass {
-		public String s1;
-		public String s2;
-
-		public StringFieldClass() {
-		}
-	}
-
-	@Test
-	public void testPackConvert01() throws Exception {
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
-				.create(StringFieldClass.class);
-		MessageUnpacker unpacker = DynamicCodeGenUnpacker
-				.create(StringFieldClass.class);
-
-		StringFieldClass src = new StringFieldClass();
-		src.s1 = "muga";
-		src.s2 = "nishizawa";
-		packer.pack(new Packer(out), src);
-
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		StringFieldClass dst = (StringFieldClass) new Unpacker(in)
-				.unpack(unpacker);
-		assertEquals(src.s1, dst.s1);
-		assertEquals(src.s2, dst.s2);
-	}
-
-	@Test
-	public void testPackConvert02() throws Exception {
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
-				.create(StringFieldClass.class);
-		Template tmpl = DynamicCodeGenTemplate.create(StringFieldClass.class);
-
-		StringFieldClass src = new StringFieldClass();
-		src.s1 = "muga";
-		src.s2 = "nishizawa";
-		packer.pack(new Packer(out), src);
-
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		StringFieldClass dst = (StringFieldClass) tmpl.unpack(new Unpacker(in));
-		assertEquals(src.s1, dst.s1);
-		assertEquals(src.s2, dst.s2);
-	}
-}
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestMessagePackUnpackable.java b/java/src/test/java/org/msgpack/util/codegen/TestMessagePackUnpackable.java
deleted file mode 100644
index 7e0bacc..0000000
--- a/java/src/test/java/org/msgpack/util/codegen/TestMessagePackUnpackable.java
+++ /dev/null
@@ -1,703 +0,0 @@
-package org.msgpack.util.codegen;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import junit.framework.TestCase;
-
-import org.junit.Test;
-import org.msgpack.MessagePackObject;
-import org.msgpack.MessageUnpackable;
-import org.msgpack.Packer;
-import org.msgpack.Unpacker;
-import org.msgpack.util.codegen.MessagePackUnpackable;
-import org.msgpack.util.codegen.PackUnpackUtil;
-import org.msgpack.util.codegen.DynamicCodeGenException;
-
-public class TestMessagePackUnpackable extends TestCase {
-
-	@Test
-	public void testPrimitiveTypeFields01() throws Exception {
-		PrimitiveTypeFieldsClass src = (PrimitiveTypeFieldsClass) PackUnpackUtil
-				.newEnhancedInstance(PrimitiveTypeFieldsClass.class);
-		src.f0 = (byte) 0;
-		src.f1 = 1;
-		src.f2 = 2;
-		src.f3 = 3;
-		src.f4 = 4;
-		src.f5 = 5;
-		src.f6 = false;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) PackUnpackUtil
-				.newEnhancedInstance(PrimitiveTypeFieldsClass.class);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		pac.unpack((MessageUnpackable) dst);
-		assertEquals(src.f0, dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2, dst.f2);
-		assertEquals(src.f3, dst.f3);
-		assertEquals(src.f4, dst.f4);
-		assertEquals(src.f5, dst.f5);
-		assertEquals(src.f6, dst.f6);
-	}
-
-	@Test
-	public void testPrimitiveTypeFields02() throws Exception {
-		PrimitiveTypeFieldsClass src = (PrimitiveTypeFieldsClass) PackUnpackUtil
-				.newEnhancedInstance(PrimitiveTypeFieldsClass.class);
-		src.f0 = (byte) 0;
-		src.f1 = 1;
-		src.f2 = 2;
-		src.f3 = 3;
-		src.f4 = 4;
-		src.f5 = 5;
-		src.f6 = false;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) PackUnpackUtil
-				.initEnhancedInstance(mpo, PrimitiveTypeFieldsClass.class);
-		assertEquals(src.f0, dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2, dst.f2);
-		assertEquals(src.f3, dst.f3);
-		assertEquals(src.f4, dst.f4);
-		assertEquals(src.f5, dst.f5);
-		assertEquals(src.f6, dst.f6);
-		assertFalse(it.hasNext());
-	}
-
-	@MessagePackUnpackable
-	public static class PrimitiveTypeFieldsClass {
-		public byte f0;
-		public short f1;
-		public int f2;
-		public long f3;
-		public float f4;
-		public double f5;
-		public boolean f6;
-
-		public PrimitiveTypeFieldsClass() {
-		}
-	}
-
-	@Test
-	public void testGeneralReferenceTypeFieldsClass01() throws Exception {
-		GeneralReferenceTypeFieldsClass src = (GeneralReferenceTypeFieldsClass) PackUnpackUtil
-				.newEnhancedInstance(GeneralReferenceTypeFieldsClass.class);
-		src.f0 = 0;
-		src.f1 = 1;
-		src.f2 = 2;
-		src.f3 = (long) 3;
-		src.f4 = (float) 4;
-		src.f5 = (double) 5;
-		src.f6 = false;
-		src.f7 = new BigInteger("7");
-		src.f8 = "8";
-		src.f9 = new byte[] { 0x01, 0x02 };
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) PackUnpackUtil
-				.newEnhancedInstance(GeneralReferenceTypeFieldsClass.class);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		pac.unpack((MessageUnpackable) dst);
-		assertEquals(src.f0, dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2, dst.f2);
-		assertEquals(src.f3, dst.f3);
-		assertEquals(src.f4, dst.f4);
-		assertEquals(src.f5, dst.f5);
-		assertEquals(src.f6, dst.f6);
-		assertEquals(src.f7, dst.f7);
-		assertEquals(src.f8, dst.f8);
-		assertEquals(src.f9[0], dst.f9[0]);
-		assertEquals(src.f9[1], dst.f9[1]);
-	}
-
-	@Test
-	public void testGeneralReferenceTypeFieldsClass02() throws Exception {
-		GeneralReferenceTypeFieldsClass src = (GeneralReferenceTypeFieldsClass) PackUnpackUtil
-				.newEnhancedInstance(GeneralReferenceTypeFieldsClass.class);
-		src.f0 = 0;
-		src.f1 = 1;
-		src.f2 = 2;
-		src.f3 = (long) 3;
-		src.f4 = (float) 4;
-		src.f5 = (double) 5;
-		src.f6 = false;
-		src.f7 = new BigInteger("7");
-		src.f8 = "8";
-		src.f9 = new byte[] { 0x01, 0x02 };
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) PackUnpackUtil
-				.initEnhancedInstance(mpo,
-						GeneralReferenceTypeFieldsClass.class);
-		assertEquals(src.f0, dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2, dst.f2);
-		assertEquals(src.f3, dst.f3);
-		assertEquals(src.f4, dst.f4);
-		assertEquals(src.f5, dst.f5);
-		assertEquals(src.f6, dst.f6);
-		assertEquals(src.f7, dst.f7);
-		assertEquals(src.f8, dst.f8);
-		assertEquals(src.f9[0], dst.f9[0]);
-		assertEquals(src.f9[1], dst.f9[1]);
-		assertFalse(it.hasNext());
-	}
-
-	@MessagePackUnpackable
-	public static class GeneralReferenceTypeFieldsClass {
-		public Byte f0;
-		public Short f1;
-		public Integer f2;
-		public Long f3;
-		public Float f4;
-		public Double f5;
-		public Boolean f6;
-		public BigInteger f7;
-		public String f8;
-		public byte[] f9;
-
-		public GeneralReferenceTypeFieldsClass() {
-		}
-	}
-
-	public void testListTypes01() throws Exception {
-		SampleListTypes src = (SampleListTypes) PackUnpackUtil
-				.newEnhancedInstance(SampleListTypes.class);
-		src.f0 = new ArrayList();
-		src.f1 = new ArrayList();
-		src.f1.add(1);
-		src.f1.add(2);
-		src.f1.add(3);
-		src.f2 = new ArrayList();
-		src.f2.add("e1");
-		src.f2.add("e2");
-		src.f2.add("e3");
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		SampleListTypes dst = (SampleListTypes) PackUnpackUtil
-				.newEnhancedInstance(SampleListTypes.class);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		pac.unpack((MessageUnpackable) dst);
-		assertEquals(src.f0.size(), dst.f0.size());
-		assertEquals(src.f1.size(), dst.f1.size());
-		for (int i = 0; i < src.f1.size(); ++i) {
-			assertEquals(src.f1.get(i), dst.f1.get(i));
-		}
-		assertEquals(src.f2.size(), dst.f2.size());
-		for (int i = 0; i < src.f2.size(); ++i) {
-			assertEquals(src.f2.get(i), dst.f2.get(i));
-		}
-	}
-
-	public void testListTypes02() throws Exception {
-		SampleListTypes src = (SampleListTypes) PackUnpackUtil
-				.newEnhancedInstance(SampleListTypes.class);
-		src.f0 = new ArrayList();
-		src.f1 = new ArrayList();
-		src.f1.add(1);
-		src.f1.add(2);
-		src.f1.add(3);
-		src.f2 = new ArrayList();
-		src.f2.add("e1");
-		src.f2.add("e2");
-		src.f2.add("e3");
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		SampleListTypes dst = (SampleListTypes) PackUnpackUtil
-				.initEnhancedInstance(mpo, SampleListTypes.class);
-		assertEquals(src.f0.size(), dst.f0.size());
-		assertEquals(src.f1.size(), dst.f1.size());
-		for (int i = 0; i < src.f1.size(); ++i) {
-			assertEquals(src.f1.get(i), dst.f1.get(i));
-		}
-		assertEquals(src.f2.size(), dst.f2.size());
-		for (int i = 0; i < src.f2.size(); ++i) {
-			assertEquals(src.f2.get(i), dst.f2.get(i));
-		}
-		assertFalse(it.hasNext());
-	}
-
-	@MessagePackUnpackable
-	public static class SampleListTypes {
-		public List f0;
-		public List f1;
-		public List f2;
-
-		public SampleListTypes() {
-		}
-	}
-
-	public void testMapTypes01() throws Exception {
-		SampleMapTypes src = (SampleMapTypes) PackUnpackUtil
-				.newEnhancedInstance(SampleMapTypes.class);
-		src.f0 = new HashMap();
-		src.f1 = new HashMap();
-		src.f1.put(1, 1);
-		src.f1.put(2, 2);
-		src.f1.put(3, 3);
-		src.f2 = new HashMap();
-		src.f2.put("k1", 1);
-		src.f2.put("k2", 2);
-		src.f2.put("k3", 3);
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		SampleMapTypes dst = (SampleMapTypes) PackUnpackUtil
-				.newEnhancedInstance(SampleMapTypes.class);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		pac.unpack((MessageUnpackable) dst);
-		assertEquals(src.f0.size(), dst.f0.size());
-		assertEquals(src.f1.size(), dst.f1.size());
-		Iterator srcf1 = src.f1.keySet().iterator();
-		Iterator dstf1 = dst.f1.keySet().iterator();
-		while (srcf1.hasNext()) {
-			Integer s1 = srcf1.next();
-			Integer d1 = dstf1.next();
-			assertEquals(s1, d1);
-			assertEquals(src.f1.get(s1), dst.f1.get(d1));
-		}
-		assertEquals(src.f2.size(), dst.f2.size());
-		Iterator srcf2 = src.f2.keySet().iterator();
-		Iterator dstf2 = dst.f2.keySet().iterator();
-		while (srcf2.hasNext()) {
-			String s2 = srcf2.next();
-			String d2 = dstf2.next();
-			assertEquals(s2, d2);
-			assertEquals(src.f2.get(s2), dst.f2.get(d2));
-		}
-	}
-
-	public void testMapTypes02() throws Exception {
-		SampleMapTypes src = (SampleMapTypes) PackUnpackUtil
-				.newEnhancedInstance(SampleMapTypes.class);
-		src.f0 = new HashMap();
-		src.f1 = new HashMap();
-		src.f1.put(1, 1);
-		src.f1.put(2, 2);
-		src.f1.put(3, 3);
-		src.f2 = new HashMap();
-		src.f2.put("k1", 1);
-		src.f2.put("k2", 2);
-		src.f2.put("k3", 3);
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		SampleMapTypes dst = (SampleMapTypes) PackUnpackUtil
-				.initEnhancedInstance(mpo, SampleMapTypes.class);
-		assertEquals(src.f0.size(), dst.f0.size());
-		assertEquals(src.f1.size(), dst.f1.size());
-		Iterator srcf1 = src.f1.keySet().iterator();
-		Iterator dstf1 = dst.f1.keySet().iterator();
-		while (srcf1.hasNext()) {
-			Integer s1 = srcf1.next();
-			Integer d1 = dstf1.next();
-			assertEquals(s1, d1);
-			assertEquals(src.f1.get(s1), dst.f1.get(d1));
-		}
-		assertEquals(src.f2.size(), dst.f2.size());
-		Iterator srcf2 = src.f2.keySet().iterator();
-		Iterator dstf2 = dst.f2.keySet().iterator();
-		while (srcf2.hasNext()) {
-			String s2 = srcf2.next();
-			String d2 = dstf2.next();
-			assertEquals(s2, d2);
-			assertEquals(src.f2.get(s2), dst.f2.get(d2));
-		}
-		assertFalse(it.hasNext());
-	}
-
-	@MessagePackUnpackable
-	public static class SampleMapTypes {
-		public Map f0;
-		public Map f1;
-		public Map f2;
-
-		public SampleMapTypes() {
-		}
-	}
-
-	@Test
-	public void testDefaultConstructorModifiers() throws Exception {
-		try {
-			PackUnpackUtil.newEnhancedInstance(NoDefaultConstructorClass.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
-		try {
-			PackUnpackUtil
-					.newEnhancedInstance(PrivateDefaultConstructorClass.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
-		try {
-			PackUnpackUtil
-					.newEnhancedInstance(ProtectedDefaultConstructorClass.class);
-			assertTrue(true);
-		} catch (DynamicCodeGenException e) {
-			fail();
-		}
-		assertTrue(true);
-		try {
-			PackUnpackUtil
-					.newEnhancedInstance(PackageDefaultConstructorClass.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
-	}
-
-	@MessagePackUnpackable
-	public static class NoDefaultConstructorClass {
-		public NoDefaultConstructorClass(int i) {
-		}
-	}
-
-	@MessagePackUnpackable
-	public static class PrivateDefaultConstructorClass {
-		private PrivateDefaultConstructorClass() {
-		}
-	}
-
-	@MessagePackUnpackable
-	public static class ProtectedDefaultConstructorClass {
-		protected ProtectedDefaultConstructorClass() {
-		}
-	}
-
-	@MessagePackUnpackable
-	public static class PackageDefaultConstructorClass {
-		PackageDefaultConstructorClass() {
-		}
-	}
-
-	@Test
-	public void testClassModifiers() throws Exception {
-		try {
-			PackUnpackUtil.newEnhancedInstance(PrivateModifierClass.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
-		try {
-			PackUnpackUtil.newEnhancedInstance(ProtectedModifierClass.class);
-			assertTrue(true);
-		} catch (DynamicCodeGenException e) {
-			fail();
-		}
-		assertTrue(true);
-		try {
-			PackUnpackUtil.newEnhancedInstance(PackageModifierClass.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
-	}
-
-	@MessagePackUnpackable
-	private static class PrivateModifierClass {
-	}
-
-	@MessagePackUnpackable
-	protected static class ProtectedModifierClass {
-		protected ProtectedModifierClass() {
-		}
-	}
-
-	@MessagePackUnpackable
-	static class PackageModifierClass {
-	}
-
-	@Test
-	public void testFinalClassAndAbstractClass() throws Exception {
-		try {
-			PackUnpackUtil.newEnhancedInstance(FinalModifierClass.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
-		try {
-			PackUnpackUtil.newEnhancedInstance(AbstractModifierClass.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
-	}
-
-	@MessagePackUnpackable
-	public final static class FinalModifierClass {
-	}
-
-	@MessagePackUnpackable
-	public abstract static class AbstractModifierClass {
-	}
-
-	@Test
-	public void testInterfaceAndEnumType() throws Exception {
-		try {
-			PackUnpackUtil.newEnhancedInstance(SampleInterface.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
-		try {
-			PackUnpackUtil.newEnhancedInstance(SampleEnum.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
-	}
-
-	@MessagePackUnpackable
-	public interface SampleInterface {
-	}
-
-	@MessagePackUnpackable
-	public enum SampleEnum {
-	}
-
-	@Test
-	public void testFieldModifiers01() throws Exception {
-		FieldModifiersClass src = (FieldModifiersClass) PackUnpackUtil
-				.newEnhancedInstance(FieldModifiersClass.class);
-		src.f0 = 0;
-		src.f2 = 2;
-		src.f3 = 3;
-		src.f4 = 4;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		FieldModifiersClass dst = (FieldModifiersClass) PackUnpackUtil
-				.newEnhancedInstance(FieldModifiersClass.class);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		pac.unpack((MessageUnpackable) dst);
-		assertTrue(src.f0 == dst.f0);
-		assertTrue(src.f1 == dst.f1);
-		assertTrue(src.f2 != dst.f2);
-		assertTrue(src.f3 == dst.f3);
-		assertTrue(src.f4 != dst.f4);
-	}
-
-	@Test
-	public void testFieldModifiers02() throws Exception {
-		FieldModifiersClass src = (FieldModifiersClass) PackUnpackUtil
-				.newEnhancedInstance(FieldModifiersClass.class);
-		src.f0 = 0;
-		src.f2 = 2;
-		src.f3 = 3;
-		src.f4 = 4;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		FieldModifiersClass dst = (FieldModifiersClass) PackUnpackUtil
-				.initEnhancedInstance(mpo, FieldModifiersClass.class);
-		assertTrue(src.f0 == dst.f0);
-		assertTrue(src.f1 == dst.f1);
-		assertTrue(src.f2 != dst.f2);
-		assertTrue(src.f3 == dst.f3);
-		assertTrue(src.f4 != dst.f4);
-		assertFalse(it.hasNext());
-	}
-
-	@MessagePackUnpackable
-	public static class FieldModifiersClass {
-		public int f0;
-		public final int f1 = 1;
-		private int f2;
-		protected int f3;
-		int f4;
-
-		public FieldModifiersClass() {
-		}
-	}
-
-	@Test
-	public void testNestedAnnotatedFieldClass01() throws Exception {
-		NestedClass src2 = (NestedClass) PackUnpackUtil
-				.newEnhancedInstance(NestedClass.class);
-		BaseClass src = (BaseClass) PackUnpackUtil
-				.newEnhancedInstance(BaseClass.class);
-		src.f0 = 0;
-		src2.f2 = 2;
-		src.f1 = src2;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		BaseClass dst = (BaseClass) PackUnpackUtil
-				.newEnhancedInstance(BaseClass.class);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		pac.unpack((MessageUnpackable) dst);
-		assertTrue(src.f0 == dst.f0);
-		assertTrue(src2.f2 == dst.f1.f2);
-	}
-
-	@Test
-	public void testNestedAnnotatedFieldClass02() throws Exception {
-		NestedClass src2 = (NestedClass) PackUnpackUtil
-				.newEnhancedInstance(NestedClass.class);
-		BaseClass src = (BaseClass) PackUnpackUtil
-				.newEnhancedInstance(BaseClass.class);
-		src.f0 = 0;
-		src2.f2 = 2;
-		src.f1 = src2;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		BaseClass dst = (BaseClass) PackUnpackUtil.initEnhancedInstance(mpo,
-				BaseClass.class);
-		assertTrue(src.f0 == dst.f0);
-		assertTrue(src2.f2 == dst.f1.f2);
-		assertFalse(it.hasNext());
-	}
-
-	@MessagePackUnpackable
-	public static class BaseClass {
-		public int f0;
-		public NestedClass f1;
-
-		public BaseClass() {
-		}
-	}
-
-	@MessagePackUnpackable
-	public static class NestedClass {
-		public int f2;
-
-		public NestedClass() {
-		}
-	}
-
-	@Test
-	public void testExtendedClass01() throws Exception {
-		SampleSubClass src = (SampleSubClass) PackUnpackUtil
-				.newEnhancedInstance(SampleSubClass.class);
-		src.f0 = 0;
-		src.f2 = 2;
-		src.f3 = 3;
-		src.f4 = 4;
-		src.f5 = 5;
-		src.f8 = 8;
-		src.f9 = 9;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		SampleSubClass dst = (SampleSubClass) PackUnpackUtil
-				.newEnhancedInstance(SampleSubClass.class);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		pac.unpack((MessageUnpackable) dst);
-		assertTrue(src.f0 == dst.f0);
-		assertTrue(src.f1 == dst.f1);
-		assertTrue(src.f2 != dst.f2);
-		assertTrue(src.f3 == dst.f3);
-		assertTrue(src.f4 != dst.f4);
-		assertTrue(src.f5 == dst.f5);
-		assertTrue(src.f6 == dst.f6);
-		assertTrue(src.f8 == dst.f8);
-		assertTrue(src.f9 != dst.f9);
-	}
-
-	@Test
-	public void testExtendedClass02() throws Exception {
-		SampleSubClass src = (SampleSubClass) PackUnpackUtil
-				.newEnhancedInstance(SampleSubClass.class);
-		src.f0 = 0;
-		src.f2 = 2;
-		src.f3 = 3;
-		src.f4 = 4;
-		src.f5 = 5;
-		src.f8 = 8;
-		src.f9 = 9;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		new Packer(out).pack(src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		SampleSubClass dst = (SampleSubClass) PackUnpackUtil
-				.initEnhancedInstance(mpo, SampleSubClass.class);
-		assertTrue(src.f0 == dst.f0);
-		assertTrue(src.f1 == dst.f1);
-		assertTrue(src.f2 != dst.f2);
-		assertTrue(src.f3 == dst.f3);
-		assertTrue(src.f4 != dst.f4);
-		assertTrue(src.f5 == dst.f5);
-		assertTrue(src.f6 == dst.f6);
-		assertTrue(src.f8 == dst.f8);
-		assertTrue(src.f9 != dst.f9);
-		assertFalse(it.hasNext());
-	}
-
-	@MessagePackUnpackable
-	public static class SampleSubClass extends SampleSuperClass {
-		public int f0;
-		public final int f1 = 1;
-		private int f2;
-		protected int f3;
-		int f4;
-
-		public SampleSubClass() {
-		}
-	}
-
-	public static class SampleSuperClass {
-		public int f5;
-		public final int f6 = 2;
-		private int f7;
-		protected int f8;
-		int f9;
-
-		public SampleSuperClass() {
-		}
-	}
-}

From 21678aeef5345a9e44cab1af1b91ccda8ce8855f Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Tue, 28 Sep 2010 19:11:54 +0900
Subject: [PATCH 0816/1648] java: moves a TestDynamicCodeGenPackerTemplate.java
 file to org.msgpack.util.codegen

---
 .../util/{ => codegen}/TestDynamicCodeGenPackerTemplate.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename java/src/test/java/org/msgpack/util/{ => codegen}/TestDynamicCodeGenPackerTemplate.java (98%)

diff --git a/java/src/test/java/org/msgpack/util/TestDynamicCodeGenPackerTemplate.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerTemplate.java
similarity index 98%
rename from java/src/test/java/org/msgpack/util/TestDynamicCodeGenPackerTemplate.java
rename to java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerTemplate.java
index 5b52c8a..aba8920 100644
--- a/java/src/test/java/org/msgpack/util/TestDynamicCodeGenPackerTemplate.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerTemplate.java
@@ -1,4 +1,4 @@
-package org.msgpack.util;
+package org.msgpack.util.codegen;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;

From 01f944e6bd67574d6094e6aafa69806250d785f2 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Tue, 28 Sep 2010 20:35:05 +0900
Subject: [PATCH 0817/1648] perl: tiny tweaks

---
 perl/xs-src/unpack.c | 40 ++++++++++++++++++++++------------------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 95b5910..065573a 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -242,14 +242,17 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c
 
 #include "msgpack/unpack_template.h"
 
-#define UNPACKER(from, name)                                              \
-    msgpack_unpack_t *name;                                               \
-    if(!(SvROK(from) && SvIOK(SvRV(from)))) {                             \
-        Perl_croak(aTHX_ "Invalid unpacker instance for " #name);         \
-    }                                                                     \
-    name = INT2PTR(msgpack_unpack_t*, SvIVX(SvRV((from))));               \
-    if(name == NULL) {                                                    \
-        Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be."); \
+#define UNPACKER(from, name)                                                  \
+    msgpack_unpack_t *name;                                                   \
+    {                                                                         \
+        SV* const obj = from;                                                 \
+        if(!(SvROK(obj) && SvIOK(SvRV(obj)))) {                               \
+            Perl_croak(aTHX_ "Invalid unpacker instance for " #name);         \
+        }                                                                     \
+        name = INT2PTR(msgpack_unpack_t*, SvIVX(SvRV((obj))));                \
+        if(name == NULL) {                                                    \
+            Perl_croak(aTHX_ "NULL found for " # name " when shouldn't be");  \
+        }                                                                     \
     }
 
 XS(xs_unpack) {
@@ -325,7 +328,7 @@ XS(xs_unpacker_new) {
     XSRETURN(1);
 }
 
-STATIC_INLINE SV*
+STATIC_INLINE size_t
 _execute_impl(SV* const self, SV* const data, UV const offset, UV const limit) {
     dTHX;
 
@@ -343,10 +346,9 @@ _execute_impl(SV* const self, SV* const data, UV const offset, UV const limit) {
 
     if(ret < 0) {
         Perl_croak(aTHX_ "Data::MessagePack::Unpacker: parse error while executing");
-    } else {
-        mp->user.finished = (ret > 0) ? true : false;
-        return sv_2mortal(newSVuv(from));
     }
+    mp->user.finished = (ret > 0) ? true : false;
+    return from;
 }
 
 XS(xs_unpacker_execute) {
@@ -365,9 +367,9 @@ XS(xs_unpacker_execute) {
         Perl_croak(aTHX_ "Usage: $unpacker->execute(data, offset = 0)");
     }
 
-    UNPACKER(self, mp);
-
-    ST(0) = _execute_impl(self, data, offset, sv_len(data));
+    dXSTARG;
+    sv_setuv(TARG, _execute_impl(self, data, offset, sv_len(data)));
+    ST(0) = TARG;
     XSRETURN(1);
 }
 
@@ -382,7 +384,9 @@ XS(xs_unpacker_execute_limit) {
     UV  const offset = SvUVx(ST(2));
     UV  const limit  = SvUVx(ST(3));
 
-    ST(0) = _execute_impl(self, data, offset, limit);
+    dXSTARG;
+    sv_setuv(TARG, _execute_impl(self, data, offset, limit));
+    ST(0) = TARG;
     XSRETURN(1);
 }
 
@@ -417,7 +421,7 @@ XS(xs_unpacker_reset) {
     UNPACKER(ST(0), mp);
 
     SV* const data = template_data(mp);
-    sv_2mortal(data);
+    SvREFCNT_dec(data);
     _reset(ST(0));
 
     XSRETURN(0);
@@ -432,7 +436,7 @@ XS(xs_unpacker_destroy) {
     UNPACKER(ST(0), mp);
 
     SV* const data = template_data(mp);
-    sv_2mortal(data);
+    SvREFCNT_dec(data);
     Safefree(mp);
 
     XSRETURN(0);

From 92ddb37ed300d86409db4852d85d2ab40d5b782e Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Tue, 28 Sep 2010 22:51:38 +0900
Subject: [PATCH 0818/1648] java: add test programs for DynamicCodeGenPacker,
 Unpacker, Converter classes

---
 .../org/msgpack/util/codegen/Constants.java   |   2 +
 .../msgpack/util/codegen/DynamicCodeGen.java  | 941 +++++++++++-------
 .../util/codegen/DynamicCodeGenConverter.java |  17 +
 .../TestDynamicCodeGenPackerConverter.java    | 591 +++++++++++
 .../TestDynamicCodeGenPackerTemplate.java     |  77 --
 .../TestDynamicCodeGenPackerUnpacker.java     | 556 +++++++++++
 6 files changed, 1722 insertions(+), 462 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenConverter.java
 create mode 100644 java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
 delete mode 100644 java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerTemplate.java
 create mode 100644 java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java

diff --git a/java/src/main/java/org/msgpack/util/codegen/Constants.java b/java/src/main/java/org/msgpack/util/codegen/Constants.java
index 2226594..91d584e 100644
--- a/java/src/main/java/org/msgpack/util/codegen/Constants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/Constants.java
@@ -5,5 +5,7 @@ public interface Constants extends BasicConstants {
 
 	String POSTFIX_TYPE_NAME_UNPACKER = "_$$_Unpacker";
 
+	String POSTFIX_TYPE_NAME_CONVERTER = "_$$_Converter";
+
 	String POSTFIX_TYPE_NAME_TEMPLATE = "_$$_Template";
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 9193b98..08851c8 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -22,6 +22,8 @@ import javassist.CtNewConstructor;
 import javassist.CtNewMethod;
 import javassist.NotFoundException;
 
+import org.msgpack.CustomConverter;
+import org.msgpack.CustomUnpacker;
 import org.msgpack.MessageConvertable;
 import org.msgpack.MessagePackObject;
 import org.msgpack.MessagePacker;
@@ -93,6 +95,26 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
+	public Class generateMessageConverterClass(Class origClass) {
+		try {
+			String origName = origClass.getName();
+			String converterName = origName + POSTFIX_TYPE_NAME_CONVERTER
+					+ inc();
+			checkClassValidation(origClass);
+			checkDefaultConstructorValidation(origClass);
+			CtClass converterCtClass = pool.makeClass(converterName);
+			setInterface(converterCtClass, MessageUnpacker.class);
+			addDefaultConstructor(converterCtClass);
+			Field[] fields = getDeclaredFields(origClass);
+			addConvertMethod(converterCtClass, origClass, fields);
+			return createClass(converterCtClass);
+		} catch (NotFoundException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (CannotCompileException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+
 	public Class generateTemplateClass(Class origClass) {
 		try {
 			String origName = origClass.getName();
@@ -252,7 +274,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				new String[] { VARIABLE_NAME_PK }, new Class[] {
 						MessageTypeException.class, IOException.class }, bsb
 						.toString());
-		// System.out.println("unpack method: " + sb.toString());
+		//System.out.println("unpack method: " + sb.toString());
 		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), unpackerCtClass);
 		unpackerCtClass.addMethod(newCtMethod);
 	}
@@ -298,6 +320,8 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		} else if (Map.class.isAssignableFrom(c)) {
 			// Map
 			insertCodeOfUnpackMethodCallForMapType(sb, f, c);
+		} else if (CustomUnpacker.isRegistered(c)) {
+			insertCodeOfUnpackMethodCallForRegisteredType(sb, f, c);
 		} else if (MessageUnpackable.class.isAssignableFrom(c)) {
 			// MessageUnpackable
 			insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, f, c);
@@ -365,6 +389,8 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 
 		// field initializer
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
 		sb.append(field.getName());
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(CHAR_NAME_EQUAL);
@@ -406,6 +432,8 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 
 		// block
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
 		sb.append(field.getName());
 		sb.append(CHAR_NAME_DOT);
 		sb.append(METHOD_NAME_ADD);
@@ -441,6 +469,8 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 
 		// field initializer
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
 		sb.append(field.getName());
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(CHAR_NAME_EQUAL);
@@ -482,6 +512,8 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 
 		// block map.
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
 		sb.append(field.getName());
 		sb.append(CHAR_NAME_DOT);
 		sb.append(METHOD_NAME_PUT);
@@ -498,15 +530,275 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 	}
 
-	// private void insertCodeOfMessageUnpackCallForEnhancedType(StringBuilder
-	// sb,
-	// Field f, Class c) {
-	// c = this.getCache(c.getName());
-	// insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, f, c);
-	// }
+	private void insertCodeOfUnpackMethodCallForRegisteredType(
+			StringBuilder sb, Field f, Class c) {
+		// if (t.fi == null) { t.fi = new Foo(); }
+		// sb.append(KEYWORD_IF);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		// sb.append(VARIABLE_NAME_TARGET);
+		// sb.append(CHAR_NAME_DOT);
+		// sb.append(f.getName());
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(CHAR_NAME_EQUAL);
+		// sb.append(CHAR_NAME_EQUAL);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(KEYWORD_NULL);
+		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(VARIABLE_NAME_TARGET);
+		// sb.append(CHAR_NAME_DOT);
+		// sb.append(f.getName());
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(CHAR_NAME_EQUAL);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(KEYWORD_NEW);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(c.getName());
+		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		// sb.append(CHAR_NAME_SEMICOLON);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		// sb.append(CHAR_NAME_SPACE);
+
+		// tmpl1.unpack(new Unpacker(in));
+		// CustomUnpacker.get(c).pack(new Packer(out), src);
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(f.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(c.getName());
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CustomUnpacker.class.getName());
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_GET);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(c.getName() + ".class");
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_UNPACK);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(VARIABLE_NAME_PK);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		insertSemicolon(sb);
+		//		
+		// sb.append(VARIABLE_NAME_PK);
+		// sb.append(CHAR_NAME_DOT);
+		// sb.append(METHOD_NAME_UNPACK);
+		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		// sb.append(MessageUnpackable.class.getName());
+		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		// sb.append(VARIABLE_NAME_TARGET);
+		// sb.append(CHAR_NAME_DOT);
+		// sb.append(f.getName());
+		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		// sb.append(CHAR_NAME_SEMICOLON);
+		// sb.append(CHAR_NAME_SPACE);
+	}
 
 	private void insertCodeOfMessageUnpackCallForMsgUnpackableType(
 			StringBuilder sb, Field f, Class c) {
+		// if (t.fi == null) { t.fi = new Foo(); }
+		sb.append(KEYWORD_IF);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(f.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(KEYWORD_NULL);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(f.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(KEYWORD_NEW);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(c.getName());
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+
+		// insert a right variable // ignore
+		sb.append(VARIABLE_NAME_PK);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_UNPACK);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(MessageUnpackable.class.getName());
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(f.getName());
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+	}
+
+	public void addConvertMethod(CtClass tmplCtClass, Class c, Field[] fs)
+			throws CannotCompileException, NotFoundException {
+		// Object convert(MessagePackObject from) throws MessageTypeException;
+		StringBuilder sb = new StringBuilder();
+		StringBuilder bsb = new StringBuilder();
+		insertConvertMethodBody(bsb, c, fs);
+		addPublicMethodDecl(sb, METHOD_NAME_CONVERT, Object.class,
+				new Class[] { MessagePackObject.class },
+				new String[] { VARIABLE_NAME_MPO },
+				new Class[] { MessageTypeException.class }, bsb.toString());
+		//System.out.println("convert method: " + sb.toString());
+		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
+		tmplCtClass.addMethod(newCtMethod);
+	}
+
+	private void insertConvertMethodBody(StringBuilder sb, Class c,
+			Field[] fields) throws CannotCompileException {
+		insertLocalVariableDecl(sb, c, VARIABLE_NAME_TARGET);
+		StringBuilder mc = new StringBuilder();
+		insertDefaultConsCall(mc, c);
+		insertValueInsertion(sb, mc.toString());
+		insertSemicolon(sb);
+		insertCodeOfMessagePackObjectArrayGet(sb);
+		insertCodeOfConvertMethodCalls(sb, fields);
+		insertReturnStat(sb, VARIABLE_NAME_TARGET);
+		insertSemicolon(sb);
+	}
+
+	private void insertCodeOfMessagePackObjectArrayGet(StringBuilder sb) {
+		// MessagePackObject[] ary = obj.asArray();
+		sb.append(MessagePackObject.class.getName());
+		sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+		sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_ARRAY);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_MPO);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_ASARRAY);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+	}
+
+	private void insertCodeOfConvertMethodCalls(StringBuilder sb, Field[] fields) {
+		for (int i = 0; i < fields.length; ++i) {
+			insertCodeOfConvertMethodCall(sb, fields[i], fields[i].getType(),
+					i, null);
+		}
+	}
+
+	private void insertCodeOfConvertMethodCall(StringBuilder sb, Field f,
+			Class c, int i, String v) {
+		if (c.isPrimitive()) { // primitive type
+			insertCodeOfConvertMethodCallForPrimTypes(sb, f, c, i, v);
+		} else { // reference type
+			if (c.equals(Boolean.class) || c.equals(Byte.class)
+					|| c.equals(Short.class) || c.equals(Integer.class)
+					|| c.equals(Float.class) || c.equals(Long.class)
+					|| c.equals(Double.class)) {
+				// wrapper type
+				insertCodeOfConvertMethodCallForWrapTypes(sb, f, c, i, v);
+			} else if (c.equals(String.class) || c.equals(byte[].class)
+					|| c.equals(BigInteger.class)) {
+				insertCodeOfConvertMethodCallForPrimTypes(sb, f, c, i, v);
+			} else if (List.class.isAssignableFrom(c)) {
+				insertCodeOfConvertMethodCallForList(sb, f, c, i);
+			} else if (Map.class.isAssignableFrom(c)) {
+				insertCodeOfConvertMethodCallForMapType(sb, f, c, i);
+			} else if (MessageConvertable.class.isAssignableFrom(c)) {
+				insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
+			} else if (CustomConverter.isRegistered(c)) {
+				insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
+			} else {
+				throw new MessageTypeException("Type error: " + c.getName());
+			}
+		}
+	}
+
+	private void insertCodeOfMessageConvertCallForRegisteredType(
+			StringBuilder sb, Field f, Class c, int i) {
+		// if (t.fi == null) { t.fi = new Foo(); }
+		// sb.append(KEYWORD_IF);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		// sb.append(VARIABLE_NAME_TARGET);
+		// sb.append(CHAR_NAME_DOT);
+		// sb.append(f.getName());
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(CHAR_NAME_EQUAL);
+		// sb.append(CHAR_NAME_EQUAL);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(KEYWORD_NULL);
+		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(VARIABLE_NAME_TARGET);
+		// sb.append(CHAR_NAME_DOT);
+		// sb.append(f.getName());
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(CHAR_NAME_EQUAL);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(KEYWORD_NEW);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(c.getName());
+		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		// sb.append(CHAR_NAME_SEMICOLON);
+		// sb.append(CHAR_NAME_SPACE);
+		// sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		// sb.append(CHAR_NAME_SPACE);
+
+		// ((MessageConvertable)f_i).messageConvert(ary[i]);
+		// obj = tmpl.convert(mpo);
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(f.getName());
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(c.getName());
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CustomConverter.class.getName());
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_GET);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(c.getName() + ".class");
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_CONVERT);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(VARIABLE_NAME_ARRAY);
+		sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+		sb.append(i);
+		sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		insertSemicolon(sb);
+	}
+
+	private void insertCodeOfMessageConvertCallForMsgConvtblType(
+			StringBuilder sb, Field f, Class c, int i) {
 		// if (fi == null) { fi = new Foo_$$_Enhanced(); }
 		sb.append(KEYWORD_IF);
 		sb.append(CHAR_NAME_SPACE);
@@ -535,215 +827,86 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
 
-		// insert a right variable // ignore
-		sb.append(VARIABLE_NAME_PK);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_UNPACK);
+		// ((MessageConvertable)f_i).messageConvert(ary[i]);
 		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(MessageUnpackable.class.getName());
+		sb.append(MessageConvertable.class.getName());
 		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
 		sb.append(f.getName());
 		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_MSGCONVERT);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(VARIABLE_NAME_ARRAY);
+		sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+		sb.append(i);
+		sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
 		sb.append(CHAR_NAME_SEMICOLON);
 		sb.append(CHAR_NAME_SPACE);
 	}
 
-	public void addConvertMethod(CtClass tmplCtClass, Class c, Field[] fs)
-			throws CannotCompileException, NotFoundException {
-		// Object convert(MessagePackObject from) throws MessageTypeException;
-		StringBuilder sb = new StringBuilder();
-		StringBuilder bsb = new StringBuilder();
-		insertConvertMethodBody(bsb, c, fs);
-		addPublicMethodDecl(sb, METHOD_NAME_CONVERT, Object.class,
-				new Class[] { MessagePackObject.class },
-				new String[] { VARIABLE_NAME_MPO },
-				new Class[] { MessageTypeException.class }, bsb.toString());
-		// System.out.println("convert method: " + sb.toString());
-		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
-		tmplCtClass.addMethod(newCtMethod);
-	}
-
-	private void insertConvertMethodBody(StringBuilder sb, Class c,
-			Field[] fields) throws CannotCompileException {
-		insertLocalVariableDecl(sb, c, VARIABLE_NAME_TARGET);
-		StringBuilder mc = new StringBuilder();
-		insertDefaultConsCall(mc, c);
-		insertValueInsertion(sb, mc.toString());
-		insertSemicolon(sb);
-		insertCodeOfMessagePackObjectArrayGet(sb);
-		insertCodeOfConvertMethodCalls(sb, fields);
-		insertReturnStat(sb, VARIABLE_NAME_TARGET);
-		insertSemicolon(sb);
-	}
-
-	private void insertCodeOfMessagePackObjectArrayGet(StringBuilder sb) {
-		// MessagePackObject[] ary = obj.asArray();
-		sb.append(MessagePackObject.class.getName());
-		sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-		sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_ARRAY);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_MPO);
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_ASARRAY);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
-	}
-
-	private void insertCodeOfConvertMethodCalls(StringBuilder sb, Field[] fields) {
-		for (int i = 0; i < fields.length; ++i) {
-			insertCodeOfMessageConvertCall(sb, fields[i], fields[i].getType(),
-					i, null);
-		}
-	}
-
-	private void insertCodeOfMessageConvertCall(StringBuilder sb, Field f,
-			Class c, int i, String v) {
-		if (c.isPrimitive()) { // primitive type
-			insertCodeOfConvertMethodCallForPrimTypes(sb, f, c, i, v);
-		} else { // reference type
-			if (c.equals(Boolean.class) || c.equals(Byte.class)
-					|| c.equals(Short.class) || c.equals(Integer.class)
-					|| c.equals(Float.class) || c.equals(Long.class)
-					|| c.equals(Double.class)) {
-				// wrapper type
-				insertCodeOfMessageConvertCallForWrapTypes(sb, f, c, i, v);
-			} else if (c.equals(String.class) || c.equals(byte[].class)
-					|| c.equals(BigInteger.class)) {
-				insertCodeOfConvertMethodCallForPrimTypes(sb, f, c, i, v);
-			} else if (List.class.isAssignableFrom(c)) {
-				insertCodeOfMessageConvertCallForList(sb, f, c, i);
-			} else if (Map.class.isAssignableFrom(c)) {
-				insertCodeOfMessageConvertCallForMap(sb, f, c, i);
-			} else if (MessageConvertable.class.isAssignableFrom(c)) {
-				insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
-			} else {
-				throw new MessageTypeException("Type error: " + c.getName());
-			}
-		}
-	}
-
-	//
-	// private void insertCodeOfMessageConvertCallForEnhancedType(
-	// StringBuilder sb, Field f, Class c, int i) {
-	// c = getCache(c.getName());
-	// insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
-	// }
-
-	private void insertCodeOfMessageConvertCallForMsgConvtblType(
-			StringBuilder sb, Field f, Class c, int i) {
-		// if (fi == null) { fi = new Foo_$$_Enhanced(); }
-		sb.append(Constants.KEYWORD_IF);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(f.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.KEYWORD_NULL);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(f.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.KEYWORD_NEW);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(c.getName());
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACKET);
-		sb.append(Constants.CHAR_NAME_SPACE);
-
-		// ((MessageConvertable)f_i).messageConvert(ary[i]);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(MessageConvertable.class.getName());
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(f.getName());
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_MSGCONVERT);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.VARIABLE_NAME_ARRAY);
-		sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
-		sb.append(i);
-		sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
-	}
-
 	private void insertCodeOfConvertMethodCallForPrimTypes(StringBuilder sb,
 			Field f, Class c, int i, String name) {
 		// target.f0 = objs[0].intValue();
 		if (f != null) {
-			sb.append(Constants.VARIABLE_NAME_TARGET);
-			sb.append(Constants.CHAR_NAME_DOT);
+			sb.append(VARIABLE_NAME_TARGET);
+			sb.append(CHAR_NAME_DOT);
 			sb.append(f.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.VARIABLE_NAME_ARRAY);
-			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(CHAR_NAME_SPACE);
+			sb.append(CHAR_NAME_EQUAL);
+			sb.append(CHAR_NAME_SPACE);
+			sb.append(VARIABLE_NAME_ARRAY);
+			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
 			sb.append(i);
-			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
 		} else {
 			sb.append(name);
 		}
-		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(CHAR_NAME_DOT);
 		if (c.equals(boolean.class)) {
-			sb.append(Constants.METHOD_NAME_ASBOOLEAN);
+			sb.append(METHOD_NAME_ASBOOLEAN);
 		} else if (c.equals(byte.class)) {
-			sb.append(Constants.METHOD_NAME_ASBYTE);
+			sb.append(METHOD_NAME_ASBYTE);
 		} else if (c.equals(short.class)) {
-			sb.append(Constants.METHOD_NAME_ASSHORT);
+			sb.append(METHOD_NAME_ASSHORT);
 		} else if (c.equals(int.class)) {
-			sb.append(Constants.METHOD_NAME_ASINT);
+			sb.append(METHOD_NAME_ASINT);
 		} else if (c.equals(float.class)) {
-			sb.append(Constants.METHOD_NAME_ASFLOAT);
+			sb.append(METHOD_NAME_ASFLOAT);
 		} else if (c.equals(long.class)) {
-			sb.append(Constants.METHOD_NAME_ASLONG);
+			sb.append(METHOD_NAME_ASLONG);
 		} else if (c.equals(double.class)) {
-			sb.append(Constants.METHOD_NAME_ASDOUBLE);
+			sb.append(METHOD_NAME_ASDOUBLE);
 		} else if (c.equals(String.class)) {
-			sb.append(Constants.METHOD_NAME_ASSTRING);
+			sb.append(METHOD_NAME_ASSTRING);
 		} else if (c.equals(byte[].class)) {
-			sb.append(Constants.METHOD_NAME_ASBYTEARRAY);
+			sb.append(METHOD_NAME_ASBYTEARRAY);
 		} else if (c.equals(BigInteger.class)) {
-			sb.append(Constants.METHOD_NAME_ASBIGINTEGER);
+			sb.append(METHOD_NAME_ASBIGINTEGER);
 		} else {
 			throw new MessageTypeException("Type error: " + c.getName());
 		}
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
 		if (f != null) {
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(CHAR_NAME_SEMICOLON);
+			sb.append(CHAR_NAME_SPACE);
 		}
 	}
 
-	private void insertCodeOfMessageConvertCallForWrapTypes(StringBuilder sb,
+	private void insertCodeOfConvertMethodCallForWrapTypes(StringBuilder sb,
 			Field f, Class c, int i, String v) {
 		if (f != null) {
+			sb.append(VARIABLE_NAME_TARGET);
+			sb.append(CHAR_NAME_DOT);
 			sb.append(f.getName());
-			sb.append(Constants.CHAR_NAME_SPACE);
-			sb.append(Constants.CHAR_NAME_EQUAL);
-			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(CHAR_NAME_SPACE);
+			sb.append(CHAR_NAME_EQUAL);
+			sb.append(CHAR_NAME_SPACE);
 		}
-		sb.append(Constants.KEYWORD_NEW);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(KEYWORD_NEW);
+		sb.append(CHAR_NAME_SPACE);
 		if (c.equals(Boolean.class)) {
 			sb.append(Boolean.class.getName());
 		} else if (c.equals(Byte.class)) {
@@ -761,155 +924,159 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		} else {
 			throw new MessageTypeException("Type error: " + c.getName());
 		}
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		if (f != null) {
-			sb.append(Constants.VARIABLE_NAME_ARRAY);
-			sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(VARIABLE_NAME_ARRAY);
+			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
 			sb.append(i);
-			sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
+			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
 		} else {
 			sb.append(v);
 		}
-		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(CHAR_NAME_DOT);
 		if (c.equals(Boolean.class)) {
-			sb.append(Constants.METHOD_NAME_ASBOOLEAN);
+			sb.append(METHOD_NAME_ASBOOLEAN);
 		} else if (c.equals(Byte.class)) {
-			sb.append(Constants.METHOD_NAME_ASBYTE);
+			sb.append(METHOD_NAME_ASBYTE);
 		} else if (c.equals(Short.class)) {
-			sb.append(Constants.METHOD_NAME_ASSHORT);
+			sb.append(METHOD_NAME_ASSHORT);
 		} else if (c.equals(Integer.class)) {
-			sb.append(Constants.METHOD_NAME_ASINT);
+			sb.append(METHOD_NAME_ASINT);
 		} else if (c.equals(Float.class)) {
-			sb.append(Constants.METHOD_NAME_ASFLOAT);
+			sb.append(METHOD_NAME_ASFLOAT);
 		} else if (c.equals(Long.class)) {
-			sb.append(Constants.METHOD_NAME_ASLONG);
+			sb.append(METHOD_NAME_ASLONG);
 		} else if (c.equals(Double.class)) {
-			sb.append(Constants.METHOD_NAME_ASDOUBLE);
+			sb.append(METHOD_NAME_ASDOUBLE);
 		} else {
 			throw new MessageTypeException("Type error: " + c.getName());
 		}
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
 		if (f != null) {
-			sb.append(Constants.CHAR_NAME_SEMICOLON);
-			sb.append(Constants.CHAR_NAME_SPACE);
+			sb.append(CHAR_NAME_SEMICOLON);
+			sb.append(CHAR_NAME_SPACE);
 		}
 	}
 
-	private void insertCodeOfMessageConvertCallForList(StringBuilder sb,
+	private void insertCodeOfConvertMethodCallForList(StringBuilder sb,
 			Field field, Class type, int i) {
 		ParameterizedType generic = (ParameterizedType) field.getGenericType();
 		Class genericType = (Class) generic.getActualTypeArguments()[0];
 
 		// List list = ary[i].asList();
 		sb.append(List.class.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_LIST);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_ARRAY);
-		sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_LIST);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_ARRAY);
+		sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
 		sb.append(i);
-		sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		sb.append(CHAR_NAME_DOT);
 		sb.append("asList");
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
 
 		// int size = list.size();
 		sb.append(int.class.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_SIZE);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_LIST);
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_SIZE);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_SIZE);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_LIST);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_SIZE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
 
 		// field initializer
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
 		sb.append(field.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.KEYWORD_NEW);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(KEYWORD_NEW);
+		sb.append(CHAR_NAME_SPACE);
 		sb.append(ArrayList.class.getName());
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
 
 		// for loop
-		sb.append(Constants.KEYWORD_FOR);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(KEYWORD_FOR);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		sb.append(int.class.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_I);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_I);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
 		sb.append(0);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_I);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_LESSTHAN);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_SIZE);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_PLUS);
-		sb.append(Constants.CHAR_NAME_PLUS);
-		sb.append(Constants.VARIABLE_NAME_I);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_I);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LESSTHAN);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_SIZE);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_PLUS);
+		sb.append(CHAR_NAME_PLUS);
+		sb.append(VARIABLE_NAME_I);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SPACE);
 
 		// block begin
-		sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
 		sb.append(MessagePackObject.class.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_VAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_VAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		sb.append(MessagePackObject.class.getName());
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.VARIABLE_NAME_LIST);
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_GET);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.VARIABLE_NAME_I);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(VARIABLE_NAME_LIST);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_GET);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(VARIABLE_NAME_I);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
 
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
 		sb.append(field.getName());
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_ADD);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		insertCodeOfMessageConvertCall(sb, null, genericType, -1,
-				Constants.VARIABLE_NAME_VAL);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_ADD);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		insertCodeOfConvertMethodCall(sb, null, genericType, -1,
+				VARIABLE_NAME_VAL);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
 		// block end
-		sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACKET);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
 	}
 
-	private void insertCodeOfMessageConvertCallForMap(StringBuilder sb,
+	private void insertCodeOfConvertMethodCallForMapType(StringBuilder sb,
 			Field f, Class c, int i) {
 		ParameterizedType generic = (ParameterizedType) f.getGenericType();
 		Class genericType0 = (Class) generic.getActualTypeArguments()[0];
@@ -917,133 +1084,137 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 		// Map map = ary[i].asMap();
 		sb.append(Map.class.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_MAP);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_ARRAY);
-		sb.append(Constants.CHAR_NAME_LEFT_SQUARE_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_MAP);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_ARRAY);
+		sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
 		sb.append(i);
-		sb.append(Constants.CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		sb.append(Constants.CHAR_NAME_DOT);
+		sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		sb.append(CHAR_NAME_DOT);
 		sb.append("asMap");
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
 
 		// int size = list.size();
 		sb.append(int.class.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_SIZE);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_MAP);
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_SIZE);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_SIZE);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_MAP);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_SIZE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
 
 		// field initializer
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
 		sb.append(f.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.KEYWORD_NEW);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(KEYWORD_NEW);
+		sb.append(CHAR_NAME_SPACE);
 		sb.append(HashMap.class.getName());
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
 
 		// for loop
-		sb.append(Constants.KEYWORD_FOR);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(KEYWORD_FOR);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		sb.append(Iterator.class.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_ITER);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_MAP);
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_KEYSET);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_ITERATOR);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_ITER);
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_HASNEXT);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_ITER);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_MAP);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_KEYSET);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_ITERATOR);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_ITER);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_HASNEXT);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
 
 		// block map.
 		sb.append(MessagePackObject.class.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_KEY);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_KEY);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		sb.append(MessagePackObject.class.getName());
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.VARIABLE_NAME_ITER);
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_NEXT);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(VARIABLE_NAME_ITER);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_NEXT);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
 		sb.append(MessagePackObject.class.getName());
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.VARIABLE_NAME_VAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_EQUAL);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(VARIABLE_NAME_VAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		sb.append(MessagePackObject.class.getName());
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.VARIABLE_NAME_MAP);
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_GET);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Constants.VARIABLE_NAME_KEY);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(VARIABLE_NAME_MAP);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_GET);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		sb.append(VARIABLE_NAME_KEY);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
 
+		sb.append(VARIABLE_NAME_TARGET);
+		sb.append(CHAR_NAME_DOT);
 		sb.append(f.getName());
-		sb.append(Constants.CHAR_NAME_DOT);
-		sb.append(Constants.METHOD_NAME_PUT);
-		sb.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-		insertCodeOfMessageConvertCall(sb, null, genericType0, -1,
-				Constants.VARIABLE_NAME_KEY);
-		sb.append(Constants.CHAR_NAME_COMMA);
-		sb.append(Constants.CHAR_NAME_SPACE);
-		insertCodeOfMessageConvertCall(sb, null, genericType1, -1,
-				Constants.VARIABLE_NAME_VAL);
-		sb.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(Constants.CHAR_NAME_SEMICOLON);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_DOT);
+		sb.append(METHOD_NAME_PUT);
+		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
+		insertCodeOfConvertMethodCall(sb, null, genericType0, -1,
+				VARIABLE_NAME_KEY);
+		sb.append(CHAR_NAME_COMMA);
+		sb.append(CHAR_NAME_SPACE);
+		insertCodeOfConvertMethodCall(sb, null, genericType1, -1,
+				VARIABLE_NAME_VAL);
+		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		sb.append(CHAR_NAME_SEMICOLON);
+		sb.append(CHAR_NAME_SPACE);
 
-		sb.append(Constants.CHAR_NAME_RIGHT_CURLY_BRACKET);
-		sb.append(Constants.CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
 	}
 
 	private Class createClass(CtClass packerCtClass)
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenConverter.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenConverter.java
new file mode 100644
index 0000000..590cde1
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenConverter.java
@@ -0,0 +1,17 @@
+package org.msgpack.util.codegen;
+
+import org.msgpack.MessageConverter;
+
+public class DynamicCodeGenConverter {
+	public static MessageConverter create(Class c) {
+		try {
+			DynamicCodeGen gen = DynamicCodeGen.getInstance();
+			Class unpackerClass = gen.generateMessageConverterClass(c);
+			return (MessageConverter) unpackerClass.newInstance();
+		} catch (InstantiationException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalAccessException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+}
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
new file mode 100644
index 0000000..7ffcd88
--- /dev/null
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
@@ -0,0 +1,591 @@
+package org.msgpack.util.codegen;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+import org.msgpack.CustomConverter;
+import org.msgpack.CustomPacker;
+import org.msgpack.CustomUnpacker;
+import org.msgpack.MessagePackObject;
+import org.msgpack.MessagePacker;
+import org.msgpack.Packer;
+import org.msgpack.Template;
+import org.msgpack.Unpacker;
+
+public class TestDynamicCodeGenPackerConverter extends TestCase {
+
+	@Test
+	public void testPrimitiveTypeFields() throws Exception {
+		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
+		src.f0 = (byte) 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f6 = false;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(PrimitiveTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate
+				.create(PrimitiveTypeFieldsClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertFalse(it.hasNext());
+	}
+
+	public static class PrimitiveTypeFieldsClass {
+		public byte f0;
+		public short f1;
+		public int f2;
+		public long f3;
+		public float f4;
+		public double f5;
+		public boolean f6;
+
+		public PrimitiveTypeFieldsClass() {
+		}
+	}
+
+	@Test
+	public void testGeneralReferenceTypeFieldsClass() throws Exception {
+		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
+		src.f0 = 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = (long) 3;
+		src.f4 = (float) 4;
+		src.f5 = (double) 5;
+		src.f6 = false;
+		src.f7 = new BigInteger("7");
+		src.f8 = "8";
+		src.f9 = new byte[] { 0x01, 0x02 };
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(GeneralReferenceTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate
+				.create(GeneralReferenceTypeFieldsClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9[0], dst.f9[0]);
+		assertEquals(src.f9[1], dst.f9[1]);
+		assertFalse(it.hasNext());
+	}
+
+	public static class GeneralReferenceTypeFieldsClass {
+		public Byte f0;
+		public Short f1;
+		public Integer f2;
+		public Long f3;
+		public Float f4;
+		public Double f5;
+		public Boolean f6;
+		public BigInteger f7;
+		public String f8;
+		public byte[] f9;
+
+		public GeneralReferenceTypeFieldsClass() {
+		}
+	}
+
+	public void testListTypes() throws Exception {
+		SampleListTypes src = new SampleListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = new ArrayList();
+		src.f1.add(1);
+		src.f1.add(2);
+		src.f1.add(3);
+		src.f2 = new ArrayList();
+		src.f2.add("e1");
+		src.f2.add("e2");
+		src.f2.add("e3");
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(SampleListTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate.create(SampleListTypes.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleListTypes dst = (SampleListTypes) tmpl.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		for (int i = 0; i < src.f1.size(); ++i) {
+			assertEquals(src.f1.get(i), dst.f1.get(i));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		for (int i = 0; i < src.f2.size(); ++i) {
+			assertEquals(src.f2.get(i), dst.f2.get(i));
+		}
+		assertFalse(it.hasNext());
+	}
+
+	public static class SampleListTypes {
+		public List f0;
+		public List f1;
+		public List f2;
+
+		public SampleListTypes() {
+		}
+	}
+
+	public void testMapTypes() throws Exception {
+		SampleMapTypes src = new SampleMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = new HashMap();
+		src.f1.put(1, 1);
+		src.f1.put(2, 2);
+		src.f1.put(3, 3);
+		src.f2 = new HashMap();
+		src.f2.put("k1", 1);
+		src.f2.put("k2", 2);
+		src.f2.put("k3", 3);
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(SampleMapTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate.create(SampleMapTypes.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleMapTypes dst = (SampleMapTypes) tmpl.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		Iterator srcf1 = src.f1.keySet().iterator();
+		Iterator dstf1 = dst.f1.keySet().iterator();
+		while (srcf1.hasNext()) {
+			Integer s1 = srcf1.next();
+			Integer d1 = dstf1.next();
+			assertEquals(s1, d1);
+			assertEquals(src.f1.get(s1), dst.f1.get(d1));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		Iterator srcf2 = src.f2.keySet().iterator();
+		Iterator dstf2 = dst.f2.keySet().iterator();
+		while (srcf2.hasNext()) {
+			String s2 = srcf2.next();
+			String d2 = dstf2.next();
+			assertEquals(s2, d2);
+			assertEquals(src.f2.get(s2), dst.f2.get(d2));
+		}
+		assertFalse(it.hasNext());
+	}
+
+	public static class SampleMapTypes {
+		public Map f0;
+		public Map f1;
+		public Map f2;
+
+		public SampleMapTypes() {
+		}
+	}
+
+	@Test
+	public void testDefaultConstructorModifiers01() throws Exception {
+		try {
+			DynamicCodeGenPacker.create(NoDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(PrivateDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(ProtectedDefaultConstructorClass.class);
+			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(PackageDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@Test
+	public void testDefaultConstructorModifiers02() throws Exception {
+		try {
+			DynamicCodeGenTemplate.create(NoDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenTemplate.create(PrivateDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenTemplate
+					.create(ProtectedDefaultConstructorClass.class);
+			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenTemplate.create(PackageDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	public static class NoDefaultConstructorClass {
+		public NoDefaultConstructorClass(int i) {
+		}
+	}
+
+	public static class PrivateDefaultConstructorClass {
+		private PrivateDefaultConstructorClass() {
+		}
+	}
+
+	public static class ProtectedDefaultConstructorClass {
+		protected ProtectedDefaultConstructorClass() {
+		}
+	}
+
+	public static class PackageDefaultConstructorClass {
+		PackageDefaultConstructorClass() {
+		}
+	}
+
+	@Test
+	public void testClassModifiers01() throws Exception {
+		try {
+			DynamicCodeGenPacker.create(PrivateModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(ProtectedModifierClass.class);
+			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(PackageModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@Test
+	public void testClassModifiers02() throws Exception {
+		try {
+			DynamicCodeGenTemplate.create(PrivateModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenTemplate.create(ProtectedModifierClass.class);
+			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenTemplate.create(PackageModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	private static class PrivateModifierClass {
+	}
+
+	protected static class ProtectedModifierClass {
+		protected ProtectedModifierClass() {
+		}
+	}
+
+	static class PackageModifierClass {
+	}
+
+	@Test
+	public void testFinalClassAndAbstractClass01() throws Exception {
+		try {
+			DynamicCodeGenPacker.create(FinalModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(AbstractModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@Test
+	public void testFinalClassAndAbstractClass02() throws Exception {
+		try {
+			DynamicCodeGenTemplate.create(FinalModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenTemplate.create(AbstractModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	public final static class FinalModifierClass {
+	}
+
+	public abstract static class AbstractModifierClass {
+	}
+
+	@Test
+	public void testInterfaceAndEnumType01() throws Exception {
+		try {
+			DynamicCodeGenPacker.create(SampleInterface.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(SampleEnum.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@Test
+	public void testInterfaceAndEnumType02() throws Exception {
+		try {
+			DynamicCodeGenTemplate.create(SampleInterface.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenTemplate.create(SampleEnum.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	public interface SampleInterface {
+	}
+
+	public enum SampleEnum {
+	}
+
+	@Test
+	public void testFieldModifiers() throws Exception {
+		FieldModifiersClass src = new FieldModifiersClass();
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(FieldModifiersClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate
+				.create(FieldModifiersClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		FieldModifiersClass dst = (FieldModifiersClass) tmpl.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f4 != dst.f4);
+		assertFalse(it.hasNext());
+	}
+
+	public static class FieldModifiersClass {
+		public int f0;
+		public final int f1 = 1;
+		private int f2;
+		protected int f3;
+		int f4;
+
+		public FieldModifiersClass() {
+		}
+	}
+
+	@Test
+	public void testNestedFieldClass() throws Exception {
+		MessagePacker packer2 = DynamicCodeGenPacker.create(NestedClass.class);
+		CustomPacker.register(NestedClass.class, packer2);
+		MessagePacker packer1 = DynamicCodeGenPacker.create(BaseClass.class);
+		CustomPacker.register(BaseClass.class, packer1);
+		Template tmpl2 = DynamicCodeGenTemplate.create(NestedClass.class);
+		CustomUnpacker.register(NestedClass.class, tmpl2);
+		CustomConverter.register(NestedClass.class, tmpl2);
+		Template tmpl1 = DynamicCodeGenTemplate.create(BaseClass.class);
+		CustomUnpacker.register(BaseClass.class, tmpl1);
+		CustomConverter.register(BaseClass.class, tmpl1);
+		BaseClass src = new BaseClass();
+		NestedClass src2 = new NestedClass();
+		src.f0 = 0;
+		src2.f2 = 2;
+		src.f1 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer1.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		BaseClass dst = (BaseClass) tmpl1.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1.f2 == dst.f1.f2);
+		assertFalse(it.hasNext());
+	}
+
+	public static class BaseClass {
+		public int f0;
+		public NestedClass f1;
+
+		public BaseClass() {
+		}
+	}
+
+	public static class NestedClass {
+		public int f2;
+
+		public NestedClass() {
+		}
+	}
+
+	@Test
+	public void testExtendedClass() throws Exception {
+		SampleSubClass src = new SampleSubClass();
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f8 = 8;
+		src.f9 = 9;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(SampleSubClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate.create(SampleSubClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleSubClass dst = (SampleSubClass) tmpl.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f4 != dst.f4);
+		assertTrue(src.f5 == dst.f5);
+		assertTrue(src.f6 == dst.f6);
+		assertTrue(src.f8 == dst.f8);
+		assertTrue(src.f9 != dst.f9);
+		assertFalse(it.hasNext());
+	}
+
+	public static class SampleSubClass extends SampleSuperClass {
+		public int f0;
+		public final int f1 = 1;
+		private int f2;
+		protected int f3;
+		int f4;
+
+		public SampleSubClass() {
+		}
+	}
+
+	public static class SampleSuperClass {
+		public int f5;
+		public final int f6 = 2;
+		private int f7;
+		protected int f8;
+		int f9;
+
+		public SampleSuperClass() {
+		}
+	}
+}
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerTemplate.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerTemplate.java
deleted file mode 100644
index aba8920..0000000
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerTemplate.java
+++ /dev/null
@@ -1,77 +0,0 @@
-package org.msgpack.util.codegen;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-
-import org.junit.Test;
-import org.msgpack.CustomPacker;
-import org.msgpack.MessagePacker;
-import org.msgpack.Packer;
-import org.msgpack.ReflectionTemplate;
-import org.msgpack.Template;
-import org.msgpack.Unpacker;
-import org.msgpack.util.codegen.DynamicCodeGenPacker;
-import org.msgpack.util.codegen.DynamicCodeGenTemplate;
-
-import static org.junit.Assert.*;
-
-public class TestDynamicCodeGenPackerTemplate {
-
-	public static class StringFieldClass {
-		public String s1;
-		public String s2;
-		public StringFieldClass() { }
-	}
-
-	@Test
-	public void testPackConvert() throws Exception {
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-
-		MessagePacker packer = DynamicCodeGenPacker.create(StringFieldClass.class);
-
-		StringFieldClass src = new StringFieldClass();
-
-		src.s1 = "kumofs";
-		src.s2 = "frsyuki";
-
-		packer.pack(new Packer(out), src);
-
-		Template tmpl = DynamicCodeGenTemplate.create(StringFieldClass.class);
-
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-
-		Object obj = tmpl.unpack(new Unpacker(in));
-		assertEquals(obj.getClass(), StringFieldClass.class);
-
-		StringFieldClass dst = (StringFieldClass)obj;
-		assertEquals(src.s1, dst.s1);
-		assertEquals(src.s2, dst.s2);
-	}
-
-	@Test
-	public void testPackConvert02() throws Exception {
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-
-		CustomPacker.register(StringFieldClass.class, DynamicCodeGenPacker.create(StringFieldClass.class));
-				
-
-		StringFieldClass src = new StringFieldClass();
-
-		src.s1 = "kumofs";
-		src.s2 = "frsyuki";
-
-		new Packer(out).pack(src);
-
-		Template tmpl = ReflectionTemplate.create(StringFieldClass.class);
-
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-
-		Object obj = tmpl.unpack(new Unpacker(in));
-		assertEquals(obj.getClass(), StringFieldClass.class);
-
-		StringFieldClass dst = (StringFieldClass)obj;
-		assertEquals(src.s1, dst.s1);
-		assertEquals(src.s2, dst.s2);
-	}
-}
-
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
new file mode 100644
index 0000000..08d0e52
--- /dev/null
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
@@ -0,0 +1,556 @@
+package org.msgpack.util.codegen;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+import org.msgpack.CustomConverter;
+import org.msgpack.CustomPacker;
+import org.msgpack.CustomUnpacker;
+import org.msgpack.MessagePacker;
+import org.msgpack.Packer;
+import org.msgpack.Template;
+import org.msgpack.Unpacker;
+
+import junit.framework.TestCase;
+
+public class TestDynamicCodeGenPackerUnpacker extends TestCase {
+
+	@Test
+	public void testPrimitiveTypeFields() throws Exception {
+		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
+		src.f0 = (byte) 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f6 = false;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(PrimitiveTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate
+				.create(PrimitiveTypeFieldsClass.class);
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+	}
+
+	public static class PrimitiveTypeFieldsClass {
+		public byte f0;
+		public short f1;
+		public int f2;
+		public long f3;
+		public float f4;
+		public double f5;
+		public boolean f6;
+
+		public PrimitiveTypeFieldsClass() {
+		}
+	}
+
+	@Test
+	public void testGeneralReferenceTypeFieldsClass() throws Exception {
+		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
+		src.f0 = 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = (long) 3;
+		src.f4 = (float) 4;
+		src.f5 = (double) 5;
+		src.f6 = false;
+		src.f7 = new BigInteger("7");
+		src.f8 = "8";
+		src.f9 = new byte[] { 0x01, 0x02 };
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(GeneralReferenceTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate
+				.create(GeneralReferenceTypeFieldsClass.class);
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9[0], dst.f9[0]);
+		assertEquals(src.f9[1], dst.f9[1]);
+	}
+
+	public static class GeneralReferenceTypeFieldsClass {
+		public Byte f0;
+		public Short f1;
+		public Integer f2;
+		public Long f3;
+		public Float f4;
+		public Double f5;
+		public Boolean f6;
+		public BigInteger f7;
+		public String f8;
+		public byte[] f9;
+
+		public GeneralReferenceTypeFieldsClass() {
+		}
+	}
+
+	public void testListTypes() throws Exception {
+		SampleListTypes src = new SampleListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = new ArrayList();
+		src.f1.add(1);
+		src.f1.add(2);
+		src.f1.add(3);
+		src.f2 = new ArrayList();
+		src.f2.add("e1");
+		src.f2.add("e2");
+		src.f2.add("e3");
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(SampleListTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate.create(SampleListTypes.class);
+		SampleListTypes dst = (SampleListTypes) tmpl.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		for (int i = 0; i < src.f1.size(); ++i) {
+			assertEquals(src.f1.get(i), dst.f1.get(i));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		for (int i = 0; i < src.f2.size(); ++i) {
+			assertEquals(src.f2.get(i), dst.f2.get(i));
+		}
+	}
+
+	public static class SampleListTypes {
+		public List f0;
+		public List f1;
+		public List f2;
+
+		public SampleListTypes() {
+		}
+	}
+
+	public void testMapTypes() throws Exception {
+		SampleMapTypes src = new SampleMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = new HashMap();
+		src.f1.put(1, 1);
+		src.f1.put(2, 2);
+		src.f1.put(3, 3);
+		src.f2 = new HashMap();
+		src.f2.put("k1", 1);
+		src.f2.put("k2", 2);
+		src.f2.put("k3", 3);
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(SampleMapTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate.create(SampleMapTypes.class);
+		SampleMapTypes dst = (SampleMapTypes) tmpl.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		Iterator srcf1 = src.f1.keySet().iterator();
+		Iterator dstf1 = dst.f1.keySet().iterator();
+		while (srcf1.hasNext()) {
+			Integer s1 = srcf1.next();
+			Integer d1 = dstf1.next();
+			assertEquals(s1, d1);
+			assertEquals(src.f1.get(s1), dst.f1.get(d1));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		Iterator srcf2 = src.f2.keySet().iterator();
+		Iterator dstf2 = dst.f2.keySet().iterator();
+		while (srcf2.hasNext()) {
+			String s2 = srcf2.next();
+			String d2 = dstf2.next();
+			assertEquals(s2, d2);
+			assertEquals(src.f2.get(s2), dst.f2.get(d2));
+		}
+	}
+
+	public static class SampleMapTypes {
+		public Map f0;
+		public Map f1;
+		public Map f2;
+
+		public SampleMapTypes() {
+		}
+	}
+
+	@Test
+	public void testDefaultConstructorModifiers01() throws Exception {
+		try {
+			DynamicCodeGenPacker.create(NoDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(PrivateDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(ProtectedDefaultConstructorClass.class);
+			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(PackageDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@Test
+	public void testDefaultConstructorModifiers02() throws Exception {
+		try {
+			DynamicCodeGenUnpacker.create(NoDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenUnpacker.create(PrivateDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenUnpacker
+					.create(ProtectedDefaultConstructorClass.class);
+			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenUnpacker.create(PackageDefaultConstructorClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	public static class NoDefaultConstructorClass {
+		public NoDefaultConstructorClass(int i) {
+		}
+	}
+
+	public static class PrivateDefaultConstructorClass {
+		private PrivateDefaultConstructorClass() {
+		}
+	}
+
+	public static class ProtectedDefaultConstructorClass {
+		protected ProtectedDefaultConstructorClass() {
+		}
+	}
+
+	public static class PackageDefaultConstructorClass {
+		PackageDefaultConstructorClass() {
+		}
+	}
+
+	@Test
+	public void testClassModifiers01() throws Exception {
+		try {
+			DynamicCodeGenPacker.create(PrivateModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(ProtectedModifierClass.class);
+			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(PackageModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@Test
+	public void testClassModifiers02() throws Exception {
+		try {
+			DynamicCodeGenUnpacker.create(PrivateModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenUnpacker.create(ProtectedModifierClass.class);
+			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenUnpacker.create(PackageModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	private static class PrivateModifierClass {
+	}
+
+	protected static class ProtectedModifierClass {
+		protected ProtectedModifierClass() {
+		}
+	}
+
+	static class PackageModifierClass {
+	}
+
+	@Test
+	public void testFinalClassAndAbstractClass01() throws Exception {
+		try {
+			DynamicCodeGenPacker.create(FinalModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(AbstractModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@Test
+	public void testFinalClassAndAbstractClass02() throws Exception {
+		try {
+			DynamicCodeGenUnpacker.create(FinalModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenUnpacker.create(AbstractModifierClass.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	public final static class FinalModifierClass {
+	}
+
+	public abstract static class AbstractModifierClass {
+	}
+
+	@Test
+	public void testInterfaceAndEnumType01() throws Exception {
+		try {
+			DynamicCodeGenPacker.create(SampleInterface.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenPacker.create(SampleEnum.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	@Test
+	public void testInterfaceAndEnumType02() throws Exception {
+		try {
+			DynamicCodeGenUnpacker.create(SampleInterface.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+		try {
+			DynamicCodeGenUnpacker.create(SampleEnum.class);
+			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
+		}
+		assertTrue(true);
+	}
+
+	public interface SampleInterface {
+	}
+
+	public enum SampleEnum {
+	}
+
+	@Test
+	public void testFieldModifiers() throws Exception {
+		FieldModifiersClass src = new FieldModifiersClass();
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(FieldModifiersClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate
+				.create(FieldModifiersClass.class);
+		FieldModifiersClass dst = (FieldModifiersClass) tmpl
+				.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f4 != dst.f4);
+	}
+
+	public static class FieldModifiersClass {
+		public int f0;
+		public final int f1 = 1;
+		private int f2;
+		protected int f3;
+		int f4;
+
+		public FieldModifiersClass() {
+		}
+	}
+
+	@Test
+	public void testNestedFieldClass() throws Exception {
+		MessagePacker packer2 = DynamicCodeGenPacker.create(NestedClass.class);
+		CustomPacker.register(NestedClass.class, packer2);
+		MessagePacker packer1 = DynamicCodeGenPacker.create(BaseClass.class);
+		CustomPacker.register(BaseClass.class, packer1);
+		Template tmpl2 = DynamicCodeGenTemplate.create(NestedClass.class);
+		CustomUnpacker.register(NestedClass.class, tmpl2);
+		CustomConverter.register(NestedClass.class, tmpl2);
+		Template tmpl1 = DynamicCodeGenTemplate.create(BaseClass.class);
+		CustomUnpacker.register(BaseClass.class, tmpl1);
+		CustomConverter.register(BaseClass.class, tmpl1);
+		BaseClass src = new BaseClass();
+		NestedClass src2 = new NestedClass();
+		src.f0 = 0;
+		src2.f2 = 2;
+		src.f1 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer1.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		BaseClass dst = (BaseClass) tmpl1.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1.f2 == dst.f1.f2);
+	}
+
+	public static class BaseClass {
+		public int f0;
+		public NestedClass f1;
+
+		public BaseClass() {
+		}
+	}
+
+	public static class NestedClass {
+		public int f2;
+
+		public NestedClass() {
+		}
+	}
+
+	@Test
+	public void testExtendedClass() throws Exception {
+		SampleSubClass src = new SampleSubClass();
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f8 = 8;
+		src.f9 = 9;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(SampleSubClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate.create(SampleSubClass.class);
+		SampleSubClass dst = (SampleSubClass) tmpl.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f4 != dst.f4);
+		assertTrue(src.f5 == dst.f5);
+		assertTrue(src.f6 == dst.f6);
+		assertTrue(src.f8 == dst.f8);
+		assertTrue(src.f9 != dst.f9);
+	}
+
+	public static class SampleSubClass extends SampleSuperClass {
+		public int f0;
+		public final int f1 = 1;
+		private int f2;
+		protected int f3;
+		int f4;
+
+		public SampleSubClass() {
+		}
+	}
+
+	public static class SampleSuperClass {
+		public int f5;
+		public final int f6 = 2;
+		private int f7;
+		protected int f8;
+		int f9;
+
+		public SampleSuperClass() {
+		}
+	}
+}

From 2a0a84763479e6eebd30f685e5f64afd7e7cb3bc Mon Sep 17 00:00:00 2001
From: advect 
Date: Wed, 29 Sep 2010 08:47:06 +0900
Subject: [PATCH 0819/1648] php: update 0.3.0

---
 php/ChangeLog                 |   15 +-
 php/config.m4                 |   45 --
 php/config.w32                |    3 -
 php/msgpack.c                 |  139 ++--
 php/msgpack/pack_define.h     |   26 +
 php/msgpack/pack_template.h   |  766 +++++++++++++++++++++
 php/msgpack/sysdep.h          |  118 ++++
 php/msgpack/unpack_define.h   |   93 +++
 php/msgpack/unpack_template.h |  409 +++++++++++
 php/msgpack/version.h         |   40 ++
 php/msgpack_class.c           |  315 +++++++--
 php/msgpack_class.h           |    6 +-
 php/msgpack_pack.c            |  132 ++--
 php/msgpack_pack.h            |    7 +-
 php/msgpack_unpack.c          | 1216 +++++++++++++++------------------
 php/msgpack_unpack.h          |  123 +++-
 php/package.xml               |   85 ++-
 php/php-msgpack.spec          |    4 +-
 php/php_msgpack.h             |    3 +-
 php/tests/009.phpt            |   18 +-
 php/tests/009b.phpt           |  101 +++
 php/tests/014.phpt            |    2 +-
 php/tests/015.phpt            |    2 +-
 php/tests/015b.phpt           |    2 +-
 php/tests/018.phpt            |    2 +-
 php/tests/024.phpt            |    6 +-
 php/tests/024b.phpt           |  165 +++++
 php/tests/026.phpt            |    6 +-
 php/tests/026b.phpt           |  107 +++
 php/tests/027.phpt            |    2 +-
 php/tests/028.phpt            |    6 +-
 php/tests/028b.phpt           |  167 +++++
 php/tests/029.phpt            |   37 +-
 php/tests/030.phpt            |    3 -
 php/tests/060.phpt            |   18 +-
 php/tests/060b.phpt           |  313 +++++++++
 php/tests/061.phpt            |    8 +-
 php/tests/061b.phpt           |  318 +++++++++
 php/tests/070.phpt            |    8 +-
 php/tests/070b.phpt           |  297 ++++++++
 php/tests/071.phpt            |    8 +-
 php/tests/071b.phpt           |  299 ++++++++
 php/tests/072.phpt            |    8 +-
 php/tests/072b.phpt           |  339 +++++++++
 php/tests/073.phpt            |  346 ++++++++++
 php/tests/073b.phpt           |  340 +++++++++
 php/tests/080.phpt            |  301 ++++++++
 php/tests/081.phpt            |  303 ++++++++
 php/tests/082.phpt            |  346 ++++++++++
 php/tests/083.phpt            |  347 ++++++++++
 php/tests/084.phpt            |  301 ++++++++
 php/tests/085.phpt            |  344 ++++++++++
 php/tests/086.phpt            |  345 ++++++++++
 php/tests/087.phpt            |  302 ++++++++
 php/tests/088.phpt            |  345 ++++++++++
 php/tests/089.phpt            |  347 ++++++++++
 56 files changed, 8804 insertions(+), 950 deletions(-)
 create mode 100644 php/msgpack/pack_define.h
 create mode 100644 php/msgpack/pack_template.h
 create mode 100644 php/msgpack/sysdep.h
 create mode 100644 php/msgpack/unpack_define.h
 create mode 100644 php/msgpack/unpack_template.h
 create mode 100644 php/msgpack/version.h
 create mode 100644 php/tests/009b.phpt
 create mode 100644 php/tests/024b.phpt
 create mode 100644 php/tests/026b.phpt
 create mode 100644 php/tests/028b.phpt
 create mode 100644 php/tests/060b.phpt
 create mode 100644 php/tests/061b.phpt
 create mode 100644 php/tests/070b.phpt
 create mode 100644 php/tests/071b.phpt
 create mode 100644 php/tests/072b.phpt
 create mode 100644 php/tests/073.phpt
 create mode 100644 php/tests/073b.phpt
 create mode 100644 php/tests/080.phpt
 create mode 100644 php/tests/081.phpt
 create mode 100644 php/tests/082.phpt
 create mode 100644 php/tests/083.phpt
 create mode 100644 php/tests/084.phpt
 create mode 100644 php/tests/085.phpt
 create mode 100644 php/tests/086.phpt
 create mode 100644 php/tests/087.phpt
 create mode 100644 php/tests/088.phpt
 create mode 100644 php/tests/089.phpt

diff --git a/php/ChangeLog b/php/ChangeLog
index 5481c1f..3cee3b6 100644
--- a/php/ChangeLog
+++ b/php/ChangeLog
@@ -1,14 +1,21 @@
 msgpack extension changelog
 
+Version 0.3.0
+-------------
+	* Change msgpack_unpack.c (used template)
+	* Add php_only ini option (true / false)
+	* Change class MessagePack and MessagePackUnpacker __construct option.
+	* Add class MessagePack and MessagePackUnpacker setOption method.
+
 Version 0.2.1
 -------------
-	* Fix stream deserializer
+	* Fix stream deserializer.
 
 Version 0.2.0
 -------------
-	* Add stream deserializer / class MessagePackUnpacker interface
-	* Add alias functions
-	* Add class MessagePack interface
+	* Add stream deserializer / class MessagePackUnpacker interface.
+	* Add alias functions.
+	* Add class MessagePack interface.
 
 Version 0.1.5
 -------------
diff --git a/php/config.m4 b/php/config.m4
index d319684..a78e1f3 100644
--- a/php/config.m4
+++ b/php/config.m4
@@ -21,52 +21,7 @@ PHP_ARG_WITH(msgpack, for msgpack support,
 Make sure that the comment is aligned:
 [  --with-msgpack             Include msgpack support])
 
-dnl PHP_ARG_ENABLE(msgpack, whether to enable msgpack support,
-dnl Make sure that the comment is aligned:
-dnl [  --enable-msgpack           Enable msgpack support])
-
 if test "$PHP_MSGPACK" != "no"; then
-  dnl Write more examples of tests here...
-
-  dnl --with-msgpack -> check with-path
-  SEARCH_PATH="/usr/local /usr"     # you might want to change this
-  SEARCH_FOR="/include/msgpack.h"  # you most likely want to change this
-  if test -r $PHP_MSGPACK/$SEARCH_FOR; then # path given as parameter
-    MSGPACK_DIR=$PHP_MSGPACK
-  else # search default path list
-    AC_MSG_CHECKING([for msgpack files in default path])
-    for i in $SEARCH_PATH ; do
-      if test -r $i/$SEARCH_FOR; then
-        MSGPACK_DIR=$i
-        AC_MSG_RESULT(found in $i)
-      fi
-    done
-  fi
-
-  if test -z "$MSGPACK_DIR"; then
-    AC_MSG_RESULT([not found])
-    AC_MSG_ERROR([Please reinstall the msgpack distribution])
-  fi
-
-  dnl --with-msgpack -> add include path
-  PHP_ADD_INCLUDE($MSGPACK_DIR/include)
-
-  dnl --with-msgpack -> check for lib and symbol presence
-  LIBNAME=msgpack # you may want to change this
-  LIBSYMBOL=msgpack_pack_object # you most likely want to change this 
-
-  PHP_CHECK_LIBRARY($LIBNAME,$LIBSYMBOL,
-  [
-    PHP_ADD_LIBRARY_WITH_PATH($LIBNAME, $MSGPACK_DIR/lib, MSGPACK_SHARED_LIBADD)
-    AC_DEFINE(HAVE_MSGPACKLIB,1,[ ])
-  ],[
-    AC_MSG_ERROR([wrong msgpack lib version or lib not found])
-  ],[
-    -L$MSGPACK_DIR/lib -lm
-  ])
-
-  PHP_SUBST(MSGPACK_SHARED_LIBADD)
-
   PHP_NEW_EXTENSION(msgpack, msgpack.c msgpack_pack.c msgpack_unpack.c msgpack_class.c, $ext_shared)
 
   PHP_INSTALL_HEADERS([ext/msgpack], [php_msgpack.h])
diff --git a/php/config.w32 b/php/config.w32
index f2153e6..726b75f 100644
--- a/php/config.w32
+++ b/php/config.w32
@@ -4,9 +4,6 @@
 // If your extension references something external, use ARG_WITH
 // ARG_WITH("msgpack", "for msgpack support", "no");
 
-// Otherwise, use ARG_ENABLE
-// ARG_ENABLE("msgpack", "enable msgpack support", "no");
-
 if (PHP_MSGPACK != "no") {
    EXTENSION("msgpack", "msgpack.c msgpack_pack.c msgpack_unpack.c msgpack_class.c");
 }
diff --git a/php/msgpack.c b/php/msgpack.c
index 62fb68d..5d4f926 100644
--- a/php/msgpack.c
+++ b/php/msgpack.c
@@ -15,6 +15,7 @@
 #include "msgpack_pack.h"
 #include "msgpack_unpack.h"
 #include "msgpack_class.h"
+#include "msgpack/version.h"
 
 static ZEND_FUNCTION(msgpack_serialize);
 static ZEND_FUNCTION(msgpack_unserialize);
@@ -27,20 +28,46 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unserialize, 0, 0, 1)
     ZEND_ARG_INFO(0, str)
 ZEND_END_ARG_INFO()
 
+PHP_INI_BEGIN()
+STD_PHP_INI_BOOLEAN(
+    "msgpack.error_display", "1", PHP_INI_ALL, OnUpdateBool,
+    error_display, zend_msgpack_globals, msgpack_globals)
+STD_PHP_INI_BOOLEAN(
+    "msgpack.php_only", "1", PHP_INI_ALL, OnUpdateBool,
+    php_only, zend_msgpack_globals, msgpack_globals)
+PHP_INI_END()
+
 PS_SERIALIZER_FUNCS(msgpack);
 
 static const zend_function_entry msgpack_functions[] = {
     ZEND_FE(msgpack_serialize, arginfo_msgpack_serialize)
     ZEND_FE(msgpack_unserialize, arginfo_msgpack_unserialize)
     ZEND_FALIAS(msgpack_pack, msgpack_serialize, arginfo_msgpack_serialize)
-    ZEND_FALIAS(msgpack_unpack, msgpack_unserialize,
-                arginfo_msgpack_unserialize)
+    ZEND_FALIAS(msgpack_unpack, msgpack_unserialize, arginfo_msgpack_unserialize)
     {NULL, NULL, NULL}
 };
 
+static void msgpack_init_globals(zend_msgpack_globals *msgpack_globals)
+{
+    TSRMLS_FETCH();
+
+    if (PG(display_errors))
+    {
+        msgpack_globals->error_display = 1;
+    }
+    else
+    {
+        msgpack_globals->error_display = 0;
+    }
+
+    msgpack_globals->php_only = 1;
+}
+
 static ZEND_MINIT_FUNCTION(msgpack)
 {
-    MSGPACK_G(error_display) = 1;
+    ZEND_INIT_MODULE_GLOBALS(msgpack, msgpack_init_globals, NULL);
+
+    REGISTER_INI_ENTRIES();
 
 #if HAVE_PHP_SESSION
     php_session_register_serializer("msgpack",
@@ -48,7 +75,14 @@ static ZEND_MINIT_FUNCTION(msgpack)
                                     PS_SERIALIZER_DECODE_NAME(msgpack));
 #endif
 
-    msgpack_init_class(TSRMLS_CC);
+    msgpack_init_class();
+
+    return SUCCESS;
+}
+
+static ZEND_MSHUTDOWN_FUNCTION(msgpack)
+{
+    UNREGISTER_INI_ENTRIES();
 
     return SUCCESS;
 }
@@ -56,12 +90,15 @@ static ZEND_MINIT_FUNCTION(msgpack)
 static ZEND_MINFO_FUNCTION(msgpack)
 {
     php_info_print_table_start();
-    php_info_print_table_row(2, "msgpack support", "enabled");
-    php_info_print_table_row(2, "msgpack version", MSGPACK_VERSION);
+    php_info_print_table_row(2, "MessagePack Support", "enabled");
 #if HAVE_PHP_SESSION
-    php_info_print_table_row(2, "msgpack Session Support", "enabled" );
+    php_info_print_table_row(2, "Session Support", "enabled" );
 #endif
+    php_info_print_table_row(2, "extension Version", MSGPACK_EXTENSION_VERSION);
+    php_info_print_table_row(2, "header Version", MSGPACK_VERSION);
     php_info_print_table_end();
+
+    DISPLAY_INI_ENTRIES();
 }
 
 zend_module_entry msgpack_module_entry = {
@@ -71,7 +108,7 @@ zend_module_entry msgpack_module_entry = {
     "msgpack",
     msgpack_functions,
     ZEND_MINIT(msgpack),
-    NULL,
+    ZEND_MSHUTDOWN(msgpack),
     NULL,
     NULL,
     ZEND_MINFO(msgpack),
@@ -109,7 +146,6 @@ PS_SERIALIZER_ENCODE_FUNC(msgpack)
 
 PS_SERIALIZER_DECODE_FUNC(msgpack)
 {
-    php_unserialize_data_t var_hash;
     int ret;
     HashTable *tmp_hash;
     HashPosition tmp_hash_pos;
@@ -117,38 +153,29 @@ PS_SERIALIZER_DECODE_FUNC(msgpack)
     ulong key_long;
     uint key_len;
     zval *tmp;
-    zval **data;
-    msgpack_unserialize_data mpsd;
+    zval **value;
+    size_t off = 0;
+    msgpack_unpack_t mp;
+    php_unserialize_data_t var_hash;
 
-    PHP_VAR_UNSERIALIZE_INIT(var_hash);
+    ALLOC_INIT_ZVAL(tmp);
 
-    MAKE_STD_ZVAL(tmp);
+    template_init(&mp);
 
-    mpsd.data = (unsigned char *)val;;
-    mpsd.length = vallen;
-    mpsd.offset = 0;
+    msgpack_unserialize_var_init(&var_hash);
 
-    ret = msgpack_unserialize_zval(&tmp, &mpsd, &var_hash TSRMLS_CC);
+    (&mp)->user.retval = (zval *)tmp;
+    (&mp)->user.var_hash = (php_unserialize_data_t *)&var_hash;
 
-    switch (ret)
-    {
-        case MSGPACK_UNPACK_EXTRA_BYTES:
-        case MSGPACK_UNPACK_SUCCESS:
-            break;
-        case MSGPACK_UNPACK_PARSE_ERROR:
-        case MSGPACK_UNPACK_CONTINUE:
-        default:
-            zval_ptr_dtor(&tmp);
-            return FAILURE;
-    }
+    ret = template_execute(&mp, (char *)val, (size_t)vallen, &off);
 
-    PHP_VAR_UNSERIALIZE_DESTROY(var_hash);
+    msgpack_unserialize_var_destroy(&var_hash);
 
     tmp_hash = HASH_OF(tmp);
 
     zend_hash_internal_pointer_reset_ex(tmp_hash, &tmp_hash_pos);
     while (zend_hash_get_current_data_ex(
-               tmp_hash, (void *)&data, &tmp_hash_pos) == SUCCESS)
+               tmp_hash, (void *)&value, &tmp_hash_pos) == SUCCESS)
     {
         ret = zend_hash_get_current_key_ex(
             tmp_hash, &key_str, &key_len, &key_long, 0, &tmp_hash_pos);
@@ -158,7 +185,8 @@ PS_SERIALIZER_DECODE_FUNC(msgpack)
                 /* ??? */
                 break;
             case HASH_KEY_IS_STRING:
-                php_set_session_var(key_str, key_len - 1, *data, NULL TSRMLS_CC);
+                php_set_session_var(
+                    key_str, key_len - 1, *value, NULL TSRMLS_CC);
                 php_add_session_var(key_str, key_len - 1 TSRMLS_CC);
                 break;
         }
@@ -185,46 +213,59 @@ PHP_MSGPACK_API void php_msgpack_unserialize(
     zval *return_value, char *str, size_t str_len TSRMLS_DC)
 {
     int ret;
+    size_t off = 0;
+    msgpack_unpack_t mp;
     php_unserialize_data_t var_hash;
-    msgpack_unserialize_data mpsd;
 
     if (str_len <= 0)
     {
         RETURN_NULL();
     }
 
-    PHP_VAR_UNSERIALIZE_INIT(var_hash);
+    template_init(&mp);
 
-    mpsd.data = (unsigned char *)str;
-    mpsd.length = str_len;
-    mpsd.offset = 0;
+    msgpack_unserialize_var_init(&var_hash);
 
-    ret = msgpack_unserialize_zval(&return_value, &mpsd, &var_hash TSRMLS_CC);
+    (&mp)->user.retval = (zval *)return_value;
+    (&mp)->user.var_hash = (php_unserialize_data_t *)&var_hash;
+
+    ret = template_execute(&mp, str, (size_t)str_len, &off);
+
+    msgpack_unserialize_var_destroy(&var_hash);
 
     switch (ret)
     {
         case MSGPACK_UNPACK_PARSE_ERROR:
-            zend_error(E_WARNING,
-                       "[msgpack] (php_msgpack_unserialize) Parse error");
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (php_msgpack_unserialize) Parse error");
+            }
             break;
         case MSGPACK_UNPACK_CONTINUE:
-            zend_error(E_WARNING,
-                       "[msgpack] (php_msgpack_unserialize) "
-                       "Insufficient data for unserializeng");
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (php_msgpack_unserialize) "
+                           "Insufficient data for unserializing");
+            }
             break;
         case MSGPACK_UNPACK_EXTRA_BYTES:
-            zend_error(E_WARNING,
-                       "[msgpack] (php_msgpack_unserialize) Extra bytes");
-            break;
         case MSGPACK_UNPACK_SUCCESS:
+            if (off < (size_t)str_len && MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (php_msgpack_unserialize) Extra bytes");
+            }
             break;
         default:
-            zend_error(E_WARNING,
-                       "[msgpack] (php_msgpack_unserialize) Unknown result");
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (php_msgpack_unserialize) Unknown result");
+            }
             break;
     }
-
-    PHP_VAR_UNSERIALIZE_DESTROY(var_hash);
 }
 
 static ZEND_FUNCTION(msgpack_serialize)
diff --git a/php/msgpack/pack_define.h b/php/msgpack/pack_define.h
new file mode 100644
index 0000000..4845d52
--- /dev/null
+++ b/php/msgpack/pack_define.h
@@ -0,0 +1,26 @@
+/*
+ * MessagePack unpacking routine template
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_PACK_DEFINE_H__
+#define MSGPACK_PACK_DEFINE_H__
+
+#include "msgpack/sysdep.h"
+#include 
+#include 
+
+#endif /* msgpack/pack_define.h */
+
diff --git a/php/msgpack/pack_template.h b/php/msgpack/pack_template.h
new file mode 100644
index 0000000..b636967
--- /dev/null
+++ b/php/msgpack/pack_template.h
@@ -0,0 +1,766 @@
+/*
+ * MessagePack packing routine template
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#if defined(__LITTLE_ENDIAN__)
+#define TAKE8_8(d)  ((uint8_t*)&d)[0]
+#define TAKE8_16(d) ((uint8_t*)&d)[0]
+#define TAKE8_32(d) ((uint8_t*)&d)[0]
+#define TAKE8_64(d) ((uint8_t*)&d)[0]
+#elif defined(__BIG_ENDIAN__)
+#define TAKE8_8(d)  ((uint8_t*)&d)[0]
+#define TAKE8_16(d) ((uint8_t*)&d)[1]
+#define TAKE8_32(d) ((uint8_t*)&d)[3]
+#define TAKE8_64(d) ((uint8_t*)&d)[7]
+#endif
+
+#ifndef msgpack_pack_inline_func
+#error msgpack_pack_inline_func template is not defined
+#endif
+
+#ifndef msgpack_pack_user
+#error msgpack_pack_user type is not defined
+#endif
+
+#ifndef msgpack_pack_append_buffer
+#error msgpack_pack_append_buffer callback is not defined
+#endif
+
+
+/*
+ * Integer
+ */
+
+#define msgpack_pack_real_uint8(x, d) \
+do { \
+	if(d < (1<<7)) { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
+	} else { \
+		/* unsigned 8 */ \
+		unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \
+		msgpack_pack_append_buffer(x, buf, 2); \
+	} \
+} while(0)
+
+#define msgpack_pack_real_uint16(x, d) \
+do { \
+	if(d < (1<<7)) { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
+	} else if(d < (1<<8)) { \
+		/* unsigned 8 */ \
+		unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
+		msgpack_pack_append_buffer(x, buf, 2); \
+	} else { \
+		/* unsigned 16 */ \
+		unsigned char buf[3]; \
+		buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+		msgpack_pack_append_buffer(x, buf, 3); \
+	} \
+} while(0)
+
+#define msgpack_pack_real_uint32(x, d) \
+do { \
+	if(d < (1<<8)) { \
+		if(d < (1<<7)) { \
+			/* fixnum */ \
+			msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
+		} else { \
+			/* unsigned 8 */ \
+			unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} \
+	} else { \
+		if(d < (1<<16)) { \
+			/* unsigned 16 */ \
+			unsigned char buf[3]; \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} else { \
+			/* unsigned 32 */ \
+			unsigned char buf[5]; \
+			buf[0] = 0xce; _msgpack_store32(&buf[1], d); \
+			msgpack_pack_append_buffer(x, buf, 5); \
+		} \
+	} \
+} while(0)
+
+#define msgpack_pack_real_uint64(x, d) \
+do { \
+	if(d < (1ULL<<8)) { \
+		if(d < (1<<7)) { \
+			/* fixnum */ \
+			msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
+		} else { \
+			/* unsigned 8 */ \
+			unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} \
+	} else { \
+		if(d < (1ULL<<16)) { \
+			/* unsigned 16 */ \
+			unsigned char buf[3]; \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} else if(d < (1ULL<<32)) { \
+			/* unsigned 32 */ \
+			unsigned char buf[5]; \
+			buf[0] = 0xce; _msgpack_store32(&buf[1], d); \
+			msgpack_pack_append_buffer(x, buf, 5); \
+		} else { \
+			/* unsigned 64 */ \
+			unsigned char buf[9]; \
+			buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
+			msgpack_pack_append_buffer(x, buf, 9); \
+		} \
+	} \
+} while(0)
+
+#define msgpack_pack_real_int8(x, d) \
+do { \
+	if(d < -(1<<5)) { \
+		/* signed 8 */ \
+		unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \
+		msgpack_pack_append_buffer(x, buf, 2); \
+	} else { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
+	} \
+} while(0)
+
+#define msgpack_pack_real_int16(x, d) \
+do { \
+	if(d < -(1<<5)) { \
+		if(d < -(1<<7)) { \
+			/* signed 16 */ \
+			unsigned char buf[3]; \
+			buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} else { \
+			/* signed 8 */ \
+			unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} \
+	} else if(d < (1<<7)) { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
+	} else { \
+		if(d < (1<<8)) { \
+			/* unsigned 8 */ \
+			unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} else { \
+			/* unsigned 16 */ \
+			unsigned char buf[3]; \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} \
+	} \
+} while(0)
+
+#define msgpack_pack_real_int32(x, d) \
+do { \
+	if(d < -(1<<5)) { \
+		if(d < -(1<<15)) { \
+			/* signed 32 */ \
+			unsigned char buf[5]; \
+			buf[0] = 0xd2; _msgpack_store32(&buf[1], d); \
+			msgpack_pack_append_buffer(x, buf, 5); \
+		} else if(d < -(1<<7)) { \
+			/* signed 16 */ \
+			unsigned char buf[3]; \
+			buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} else { \
+			/* signed 8 */ \
+			unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} \
+	} else if(d < (1<<7)) { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
+	} else { \
+		if(d < (1<<8)) { \
+			/* unsigned 8 */ \
+			unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
+			msgpack_pack_append_buffer(x, buf, 2); \
+		} else if(d < (1<<16)) { \
+			/* unsigned 16 */ \
+			unsigned char buf[3]; \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+			msgpack_pack_append_buffer(x, buf, 3); \
+		} else { \
+			/* unsigned 32 */ \
+			unsigned char buf[5]; \
+			buf[0] = 0xce; _msgpack_store32(&buf[1], d); \
+			msgpack_pack_append_buffer(x, buf, 5); \
+		} \
+	} \
+} while(0)
+
+#define msgpack_pack_real_int64(x, d) \
+do { \
+	if(d < -(1LL<<5)) { \
+		if(d < -(1LL<<15)) { \
+			if(d < -(1LL<<31)) { \
+				/* signed 64 */ \
+				unsigned char buf[9]; \
+				buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \
+				msgpack_pack_append_buffer(x, buf, 9); \
+			} else { \
+				/* signed 32 */ \
+				unsigned char buf[5]; \
+				buf[0] = 0xd2; _msgpack_store32(&buf[1], d); \
+				msgpack_pack_append_buffer(x, buf, 5); \
+			} \
+		} else { \
+			if(d < -(1<<7)) { \
+				/* signed 16 */ \
+				unsigned char buf[3]; \
+				buf[0] = 0xd1; _msgpack_store16(&buf[1], d); \
+				msgpack_pack_append_buffer(x, buf, 3); \
+			} else { \
+				/* signed 8 */ \
+				unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \
+				msgpack_pack_append_buffer(x, buf, 2); \
+			} \
+		} \
+	} else if(d < (1<<7)) { \
+		/* fixnum */ \
+		msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
+	} else { \
+		if(d < (1LL<<16)) { \
+			if(d < (1<<8)) { \
+				/* unsigned 8 */ \
+				unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
+				msgpack_pack_append_buffer(x, buf, 2); \
+			} else { \
+				/* unsigned 16 */ \
+				unsigned char buf[3]; \
+				buf[0] = 0xcd; _msgpack_store16(&buf[1], d); \
+				msgpack_pack_append_buffer(x, buf, 3); \
+			} \
+		} else { \
+			if(d < (1LL<<32)) { \
+				/* unsigned 32 */ \
+				unsigned char buf[5]; \
+				buf[0] = 0xce; _msgpack_store32(&buf[1], d); \
+				msgpack_pack_append_buffer(x, buf, 5); \
+			} else { \
+				/* unsigned 64 */ \
+				unsigned char buf[9]; \
+				buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
+				msgpack_pack_append_buffer(x, buf, 9); \
+			} \
+		} \
+	} \
+} while(0)
+
+
+#ifdef msgpack_pack_inline_func_fixint
+
+msgpack_pack_inline_func_fixint(_uint8)(msgpack_pack_user x, uint8_t d)
+{
+	unsigned char buf[2] = {0xcc, TAKE8_8(d)};
+	msgpack_pack_append_buffer(x, buf, 2);
+}
+
+msgpack_pack_inline_func_fixint(_uint16)(msgpack_pack_user x, uint16_t d)
+{
+	unsigned char buf[3];
+	buf[0] = 0xcd; _msgpack_store16(&buf[1], d);
+	msgpack_pack_append_buffer(x, buf, 3);
+}
+
+msgpack_pack_inline_func_fixint(_uint32)(msgpack_pack_user x, uint32_t d)
+{
+	unsigned char buf[5];
+	buf[0] = 0xce; _msgpack_store32(&buf[1], d);
+	msgpack_pack_append_buffer(x, buf, 5);
+}
+
+msgpack_pack_inline_func_fixint(_uint64)(msgpack_pack_user x, uint64_t d)
+{
+	unsigned char buf[9];
+	buf[0] = 0xcf; _msgpack_store64(&buf[1], d);
+	msgpack_pack_append_buffer(x, buf, 9);
+}
+
+msgpack_pack_inline_func_fixint(_int8)(msgpack_pack_user x, int8_t d)
+{
+	unsigned char buf[2] = {0xd0, TAKE8_8(d)};
+	msgpack_pack_append_buffer(x, buf, 2);
+}
+
+msgpack_pack_inline_func_fixint(_int16)(msgpack_pack_user x, int16_t d)
+{
+	unsigned char buf[3];
+	buf[0] = 0xd1; _msgpack_store16(&buf[1], d);
+	msgpack_pack_append_buffer(x, buf, 3);
+}
+
+msgpack_pack_inline_func_fixint(_int32)(msgpack_pack_user x, int32_t d)
+{
+	unsigned char buf[5];
+	buf[0] = 0xd2; _msgpack_store32(&buf[1], d);
+	msgpack_pack_append_buffer(x, buf, 5);
+}
+
+msgpack_pack_inline_func_fixint(_int64)(msgpack_pack_user x, int64_t d)
+{
+	unsigned char buf[9];
+	buf[0] = 0xd3; _msgpack_store64(&buf[1], d);
+	msgpack_pack_append_buffer(x, buf, 9);
+}
+
+#undef msgpack_pack_inline_func_fixint
+#endif
+
+
+msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d)
+{
+	msgpack_pack_real_uint8(x, d);
+}
+
+msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d)
+{
+	msgpack_pack_real_uint16(x, d);
+}
+
+msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d)
+{
+	msgpack_pack_real_uint32(x, d);
+}
+
+msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d)
+{
+	msgpack_pack_real_uint64(x, d);
+}
+
+msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d)
+{
+	msgpack_pack_real_int8(x, d);
+}
+
+msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d)
+{
+	msgpack_pack_real_int16(x, d);
+}
+
+msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d)
+{
+	msgpack_pack_real_int32(x, d);
+}
+
+msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d)
+{
+	msgpack_pack_real_int64(x, d);
+}
+
+
+#ifdef msgpack_pack_inline_func_cint
+
+msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d)
+{
+#if defined(SIZEOF_SHORT)
+#if SIZEOF_SHORT == 2
+	msgpack_pack_real_int16(x, d);
+#elif SIZEOF_SHORT == 4
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(SHRT_MAX)
+#if SHRT_MAX == 0x7fff
+	msgpack_pack_real_int16(x, d);
+#elif SHRT_MAX == 0x7fffffff
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(short) == 2) {
+	msgpack_pack_real_int16(x, d);
+} else if(sizeof(short) == 4) {
+	msgpack_pack_real_int32(x, d);
+} else {
+	msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d)
+{
+#if defined(SIZEOF_INT)
+#if SIZEOF_INT == 2
+	msgpack_pack_real_int16(x, d);
+#elif SIZEOF_INT == 4
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(INT_MAX)
+#if INT_MAX == 0x7fff
+	msgpack_pack_real_int16(x, d);
+#elif INT_MAX == 0x7fffffff
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(int) == 2) {
+	msgpack_pack_real_int16(x, d);
+} else if(sizeof(int) == 4) {
+	msgpack_pack_real_int32(x, d);
+} else {
+	msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d)
+{
+#if defined(SIZEOF_LONG)
+#if SIZEOF_LONG == 2
+	msgpack_pack_real_int16(x, d);
+#elif SIZEOF_LONG == 4
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(LONG_MAX)
+#if LONG_MAX == 0x7fffL
+	msgpack_pack_real_int16(x, d);
+#elif LONG_MAX == 0x7fffffffL
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(long) == 2) {
+	msgpack_pack_real_int16(x, d);
+} else if(sizeof(long) == 4) {
+	msgpack_pack_real_int32(x, d);
+} else {
+	msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d)
+{
+#if defined(SIZEOF_LONG_LONG)
+#if SIZEOF_LONG_LONG == 2
+	msgpack_pack_real_int16(x, d);
+#elif SIZEOF_LONG_LONG == 4
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
+#elif defined(LLONG_MAX)
+#if LLONG_MAX == 0x7fffL
+	msgpack_pack_real_int16(x, d);
+#elif LLONG_MAX == 0x7fffffffL
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
+#else
+if(sizeof(long long) == 2) {
+	msgpack_pack_real_int16(x, d);
+} else if(sizeof(long long) == 4) {
+	msgpack_pack_real_int32(x, d);
+} else {
+	msgpack_pack_real_int64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d)
+{
+#if defined(SIZEOF_SHORT)
+#if SIZEOF_SHORT == 2
+	msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_SHORT == 4
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(USHRT_MAX)
+#if USHRT_MAX == 0xffffU
+	msgpack_pack_real_uint16(x, d);
+#elif USHRT_MAX == 0xffffffffU
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned short) == 2) {
+	msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned short) == 4) {
+	msgpack_pack_real_uint32(x, d);
+} else {
+	msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d)
+{
+#if defined(SIZEOF_INT)
+#if SIZEOF_INT == 2
+	msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_INT == 4
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(UINT_MAX)
+#if UINT_MAX == 0xffffU
+	msgpack_pack_real_uint16(x, d);
+#elif UINT_MAX == 0xffffffffU
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned int) == 2) {
+	msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned int) == 4) {
+	msgpack_pack_real_uint32(x, d);
+} else {
+	msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d)
+{
+#if defined(SIZEOF_LONG)
+#if SIZEOF_LONG == 2
+	msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_LONG == 4
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(ULONG_MAX)
+#if ULONG_MAX == 0xffffUL
+	msgpack_pack_real_uint16(x, d);
+#elif ULONG_MAX == 0xffffffffUL
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned long) == 2) {
+	msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned long) == 4) {
+	msgpack_pack_real_uint32(x, d);
+} else {
+	msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d)
+{
+#if defined(SIZEOF_LONG_LONG)
+#if SIZEOF_LONG_LONG == 2
+	msgpack_pack_real_uint16(x, d);
+#elif SIZEOF_LONG_LONG == 4
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
+#elif defined(ULLONG_MAX)
+#if ULLONG_MAX == 0xffffUL
+	msgpack_pack_real_uint16(x, d);
+#elif ULLONG_MAX == 0xffffffffUL
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned long long) == 2) {
+	msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned long long) == 4) {
+	msgpack_pack_real_uint32(x, d);
+} else {
+	msgpack_pack_real_uint64(x, d);
+}
+#endif
+}
+
+#undef msgpack_pack_inline_func_cint
+#endif
+
+
+
+/*
+ * Float
+ */
+
+msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d)
+{
+	union { float f; uint32_t i; } mem;
+	mem.f = d;
+	unsigned char buf[5];
+	buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i);
+	msgpack_pack_append_buffer(x, buf, 5);
+}
+
+msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d)
+{
+	union { double f; uint64_t i; } mem;
+	mem.f = d;
+	unsigned char buf[9];
+	buf[0] = 0xcb; _msgpack_store64(&buf[1], mem.i);
+	msgpack_pack_append_buffer(x, buf, 9);
+}
+
+
+/*
+ * Nil
+ */
+
+msgpack_pack_inline_func(_nil)(msgpack_pack_user x)
+{
+	static const unsigned char d = 0xc0;
+	msgpack_pack_append_buffer(x, &d, 1);
+}
+
+
+/*
+ * Boolean
+ */
+
+msgpack_pack_inline_func(_true)(msgpack_pack_user x)
+{
+	static const unsigned char d = 0xc3;
+	msgpack_pack_append_buffer(x, &d, 1);
+}
+
+msgpack_pack_inline_func(_false)(msgpack_pack_user x)
+{
+	static const unsigned char d = 0xc2;
+	msgpack_pack_append_buffer(x, &d, 1);
+}
+
+
+/*
+ * Array
+ */
+
+msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n)
+{
+	if(n < 16) {
+		unsigned char d = 0x90 | n;
+		msgpack_pack_append_buffer(x, &d, 1);
+	} else if(n < 65536) {
+		unsigned char buf[3];
+		buf[0] = 0xdc; _msgpack_store16(&buf[1], n);
+		msgpack_pack_append_buffer(x, buf, 3);
+	} else {
+		unsigned char buf[5];
+		buf[0] = 0xdd; _msgpack_store32(&buf[1], n);
+		msgpack_pack_append_buffer(x, buf, 5);
+	}
+}
+
+
+/*
+ * Map
+ */
+
+msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n)
+{
+	if(n < 16) {
+		unsigned char d = 0x80 | n;
+		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
+	} else if(n < 65536) {
+		unsigned char buf[3];
+		buf[0] = 0xde; _msgpack_store16(&buf[1], n);
+		msgpack_pack_append_buffer(x, buf, 3);
+	} else {
+		unsigned char buf[5];
+		buf[0] = 0xdf; _msgpack_store32(&buf[1], n);
+		msgpack_pack_append_buffer(x, buf, 5);
+	}
+}
+
+
+/*
+ * Raw
+ */
+
+msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l)
+{
+	if(l < 32) {
+		unsigned char d = 0xa0 | l;
+		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
+	} else if(l < 65536) {
+		unsigned char buf[3];
+		buf[0] = 0xda; _msgpack_store16(&buf[1], l);
+		msgpack_pack_append_buffer(x, buf, 3);
+	} else {
+		unsigned char buf[5];
+		buf[0] = 0xdb; _msgpack_store32(&buf[1], l);
+		msgpack_pack_append_buffer(x, buf, 5);
+	}
+}
+
+msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l)
+{
+	msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
+}
+
+#undef msgpack_pack_inline_func
+#undef msgpack_pack_user
+#undef msgpack_pack_append_buffer
+
+#undef TAKE8_8
+#undef TAKE8_16
+#undef TAKE8_32
+#undef TAKE8_64
+
+#undef msgpack_pack_real_uint8
+#undef msgpack_pack_real_uint16
+#undef msgpack_pack_real_uint32
+#undef msgpack_pack_real_uint64
+#undef msgpack_pack_real_int8
+#undef msgpack_pack_real_int16
+#undef msgpack_pack_real_int32
+#undef msgpack_pack_real_int64
+
diff --git a/php/msgpack/sysdep.h b/php/msgpack/sysdep.h
new file mode 100644
index 0000000..2bc01c9
--- /dev/null
+++ b/php/msgpack/sysdep.h
@@ -0,0 +1,118 @@
+/*
+ * MessagePack system dependencies
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_SYSDEP_H__
+#define MSGPACK_SYSDEP_H__
+
+
+#ifdef _MSC_VER
+typedef __int8 int8_t;
+typedef unsigned __int8 uint8_t;
+typedef __int16 int16_t;
+typedef unsigned __int16 uint16_t;
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
+typedef __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+#else
+#include 
+#include 
+#include 
+#endif
+
+
+#ifdef _WIN32
+typedef long _msgpack_atomic_counter_t;
+#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr)
+#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr)
+#else
+typedef unsigned int _msgpack_atomic_counter_t;
+#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1)
+#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1)
+#endif
+
+
+#ifdef _WIN32
+#include 
+
+#ifdef __cplusplus
+/* numeric_limits::min,max */
+#ifdef max
+#undef max
+#endif
+#ifdef min
+#undef min
+#endif
+#endif
+
+#else
+#include   /* __BYTE_ORDER */
+#endif
+
+#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+#define __LITTLE_ENDIAN__
+#elif __BYTE_ORDER == __BIG_ENDIAN
+#define __BIG_ENDIAN__
+#endif
+#endif
+
+#ifdef __LITTLE_ENDIAN__
+
+#define _msgpack_be16(x) ntohs(x)
+#define _msgpack_be32(x) ntohl(x)
+
+#if defined(_byteswap_uint64)
+#  define _msgpack_be64(x) (_byteswap_uint64(x))
+#elif defined(bswap_64)
+#  define _msgpack_be64(x) bswap_64(x)
+#elif defined(__DARWIN_OSSwapInt64)
+#  define _msgpack_be64(x) __DARWIN_OSSwapInt64(x)
+#else
+#define _msgpack_be64(x) \
+	( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \
+	  ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \
+	  ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \
+	  ((((uint64_t)x) <<  8) & 0x000000ff00000000ULL ) | \
+	  ((((uint64_t)x) >>  8) & 0x00000000ff000000ULL ) | \
+	  ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \
+	  ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \
+	  ((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) )
+#endif
+
+#else
+#define _msgpack_be16(x) (x)
+#define _msgpack_be32(x) (x)
+#define _msgpack_be64(x) (x)
+#endif
+
+
+#define _msgpack_store16(to, num) \
+	do { uint16_t val = _msgpack_be16(num); memcpy(to, &val, 2); } while(0);
+#define _msgpack_store32(to, num) \
+	do { uint32_t val = _msgpack_be32(num); memcpy(to, &val, 4); } while(0);
+#define _msgpack_store64(to, num) \
+	do { uint64_t val = _msgpack_be64(num); memcpy(to, &val, 8); } while(0);
+
+
+#define _msgpack_load16(cast, from) ((cast)_msgpack_be16(*(uint16_t*)from))
+#define _msgpack_load32(cast, from) ((cast)_msgpack_be32(*(uint32_t*)from))
+#define _msgpack_load64(cast, from) ((cast)_msgpack_be64(*(uint64_t*)from))
+
+
+#endif /* msgpack/sysdep.h */
+
diff --git a/php/msgpack/unpack_define.h b/php/msgpack/unpack_define.h
new file mode 100644
index 0000000..959d351
--- /dev/null
+++ b/php/msgpack/unpack_define.h
@@ -0,0 +1,93 @@
+/*
+ * MessagePack unpacking routine template
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_UNPACK_DEFINE_H__
+#define MSGPACK_UNPACK_DEFINE_H__
+
+#include "msgpack/sysdep.h"
+#include 
+#include 
+#include 
+#include 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#ifndef MSGPACK_EMBED_STACK_SIZE
+#define MSGPACK_EMBED_STACK_SIZE 32
+#endif
+
+
+typedef enum {
+	CS_HEADER            = 0x00,  // nil
+
+	//CS_                = 0x01,
+	//CS_                = 0x02,  // false
+	//CS_                = 0x03,  // true
+
+	//CS_                = 0x04,
+	//CS_                = 0x05,
+	//CS_                = 0x06,
+	//CS_                = 0x07,
+
+	//CS_                = 0x08,
+	//CS_                = 0x09,
+	CS_FLOAT             = 0x0a,
+	CS_DOUBLE            = 0x0b,
+	CS_UINT_8            = 0x0c,
+	CS_UINT_16           = 0x0d,
+	CS_UINT_32           = 0x0e,
+	CS_UINT_64           = 0x0f,
+	CS_INT_8             = 0x10,
+	CS_INT_16            = 0x11,
+	CS_INT_32            = 0x12,
+	CS_INT_64            = 0x13,
+
+	//CS_                = 0x14,
+	//CS_                = 0x15,
+	//CS_BIG_INT_16        = 0x16,
+	//CS_BIG_INT_32        = 0x17,
+	//CS_BIG_FLOAT_16      = 0x18,
+	//CS_BIG_FLOAT_32      = 0x19,
+	CS_RAW_16            = 0x1a,
+	CS_RAW_32            = 0x1b,
+	CS_ARRAY_16          = 0x1c,
+	CS_ARRAY_32          = 0x1d,
+	CS_MAP_16            = 0x1e,
+	CS_MAP_32            = 0x1f,
+
+	//ACS_BIG_INT_VALUE,
+	//ACS_BIG_FLOAT_VALUE,
+	ACS_RAW_VALUE,
+} msgpack_unpack_state;
+
+
+typedef enum {
+	CT_ARRAY_ITEM,
+	CT_MAP_KEY,
+	CT_MAP_VALUE,
+} msgpack_container_type;
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* msgpack/unpack_define.h */
+
diff --git a/php/msgpack/unpack_template.h b/php/msgpack/unpack_template.h
new file mode 100644
index 0000000..0fbfbb7
--- /dev/null
+++ b/php/msgpack/unpack_template.h
@@ -0,0 +1,409 @@
+/*
+ * MessagePack unpacking routine template
+ *
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#ifndef msgpack_unpack_func
+#error msgpack_unpack_func template is not defined
+#endif
+
+#ifndef msgpack_unpack_callback
+#error msgpack_unpack_callback template is not defined
+#endif
+
+#ifndef msgpack_unpack_struct
+#error msgpack_unpack_struct template is not defined
+#endif
+
+#ifndef msgpack_unpack_struct_decl
+#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name)
+#endif
+
+#ifndef msgpack_unpack_object
+#error msgpack_unpack_object type is not defined
+#endif
+
+#ifndef msgpack_unpack_user
+#error msgpack_unpack_user type is not defined
+#endif
+
+#ifndef USE_CASE_RANGE
+#if !defined(_MSC_VER)
+#define USE_CASE_RANGE
+#endif
+#endif
+
+msgpack_unpack_struct_decl(_stack) {
+	msgpack_unpack_object obj;
+	size_t count;
+	unsigned int ct;
+	msgpack_unpack_object map_key;
+};
+
+msgpack_unpack_struct_decl(_context) {
+	msgpack_unpack_user user;
+	unsigned int cs;
+	unsigned int trail;
+	unsigned int top;
+	/*
+	msgpack_unpack_struct(_stack)* stack;
+	unsigned int stack_size;
+	msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE];
+	*/
+	msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE];
+};
+
+
+msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx)
+{
+	ctx->cs = CS_HEADER;
+	ctx->trail = 0;
+	ctx->top = 0;
+	/*
+	ctx->stack = ctx->embed_stack;
+	ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
+	*/
+	ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user);
+}
+
+/*
+msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx)
+{
+	if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
+		free(ctx->stack);
+	}
+}
+*/
+
+msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx)
+{
+	return (ctx)->stack[0].obj;
+}
+
+
+msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
+{
+	assert(len >= *off);
+
+	const unsigned char* p = (unsigned char*)data + *off;
+	const unsigned char* const pe = (unsigned char*)data + len;
+	const void* n = NULL;
+
+	unsigned int trail = ctx->trail;
+	unsigned int cs = ctx->cs;
+	unsigned int top = ctx->top;
+	msgpack_unpack_struct(_stack)* stack = ctx->stack;
+	/*
+	unsigned int stack_size = ctx->stack_size;
+	*/
+	msgpack_unpack_user* user = &ctx->user;
+
+	msgpack_unpack_object obj;
+	msgpack_unpack_struct(_stack)* c = NULL;
+
+	int ret;
+
+#define push_simple_value(func) \
+	if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \
+	goto _push
+#define push_fixed_value(func, arg) \
+	if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \
+	goto _push
+#define push_variable_value(func, base, pos, len) \
+	if(msgpack_unpack_callback(func)(user, \
+		(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
+	goto _push
+
+#define again_fixed_trail(_cs, trail_len) \
+	trail = trail_len; \
+	cs = _cs; \
+	goto _fixed_trail_again
+#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \
+	trail = trail_len; \
+	if(trail == 0) { goto ifzero; } \
+	cs = _cs; \
+	goto _fixed_trail_again
+
+#define start_container(func, count_, ct_) \
+	if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
+	if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
+	if((count_) == 0) { obj = stack[top].obj; goto _push; } \
+	stack[top].ct = ct_; \
+	stack[top].count = count_; \
+	++top; \
+	/*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
+	/*printf("stack push %d\n", top);*/ \
+	/* FIXME \
+	if(top >= stack_size) { \
+		if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
+			size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \
+			size_t nsize = csize * 2; \
+			msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \
+			if(tmp == NULL) { goto _failed; } \
+			memcpy(tmp, ctx->stack, csize); \
+			ctx->stack = stack = tmp; \
+			ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
+		} else { \
+			size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \
+			msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \
+			if(tmp == NULL) { goto _failed; } \
+			ctx->stack = stack = tmp; \
+			ctx->stack_size = stack_size = stack_size * 2; \
+		} \
+	} \
+	*/ \
+	goto _header_again
+
+#define NEXT_CS(p) \
+	((unsigned int)*p & 0x1f)
+
+#ifdef USE_CASE_RANGE
+#define SWITCH_RANGE_BEGIN     switch(*p) {
+#define SWITCH_RANGE(FROM, TO) case FROM ... TO:
+#define SWITCH_RANGE_DEFAULT   default:
+#define SWITCH_RANGE_END       }
+#else
+#define SWITCH_RANGE_BEGIN     { if(0) {
+#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) {
+#define SWITCH_RANGE_DEFAULT   } else {
+#define SWITCH_RANGE_END       } }
+#endif
+
+	if(p == pe) { goto _out; }
+	do {
+		switch(cs) {
+		case CS_HEADER:
+			SWITCH_RANGE_BEGIN
+			SWITCH_RANGE(0x00, 0x7f)  // Positive Fixnum
+				push_fixed_value(_uint8, *(uint8_t*)p);
+			SWITCH_RANGE(0xe0, 0xff)  // Negative Fixnum
+				push_fixed_value(_int8, *(int8_t*)p);
+			SWITCH_RANGE(0xc0, 0xdf)  // Variable
+				switch(*p) {
+				case 0xc0:  // nil
+					push_simple_value(_nil);
+				//case 0xc1:  // string
+				//	again_terminal_trail(NEXT_CS(p), p+1);
+				case 0xc2:  // false
+					push_simple_value(_false);
+				case 0xc3:  // true
+					push_simple_value(_true);
+				//case 0xc4:
+				//case 0xc5:
+				//case 0xc6:
+				//case 0xc7:
+				//case 0xc8:
+				//case 0xc9:
+				case 0xca:  // float
+				case 0xcb:  // double
+				case 0xcc:  // unsigned int  8
+				case 0xcd:  // unsigned int 16
+				case 0xce:  // unsigned int 32
+				case 0xcf:  // unsigned int 64
+				case 0xd0:  // signed int  8
+				case 0xd1:  // signed int 16
+				case 0xd2:  // signed int 32
+				case 0xd3:  // signed int 64
+					again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
+				//case 0xd4:
+				//case 0xd5:
+				//case 0xd6:  // big integer 16
+				//case 0xd7:  // big integer 32
+				//case 0xd8:  // big float 16
+				//case 0xd9:  // big float 32
+				case 0xda:  // raw 16
+				case 0xdb:  // raw 32
+				case 0xdc:  // array 16
+				case 0xdd:  // array 32
+				case 0xde:  // map 16
+				case 0xdf:  // map 32
+					again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
+				default:
+					goto _failed;
+				}
+			SWITCH_RANGE(0xa0, 0xbf)  // FixRaw
+				again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
+			SWITCH_RANGE(0x90, 0x9f)  // FixArray
+				start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM);
+			SWITCH_RANGE(0x80, 0x8f)  // FixMap
+				start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
+
+			SWITCH_RANGE_DEFAULT
+				goto _failed;
+			SWITCH_RANGE_END
+			// end CS_HEADER
+
+
+		_fixed_trail_again:
+			++p;
+
+		default:
+			if((size_t)(pe - p) < trail) { goto _out; }
+			n = p;  p += trail - 1;
+			switch(cs) {
+			//case CS_
+			//case CS_
+			case CS_FLOAT: {
+					union { uint32_t i; float f; } mem;
+					mem.i = _msgpack_load32(uint32_t,n);
+					push_fixed_value(_float, mem.f); }
+			case CS_DOUBLE: {
+					union { uint64_t i; double f; } mem;
+					mem.i = _msgpack_load64(uint64_t,n);
+					push_fixed_value(_double, mem.f); }
+			case CS_UINT_8:
+				push_fixed_value(_uint8, *(uint8_t*)n);
+			case CS_UINT_16:
+				push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
+			case CS_UINT_32:
+				push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
+			case CS_UINT_64:
+				push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
+
+			case CS_INT_8:
+				push_fixed_value(_int8, *(int8_t*)n);
+			case CS_INT_16:
+				push_fixed_value(_int16, _msgpack_load16(int16_t,n));
+			case CS_INT_32:
+				push_fixed_value(_int32, _msgpack_load32(int32_t,n));
+			case CS_INT_64:
+				push_fixed_value(_int64, _msgpack_load64(int64_t,n));
+
+			//case CS_
+			//case CS_
+			//case CS_BIG_INT_16:
+			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero);
+			//case CS_BIG_INT_32:
+			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero);
+			//case ACS_BIG_INT_VALUE:
+			//_big_int_zero:
+			//	// FIXME
+			//	push_variable_value(_big_int, data, n, trail);
+
+			//case CS_BIG_FLOAT_16:
+			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero);
+			//case CS_BIG_FLOAT_32:
+			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero);
+			//case ACS_BIG_FLOAT_VALUE:
+			//_big_float_zero:
+			//	// FIXME
+			//	push_variable_value(_big_float, data, n, trail);
+
+			case CS_RAW_16:
+				again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
+			case CS_RAW_32:
+				again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
+			case ACS_RAW_VALUE:
+			_raw_zero:
+				push_variable_value(_raw, data, n, trail);
+
+			case CS_ARRAY_16:
+				start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
+			case CS_ARRAY_32:
+				/* FIXME security guard */
+				start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
+
+			case CS_MAP_16:
+				start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
+			case CS_MAP_32:
+				/* FIXME security guard */
+				start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
+
+			default:
+				goto _failed;
+			}
+		}
+
+_push:
+	if(top == 0) { goto _finish; }
+	c = &stack[top-1];
+	switch(c->ct) {
+	case CT_ARRAY_ITEM:
+		if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; }
+		if(--c->count == 0) {
+			obj = c->obj;
+			--top;
+			/*printf("stack pop %d\n", top);*/
+			goto _push;
+		}
+		goto _header_again;
+	case CT_MAP_KEY:
+		c->map_key = obj;
+		c->ct = CT_MAP_VALUE;
+		goto _header_again;
+	case CT_MAP_VALUE:
+		if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
+		if(--c->count == 0) {
+			obj = c->obj;
+			--top;
+			/*printf("stack pop %d\n", top);*/
+			goto _push;
+		}
+		c->ct = CT_MAP_KEY;
+		goto _header_again;
+
+	default:
+		goto _failed;
+	}
+
+_header_again:
+		cs = CS_HEADER;
+		++p;
+	} while(p != pe);
+	goto _out;
+
+
+_finish:
+	stack[0].obj = obj;
+	++p;
+	ret = 1;
+	/*printf("-- finish --\n"); */
+	goto _end;
+
+_failed:
+	/*printf("** FAILED **\n"); */
+	ret = -1;
+	goto _end;
+
+_out:
+	ret = 0;
+	goto _end;
+
+_end:
+	ctx->cs = cs;
+	ctx->trail = trail;
+	ctx->top = top;
+	*off = p - (const unsigned char*)data;
+
+	return ret;
+}
+
+
+#undef msgpack_unpack_func
+#undef msgpack_unpack_callback
+#undef msgpack_unpack_struct
+#undef msgpack_unpack_object
+#undef msgpack_unpack_user
+
+#undef push_simple_value
+#undef push_fixed_value
+#undef push_variable_value
+#undef again_fixed_trail
+#undef again_fixed_trail_if_zero
+#undef start_container
+
+#undef NEXT_CS
+
diff --git a/php/msgpack/version.h b/php/msgpack/version.h
new file mode 100644
index 0000000..13671d1
--- /dev/null
+++ b/php/msgpack/version.h
@@ -0,0 +1,40 @@
+/*
+ * MessagePack for C version information
+ *
+ * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+#ifndef MSGPACK_VERSION_H__
+#define MSGPACK_VERSION_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+const char* msgpack_version(void);
+int msgpack_version_major(void);
+int msgpack_version_minor(void);
+
+#define MSGPACK_VERSION "0.5.4"
+#define MSGPACK_VERSION_MAJOR 0
+#define MSGPACK_VERSION_MINOR 5
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* msgpack/version.h */
+
diff --git a/php/msgpack_class.c b/php/msgpack_class.c
index b728cb5..5cfff8d 100644
--- a/php/msgpack_class.c
+++ b/php/msgpack_class.c
@@ -6,33 +6,81 @@
 #include "msgpack_unpack.h"
 #include "msgpack_class.h"
 
+typedef struct {
+    zend_object object;
+    long php_only;
+} php_msgpack_base_t;
+
 typedef struct {
     zend_object object;
     smart_str buffer;
     zval *retval;
     long offset;
+    msgpack_unpack_t mp;
+    php_unserialize_data_t var_hash;
+    long php_only;
 } php_msgpack_unpacker_t;
 
 #if ZEND_MODULE_API_NO >= 20060613
-#define MAGPACK_METHOD_BASE(classname, name) zim_##classname##_##name
+#   define MSGPACK_METHOD_BASE(classname, name) zim_##classname##_##name
 #else
-#define MSGPACK_METHOD_BASE(classname, name) zif_##classname##_##name
+#   define MSGPACK_METHOD_BASE(classname, name) zif_##classname##_##name
 #endif
 
+#if ZEND_MODULE_API_NO >= 20090115
+#   define PUSH_PARAM(arg) zend_vm_stack_push(arg TSRMLS_CC)
+#   define POP_PARAM() (void)zend_vm_stack_pop(TSRMLS_C)
+#   define PUSH_EO_PARAM()
+#   define POP_EO_PARAM()
+#else
+#   define PUSH_PARAM(arg) zend_ptr_stack_push(&EG(argument_stack), arg)
+#   define POP_PARAM() (void)zend_ptr_stack_pop(&EG(argument_stack))
+#   define PUSH_EO_PARAM() zend_ptr_stack_push(&EG(argument_stack), NULL)
+#   define POP_EO_PARAM() (void)zend_ptr_stack_pop(&EG(argument_stack))
+#endif
+
+#define MSGPACK_METHOD_HELPER(classname, name, retval, thisptr, num, param) \
+    PUSH_PARAM(param); PUSH_PARAM((void*)num);                              \
+    PUSH_EO_PARAM();                                                        \
+    MSGPACK_METHOD_BASE(classname, name)(num, retval, NULL, thisptr, 0 TSRMLS_CC); \
+    POP_EO_PARAM();                                                         \
+    POP_PARAM();                                                            \
+    POP_PARAM();
+
 #define MSGPACK_METHOD(classname, name, retval, thisptr) \
-    MAGPACK_METHOD_BASE(classname, name)(0, retval, NULL, thisptr, 0 TSRMLS_CC)
+    MSGPACK_METHOD_BASE(classname, name)(0, retval, NULL, thisptr, 0 TSRMLS_CC)
+
+#define MSGPACK_METHOD1(classname, name, retval, thisptr, param1) \
+    MSGPACK_METHOD_HELPER(classname, name, retval, thisptr, 1, param1);
+
+#define MSGPACK_BASE_OBJECT   \
+    php_msgpack_base_t *base; \
+    base = (php_msgpack_base_t *)zend_object_store_get_object(getThis() TSRMLS_CC);
 
 #define MSGPACK_UNPACKER_OBJECT       \
     php_msgpack_unpacker_t *unpacker; \
-    unpacker =(php_msgpack_unpacker_t *)zend_object_store_get_object(getThis() TSRMLS_CC);
+    unpacker = (php_msgpack_unpacker_t *)zend_object_store_get_object(getThis() TSRMLS_CC);
+
+#define MSGPACK_CLASS_OPT_PHPONLY -1001
 
 /* MessagePack */
 static zend_class_entry *msgpack_ce = NULL;
 
+static ZEND_METHOD(msgpack, __construct);
+static ZEND_METHOD(msgpack, setOption);
 static ZEND_METHOD(msgpack, pack);
 static ZEND_METHOD(msgpack, unpack);
 static ZEND_METHOD(msgpack, unpacker);
 
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_base___construct, 0, 0, 0)
+    ZEND_ARG_INFO(0, opt)
+ZEND_END_ARG_INFO()
+
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_base_setOption, 0, 0, 2)
+    ZEND_ARG_INFO(0, option)
+    ZEND_ARG_INFO(0, value)
+ZEND_END_ARG_INFO()
+
 ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_base_pack, 0, 0, 1)
     ZEND_ARG_INFO(0, value)
 ZEND_END_ARG_INFO()
@@ -45,6 +93,9 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_base_unpacker, 0, 0, 0)
 ZEND_END_ARG_INFO()
 
 static const zend_function_entry msgpack_base_methods[] = {
+    ZEND_ME(msgpack, __construct,
+            arginfo_msgpack_base___construct, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack, setOption, arginfo_msgpack_base_setOption, ZEND_ACC_PUBLIC)
     ZEND_ME(msgpack, pack, arginfo_msgpack_base_pack, ZEND_ACC_PUBLIC)
     ZEND_ME(msgpack, unpack, arginfo_msgpack_base_unpack, ZEND_ACC_PUBLIC)
     ZEND_ME(msgpack, unpacker, arginfo_msgpack_base_unpacker, ZEND_ACC_PUBLIC)
@@ -54,19 +105,26 @@ static const zend_function_entry msgpack_base_methods[] = {
 /* MessagePackUnpacker */
 static zend_class_entry *msgpack_unpacker_ce = NULL;
 
-static ZEND_METHOD(msgpack, __construct);
-static ZEND_METHOD(msgpack, __destruct);
-static ZEND_METHOD(msgpack, feed);
-static ZEND_METHOD(msgpack, execute);
-static ZEND_METHOD(msgpack, data);
-static ZEND_METHOD(msgpack, reset);
+static ZEND_METHOD(msgpack_unpacker, __construct);
+static ZEND_METHOD(msgpack_unpacker, __destruct);
+static ZEND_METHOD(msgpack_unpacker, setOption);
+static ZEND_METHOD(msgpack_unpacker, feed);
+static ZEND_METHOD(msgpack_unpacker, execute);
+static ZEND_METHOD(msgpack_unpacker, data);
+static ZEND_METHOD(msgpack_unpacker, reset);
 
 ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unpacker___construct, 0, 0, 0)
+    ZEND_ARG_INFO(0, opt)
 ZEND_END_ARG_INFO()
 
 ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unpacker___destruct, 0, 0, 0)
 ZEND_END_ARG_INFO()
 
+ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unpacker_setOption, 0, 0, 2)
+    ZEND_ARG_INFO(0, option)
+    ZEND_ARG_INFO(0, value)
+ZEND_END_ARG_INFO()
+
 ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unpacker_feed, 0, 0, 1)
     ZEND_ARG_INFO(0, str)
 ZEND_END_ARG_INFO()
@@ -83,17 +141,52 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_msgpack_unpacker_reset, 0, 0, 0)
 ZEND_END_ARG_INFO()
 
 static const zend_function_entry msgpack_unpacker_methods[] = {
-    ZEND_ME(msgpack, __construct,
+    ZEND_ME(msgpack_unpacker, __construct,
             arginfo_msgpack_unpacker___construct, ZEND_ACC_PUBLIC)
-    ZEND_ME(msgpack, __destruct,
+    ZEND_ME(msgpack_unpacker, __destruct,
             arginfo_msgpack_unpacker___destruct, ZEND_ACC_PUBLIC)
-    ZEND_ME(msgpack, feed, arginfo_msgpack_unpacker_feed, ZEND_ACC_PUBLIC)
-    ZEND_ME(msgpack, execute, arginfo_msgpack_unpacker_execute, ZEND_ACC_PUBLIC)
-    ZEND_ME(msgpack, data, arginfo_msgpack_unpacker_data, ZEND_ACC_PUBLIC)
-    ZEND_ME(msgpack, reset, arginfo_msgpack_unpacker_reset, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack_unpacker, setOption,
+            arginfo_msgpack_unpacker_setOption, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack_unpacker, feed,
+            arginfo_msgpack_unpacker_feed, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack_unpacker, execute,
+            arginfo_msgpack_unpacker_execute, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack_unpacker, data,
+            arginfo_msgpack_unpacker_data, ZEND_ACC_PUBLIC)
+    ZEND_ME(msgpack_unpacker, reset,
+            arginfo_msgpack_unpacker_reset, ZEND_ACC_PUBLIC)
     {NULL, NULL, NULL}
 };
 
+static void php_msgpack_base_free(php_msgpack_base_t *base TSRMLS_DC)
+{
+    zend_object_std_dtor(&base->object TSRMLS_CC);
+    efree(base);
+}
+
+static zend_object_value php_msgpack_base_new(zend_class_entry *ce TSRMLS_DC)
+{
+    zend_object_value retval;
+    zval *tmp;
+    php_msgpack_base_t *base;
+
+    base = emalloc(sizeof(php_msgpack_base_t));
+
+    zend_object_std_init(&base->object, ce TSRMLS_CC);
+
+    zend_hash_copy(
+        base->object.properties, &ce->default_properties,
+        (copy_ctor_func_t)zval_add_ref, (void *)&tmp, sizeof(zval *));
+
+    retval.handle = zend_objects_store_put(
+        base, (zend_objects_store_dtor_t)zend_objects_destroy_object,
+        (zend_objects_free_object_storage_t)php_msgpack_base_free,
+        NULL TSRMLS_CC);
+    retval.handlers = zend_get_std_object_handlers();
+
+    return retval;
+}
+
 static void php_msgpack_unpacker_free(
     php_msgpack_unpacker_t *unpacker TSRMLS_DC)
 {
@@ -126,10 +219,58 @@ static zend_object_value php_msgpack_unpacker_new(
 }
 
 /* MessagePack */
+static ZEND_METHOD(msgpack, __construct)
+{
+    bool php_only = MSGPACK_G(php_only);
+    MSGPACK_BASE_OBJECT;
+
+    if (zend_parse_parameters(
+            ZEND_NUM_ARGS() TSRMLS_CC, "|b", &php_only) == FAILURE)
+    {
+        return;
+    }
+
+    base->php_only = php_only;
+}
+
+static ZEND_METHOD(msgpack, setOption)
+{
+    long option;
+    zval *value;
+    MSGPACK_BASE_OBJECT;
+
+    if (zend_parse_parameters(
+            ZEND_NUM_ARGS() TSRMLS_CC, "lz", &option, &value) == FAILURE)
+    {
+        return;
+    }
+
+    switch (option)
+    {
+        case MSGPACK_CLASS_OPT_PHPONLY:
+            convert_to_boolean(value);
+            base->php_only = Z_BVAL_P(value);
+            break;
+        default:
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (MessagePack::setOption) "
+                           "error setting msgpack option");
+            }
+            RETURN_FALSE;
+            break;
+    }
+
+    RETURN_TRUE;
+}
+
 static ZEND_METHOD(msgpack, pack)
 {
     zval *parameter;
     smart_str buf = {0};
+    int php_only = MSGPACK_G(php_only);
+    MSGPACK_BASE_OBJECT;
 
     if (zend_parse_parameters(
             ZEND_NUM_ARGS() TSRMLS_CC, "z", ¶meter) == FAILURE)
@@ -137,8 +278,12 @@ static ZEND_METHOD(msgpack, pack)
         return;
     }
 
+    MSGPACK_G(php_only) = base->php_only;
+
     php_msgpack_serialize(&buf, parameter TSRMLS_CC);
 
+    MSGPACK_G(php_only) = php_only;
+
     ZVAL_STRINGL(return_value, buf.c, buf.len, 1);
 
     smart_str_free(&buf);
@@ -148,6 +293,8 @@ static ZEND_METHOD(msgpack, unpack)
 {
     char *str;
     int str_len;
+    int php_only = MSGPACK_G(php_only);
+    MSGPACK_BASE_OBJECT;
 
     if (zend_parse_parameters(
             ZEND_NUM_ARGS() TSRMLS_CC, "s", &str, &str_len) == FAILURE)
@@ -160,31 +307,57 @@ static ZEND_METHOD(msgpack, unpack)
         RETURN_NULL();
     }
 
+    MSGPACK_G(php_only) = base->php_only;
+
     php_msgpack_unserialize(return_value, str, str_len TSRMLS_CC);
+
+    MSGPACK_G(php_only) = php_only;
 }
 
 static ZEND_METHOD(msgpack, unpacker)
 {
-    zval temp;
+    zval temp, *opt;
+    MSGPACK_BASE_OBJECT;
+
+    ALLOC_INIT_ZVAL(opt);
+    ZVAL_BOOL(opt, base->php_only);
 
     object_init_ex(return_value, msgpack_unpacker_ce);
 
-    MSGPACK_METHOD(msgpack, __construct, &temp, return_value);
+    MSGPACK_METHOD1(msgpack_unpacker, __construct, &temp, return_value, opt);
+
+    zval_ptr_dtor(&opt);
 }
 
 /* MessagePackUnpacker */
-static ZEND_METHOD(msgpack, __construct)
+static ZEND_METHOD(msgpack_unpacker, __construct)
 {
+    bool php_only = MSGPACK_G(php_only);
     MSGPACK_UNPACKER_OBJECT;
 
+    if (zend_parse_parameters(
+            ZEND_NUM_ARGS() TSRMLS_CC, "|b", &php_only) == FAILURE)
+    {
+        return;
+    }
+
+    unpacker->php_only = php_only;
+
     unpacker->buffer.c = NULL;
     unpacker->buffer.len = 0;
     unpacker->buffer.a = 0;
     unpacker->retval = NULL;
     unpacker->offset = 0;
+
+    template_init(&unpacker->mp);
+
+    msgpack_unserialize_var_init(&unpacker->var_hash);
+
+    (&unpacker->mp)->user.var_hash =
+        (php_unserialize_data_t *)&unpacker->var_hash;
 }
 
-static ZEND_METHOD(msgpack, __destruct)
+static ZEND_METHOD(msgpack_unpacker, __destruct)
 {
     MSGPACK_UNPACKER_OBJECT;
 
@@ -194,9 +367,43 @@ static ZEND_METHOD(msgpack, __destruct)
     {
         zval_ptr_dtor(&unpacker->retval);
     }
+
+    msgpack_unserialize_var_destroy(&unpacker->var_hash);
 }
 
-static ZEND_METHOD(msgpack, feed)
+static ZEND_METHOD(msgpack_unpacker, setOption)
+{
+    long option;
+    zval *value;
+    MSGPACK_UNPACKER_OBJECT;
+
+    if (zend_parse_parameters(
+            ZEND_NUM_ARGS() TSRMLS_CC, "lz", &option, &value) == FAILURE)
+    {
+        return;
+    }
+
+    switch (option)
+    {
+        case MSGPACK_CLASS_OPT_PHPONLY:
+            convert_to_boolean(value);
+            unpacker->php_only = Z_BVAL_P(value);
+            break;
+        default:
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (MessagePackUnpacker::setOption) "
+                           "error setting msgpack option");
+            }
+            RETURN_FALSE;
+            break;
+    }
+
+    RETURN_TRUE;
+}
+
+static ZEND_METHOD(msgpack_unpacker, feed)
 {
     char *str;
     int str_len;
@@ -218,14 +425,15 @@ static ZEND_METHOD(msgpack, feed)
     RETURN_TRUE;
 }
 
-static ZEND_METHOD(msgpack, execute)
+static ZEND_METHOD(msgpack_unpacker, execute)
 {
-    char *str = NULL;
+    char *str = NULL, *data;
     long str_len = 0;
     zval *offset;
     int ret;
-    php_unserialize_data_t var_hash;
-    msgpack_unserialize_data mpsd;
+    size_t len, off;
+    int error_display = MSGPACK_G(error_display);
+    int php_only = MSGPACK_G(php_only);
     MSGPACK_UNPACKER_OBJECT;
 
     if (zend_parse_parameters(
@@ -237,45 +445,38 @@ static ZEND_METHOD(msgpack, execute)
 
     if (str != NULL)
     {
-        mpsd.data = (unsigned char *)str;
-        mpsd.length = str_len;
-        mpsd.offset = Z_LVAL_P(offset);
+        data = (char *)str;
+        len = (size_t)str_len;
+        off = Z_LVAL_P(offset);
     }
     else
     {
-        mpsd.data = (unsigned char *)unpacker->buffer.c;
-        mpsd.length = unpacker->buffer.len;
-        mpsd.offset = unpacker->offset;
+        data = (char *)unpacker->buffer.c;
+        len = unpacker->buffer.len;
+        off = unpacker->offset;
     }
 
-    if (mpsd.length <= 0 || mpsd.length == mpsd.offset)
-    {
-        RETURN_FALSE;
-    }
-
-    PHP_VAR_UNSERIALIZE_INIT(var_hash);
-
     if (unpacker->retval == NULL)
     {
         ALLOC_INIT_ZVAL(unpacker->retval);
     }
+    (&unpacker->mp)->user.retval = (zval *)unpacker->retval;
 
     MSGPACK_G(error_display) = 0;
+    MSGPACK_G(php_only) = unpacker->php_only;
 
-    ret = msgpack_unserialize_zval(
-        &unpacker->retval, &mpsd, &var_hash TSRMLS_CC);
+    ret = template_execute(&unpacker->mp, data, len, &off);
 
-    MSGPACK_G(error_display) = 1;
-
-    PHP_VAR_UNSERIALIZE_DESTROY(var_hash);
+    MSGPACK_G(error_display) = error_display;
+    MSGPACK_G(php_only) = php_only;
 
     if (str != NULL)
     {
-        ZVAL_LONG(offset, mpsd.offset);
+        ZVAL_LONG(offset, off);
     }
     else
     {
-        unpacker->offset = mpsd.offset;
+        unpacker->offset = off;
     }
 
     switch (ret)
@@ -288,14 +489,14 @@ static ZEND_METHOD(msgpack, execute)
     }
 }
 
-static ZEND_METHOD(msgpack, data)
+static ZEND_METHOD(msgpack_unpacker, data)
 {
     MSGPACK_UNPACKER_OBJECT;
 
     RETURN_ZVAL(unpacker->retval, 1, 1);
 }
 
-static ZEND_METHOD(msgpack, reset)
+static ZEND_METHOD(msgpack_unpacker, reset)
 {
     smart_str buffer = {0};
     MSGPACK_UNPACKER_OBJECT;
@@ -325,15 +526,33 @@ static ZEND_METHOD(msgpack, reset)
         zval_ptr_dtor(&unpacker->retval);
         unpacker->retval = NULL;
     }
+
+    msgpack_unserialize_var_destroy(&unpacker->var_hash);
+
+
+    msgpack_unserialize_var_init(&unpacker->var_hash);
+
+    (&unpacker->mp)->user.var_hash =
+        (php_unserialize_data_t *)&unpacker->var_hash;
+
+    msgpack_unserialize_init(&((&unpacker->mp)->user));
 }
 
-void msgpack_init_class(TSRMLS_DC)
+void msgpack_init_class()
 {
     zend_class_entry ce;
+    TSRMLS_FETCH();
 
+    /* base */
     INIT_CLASS_ENTRY(ce, "MessagePack", msgpack_base_methods);
     msgpack_ce = zend_register_internal_class(&ce TSRMLS_CC);
+    msgpack_ce->create_object = php_msgpack_base_new;
 
+    zend_declare_class_constant_long(
+        msgpack_ce, ZEND_STRS("OPT_PHPONLY") - 1,
+        MSGPACK_CLASS_OPT_PHPONLY TSRMLS_CC);
+
+    /* unpacker */
     INIT_CLASS_ENTRY(ce, "MessagePackUnpacker", msgpack_unpacker_methods);
     msgpack_unpacker_ce = zend_register_internal_class(&ce TSRMLS_CC);
     msgpack_unpacker_ce->create_object = php_msgpack_unpacker_new;
diff --git a/php/msgpack_class.h b/php/msgpack_class.h
index 00ac9e7..fbebaf4 100644
--- a/php/msgpack_class.h
+++ b/php/msgpack_class.h
@@ -1,7 +1,7 @@
 
-#ifndef MSGPACL_CLASS_H
-#define MSGPACL_CLASS_H
+#ifndef MSGPACK_CLASS_H
+#define MSGPACK_CLASS_H
 
-void msgpack_init_class(TSRMLS_DC);
+void msgpack_init_class();
 
 #endif
diff --git a/php/msgpack_pack.c b/php/msgpack_pack.c
index 91a3d39..d2d4ba3 100644
--- a/php/msgpack_pack.c
+++ b/php/msgpack_pack.c
@@ -119,13 +119,14 @@ inline static void msgpack_serialize_class(
 
             if (Z_TYPE_PP(name) != IS_STRING)
             {
-                zend_error(E_NOTICE,
-                           "[msgpack] (msgpack_serialize_class) "
-                           "__sleep should return an array only "
-                           "containing the names of "
-                           "instance-variables to serialize.");
-
-                msgpack_pack_nil(buf);
+                if (MSGPACK_G(error_display))
+                {
+                    zend_error(E_NOTICE,
+                               "[msgpack] (msgpack_serialize_class) "
+                               "__sleep should return an array only "
+                               "containing the names of "
+                               "instance-variables to serialize.");
+                }
                 continue;
             }
 
@@ -195,11 +196,14 @@ inline static void msgpack_serialize_class(
 
                         pefree(prot_name, ce->type & ZEND_INTERNAL_CLASS);
 
-                        zend_error(E_NOTICE,
-                                   "[msgpack] (msgpack_serialize_class) "
-                                   "\"%s\" returned as member variable from "
-                                   "__sleep() but does not exist",
-                                   Z_STRVAL_PP(name));
+                        if (MSGPACK_G(error_display))
+                        {
+                            zend_error(E_NOTICE,
+                                       "[msgpack] (msgpack_serialize_class) "
+                                       "\"%s\" returned as member variable from "
+                                       "__sleep() but does not exist",
+                                       Z_STRVAL_PP(name));
+                        }
 
                         msgpack_serialize_string(
                             buf, Z_STRVAL_PP(name), Z_STRLEN_PP(name));
@@ -227,6 +231,7 @@ inline static void msgpack_serialize_array(
 {
     HashTable *ht;
     size_t n;
+    bool hash = true;
 
     if (object)
     {
@@ -253,19 +258,46 @@ inline static void msgpack_serialize_array(
 
     if (object)
     {
-        msgpack_pack_map(buf, n + 1);
+        if (MSGPACK_G(php_only))
+        {
+            if (Z_ISREF_P(val))
+            {
+                msgpack_pack_map(buf, n + 2);
+                msgpack_pack_nil(buf);
+                msgpack_pack_long(buf, MSGPACK_SERIALIZE_TYPE_REFERENCE);
+            }
+            else
+            {
+                msgpack_pack_map(buf, n + 1);
+            }
 
-        msgpack_pack_nil(buf);
+            msgpack_pack_nil(buf);
 
-        msgpack_serialize_string(buf, class_name, name_len);
+            msgpack_serialize_string(buf, class_name, name_len);
+        }
+        else
+        {
+            msgpack_pack_array(buf, n);
+            hash = false;
+        }
     }
     else if (n == 0)
     {
+        hash = false;
         msgpack_pack_array(buf, n);
     }
     else
     {
-        msgpack_pack_map(buf, n);
+        if (Z_ISREF_P(val) && MSGPACK_G(php_only))
+        {
+            msgpack_pack_map(buf, n + 1);
+            msgpack_pack_nil(buf);
+            msgpack_pack_long(buf, MSGPACK_SERIALIZE_TYPE_REFERENCE);
+        }
+        else
+        {
+            msgpack_pack_map(buf, n);
+        }
     }
 
     if (n > 0)
@@ -292,20 +324,26 @@ inline static void msgpack_serialize_array(
                 continue;
             }
 
-            switch (key_type)
+            if (hash)
             {
-                case HASH_KEY_IS_LONG:
-                    msgpack_pack_long(buf, key_index);
-                    break;
-                case HASH_KEY_IS_STRING:
-                    msgpack_serialize_string(buf, key, key_len - 1);
-                    break;
-                default:
-                    msgpack_serialize_string(buf, "", sizeof(""));
-
-                    zend_error(E_WARNING, "[msgpack] (msgpack_serialize_array) "
-                               "key is not string nor array");
-                    break;
+                switch (key_type)
+                {
+                    case HASH_KEY_IS_LONG:
+                        msgpack_pack_long(buf, key_index);
+                        break;
+                    case HASH_KEY_IS_STRING:
+                        msgpack_serialize_string(buf, key, key_len - 1);
+                        break;
+                    default:
+                        msgpack_serialize_string(buf, "", sizeof(""));
+                        if (MSGPACK_G(error_display))
+                        {
+                            zend_error(E_WARNING,
+                                       "[msgpack] (msgpack_serialize_array) "
+                                       "key is not string nor array");
+                        }
+                        break;
+                }
             }
 
             if (zend_hash_get_current_data_ex(
@@ -365,7 +403,6 @@ inline static void msgpack_serialize_object(
             msgpack_pack_long(buf, MSGPACK_SERIALIZE_TYPE_CUSTOM_OBJECT);
 
             msgpack_serialize_string(buf, ce->name, ce->name_length);
-
             msgpack_pack_raw(buf, serialized_length);
             msgpack_pack_raw_body(buf, serialized_data, serialized_length);
         }
@@ -401,11 +438,14 @@ inline static void msgpack_serialize_object(
                 }
                 else
                 {
-                    zend_error(E_NOTICE,
-                               "[msgpack] (msgpack_serialize_object) "
-                               "__sleep should return an array only "
-                               "containing the names of instance-variables "
-                               "to serialize");
+                    if (MSGPACK_G(error_display))
+                    {
+                        zend_error(E_NOTICE,
+                                   "[msgpack] (msgpack_serialize_object) "
+                                   "__sleep should return an array only "
+                                   "containing the names of instance-variables "
+                                   "to serialize");
+                    }
                     msgpack_pack_nil(buf);
                 }
                 zval_ptr_dtor(&retval_ptr);
@@ -429,18 +469,19 @@ void msgpack_serialize_zval(
 {
     ulong *var_already;
 
-    if (var_hash &&
+    if (MSGPACK_G(php_only) &&
+        var_hash &&
         msgpack_var_add(
             var_hash, val, (void *)&var_already TSRMLS_CC) == FAILURE)
     {
-        if (Z_ISREF_P(val))
+        if (Z_ISREF_P(val) && Z_TYPE_P(val) == IS_ARRAY)
         {
             msgpack_pack_map(buf, 2);
 
             msgpack_pack_nil(buf);
-            msgpack_pack_long(buf, MSGPACK_SERIALIZE_TYPE_REFERENCE);
+            msgpack_pack_long(buf, MSGPACK_SERIALIZE_TYPE_RECURSIVE);
 
-            msgpack_pack_nil(buf);
+            msgpack_pack_long(buf, 0);
             msgpack_pack_long(buf, *var_already);
 
             return;
@@ -450,9 +491,9 @@ void msgpack_serialize_zval(
             msgpack_pack_map(buf, 2);
 
             msgpack_pack_nil(buf);
-            msgpack_pack_long(buf, MSGPACK_SERIALIZE_TYPE_OBJECT);
+            msgpack_pack_long(buf, MSGPACK_SERIALIZE_TYPE_RECURSIVE);
 
-            msgpack_pack_nil(buf);
+            msgpack_pack_long(buf, 0);
             msgpack_pack_long(buf, *var_already);
 
             return;
@@ -504,9 +545,12 @@ void msgpack_serialize_zval(
             }
             break;
         default:
-            zend_error(E_WARNING,
-                       "[msgpack] (php_msgpack_serialize) "
-                       "type is unsupported, encoded as null");
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (php_msgpack_serialize) "
+                           "type is unsupported, encoded as null");
+            }
             msgpack_pack_nil(buf);
             break;
     }
diff --git a/php/msgpack_pack.h b/php/msgpack_pack.h
index 96cfd4e..16f3e7b 100644
--- a/php/msgpack_pack.h
+++ b/php/msgpack_pack.h
@@ -1,13 +1,14 @@
 
-#ifndef MSGPACL_PACK_H
-#define MSGPACL_PACK_H
+#ifndef MSGPACK_PACK_H
+#define MSGPACK_PACK_H
 
 #include "ext/standard/php_smart_str.h"
 
 enum msgpack_serialize_type
 {
+    MSGPACK_SERIALIZE_TYPE_NONE =  0,
     MSGPACK_SERIALIZE_TYPE_REFERENCE =  1,
-    MSGPACK_SERIALIZE_TYPE_OBJECT,
+    MSGPACK_SERIALIZE_TYPE_RECURSIVE,
     MSGPACK_SERIALIZE_TYPE_CUSTOM_OBJECT,
 };
 
diff --git a/php/msgpack_unpack.c b/php/msgpack_unpack.c
index c6b9b3f..f3b610d 100644
--- a/php/msgpack_unpack.c
+++ b/php/msgpack_unpack.c
@@ -2,31 +2,69 @@
 #include "php.h"
 #include "php_ini.h"
 #include "ext/standard/php_incomplete_class.h"
-#include "ext/standard/php_var.h"
 
 #include "php_msgpack.h"
 #include "msgpack_pack.h"
 #include "msgpack_unpack.h"
 
-#include "msgpack/unpack_define.h"
-
-#define VAR_ENTRIES_MAX 1024
-
 #if (PHP_MAJOR_VERSION == 5 && PHP_MINOR_VERSION < 3)
 #   define Z_ADDREF_PP(ppz)      ZVAL_ADDREF(*(ppz))
 #   define Z_SET_ISREF_PP(ppz)   (*(ppz))->is_ref = 1
 #   define Z_UNSET_ISREF_PP(ppz) (*(ppz))->is_ref = 0
 #endif
 
+#define VAR_ENTRIES_MAX 1024
+
 typedef struct
 {
     zval *data[VAR_ENTRIES_MAX];
     long used_slots;
+    long value_slots;
+    long access_slots[VAR_ENTRIES_MAX];
+    bool alloc_slots[VAR_ENTRIES_MAX];
     void *next;
 } var_entries;
 
+#define MSGPACK_UNSERIALIZE_ALLOC(_unpack)                     \
+    if (_unpack->deps <= 0) {                                  \
+        *obj = _unpack->retval;                                \
+        msgpack_var_push(_unpack->var_hash, obj, true, false); \
+    } else {                                                   \
+        ALLOC_INIT_ZVAL(*obj);                                 \
+        msgpack_var_push(_unpack->var_hash, obj, false, true); \
+    }
+
+#define MSGPACK_UNSERIALIZE_ALLOC_VALUE(_unpack)               \
+    if (_unpack->deps <= 0) {                                  \
+        *obj = _unpack->retval;                                \
+        msgpack_var_push(_unpack->var_hash, obj, true, false); \
+    } else {                                                   \
+        ALLOC_INIT_ZVAL(*obj);                                 \
+        msgpack_var_push(_unpack->var_hash, obj, true, true);  \
+    }
+
+#define MSGPACK_UNSERIALIZE_PUSH_ITEM(_unpack, _count, _val)         \
+    msgpack_var_alloc(_unpack->var_hash, _count);                    \
+    if (Z_TYPE_P(_val) != IS_ARRAY && Z_TYPE_P(_val) != IS_OBJECT) { \
+        msgpack_var_push(_unpack->var_hash, &_val, true, false);     \
+    }
+
+#define MSGPACK_UNSERIALIZE_FINISH_ITEM(_unpack) \
+    long deps = _unpack->deps - 1;               \
+    _unpack->stack[deps]--;                      \
+    if (_unpack->stack[deps] == 0) {             \
+        _unpack->deps--;                         \
+    }
+
+#define MSGPACK_UNSERIALIZE_FINISH_MAP_ITEM(_unpack, _key, _val) \
+    zval_ptr_dtor(&_key);                                        \
+    zval_ptr_dtor(&_val);                                        \
+    msgpack_var_alloc(_unpack->var_hash, 2);                     \
+    MSGPACK_UNSERIALIZE_FINISH_ITEM(_unpack);
+
+
 inline static void msgpack_var_push(
-    php_unserialize_data_t *var_hashx, zval **rval)
+    php_unserialize_data_t *var_hashx, zval **rval, bool value, bool alloc)
 {
     var_entries *var_hash, *prev = NULL;
 
@@ -47,6 +85,7 @@ inline static void msgpack_var_push(
     {
         var_hash = emalloc(sizeof(var_entries));
         var_hash->used_slots = 0;
+        var_hash->value_slots = 0;
         var_hash->next = 0;
 
         if (!var_hashx->first)
@@ -59,9 +98,45 @@ inline static void msgpack_var_push(
         }
     }
 
+    var_hash->alloc_slots[var_hash->used_slots] = alloc;
+
+    if (value)
+    {
+        var_hash->access_slots[var_hash->value_slots++] = var_hash->used_slots;
+    }
+
     var_hash->data[var_hash->used_slots++] = *rval;
 }
 
+inline static void msgpack_var_alloc(
+    php_unserialize_data_t *var_hashx, long count)
+{
+    long i;
+    var_entries *var_hash = var_hashx->first;
+
+    while (var_hash && var_hash->used_slots == VAR_ENTRIES_MAX)
+    {
+        var_hash = var_hash->next;
+    }
+
+    if (!var_hash || count <= 0)
+    {
+        return;
+    }
+
+    for (i = var_hash->used_slots - 1; i >= 0; i--)
+    {
+        if (var_hash->alloc_slots[i])
+        {
+            var_hash->alloc_slots[i] = false;
+            if (--count <= 0)
+            {
+                break;
+            }
+        }
+    }
+}
+
 inline static int msgpack_var_access(
     php_unserialize_data_t *var_hashx, long id, zval ***store)
 {
@@ -79,6 +154,13 @@ inline static int msgpack_var_access(
         return !SUCCESS;
     }
 
+    if (id < 0 || id >= var_hash->value_slots)
+    {
+        return !SUCCESS;
+    }
+
+    id = var_hash->access_slots[id];
+
     if (id < 0 || id >= var_hash->used_slots)
     {
         return !SUCCESS;
@@ -89,741 +171,533 @@ inline static int msgpack_var_access(
     return SUCCESS;
 }
 
-inline static int msgpack_unserialize_array(
-    zval **return_value, msgpack_unserialize_data *mpsd,
-    ulong ct, php_unserialize_data_t *var_hash TSRMLS_DC)
+inline static zend_class_entry* msgpack_unserialize_class(
+    zval **container, char *class_name, size_t name_len)
 {
-    ulong i;
-    HashTable *ht;
-
-    msgpack_var_push(var_hash, return_value);
-
-    if (Z_TYPE_PP(return_value) != IS_ARRAY)
-    {
-        array_init(*return_value);
-    }
-
-    ht = HASH_OF(*return_value);
-
-    for (i = 0; i < ct; i++)
-    {
-        zval *val;
-
-        /* value */
-        ALLOC_INIT_ZVAL(val);
-
-        if (msgpack_unserialize_zval(&val, mpsd, var_hash TSRMLS_CC) <= 0)
-        {
-            if (MSGPACK_G(error_display))
-            {
-                zend_error(E_WARNING,
-                           "[msgpack] (msgpack_unserialize_array) "
-                           "Invalid value");
-            }
-            zval_ptr_dtor(&val);
-
-            return MSGPACK_UNPACK_PARSE_ERROR;
-        }
-
-        /* update */
-        zend_hash_index_update(ht, i, &val, sizeof(zval *), NULL);
-    }
-
-    return MSGPACK_UNPACK_SUCCESS;
-}
-
-inline static int msgpack_unserialize_object_type(
-    zval **return_value, msgpack_unserialize_data *mpsd,
-    php_unserialize_data_t *var_hash, zend_bool is_ref TSRMLS_DC)
-{
-    int ret = MSGPACK_UNPACK_SUCCESS;
-    zval *key, *val, **rval;
-
-    ALLOC_INIT_ZVAL(key);
-
-    if (msgpack_unserialize_zval(&key, mpsd, NULL TSRMLS_CC) <= 0)
-    {
-        zval_ptr_dtor(&key);
-
-        ZVAL_BOOL(*return_value, 0);
-
-        return MSGPACK_UNPACK_PARSE_ERROR;
-    }
-
-    ALLOC_INIT_ZVAL(val);
-
-    if (is_ref)
-    {
-        ret = msgpack_unserialize_zval(&val, mpsd, NULL TSRMLS_CC);
-    }
-    else
-    {
-        ret = msgpack_unserialize_zval(&val, mpsd, var_hash TSRMLS_CC);
-    }
-
-    if (ret <= 0)
-    {
-        zval_ptr_dtor(&val);
-        zval_ptr_dtor(&key);
-
-        ZVAL_BOOL(*return_value, 0);
-
-        return MSGPACK_UNPACK_PARSE_ERROR;
-    }
-
-    if (!var_hash ||
-        msgpack_var_access(var_hash, Z_LVAL_P(val) - 1, &rval) != SUCCESS)
-    {
-        if (MSGPACK_G(error_display))
-        {
-            zend_error(E_WARNING,
-                       "[msgpack] (msgpack_unserialize_object_key) "
-                       "Invalid references value: %ld",
-                       Z_LVAL_P(val) - 1);
-        }
-        ret = MSGPACK_UNPACK_CONTINUE;
-    }
-    else
-    {
-        if (*return_value != NULL)
-        {
-            zval_ptr_dtor(return_value);
-        }
-
-        *return_value = *rval;
-
-        Z_ADDREF_PP(return_value);
-
-        if (is_ref)
-        {
-            Z_SET_ISREF_PP(return_value);
-        }
-        else
-        {
-            Z_UNSET_ISREF_PP(return_value);
-        }
-    }
-
-    zval_ptr_dtor(&key);
-    zval_ptr_dtor(&val);
-
-    return ret;
-}
-
-inline static int msgpack_unserialize_object(
-    zval **return_value, msgpack_unserialize_data *mpsd,
-    ulong ct, php_unserialize_data_t *var_hash TSRMLS_DC)
-{
-    int ret = MSGPACK_UNPACK_SUCCESS;
-
     zend_class_entry *ce, **pce;
     bool incomplete_class = false;
     zval *user_func, *retval_ptr, **args[1], *arg_func_name;
-    HashTable *ht;
+    TSRMLS_FETCH();
 
-    zval *key, *val;
+    do
+    {
+        /* Try to find class directly */
+        if (zend_lookup_class(class_name, name_len, &pce TSRMLS_CC) == SUCCESS)
+        {
+            ce = *pce;
+            break;
+        }
 
-    int object = 1;
-    int custom_object = 0;
+        /* Check for unserialize callback */
+        if ((PG(unserialize_callback_func) == NULL) ||
+            (PG(unserialize_callback_func)[0] == '\0'))
+        {
+            incomplete_class = 1;
+            ce = PHP_IC_ENTRY;
+            break;
+        }
 
-    /* Get class */
-    ALLOC_INIT_ZVAL(key);
+        /* Call unserialize callback */
+        ALLOC_INIT_ZVAL(user_func);
+        ZVAL_STRING(user_func, PG(unserialize_callback_func), 1);
+        args[0] = &arg_func_name;
+        ALLOC_INIT_ZVAL(arg_func_name);
+        ZVAL_STRING(arg_func_name, class_name, 1);
+        if (call_user_function_ex(
+                CG(function_table), NULL, user_func, &retval_ptr,
+                1, args, 0, NULL TSRMLS_CC) != SUCCESS)
+        {
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (msgpack_unserialize_class) "
+                           "defined (%s) but not found", class_name);
+            }
 
-    if (msgpack_unserialize_zval(&key, mpsd, NULL TSRMLS_CC) <= 0)
+            incomplete_class = 1;
+            ce = PHP_IC_ENTRY;
+            zval_ptr_dtor(&user_func);
+            zval_ptr_dtor(&arg_func_name);
+            break;
+        }
+        if (retval_ptr)
+        {
+            zval_ptr_dtor(&retval_ptr);
+        }
+
+        /* The callback function may have defined the class */
+        if (zend_lookup_class(class_name, name_len, &pce TSRMLS_CC) == SUCCESS)
+        {
+            ce = *pce;
+        }
+        else
+        {
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (msgpack_unserialize_class) "
+                           "Function %s() hasn't defined the class "
+                           "it was called for", class_name);
+            }
+
+            incomplete_class = true;
+            ce = PHP_IC_ENTRY;
+        }
+
+        zval_ptr_dtor(&user_func);
+        zval_ptr_dtor(&arg_func_name);
+    }
+    while(0);
+
+    if (EG(exception))
     {
         if (MSGPACK_G(error_display))
         {
             zend_error(E_WARNING,
-                       "[msgpack] (msgpack_unserialize_object) "
-                       "Invalid sign key");
+                       "[msgpack] (msgpack_unserialize_class) "
+                       "Exception error");
         }
 
-        zval_ptr_dtor(&key);
-
-        ZVAL_BOOL(*return_value, 0);
-
-        return MSGPACK_UNPACK_PARSE_ERROR;
+        return NULL;
     }
 
-    ct--;
+    object_init_ex(*container, ce);
 
-    if (Z_TYPE_P(key) == IS_NULL)
+    /* store incomplete class name */
+    if (incomplete_class)
     {
-        ALLOC_INIT_ZVAL(val);
+        php_store_class_name(*container, class_name, name_len);
+    }
 
-        if (msgpack_unserialize_zval(&val, mpsd, NULL TSRMLS_CC) <= 0)
+    return ce;
+}
+
+void msgpack_unserialize_var_init(php_unserialize_data_t *var_hashx)
+{
+    var_hashx->first = 0;
+    var_hashx->first_dtor = 0;
+}
+
+void msgpack_unserialize_var_destroy(php_unserialize_data_t *var_hashx)
+{
+    void *next;
+    long i;
+    var_entries *var_hash = var_hashx->first;
+
+    while (var_hash)
+    {
+        for (i = 0; i < var_hash->used_slots; i++)
         {
-            if (MSGPACK_G(error_display))
+            if (var_hash->alloc_slots[i] && var_hash->data[i])
             {
-                zend_error(E_WARNING,
-                           "[msgpack] (msgpack_unserialize_object) "
-                           "Invalid sign value");
-            }
-
-            zval_ptr_dtor(&val);
-            zval_ptr_dtor(&key);
-
-            ZVAL_BOOL(*return_value, 0);
-
-            return MSGPACK_UNPACK_PARSE_ERROR;
-        }
-
-        if (Z_TYPE_P(val) == IS_LONG)
-        {
-            switch (Z_LVAL_P(val))
-            {
-                case MSGPACK_SERIALIZE_TYPE_REFERENCE:
-                    ret = msgpack_unserialize_object_type(
-                        return_value, mpsd, var_hash, 1 TSRMLS_CC);
-
-                    zval_ptr_dtor(&key);
-                    zval_ptr_dtor(&val);
-
-                    return ret;
-                case MSGPACK_SERIALIZE_TYPE_OBJECT:
-                    ret = msgpack_unserialize_object_type(
-                        return_value, mpsd, var_hash, 0 TSRMLS_CC);
-
-                    zval_ptr_dtor(&key);
-                    zval_ptr_dtor(&val);
-
-                    return ret;
-                case MSGPACK_SERIALIZE_TYPE_CUSTOM_OBJECT:
-                    custom_object = 1;
-
-                    zval_ptr_dtor(&val);
-
-                    ALLOC_INIT_ZVAL(val);
-
-                    if (msgpack_unserialize_zval(
-                            &val, mpsd, NULL TSRMLS_CC) <= 0)
-                    {
-                        zval_ptr_dtor(&key);
-                        zval_ptr_dtor(&val);
-
-                        ZVAL_BOOL(*return_value, 0);
-
-                        return MSGPACK_UNPACK_PARSE_ERROR;
-                    }
-                    break;
-                default:
-                    zval_ptr_dtor(&key);
-                    zval_ptr_dtor(&val);
-
-                    ZVAL_BOOL(*return_value, 0);
-
-                    return MSGPACK_UNPACK_PARSE_ERROR;
+                zval_ptr_dtor(&var_hash->data[i]);
             }
         }
+
+        next = var_hash->next;
+        efree(var_hash);
+        var_hash = next;
+    }
+
+    /*
+    var_hash = var_hashx->first_dtor;
+
+    while (var_hash)
+    {
+        for (i = 0; i < var_hash->used_slots; i++)
+        {
+            zval_ptr_dtor(&var_hash->data[i]);
+        }
+        next = var_hash->next;
+        efree(var_hash);
+        var_hash = next;
+    }
+    */
+}
+
+void msgpack_unserialize_init(msgpack_unserialize_data *unpack)
+{
+    unpack->deps = 0;
+    unpack->type = MSGPACK_SERIALIZE_TYPE_NONE;
+}
+
+int msgpack_unserialize_uint8(
+    msgpack_unserialize_data *unpack, uint8_t data, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_LONG(*obj, data);
+
+    return 0;
+}
+
+int msgpack_unserialize_uint16(
+    msgpack_unserialize_data *unpack, uint16_t data, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_LONG(*obj, data);
+
+    return 0;
+}
+
+int msgpack_unserialize_uint32(
+    msgpack_unserialize_data *unpack, uint32_t data, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_LONG(*obj, data);
+
+    return 0;
+}
+
+int msgpack_unserialize_uint64(
+    msgpack_unserialize_data *unpack, uint64_t data, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_LONG(*obj, data);
+
+    return 0;
+}
+
+int msgpack_unserialize_int8(
+    msgpack_unserialize_data *unpack, int8_t data, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_LONG(*obj, data);
+
+    return 0;
+}
+
+int msgpack_unserialize_int16(
+    msgpack_unserialize_data *unpack, int16_t data, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_LONG(*obj, data);
+
+    return 0;
+}
+
+int msgpack_unserialize_int32(
+    msgpack_unserialize_data *unpack, int32_t data, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_LONG(*obj, data);
+
+    return 0;
+}
+
+int msgpack_unserialize_int64(
+    msgpack_unserialize_data *unpack, int64_t data, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_LONG(*obj, data);
+
+    return 0;
+}
+
+int msgpack_unserialize_float(
+    msgpack_unserialize_data *unpack, float data, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_DOUBLE(*obj, data);
+
+    return 0;
+}
+
+int msgpack_unserialize_double(
+    msgpack_unserialize_data *unpack, double data, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_DOUBLE(*obj, data);
+
+    return 0;
+}
+
+int msgpack_unserialize_nil(msgpack_unserialize_data *unpack, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_NULL(*obj);
+
+    return 0;
+}
+
+int msgpack_unserialize_true(msgpack_unserialize_data *unpack, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_BOOL(*obj, 1);
+
+    return 0;
+}
+
+int msgpack_unserialize_false(msgpack_unserialize_data *unpack, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    ZVAL_BOOL(*obj, 0);
+
+    return 0;
+}
+
+int msgpack_unserialize_raw(
+    msgpack_unserialize_data *unpack, const char* base,
+    const char* data, unsigned int len, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC(unpack);
+
+    if (len == 0)
+    {
+        ZVAL_STRINGL(*obj, "", 0, 1);
     }
     else
     {
-        object = 0;
-
-        msgpack_var_push(var_hash, return_value);
-
-        if (Z_TYPE_PP(return_value) != IS_ARRAY)
-        {
-            array_init(*return_value);
-        }
-
-        ht = HASH_OF(*return_value);
-
-        ALLOC_INIT_ZVAL(val);
-
-        if (msgpack_unserialize_zval(&val, mpsd, var_hash TSRMLS_CC) <= 0)
-        {
-            if (MSGPACK_G(error_display))
-            {
-                zend_error(E_WARNING,
-                           "[msgpack] (msgpack_unserialize_object) "
-                           "Invalid sign value");
-            }
-
-            zval_ptr_dtor(&val);
-            zval_ptr_dtor(&key);
-
-            return MSGPACK_UNPACK_PARSE_ERROR;
-        }
-
-        /* update */
-        switch (Z_TYPE_P(key))
-        {
-            case IS_LONG:
-                zend_hash_index_update(
-                    ht, Z_LVAL_P(key), &val, sizeof(zval *), NULL);
-                break;
-            case IS_STRING:
-                zend_symtable_update(
-                    ht, Z_STRVAL_P(key), Z_STRLEN_P(key) + 1,
-                    &val, sizeof(zval *), NULL);
-                break;
-            default:
-                zval_ptr_dtor(&key);
-                zval_ptr_dtor(&val);
-
-                return MSGPACK_UNPACK_PARSE_ERROR;
-        }
+        ZVAL_STRINGL(*obj, data, len, 1);
     }
 
-    zval_ptr_dtor(&key);
+    return 0;
+}
 
-    if (object)
+int msgpack_unserialize_array(
+    msgpack_unserialize_data *unpack, unsigned int count, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC_VALUE(unpack);
+
+    array_init(*obj);
+
+    unpack->stack[unpack->deps++] = count;
+
+    return 0;
+}
+
+int msgpack_unserialize_array_item(
+    msgpack_unserialize_data *unpack, zval **container, zval *obj)
+{
+    MSGPACK_UNSERIALIZE_PUSH_ITEM(unpack, 1, obj);
+
+    add_next_index_zval(*container, obj);
+
+    MSGPACK_UNSERIALIZE_FINISH_ITEM(unpack);
+
+    return 0;
+}
+
+int msgpack_unserialize_map(
+    msgpack_unserialize_data *unpack, unsigned int count, zval **obj)
+{
+    MSGPACK_UNSERIALIZE_ALLOC_VALUE(unpack);
+
+    unpack->stack[unpack->deps++] = count;
+
+    unpack->type = MSGPACK_SERIALIZE_TYPE_NONE;
+
+    return 0;
+}
+
+int msgpack_unserialize_map_item(
+    msgpack_unserialize_data *unpack, zval **container, zval *key, zval *val)
+{
+    TSRMLS_FETCH();
+
+    if (MSGPACK_G(php_only))
     {
-        convert_to_string(val);
+        if (Z_TYPE_P(key) == IS_NULL)
+        {
+            unpack->type = MSGPACK_SERIALIZE_TYPE_NONE;
 
-        do {
-            /* Try to find class directly */
-            if (zend_lookup_class(
-                    Z_STRVAL_P(val), Z_STRLEN_P(val), &pce TSRMLS_CC) == SUCCESS)
+            if (Z_TYPE_P(val) == IS_LONG)
             {
-                ce = *pce;
-                break;
-            }
-
-            /* Check for unserialize callback */
-            if ((PG(unserialize_callback_func) == NULL) ||
-                (PG(unserialize_callback_func)[0] == '\0'))
-            {
-                incomplete_class = 1;
-                ce = PHP_IC_ENTRY;
-                break;
-            }
-
-            /* Call unserialize callback */
-            ALLOC_INIT_ZVAL(user_func);
-            ZVAL_STRING(user_func, PG(unserialize_callback_func), 1);
-            args[0] = &arg_func_name;
-            ALLOC_INIT_ZVAL(arg_func_name);
-            ZVAL_STRING(arg_func_name, Z_STRVAL_P(val), 1);
-            if (call_user_function_ex(
-                    CG(function_table), NULL, user_func, &retval_ptr,
-                    1, args, 0, NULL TSRMLS_CC) != SUCCESS)
-            {
-                if (MSGPACK_G(error_display))
+                switch (Z_LVAL_P(val))
                 {
-                    zend_error(E_WARNING,
-                               "[msgpack] (msgpack_unserialize_object) "
-                               "defined (%s) but not found", Z_STRVAL_P(val));
+                    case MSGPACK_SERIALIZE_TYPE_REFERENCE:
+                        Z_SET_ISREF_PP(container);
+                        break;
+                    case MSGPACK_SERIALIZE_TYPE_RECURSIVE:
+                        unpack->type = MSGPACK_SERIALIZE_TYPE_RECURSIVE;
+                        break;
+                    case MSGPACK_SERIALIZE_TYPE_CUSTOM_OBJECT:
+                        unpack->type = MSGPACK_SERIALIZE_TYPE_CUSTOM_OBJECT;
+                        break;
+                    default:
+                        break;
                 }
+            }
+            else if (Z_TYPE_P(val) == IS_STRING)
+            {
+                zend_class_entry *ce = msgpack_unserialize_class(
+                    container, Z_STRVAL_P(val), Z_STRLEN_P(val));
 
-                incomplete_class = 1;
-                ce = PHP_IC_ENTRY;
-                zval_ptr_dtor(&user_func);
-                zval_ptr_dtor(&arg_func_name);
-                break;
-            }
-            if (retval_ptr)
-            {
-                zval_ptr_dtor(&retval_ptr);
-            }
-
-            /* The callback function may have defined the class */
-            if (zend_lookup_class(
-                    Z_STRVAL_P(val), Z_STRLEN_P(val), &pce TSRMLS_CC) == SUCCESS)
-            {
-                ce = *pce;
-            }
-            else
-            {
-                if (MSGPACK_G(error_display))
+                if (ce == NULL)
                 {
-                    zend_error(E_WARNING,
-                               "[msgpack] (msgpack_unserialize_object) "
-                               "Function %s() hasn't defined the class"
-                               "it was called for", Z_STRVAL_P(val));
+                    MSGPACK_UNSERIALIZE_FINISH_MAP_ITEM(unpack, key, val);
+
+                    return 0;
                 }
-
-                incomplete_class = true;
-                ce = PHP_IC_ENTRY;
             }
 
-            zval_ptr_dtor(&user_func);
-            zval_ptr_dtor(&arg_func_name);
-        } while(0);
+            MSGPACK_UNSERIALIZE_FINISH_MAP_ITEM(unpack, key, val);
 
-        if (EG(exception))
+            return 0;
+        }
+        else if (unpack->type == MSGPACK_SERIALIZE_TYPE_CUSTOM_OBJECT)
         {
-            if (MSGPACK_G(error_display))
+            unpack->type = MSGPACK_SERIALIZE_TYPE_NONE;
+
+            zend_class_entry *ce = msgpack_unserialize_class(
+                container, Z_STRVAL_P(key), Z_STRLEN_P(key));
+
+            if (ce == NULL)
             {
-                zend_error(E_WARNING,
-                           "[msgpack] (msgpack_unserialize_object) "
-                           "Exception error");
+                MSGPACK_UNSERIALIZE_FINISH_MAP_ITEM(unpack, key, val);
+
+                return 0;
             }
 
-            zval_ptr_dtor(&val);
-
-            ZVAL_BOOL(*return_value, 0);
-
-            return MSGPACK_UNPACK_PARSE_ERROR;
-        }
-
-        msgpack_var_push(var_hash, return_value);
-
-        object_init_ex(*return_value, ce);
-
-        /* store incomplete class name */
-        if (incomplete_class)
-        {
-            php_store_class_name(
-                *return_value, Z_STRVAL_P(val), Z_STRLEN_P(val));
-        }
-
-        zval_ptr_dtor(&val);
-
-        /* implementing Serializable */
-        if (custom_object)
-        {
-            zval *rval;
-
+            /* implementing Serializable */
             if (ce->unserialize == NULL)
             {
                 if (MSGPACK_G(error_display))
                 {
                     zend_error(E_WARNING,
-                               "[msgpack] (msgpack_unserialize_object) "
+                               "[msgpack] (msgpack_unserialize_map_item) "
                                "Class %s has no unserializer", ce->name);
                 }
 
-                return MSGPACK_UNPACK_PARSE_ERROR;
-            }
+                MSGPACK_UNSERIALIZE_FINISH_MAP_ITEM(unpack, key, val);
 
-            /* value */
-            ALLOC_INIT_ZVAL(rval);
-
-            if (msgpack_unserialize_zval(&rval, mpsd, var_hash TSRMLS_CC) <= 0)
-            {
-                zval_ptr_dtor(&rval);
-
-                return MSGPACK_UNPACK_PARSE_ERROR;
+                return 0;
             }
 
             ce->unserialize(
-                return_value, ce,
-                (const unsigned char *)Z_STRVAL_P(rval), Z_STRLEN_P(rval) + 1,
-                (zend_unserialize_data *)var_hash TSRMLS_CC);
+                container, ce,
+                (const unsigned char *)Z_STRVAL_P(val), Z_STRLEN_P(val) + 1,
+                NULL TSRMLS_CC);
 
-            zval_ptr_dtor(&key);
-            zval_ptr_dtor(&rval);
+            MSGPACK_UNSERIALIZE_FINISH_MAP_ITEM(unpack, key, val);
 
-            return ret;
+            return 0;
         }
+        else if (unpack->type == MSGPACK_SERIALIZE_TYPE_RECURSIVE)
+        {
+            zval **rval;
 
-        ht = HASH_OF(*return_value);
+            unpack->type = MSGPACK_SERIALIZE_TYPE_NONE;
+
+            if (msgpack_var_access(
+                    unpack->var_hash, Z_LVAL_P(val) - 1, &rval) != SUCCESS)
+            {
+                if (MSGPACK_G(error_display))
+                {
+                    zend_error(E_WARNING,
+                               "[msgpack] (msgpack_unserialize_map_item) "
+                               "Invalid references value: %ld",
+                               Z_LVAL_P(val) - 1);
+                }
+
+                MSGPACK_UNSERIALIZE_FINISH_MAP_ITEM(unpack, key, val);
+
+                return 0;
+            }
+
+            if (container != NULL)
+            {
+                zval_ptr_dtor(container);
+            }
+
+            *container = *rval;
+
+            Z_ADDREF_PP(container);
+
+            MSGPACK_UNSERIALIZE_FINISH_MAP_ITEM(unpack, key, val);
+
+            return 0;
+        }
     }
 
-    /* object property */
-    while (ct-- > 0)
+    MSGPACK_UNSERIALIZE_PUSH_ITEM(unpack, 2, val);
+
+    if (Z_TYPE_PP(container) != IS_ARRAY && Z_TYPE_PP(container) != IS_OBJECT)
     {
-        zval *rval;
-
-        /* key */
-        ALLOC_INIT_ZVAL(key);
-
-        if (msgpack_unserialize_zval(&key, mpsd, NULL TSRMLS_CC) <= 0)
-        {
-            zval_ptr_dtor(&key);
-
-            return MSGPACK_UNPACK_PARSE_ERROR;
-        }
-
-        /* value */
-        ALLOC_INIT_ZVAL(rval);
-
-        if (msgpack_unserialize_zval(&rval, mpsd, var_hash TSRMLS_CC) <= 0)
-        {
-            zval_ptr_dtor(&rval);
-            zval_ptr_dtor(&key);
-
-            return MSGPACK_UNPACK_PARSE_ERROR;
-
-        }
-
-        /* update */
-        switch (Z_TYPE_P(key))
-        {
-            case IS_LONG:
-                zend_hash_index_update(
-                    ht, Z_LVAL_P(key), &rval, sizeof(zval *), NULL);
-                break;
-            case IS_STRING:
-                zend_symtable_update(
-                    ht, Z_STRVAL_P(key), Z_STRLEN_P(key) + 1,
-                    &rval, sizeof(zval *), NULL);
-                break;
-            default:
-                zval_ptr_dtor(&key);
-                zval_ptr_dtor(&rval);
-
-                return MSGPACK_UNPACK_PARSE_ERROR;
-        }
-
-        zval_ptr_dtor(&key);
+        array_init(*container);
     }
 
-    /* wakeup */
-    if (object && Z_OBJCE_PP(return_value) != PHP_IC_ENTRY &&
-        zend_hash_exists(&Z_OBJCE_PP(return_value)->function_table,
-                         "__wakeup", sizeof("__wakeup")))
+    switch (Z_TYPE_P(key))
     {
-        zval f, *h = NULL;
+        case IS_LONG:
+            if (zend_hash_index_update(
+                    HASH_OF(*container), Z_LVAL_P(key), &val,
+                    sizeof(val), NULL) == FAILURE)
+            {
+                zval_ptr_dtor(&val);
+                if (MSGPACK_G(error_display))
+                {
+                    zend_error(E_WARNING,
+                               "[msgpack] (msgpack_unserialize_map_item) "
+                               "illegal offset type, skip this decoding");
+                }
+            }
+            break;
+        case IS_STRING:
+            if (zend_symtable_update(
+                    HASH_OF(*container), Z_STRVAL_P(key), Z_STRLEN_P(key) + 1,
+                    &val, sizeof(val), NULL) == FAILURE)
+            {
+                zval_ptr_dtor(&val);
+                if (MSGPACK_G(error_display))
+                {
+                    zend_error(E_WARNING,
+                               "[msgpack] (msgpack_unserialize_map_item) "
+                               "illegal offset type, skip this decoding");
+                }
+            }
+            break;
+        default:
+            zval_ptr_dtor(&val);
+            if (MSGPACK_G(error_display))
+            {
+                zend_error(E_WARNING,
+                           "[msgpack] (msgpack_unserialize_map_item) "
+                           "illegal offset type, skip this decoding");
+            }
+            break;
+    }
 
-        INIT_PZVAL(&f);
-        ZVAL_STRINGL(&f, "__wakeup", sizeof("__wakeup") - 1, 0);
-        call_user_function_ex(
-            CG(function_table), return_value, &f, &h, 0, 0, 1, NULL TSRMLS_CC);
-        if (h)
-        {
-            zval_ptr_dtor(&h);
-        }
+    zval_ptr_dtor(&key);
 
-        if (EG(exception))
+    long deps = unpack->deps - 1;
+    unpack->stack[deps]--;
+    if (unpack->stack[deps] == 0)
+    {
+        unpack->deps--;
+
+        /* wakeup */
+        if (MSGPACK_G(php_only) &&
+            Z_TYPE_PP(container) == IS_OBJECT &&
+            Z_OBJCE_PP(container) != PHP_IC_ENTRY &&
+            zend_hash_exists(
+                &Z_OBJCE_PP(container)->function_table,
+                "__wakeup", sizeof("__wakeup")))
         {
-            ret = MSGPACK_UNPACK_PARSE_ERROR;
+            zval f, *h = NULL;
+
+            INIT_PZVAL(&f);
+            ZVAL_STRINGL(&f, "__wakeup", sizeof("__wakeup") - 1, 0);
+            call_user_function_ex(
+                CG(function_table), container, &f, &h, 0, 0, 1, NULL TSRMLS_CC);
+            if (h)
+            {
+                zval_ptr_dtor(&h);
+            }
         }
     }
 
-    return ret;
+    return 0;
 }
-
-int msgpack_unserialize_zval(
-    zval **return_value, msgpack_unserialize_data *mpsd,
-    php_unserialize_data_t *var_hash TSRMLS_DC)
-{
-    const unsigned char* data = mpsd->data;
-    const unsigned char* const pe = mpsd->data + mpsd->length;
-    const void* n = NULL;
-
-    unsigned int trail = 0;
-    unsigned int cs = CS_HEADER;
-
-    int ret;
-    unsigned int ct;
-
-#define next_cs(p) ((unsigned int)*p & 0x1f)
-
-#define finish_zval_long(val_) \
-    msgpack_var_push(var_hash, return_value); \
-    ZVAL_LONG(*return_value, val_); \
-    goto _finish
-
-#define finish_zval_null() \
-    msgpack_var_push(var_hash, return_value); \
-    ZVAL_NULL(*return_value); \
-    goto _finish
-
-#define finish_zval_bool(val_) \
-    msgpack_var_push(var_hash, return_value); \
-    ZVAL_BOOL(*return_value, val_); \
-    goto _finish
-
-#define finish_zval_double(val_) \
-    msgpack_var_push(var_hash, return_value); \
-    ZVAL_DOUBLE(*return_value, val_); \
-    goto _finish
-
-#define finish_zval_string(val_, len_) \
-    msgpack_var_push(var_hash, return_value); \
-    if (len_ == 0) { ZVAL_STRINGL(*return_value, "", 0, 1); } \
-    else { ZVAL_STRINGL(*return_value, val_, len_, 1); } \
-    goto _finish
-
-#define finish_zval_array(count_) \
-    ct = count_; \
-    cs = CS_HEADER; \
-    ++(mpsd->data); \
-    mpsd->length = (pe - mpsd->data); \
-    if (msgpack_unserialize_array(return_value, mpsd, ct, var_hash TSRMLS_CC) <= 0) { goto _failed; } \
-    goto _finish_end
-
-#define finish_zval_object(count_) \
-    ct = count_; \
-    cs = CS_HEADER; \
-    ++(mpsd->data); \
-    mpsd->length = (pe - mpsd->data); \
-    if (msgpack_unserialize_object(return_value, mpsd, ct, var_hash TSRMLS_CC) <= 0) { goto _failed; } \
-    goto _finish_end
-
-#define again_fixed_trail(_cs, trail_len) \
-    trail = trail_len; \
-    cs = _cs; \
-    goto _fixed_trail_again
-
-#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \
-    trail = trail_len; \
-    if(trail == 0) { goto ifzero; } \
-    cs = _cs; \
-    goto _fixed_trail_again
-
-    if (mpsd->length <= 0)
-    {
-        goto _failed;
-    }
-
-    if (mpsd->data == pe)
-    {
-        goto _out;
-    }
-    do
-    {
-        switch (cs)
-        {
-            case CS_HEADER:
-                switch (*(mpsd->data))
-                {
-                    case 0x00 ... 0x7f: /* Positive Fixnum */
-                        finish_zval_long(*(uint8_t*)mpsd->data);
-                    case 0xe0 ... 0xff: /* Negative Fixnum */
-                        finish_zval_long(*(int8_t*)mpsd->data);
-                    case 0xc0 ... 0xdf: /* Variable */
-                        switch (*(mpsd->data))
-                        {
-                            case 0xc0:  /* nil */
-                                finish_zval_null();
-                            case 0xc2:  /* false */
-                                finish_zval_bool(0);
-                            case 0xc3:  /* true */
-                                finish_zval_bool(1);
-                            case 0xca:  /* float */
-                            case 0xcb:  /* double */
-                            case 0xcc:  /* unsigned int  8 */
-                            case 0xcd:  /* unsigned int 16 */
-                            case 0xce:  /* unsigned int 32 */
-                            case 0xcf:  /* unsigned int 64 */
-                            case 0xd0:  /* signed int  8 */
-                            case 0xd1:  /* signed int 16 */
-                            case 0xd2:  /* signed int 32 */
-                            case 0xd3:  /* signed int 64 */
-                                again_fixed_trail(
-                                    next_cs(mpsd->data),
-                                    1 << (((unsigned int)*(mpsd->data)) & 0x03));
-                            case 0xda:  /* raw 16 */
-                            case 0xdb:  /* raw 32 */
-                            case 0xdc:  /* array 16 */
-                            case 0xdd:  /* array 32 */
-                            case 0xde:  /* map 16 */
-                            case 0xdf:  /* map 32 */
-                                again_fixed_trail(
-                                    next_cs(mpsd->data),
-                                    2 << (((unsigned int)*(mpsd->data)) & 0x01));
-                            default:
-                                goto _failed;
-                        }
-                    case 0xa0 ... 0xbf: /* FixRaw */
-                        again_fixed_trail_if_zero(
-                            ACS_RAW_VALUE, (unsigned int)*(mpsd->data) & 0x1f,
-                            _raw_zero);
-                    case 0x90 ... 0x9f: /* FixArray */
-                        finish_zval_array(((unsigned int)*(mpsd->data)) & 0x0f);
-                    case 0x80 ... 0x8f: /* FixMap */
-                        finish_zval_object(((unsigned int)*(mpsd->data)) & 0x0f);
-                    default:
-                        goto _failed;
-                }
-_fixed_trail_again:
-                ++(mpsd->data);
-            default:
-                if ((size_t)(pe - (mpsd->data)) < trail)
-                {
-                    goto _out;
-                }
-                n = (mpsd->data);
-                mpsd->data += trail - 1;
-                switch (cs)
-                {
-                    case CS_FLOAT:
-                    {
-                        union { uint32_t i; float f; } mem;
-                        mem.i = _msgpack_load32(uint32_t, n);
-                        finish_zval_double(mem.f);
-                    }
-                    case CS_DOUBLE:
-                    {
-                        union { uint64_t i; double f; } mem;
-                        mem.i = _msgpack_load64(uint64_t, n);
-                        finish_zval_double(mem.f);
-                    }
-                    case CS_UINT_8:
-                        finish_zval_long(*(uint8_t*)n);
-                    case CS_UINT_16:
-                        finish_zval_long(_msgpack_load16(uint16_t, n));
-                    case CS_UINT_32:
-                        finish_zval_long(_msgpack_load32(uint32_t, n));
-                    case CS_UINT_64:
-                        finish_zval_long(_msgpack_load64(uint64_t, n));
-                    case CS_INT_8:
-                        finish_zval_long(*(int8_t*)n);
-                    case CS_INT_16:
-                        finish_zval_long(_msgpack_load16(int16_t, n));
-                    case CS_INT_32:
-                        finish_zval_long(_msgpack_load32(int32_t, n));
-                    case CS_INT_64:
-                        finish_zval_long(_msgpack_load64(int64_t, n));
-                    case CS_RAW_16:
-                        again_fixed_trail_if_zero(
-                            ACS_RAW_VALUE, _msgpack_load16(uint16_t, n),
-                            _raw_zero);
-                    case CS_RAW_32:
-                        again_fixed_trail_if_zero(
-                            ACS_RAW_VALUE, _msgpack_load32(uint32_t, n),
-                            _raw_zero);
-                    case ACS_RAW_VALUE:
-_raw_zero:
-                        finish_zval_string(n, trail);
-                    case CS_ARRAY_16:
-                        finish_zval_array(_msgpack_load16(uint16_t, n));
-                    case CS_ARRAY_32:
-                        finish_zval_array(_msgpack_load32(uint32_t, n));
-                    /* FIXME security guard */
-                    case CS_MAP_16:
-                        finish_zval_object(_msgpack_load16(uint16_t, n));
-                    case CS_MAP_32:
-                        finish_zval_object(_msgpack_load32(uint32_t, n));
-                    /* FIXME security guard */
-                    default:
-                        goto _failed;
-                }
-        }
-        cs = CS_HEADER;
-        ++(mpsd->data);
-    }
-    while (mpsd->data != pe);
-    goto _out;
-
-_finish:
-    ++(mpsd->data);
-_finish_end:
-    ret = MSGPACK_UNPACK_EXTRA_BYTES;
-    goto _end;
-
-_failed:
-    ret = MSGPACK_UNPACK_PARSE_ERROR;
-    goto _end;
-
-_out:
-    ret = MSGPACK_UNPACK_CONTINUE;
-    goto _end;
-
-_end:
-    mpsd->offset = mpsd->data - data;
-    mpsd->length = pe - mpsd->data;
-
-    if (ret == MSGPACK_UNPACK_EXTRA_BYTES && mpsd->length == 0)
-    {
-        ret = MSGPACK_UNPACK_SUCCESS;
-    }
-
-    return ret;
-}
-
-#undef finish_zval_long
-#undef finish_zval_null
-#undef finish_zval_bool
-#undef finish_zval_double
-#undef finish_zval_string
-#undef finish_zval_array
-#undef finish_zval_object
-#undef again_fixed_trail
-#undef again_fixed_trail_if_zero
-
-#undef next_cs
diff --git a/php/msgpack_unpack.h b/php/msgpack_unpack.h
index 6c85d73..da963eb 100644
--- a/php/msgpack_unpack.h
+++ b/php/msgpack_unpack.h
@@ -1,9 +1,13 @@
 
-#ifndef MSGPACL_UNPACK_H
-#define MSGPACL_UNPACK_H
+#ifndef MSGPACK_UNPACK_H
+#define MSGPACK_UNPACK_H
 
 #include "ext/standard/php_var.h"
 
+#define MSGPACK_EMBED_STACK_SIZE 1024
+
+#include "msgpack/unpack_define.h"
+
 typedef enum
 {
     MSGPACK_UNPACK_SUCCESS     =  2,
@@ -12,15 +16,114 @@ typedef enum
     MSGPACK_UNPACK_PARSE_ERROR = -1,
 } msgpack_unpack_return;
 
-typedef struct
-{
-    unsigned char *data;
-    size_t length;
-    size_t offset;
+typedef struct {
+    zval *retval;
+    long deps;
+    php_unserialize_data_t *var_hash;
+    long stack[MSGPACK_EMBED_STACK_SIZE];
+    int type;
 } msgpack_unserialize_data;
 
-int msgpack_unserialize_zval(
-    zval **return_value, msgpack_unserialize_data *mpsd,
-    php_unserialize_data_t *var_hash TSRMLS_DC);
+void msgpack_unserialize_var_init(php_unserialize_data_t *var_hashx);
+void msgpack_unserialize_var_destroy(php_unserialize_data_t *var_hashx);
+
+void msgpack_unserialize_init(msgpack_unserialize_data *unpack);
+
+int msgpack_unserialize_uint8(
+    msgpack_unserialize_data *unpack, uint8_t data, zval **obj);
+int msgpack_unserialize_uint16(
+    msgpack_unserialize_data *unpack, uint16_t data, zval **obj);
+int msgpack_unserialize_uint32(
+    msgpack_unserialize_data *unpack, uint32_t data, zval **obj);
+int msgpack_unserialize_uint64(
+    msgpack_unserialize_data *unpack, uint64_t data, zval **obj);
+int msgpack_unserialize_int8(
+    msgpack_unserialize_data *unpack, int8_t data, zval **obj);
+int msgpack_unserialize_int16(
+    msgpack_unserialize_data *unpack, int16_t data, zval **obj);
+int msgpack_unserialize_int32(
+    msgpack_unserialize_data *unpack, int32_t data, zval **obj);
+int msgpack_unserialize_int64(
+    msgpack_unserialize_data *unpack, int64_t data, zval **obj);
+int msgpack_unserialize_float(
+    msgpack_unserialize_data *unpack, float data, zval **obj);
+int msgpack_unserialize_double(
+    msgpack_unserialize_data *unpack, double data, zval **obj);
+int msgpack_unserialize_nil(msgpack_unserialize_data *unpack, zval **obj);
+int msgpack_unserialize_true(msgpack_unserialize_data *unpack, zval **obj);
+int msgpack_unserialize_false(msgpack_unserialize_data *unpack, zval **obj);
+int msgpack_unserialize_raw(
+    msgpack_unserialize_data *unpack, const char* base, const char* data,
+    unsigned int len, zval **obj);
+int msgpack_unserialize_array(
+    msgpack_unserialize_data *unpack, unsigned int count, zval **obj);
+int msgpack_unserialize_array_item(
+    msgpack_unserialize_data *unpack, zval **container, zval *obj);
+int msgpack_unserialize_map(
+    msgpack_unserialize_data *unpack, unsigned int count, zval **obj);
+int msgpack_unserialize_map_item(
+    msgpack_unserialize_data *unpack, zval **container, zval *key, zval *val);
+
+/* template functions */
+#define msgpack_unpack_struct(name)    struct template ## name
+#define msgpack_unpack_func(ret, name) ret template ## name
+#define msgpack_unpack_callback(name)  template_callback ## name
+
+#define msgpack_unpack_object zval*
+#define unpack_user           msgpack_unserialize_data
+#define msgpack_unpack_user   msgpack_unserialize_data
+
+struct template_context;
+typedef struct template_context msgpack_unpack_t;
+
+static void template_init(msgpack_unpack_t* unpack);
+static msgpack_unpack_object template_data(msgpack_unpack_t* unpack);
+static int template_execute(
+    msgpack_unpack_t* unpack, const char* data, size_t len, size_t* off);
+
+static inline msgpack_unpack_object template_callback_root(unpack_user* user)
+{
+    msgpack_unserialize_init(user);
+    return NULL;
+}
+
+#define template_callback_uint8(user, data, obj) \
+    msgpack_unserialize_uint8(user, data, obj)
+#define template_callback_uint16(user, data, obj) \
+    msgpack_unserialize_uint16(user, data, obj)
+#define template_callback_uint32(user, data, obj) \
+    msgpack_unserialize_uint32(user, data, obj)
+#define template_callback_uint64(user, data, obj) \
+    msgpack_unserialize_uint64(user, data, obj)
+#define template_callback_int8(user, data, obj) \
+    msgpack_unserialize_int8(user, data, obj)
+#define template_callback_int16(user, data, obj) \
+    msgpack_unserialize_int16(user, data, obj)
+#define template_callback_int32(user, data, obj) \
+    msgpack_unserialize_int32(user, data, obj)
+#define template_callback_int64(user, data, obj) \
+    msgpack_unserialize_int64(user, data, obj)
+#define template_callback_float(user, data, obj) \
+    msgpack_unserialize_float(user, data, obj)
+#define template_callback_double(user, data, obj) \
+    msgpack_unserialize_double(user, data, obj)
+#define template_callback_nil(user, obj) \
+    msgpack_unserialize_nil(user, obj)
+#define template_callback_true(user, obj) \
+    msgpack_unserialize_true(user, obj)
+#define template_callback_false(user, obj) \
+    msgpack_unserialize_false(user, obj)
+#define template_callback_raw(user, base, data, len, obj) \
+    msgpack_unserialize_raw(user, base, data, len, obj)
+#define template_callback_array(user, count, obj) \
+    msgpack_unserialize_array(user, count, obj)
+#define template_callback_array_item(user, container, obj) \
+    msgpack_unserialize_array_item(user, container, obj)
+#define template_callback_map(user, count, obj) \
+    msgpack_unserialize_map(user, count, obj)
+#define template_callback_map_item(user, container, key, val) \
+    msgpack_unserialize_map_item(user, container, key, val)
+
+#include "msgpack/unpack_template.h"
 
 #endif
diff --git a/php/package.xml b/php/package.xml
index 4869770..6172a4c 100644
--- a/php/package.xml
+++ b/php/package.xml
@@ -10,11 +10,11 @@
     advect@gmail.com
     yes
   
-  2010-07-17
-  
+  2010-09-28
+  
   
-    0.2.1
-    0.2.1
+    0.3.0
+    0.3.0
   
   
     beta
@@ -26,15 +26,21 @@
     
       
       
-      
-      
-      
-      
-      
-      
-      
-      
-      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
       
       
       
@@ -43,28 +49,32 @@
       
       
       
-      
+      
+      
       
       
       
-      
-      
-      
+      
+      
+      
       
       
-      
+      
       
       
       
       
       
-      
+      
+      
       
-      
-      
-      
-      
-      
+      
+      
+      
+      
+      
+      
+      
       
       
       
@@ -74,13 +84,30 @@
       
       
       
-      
-      
+      
+      
+      
+      
       
       
-      
-      
-      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
     
   
   
diff --git a/php/php-msgpack.spec b/php/php-msgpack.spec
index 5726c43..6a98288 100644
--- a/php/php-msgpack.spec
+++ b/php/php-msgpack.spec
@@ -3,7 +3,7 @@
 
 Summary: PHP extension for interfacing with MessagePack
 Name: php-msgpack
-Version: 0.2.1
+Version: 0.3.0
 Release: 1%{?dist}
 Source: php-msgpack-%{version}.tar.gz
 License: New BSD License
@@ -11,7 +11,7 @@ Group: Development/Libraries
 Packager: advect 
 Provides: php-pecl-msgpack
 BuildRoot: %{_tmppath}/%{name}-%{version}-root
-BuildRequires: php-devel, msgpack-devel
+BuildRequires: php-devel
 Requires: msgpack
 %if 0%{?php_zend_api}
 Requires: php(zend-abi) = %{php_zend_api}
diff --git a/php/php_msgpack.h b/php/php_msgpack.h
index 91b2d30..61badea 100644
--- a/php/php_msgpack.h
+++ b/php/php_msgpack.h
@@ -2,7 +2,7 @@
 #ifndef PHP_MSGPACK_H
 #define PHP_MSGPACK_H
 
-#define MSGPACK_VERSION "0.2.1"
+#define MSGPACK_EXTENSION_VERSION "0.3.0"
 
 #include "ext/standard/php_smart_str.h"
 
@@ -23,6 +23,7 @@ extern zend_module_entry msgpack_module_entry;
 
 ZEND_BEGIN_MODULE_GLOBALS(msgpack)
     zend_bool error_display;
+    zend_bool php_only;
 ZEND_END_MODULE_GLOBALS(msgpack)
 
 ZEND_DECLARE_MODULE_GLOBALS(msgpack)
diff --git a/php/tests/009.phpt b/php/tests/009.phpt
index 6baad5e..9992988 100644
--- a/php/tests/009.phpt
+++ b/php/tests/009.phpt
@@ -1,6 +1,10 @@
 --TEST--
-Check for reference serialisation
+Check for reference serialization
 --SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 and lower";
+}
 --FILE--
 
   &array(1) {
@@ -63,16 +67,16 @@ array(2) {
 }
 OK
 cyclic
-81008100810082c001c002
+810082c0010082c0010082c0020002
 array(1) {
   [0]=>
   &array(1) {
     [0]=>
-    array(1) {
+    &array(1) {
       [0]=>
       &array(1) {
         [0]=>
-        array(1) {
+        &array(1) {
           [0]=>
           *RECURSION*
         }
@@ -101,11 +105,11 @@ array(1) {
   [0]=>
   &array(1) {
     [0]=>
-    array(1) {
+    &array(1) {
       [0]=>
       &array(1) {
         [0]=>
-        array(1) {
+        &array(1) {
           [0]=>
           *RECURSION*
         }
diff --git a/php/tests/009b.phpt b/php/tests/009b.phpt
new file mode 100644
index 0000000..7765d62
--- /dev/null
+++ b/php/tests/009b.phpt
@@ -0,0 +1,101 @@
+--TEST--
+Check for reference serialization
+--SKIPIF--
+
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(&$a, &$a)
+820082c00100a3666f6f0182c0020002
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+cyclic
+810082c0010082c0010082c0020002
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      *RECURSION*
+    }
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      *RECURSION*
+    }
+  }
+}
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      *RECURSION*
+    }
+  }
+}
diff --git a/php/tests/014.phpt b/php/tests/014.phpt
index c663579..b9c7c67 100644
--- a/php/tests/014.phpt
+++ b/php/tests/014.phpt
@@ -34,7 +34,7 @@ test('object', $a, false);
 ?>
 --EXPECTF--
 object
-820083c0a34f626aa16101a162020182c001c002
+820084c001c0a34f626aa16101a162020182c0020002
 array(2) {
   [0]=>
   &object(Obj)#%d (2) {
diff --git a/php/tests/015.phpt b/php/tests/015.phpt
index 828800a..634a8b1 100644
--- a/php/tests/015.phpt
+++ b/php/tests/015.phpt
@@ -51,7 +51,7 @@ var_dump($_SESSION);
 ?>
 --EXPECT--
 2
-81a3666f6f02
+82c001a3666f6f02
 array(1) {
   ["foo"]=>
   int(2)
diff --git a/php/tests/015b.phpt b/php/tests/015b.phpt
index 829584b..7ced312 100644
--- a/php/tests/015b.phpt
+++ b/php/tests/015b.phpt
@@ -51,7 +51,7 @@ var_dump($_SESSION);
 ?>
 --EXPECT--
 2
-81a3666f6f02
+82c001a3666f6f02
 array(1) {
   ["foo"]=>
   int(2)
diff --git a/php/tests/018.phpt b/php/tests/018.phpt
index 84ccafa..f5ff4b7 100644
--- a/php/tests/018.phpt
+++ b/php/tests/018.phpt
@@ -74,7 +74,7 @@ object(Obj)#%d (3) {
 }
 OK
 wrong
-82c0a34f706ac0
+82c0a34f706a
 object(Opj)#%d (2) {
   ["a"]=>
   NULL
diff --git a/php/tests/024.phpt b/php/tests/024.phpt
index 30fab75..9b185f7 100644
--- a/php/tests/024.phpt
+++ b/php/tests/024.phpt
@@ -1,6 +1,10 @@
 --TEST--
 Recursive objects
 --SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 and lower";
+}
 --FILE--
 
   int(100)
diff --git a/php/tests/024b.phpt b/php/tests/024b.phpt
new file mode 100644
index 0000000..7c691be
--- /dev/null
+++ b/php/tests/024b.phpt
@@ -0,0 +1,165 @@
+--TEST--
+Recursive objects
+--SKIPIF--
+a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+class Obj2 {
+    public $aa;
+    protected $bb;
+    private $cc;
+    private $obj;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+
+        $this->obj = new Obj($a, $b, $c);
+    }
+}
+
+class Obj3 {
+    private $objs;
+
+    function __construct($a, $b, $c) {
+        $this->objs = array();
+
+        for ($i = $a; $i < $c; $i += $b) {
+            $this->objs[] = new Obj($a, $i, $c);
+        }
+    }
+}
+
+class Obj4 {
+    private $a;
+    private $obj;
+
+    function __construct($a) {
+        $this->a = $a;
+    }
+
+    public function set($obj) {
+        $this->obj = $obj;
+    }
+}
+
+$o2 = new Obj2(1, 2, 3);
+test('objectrec', $o2, false);
+
+$o3 = new Obj3(0, 1, 4);
+test('objectrecarr', $o3, false);
+
+$o4 = new Obj4(100);
+$o4->set($o4);
+test('objectselfrec', $o4, true);
+?>
+--EXPECTF--
+objectrec
+88c0a44f626a32a26161c0a5002a006262c0a8004f626a32006363c0a9004f626a32006f626a84c0a34f626aa16101a4002a006202a6004f626a006303a16101a16202a16303
+object(Obj2)#%d (7) {
+  ["aa"]=>
+  NULL
+  [%r"?bb"?:protected"?%r]=>
+  NULL
+  [%r"?cc"?:("Obj2":)?private"?%r]=>
+  NULL
+  [%r"?obj"?:("Obj2":)?private"?%r]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  ["a"]=>
+  int(1)
+  ["b"]=>
+  int(2)
+  ["c"]=>
+  int(3)
+}
+OK
+objectrecarr
+82c0a44f626a33aa004f626a33006f626a73840084c0a34f626aa16100a4002a006200a6004f626a0063040184c0a34f626aa16100a4002a006201a6004f626a0063040284c0a34f626aa16100a4002a006202a6004f626a0063040384c0a34f626aa16100a4002a006203a6004f626a006304
+object(Obj3)#%d (1) {
+  [%r"?objs"?:("Obj3":)?private"?%r]=>
+  array(4) {
+    [0]=>
+    object(Obj)#%d (3) {
+      ["a"]=>
+      int(0)
+      [%r"?b"?:protected"?%r]=>
+      int(0)
+      [%r"?c"?:("Obj":)?private"?%r]=>
+      int(4)
+    }
+    [1]=>
+    object(Obj)#%d (3) {
+      ["a"]=>
+      int(0)
+      [%r"?b"?:protected"?%r]=>
+      int(1)
+      [%r"?c"?:("Obj":)?private"?%r]=>
+      int(4)
+    }
+    [2]=>
+    object(Obj)#%d (3) {
+      ["a"]=>
+      int(0)
+      [%r"?b"?:protected"?%r]=>
+      int(2)
+      [%r"?c"?:("Obj":)?private"?%r]=>
+      int(4)
+    }
+    [3]=>
+    object(Obj)#%d (3) {
+      ["a"]=>
+      int(0)
+      [%r"?b"?:protected"?%r]=>
+      int(3)
+      [%r"?c"?:("Obj":)?private"?%r]=>
+      int(4)
+    }
+  }
+}
+OK
+objectselfrec
+83c0a44f626a34a7004f626a34006164a9004f626a34006f626a82c0020001
+object(Obj4)#%d (2) {
+  [%r"?a"?:("Obj4":)?private"?%r]=>
+  int(100)
+  [%r"?obj"?:("Obj4":)?private"?%r]=>
+  *RECURSION*
+}
+OK
diff --git a/php/tests/026.phpt b/php/tests/026.phpt
index 2d3bb73..c243ef5 100644
--- a/php/tests/026.phpt
+++ b/php/tests/026.phpt
@@ -2,6 +2,10 @@
 Cyclic array test
 --INI--
 --SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 and lower";
+}
 --FILE--
 
 --EXPECT--
 array
-82a16182a162a163a164a165a16682a16182a162a163a164a165a16682c001c005
+82a16182a162a163a164a165a16683c001a16182a162a163a164a165a16682c0020005
 array(2) {
   ["a"]=>
   array(2) {
diff --git a/php/tests/026b.phpt b/php/tests/026b.phpt
new file mode 100644
index 0000000..f7a3381
--- /dev/null
+++ b/php/tests/026b.phpt
@@ -0,0 +1,107 @@
+--TEST--
+Cyclic array test
+--INI--
+--SKIPIF--
+ array(
+        'b' => 'c',
+        'd' => 'e'
+    ),
+);
+
+$a['f'] = &$a;
+
+test('array', $a, true);
+
+$a = array("foo" => &$b);
+$b = array(1, 2, $a);
+var_dump($a);
+var_dump($k = msgpack_unserialize(msgpack_serialize($a)));
+
+$k["foo"][1] = "b";
+var_dump($k);
+?>
+--EXPECT--
+array
+82a16182a162a163a164a165a16683c001a16182a162a163a164a165a16682c0020005
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  &array(2) {
+    ["a"]=>
+    array(2) {
+      ["b"]=>
+      string(1) "c"
+      ["d"]=>
+      string(1) "e"
+    }
+    ["f"]=>
+    *RECURSION*
+  }
+}
+OK
+array(1) {
+  ["foo"]=>
+  &array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    *RECURSION*
+  }
+}
+array(1) {
+  ["foo"]=>
+  &array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    array(1) {
+      ["foo"]=>
+      *RECURSION*
+    }
+  }
+}
+array(1) {
+  ["foo"]=>
+  &array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    string(1) "b"
+    [2]=>
+    array(1) {
+      ["foo"]=>
+      *RECURSION*
+    }
+  }
+}
diff --git a/php/tests/027.phpt b/php/tests/027.phpt
index ecb8784..c9c7cbd 100644
--- a/php/tests/027.phpt
+++ b/php/tests/027.phpt
@@ -64,7 +64,7 @@ var_dump($_SESSION);
 --EXPECT--
 bool(true)
 read
-wrote: 82a3666f6f01a474657374a6666f6f626172
+wrote: 83c001a3666f6f01a474657374a6666f6f626172
 array(2) {
   ["foo"]=>
   int(1)
diff --git a/php/tests/028.phpt b/php/tests/028.phpt
index 56805f9..00db675 100644
--- a/php/tests/028.phpt
+++ b/php/tests/028.phpt
@@ -1,6 +1,10 @@
 --TEST--
 Serialize object into session, full set
 --SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 and lower";
+}
 --FILE--
 
 --EXPECTF--
 read
-write: 83a36f6c6484c0a3466f6fa700466f6f00643184c0a3426172a2643182c002c002a70042617200643282c002c002a5002a00643382c002c002a5002a00643282c002c003a2643382c002c003a474657374a6666f6f626172a36e657784c0a3426172a2643184c0a3466f6fa700466f6f00643182c002c00aa5002a00643282c002c00aa2643382c002c00aa70042617200643282c002c00ba5002a00643382c002c00b
+write: 84c001a36f6c6484c0a3466f6fa700466f6f00643184c0a3426172a2643182c0020002a70042617200643282c0020002a5002a00643382c0020002a5002a00643282c0020003a2643382c0020003a474657374a6666f6f626172a36e657784c0a3426172a2643184c0a3466f6fa700466f6f00643182c002000aa5002a00643282c002000aa2643382c002000aa70042617200643282c002000ba5002a00643382c002000b
 array(3) {
   ["old"]=>
   object(Foo)#3 (3) {
diff --git a/php/tests/028b.phpt b/php/tests/028b.phpt
new file mode 100644
index 0000000..7331a57
--- /dev/null
+++ b/php/tests/028b.phpt
@@ -0,0 +1,167 @@
+--TEST--
+Serialize object into session, full set
+--SKIPIF--
+d1 = $foo;
+        $this->d2 = $foo;
+        $this->d3 = $foo;
+    }
+}
+
+class Bar {
+    private static $s1 = array();
+    protected static $s2 = array();
+    public static $s3 = array();
+
+    public $d1;
+    private $d2;
+    protected $d3;
+
+    public function __construct() {
+    }
+
+    public function set($foo) {
+        $this->d1 = $foo;
+        $this->d2 = $foo;
+        $this->d3 = $foo;
+    }
+}
+
+$output = '';
+
+function open($path, $name) {
+    return true;
+}
+
+function close() {
+    return true;
+}
+
+function read($id) {
+    global $output;
+    $output .= "read" . PHP_EOL;
+    $a = new Bar();
+    $b = new Foo($a);
+    $a->set($b);
+    $session = array('old' => $b);
+    return msgpack_serialize($session);
+}
+
+function write($id, $data) {
+    global $output;
+    $output .= "write: ";
+    $output .= bin2hex($data) . PHP_EOL;
+    return true;
+}
+
+function destroy($id) {
+    return true;
+}
+
+function gc($time) {
+    return true;
+}
+
+ini_set('session.serialize_handler', 'msgpack');
+
+session_set_save_handler('open', 'close', 'read', 'write', 'destroy', 'gc');
+
+session_start();
+
+$_SESSION['test'] = "foobar";
+$a = new Bar();
+$b = new Foo($a);
+$a->set($b);
+$_SESSION['new'] = $a;
+
+session_write_close();
+
+echo $output;
+var_dump($_SESSION);
+?>
+--EXPECTF--
+read
+write: 84c001a36f6c6484c0a3466f6fa700466f6f00643184c0a3426172a2643182c0020002a70042617200643282c0020002a5002a00643382c0020002a5002a00643282c0020003a2643382c0020003a474657374a6666f6f626172a36e657784c0a3426172a2643184c0a3466f6fa700466f6f00643182c002000aa5002a00643282c002000aa2643382c002000aa70042617200643282c002000ba5002a00643382c002000b
+array(3) {
+  ["old"]=>
+  object(Foo)#3 (3) {
+    [%r"?d1"?:("Foo":)?private"?%r]=>
+    object(Bar)#4 (3) {
+      ["d1"]=>
+      *RECURSION*
+      [%r"?d2"?:("Bar":)?private"?%r]=>
+      *RECURSION*
+      [%r"?d3"?:protected"?%r]=>
+      *RECURSION*
+    }
+    [%r"?d2"?:protected"?%r]=>
+    object(Bar)#4 (3) {
+      ["d1"]=>
+      *RECURSION*
+      [%r"?d2"?:("Bar":)?private"?%r]=>
+      *RECURSION*
+      [%r"?d3"?:protected"?%r]=>
+      *RECURSION*
+    }
+    ["d3"]=>
+    object(Bar)#4 (3) {
+      ["d1"]=>
+      *RECURSION*
+      [%r"?d2"?:("Bar":)?private"?%r]=>
+      *RECURSION*
+      [%r"?d3"?:protected"?%r]=>
+      *RECURSION*
+    }
+  }
+  ["test"]=>
+  string(6) "foobar"
+  ["new"]=>
+  object(Bar)#5 (3) {
+    ["d1"]=>
+    object(Foo)#6 (3) {
+      [%r"?d1"?:("Foo":)?private"?%r]=>
+      *RECURSION*
+      [%r"?d2"?:protected"?%r]=>
+      *RECURSION*
+      ["d3"]=>
+      *RECURSION*
+    }
+    [%r"?d2"?:("Bar":)?private"?%r]=>
+    object(Foo)#6 (3) {
+      [%r"?d1"?:("Foo":)?private"?%r]=>
+      *RECURSION*
+      [%r"?d2"?:protected"?%r]=>
+      *RECURSION*
+      ["d3"]=>
+      *RECURSION*
+    }
+    [%r"?d3"?:protected"?%r]=>
+    object(Foo)#6 (3) {
+      [%r"?d1"?:("Foo":)?private"?%r]=>
+      *RECURSION*
+      [%r"?d2"?:protected"?%r]=>
+      *RECURSION*
+      ["d3"]=>
+      *RECURSION*
+    }
+  }
+}
diff --git a/php/tests/029.phpt b/php/tests/029.phpt
index 3bd1e9d..686abb9 100644
--- a/php/tests/029.phpt
+++ b/php/tests/029.phpt
@@ -9,12 +9,39 @@ phpinfo(INFO_MODULES);
 $str = ob_get_clean();
 
 $array = explode("\n", $str);
-$array = preg_grep('/^msgpack/', $array);
 
-echo implode("\n", $array), PHP_EOL;
+$section = false;
+$blank = 0;
+foreach ($array as $key => $val)
+{
+    if (strcmp($val, 'msgpack') == 0 || $section)
+    {
+        $section = true;
+    }
+    else
+    {
+        continue;
+    }
 
+    if (empty($val))
+    {
+        $blank++;
+        if ($blank == 3)
+        {
+            $section = false;
+        }
+    }
+
+    echo $val, PHP_EOL;
+}
 --EXPECTF--
 msgpack
-msgpack support => enabled
-msgpack version => %s
-msgpack Session Support => enabled
+
+MessagePack Support => enabled
+Session Support => enabled
+extension Version => %s
+header Version => %s
+
+Directive => Local Value => Master Value
+msgpack.error_display => On => On
+msgpack.php_only => On => On
diff --git a/php/tests/030.phpt b/php/tests/030.phpt
index ec8622d..3b8d986 100644
--- a/php/tests/030.phpt
+++ b/php/tests/030.phpt
@@ -56,9 +56,6 @@ foreach ($datas as $data) {
 }
 ?>
 --EXPECTF--
-output at 2:
-array(0) {
-}
 output at 3:
 array(1) {
   [0]=>
diff --git a/php/tests/060.phpt b/php/tests/060.phpt
index c0fda4c..649d788 100644
--- a/php/tests/060.phpt
+++ b/php/tests/060.phpt
@@ -1,6 +1,10 @@
 --TEST--
 Check for buffered streaming unserialization
 --SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 and lower";
+}
 --FILE--
  1, "two" => 2))', array("one" => 1, "two" => 2), false);
@@ -234,11 +238,11 @@ array(1) {
   [0]=>
   &array(1) {
     [0]=>
-    array(1) {
+    &array(1) {
       [0]=>
       &array(1) {
         [0]=>
-        array(1) {
+        &array(1) {
           [0]=>
           *RECURSION*
         }
diff --git a/php/tests/060b.phpt b/php/tests/060b.phpt
new file mode 100644
index 0000000..0b947c8
--- /dev/null
+++ b/php/tests/060b.phpt
@@ -0,0 +1,313 @@
+--TEST--
+Check for buffered streaming unserialization
+--SKIPIF--
+feed($str);
+        if ($unpacker->execute())
+        {
+            $unserialized = $unpacker->data();
+            var_dump($unserialized);
+            $unpacker->reset();
+        }
+
+        $i += $len;
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('bool: true', true);
+test('bool: false', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('array', array(), false);
+test('array(1, 2, 3)', array(1, 2, 3), false);
+test('array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      *RECURSION*
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/061.phpt b/php/tests/061.phpt
index ded494e..7ecc0c9 100644
--- a/php/tests/061.phpt
+++ b/php/tests/061.phpt
@@ -1,6 +1,10 @@
 --TEST--
 Check for unbuffered streaming unserialization
 --SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 and lower";
+}
 --FILE--
 
   &array(1) {
     [0]=>
-    array(1) {
+    &array(1) {
       [0]=>
       &array(1) {
         [0]=>
-        array(1) {
+        &array(1) {
           [0]=>
           *RECURSION*
         }
diff --git a/php/tests/061b.phpt b/php/tests/061b.phpt
new file mode 100644
index 0000000..efb3a9b
--- /dev/null
+++ b/php/tests/061b.phpt
@@ -0,0 +1,318 @@
+--TEST--
+Check for unbuffered streaming unserialization
+--SKIPIF--
+execute($str, $offset))
+        {
+            $unserialized = $unpacker->data();
+            var_dump($unserialized);
+
+            $unpacker->reset();
+            $str = "";
+            $offset = 0;
+        }
+
+        $i += $len;
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      *RECURSION*
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/070.phpt b/php/tests/070.phpt
index 4a00ddf..893023b 100644
--- a/php/tests/070.phpt
+++ b/php/tests/070.phpt
@@ -1,6 +1,10 @@
 --TEST--
 Check for alias functions
 --SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 and lower";
+}
 --FILE--
 
   &array(1) {
     [0]=>
-    array(1) {
+    &array(1) {
       [0]=>
       &array(1) {
         [0]=>
-        array(1) {
+        &array(1) {
           [0]=>
           *RECURSION*
         }
diff --git a/php/tests/070b.phpt b/php/tests/070b.phpt
new file mode 100644
index 0000000..109ddc1
--- /dev/null
+++ b/php/tests/070b.phpt
@@ -0,0 +1,297 @@
+--TEST--
+Check for alias functions
+--SKIPIF--
+ 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      *RECURSION*
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/071.phpt b/php/tests/071.phpt
index cca351a..431303b 100644
--- a/php/tests/071.phpt
+++ b/php/tests/071.phpt
@@ -1,6 +1,10 @@
 --TEST--
 Check for class methods
 --SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 and lower";
+}
 --FILE--
 
   &array(1) {
     [0]=>
-    array(1) {
+    &array(1) {
       [0]=>
       &array(1) {
         [0]=>
-        array(1) {
+        &array(1) {
           [0]=>
           *RECURSION*
         }
diff --git a/php/tests/071b.phpt b/php/tests/071b.phpt
new file mode 100644
index 0000000..770d06e
--- /dev/null
+++ b/php/tests/071b.phpt
@@ -0,0 +1,299 @@
+--TEST--
+Check for class methods
+--SKIPIF--
+pack($variable);
+    $unserialized = $msgpack->unpack($serialized);
+
+    var_dump($unserialized);
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      *RECURSION*
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/072.phpt b/php/tests/072.phpt
index d8cb1b9..ea79a56 100644
--- a/php/tests/072.phpt
+++ b/php/tests/072.phpt
@@ -1,6 +1,10 @@
 --TEST--
 Check for class methods unpacker
 --SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 and lower";
+}
 --FILE--
 
   &array(1) {
     [0]=>
-    array(1) {
+    &array(1) {
       [0]=>
       &array(1) {
         [0]=>
-        array(1) {
+        &array(1) {
           [0]=>
           *RECURSION*
         }
diff --git a/php/tests/072b.phpt b/php/tests/072b.phpt
new file mode 100644
index 0000000..c7b0c3c
--- /dev/null
+++ b/php/tests/072b.phpt
@@ -0,0 +1,339 @@
+--TEST--
+Check for class methods unpacker
+--SKIPIF--
+pack($variable);
+    $unpacker = $msgpack->unpacker();
+
+    $length = strlen($serialized);
+
+    if (rand(0, 1))
+    {
+        for ($i = 0; $i < $length;) {
+            $len = rand(1, 10);
+            $str = substr($serialized, $i, $len);
+
+            $unpacker->feed($str);
+            if ($unpacker->execute())
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+                $unpacker->reset();
+            }
+
+            $i += $len;
+        }
+    }
+    else
+    {
+        $str = "";
+        $offset = 0;
+
+        for ($i = 0; $i < $length;) {
+            $len = rand(1, 10);
+            $str .= substr($serialized, $i, $len);
+
+            if ($unpacker->execute($str, $offset))
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+
+                $unpacker->reset();
+                $str = "";
+                $offset = 0;
+            }
+
+            $i += $len;
+        }
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      *RECURSION*
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/073.phpt b/php/tests/073.phpt
new file mode 100644
index 0000000..d3a7637
--- /dev/null
+++ b/php/tests/073.phpt
@@ -0,0 +1,346 @@
+--TEST--
+Check for class unpacker
+--SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 and lower";
+}
+--FILE--
+pack($variable);
+
+    $unpacker = new MessagePackUnpacker();
+
+    $length = strlen($serialized);
+
+    if (rand(0, 1))
+    {
+        for ($i = 0; $i < $length;) {
+            $len = rand(1, 10);
+            $str = substr($serialized, $i, $len);
+
+            $unpacker->feed($str);
+            if ($unpacker->execute())
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+                $unpacker->reset();
+            }
+
+            $i += $len;
+        }
+    }
+    else
+    {
+        $str = "";
+        $offset = 0;
+
+        for ($i = 0; $i < $length;) {
+            $len = rand(1, 10);
+            $str .= substr($serialized, $i, $len);
+
+            if ($unpacker->execute($str, $offset))
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+
+                $unpacker->reset();
+                $str = "";
+                $offset = 0;
+            }
+
+            $i += $len;
+        }
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/073b.phpt b/php/tests/073b.phpt
new file mode 100644
index 0000000..131a534
--- /dev/null
+++ b/php/tests/073b.phpt
@@ -0,0 +1,340 @@
+--TEST--
+Check for class unpacker
+--SKIPIF--
+pack($variable);
+
+    $unpacker = new MessagePackUnpacker();
+
+    $length = strlen($serialized);
+
+    if (rand(0, 1))
+    {
+        for ($i = 0; $i < $length;) {
+            $len = rand(1, 10);
+            $str = substr($serialized, $i, $len);
+
+            $unpacker->feed($str);
+            if ($unpacker->execute())
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+                $unpacker->reset();
+            }
+
+            $i += $len;
+        }
+    }
+    else
+    {
+        $str = "";
+        $offset = 0;
+
+        for ($i = 0; $i < $length;) {
+            $len = rand(1, 10);
+            $str .= substr($serialized, $i, $len);
+
+            if ($unpacker->execute($str, $offset))
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+
+                $unpacker->reset();
+                $str = "";
+                $offset = 0;
+            }
+
+            $i += $len;
+        }
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      *RECURSION*
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/080.phpt b/php/tests/080.phpt
new file mode 100644
index 0000000..aba1cb6
--- /dev/null
+++ b/php/tests/080.phpt
@@ -0,0 +1,301 @@
+--TEST--
+disabled php only (ini_set)
+--SKIPIF--
+--FILE--
+ 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), true);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), true);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), true);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          NULL
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/081.phpt b/php/tests/081.phpt
new file mode 100644
index 0000000..d22daaa
--- /dev/null
+++ b/php/tests/081.phpt
@@ -0,0 +1,303 @@
+--TEST--
+disabled php only for class methods (ini_set)
+--SKIPIF--
+--FILE--
+pack($variable);
+    $unserialized = $msgpack->unpack($serialized);
+
+    var_dump($unserialized);
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), true);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), true);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), true);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          NULL
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/082.phpt b/php/tests/082.phpt
new file mode 100644
index 0000000..45a9d93
--- /dev/null
+++ b/php/tests/082.phpt
@@ -0,0 +1,346 @@
+--TEST--
+disabled php only for class methods unpacker (ini_set)
+--SKIPIF--
+--FILE--
+pack($variable);
+    $unpacker = $msgpack->unpacker();
+
+    $length = strlen($serialized);
+
+    if (rand(0, 1))
+    {
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str = substr($serialized, $i, $len);
+
+            $unpacker->feed($str);
+            if ($unpacker->execute())
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+                $unpacker->reset();
+            }
+
+            $i += $len;
+        }
+    }
+    else
+    {
+        $str = "";
+        $offset = 0;
+
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str .= substr($serialized, $i, $len);
+
+            if ($unpacker->execute($str, $offset))
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+
+                $unpacker->reset();
+                $str = "";
+                $offset = 0;
+            }
+
+            $i += $len;
+        }
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), true);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), true);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), true);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          NULL
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/083.phpt b/php/tests/083.phpt
new file mode 100644
index 0000000..203d829
--- /dev/null
+++ b/php/tests/083.phpt
@@ -0,0 +1,347 @@
+--TEST--
+disabled php only for class unpacker (ini_set)
+--SKIPIF--
+--FILE--
+pack($variable);
+
+    $unpacker = new MessagePackUnpacker();
+
+    $length = strlen($serialized);
+
+    if (rand(0, 1))
+    {
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str = substr($serialized, $i, $len);
+
+            $unpacker->feed($str);
+            if ($unpacker->execute())
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+                $unpacker->reset();
+            }
+
+            $i += $len;
+        }
+    }
+    else
+    {
+        $str = "";
+        $offset = 0;
+
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str .= substr($serialized, $i, $len);
+
+            if ($unpacker->execute($str, $offset))
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+
+                $unpacker->reset();
+                $str = "";
+                $offset = 0;
+            }
+
+            $i += $len;
+        }
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), true);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), true);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), true);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          NULL
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/084.phpt b/php/tests/084.phpt
new file mode 100644
index 0000000..74d061b
--- /dev/null
+++ b/php/tests/084.phpt
@@ -0,0 +1,301 @@
+--TEST--
+disabled php only for class methods (constract)
+--SKIPIF--
+--FILE--
+pack($variable);
+    $unserialized = $msgpack->unpack($serialized);
+
+    var_dump($unserialized);
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), true);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), true);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), true);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          NULL
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/085.phpt b/php/tests/085.phpt
new file mode 100644
index 0000000..72aacc8
--- /dev/null
+++ b/php/tests/085.phpt
@@ -0,0 +1,344 @@
+--TEST--
+disabled php only for class methods unpacker (constract)
+--SKIPIF--
+--FILE--
+pack($variable);
+    $unpacker = $msgpack->unpacker();
+
+    $length = strlen($serialized);
+
+    if (rand(0, 1))
+    {
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str = substr($serialized, $i, $len);
+
+            $unpacker->feed($str);
+            if ($unpacker->execute())
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+                $unpacker->reset();
+            }
+
+            $i += $len;
+        }
+    }
+    else
+    {
+        $str = "";
+        $offset = 0;
+
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str .= substr($serialized, $i, $len);
+
+            if ($unpacker->execute($str, $offset))
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+
+                $unpacker->reset();
+                $str = "";
+                $offset = 0;
+            }
+
+            $i += $len;
+        }
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), true);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), true);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), true);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          NULL
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/086.phpt b/php/tests/086.phpt
new file mode 100644
index 0000000..aeaa332
--- /dev/null
+++ b/php/tests/086.phpt
@@ -0,0 +1,345 @@
+--TEST--
+disabled php only for class unpacker (constract)
+--SKIPIF--
+--FILE--
+pack($variable);
+
+    $unpacker = new MessagePackUnpacker(false);
+
+    $length = strlen($serialized);
+
+    if (rand(0, 1))
+    {
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str = substr($serialized, $i, $len);
+
+            $unpacker->feed($str);
+            if ($unpacker->execute())
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+                $unpacker->reset();
+            }
+
+            $i += $len;
+        }
+    }
+    else
+    {
+        $str = "";
+        $offset = 0;
+
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str .= substr($serialized, $i, $len);
+
+            if ($unpacker->execute($str, $offset))
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+
+                $unpacker->reset();
+                $str = "";
+                $offset = 0;
+            }
+
+            $i += $len;
+        }
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), true);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), true);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), true);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          NULL
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/087.phpt b/php/tests/087.phpt
new file mode 100644
index 0000000..9bb4e49
--- /dev/null
+++ b/php/tests/087.phpt
@@ -0,0 +1,302 @@
+--TEST--
+disabled php only for class methods (set option)
+--SKIPIF--
+--FILE--
+setOption(MessagePack::OPT_PHPONLY, false);
+
+    $serialized = $msgpack->pack($variable);
+    $unserialized = $msgpack->unpack($serialized);
+
+    var_dump($unserialized);
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), true);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), true);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), true);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          NULL
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/088.phpt b/php/tests/088.phpt
new file mode 100644
index 0000000..7cbabb9
--- /dev/null
+++ b/php/tests/088.phpt
@@ -0,0 +1,345 @@
+--TEST--
+disabled php only for class methods unpacker (set option)
+--SKIPIF--
+--FILE--
+setOption(MessagePack::OPT_PHPONLY, false);
+
+    $serialized = $msgpack->pack($variable);
+    $unpacker = $msgpack->unpacker();
+
+    $length = strlen($serialized);
+
+    if (rand(0, 1))
+    {
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str = substr($serialized, $i, $len);
+
+            $unpacker->feed($str);
+            if ($unpacker->execute())
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+                $unpacker->reset();
+            }
+
+            $i += $len;
+        }
+    }
+    else
+    {
+        $str = "";
+        $offset = 0;
+
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str .= substr($serialized, $i, $len);
+
+            if ($unpacker->execute($str, $offset))
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+
+                $unpacker->reset();
+                $str = "";
+                $offset = 0;
+            }
+
+            $i += $len;
+        }
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), true);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), true);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), true);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          NULL
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/089.phpt b/php/tests/089.phpt
new file mode 100644
index 0000000..f3a0537
--- /dev/null
+++ b/php/tests/089.phpt
@@ -0,0 +1,347 @@
+--TEST--
+disabled php only for class unpacker (set option)
+--SKIPIF--
+--FILE--
+setOption(MessagePack::OPT_PHPONLY, false);
+
+    $serialized = $msgpack->pack($variable);
+
+    $unpacker = new MessagePackUnpacker();
+    $unpacker->setOption(MessagePack::OPT_PHPONLY, false);
+
+    $length = strlen($serialized);
+
+    if (rand(0, 1))
+    {
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str = substr($serialized, $i, $len);
+
+            $unpacker->feed($str);
+            if ($unpacker->execute())
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+                $unpacker->reset();
+            }
+
+            $i += $len;
+        }
+    }
+    else
+    {
+        $str = "";
+        $offset = 0;
+
+        for ($i = 0; $i < $length;)
+        {
+            $len = rand(1, 10);
+            $str .= substr($serialized, $i, $len);
+
+            if ($unpacker->execute($str, $offset))
+            {
+                $unserialized = $unpacker->data();
+                var_dump($unserialized);
+
+                $unpacker->reset();
+                $str = "";
+                $offset = 0;
+            }
+
+            $i += $len;
+        }
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), true);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), true);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), true);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  array(1) {
+    [0]=>
+    array(1) {
+      [0]=>
+      array(1) {
+        [0]=>
+        array(1) {
+          [0]=>
+          NULL
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+}
+OK

From 7f7f5253f2e45927f119065e766d499feeb8c566 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 30 Sep 2010 00:55:56 +0900
Subject: [PATCH 0820/1648] java: adds a test program of the DynamicCodeGen
 class for packing and unpacking an object that has a field anntated by
 @MessagePackMessage

---
 .../main/java/org/msgpack/CustomMessage.java  |   2 +-
 .../msgpack/util/codegen/DynamicCodeGen.java  | 125 ++++++++----------
 .../TestDynamicCodeGenPackerConverter.java    |  40 ++++++
 .../TestDynamicCodeGenPackerUnpacker.java     |  35 +++++
 4 files changed, 129 insertions(+), 73 deletions(-)

diff --git a/java/src/main/java/org/msgpack/CustomMessage.java b/java/src/main/java/org/msgpack/CustomMessage.java
index 53f8323..959c9cd 100644
--- a/java/src/main/java/org/msgpack/CustomMessage.java
+++ b/java/src/main/java/org/msgpack/CustomMessage.java
@@ -29,7 +29,7 @@ public class CustomMessage {
 		CustomConverter.register(target, tmpl);
 	}
 
-	static boolean isAnnotated(Class target, Class with) {
+	public static boolean isAnnotated(Class target, Class with) {
 		return target.getAnnotation(with) != null;
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 08851c8..ba9b0ba 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -23,9 +23,11 @@ import javassist.CtNewMethod;
 import javassist.NotFoundException;
 
 import org.msgpack.CustomConverter;
+import org.msgpack.CustomMessage;
 import org.msgpack.CustomUnpacker;
 import org.msgpack.MessageConvertable;
 import org.msgpack.MessagePackObject;
+import org.msgpack.MessagePackable;
 import org.msgpack.MessagePacker;
 import org.msgpack.MessageTypeException;
 import org.msgpack.MessageUnpackable;
@@ -33,6 +35,9 @@ import org.msgpack.MessageUnpacker;
 import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
+import org.msgpack.annotation.MessagePackDelegate;
+import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.annotation.MessagePackOrdinalEnum;
 
 public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
@@ -98,11 +103,10 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	public Class generateMessageConverterClass(Class origClass) {
 		try {
 			String origName = origClass.getName();
-			String converterName = origName + POSTFIX_TYPE_NAME_CONVERTER
-					+ inc();
+			String convName = origName + POSTFIX_TYPE_NAME_CONVERTER + inc();
 			checkClassValidation(origClass);
 			checkDefaultConstructorValidation(origClass);
-			CtClass converterCtClass = pool.makeClass(converterName);
+			CtClass converterCtClass = pool.makeClass(convName);
 			setInterface(converterCtClass, MessageUnpacker.class);
 			addDefaultConstructor(converterCtClass);
 			Field[] fields = getDeclaredFields(origClass);
@@ -162,7 +166,6 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		} catch (Exception e1) {
 			throwConstructoValidationException(origClass);
 		}
-
 		int mod = cons.getModifiers();
 		if (!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) {
 			throwConstructoValidationException(origClass);
@@ -256,6 +259,28 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	}
 
 	private void insertCodeOfPackMethodCall(StringBuilder sb, Field field) {
+		Class c = field.getType();
+		if (c.isPrimitive() || c.equals(Boolean.class) || c.equals(Byte.class)
+				|| c.equals(Double.class) || c.equals(Float.class)
+				|| c.equals(Integer.class) || c.equals(Long.class)
+				|| c.equals(Short.class) || List.class.isAssignableFrom(c)
+				|| Map.class.isAssignableFrom(c)
+				|| CustomUnpacker.isRegistered(c)
+				|| MessagePackable.class.isAssignableFrom(c)) {
+			;
+		} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
+			// @MessagePackMessage
+			MessagePacker packer = DynamicCodeGenPacker.create(c);
+			CustomMessage.registerPacker(c, packer);
+		} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
+			// FIXME DelegatePacker
+			throw new UnsupportedOperationException("not supported yet. : "
+					+ c.getName());
+		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
+			// FIXME OrdinalEnumPacker
+			throw new UnsupportedOperationException("not supported yet. : "
+					+ c.getName());
+		}
 		StringBuilder fa = new StringBuilder();
 		insertFieldAccess(fa, VARIABLE_NAME_TARGET, field.getName());
 		insertMethodCall(sb, VARIABLE_NAME_PK, METHOD_NAME_PACK,
@@ -265,6 +290,9 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private void addUnpackMethod(CtClass unpackerCtClass, Class c, Field[] fs)
 			throws CannotCompileException, NotFoundException {
+		if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
+			// TODO
+		}
 		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
@@ -324,7 +352,12 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			insertCodeOfUnpackMethodCallForRegisteredType(sb, f, c);
 		} else if (MessageUnpackable.class.isAssignableFrom(c)) {
 			// MessageUnpackable
-			insertCodeOfMessageUnpackCallForMsgUnpackableType(sb, f, c);
+			insertCodeOfUnpackMethodCallForMsgUnpackableType(sb, f, c);
+		} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
+			// @MessagePackMessage
+			Template tmpl = DynamicCodeGenTemplate.create(c);
+			CustomMessage.registerTemplate(c, tmpl);
+			insertCodeOfUnpackMethodCallForRegisteredType(sb, f, c);
 		} else {
 			throw new MessageTypeException("unknown type:  " + c.getName());
 		}
@@ -532,78 +565,26 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private void insertCodeOfUnpackMethodCallForRegisteredType(
 			StringBuilder sb, Field f, Class c) {
-		// if (t.fi == null) { t.fi = new Foo(); }
-		// sb.append(KEYWORD_IF);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		// sb.append(VARIABLE_NAME_TARGET);
-		// sb.append(CHAR_NAME_DOT);
-		// sb.append(f.getName());
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(CHAR_NAME_EQUAL);
-		// sb.append(CHAR_NAME_EQUAL);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(KEYWORD_NULL);
-		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(VARIABLE_NAME_TARGET);
-		// sb.append(CHAR_NAME_DOT);
-		// sb.append(f.getName());
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(CHAR_NAME_EQUAL);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(KEYWORD_NEW);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(c.getName());
-		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		// sb.append(CHAR_NAME_SEMICOLON);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-		// sb.append(CHAR_NAME_SPACE);
-
-		// tmpl1.unpack(new Unpacker(in));
-		// CustomUnpacker.get(c).pack(new Packer(out), src);
+		// target.field = (Cast) CustomUnpacker.get(C.class).unpack(pk);
+		StringBuilder mc = new StringBuilder();
+		insertTypeCast(mc, c);
+		mc.append(CustomUnpacker.class.getName());
+		String t = mc.toString();
+		mc = new StringBuilder();
+		insertMethodCall(mc, t, METHOD_NAME_GET, new String[] { c.getName()
+				+ ".class" });
+		t = mc.toString();
+		mc = new StringBuilder();
+		insertMethodCall(mc, t, METHOD_NAME_UNPACK,
+				new String[] { VARIABLE_NAME_PK });
 		sb.append(VARIABLE_NAME_TARGET);
 		sb.append(CHAR_NAME_DOT);
 		sb.append(f.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(c.getName());
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CustomUnpacker.class.getName());
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_GET);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(c.getName() + ".class");
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_UNPACK);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(VARIABLE_NAME_PK);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		insertValueInsertion(sb, mc.toString());
 		insertSemicolon(sb);
-		//		
-		// sb.append(VARIABLE_NAME_PK);
-		// sb.append(CHAR_NAME_DOT);
-		// sb.append(METHOD_NAME_UNPACK);
-		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		// sb.append(MessageUnpackable.class.getName());
-		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		// sb.append(VARIABLE_NAME_TARGET);
-		// sb.append(CHAR_NAME_DOT);
-		// sb.append(f.getName());
-		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		// sb.append(CHAR_NAME_SEMICOLON);
-		// sb.append(CHAR_NAME_SPACE);
 	}
 
-	private void insertCodeOfMessageUnpackCallForMsgUnpackableType(
+	private void insertCodeOfUnpackMethodCallForMsgUnpackableType(
 			StringBuilder sb, Field f, Class c) {
 		// if (t.fi == null) { t.fi = new Foo(); }
 		sb.append(KEYWORD_IF);
@@ -663,7 +644,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				new Class[] { MessagePackObject.class },
 				new String[] { VARIABLE_NAME_MPO },
 				new Class[] { MessageTypeException.class }, bsb.toString());
-		//System.out.println("convert method: " + sb.toString());
+		// System.out.println("convert method: " + sb.toString());
 		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
 		tmplCtClass.addMethod(newCtMethod);
 	}
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
index 7ffcd88..b737674 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
@@ -20,6 +20,7 @@ import org.msgpack.MessagePacker;
 import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
+import org.msgpack.annotation.MessagePackMessage;
 
 public class TestDynamicCodeGenPackerConverter extends TestCase {
 
@@ -534,6 +535,44 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		}
 	}
 
+	@Test
+	public void testMessagePackMessageFieldClass() throws Exception {
+		BaseClass2 src = new BaseClass2();
+		MessagePackMessageClass2 src2 = new MessagePackMessageClass2();
+		src.f0 = 0;
+		src2.f2 = 2;
+		src.f1 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker.create(BaseClass2.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		Template tmpl = DynamicCodeGenTemplate.create(BaseClass2.class);
+		BaseClass2 dst = (BaseClass2) tmpl.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1.f2 == dst.f1.f2);
+		assertFalse(it.hasNext());
+	}
+
+	public static class BaseClass2 {
+		public int f0;
+		public MessagePackMessageClass2 f1;
+
+		public BaseClass2() {
+		}
+	}
+
+	@MessagePackMessage
+	public static class MessagePackMessageClass2 {
+		public int f2;
+
+		public MessagePackMessageClass2() {
+		}
+	}
+
 	@Test
 	public void testExtendedClass() throws Exception {
 		SampleSubClass src = new SampleSubClass();
@@ -581,6 +620,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	public static class SampleSuperClass {
 		public int f5;
 		public final int f6 = 2;
+		@SuppressWarnings("unused")
 		private int f7;
 		protected int f8;
 		int f9;
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
index 08d0e52..ebc6d15 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
@@ -17,6 +17,7 @@ import org.msgpack.MessagePacker;
 import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
+import org.msgpack.annotation.MessagePackMessage;
 
 import junit.framework.TestCase;
 
@@ -504,6 +505,39 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		}
 	}
 
+	@Test
+	public void testMessagePackMessageFieldClass() throws Exception {
+		BaseClass2 src = new BaseClass2();
+		MessagePackMessageClass2 src2 = new MessagePackMessageClass2();
+		src.f0 = 0;
+		src2.f2 = 2;
+		src.f1 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker.create(BaseClass2.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate.create(BaseClass2.class);
+		BaseClass2 dst = (BaseClass2) tmpl.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1.f2 == dst.f1.f2);
+	}
+
+	public static class BaseClass2 {
+		public int f0;
+		public MessagePackMessageClass2 f1;
+
+		public BaseClass2() {
+		}
+	}
+
+	@MessagePackMessage
+	public static class MessagePackMessageClass2 {
+		public int f2;
+
+		public MessagePackMessageClass2() {
+		}
+	}
+
 	@Test
 	public void testExtendedClass() throws Exception {
 		SampleSubClass src = new SampleSubClass();
@@ -546,6 +580,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	public static class SampleSuperClass {
 		public int f5;
 		public final int f6 = 2;
+		@SuppressWarnings("unused")
 		private int f7;
 		protected int f8;
 		int f9;

From 0bd4150a801930d12594e12442b424b0f820a202 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 30 Sep 2010 01:17:44 +0900
Subject: [PATCH 0821/1648] java: changes several methods declared in a
 DynamicCodeGen class

---
 .../msgpack/util/codegen/DynamicCodeGen.java  | 59 ++++++++-----------
 1 file changed, 23 insertions(+), 36 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index ba9b0ba..d16fd7b 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -290,9 +290,6 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private void addUnpackMethod(CtClass unpackerCtClass, Class c, Field[] fs)
 			throws CannotCompileException, NotFoundException {
-		if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
-			// TODO
-		}
 		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
@@ -302,7 +299,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				new String[] { VARIABLE_NAME_PK }, new Class[] {
 						MessageTypeException.class, IOException.class }, bsb
 						.toString());
-		//System.out.println("unpack method: " + sb.toString());
+		// System.out.println("unpack method: " + sb.toString());
 		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), unpackerCtClass);
 		unpackerCtClass.addMethod(newCtMethod);
 	}
@@ -358,6 +355,14 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			Template tmpl = DynamicCodeGenTemplate.create(c);
 			CustomMessage.registerTemplate(c, tmpl);
 			insertCodeOfUnpackMethodCallForRegisteredType(sb, f, c);
+		} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
+			// FIXME DelegatePacker
+			throw new UnsupportedOperationException("not supported yet. : "
+					+ c.getName());
+		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
+			// FIXME OrdinalEnumPacker
+			throw new UnsupportedOperationException("not supported yet. : "
+					+ c.getName());
 		} else {
 			throw new MessageTypeException("unknown type:  " + c.getName());
 		}
@@ -710,6 +715,20 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
 			} else if (CustomConverter.isRegistered(c)) {
 				insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
+			} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
+				// @MessagePackMessage
+				Template tmpl = DynamicCodeGenTemplate.create(c);
+				CustomMessage.registerTemplate(c, tmpl);
+				insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
+			} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
+				// FIXME DelegatePacker
+				throw new UnsupportedOperationException("not supported yet. : "
+						+ c.getName());
+			} else if (CustomMessage.isAnnotated(c,
+					MessagePackOrdinalEnum.class)) {
+				// FIXME OrdinalEnumPacker
+				throw new UnsupportedOperationException("not supported yet. : "
+						+ c.getName());
 			} else {
 				throw new MessageTypeException("Type error: " + c.getName());
 			}
@@ -718,38 +737,6 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private void insertCodeOfMessageConvertCallForRegisteredType(
 			StringBuilder sb, Field f, Class c, int i) {
-		// if (t.fi == null) { t.fi = new Foo(); }
-		// sb.append(KEYWORD_IF);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		// sb.append(VARIABLE_NAME_TARGET);
-		// sb.append(CHAR_NAME_DOT);
-		// sb.append(f.getName());
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(CHAR_NAME_EQUAL);
-		// sb.append(CHAR_NAME_EQUAL);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(KEYWORD_NULL);
-		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(VARIABLE_NAME_TARGET);
-		// sb.append(CHAR_NAME_DOT);
-		// sb.append(f.getName());
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(CHAR_NAME_EQUAL);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(KEYWORD_NEW);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(c.getName());
-		// sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		// sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		// sb.append(CHAR_NAME_SEMICOLON);
-		// sb.append(CHAR_NAME_SPACE);
-		// sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-		// sb.append(CHAR_NAME_SPACE);
-
 		// ((MessageConvertable)f_i).messageConvert(ary[i]);
 		// obj = tmpl.convert(mpo);
 		sb.append(VARIABLE_NAME_TARGET);

From a3b1ef95279df7fc498e3da857afb225f0fedcaa Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Fri, 1 Oct 2010 12:51:09 +0900
Subject: [PATCH 0822/1648] java: Packer: supports Set

---
 java/src/main/java/org/msgpack/Packer.java                 | 6 ++++++
 java/src/main/java/org/msgpack/template/ClassTemplate.java | 4 ++++
 2 files changed, 10 insertions(+)

diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index 52bd29d..f443f53 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -21,6 +21,7 @@ import java.io.OutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.List;
+import java.util.Set;
 import java.util.Map;
 import java.math.BigInteger;
 
@@ -449,6 +450,11 @@ public class Packer {
 			packArray(l.size());
 			for(Object i : l) { pack(i); }
 			return this;
+		} else if(o instanceof Set) {
+			Set l = (Set)o;
+			packArray(l.size());
+			for(Object i : l) { pack(i); }
+			return this;
 		} else if(o instanceof Map) {
 			Map m = (Map)o;
 			packMap(m.size());
diff --git a/java/src/main/java/org/msgpack/template/ClassTemplate.java b/java/src/main/java/org/msgpack/template/ClassTemplate.java
index c529edd..9bb957c 100644
--- a/java/src/main/java/org/msgpack/template/ClassTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ClassTemplate.java
@@ -21,6 +21,10 @@ import java.io.IOException;
 import org.msgpack.*;
 
 public class ClassTemplate implements Template {
+	static {
+		Templates.load();
+	}
+
 	private Class klass;
 
 	public ClassTemplate(Class klass) {

From 7c76f073841f309056e596811ec9851beb78ba7d Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Fri, 1 Oct 2010 13:12:31 +0900
Subject: [PATCH 0823/1648] java: adds standard packer classes

---
 java/src/main/java/org/msgpack/Packer.java    | 73 ++++++++++++-------
 java/src/main/java/org/msgpack/Unpacker.java  |  3 +-
 .../org/msgpack/packer/BigIntegerPacker.java  | 41 +++++++++++
 .../org/msgpack/packer/BooleanPacker.java     | 40 ++++++++++
 .../org/msgpack/packer/ByteArrayPacker.java   | 40 ++++++++++
 .../java/org/msgpack/packer/BytePacker.java   | 40 ++++++++++
 .../java/org/msgpack/packer/DoublePacker.java | 40 ++++++++++
 .../java/org/msgpack/packer/FloatPacker.java  | 40 ++++++++++
 .../org/msgpack/packer/IntegerPacker.java     | 40 ++++++++++
 .../java/org/msgpack/packer/LongPacker.java   | 40 ++++++++++
 .../java/org/msgpack/packer/ShortPacker.java  | 40 ++++++++++
 .../java/org/msgpack/packer/StringPacker.java | 40 ++++++++++
 12 files changed, 448 insertions(+), 29 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/packer/BigIntegerPacker.java
 create mode 100644 java/src/main/java/org/msgpack/packer/BooleanPacker.java
 create mode 100644 java/src/main/java/org/msgpack/packer/ByteArrayPacker.java
 create mode 100644 java/src/main/java/org/msgpack/packer/BytePacker.java
 create mode 100644 java/src/main/java/org/msgpack/packer/DoublePacker.java
 create mode 100644 java/src/main/java/org/msgpack/packer/FloatPacker.java
 create mode 100644 java/src/main/java/org/msgpack/packer/IntegerPacker.java
 create mode 100644 java/src/main/java/org/msgpack/packer/LongPacker.java
 create mode 100644 java/src/main/java/org/msgpack/packer/ShortPacker.java
 create mode 100644 java/src/main/java/org/msgpack/packer/StringPacker.java

diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index f443f53..704481d 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -28,6 +28,7 @@ import java.math.BigInteger;
 import org.msgpack.annotation.MessagePackDelegate;
 import org.msgpack.annotation.MessagePackMessage;
 import org.msgpack.annotation.MessagePackOrdinalEnum;
+import org.msgpack.packer.*;
 
 /**
  * Packer enables you to serialize objects into OutputStream.
@@ -46,6 +47,22 @@ import org.msgpack.annotation.MessagePackOrdinalEnum;
  * You can serialize objects that implements {@link MessagePackable} interface.
  */
 public class Packer {
+	static {
+		// final classes
+		BooleanPacker.getInstance();
+		ByteArrayPacker.getInstance();
+		BytePacker.getInstance();
+		DoublePacker.getInstance();
+		FloatPacker.getInstance();
+		IntegerPacker.getInstance();
+		LongPacker.getInstance();
+		ShortPacker.getInstance();
+		StringPacker.getInstance();
+		//BigIntegerPacker.getInstance();  // BigInteger is not final
+	}
+
+	public static void load() { }
+
 	protected byte[] castBytes = new byte[9];
 	protected ByteBuffer castBuffer = ByteBuffer.wrap(castBytes);
 	protected OutputStream out;
@@ -434,17 +451,17 @@ public class Packer {
 	public Packer pack(Object o) throws IOException {
 		if(o == null) {
 			return packNil();
-		} else if(o instanceof String) {
-			byte[] b = ((String)o).getBytes("UTF-8");
-			packRaw(b.length);
-			return packRawBody(b);
+		//} else if(o instanceof String) {
+		//	byte[] b = ((String)o).getBytes("UTF-8");
+		//	packRaw(b.length);
+		//	return packRawBody(b);
 		} else if(o instanceof MessagePackable) {
 			((MessagePackable)o).messagePack(this);
 			return this;
-		} else if(o instanceof byte[]) {
-			byte[] b = (byte[])o;
-			packRaw(b.length);
-			return packRawBody(b);
+		//} else if(o instanceof byte[]) {
+		//	byte[] b = (byte[])o;
+		//	packRaw(b.length);
+		//	return packRawBody(b);
 		} else if(o instanceof List) {
 			List l = (List)o;
 			packArray(l.size());
@@ -463,27 +480,27 @@ public class Packer {
 				pack(e.getValue());
 			}
 			return this;
-		} else if(o instanceof Boolean) {
-			if((Boolean)o) {
-				return packTrue();
-			} else {
-				return packFalse();
-			}
-		} else if(o instanceof Integer) {
-			return packInt((Integer)o);
-		} else if(o instanceof Long) {
-			return packLong((Long)o);
-		} else if(o instanceof Short) {
-			return packShort((Short)o);
-		} else if(o instanceof Byte) {
-			return packByte((Byte)o);
-		} else if(o instanceof Float) {
-			return packFloat((Float)o);
-		} else if(o instanceof Double) {
-			return packDouble((Double)o);
+		//} else if(o instanceof Boolean) {
+		//	if((Boolean)o) {
+		//		return packTrue();
+		//	} else {
+		//		return packFalse();
+		//	}
+		//} else if(o instanceof Integer) {
+		//	return packInt((Integer)o);
+		//} else if(o instanceof Long) {
+		//	return packLong((Long)o);
+		//} else if(o instanceof Short) {
+		//	return packShort((Short)o);
+		//} else if(o instanceof Byte) {
+		//	return packByte((Byte)o);
+		//} else if(o instanceof Float) {
+		//	return packFloat((Float)o);
+		//} else if(o instanceof Double) {
+		//	return packDouble((Double)o);
 		} else if(o instanceof BigInteger) {
 			return packBigInteger((BigInteger)o);
-		} 
+		}
 
 		Class klass = o.getClass();
 		MessagePacker packer = CustomPacker.get(klass);
@@ -506,5 +523,5 @@ public class Packer {
 		}
 		throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")");
 	}
-
 }
+
diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index d91fc0e..8bbafb3 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -588,10 +588,11 @@ public class Unpacker implements Iterable {
 		}
 
 		MessageUnpacker unpacker = CustomUnpacker.get(klass);
+
 		if(unpacker != null) {
 			return unpacker.unpack(this);
 		}
-		
+
 		Template tmpl = null;
 		if (CustomMessage.isAnnotated(klass, MessagePackMessage.class)) {
 			tmpl = ReflectionTemplate.create(klass);
diff --git a/java/src/main/java/org/msgpack/packer/BigIntegerPacker.java b/java/src/main/java/org/msgpack/packer/BigIntegerPacker.java
new file mode 100644
index 0000000..60bbab9
--- /dev/null
+++ b/java/src/main/java/org/msgpack/packer/BigIntegerPacker.java
@@ -0,0 +1,41 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.packer;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import org.msgpack.*;
+
+public class BigIntegerPacker implements MessagePacker {
+	private BigIntegerPacker() { }
+
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.pack((BigInteger)target);
+	}
+
+	static public BigIntegerPacker getInstance() {
+		return instance;
+	}
+
+	static final BigIntegerPacker instance = new BigIntegerPacker();
+
+	static {
+		CustomMessage.registerPacker(BigInteger.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/packer/BooleanPacker.java b/java/src/main/java/org/msgpack/packer/BooleanPacker.java
new file mode 100644
index 0000000..6f300c9
--- /dev/null
+++ b/java/src/main/java/org/msgpack/packer/BooleanPacker.java
@@ -0,0 +1,40 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.packer;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class BooleanPacker implements MessagePacker {
+	private BooleanPacker() { }
+
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.pack((Boolean)target);
+	}
+
+	static public BooleanPacker getInstance() {
+		return instance;
+	}
+
+	static final BooleanPacker instance = new BooleanPacker();
+
+	static {
+		CustomMessage.registerPacker(Boolean.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/packer/ByteArrayPacker.java b/java/src/main/java/org/msgpack/packer/ByteArrayPacker.java
new file mode 100644
index 0000000..f5f38f8
--- /dev/null
+++ b/java/src/main/java/org/msgpack/packer/ByteArrayPacker.java
@@ -0,0 +1,40 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.packer;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class ByteArrayPacker implements MessagePacker {
+	private ByteArrayPacker() { }
+
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.pack((byte[])target);
+	}
+
+	static public ByteArrayPacker getInstance() {
+		return instance;
+	}
+
+	static final ByteArrayPacker instance = new ByteArrayPacker();
+
+	static {
+		CustomMessage.registerPacker(byte[].class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/packer/BytePacker.java b/java/src/main/java/org/msgpack/packer/BytePacker.java
new file mode 100644
index 0000000..a005f76
--- /dev/null
+++ b/java/src/main/java/org/msgpack/packer/BytePacker.java
@@ -0,0 +1,40 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.packer;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class BytePacker implements MessagePacker {
+	private BytePacker() { }
+
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.pack(((Byte)target).byteValue());
+	}
+
+	static public BytePacker getInstance() {
+		return instance;
+	}
+
+	static final BytePacker instance = new BytePacker();
+
+	static {
+		CustomMessage.registerPacker(Byte.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/packer/DoublePacker.java b/java/src/main/java/org/msgpack/packer/DoublePacker.java
new file mode 100644
index 0000000..df53e33
--- /dev/null
+++ b/java/src/main/java/org/msgpack/packer/DoublePacker.java
@@ -0,0 +1,40 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.packer;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class DoublePacker implements MessagePacker {
+	private DoublePacker() { }
+
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.pack(((Double)target).doubleValue());
+	}
+
+	static public DoublePacker getInstance() {
+		return instance;
+	}
+
+	static final DoublePacker instance = new DoublePacker();
+
+	static {
+		CustomMessage.registerPacker(Double.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/packer/FloatPacker.java b/java/src/main/java/org/msgpack/packer/FloatPacker.java
new file mode 100644
index 0000000..3aa47e1
--- /dev/null
+++ b/java/src/main/java/org/msgpack/packer/FloatPacker.java
@@ -0,0 +1,40 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.packer;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class FloatPacker implements MessagePacker {
+	private FloatPacker() { }
+
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.pack(((Float)target).floatValue());
+	}
+
+	static public FloatPacker getInstance() {
+		return instance;
+	}
+
+	static final FloatPacker instance = new FloatPacker();
+
+	static {
+		CustomMessage.registerPacker(Float.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/packer/IntegerPacker.java b/java/src/main/java/org/msgpack/packer/IntegerPacker.java
new file mode 100644
index 0000000..1e64429
--- /dev/null
+++ b/java/src/main/java/org/msgpack/packer/IntegerPacker.java
@@ -0,0 +1,40 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.packer;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class IntegerPacker implements MessagePacker {
+	private IntegerPacker() { }
+
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.pack(((Integer)target).intValue());
+	}
+
+	static public IntegerPacker getInstance() {
+		return instance;
+	}
+
+	static final IntegerPacker instance = new IntegerPacker();
+
+	static {
+		CustomMessage.registerPacker(Integer.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/packer/LongPacker.java b/java/src/main/java/org/msgpack/packer/LongPacker.java
new file mode 100644
index 0000000..146e42d
--- /dev/null
+++ b/java/src/main/java/org/msgpack/packer/LongPacker.java
@@ -0,0 +1,40 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.packer;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class LongPacker implements MessagePacker {
+	private LongPacker() { }
+
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.pack(((Long)target).longValue());
+	}
+
+	static public LongPacker getInstance() {
+		return instance;
+	}
+
+	static final LongPacker instance = new LongPacker();
+
+	static {
+		CustomMessage.registerPacker(Long.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/packer/ShortPacker.java b/java/src/main/java/org/msgpack/packer/ShortPacker.java
new file mode 100644
index 0000000..e4be9ec
--- /dev/null
+++ b/java/src/main/java/org/msgpack/packer/ShortPacker.java
@@ -0,0 +1,40 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.packer;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class ShortPacker implements MessagePacker {
+	private ShortPacker() { }
+
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.pack(((Short)target).shortValue());
+	}
+
+	static public ShortPacker getInstance() {
+		return instance;
+	}
+
+	static final ShortPacker instance = new ShortPacker();
+
+	static {
+		CustomMessage.registerPacker(Short.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/packer/StringPacker.java b/java/src/main/java/org/msgpack/packer/StringPacker.java
new file mode 100644
index 0000000..17a0500
--- /dev/null
+++ b/java/src/main/java/org/msgpack/packer/StringPacker.java
@@ -0,0 +1,40 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.packer;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class StringPacker implements MessagePacker {
+	private StringPacker() { }
+
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.pack((String)target);
+	}
+
+	static public StringPacker getInstance() {
+		return instance;
+	}
+
+	static final StringPacker instance = new StringPacker();
+
+	static {
+		CustomMessage.registerPacker(String.class, instance);
+	}
+}
+

From cdd60e5f9cff77af78eb2b19f4a701d522e3c467 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 2 Oct 2010 17:43:31 +0900
Subject: [PATCH 0824/1648] add a packer and unpacker for Enum types

---
 .../org/msgpack/MessageTypeException.java     |   4 +
 .../msgpack/util/codegen/BasicConstants.java  |   6 +-
 .../msgpack/util/codegen/DynamicCodeGen.java  | 205 ++++++++++++++++--
 .../DynamicCodeGenOrdinalEnumConverter.java   |  17 ++
 .../DynamicCodeGenOrdinalEnumPacker.java      |  17 ++
 .../DynamicCodeGenOrdinalEnumTemplate.java    |  17 ++
 .../DynamicCodeGenOrdinalEnumUnpacker.java    |  17 ++
 .../TestDynamicCodeGenPackerConverter.java    |  50 ++++-
 .../TestDynamicCodeGenPackerUnpacker.java     |  56 +++--
 9 files changed, 332 insertions(+), 57 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumConverter.java
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumPacker.java
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumTemplate.java
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumUnpacker.java

diff --git a/java/src/main/java/org/msgpack/MessageTypeException.java b/java/src/main/java/org/msgpack/MessageTypeException.java
index 698ef6d..7a06a3e 100644
--- a/java/src/main/java/org/msgpack/MessageTypeException.java
+++ b/java/src/main/java/org/msgpack/MessageTypeException.java
@@ -23,5 +23,9 @@ public class MessageTypeException extends RuntimeException {
 	public MessageTypeException(String s) {
 		super(s);
 	}
+	
+	public MessageTypeException(String s, Throwable t) {
+		super(s, t);
+	}
 }
 
diff --git a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
index e568a76..cd667a6 100644
--- a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
@@ -19,7 +19,7 @@ public interface BasicConstants {
 	String KEYWORD_NEW = "new";
 
 	String KEYWORD_NULL = "null";
-	
+
 	String KEYWORD_RETURN = "return";
 
 	String KEYWORD_THROW = "throw";
@@ -98,8 +98,10 @@ public interface BasicConstants {
 
 	String METHOD_NAME_NEXT = "next";
 
+	String METHOD_NAME_ORDINAL = "ordinal";
+
 	String METHOD_NAME_CONVERT = "convert";
-	
+
 	String METHOD_NAME_MSGCONVERT = "messageConvert";
 
 	String METHOD_NAME_PACK = "pack";
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index d16fd7b..c7f5dd3 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -81,6 +81,24 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
+	public Class generateOrdinalEnumPackerClass(Class origClass) {
+		try {
+			String origName = origClass.getName();
+			String packerName = origName + POSTFIX_TYPE_NAME_PACKER + inc();
+			checkClassValidation(origClass);
+			//checkDefaultConstructorValidation(origClass);
+			CtClass packerCtClass = pool.makeClass(packerName);
+			setInterface(packerCtClass, MessagePacker.class);
+			addDefaultConstructor(packerCtClass);
+			addPackMethodForOrdinalEnumTypes(packerCtClass, origClass);
+			return createClass(packerCtClass);
+		} catch (NotFoundException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (CannotCompileException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+
 	public Class generateMessageUnpackerClass(Class origClass) {
 		try {
 			String origName = origClass.getName();
@@ -100,6 +118,24 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
+	public Class generateOrdinalEnumUnpackerClass(Class origClass) {
+		try {
+			String origName = origClass.getName();
+			String unpackerName = origName + POSTFIX_TYPE_NAME_UNPACKER + inc();
+			checkClassValidation(origClass);
+			checkDefaultConstructorValidation(origClass);
+			CtClass unpackerCtClass = pool.makeClass(unpackerName);
+			setInterface(unpackerCtClass, MessageUnpacker.class);
+			addDefaultConstructor(unpackerCtClass);
+			addUnpackMethodForOrdinalEnumTypes(unpackerCtClass, origClass);
+			return createClass(unpackerCtClass);
+		} catch (NotFoundException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (CannotCompileException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+
 	public Class generateMessageConverterClass(Class origClass) {
 		try {
 			String origName = origClass.getName();
@@ -119,6 +155,24 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
+	public Class generateOrdinalEnumConverterClass(Class origClass) {
+		try {
+			String origName = origClass.getName();
+			String convName = origName + POSTFIX_TYPE_NAME_CONVERTER + inc();
+			checkClassValidation(origClass);
+			//checkDefaultConstructorValidation(origClass);
+			CtClass converterCtClass = pool.makeClass(convName);
+			setInterface(converterCtClass, MessageUnpacker.class);
+			addDefaultConstructor(converterCtClass);
+			addConvertMethodForOrdinalEnumTypes(converterCtClass, origClass);
+			return createClass(converterCtClass);
+		} catch (NotFoundException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (CannotCompileException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+
 	public Class generateTemplateClass(Class origClass) {
 		try {
 			String origName = origClass.getName();
@@ -139,18 +193,37 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
+	public Class generateOrdinalEnumTemplateClass(Class origClass) {
+		try {
+			String origName = origClass.getName();
+			String tmplName = origName + POSTFIX_TYPE_NAME_TEMPLATE + inc();
+			checkClassValidation(origClass);
+			//checkDefaultConstructorValidation(origClass);
+			CtClass tmplCtClass = pool.makeClass(tmplName);
+			setInterface(tmplCtClass, Template.class);
+			addDefaultConstructor(tmplCtClass);
+			addUnpackMethodForOrdinalEnumTypes(tmplCtClass, origClass);
+			addConvertMethodForOrdinalEnumTypes(tmplCtClass, origClass);
+			return createClass(tmplCtClass);
+		} catch (NotFoundException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (CannotCompileException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+
 	private void checkClassValidation(Class origClass) {
-		// not public, abstract, final
+		// not public, abstract
 		int mod = origClass.getModifiers();
 		if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
-				|| Modifier.isAbstract(mod) || Modifier.isFinal(mod)) {
+				|| Modifier.isAbstract(mod)) {
 			throwClassValidationException(origClass,
-					"it must be a public class");
+					"a class must have a public modifier");
 		}
-		// interface, enum
-		if (origClass.isInterface() || origClass.isEnum()) {
+		// interface
+		if (origClass.isInterface()) {
 			throwClassValidationException(origClass,
-					"it must not be an interface or enum");
+					"cannot generate packer and unpacker for an interface");
 		}
 	}
 
@@ -239,7 +312,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				Packer.class, Object.class }, new String[] { VARIABLE_NAME_PK,
 				VARIABLE_NAME_OBJECT }, new Class[] { IOException.class }, bsb
 				.toString());
-		// System.out.println("pack method: " + sb.toString());
+		System.out.println("pack method: " + sb.toString());
 		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), packerCtClass);
 		packerCtClass.addMethod(newCtMethod);
 	}
@@ -277,9 +350,9 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			throw new UnsupportedOperationException("not supported yet. : "
 					+ c.getName());
 		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
-			// FIXME OrdinalEnumPacker
-			throw new UnsupportedOperationException("not supported yet. : "
-					+ c.getName());
+			// @MessagePackOrdinalEnum
+			MessagePacker packer = DynamicCodeGenOrdinalEnumPacker.create(c);
+			CustomMessage.registerPacker(c, packer);
 		}
 		StringBuilder fa = new StringBuilder();
 		insertFieldAccess(fa, VARIABLE_NAME_TARGET, field.getName());
@@ -288,9 +361,42 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		insertSemicolon(sb);
 	}
 
+	private void addPackMethodForOrdinalEnumTypes(CtClass packerCtClass,
+			Class c) throws CannotCompileException, NotFoundException {
+		// void pack(Packer pk, Object target) throws IOException;
+		StringBuilder sb = new StringBuilder();
+		StringBuilder bsb = new StringBuilder();
+		// FIXME
+		insertLocalVariableDecl(bsb, c, VARIABLE_NAME_TARGET);
+		StringBuilder mc = new StringBuilder();
+		insertTypeCast(mc, c, VARIABLE_NAME_OBJECT);
+		insertValueInsertion(bsb, mc.toString());
+		insertSemicolon(bsb);
+		insertMethodCall(bsb, VARIABLE_NAME_PK, METHOD_NAME_PACKARRAY,
+				new String[] { new Integer(1).toString() });
+		insertSemicolon(bsb);
+		StringBuilder fa = new StringBuilder();
+		insertTypeCast(fa, Enum.class, VARIABLE_NAME_TARGET);
+		String t = fa.toString();
+		fa = new StringBuilder();
+		insertMethodCall(fa, t, METHOD_NAME_ORDINAL, new String[0]);
+		insertMethodCall(bsb, VARIABLE_NAME_PK, METHOD_NAME_PACK, new String[] { fa.toString() });
+		insertSemicolon(bsb);
+		addPublicMethodDecl(sb, METHOD_NAME_PACK, void.class, new Class[] {
+				Packer.class, Object.class }, new String[] { VARIABLE_NAME_PK,
+				VARIABLE_NAME_OBJECT }, new Class[] { IOException.class }, bsb
+				.toString());
+		System.out.println("pack method: " + sb.toString());
+		try {
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), packerCtClass);
+			packerCtClass.addMethod(newCtMethod);
+		} catch (CannotCompileException e) {
+			throw new CannotCompileException(e.getMessage() + ": " + sb.toString(), e);
+		}
+	}
+
 	private void addUnpackMethod(CtClass unpackerCtClass, Class c, Field[] fs)
 			throws CannotCompileException, NotFoundException {
-		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
 		insertUnpackMethodBody(bsb, c, fs);
@@ -299,9 +405,13 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				new String[] { VARIABLE_NAME_PK }, new Class[] {
 						MessageTypeException.class, IOException.class }, bsb
 						.toString());
-		// System.out.println("unpack method: " + sb.toString());
-		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), unpackerCtClass);
-		unpackerCtClass.addMethod(newCtMethod);
+		System.out.println("unpack method: " + sb.toString());
+		try {
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), unpackerCtClass);
+			unpackerCtClass.addMethod(newCtMethod);
+		} catch (CannotCompileException e) {
+			throw new CannotCompileException(e.getMessage() + ": " + sb.toString(), e);
+		}
 	}
 
 	private void insertUnpackMethodBody(StringBuilder sb, Class c, Field[] fs) {
@@ -360,9 +470,10 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			throw new UnsupportedOperationException("not supported yet. : "
 					+ c.getName());
 		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
-			// FIXME OrdinalEnumPacker
-			throw new UnsupportedOperationException("not supported yet. : "
-					+ c.getName());
+			// @MessagePackOrdinalEnum
+			Template tmpl = DynamicCodeGenOrdinalEnumTemplate.create(c);
+			CustomMessage.registerTemplate(c, tmpl);
+			insertCodeOfUnpackMethodCallForRegisteredType(sb, f, c);
 		} else {
 			throw new MessageTypeException("unknown type:  " + c.getName());
 		}
@@ -639,6 +750,29 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 	}
 
+	private void addUnpackMethodForOrdinalEnumTypes(CtClass unpackerCtClass,
+			Class c) throws CannotCompileException, NotFoundException {
+		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
+		StringBuilder sb = new StringBuilder();
+		StringBuilder bsb = new StringBuilder();
+		// FIXME
+		insertMethodCall(bsb, VARIABLE_NAME_PK, METHOD_NAME_UNPACKARRAY,
+				new String[0]);
+		insertSemicolon(bsb);
+		//insertUnpackMethodBody(bsb, c, new Field[0]);
+		bsb.append("int _$$_i = _$$_pk.unpackInt();");
+		bsb.append("return " + c.getName()
+				+ ".class.getEnumConstants()[_$$_i];");
+		addPublicMethodDecl(sb, METHOD_NAME_UNPACK, Object.class,
+				new Class[] { Unpacker.class },
+				new String[] { VARIABLE_NAME_PK }, new Class[] {
+						MessageTypeException.class, IOException.class }, bsb
+						.toString());
+		System.out.println("unpack method: " + sb.toString());
+		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), unpackerCtClass);
+		unpackerCtClass.addMethod(newCtMethod);
+	}
+
 	public void addConvertMethod(CtClass tmplCtClass, Class c, Field[] fs)
 			throws CannotCompileException, NotFoundException {
 		// Object convert(MessagePackObject from) throws MessageTypeException;
@@ -649,20 +783,20 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				new Class[] { MessagePackObject.class },
 				new String[] { VARIABLE_NAME_MPO },
 				new Class[] { MessageTypeException.class }, bsb.toString());
-		// System.out.println("convert method: " + sb.toString());
+		System.out.println("convert method: " + sb.toString());
 		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
 		tmplCtClass.addMethod(newCtMethod);
 	}
 
 	private void insertConvertMethodBody(StringBuilder sb, Class c,
-			Field[] fields) throws CannotCompileException {
+			Field[] fs) throws CannotCompileException {
 		insertLocalVariableDecl(sb, c, VARIABLE_NAME_TARGET);
 		StringBuilder mc = new StringBuilder();
 		insertDefaultConsCall(mc, c);
 		insertValueInsertion(sb, mc.toString());
 		insertSemicolon(sb);
 		insertCodeOfMessagePackObjectArrayGet(sb);
-		insertCodeOfConvertMethodCalls(sb, fields);
+		insertCodeOfConvertMethodCalls(sb, fs);
 		insertReturnStat(sb, VARIABLE_NAME_TARGET);
 		insertSemicolon(sb);
 	}
@@ -726,9 +860,10 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 						+ c.getName());
 			} else if (CustomMessage.isAnnotated(c,
 					MessagePackOrdinalEnum.class)) {
-				// FIXME OrdinalEnumPacker
-				throw new UnsupportedOperationException("not supported yet. : "
-						+ c.getName());
+				// @MessagePackMessage
+				Template tmpl = DynamicCodeGenOrdinalEnumTemplate.create(c);
+				CustomMessage.registerTemplate(c, tmpl);
+				insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
 			} else {
 				throw new MessageTypeException("Type error: " + c.getName());
 			}
@@ -1185,6 +1320,30 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 	}
 
+	public void addConvertMethodForOrdinalEnumTypes(CtClass tmplCtClass,
+			Class c) throws CannotCompileException, NotFoundException {
+		// Object convert(MessagePackObject from) throws MessageTypeException;
+		StringBuilder sb = new StringBuilder();
+		StringBuilder bsb = new StringBuilder();
+		insertCodeOfMessagePackObjectArrayGet(bsb);
+		//insertConvertMethodBody(bsb, c, new Field[0]);
+		// FIXME
+		//bsb.append("_$$_ary[0].asInt(); ");
+		//bsb.append("int i = _$$_ary[1].asInt(); ");
+		bsb.append("int i = _$$_ary[0].asInt(); ");
+		bsb.append("java.lang.Object o = ").append(c.getName()).append(
+				".class.getEnumConstants()[i]; ");
+		bsb.append("return (").append(c.getName()).append(") o; ");
+
+		addPublicMethodDecl(sb, METHOD_NAME_CONVERT, Object.class,
+				new Class[] { MessagePackObject.class },
+				new String[] { VARIABLE_NAME_MPO },
+				new Class[] { MessageTypeException.class }, bsb.toString());
+		System.out.println("convert method: " + sb.toString());
+		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
+		tmplCtClass.addMethod(newCtMethod);
+	}
+
 	private Class createClass(CtClass packerCtClass)
 			throws CannotCompileException {
 		return packerCtClass.toClass(null, null);
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumConverter.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumConverter.java
new file mode 100644
index 0000000..298d96c
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumConverter.java
@@ -0,0 +1,17 @@
+package org.msgpack.util.codegen;
+
+import org.msgpack.MessageConverter;
+
+public class DynamicCodeGenOrdinalEnumConverter {
+	public static MessageConverter create(Class c) {
+		try {
+			DynamicCodeGen gen = DynamicCodeGen.getInstance();
+			Class unpackerClass = gen.generateOrdinalEnumConverterClass(c);
+			return (MessageConverter) unpackerClass.newInstance();
+		} catch (InstantiationException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalAccessException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumPacker.java
new file mode 100644
index 0000000..73da950
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumPacker.java
@@ -0,0 +1,17 @@
+package org.msgpack.util.codegen;
+
+import org.msgpack.MessagePacker;
+
+public class DynamicCodeGenOrdinalEnumPacker {
+	public static MessagePacker create(Class c) {
+		try {
+			DynamicCodeGen gen = DynamicCodeGen.getInstance();
+			Class packerClass = gen.generateOrdinalEnumPackerClass(c);
+			return (MessagePacker)packerClass.newInstance();
+		} catch (InstantiationException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalAccessException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumTemplate.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumTemplate.java
new file mode 100644
index 0000000..5b53d8e
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumTemplate.java
@@ -0,0 +1,17 @@
+package org.msgpack.util.codegen;
+
+import org.msgpack.Template;
+
+public class DynamicCodeGenOrdinalEnumTemplate {
+	public static Template create(Class c) {
+		try {
+			DynamicCodeGen gen = DynamicCodeGen.getInstance();
+			Class tmplClass = gen.generateOrdinalEnumTemplateClass(c);
+			return (Template) tmplClass.newInstance();
+		} catch (InstantiationException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalAccessException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumUnpacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumUnpacker.java
new file mode 100644
index 0000000..4ec4880
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumUnpacker.java
@@ -0,0 +1,17 @@
+package org.msgpack.util.codegen;
+
+import org.msgpack.MessageUnpacker;
+
+public class DynamicCodeGenOrdinalEnumUnpacker {
+	public static MessageUnpacker create(Class c) {
+		try {
+			DynamicCodeGen gen = DynamicCodeGen.getInstance();
+			Class unpackerClass = gen.generateOrdinalEnumUnpackerClass(c);
+			return (MessageUnpacker) unpackerClass.newInstance();
+		} catch (InstantiationException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalAccessException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		}
+	}
+}
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
index b737674..55e761b 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
@@ -21,6 +21,7 @@ import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
 import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.annotation.MessagePackOrdinalEnum;
 
 public class TestDynamicCodeGenPackerConverter extends TestCase {
 
@@ -372,9 +373,9 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	public void testFinalClassAndAbstractClass01() throws Exception {
 		try {
 			DynamicCodeGenPacker.create(FinalModifierClass.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
 		}
 		assertTrue(true);
 		try {
@@ -390,9 +391,9 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	public void testFinalClassAndAbstractClass02() throws Exception {
 		try {
 			DynamicCodeGenTemplate.create(FinalModifierClass.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
 		}
 		assertTrue(true);
 		try {
@@ -429,7 +430,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	}
 
 	@Test
-	public void testInterfaceAndEnumType02() throws Exception {
+	public void testInterfaceType() throws Exception {
 		try {
 			DynamicCodeGenTemplate.create(SampleInterface.class);
 			fail();
@@ -437,19 +438,44 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 			assertTrue(true);
 		}
 		assertTrue(true);
-		try {
-			DynamicCodeGenTemplate.create(SampleEnum.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
 	}
 
 	public interface SampleInterface {
 	}
 
+	@Test
+	public void testEnumTypeForOrdinal() throws Exception {
+		SampleEnumFieldClass src = new SampleEnumFieldClass();
+		src.f0 = 0;
+		src.f1 = SampleEnum.ONE;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(SampleEnumFieldClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate
+				.create(SampleEnumFieldClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+	}
+
+	public static class SampleEnumFieldClass {
+		public int f0;
+
+		public SampleEnum f1;
+
+		public SampleEnumFieldClass() {
+		}
+	}
+
+	@MessagePackOrdinalEnum
 	public enum SampleEnum {
+		ONE, TWO, THREE;
 	}
 
 	@Test
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
index ebc6d15..8206aa4 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
@@ -18,6 +18,7 @@ import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
 import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.annotation.MessagePackOrdinalEnum;
 
 import junit.framework.TestCase;
 
@@ -351,9 +352,9 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	public void testFinalClassAndAbstractClass01() throws Exception {
 		try {
 			DynamicCodeGenPacker.create(FinalModifierClass.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
 		}
 		assertTrue(true);
 		try {
@@ -369,9 +370,9 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	public void testFinalClassAndAbstractClass02() throws Exception {
 		try {
 			DynamicCodeGenUnpacker.create(FinalModifierClass.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
+		} catch (DynamicCodeGenException e) {
+			fail();
 		}
 		assertTrue(true);
 		try {
@@ -390,7 +391,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testInterfaceAndEnumType01() throws Exception {
+	public void testInterfaceType01() throws Exception {
 		try {
 			DynamicCodeGenPacker.create(SampleInterface.class);
 			fail();
@@ -398,17 +399,10 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 			assertTrue(true);
 		}
 		assertTrue(true);
-		try {
-			DynamicCodeGenPacker.create(SampleEnum.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
 	}
 
 	@Test
-	public void testInterfaceAndEnumType02() throws Exception {
+	public void testInterfaceType02() throws Exception {
 		try {
 			DynamicCodeGenUnpacker.create(SampleInterface.class);
 			fail();
@@ -416,19 +410,41 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 			assertTrue(true);
 		}
 		assertTrue(true);
-		try {
-			DynamicCodeGenUnpacker.create(SampleEnum.class);
-			fail();
-		} catch (DynamicCodeGenException e) {
-			assertTrue(true);
-		}
-		assertTrue(true);
 	}
 
 	public interface SampleInterface {
 	}
 
+	@Test
+	public void testEnumTypeForOrdinal() throws Exception {
+		SampleEnumFieldClass src = new SampleEnumFieldClass();
+		src.f0 = 0;
+		src.f1 = SampleEnum.ONE;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicCodeGenPacker
+				.create(SampleEnumFieldClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicCodeGenTemplate
+				.create(SampleEnumFieldClass.class);
+		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl
+				.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+	}
+
+	public static class SampleEnumFieldClass {
+		public int f0;
+
+		public SampleEnum f1;
+
+		public SampleEnumFieldClass() {
+		}
+	}
+
+	@MessagePackOrdinalEnum
 	public enum SampleEnum {
+		ONE, TWO, THREE;
 	}
 
 	@Test

From d8b4051d6d78ad1437f24bffdf31beac4040d128 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 2 Oct 2010 21:24:22 +0900
Subject: [PATCH 0825/1648] java: edit pom.xml to enable to using slf4j logger

---
 java/pom.xml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/java/pom.xml b/java/pom.xml
index eb7930d..beec503 100755
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -28,6 +28,16 @@
       junit
       4.8.1
       test
+    
+      
+      org.slf4j
+      slf4j-api
+      1.4.3
+    
+    
+      org.slf4j
+      slf4j-log4j12
+      1.4.3
     
     
       javassist

From fc5bc84207247cd07e5c7b3c938f307f364e7bff Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 2 Oct 2010 21:25:30 +0900
Subject: [PATCH 0826/1648] java: add a log4j.properties file in
 src/main/java/resources

---
 java/src/main/resources/log4j.properties | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 java/src/main/resources/log4j.properties

diff --git a/java/src/main/resources/log4j.properties b/java/src/main/resources/log4j.properties
new file mode 100644
index 0000000..6fae205
--- /dev/null
+++ b/java/src/main/resources/log4j.properties
@@ -0,0 +1,14 @@
+### direct log messages to stdout ###
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d %5p %c{1} - %m%n
+
+### direct messages to file mylog.log ###
+log4j.appender.file=org.apache.log4j.FileAppender
+log4j.appender.file.File=mylog.log
+log4j.appender.file.Append=true
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d %5p %c{1} - %m%n
+
+log4j.rootLogger=info, stdout

From a3accd28ea2c1d6485e25ee7dfbc1168b2428fb4 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 2 Oct 2010 21:28:21 +0900
Subject: [PATCH 0827/1648] java: insert logging codes into
 CustomConverter.java, CustomPacker.java, CustomUnpacker.java and util.codegen
 programs

---
 .../java/org/msgpack/CustomConverter.java     |   8 +-
 .../main/java/org/msgpack/CustomPacker.java   |   7 +
 .../main/java/org/msgpack/CustomUnpacker.java |   8 +-
 .../msgpack/util/codegen/DynamicCodeGen.java  | 440 ++++++++++--------
 .../util/codegen/DynamicCodeGenBase.java      |   9 +-
 .../TestDynamicCodeGenPackerConverter.java    |  22 +-
 .../TestDynamicCodeGenPackerUnpacker.java     |  22 +-
 7 files changed, 288 insertions(+), 228 deletions(-)

diff --git a/java/src/main/java/org/msgpack/CustomConverter.java b/java/src/main/java/org/msgpack/CustomConverter.java
index abbc88a..a82e3e1 100644
--- a/java/src/main/java/org/msgpack/CustomConverter.java
+++ b/java/src/main/java/org/msgpack/CustomConverter.java
@@ -19,11 +19,17 @@ package org.msgpack;
 
 import java.util.concurrent.ConcurrentHashMap;
 
-// FIXME package private?
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class CustomConverter {
+	private static Logger LOG = LoggerFactory.getLogger(CustomConverter.class);
+	
 	private static ConcurrentHashMap, MessageConverter> map = new ConcurrentHashMap, MessageConverter>();
 	
 	public static void register(Class target, MessageConverter converter) {
+		LOG.debug("register a MessageConverter object for the type: "
+				+ target.getName());
 		map.putIfAbsent(target, converter);
 	}
 
diff --git a/java/src/main/java/org/msgpack/CustomPacker.java b/java/src/main/java/org/msgpack/CustomPacker.java
index 743a52b..6a65fc3 100644
--- a/java/src/main/java/org/msgpack/CustomPacker.java
+++ b/java/src/main/java/org/msgpack/CustomPacker.java
@@ -19,10 +19,17 @@ package org.msgpack;
 
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class CustomPacker {
+	private static Logger LOG = LoggerFactory.getLogger(CustomPacker.class);
+
 	private static ConcurrentHashMap, MessagePacker> map = new ConcurrentHashMap, MessagePacker>();
 
 	public static void register(Class target, MessagePacker packer) {
+		LOG.debug("register a MessagePacker object for the type: "
+				+ target.getName());
 		map.putIfAbsent(target, packer);
 	}
 
diff --git a/java/src/main/java/org/msgpack/CustomUnpacker.java b/java/src/main/java/org/msgpack/CustomUnpacker.java
index b45292b..8e80115 100644
--- a/java/src/main/java/org/msgpack/CustomUnpacker.java
+++ b/java/src/main/java/org/msgpack/CustomUnpacker.java
@@ -19,11 +19,17 @@ package org.msgpack;
 
 import java.util.concurrent.ConcurrentHashMap;
 
-// FIXME package private?
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class CustomUnpacker {
+	private static Logger LOG = LoggerFactory.getLogger(CustomUnpacker.class);
+
 	private static ConcurrentHashMap, MessageUnpacker> map = new ConcurrentHashMap, MessageUnpacker>();
 
 	public static void register(Class target, MessageUnpacker converter) {
+		LOG.debug("register a MessageUnpacker object for the type: "
+				+ target.getName());
 		map.putIfAbsent(target, converter);
 	}
 
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index c7f5dd3..efe9a69 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -24,6 +24,7 @@ import javassist.NotFoundException;
 
 import org.msgpack.CustomConverter;
 import org.msgpack.CustomMessage;
+import org.msgpack.CustomPacker;
 import org.msgpack.CustomUnpacker;
 import org.msgpack.MessageConvertable;
 import org.msgpack.MessagePackObject;
@@ -38,9 +39,13 @@ import org.msgpack.Unpacker;
 import org.msgpack.annotation.MessagePackDelegate;
 import org.msgpack.annotation.MessagePackMessage;
 import org.msgpack.annotation.MessagePackOrdinalEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
+	private static Logger LOG = LoggerFactory.getLogger(DynamicCodeGen.class);
+
 	private static DynamicCodeGen INSTANCE;
 
 	private static AtomicInteger COUNTER = new AtomicInteger(0);
@@ -51,6 +56,8 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	public static DynamicCodeGen getInstance() {
 		if (INSTANCE == null) {
+			LOG.info("create an instance of the type: "
+					+ DynamicCodeGen.class.getName());
 			INSTANCE = new DynamicCodeGen();
 		}
 		return INSTANCE;
@@ -63,6 +70,8 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	}
 
 	public Class generateMessagePackerClass(Class origClass) {
+		LOG.debug("start generating a MessagePacker impl.: "
+				+ origClass.getName());
 		try {
 			String origName = origClass.getName();
 			String packerName = origName + POSTFIX_TYPE_NAME_PACKER + inc();
@@ -75,31 +84,38 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			addPackMethod(packerCtClass, origClass, fields);
 			return createClass(packerCtClass);
 		} catch (NotFoundException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (CannotCompileException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 
 	public Class generateOrdinalEnumPackerClass(Class origClass) {
+		LOG.debug("start generating a OrdinalEnumPacker impl.: "
+				+ origClass.getName());
 		try {
 			String origName = origClass.getName();
 			String packerName = origName + POSTFIX_TYPE_NAME_PACKER + inc();
 			checkClassValidation(origClass);
-			//checkDefaultConstructorValidation(origClass);
 			CtClass packerCtClass = pool.makeClass(packerName);
 			setInterface(packerCtClass, MessagePacker.class);
 			addDefaultConstructor(packerCtClass);
 			addPackMethodForOrdinalEnumTypes(packerCtClass, origClass);
 			return createClass(packerCtClass);
 		} catch (NotFoundException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (CannotCompileException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 
 	public Class generateMessageUnpackerClass(Class origClass) {
+		LOG.debug("start generating a MessageUnpacker impl.: "
+				+ origClass.getName());
 		try {
 			String origName = origClass.getName();
 			String unpackerName = origName + POSTFIX_TYPE_NAME_UNPACKER + inc();
@@ -112,13 +128,17 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			addUnpackMethod(unpackerCtClass, origClass, fields);
 			return createClass(unpackerCtClass);
 		} catch (NotFoundException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (CannotCompileException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 
 	public Class generateOrdinalEnumUnpackerClass(Class origClass) {
+		LOG.debug("start generating a OrdinalEnumUnpacker impl.: "
+				+ origClass.getName());
 		try {
 			String origName = origClass.getName();
 			String unpackerName = origName + POSTFIX_TYPE_NAME_UNPACKER + inc();
@@ -130,13 +150,17 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			addUnpackMethodForOrdinalEnumTypes(unpackerCtClass, origClass);
 			return createClass(unpackerCtClass);
 		} catch (NotFoundException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (CannotCompileException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 
 	public Class generateMessageConverterClass(Class origClass) {
+		LOG.debug("start generating a MessageConverter impl.: "
+				+ origClass.getName());
 		try {
 			String origName = origClass.getName();
 			String convName = origName + POSTFIX_TYPE_NAME_CONVERTER + inc();
@@ -149,31 +173,37 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			addConvertMethod(converterCtClass, origClass, fields);
 			return createClass(converterCtClass);
 		} catch (NotFoundException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (CannotCompileException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 
 	public Class generateOrdinalEnumConverterClass(Class origClass) {
+		LOG.debug("start generating a OrdinalEnumConverter impl.: "
+				+ origClass.getName());
 		try {
 			String origName = origClass.getName();
 			String convName = origName + POSTFIX_TYPE_NAME_CONVERTER + inc();
 			checkClassValidation(origClass);
-			//checkDefaultConstructorValidation(origClass);
 			CtClass converterCtClass = pool.makeClass(convName);
 			setInterface(converterCtClass, MessageUnpacker.class);
 			addDefaultConstructor(converterCtClass);
 			addConvertMethodForOrdinalEnumTypes(converterCtClass, origClass);
 			return createClass(converterCtClass);
 		} catch (NotFoundException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (CannotCompileException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 
 	public Class generateTemplateClass(Class origClass) {
+		LOG.debug("start generating a Template impl.: " + origClass.getName());
 		try {
 			String origName = origClass.getName();
 			String tmplName = origName + POSTFIX_TYPE_NAME_TEMPLATE + inc();
@@ -187,18 +217,21 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			addConvertMethod(tmplCtClass, origClass, fields);
 			return createClass(tmplCtClass);
 		} catch (NotFoundException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (CannotCompileException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 
 	public Class generateOrdinalEnumTemplateClass(Class origClass) {
+		LOG.debug("start generating a OrdinalEnumTemplate impl.: "
+				+ origClass.getName());
 		try {
 			String origName = origClass.getName();
 			String tmplName = origName + POSTFIX_TYPE_NAME_TEMPLATE + inc();
 			checkClassValidation(origClass);
-			//checkDefaultConstructorValidation(origClass);
 			CtClass tmplCtClass = pool.makeClass(tmplName);
 			setInterface(tmplCtClass, Template.class);
 			addDefaultConstructor(tmplCtClass);
@@ -206,8 +239,10 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			addConvertMethodForOrdinalEnumTypes(tmplCtClass, origClass);
 			return createClass(tmplCtClass);
 		} catch (NotFoundException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (CannotCompileException e) {
+			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
@@ -215,8 +250,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	private void checkClassValidation(Class origClass) {
 		// not public, abstract
 		int mod = origClass.getModifiers();
-		if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
-				|| Modifier.isAbstract(mod)) {
+		if ((!Modifier.isPublic(mod)) || Modifier.isAbstract(mod)) {
 			throwClassValidationException(origClass,
 					"a class must have a public modifier");
 		}
@@ -229,7 +263,10 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private static void throwClassValidationException(Class origClass,
 			String msg) {
-		throw new DynamicCodeGenException(msg + ": " + origClass.getName());
+		DynamicCodeGenException e = new DynamicCodeGenException(msg + ": "
+				+ origClass.getName());
+		LOG.error(e.getMessage(), e);
+		throw e;
 	}
 
 	private void checkDefaultConstructorValidation(Class origClass) {
@@ -237,18 +274,20 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		try {
 			cons = origClass.getDeclaredConstructor(new Class[0]);
 		} catch (Exception e1) {
-			throwConstructoValidationException(origClass);
+			throwConstructorValidationException(origClass);
 		}
 		int mod = cons.getModifiers();
-		if (!(Modifier.isPublic(mod) || Modifier.isProtected(mod))) {
-			throwConstructoValidationException(origClass);
+		if (!Modifier.isPublic(mod)) {
+			throwConstructorValidationException(origClass);
 		}
 	}
 
-	private static void throwConstructoValidationException(Class origClass) {
-		throw new DynamicCodeGenException(
+	private static void throwConstructorValidationException(Class origClass) {
+		DynamicCodeGenException e = new DynamicCodeGenException(
 				"it must have a public zero-argument constructor: "
 						+ origClass.getName());
+		LOG.error(e.getMessage(), e);
+		throw e;
 	}
 
 	private void setInterface(CtClass packerCtClass, Class infClass)
@@ -257,11 +296,11 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		packerCtClass.addInterface(infCtClass);
 	}
 
-	private void addDefaultConstructor(CtClass enhCtClass)
+	private void addDefaultConstructor(CtClass dstCtClass)
 			throws CannotCompileException {
 		CtConstructor newCtCons = CtNewConstructor
-				.defaultConstructor(enhCtClass);
-		enhCtClass.addConstructor(newCtCons);
+				.defaultConstructor(dstCtClass);
+		dstCtClass.addConstructor(newCtCons);
 	}
 
 	private Field[] getDeclaredFields(Class origClass) {
@@ -281,29 +320,30 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		return allFields.toArray(new Field[0]);
 	}
 
-	private void checkFieldValidation(Field field, List fields) {
-		// check modifiers (public or protected)
-		int mod = field.getModifiers();
-		if ((!(Modifier.isPublic(mod) || Modifier.isProtected(mod)))
-				|| Modifier.isStatic(mod) || Modifier.isFinal(mod)
-				|| Modifier.isTransient(mod) || field.isSynthetic()) {
-			throwFieldValidationException(field);
+	private void checkFieldValidation(Field f, List fs) {
+		// check that it has a public modifier
+		int mod = f.getModifiers();
+		if ((!(Modifier.isPublic(mod))) || Modifier.isStatic(mod)
+				|| Modifier.isFinal(mod) || Modifier.isTransient(mod)
+				|| f.isSynthetic()) {
+			throwFieldValidationException(f);
 		}
 		// check same name
-		for (Field f : fields) {
-			if (f.getName().equals(field.getName())) {
-				throwFieldValidationException(field);
+		for (Field f0 : fs) {
+			if (f0.getName().equals(f.getName())) {
+				throwFieldValidationException(f);
 			}
 		}
 	}
 
-	private static void throwFieldValidationException(Field field) {
-		throw new DynamicCodeGenException("it must be a public field: "
-				+ field.getName());
+	private static void throwFieldValidationException(Field f) {
+		DynamicCodeGenException e = new DynamicCodeGenException(
+				"it must be a public field: " + f.getName());
+		LOG.debug(e.getMessage(), e);
+		throw e;
 	}
 
-	private void addPackMethod(CtClass packerCtClass, Class c, Field[] fs)
-			throws CannotCompileException, NotFoundException {
+	private void addPackMethod(CtClass packerCtClass, Class c, Field[] fs) {
 		// void pack(Packer pk, Object target) throws IOException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
@@ -312,9 +352,18 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				Packer.class, Object.class }, new String[] { VARIABLE_NAME_PK,
 				VARIABLE_NAME_OBJECT }, new Class[] { IOException.class }, bsb
 				.toString());
-		System.out.println("pack method: " + sb.toString());
-		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), packerCtClass);
-		packerCtClass.addMethod(newCtMethod);
+		LOG.trace("pack method src: " + sb.toString());
+		try {
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(),
+					packerCtClass);
+			packerCtClass.addMethod(newCtMethod);
+		} catch (CannotCompileException e) {
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage()
+					+ ": " + sb.toString(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
+		}
 	}
 
 	private void insertPackMethodBody(StringBuilder sb, Class c, Field[] fs) {
@@ -333,26 +382,41 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private void insertCodeOfPackMethodCall(StringBuilder sb, Field field) {
 		Class c = field.getType();
-		if (c.isPrimitive() || c.equals(Boolean.class) || c.equals(Byte.class)
+		if (c.isPrimitive()) {
+		} else if (c.equals(Boolean.class) || c.equals(Byte.class)
 				|| c.equals(Double.class) || c.equals(Float.class)
 				|| c.equals(Integer.class) || c.equals(Long.class)
-				|| c.equals(Short.class) || List.class.isAssignableFrom(c)
-				|| Map.class.isAssignableFrom(c)
-				|| CustomUnpacker.isRegistered(c)
-				|| MessagePackable.class.isAssignableFrom(c)) {
-			;
+				|| c.equals(Short.class)) {
+			; // ignore
+		} else if (c.equals(String.class) || c.equals(BigInteger.class)
+				|| c.equals(byte[].class)) {
+			; // ignore
+		} else if (List.class.isAssignableFrom(c)
+				|| Map.class.isAssignableFrom(c)) {
+			; // ignore
+		} else if (CustomPacker.isRegistered(c)) {
+			; // ignore
+		} else if (MessagePackable.class.isAssignableFrom(c)) {
+			; // ignore
 		} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
 			// @MessagePackMessage
 			MessagePacker packer = DynamicCodeGenPacker.create(c);
 			CustomMessage.registerPacker(c, packer);
 		} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
 			// FIXME DelegatePacker
-			throw new UnsupportedOperationException("not supported yet. : "
-					+ c.getName());
+			UnsupportedOperationException e = new UnsupportedOperationException(
+					"not supported yet. : " + c.getName());
+			LOG.error(e.getMessage(), e);
+			throw e;
 		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
 			// @MessagePackOrdinalEnum
 			MessagePacker packer = DynamicCodeGenOrdinalEnumPacker.create(c);
 			CustomMessage.registerPacker(c, packer);
+		} else {
+			MessageTypeException e = new MessageTypeException("unknown type: "
+					+ c.getName());
+			LOG.error(e.getMessage(), e);
+			throw e;
 		}
 		StringBuilder fa = new StringBuilder();
 		insertFieldAccess(fa, VARIABLE_NAME_TARGET, field.getName());
@@ -366,37 +430,43 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// void pack(Packer pk, Object target) throws IOException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
-		// FIXME
 		insertLocalVariableDecl(bsb, c, VARIABLE_NAME_TARGET);
-		StringBuilder mc = new StringBuilder();
-		insertTypeCast(mc, c, VARIABLE_NAME_OBJECT);
-		insertValueInsertion(bsb, mc.toString());
+		StringBuilder fa = new StringBuilder();
+		insertTypeCast(fa, c, VARIABLE_NAME_OBJECT);
+		insertValueInsertion(bsb, fa.toString());
 		insertSemicolon(bsb);
 		insertMethodCall(bsb, VARIABLE_NAME_PK, METHOD_NAME_PACKARRAY,
 				new String[] { new Integer(1).toString() });
 		insertSemicolon(bsb);
-		StringBuilder fa = new StringBuilder();
+		fa = new StringBuilder();
 		insertTypeCast(fa, Enum.class, VARIABLE_NAME_TARGET);
 		String t = fa.toString();
 		fa = new StringBuilder();
 		insertMethodCall(fa, t, METHOD_NAME_ORDINAL, new String[0]);
-		insertMethodCall(bsb, VARIABLE_NAME_PK, METHOD_NAME_PACK, new String[] { fa.toString() });
+		insertMethodCall(bsb, VARIABLE_NAME_PK, METHOD_NAME_PACK,
+				new String[] { fa.toString() });
 		insertSemicolon(bsb);
 		addPublicMethodDecl(sb, METHOD_NAME_PACK, void.class, new Class[] {
 				Packer.class, Object.class }, new String[] { VARIABLE_NAME_PK,
 				VARIABLE_NAME_OBJECT }, new Class[] { IOException.class }, bsb
 				.toString());
-		System.out.println("pack method: " + sb.toString());
+		LOG.trace("pack method src: " + sb.toString());
 		try {
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), packerCtClass);
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(),
+					packerCtClass);
 			packerCtClass.addMethod(newCtMethod);
 		} catch (CannotCompileException e) {
-			throw new CannotCompileException(e.getMessage() + ": " + sb.toString(), e);
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage()
+					+ ": " + sb.toString(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
 		}
 	}
 
 	private void addUnpackMethod(CtClass unpackerCtClass, Class c, Field[] fs)
 			throws CannotCompileException, NotFoundException {
+		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
 		insertUnpackMethodBody(bsb, c, fs);
@@ -405,12 +475,17 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				new String[] { VARIABLE_NAME_PK }, new Class[] {
 						MessageTypeException.class, IOException.class }, bsb
 						.toString());
-		System.out.println("unpack method: " + sb.toString());
+		LOG.trace("unpack method src: " + sb.toString());
 		try {
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), unpackerCtClass);
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(),
+					unpackerCtClass);
 			unpackerCtClass.addMethod(newCtMethod);
 		} catch (CannotCompileException e) {
-			throw new CannotCompileException(e.getMessage() + ": " + sb.toString(), e);
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage()
+					+ ": " + sb.toString(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
 		}
 	}
 
@@ -435,18 +510,14 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		if (c.isPrimitive()) {
 			// primitive type
 			insertCodeOfUnpackMethodCallForPrimTypes(sb, f, c);
-		} else if (c.equals(Boolean.class) || // Boolean
-				c.equals(Byte.class) || // Byte
-				c.equals(Double.class) || // Double
-				c.equals(Float.class) || // Float
-				c.equals(Integer.class) || // Integer
-				c.equals(Long.class) || // Long
-				c.equals(Short.class)) { // Short
+		} else if (c.equals(Boolean.class) || c.equals(Byte.class)
+				|| c.equals(Double.class) || c.equals(Float.class)
+				|| c.equals(Integer.class) || c.equals(Long.class)
+				|| c.equals(Short.class)) {
 			// reference type (wrapper type)
 			insertCodeOfUnpackMethodCallForWrapTypes(sb, f, c);
-		} else if (c.equals(BigInteger.class) || // BigInteger
-				c.equals(String.class) || // String
-				c.equals(byte[].class)) { // byte[]
+		} else if (c.equals(BigInteger.class) || c.equals(String.class)
+				|| c.equals(byte[].class)) {
 			// reference type (other type)
 			insertCodeOfUnpackMethodCallForPrimTypes(sb, f, c);
 		} else if (List.class.isAssignableFrom(c)) {
@@ -467,27 +538,28 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			insertCodeOfUnpackMethodCallForRegisteredType(sb, f, c);
 		} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
 			// FIXME DelegatePacker
-			throw new UnsupportedOperationException("not supported yet. : "
-					+ c.getName());
+			UnsupportedOperationException e = new UnsupportedOperationException(
+					"not supported yet. : " + c.getName());
+			LOG.error(e.getMessage(), e);
+			throw e;
 		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
 			// @MessagePackOrdinalEnum
 			Template tmpl = DynamicCodeGenOrdinalEnumTemplate.create(c);
 			CustomMessage.registerTemplate(c, tmpl);
 			insertCodeOfUnpackMethodCallForRegisteredType(sb, f, c);
 		} else {
-			throw new MessageTypeException("unknown type:  " + c.getName());
+			MessageTypeException e = new MessageTypeException("unknown type:  "
+					+ c.getName());
+			LOG.error(e.getMessage(), e);
+			throw e;
 		}
 	}
 
 	private void insertCodeOfUnpackMethodCallForPrimTypes(StringBuilder sb,
 			Field f, Class c) {
 		if (f != null) {
-			sb.append(VARIABLE_NAME_TARGET);
-			sb.append(CHAR_NAME_DOT);
-			sb.append(f.getName());
-			sb.append(CHAR_NAME_SPACE);
-			sb.append(CHAR_NAME_EQUAL);
-			sb.append(CHAR_NAME_SPACE);
+			insertFieldAccess(sb, VARIABLE_NAME_TARGET, f.getName());
+			insertInsertion(sb);
 		}
 		insertMethodCall(sb, VARIABLE_NAME_PK, getUnpackMethodName(c),
 				new String[0]);
@@ -499,12 +571,8 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	private void insertCodeOfUnpackMethodCallForWrapTypes(StringBuilder sb,
 			Field f, Class c) {
 		if (f != null) {
-			sb.append(VARIABLE_NAME_TARGET);
-			sb.append(CHAR_NAME_DOT);
-			sb.append(f.getName());
-			sb.append(CHAR_NAME_SPACE);
-			sb.append(CHAR_NAME_EQUAL);
-			sb.append(CHAR_NAME_SPACE);
+			insertFieldAccess(sb, VARIABLE_NAME_TARGET, f.getName());
+			insertInsertion(sb);
 		}
 		StringBuilder mc = new StringBuilder();
 		insertMethodCall(mc, VARIABLE_NAME_PK, getUnpackMethodName(c),
@@ -512,8 +580,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		insertMethodCall(sb, c.getName(), METHOD_NAME_VALUEOF,
 				new String[] { mc.toString() });
 		if (f != null) {
-			sb.append(CHAR_NAME_SEMICOLON);
-			sb.append(CHAR_NAME_SPACE);
+			insertSemicolon(sb);
 		}
 	}
 
@@ -683,19 +750,16 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			StringBuilder sb, Field f, Class c) {
 		// target.field = (Cast) CustomUnpacker.get(C.class).unpack(pk);
 		StringBuilder mc = new StringBuilder();
-		insertTypeCast(mc, c);
-		mc.append(CustomUnpacker.class.getName());
+		insertMethodCall(mc, CustomUnpacker.class.getName(), METHOD_NAME_GET,
+				new String[] { c.getName() + ".class" });
 		String t = mc.toString();
 		mc = new StringBuilder();
-		insertMethodCall(mc, t, METHOD_NAME_GET, new String[] { c.getName()
-				+ ".class" });
-		t = mc.toString();
-		mc = new StringBuilder();
 		insertMethodCall(mc, t, METHOD_NAME_UNPACK,
 				new String[] { VARIABLE_NAME_PK });
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(f.getName());
+		t = mc.toString();
+		mc = new StringBuilder();
+		insertTypeCast(mc, c, t);
+		insertFieldAccess(sb, VARIABLE_NAME_TARGET, f.getName());
 		insertValueInsertion(sb, mc.toString());
 		insertSemicolon(sb);
 	}
@@ -751,7 +815,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	}
 
 	private void addUnpackMethodForOrdinalEnumTypes(CtClass unpackerCtClass,
-			Class c) throws CannotCompileException, NotFoundException {
+			Class c) {
 		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
@@ -759,7 +823,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		insertMethodCall(bsb, VARIABLE_NAME_PK, METHOD_NAME_UNPACKARRAY,
 				new String[0]);
 		insertSemicolon(bsb);
-		//insertUnpackMethodBody(bsb, c, new Field[0]);
+		// insertUnpackMethodBody(bsb, c, new Field[0]);
 		bsb.append("int _$$_i = _$$_pk.unpackInt();");
 		bsb.append("return " + c.getName()
 				+ ".class.getEnumConstants()[_$$_i];");
@@ -768,13 +832,21 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				new String[] { VARIABLE_NAME_PK }, new Class[] {
 						MessageTypeException.class, IOException.class }, bsb
 						.toString());
-		System.out.println("unpack method: " + sb.toString());
-		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), unpackerCtClass);
-		unpackerCtClass.addMethod(newCtMethod);
+		LOG.trace("unpack method src: " + sb.toString());
+		try {
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(),
+					unpackerCtClass);
+			unpackerCtClass.addMethod(newCtMethod);
+		} catch (CannotCompileException e) {
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage()
+					+ ": " + sb.toString(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
+		}
 	}
 
-	public void addConvertMethod(CtClass tmplCtClass, Class c, Field[] fs)
-			throws CannotCompileException, NotFoundException {
+	public void addConvertMethod(CtClass tmplCtClass, Class c, Field[] fs) {
 		// Object convert(MessagePackObject from) throws MessageTypeException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
@@ -783,13 +855,21 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				new Class[] { MessagePackObject.class },
 				new String[] { VARIABLE_NAME_MPO },
 				new Class[] { MessageTypeException.class }, bsb.toString());
-		System.out.println("convert method: " + sb.toString());
-		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
-		tmplCtClass.addMethod(newCtMethod);
+		LOG.trace("convert method src: " + sb.toString());
+		try {
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
+			tmplCtClass.addMethod(newCtMethod);
+		} catch (CannotCompileException e) {
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage()
+					+ ": " + sb.toString(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
+		}
 	}
 
 	private void insertConvertMethodBody(StringBuilder sb, Class c,
-			Field[] fs) throws CannotCompileException {
+			Field[] fs) {
 		insertLocalVariableDecl(sb, c, VARIABLE_NAME_TARGET);
 		StringBuilder mc = new StringBuilder();
 		insertDefaultConsCall(mc, c);
@@ -831,42 +911,44 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			Class c, int i, String v) {
 		if (c.isPrimitive()) { // primitive type
 			insertCodeOfConvertMethodCallForPrimTypes(sb, f, c, i, v);
-		} else { // reference type
-			if (c.equals(Boolean.class) || c.equals(Byte.class)
-					|| c.equals(Short.class) || c.equals(Integer.class)
-					|| c.equals(Float.class) || c.equals(Long.class)
-					|| c.equals(Double.class)) {
-				// wrapper type
-				insertCodeOfConvertMethodCallForWrapTypes(sb, f, c, i, v);
-			} else if (c.equals(String.class) || c.equals(byte[].class)
-					|| c.equals(BigInteger.class)) {
-				insertCodeOfConvertMethodCallForPrimTypes(sb, f, c, i, v);
-			} else if (List.class.isAssignableFrom(c)) {
-				insertCodeOfConvertMethodCallForList(sb, f, c, i);
-			} else if (Map.class.isAssignableFrom(c)) {
-				insertCodeOfConvertMethodCallForMapType(sb, f, c, i);
-			} else if (MessageConvertable.class.isAssignableFrom(c)) {
-				insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
-			} else if (CustomConverter.isRegistered(c)) {
-				insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
-			} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
-				// @MessagePackMessage
-				Template tmpl = DynamicCodeGenTemplate.create(c);
-				CustomMessage.registerTemplate(c, tmpl);
-				insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
-			} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
-				// FIXME DelegatePacker
-				throw new UnsupportedOperationException("not supported yet. : "
-						+ c.getName());
-			} else if (CustomMessage.isAnnotated(c,
-					MessagePackOrdinalEnum.class)) {
-				// @MessagePackMessage
-				Template tmpl = DynamicCodeGenOrdinalEnumTemplate.create(c);
-				CustomMessage.registerTemplate(c, tmpl);
-				insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
-			} else {
-				throw new MessageTypeException("Type error: " + c.getName());
-			}
+		} else if (c.equals(Boolean.class) || c.equals(Byte.class)
+				|| c.equals(Short.class) || c.equals(Integer.class)
+				|| c.equals(Float.class) || c.equals(Long.class)
+				|| c.equals(Double.class)) {
+			// reference type (wrapper)
+			insertCodeOfConvertMethodCallForWrapTypes(sb, f, c, i, v);
+		} else if (c.equals(String.class) || c.equals(byte[].class)
+				|| c.equals(BigInteger.class)) {
+			insertCodeOfConvertMethodCallForPrimTypes(sb, f, c, i, v);
+		} else if (List.class.isAssignableFrom(c)) {
+			insertCodeOfConvertMethodCallForList(sb, f, c, i);
+		} else if (Map.class.isAssignableFrom(c)) {
+			insertCodeOfConvertMethodCallForMapType(sb, f, c, i);
+		} else if (MessageConvertable.class.isAssignableFrom(c)) {
+			insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
+		} else if (CustomConverter.isRegistered(c)) {
+			insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
+		} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
+			// @MessagePackMessage
+			Template tmpl = DynamicCodeGenTemplate.create(c);
+			CustomMessage.registerTemplate(c, tmpl);
+			insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
+		} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
+			// FIXME DelegatePacker
+			UnsupportedOperationException e = new UnsupportedOperationException(
+					"not supported yet. : " + c.getName());
+			LOG.error(e.getMessage(), e);
+			throw e;
+		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
+			// @MessagePackMessage
+			Template tmpl = DynamicCodeGenOrdinalEnumTemplate.create(c);
+			CustomMessage.registerTemplate(c, tmpl);
+			insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
+		} else {
+			MessageTypeException e = new MessageTypeException("Type error: "
+					+ c.getName());
+			LOG.error(e.getMessage(), e);
+			throw e;
 		}
 	}
 
@@ -967,34 +1049,11 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			sb.append(name);
 		}
 		sb.append(CHAR_NAME_DOT);
-		if (c.equals(boolean.class)) {
-			sb.append(METHOD_NAME_ASBOOLEAN);
-		} else if (c.equals(byte.class)) {
-			sb.append(METHOD_NAME_ASBYTE);
-		} else if (c.equals(short.class)) {
-			sb.append(METHOD_NAME_ASSHORT);
-		} else if (c.equals(int.class)) {
-			sb.append(METHOD_NAME_ASINT);
-		} else if (c.equals(float.class)) {
-			sb.append(METHOD_NAME_ASFLOAT);
-		} else if (c.equals(long.class)) {
-			sb.append(METHOD_NAME_ASLONG);
-		} else if (c.equals(double.class)) {
-			sb.append(METHOD_NAME_ASDOUBLE);
-		} else if (c.equals(String.class)) {
-			sb.append(METHOD_NAME_ASSTRING);
-		} else if (c.equals(byte[].class)) {
-			sb.append(METHOD_NAME_ASBYTEARRAY);
-		} else if (c.equals(BigInteger.class)) {
-			sb.append(METHOD_NAME_ASBIGINTEGER);
-		} else {
-			throw new MessageTypeException("Type error: " + c.getName());
-		}
+		sb.append(getAsMethodName(c));
 		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
 		if (f != null) {
-			sb.append(CHAR_NAME_SEMICOLON);
-			sb.append(CHAR_NAME_SPACE);
+			insertSemicolon(sb);
 		}
 	}
 
@@ -1010,23 +1069,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 		sb.append(KEYWORD_NEW);
 		sb.append(CHAR_NAME_SPACE);
-		if (c.equals(Boolean.class)) {
-			sb.append(Boolean.class.getName());
-		} else if (c.equals(Byte.class)) {
-			sb.append(Byte.class.getName());
-		} else if (c.equals(Short.class)) {
-			sb.append(Short.class.getName());
-		} else if (c.equals(Integer.class)) {
-			sb.append(Integer.class.getName());
-		} else if (c.equals(Float.class)) {
-			sb.append(Float.class.getName());
-		} else if (c.equals(Long.class)) {
-			sb.append(Long.class.getName());
-		} else if (c.equals(Double.class)) {
-			sb.append(Double.class.getName());
-		} else {
-			throw new MessageTypeException("Type error: " + c.getName());
-		}
+		sb.append(c.getName());
 		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		if (f != null) {
 			sb.append(VARIABLE_NAME_ARRAY);
@@ -1037,29 +1080,12 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			sb.append(v);
 		}
 		sb.append(CHAR_NAME_DOT);
-		if (c.equals(Boolean.class)) {
-			sb.append(METHOD_NAME_ASBOOLEAN);
-		} else if (c.equals(Byte.class)) {
-			sb.append(METHOD_NAME_ASBYTE);
-		} else if (c.equals(Short.class)) {
-			sb.append(METHOD_NAME_ASSHORT);
-		} else if (c.equals(Integer.class)) {
-			sb.append(METHOD_NAME_ASINT);
-		} else if (c.equals(Float.class)) {
-			sb.append(METHOD_NAME_ASFLOAT);
-		} else if (c.equals(Long.class)) {
-			sb.append(METHOD_NAME_ASLONG);
-		} else if (c.equals(Double.class)) {
-			sb.append(METHOD_NAME_ASDOUBLE);
-		} else {
-			throw new MessageTypeException("Type error: " + c.getName());
-		}
+		sb.append(getAsMethodName(c));
 		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
 		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
 		if (f != null) {
-			sb.append(CHAR_NAME_SEMICOLON);
-			sb.append(CHAR_NAME_SPACE);
+			insertSemicolon(sb);
 		}
 	}
 
@@ -1321,15 +1347,15 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	}
 
 	public void addConvertMethodForOrdinalEnumTypes(CtClass tmplCtClass,
-			Class c) throws CannotCompileException, NotFoundException {
+			Class c) {
 		// Object convert(MessagePackObject from) throws MessageTypeException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
 		insertCodeOfMessagePackObjectArrayGet(bsb);
-		//insertConvertMethodBody(bsb, c, new Field[0]);
+		// insertConvertMethodBody(bsb, c, new Field[0]);
 		// FIXME
-		//bsb.append("_$$_ary[0].asInt(); ");
-		//bsb.append("int i = _$$_ary[1].asInt(); ");
+		// bsb.append("_$$_ary[0].asInt(); ");
+		// bsb.append("int i = _$$_ary[1].asInt(); ");
 		bsb.append("int i = _$$_ary[0].asInt(); ");
 		bsb.append("java.lang.Object o = ").append(c.getName()).append(
 				".class.getEnumConstants()[i]; ");
@@ -1339,9 +1365,17 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				new Class[] { MessagePackObject.class },
 				new String[] { VARIABLE_NAME_MPO },
 				new Class[] { MessageTypeException.class }, bsb.toString());
-		System.out.println("convert method: " + sb.toString());
-		CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
-		tmplCtClass.addMethod(newCtMethod);
+		LOG.trace("convert method src: " + sb.toString());
+		try {
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
+			tmplCtClass.addMethod(newCtMethod);
+		} catch (CannotCompileException e) {
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage()
+					+ ": " + sb.toString(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
+		}
 	}
 
 	private Class createClass(CtClass packerCtClass)
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 9e10cf7..6184883 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -74,6 +74,13 @@ public class DynamicCodeGenBase implements BasicConstants {
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(expr);
 	}
+	
+	public void insertInsertion(StringBuilder sb) {
+		// = 
+		sb.append(CHAR_NAME_SPACE);
+		sb.append(CHAR_NAME_EQUAL);
+		sb.append(CHAR_NAME_SPACE);
+	}
 
 	public void insertFieldAccess(StringBuilder sb, String target, String field) {
 		// target.field
@@ -226,7 +233,7 @@ public class DynamicCodeGenBase implements BasicConstants {
 		}
 	}
 
-	public String getAsMethodName(Class c) throws DynamicCodeGenException {
+	public String getAsMethodName(Class c) {
 		if (c.equals(boolean.class) || c.equals(Boolean.class)) {
 			return METHOD_NAME_ASBOOLEAN;
 		} else if (c.equals(byte.class) || c.equals(Byte.class)) {
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
index 55e761b..54dc43b 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
@@ -241,9 +241,9 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertTrue(true);
 		try {
 			DynamicCodeGenPacker.create(ProtectedDefaultConstructorClass.class);
-			assertTrue(true);
-		} catch (DynamicCodeGenException e) {
 			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
@@ -274,9 +274,9 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		try {
 			DynamicCodeGenTemplate
 					.create(ProtectedDefaultConstructorClass.class);
-			assertTrue(true);
-		} catch (DynamicCodeGenException e) {
 			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
@@ -319,9 +319,9 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertTrue(true);
 		try {
 			DynamicCodeGenPacker.create(ProtectedModifierClass.class);
-			assertTrue(true);
-		} catch (DynamicCodeGenException e) {
 			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
@@ -344,9 +344,9 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertTrue(true);
 		try {
 			DynamicCodeGenTemplate.create(ProtectedModifierClass.class);
-			assertTrue(true);
-		} catch (DynamicCodeGenException e) {
 			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
@@ -500,7 +500,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1 == dst.f1);
 		assertTrue(src.f2 != dst.f2);
-		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f3 != dst.f3);
 		assertTrue(src.f4 != dst.f4);
 		assertFalse(it.hasNext());
 	}
@@ -623,11 +623,11 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1 == dst.f1);
 		assertTrue(src.f2 != dst.f2);
-		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f3 != dst.f3);
 		assertTrue(src.f4 != dst.f4);
 		assertTrue(src.f5 == dst.f5);
 		assertTrue(src.f6 == dst.f6);
-		assertTrue(src.f8 == dst.f8);
+		assertTrue(src.f8 != dst.f8);
 		assertTrue(src.f9 != dst.f9);
 		assertFalse(it.hasNext());
 	}
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
index 8206aa4..104194e 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
@@ -220,9 +220,9 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		assertTrue(true);
 		try {
 			DynamicCodeGenPacker.create(ProtectedDefaultConstructorClass.class);
-			assertTrue(true);
-		} catch (DynamicCodeGenException e) {
 			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
@@ -253,9 +253,9 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		try {
 			DynamicCodeGenUnpacker
 					.create(ProtectedDefaultConstructorClass.class);
-			assertTrue(true);
-		} catch (DynamicCodeGenException e) {
 			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
@@ -298,9 +298,9 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		assertTrue(true);
 		try {
 			DynamicCodeGenPacker.create(ProtectedModifierClass.class);
-			assertTrue(true);
-		} catch (DynamicCodeGenException e) {
 			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
@@ -323,9 +323,9 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		assertTrue(true);
 		try {
 			DynamicCodeGenUnpacker.create(ProtectedModifierClass.class);
-			assertTrue(true);
-		} catch (DynamicCodeGenException e) {
 			fail();
+		} catch (DynamicCodeGenException e) {
+			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
@@ -466,7 +466,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1 == dst.f1);
 		assertTrue(src.f2 != dst.f2);
-		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f3 != dst.f3);
 		assertTrue(src.f4 != dst.f4);
 	}
 
@@ -574,11 +574,11 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1 == dst.f1);
 		assertTrue(src.f2 != dst.f2);
-		assertTrue(src.f3 == dst.f3);
+		assertTrue(src.f3 != dst.f3);
 		assertTrue(src.f4 != dst.f4);
 		assertTrue(src.f5 == dst.f5);
 		assertTrue(src.f6 == dst.f6);
-		assertTrue(src.f8 == dst.f8);
+		assertTrue(src.f8 != dst.f8);
 		assertTrue(src.f9 != dst.f9);
 	}
 

From 3ec55791c6a3234db0381b706ac30425d34536b5 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 2 Oct 2010 22:12:40 +0900
Subject: [PATCH 0828/1648] java: refactor DynamicCodeGen.java

---
 .../msgpack/util/codegen/BasicConstants.java  |  2 +
 .../msgpack/util/codegen/DynamicCodeGen.java  | 69 +++++++++++--------
 .../util/codegen/DynamicCodeGenBase.java      | 10 +++
 3 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
index cd667a6..a293056 100644
--- a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
@@ -100,6 +100,8 @@ public interface BasicConstants {
 
 	String METHOD_NAME_ORDINAL = "ordinal";
 
+	String METHOD_NAME_GETENUMCONSTANTS = "getEnumConstants";
+
 	String METHOD_NAME_CONVERT = "convert";
 
 	String METHOD_NAME_MSGCONVERT = "messageConvert";
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index efe9a69..c57b3db 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -819,14 +819,23 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
-		// FIXME
 		insertMethodCall(bsb, VARIABLE_NAME_PK, METHOD_NAME_UNPACKARRAY,
 				new String[0]);
 		insertSemicolon(bsb);
-		// insertUnpackMethodBody(bsb, c, new Field[0]);
-		bsb.append("int _$$_i = _$$_pk.unpackInt();");
-		bsb.append("return " + c.getName()
-				+ ".class.getEnumConstants()[_$$_i];");
+		StringBuilder mc = new StringBuilder();
+		insertMethodCall(mc, VARIABLE_NAME_PK, METHOD_NAME_UNPACKINT,
+				new String[0]);
+		insertLocalVariableDecl(bsb, int.class, VARIABLE_NAME_I);
+		insertValueInsertion(bsb, mc.toString());
+		insertSemicolon(bsb);
+		mc = new StringBuilder();
+		insertMethodCall(mc, c.getName() + ".class",
+				METHOD_NAME_GETENUMCONSTANTS, new String[0]);
+		mc.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+		mc.append(VARIABLE_NAME_I);
+		mc.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		insertReturnStat(bsb, mc.toString());
+		insertSemicolon(bsb);
 		addPublicMethodDecl(sb, METHOD_NAME_UNPACK, Object.class,
 				new Class[] { Unpacker.class },
 				new String[] { VARIABLE_NAME_PK }, new Class[] {
@@ -883,21 +892,11 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private void insertCodeOfMessagePackObjectArrayGet(StringBuilder sb) {
 		// MessagePackObject[] ary = obj.asArray();
-		sb.append(MessagePackObject.class.getName());
-		sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-		sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_ARRAY);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_MPO);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_ASARRAY);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
+		insertLocalVariableDecl(sb, MessagePackObject.class, VARIABLE_NAME_ARRAY, 1);
+		StringBuilder mc = new StringBuilder();
+		insertMethodCall(mc, VARIABLE_NAME_MPO, METHOD_NAME_ASARRAY, new String[0]);
+		insertValueInsertion(sb, mc.toString());
+		insertSemicolon(sb);
 	}
 
 	private void insertCodeOfConvertMethodCalls(StringBuilder sb, Field[] fields) {
@@ -1106,7 +1105,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(i);
 		sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
 		sb.append(CHAR_NAME_DOT);
-		sb.append("asList");
+		sb.append(METHOD_NAME_ASLIST);
 		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
 		sb.append(CHAR_NAME_SEMICOLON);
@@ -1352,15 +1351,25 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
 		insertCodeOfMessagePackObjectArrayGet(bsb);
-		// insertConvertMethodBody(bsb, c, new Field[0]);
-		// FIXME
-		// bsb.append("_$$_ary[0].asInt(); ");
-		// bsb.append("int i = _$$_ary[1].asInt(); ");
-		bsb.append("int i = _$$_ary[0].asInt(); ");
-		bsb.append("java.lang.Object o = ").append(c.getName()).append(
-				".class.getEnumConstants()[i]; ");
-		bsb.append("return (").append(c.getName()).append(") o; ");
-
+		StringBuilder mc = new StringBuilder();
+		insertMethodCall(mc, VARIABLE_NAME_ARRAY + "[0]", METHOD_NAME_ASINT,
+				new String[0]);
+		insertLocalVariableDecl(bsb, int.class, VARIABLE_NAME_I);
+		insertValueInsertion(bsb, mc.toString());
+		insertSemicolon(bsb);
+		mc = new StringBuilder();
+		insertMethodCall(mc, c.getName() + ".class",
+				METHOD_NAME_GETENUMCONSTANTS, new String[0]);
+		mc.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+		mc.append(VARIABLE_NAME_I);
+		mc.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		insertLocalVariableDecl(bsb, Object.class, VARIABLE_NAME_OBJECT);
+		insertValueInsertion(bsb, mc.toString());
+		insertSemicolon(bsb);
+		mc = new StringBuilder();
+		insertTypeCast(mc, c, VARIABLE_NAME_OBJECT);
+		insertReturnStat(bsb, mc.toString());
+		insertSemicolon(bsb);
 		addPublicMethodDecl(sb, METHOD_NAME_CONVERT, Object.class,
 				new Class[] { MessagePackObject.class },
 				new String[] { VARIABLE_NAME_MPO },
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 6184883..e460d6a 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -62,7 +62,17 @@ public class DynamicCodeGenBase implements BasicConstants {
 	public void insertLocalVariableDecl(StringBuilder sb, Class type,
 			String name) {
 		// int lv
+		insertLocalVariableDecl(sb, type, name, 0);
+	}
+	
+	public void insertLocalVariableDecl(StringBuilder sb, Class type,
+			String name, int dim) {
+		// int[] lv
 		sb.append(type.getName());
+		for (int i = 0; i < dim; ++i) {
+			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		}
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(name);
 	}

From 1ae6a413368957184af1aa7f421368c3281bc51b Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 2 Oct 2010 23:37:08 +0900
Subject: [PATCH 0829/1648] java: edit BasicConstants.java in
 org.msgpack.util.codegen

---
 .../main/java/org/msgpack/util/codegen/BasicConstants.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
index a293056..92497eb 100644
--- a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
@@ -37,6 +37,8 @@ public interface BasicConstants {
 	String CHAR_NAME_PLUS = "+";
 
 	String CHAR_NAME_MINUS = "-";
+	
+	String CHAR_NAME_UNDERSCORE = "_";
 
 	String CHAR_NAME_LESSTHAN = "<";
 
@@ -60,7 +62,7 @@ public interface BasicConstants {
 
 	String VARIABLE_NAME_OBJECT = "_$$_obj";
 
-	String VARIABLE_NAME_TARGET = "_$$_target";
+	String VARIABLE_NAME_TARGET = "_$$_tt";
 
 	String VARIABLE_NAME_SIZE = "_$$_len";
 

From a85f6e72fe0c8f72ddaca3c21e7db2619e161b73 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sun, 3 Oct 2010 02:09:01 +0900
Subject: [PATCH 0830/1648] java: edit DynamicCodeGen.java

---
 .../main/java/org/msgpack/util/codegen/BasicConstants.java  | 2 +-
 .../main/java/org/msgpack/util/codegen/DynamicCodeGen.java  | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
index 92497eb..efaddab 100644
--- a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
@@ -62,7 +62,7 @@ public interface BasicConstants {
 
 	String VARIABLE_NAME_OBJECT = "_$$_obj";
 
-	String VARIABLE_NAME_TARGET = "_$$_tt";
+	String VARIABLE_NAME_TARGET = "_$$_t";
 
 	String VARIABLE_NAME_SIZE = "_$$_len";
 
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index c57b3db..581a6d7 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -939,7 +939,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			LOG.error(e.getMessage(), e);
 			throw e;
 		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
-			// @MessagePackMessage
+			// @MessagePackOrdinalEnum
 			Template tmpl = DynamicCodeGenOrdinalEnumTemplate.create(c);
 			CustomMessage.registerTemplate(c, tmpl);
 			insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
@@ -953,8 +953,8 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private void insertCodeOfMessageConvertCallForRegisteredType(
 			StringBuilder sb, Field f, Class c, int i) {
-		// ((MessageConvertable)f_i).messageConvert(ary[i]);
-		// obj = tmpl.convert(mpo);
+		// target.f0 = (Class) CustomConverter.get(C.class).convert(mpo[i]);
+		// obj = tmpl.convert(mpo[i]);
 		sb.append(VARIABLE_NAME_TARGET);
 		sb.append(CHAR_NAME_DOT);
 		sb.append(f.getName());

From 9346908485b30df552df6b5bc18337449347c471 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Tue, 5 Oct 2010 15:55:38 +0900
Subject: [PATCH 0831/1648] perl: remove tricky hacks

---
 perl/lib/Data/MessagePack.pm | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 1fccbf7..7d1bda7 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -9,17 +9,13 @@ our $PreferInteger = 0;
 sub true () {
     require Data::MessagePack::Boolean;
     no warnings 'once', 'redefine';
-    my $t = $Data::MessagePack::Boolean::true;
-    *true = sub (){ $t };
-    return $t;
+    return $Data::MessagePack::Boolean::true;
 }
 
 sub false () {
     require Data::MessagePack::Boolean;
     no warnings 'once', 'redefine';
-    my $f = $Data::MessagePack::Boolean::false;
-    *false = sub (){ $f };
-    return $f;
+    return $Data::MessagePack::Boolean::false;
 }
 
 if ( !__PACKAGE__->can('pack') ) { # this idea comes from Text::Xslate

From f2d13cd6476a0d09525749803461743985ca766d Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Tue, 5 Oct 2010 15:58:49 +0900
Subject: [PATCH 0832/1648] perl: make test code more simple

---
 perl/t/Util.pm | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/perl/t/Util.pm b/perl/t/Util.pm
index ad69c4d..0aa88bb 100644
--- a/perl/t/Util.pm
+++ b/perl/t/Util.pm
@@ -10,13 +10,9 @@ sub import {
     warnings->import;
 
     no strict 'refs';
-    *{"$pkg\::true"} = sub () {
-        Data::MessagePack::true()
-    };
-    *{"$pkg\::false"} = sub () {
-        Data::MessagePack::false()
-    };
-    *{"$pkg\::null"} = sub() { undef };
+    *{"$pkg\::true"}  = \&Data::MessagePack::true;
+    *{"$pkg\::false"} = \&Data::MessagePack::false;
+    *{"$pkg\::null"}  = sub() { undef };
 }
 
 1;

From a4a04872a3e7d331c722938a719711cb3178a5c7 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Tue, 5 Oct 2010 17:10:10 +0900
Subject: [PATCH 0833/1648] perl: add $unpacker->utf8 mode, decoding strings as
 UTF-8.

---
 perl/lib/Data/MessagePack/PP.pm        | 21 ++++++++++++++++---
 perl/lib/Data/MessagePack/Unpacker.pod | 11 ++++++++++
 perl/t/15_utf8.t                       | 27 +++++++++++++++++++++++++
 perl/xs-src/MessagePack.c              |  4 ++++
 perl/xs-src/unpack.c                   | 28 ++++++++++++++++++++++++--
 5 files changed, 86 insertions(+), 5 deletions(-)
 create mode 100644 perl/t/15_utf8.t

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 5dccc0b..00e58b9 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -248,6 +248,7 @@ sub _pack {
 # UNPACK
 #
 
+our $_utf8 = 0;
 my $p; # position variables for speed.
 
 sub unpack :method {
@@ -358,7 +359,9 @@ sub _unpack {
             $num = $byte & ~0xa0;
             $p += $num;
         }
-        return substr( $value, $p - $num, $num );
+        my $s = substr( $value, $p - $num, $num );
+        utf8::decode($s) if $_utf8;
+        return $s;
     }
 
     elsif ( $byte == 0xc0 ) { # nil
@@ -396,9 +399,19 @@ package
     Data::MessagePack::PP::Unpacker;
 
 sub new {
-    bless { pos => 0 }, shift;
+    bless { pos => 0, utf8 => 0 }, shift;
 }
 
+sub utf8 {
+    my $self = shift;
+    $self->{utf8} = (@_ ? shift : 1);
+    return $self;
+}
+
+sub get_utf8 {
+    my($self) = @_;
+    return $self->{utf8};
+}
 
 sub execute_limit {
     execute( @_ );
@@ -540,7 +553,9 @@ sub _count {
 
 
 sub data {
-    return Data::MessagePack->unpack( substr($_[0]->{ data }, 0, $_[0]->{pos}) );
+    my($self) = @_;
+    local $Data::MessagePack::PP::_utf8 = $self->{utf8};
+    return Data::MessagePack->unpack( substr($self->{ data }, 0, $self->{pos}) );
 }
 
 
diff --git a/perl/lib/Data/MessagePack/Unpacker.pod b/perl/lib/Data/MessagePack/Unpacker.pod
index 2bc4549..37ab3db 100644
--- a/perl/lib/Data/MessagePack/Unpacker.pod
+++ b/perl/lib/Data/MessagePack/Unpacker.pod
@@ -24,6 +24,17 @@ This is a streaming deserializer for messagepack.
 
 creates a new instance of stream deserializer.
 
+=item $up->utf8([$bool])
+
+sets utf8 mode. true if I<$bool> is omitted.
+returns I<$up> itself.
+
+If utf8 mode is enabled, strings will be decoded as UTF-8.
+
+=item my $ret = $up->get_utf8()
+
+returns the utf8 mode flag of I<$up>.
+
 =item my $ret = $up->execute($data, $offset);
 
 =item my $ret = $up->execute_limit($data, $offset, $limit)
diff --git a/perl/t/15_utf8.t b/perl/t/15_utf8.t
new file mode 100644
index 0000000..d7d17b8
--- /dev/null
+++ b/perl/t/15_utf8.t
@@ -0,0 +1,27 @@
+#!perl -w
+use strict;
+use Test::More;
+use Data::MessagePack;
+use utf8;
+
+my $data = [42, undef, 'foo', "\x{99f1}\x{99dd}"];
+my $packed = Data::MessagePack->pack($data);
+
+my $u = Data::MessagePack::Unpacker->new()->utf8();
+ok $u->get_utf8();
+$u->execute($packed);
+my $d = $u->data();
+$u->reset();
+is_deeply $d, $data, 'decoded';
+
+is $u->utf8(0), $u, 'utf8(0)';
+ok !$u->get_utf8();
+$u->execute($packed);
+$d = $u->data();
+$u->reset();
+my $s = $data->[3];
+utf8::encode($s);
+is_deeply $d->[3], $s, 'not decoded';
+
+done_testing;
+
diff --git a/perl/xs-src/MessagePack.c b/perl/xs-src/MessagePack.c
index 69337f4..0c3c0b1 100644
--- a/perl/xs-src/MessagePack.c
+++ b/perl/xs-src/MessagePack.c
@@ -7,6 +7,8 @@
 XS(xs_pack);
 XS(xs_unpack);
 XS(xs_unpacker_new);
+XS(xs_unpacker_utf8);
+XS(xs_unpacker_get_utf8);
 XS(xs_unpacker_execute);
 XS(xs_unpacker_execute_limit);
 XS(xs_unpacker_is_finished);
@@ -28,6 +30,8 @@ XS(boot_Data__MessagePack) {
     newXS("Data::MessagePack::unpack", xs_unpack, __FILE__);
 
     newXS("Data::MessagePack::Unpacker::new",           xs_unpacker_new, __FILE__);
+    newXS("Data::MessagePack::Unpacker::utf8",          xs_unpacker_utf8, __FILE__);
+    newXS("Data::MessagePack::Unpacker::get_utf8",      xs_unpacker_get_utf8, __FILE__);
     newXS("Data::MessagePack::Unpacker::execute",       xs_unpacker_execute, __FILE__);
     newXS("Data::MessagePack::Unpacker::execute_limit", xs_unpacker_execute_limit, __FILE__);
     newXS("Data::MessagePack::Unpacker::is_finished",   xs_unpacker_is_finished, __FILE__);
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 065573a..f39d8c1 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -13,6 +13,7 @@ START_MY_CXT
 typedef struct {
     bool finished;
     bool incremented;
+    bool utf8;
 } unpack_user;
 
 #include "msgpack/unpack_define.h"
@@ -237,6 +238,9 @@ STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const c
     dTHX;
     /*  newSVpvn(p, l) returns an undef if p == NULL */
     *o = ((l==0) ? newSVpvs("") : newSVpvn(p, l));
+    if(u->utf8) {
+        sv_utf8_decode(*o);
+    }
     return 0;
 }
 
@@ -276,7 +280,7 @@ XS(xs_unpack) {
     msgpack_unpack_t mp;
     template_init(&mp);
 
-    unpack_user const u = {false, false};
+    unpack_user const u = {false, false, false};
     mp.user = u;
 
     size_t from = 0;
@@ -303,7 +307,7 @@ XS(xs_unpack) {
 
 STATIC_INLINE void _reset(SV* const self) {
     dTHX;
-	unpack_user const u = {false, false};
+	unpack_user const u = {false, false, false};
 
 	UNPACKER(self, mp);
 	template_init(mp);
@@ -328,6 +332,26 @@ XS(xs_unpacker_new) {
     XSRETURN(1);
 }
 
+XS(xs_unpacker_utf8) {
+    dXSARGS;
+    if (!(items == 1 || items == 2)) {
+        Perl_croak(aTHX_ "Usage: $unpacker->utf8([$bool)");
+    }
+    UNPACKER(ST(0), mp);
+    mp->user.utf8 = (items == 1 || sv_true(ST(1))) ? true : false;
+    XSRETURN(1); // returns $self
+}
+
+XS(xs_unpacker_get_utf8) {
+    dXSARGS;
+    if (items != 1) {
+        Perl_croak(aTHX_ "Usage: $unpacker->get_utf8()");
+    }
+    UNPACKER(ST(0), mp);
+    ST(0) = boolSV(mp->user.utf8);
+    XSRETURN(1);
+}
+
 STATIC_INLINE size_t
 _execute_impl(SV* const self, SV* const data, UV const offset, UV const limit) {
     dTHX;

From 7c92f8a90b54a7cf2279211fc4601e11872bee38 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Tue, 5 Oct 2010 17:47:27 +0900
Subject: [PATCH 0834/1648] perl: improve docs

---
 perl/lib/Data/MessagePack.pm           | 6 ++++++
 perl/lib/Data/MessagePack/Unpacker.pod | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 7d1bda7..3d2d695 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -165,6 +165,12 @@ will astonish those who try to unpack byte streams with an arbitrary buffer size
 (e.g. C<< while(read($socket, $buffer, $arbitrary_buffer_size)) { ... } >>).
 We should implement the internal buffer for the unpacker.
 
+=item UTF8 mode
+
+Data::MessagePack::Unpacker supports utf8 mode, which decodes strings
+as UTF8-8. << Data::MessagePack->unpack >> should support utf8 mode in a
+future.
+
 =back
 
 =head1 AUTHORS
diff --git a/perl/lib/Data/MessagePack/Unpacker.pod b/perl/lib/Data/MessagePack/Unpacker.pod
index 37ab3db..24dafd0 100644
--- a/perl/lib/Data/MessagePack/Unpacker.pod
+++ b/perl/lib/Data/MessagePack/Unpacker.pod
@@ -31,6 +31,8 @@ returns I<$up> itself.
 
 If utf8 mode is enabled, strings will be decoded as UTF-8.
 
+The utf8 mode is disabled by default.
+
 =item my $ret = $up->get_utf8()
 
 returns the utf8 mode flag of I<$up>.

From b01c270889a35ebe0b05d4843def94bc95994de5 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Tue, 5 Oct 2010 20:45:51 +0900
Subject: [PATCH 0835/1648] java: refactor the programs in a
 org.msgpack.util.codegen package

---
 .../msgpack/util/codegen/BasicConstants.java  |   8 +-
 .../msgpack/util/codegen/DynamicCodeGen.java  | 953 ++++--------------
 .../util/codegen/DynamicCodeGenBase.java      | 140 ++-
 .../util/codegen/DynamicCodeGenConverter.java |  10 +-
 .../util/codegen/DynamicCodeGenException.java |   2 +-
 .../DynamicCodeGenOrdinalEnumConverter.java   |   2 +-
 .../DynamicCodeGenOrdinalEnumPacker.java      |   2 +-
 .../DynamicCodeGenOrdinalEnumUnpacker.java    |   2 +-
 .../util/codegen/DynamicCodeGenPacker.java    |   2 +-
 .../util/codegen/DynamicCodeGenTemplate.java  |   5 +-
 .../util/codegen/DynamicCodeGenUnpacker.java  |  10 +-
 11 files changed, 361 insertions(+), 775 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
index efaddab..8448c02 100644
--- a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
@@ -37,7 +37,7 @@ public interface BasicConstants {
 	String CHAR_NAME_PLUS = "+";
 
 	String CHAR_NAME_MINUS = "-";
-	
+
 	String CHAR_NAME_UNDERSCORE = "_";
 
 	String CHAR_NAME_LESSTHAN = "<";
@@ -82,6 +82,10 @@ public interface BasicConstants {
 
 	String VARIABLE_NAME_I = "i";
 
+	String VARIABLE_NAME_TEMPLATES = "_$$_templates";
+
+	String VARIABLE_NAME_TEMPLATES0 = "_$$_tmpls";
+
 	String METHOD_NAME_VALUEOF = "valueOf";
 
 	String METHOD_NAME_ADD = "add";
@@ -108,6 +112,8 @@ public interface BasicConstants {
 
 	String METHOD_NAME_MSGCONVERT = "messageConvert";
 
+	String METHOD_NAME_SETTEMPLATES = "setTemplates";
+
 	String METHOD_NAME_PACK = "pack";
 
 	String METHOD_NAME_PACKARRAY = "packArray";
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 581a6d7..1877607 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -4,35 +4,31 @@ import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
-import java.lang.reflect.ParameterizedType;
 import java.math.BigInteger;
 import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import javassist.CannotCompileException;
 import javassist.ClassPool;
 import javassist.CtClass;
 import javassist.CtConstructor;
+import javassist.CtField;
 import javassist.CtMethod;
 import javassist.CtNewConstructor;
 import javassist.CtNewMethod;
 import javassist.NotFoundException;
 
-import org.msgpack.CustomConverter;
 import org.msgpack.CustomMessage;
 import org.msgpack.CustomPacker;
-import org.msgpack.CustomUnpacker;
 import org.msgpack.MessageConvertable;
 import org.msgpack.MessagePackObject;
 import org.msgpack.MessagePackable;
 import org.msgpack.MessagePacker;
 import org.msgpack.MessageTypeException;
 import org.msgpack.MessageUnpackable;
-import org.msgpack.MessageUnpacker;
 import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
@@ -65,8 +61,19 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private ClassPool pool;
 
-	private DynamicCodeGen() {
-		this.pool = ClassPool.getDefault();
+	private ConcurrentHashMap tmplMap;
+
+	DynamicCodeGen() {
+		pool = ClassPool.getDefault();
+		tmplMap = new ConcurrentHashMap();
+	}
+
+	public void setTemplates(Class origClass, Template[] tmpls) {
+		tmplMap.putIfAbsent(origClass.getName(), tmpls);
+	}
+
+	public Template[] getTemplates(Class origClass) {
+		return tmplMap.get(origClass.getName());
 	}
 
 	public Class generateMessagePackerClass(Class origClass) {
@@ -113,95 +120,6 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	public Class generateMessageUnpackerClass(Class origClass) {
-		LOG.debug("start generating a MessageUnpacker impl.: "
-				+ origClass.getName());
-		try {
-			String origName = origClass.getName();
-			String unpackerName = origName + POSTFIX_TYPE_NAME_UNPACKER + inc();
-			checkClassValidation(origClass);
-			checkDefaultConstructorValidation(origClass);
-			CtClass unpackerCtClass = pool.makeClass(unpackerName);
-			setInterface(unpackerCtClass, MessageUnpacker.class);
-			addDefaultConstructor(unpackerCtClass);
-			Field[] fields = getDeclaredFields(origClass);
-			addUnpackMethod(unpackerCtClass, origClass, fields);
-			return createClass(unpackerCtClass);
-		} catch (NotFoundException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (CannotCompileException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
-	}
-
-	public Class generateOrdinalEnumUnpackerClass(Class origClass) {
-		LOG.debug("start generating a OrdinalEnumUnpacker impl.: "
-				+ origClass.getName());
-		try {
-			String origName = origClass.getName();
-			String unpackerName = origName + POSTFIX_TYPE_NAME_UNPACKER + inc();
-			checkClassValidation(origClass);
-			checkDefaultConstructorValidation(origClass);
-			CtClass unpackerCtClass = pool.makeClass(unpackerName);
-			setInterface(unpackerCtClass, MessageUnpacker.class);
-			addDefaultConstructor(unpackerCtClass);
-			addUnpackMethodForOrdinalEnumTypes(unpackerCtClass, origClass);
-			return createClass(unpackerCtClass);
-		} catch (NotFoundException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (CannotCompileException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
-	}
-
-	public Class generateMessageConverterClass(Class origClass) {
-		LOG.debug("start generating a MessageConverter impl.: "
-				+ origClass.getName());
-		try {
-			String origName = origClass.getName();
-			String convName = origName + POSTFIX_TYPE_NAME_CONVERTER + inc();
-			checkClassValidation(origClass);
-			checkDefaultConstructorValidation(origClass);
-			CtClass converterCtClass = pool.makeClass(convName);
-			setInterface(converterCtClass, MessageUnpacker.class);
-			addDefaultConstructor(converterCtClass);
-			Field[] fields = getDeclaredFields(origClass);
-			addConvertMethod(converterCtClass, origClass, fields);
-			return createClass(converterCtClass);
-		} catch (NotFoundException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (CannotCompileException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
-	}
-
-	public Class generateOrdinalEnumConverterClass(Class origClass) {
-		LOG.debug("start generating a OrdinalEnumConverter impl.: "
-				+ origClass.getName());
-		try {
-			String origName = origClass.getName();
-			String convName = origName + POSTFIX_TYPE_NAME_CONVERTER + inc();
-			checkClassValidation(origClass);
-			CtClass converterCtClass = pool.makeClass(convName);
-			setInterface(converterCtClass, MessageUnpacker.class);
-			addDefaultConstructor(converterCtClass);
-			addConvertMethodForOrdinalEnumTypes(converterCtClass, origClass);
-			return createClass(converterCtClass);
-		} catch (NotFoundException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (CannotCompileException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
-	}
-
 	public Class generateTemplateClass(Class origClass) {
 		LOG.debug("start generating a Template impl.: " + origClass.getName());
 		try {
@@ -211,8 +129,13 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			checkDefaultConstructorValidation(origClass);
 			CtClass tmplCtClass = pool.makeClass(tmplName);
 			setInterface(tmplCtClass, Template.class);
+			setInterface(tmplCtClass, DynamicCodeGenBase.TemplateAccessor.class);
 			addDefaultConstructor(tmplCtClass);
 			Field[] fields = getDeclaredFields(origClass);
+			Template[] tmpls = createTemplates(fields);
+			setTemplates(origClass, tmpls);
+			addTemplateArrayField(tmplCtClass, origClass);
+			addSetTemplatesMethod(tmplCtClass, origClass);
 			addUnpackMethod(tmplCtClass, origClass, fields);
 			addConvertMethod(tmplCtClass, origClass, fields);
 			return createClass(tmplCtClass);
@@ -343,6 +266,23 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		throw e;
 	}
 
+	Template[] createTemplates(Field[] fields) {
+		Template[] tmpls = new Template[fields.length];
+		for (int i = 0; i < tmpls.length; ++i) {
+			tmpls[i] = createTemplate(fields[i]);
+		}
+		return tmpls;
+	}
+
+	Template createTemplate(Field field) {
+		Class c = field.getType();
+		if (List.class.isAssignableFrom(c) || Map.class.isAssignableFrom(c)) {
+			return createTemplate(field.getGenericType());
+		} else {
+			return createTemplate(c);
+		}
+	}
+
 	private void addPackMethod(CtClass packerCtClass, Class c, Field[] fs) {
 		// void pack(Packer pk, Object target) throws IOException;
 		StringBuilder sb = new StringBuilder();
@@ -383,6 +323,7 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	private void insertCodeOfPackMethodCall(StringBuilder sb, Field field) {
 		Class c = field.getType();
 		if (c.isPrimitive()) {
+			; // ignore
 		} else if (c.equals(Boolean.class) || c.equals(Byte.class)
 				|| c.equals(Double.class) || c.equals(Float.class)
 				|| c.equals(Integer.class) || c.equals(Long.class)
@@ -464,6 +405,44 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
+	private void addTemplateArrayField(CtClass newCtClass, Class origClass) {
+		StringBuilder sb = new StringBuilder();
+		addPublicFieldDecl(sb, Template.class, VARIABLE_NAME_TEMPLATES, 1);
+		insertSemicolon(sb);
+		LOG.trace("templates field src: " + sb.toString());
+		try {
+			CtField templatesCtField = CtField.make(sb.toString(), newCtClass);
+			newCtClass.addField(templatesCtField);
+		} catch (CannotCompileException e) {
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage()
+					+ ": " + sb.toString(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
+		}
+	}
+
+	private void addSetTemplatesMethod(CtClass newCtClass, Class origClass) {
+		StringBuilder sb = new StringBuilder();
+		StringBuilder body = new StringBuilder();
+		body.append("_$$_templates = _$$_tmpls;");
+		addPublicMethodDecl(sb, METHOD_NAME_SETTEMPLATES, void.class,
+				new Class[] { Template.class }, new int[] { 1 },
+				new String[] { VARIABLE_NAME_TEMPLATES0 }, new Class[0],
+				body.toString());
+		LOG.trace("settemplates method src: " + sb.toString());
+		try {
+			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), newCtClass);
+			newCtClass.addMethod(newCtMethod);
+		} catch (CannotCompileException e) {
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage()
+					+ ": " + sb.toString(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
+		}
+	}
+
 	private void addUnpackMethod(CtClass unpackerCtClass, Class c, Field[] fs)
 			throws CannotCompileException, NotFoundException {
 		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
@@ -498,268 +477,81 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		insertMethodCall(sb, VARIABLE_NAME_PK, METHOD_NAME_UNPACKARRAY,
 				new String[0]);
 		insertSemicolon(sb);
-		for (Field f : fs) {
-			insertCodeOfUnpackMethodCall(sb, f, f.getType());
-		}
+		insertCodeOfUnpackMethodCalls(sb, fs);
 		insertReturnStat(sb, VARIABLE_NAME_TARGET);
 		insertSemicolon(sb);
 	}
 
-	private void insertCodeOfUnpackMethodCall(StringBuilder sb, Field f,
-			Class c) {
-		if (c.isPrimitive()) {
-			// primitive type
-			insertCodeOfUnpackMethodCallForPrimTypes(sb, f, c);
-		} else if (c.equals(Boolean.class) || c.equals(Byte.class)
-				|| c.equals(Double.class) || c.equals(Float.class)
-				|| c.equals(Integer.class) || c.equals(Long.class)
-				|| c.equals(Short.class)) {
-			// reference type (wrapper type)
-			insertCodeOfUnpackMethodCallForWrapTypes(sb, f, c);
-		} else if (c.equals(BigInteger.class) || c.equals(String.class)
-				|| c.equals(byte[].class)) {
-			// reference type (other type)
-			insertCodeOfUnpackMethodCallForPrimTypes(sb, f, c);
-		} else if (List.class.isAssignableFrom(c)) {
-			// List
-			insertCodeOfUnpackMethodCallForListType(sb, f, c);
-		} else if (Map.class.isAssignableFrom(c)) {
-			// Map
-			insertCodeOfUnpackMethodCallForMapType(sb, f, c);
-		} else if (CustomUnpacker.isRegistered(c)) {
-			insertCodeOfUnpackMethodCallForRegisteredType(sb, f, c);
-		} else if (MessageUnpackable.class.isAssignableFrom(c)) {
-			// MessageUnpackable
-			insertCodeOfUnpackMethodCallForMsgUnpackableType(sb, f, c);
-		} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
-			// @MessagePackMessage
-			Template tmpl = DynamicCodeGenTemplate.create(c);
-			CustomMessage.registerTemplate(c, tmpl);
-			insertCodeOfUnpackMethodCallForRegisteredType(sb, f, c);
-		} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
-			// FIXME DelegatePacker
-			UnsupportedOperationException e = new UnsupportedOperationException(
-					"not supported yet. : " + c.getName());
-			LOG.error(e.getMessage(), e);
-			throw e;
-		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
-			// @MessagePackOrdinalEnum
-			Template tmpl = DynamicCodeGenOrdinalEnumTemplate.create(c);
-			CustomMessage.registerTemplate(c, tmpl);
-			insertCodeOfUnpackMethodCallForRegisteredType(sb, f, c);
+	private void insertCodeOfUnpackMethodCalls(StringBuilder sb, Field[] fields) {
+		for (int i = 0; i < fields.length; ++i) {
+			insertCodeOfUnpackMethodCall(sb, fields[i], i);
+		}
+	}
+
+	private void insertCodeOfUnpackMethodCall(StringBuilder sb, Field field,
+			int i) {
+		// target.fi = ((Integer)_$$_tmpls[i].unpack(_$$_pk)).intValue();
+		Class type = field.getType();
+		insertFieldAccess(sb, VARIABLE_NAME_TARGET, field.getName());
+		String castType = null;
+		String rawValueGetter = null;
+		if (type.isPrimitive()) {
+			if (type.equals(byte.class)) {
+				castType = "(Byte)";
+				rawValueGetter = "byteValue";
+			} else if (type.equals(boolean.class)) {
+				castType = "(Boolean)";
+				rawValueGetter = "booleanValue";
+			} else if (type.equals(short.class)) {
+				castType = "(Short)";
+				rawValueGetter = "shortValue";
+			} else if (type.equals(int.class)) {
+				castType = "(Integer)";
+				rawValueGetter = "intValue";
+			} else if (type.equals(long.class)) {
+				castType = "(Long)";
+				rawValueGetter = "longValue";
+			} else if (type.equals(float.class)) {
+				castType = "(Float)";
+				rawValueGetter = "floatValue";
+			} else if (type.equals(double.class)) {
+				castType = "(Double)";
+				rawValueGetter = "doubleValue";
+			} else {
+				throw new DynamicCodeGenException("Fatal error: "
+						+ type.getName());
+			}
+		} else if (type.isArray()) {
+			Class ct = type.getComponentType();
+			if (ct.equals(byte.class)) {
+				castType = "(byte[])";
+			} else {
+				throw new UnsupportedOperationException("Not supported yet: "
+						+ type.getName());
+			}
 		} else {
-			MessageTypeException e = new MessageTypeException("unknown type:  "
-					+ c.getName());
-			LOG.error(e.getMessage(), e);
-			throw e;
-		}
-	}
-
-	private void insertCodeOfUnpackMethodCallForPrimTypes(StringBuilder sb,
-			Field f, Class c) {
-		if (f != null) {
-			insertFieldAccess(sb, VARIABLE_NAME_TARGET, f.getName());
-			insertInsertion(sb);
-		}
-		insertMethodCall(sb, VARIABLE_NAME_PK, getUnpackMethodName(c),
-				new String[0]);
-		if (f != null) {
-			insertSemicolon(sb);
-		}
-	}
-
-	private void insertCodeOfUnpackMethodCallForWrapTypes(StringBuilder sb,
-			Field f, Class c) {
-		if (f != null) {
-			insertFieldAccess(sb, VARIABLE_NAME_TARGET, f.getName());
-			insertInsertion(sb);
+			castType = "(" + type.getName() + ")";
 		}
 		StringBuilder mc = new StringBuilder();
-		insertMethodCall(mc, VARIABLE_NAME_PK, getUnpackMethodName(c),
-				new String[0]);
-		insertMethodCall(sb, c.getName(), METHOD_NAME_VALUEOF,
-				new String[] { mc.toString() });
-		if (f != null) {
-			insertSemicolon(sb);
-		}
-	}
-
-	private void insertCodeOfUnpackMethodCallForListType(StringBuilder sb,
-			Field field, Class type) {
-		ParameterizedType generic = (ParameterizedType) field.getGenericType();
-		Class genericType = (Class) generic.getActualTypeArguments()[0];
-
-		// len
-		sb.append(int.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_SIZE);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_PK);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_UNPACKARRAY);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		// field initializer
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(field.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(KEYWORD_NEW);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(ArrayList.class.getName());
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		// for loop
-		sb.append(KEYWORD_FOR);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(int.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_I);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(0);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_I);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LESSTHAN);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_SIZE);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_PLUS);
-		sb.append(CHAR_NAME_PLUS);
-		sb.append(VARIABLE_NAME_I);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-
-		// block
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(field.getName());
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_ADD);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		insertCodeOfUnpackMethodCall(sb, null, genericType);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-	}
-
-	private void insertCodeOfUnpackMethodCallForMapType(StringBuilder sb,
-			Field field, Class type) {
-		ParameterizedType generic = (ParameterizedType) field.getGenericType();
-		Class genericType0 = (Class) generic.getActualTypeArguments()[0];
-		Class genericType1 = (Class) generic.getActualTypeArguments()[1];
-
-		// len
-		sb.append(int.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_SIZE);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_PK);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_UNPACKMAP);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		// field initializer
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(field.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(KEYWORD_NEW);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(HashMap.class.getName());
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		// for loop
-		sb.append(KEYWORD_FOR);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(int.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_I);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(0);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_I);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LESSTHAN);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_SIZE);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_PLUS);
-		sb.append(CHAR_NAME_PLUS);
-		sb.append(VARIABLE_NAME_I);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-
-		// block map.
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(field.getName());
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_PUT);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		insertCodeOfUnpackMethodCall(sb, null, genericType0);
-		sb.append(CHAR_NAME_COMMA);
-		sb.append(CHAR_NAME_SPACE);
-		insertCodeOfUnpackMethodCall(sb, null, genericType1);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-	}
-
-	private void insertCodeOfUnpackMethodCallForRegisteredType(
-			StringBuilder sb, Field f, Class c) {
-		// target.field = (Cast) CustomUnpacker.get(C.class).unpack(pk);
-		StringBuilder mc = new StringBuilder();
-		insertMethodCall(mc, CustomUnpacker.class.getName(), METHOD_NAME_GET,
-				new String[] { c.getName() + ".class" });
-		String t = mc.toString();
+		mc.append(castType);
+		mc.append(VARIABLE_NAME_TEMPLATES);
+		mc.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+		mc.append(i);
+		mc.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		String tname = mc.toString();
 		mc = new StringBuilder();
-		insertMethodCall(mc, t, METHOD_NAME_UNPACK,
+		insertMethodCall(mc, tname, METHOD_NAME_UNPACK,
 				new String[] { VARIABLE_NAME_PK });
-		t = mc.toString();
-		mc = new StringBuilder();
-		insertTypeCast(mc, c, t);
-		insertFieldAccess(sb, VARIABLE_NAME_TARGET, f.getName());
+		if (type.isPrimitive()) {
+			tname = mc.toString();
+			mc = new StringBuilder();
+			mc.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			mc.append(tname);
+			mc.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			tname = mc.toString();
+			mc = new StringBuilder();
+			insertMethodCall(mc, tname, rawValueGetter, new String[0]);
+		}
 		insertValueInsertion(sb, mc.toString());
 		insertSemicolon(sb);
 	}
@@ -892,92 +684,90 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private void insertCodeOfMessagePackObjectArrayGet(StringBuilder sb) {
 		// MessagePackObject[] ary = obj.asArray();
-		insertLocalVariableDecl(sb, MessagePackObject.class, VARIABLE_NAME_ARRAY, 1);
+		insertLocalVariableDecl(sb, MessagePackObject.class,
+				VARIABLE_NAME_ARRAY, 1);
 		StringBuilder mc = new StringBuilder();
-		insertMethodCall(mc, VARIABLE_NAME_MPO, METHOD_NAME_ASARRAY, new String[0]);
+		insertMethodCall(mc, VARIABLE_NAME_MPO, METHOD_NAME_ASARRAY,
+				new String[0]);
 		insertValueInsertion(sb, mc.toString());
 		insertSemicolon(sb);
 	}
 
 	private void insertCodeOfConvertMethodCalls(StringBuilder sb, Field[] fields) {
 		for (int i = 0; i < fields.length; ++i) {
-			insertCodeOfConvertMethodCall(sb, fields[i], fields[i].getType(),
-					i, null);
+			insertCodeOfConvMethodCall(sb, fields[i], i);
 		}
 	}
 
-	private void insertCodeOfConvertMethodCall(StringBuilder sb, Field f,
-			Class c, int i, String v) {
-		if (c.isPrimitive()) { // primitive type
-			insertCodeOfConvertMethodCallForPrimTypes(sb, f, c, i, v);
-		} else if (c.equals(Boolean.class) || c.equals(Byte.class)
-				|| c.equals(Short.class) || c.equals(Integer.class)
-				|| c.equals(Float.class) || c.equals(Long.class)
-				|| c.equals(Double.class)) {
-			// reference type (wrapper)
-			insertCodeOfConvertMethodCallForWrapTypes(sb, f, c, i, v);
-		} else if (c.equals(String.class) || c.equals(byte[].class)
-				|| c.equals(BigInteger.class)) {
-			insertCodeOfConvertMethodCallForPrimTypes(sb, f, c, i, v);
-		} else if (List.class.isAssignableFrom(c)) {
-			insertCodeOfConvertMethodCallForList(sb, f, c, i);
-		} else if (Map.class.isAssignableFrom(c)) {
-			insertCodeOfConvertMethodCallForMapType(sb, f, c, i);
-		} else if (MessageConvertable.class.isAssignableFrom(c)) {
-			insertCodeOfMessageConvertCallForMsgConvtblType(sb, f, c, i);
-		} else if (CustomConverter.isRegistered(c)) {
-			insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
-		} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
-			// @MessagePackMessage
-			Template tmpl = DynamicCodeGenTemplate.create(c);
-			CustomMessage.registerTemplate(c, tmpl);
-			insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
-		} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
-			// FIXME DelegatePacker
-			UnsupportedOperationException e = new UnsupportedOperationException(
-					"not supported yet. : " + c.getName());
-			LOG.error(e.getMessage(), e);
-			throw e;
-		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
-			// @MessagePackOrdinalEnum
-			Template tmpl = DynamicCodeGenOrdinalEnumTemplate.create(c);
-			CustomMessage.registerTemplate(c, tmpl);
-			insertCodeOfMessageConvertCallForRegisteredType(sb, f, c, i);
+	private void insertCodeOfConvMethodCall(StringBuilder sb, Field field, int i) {
+		// target.f0 = ((Integer)_$$_tmpls[i].convert(_$$_ary[i])).intValue();
+		Class type = field.getType();
+		insertFieldAccess(sb, VARIABLE_NAME_TARGET, field.getName());
+		String castType = null;
+		String rawValueGetter = null;
+		if (type.isPrimitive()) {
+			if (type.equals(byte.class)) {
+				castType = "(Byte)";
+				rawValueGetter = "byteValue";
+			} else if (type.equals(boolean.class)) {
+				castType = "(Boolean)";
+				rawValueGetter = "booleanValue";
+			} else if (type.equals(short.class)) {
+				castType = "(Short)";
+				rawValueGetter = "shortValue";
+			} else if (type.equals(int.class)) {
+				castType = "(Integer)";
+				rawValueGetter = "intValue";
+			} else if (type.equals(long.class)) {
+				castType = "(Long)";
+				rawValueGetter = "longValue";
+			} else if (type.equals(float.class)) {
+				castType = "(Float)";
+				rawValueGetter = "floatValue";
+			} else if (type.equals(double.class)) {
+				castType = "(Double)";
+				rawValueGetter = "doubleValue";
+			} else {
+				throw new DynamicCodeGenException("Fatal error: "
+						+ type.getName());
+			}
+		} else if (type.isArray()) {
+			Class ct = type.getComponentType();
+			if (ct.equals(byte.class)) {
+				castType = "(byte[])";
+			} else {
+				throw new UnsupportedOperationException("Not supported yet: "
+						+ type.getName());
+			}
 		} else {
-			MessageTypeException e = new MessageTypeException("Type error: "
-					+ c.getName());
-			LOG.error(e.getMessage(), e);
-			throw e;
+			castType = "(" + type.getName() + ")";
 		}
-	}
-
-	private void insertCodeOfMessageConvertCallForRegisteredType(
-			StringBuilder sb, Field f, Class c, int i) {
-		// target.f0 = (Class) CustomConverter.get(C.class).convert(mpo[i]);
-		// obj = tmpl.convert(mpo[i]);
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(f.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(c.getName());
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CustomConverter.class.getName());
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_GET);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(c.getName() + ".class");
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_CONVERT);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(VARIABLE_NAME_ARRAY);
-		sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-		sb.append(i);
-		sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
+		StringBuilder mc = new StringBuilder();
+		mc.append(castType);
+		mc.append(VARIABLE_NAME_TEMPLATES);
+		mc.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+		mc.append(i);
+		mc.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		String tname = mc.toString();
+		mc = new StringBuilder();
+		mc.append(VARIABLE_NAME_ARRAY);
+		mc.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+		mc.append(i);
+		mc.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		String aname = mc.toString();
+		mc = new StringBuilder();
+		insertMethodCall(mc, tname, METHOD_NAME_CONVERT, new String[] { aname });
+		if (type.isPrimitive()) {
+			tname = mc.toString();
+			mc = new StringBuilder();
+			mc.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
+			mc.append(tname);
+			mc.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
+			tname = mc.toString();
+			mc = new StringBuilder();
+			insertMethodCall(mc, tname, rawValueGetter, new String[0]);
+		}
+		insertValueInsertion(sb, mc.toString());
 		insertSemicolon(sb);
 	}
 
@@ -1030,321 +820,6 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 	}
 
-	private void insertCodeOfConvertMethodCallForPrimTypes(StringBuilder sb,
-			Field f, Class c, int i, String name) {
-		// target.f0 = objs[0].intValue();
-		if (f != null) {
-			sb.append(VARIABLE_NAME_TARGET);
-			sb.append(CHAR_NAME_DOT);
-			sb.append(f.getName());
-			sb.append(CHAR_NAME_SPACE);
-			sb.append(CHAR_NAME_EQUAL);
-			sb.append(CHAR_NAME_SPACE);
-			sb.append(VARIABLE_NAME_ARRAY);
-			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-			sb.append(i);
-			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		} else {
-			sb.append(name);
-		}
-		sb.append(CHAR_NAME_DOT);
-		sb.append(getAsMethodName(c));
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		if (f != null) {
-			insertSemicolon(sb);
-		}
-	}
-
-	private void insertCodeOfConvertMethodCallForWrapTypes(StringBuilder sb,
-			Field f, Class c, int i, String v) {
-		if (f != null) {
-			sb.append(VARIABLE_NAME_TARGET);
-			sb.append(CHAR_NAME_DOT);
-			sb.append(f.getName());
-			sb.append(CHAR_NAME_SPACE);
-			sb.append(CHAR_NAME_EQUAL);
-			sb.append(CHAR_NAME_SPACE);
-		}
-		sb.append(KEYWORD_NEW);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(c.getName());
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		if (f != null) {
-			sb.append(VARIABLE_NAME_ARRAY);
-			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-			sb.append(i);
-			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		} else {
-			sb.append(v);
-		}
-		sb.append(CHAR_NAME_DOT);
-		sb.append(getAsMethodName(c));
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		if (f != null) {
-			insertSemicolon(sb);
-		}
-	}
-
-	private void insertCodeOfConvertMethodCallForList(StringBuilder sb,
-			Field field, Class type, int i) {
-		ParameterizedType generic = (ParameterizedType) field.getGenericType();
-		Class genericType = (Class) generic.getActualTypeArguments()[0];
-
-		// List list = ary[i].asList();
-		sb.append(List.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_LIST);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_ARRAY);
-		sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-		sb.append(i);
-		sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_ASLIST);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		// int size = list.size();
-		sb.append(int.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_SIZE);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_LIST);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_SIZE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		// field initializer
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(field.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(KEYWORD_NEW);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(ArrayList.class.getName());
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		// for loop
-		sb.append(KEYWORD_FOR);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(int.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_I);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(0);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_I);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LESSTHAN);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_SIZE);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_PLUS);
-		sb.append(CHAR_NAME_PLUS);
-		sb.append(VARIABLE_NAME_I);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SPACE);
-
-		// block begin
-		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(MessagePackObject.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_VAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(MessagePackObject.class.getName());
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(VARIABLE_NAME_LIST);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_GET);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(VARIABLE_NAME_I);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(field.getName());
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_ADD);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		insertCodeOfConvertMethodCall(sb, null, genericType, -1,
-				VARIABLE_NAME_VAL);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-		// block end
-		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-	}
-
-	private void insertCodeOfConvertMethodCallForMapType(StringBuilder sb,
-			Field f, Class c, int i) {
-		ParameterizedType generic = (ParameterizedType) f.getGenericType();
-		Class genericType0 = (Class) generic.getActualTypeArguments()[0];
-		Class genericType1 = (Class) generic.getActualTypeArguments()[1];
-
-		// Map map = ary[i].asMap();
-		sb.append(Map.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_MAP);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_ARRAY);
-		sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-		sb.append(i);
-		sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append("asMap");
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		// int size = list.size();
-		sb.append(int.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_SIZE);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_MAP);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_SIZE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		// field initializer
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(f.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(KEYWORD_NEW);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(HashMap.class.getName());
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		// for loop
-		sb.append(KEYWORD_FOR);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(Iterator.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_ITER);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_MAP);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_KEYSET);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_ITERATOR);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_ITER);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_HASNEXT);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-
-		// block map.
-		sb.append(MessagePackObject.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_KEY);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(MessagePackObject.class.getName());
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(VARIABLE_NAME_ITER);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_NEXT);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(MessagePackObject.class.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_VAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(MessagePackObject.class.getName());
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(VARIABLE_NAME_MAP);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_GET);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(VARIABLE_NAME_KEY);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(f.getName());
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_PUT);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		insertCodeOfConvertMethodCall(sb, null, genericType0, -1,
-				VARIABLE_NAME_KEY);
-		sb.append(CHAR_NAME_COMMA);
-		sb.append(CHAR_NAME_SPACE);
-		insertCodeOfConvertMethodCall(sb, null, genericType1, -1,
-				VARIABLE_NAME_VAL);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-
-		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-	}
-
 	public void addConvertMethodForOrdinalEnumTypes(CtClass tmplCtClass,
 			Class c) {
 		// Object convert(MessagePackObject from) throws MessageTypeException;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index e460d6a..2e5b204 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -1,17 +1,46 @@
 package org.msgpack.util.codegen;
 
+import java.lang.reflect.GenericArrayType;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
 import java.math.BigInteger;
 import java.util.List;
 import java.util.Map;
 
+import org.msgpack.CustomConverter;
+import org.msgpack.CustomMessage;
+import org.msgpack.Template;
+import org.msgpack.Templates;
+import org.msgpack.annotation.MessagePackDelegate;
+import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.annotation.MessagePackOrdinalEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class DynamicCodeGenBase implements BasicConstants {
+	public static interface TemplateAccessor {
+		void setTemplates(Template[] templates);
+	}
+
+	private static Logger LOG = LoggerFactory
+			.getLogger(DynamicCodeGenBase.class);
+
 	public DynamicCodeGenBase() {
 	}
 
 	public void addPublicFieldDecl(StringBuilder sb, Class type, String name) {
+		addPublicFieldDecl(sb, type, name, 0);
+	}
+
+	public void addPublicFieldDecl(StringBuilder sb, Class type,
+			String name, int dim) {
 		sb.append(KEYWORD_MODIFIER_PUBLIC);
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(type.getName());
+		for (int i = 0; i < dim; ++i) {
+			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		}
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(name);
 	}
@@ -19,6 +48,17 @@ public class DynamicCodeGenBase implements BasicConstants {
 	public void addPublicMethodDecl(StringBuilder sb, String mname,
 			Class returnType, Class[] paramTypes, String[] anames,
 			Class[] exceptTypes, String methodBody) {
+		int[] dims = new int[paramTypes.length];
+		for (int i = 0; i < paramTypes.length; ++i) {
+			dims[i] = 0;
+		}
+		addPublicMethodDecl(sb, mname, returnType, paramTypes, dims, anames,
+				exceptTypes, methodBody);
+	}
+
+	public void addPublicMethodDecl(StringBuilder sb, String mname,
+			Class returnType, Class[] paramTypes, int[] dims,
+			String[] anames, Class[] exceptTypes, String methodBody) {
 		sb.append(KEYWORD_MODIFIER_PUBLIC);
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(returnType.getName());
@@ -27,6 +67,10 @@ public class DynamicCodeGenBase implements BasicConstants {
 		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
 		for (int i = 0; i < paramTypes.length; ++i) {
 			sb.append(paramTypes[i].getName());
+			for (int j = 0; j < dims[i]; ++j) {
+				sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+				sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+			}
 			sb.append(CHAR_NAME_SPACE);
 			sb.append(anames[i]);
 			if (i + 1 != paramTypes.length) {
@@ -36,16 +80,18 @@ public class DynamicCodeGenBase implements BasicConstants {
 		}
 		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
 		sb.append(CHAR_NAME_SPACE);
-		sb.append(KEYWORD_THROWS);
-		sb.append(CHAR_NAME_SPACE);
-		for (int i = 0; i < exceptTypes.length; ++i) {
-			sb.append(exceptTypes[i].getName());
-			if (i + 1 != exceptTypes.length) {
-				sb.append(CHAR_NAME_COMMA);
-				sb.append(CHAR_NAME_SPACE);
+		if (exceptTypes.length != 0) {
+			sb.append(KEYWORD_THROWS);
+			sb.append(CHAR_NAME_SPACE);
+			for (int i = 0; i < exceptTypes.length; ++i) {
+				sb.append(exceptTypes[i].getName());
+				if (i + 1 != exceptTypes.length) {
+					sb.append(CHAR_NAME_COMMA);
+					sb.append(CHAR_NAME_SPACE);
+				}
 			}
+			sb.append(CHAR_NAME_SPACE);
 		}
-		sb.append(CHAR_NAME_SPACE);
 		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(methodBody);
@@ -64,7 +110,7 @@ public class DynamicCodeGenBase implements BasicConstants {
 		// int lv
 		insertLocalVariableDecl(sb, type, name, 0);
 	}
-	
+
 	public void insertLocalVariableDecl(StringBuilder sb, Class type,
 			String name, int dim) {
 		// int[] lv
@@ -84,9 +130,9 @@ public class DynamicCodeGenBase implements BasicConstants {
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(expr);
 	}
-	
+
 	public void insertInsertion(StringBuilder sb) {
-		// = 
+		// =
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(CHAR_NAME_EQUAL);
 		sb.append(CHAR_NAME_SPACE);
@@ -272,4 +318,76 @@ public class DynamicCodeGenBase implements BasicConstants {
 			throw new DynamicCodeGenException("Type error: " + c.getName());
 		}
 	}
+
+	public Template createTemplate(Type t) {
+		if (t.getClass().equals(Class.class)) {
+			Class c = (Class) t;
+			if (c.equals(boolean.class) || c.equals(Boolean.class)) {
+				return Templates.tBoolean();
+			} else if (c.equals(byte.class) || c.equals(Byte.class)) {
+				return Templates.tByte();
+			} else if (c.equals(short.class) || c.equals(Short.class)) {
+				return Templates.tShort();
+			} else if (c.equals(int.class) || c.equals(Integer.class)) {
+				return Templates.tInteger();
+			} else if (c.equals(float.class) || c.equals(Float.class)) {
+				return Templates.tFloat();
+			} else if (c.equals(long.class) || c.equals(Long.class)) {
+				return Templates.tLong();
+			} else if (c.equals(double.class) || c.equals(Double.class)) {
+				return Templates.tDouble();
+			} else if (c.equals(String.class)) {
+				return Templates.tString();
+			} else if (c.equals(BigInteger.class)) {
+				return Templates.tBigInteger();
+			} else if (CustomConverter.isRegistered(c)) {
+				return (Template) CustomConverter.get(c);
+			} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
+				// @MessagePackMessage
+				Template tmpl = DynamicCodeGenTemplate.create(c);
+				CustomMessage.registerTemplate(c, tmpl);
+				return tmpl;
+			} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
+				// FIXME DelegatePacker
+				UnsupportedOperationException e = new UnsupportedOperationException(
+						"not supported yet. : " + c.getName());
+				LOG.error(e.getMessage(), e);
+				throw e;
+			} else if (CustomMessage.isAnnotated(c,
+					MessagePackOrdinalEnum.class)) {
+				// @MessagePackOrdinalEnum
+				Template tmpl = DynamicCodeGenOrdinalEnumTemplate.create(c);
+				CustomMessage.registerTemplate(c, tmpl);
+				return tmpl;
+			} else {
+				throw new DynamicCodeGenException("Type error: "
+						+ ((Class) t).getName());
+			}
+		} else if (t instanceof GenericArrayType) {
+			GenericArrayType gat = (GenericArrayType) t;
+			Type gct = gat.getGenericComponentType();
+			if (gct.equals(byte.class)) {
+				return Templates.tByteArray();
+			} else {
+				throw new DynamicCodeGenException("Not supported yet: " + gat);
+			}
+		} else if (t instanceof ParameterizedType) {
+			ParameterizedType pt = (ParameterizedType) t;
+			Class rawType = (Class) pt.getRawType();
+			if (rawType.equals(List.class)) {
+				Type[] ats = pt.getActualTypeArguments();
+				return Templates.tList(createTemplate(ats[0]));
+			} else if (rawType.equals(Map.class)) {
+				Type[] ats = pt.getActualTypeArguments();
+				return Templates.tMap(createTemplate(ats[0]),
+						createTemplate(ats[1]));
+			} else {
+				throw new DynamicCodeGenException("Type error: "
+						+ t.getClass().getName());
+			}
+		} else {
+			throw new DynamicCodeGenException("Type error: "
+					+ t.getClass().getName());
+		}
+	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenConverter.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenConverter.java
index 590cde1..4e03714 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenConverter.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenConverter.java
@@ -4,14 +4,6 @@ import org.msgpack.MessageConverter;
 
 public class DynamicCodeGenConverter {
 	public static MessageConverter create(Class c) {
-		try {
-			DynamicCodeGen gen = DynamicCodeGen.getInstance();
-			Class unpackerClass = gen.generateMessageConverterClass(c);
-			return (MessageConverter) unpackerClass.newInstance();
-		} catch (InstantiationException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (IllegalAccessException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
+		return DynamicCodeGenTemplate.create(c);
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
index 1877ed5..defb3c9 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
@@ -5,7 +5,7 @@ public class DynamicCodeGenException extends RuntimeException {
 	public DynamicCodeGenException(String reason) {
 		super(reason);
 	}
-	
+
 	public DynamicCodeGenException(String reason, Throwable t) {
 		super(reason, t);
 	}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumConverter.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumConverter.java
index 298d96c..e06a5bb 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumConverter.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumConverter.java
@@ -6,7 +6,7 @@ public class DynamicCodeGenOrdinalEnumConverter {
 	public static MessageConverter create(Class c) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
-			Class unpackerClass = gen.generateOrdinalEnumConverterClass(c);
+			Class unpackerClass = gen.generateOrdinalEnumTemplateClass(c);
 			return (MessageConverter) unpackerClass.newInstance();
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumPacker.java
index 73da950..01da86a 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumPacker.java
@@ -7,7 +7,7 @@ public class DynamicCodeGenOrdinalEnumPacker {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
 			Class packerClass = gen.generateOrdinalEnumPackerClass(c);
-			return (MessagePacker)packerClass.newInstance();
+			return (MessagePacker) packerClass.newInstance();
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (IllegalAccessException e) {
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumUnpacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumUnpacker.java
index 4ec4880..8f290bd 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumUnpacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumUnpacker.java
@@ -6,7 +6,7 @@ public class DynamicCodeGenOrdinalEnumUnpacker {
 	public static MessageUnpacker create(Class c) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
-			Class unpackerClass = gen.generateOrdinalEnumUnpackerClass(c);
+			Class unpackerClass = gen.generateOrdinalEnumTemplateClass(c);
 			return (MessageUnpacker) unpackerClass.newInstance();
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenPacker.java
index 2a014b8..4bd70f4 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenPacker.java
@@ -8,7 +8,7 @@ public class DynamicCodeGenPacker {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
 			Class packerClass = gen.generateMessagePackerClass(c);
-			return (MessagePacker)packerClass.newInstance();
+			return (MessagePacker) packerClass.newInstance();
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (IllegalAccessException e) {
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java
index 1e12f2e..98a1f8d 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java
@@ -1,13 +1,16 @@
 package org.msgpack.util.codegen;
 
 import org.msgpack.Template;
+import org.msgpack.util.codegen.DynamicCodeGenBase.TemplateAccessor;
 
 public class DynamicCodeGenTemplate {
 	public static Template create(Class c) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
 			Class tmplClass = gen.generateTemplateClass(c);
-			return (Template) tmplClass.newInstance();
+			Object obj = tmplClass.newInstance();
+			((DynamicCodeGenBase.TemplateAccessor)obj).setTemplates(gen.getTemplates(c));
+			return (Template) obj;
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (IllegalAccessException e) {
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenUnpacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenUnpacker.java
index e4fe76b..3cb0fce 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenUnpacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenUnpacker.java
@@ -4,14 +4,6 @@ import org.msgpack.MessageUnpacker;
 
 public class DynamicCodeGenUnpacker {
 	public static MessageUnpacker create(Class c) {
-		try {
-			DynamicCodeGen gen = DynamicCodeGen.getInstance();
-			Class unpackerClass = gen.generateMessageUnpackerClass(c);
-			return (MessageUnpacker) unpackerClass.newInstance();
-		} catch (InstantiationException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (IllegalAccessException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
+		return DynamicCodeGenTemplate.create(c);
 	}
 }

From 28f4bd5a67f06469ed7cf674851635b254fc156a Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Tue, 5 Oct 2010 20:47:03 +0900
Subject: [PATCH 0836/1648] java: edit test programs for
 org.msgpack.util.codegen.*

---
 .../TestDynamicCodeGenPackerConverter.java    | 35 +++++++++++++++++++
 .../TestDynamicCodeGenPackerUnpacker.java     | 35 +++++++++++++++++++
 2 files changed, 70 insertions(+)

diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
index 54dc43b..5514c27 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
@@ -138,6 +138,13 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		src.f2.add("e1");
 		src.f2.add("e2");
 		src.f2.add("e3");
+		src.f3 = new ArrayList>();
+		src.f3.add(src.f2);
+		src.f4 = new ArrayList();
+		SampleListNestedType slnt = new SampleListNestedType();
+		slnt.f0 = new byte[] { 0x01, 0x02 };
+		slnt.f1 = "muga";
+		src.f4.add(slnt);
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		MessagePacker packer = DynamicCodeGenPacker
 				.create(SampleListTypes.class);
@@ -158,6 +165,23 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		for (int i = 0; i < src.f2.size(); ++i) {
 			assertEquals(src.f2.get(i), dst.f2.get(i));
 		}
+		assertEquals(src.f3.size(), dst.f3.size());
+		for (int i = 0; i < src.f3.size(); ++i) {
+			List srclist = src.f3.get(i);
+			List dstlist = dst.f3.get(i);
+			assertEquals(srclist.size(), dstlist.size());
+			for (int j = 0; j < srclist.size(); ++j) {
+				assertEquals(srclist.get(j), dstlist.get(j));
+			}
+		}
+		assertEquals(src.f4.size(), dst.f4.size());
+		for (int i = 0; i < src.f4.size(); ++i) {
+			SampleListNestedType s = src.f4.get(i);
+			SampleListNestedType d = dst.f4.get(i);
+			assertEquals(s.f0[0], d.f0[0]);
+			assertEquals(s.f0[1], d.f0[1]);
+			assertEquals(s.f1, d.f1);
+		}
 		assertFalse(it.hasNext());
 	}
 
@@ -165,11 +189,22 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		public List f0;
 		public List f1;
 		public List f2;
+		public List> f3;
+		public List f4;
 
 		public SampleListTypes() {
 		}
 	}
 
+	@MessagePackMessage
+	public static class SampleListNestedType {
+		public byte[] f0;
+		public String f1;
+
+		public SampleListNestedType() {
+		}
+	}
+
 	public void testMapTypes() throws Exception {
 		SampleMapTypes src = new SampleMapTypes();
 		src.f0 = new HashMap();
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
index 104194e..7876c7f 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
@@ -127,6 +127,13 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		src.f2.add("e1");
 		src.f2.add("e2");
 		src.f2.add("e3");
+		src.f3 = new ArrayList>();
+		src.f3.add(src.f2);
+		src.f4 = new ArrayList();
+		SampleListNestedType slnt = new SampleListNestedType();
+		slnt.f0 = new byte[] { 0x01, 0x02 };
+		slnt.f1 = "muga";
+		src.f4.add(slnt);
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		MessagePacker packer = DynamicCodeGenPacker
 				.create(SampleListTypes.class);
@@ -143,17 +150,45 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		for (int i = 0; i < src.f2.size(); ++i) {
 			assertEquals(src.f2.get(i), dst.f2.get(i));
 		}
+		assertEquals(src.f3.size(), dst.f3.size());
+		for (int i = 0; i < src.f3.size(); ++i) {
+			List srclist = src.f3.get(i);
+			List dstlist = dst.f3.get(i);
+			assertEquals(srclist.size(), dstlist.size());
+			for (int j = 0; j < srclist.size(); ++j) {
+				assertEquals(srclist.get(j), dstlist.get(j));
+			}
+		}
+		assertEquals(src.f4.size(), dst.f4.size());
+		for (int i = 0; i < src.f4.size(); ++i) {
+			SampleListNestedType s = src.f4.get(i);
+			SampleListNestedType d = dst.f4.get(i);
+			assertEquals(s.f0[0], d.f0[0]);
+			assertEquals(s.f0[1], d.f0[1]);
+			assertEquals(s.f1, d.f1);
+		}
 	}
 
 	public static class SampleListTypes {
 		public List f0;
 		public List f1;
 		public List f2;
+		public List> f3;
+		public List f4;
 
 		public SampleListTypes() {
 		}
 	}
 
+	@MessagePackMessage
+	public static class SampleListNestedType {
+		public byte[] f0;
+		public String f1;
+
+		public SampleListNestedType() {
+		}
+	}
+
 	public void testMapTypes() throws Exception {
 		SampleMapTypes src = new SampleMapTypes();
 		src.f0 = new HashMap();

From 4b93b0632379a30ac0d8c5debd26f7ad28d64faa Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Wed, 6 Oct 2010 12:27:04 +0900
Subject: [PATCH 0837/1648] perl: regenerated README file

---
 perl/README | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/perl/README b/perl/README
index 224ff08..ae924ee 100644
--- a/perl/README
+++ b/perl/README
@@ -102,6 +102,11 @@ TODO
         "while(read($socket, $buffer, $arbitrary_buffer_size)) { ... }"). We
         should implement the internal buffer for the unpacker.
 
+    UTF8 mode
+        Data::MessagePack::Unpacker supports utf8 mode, which decodes
+        strings as UTF8-8. << Data::MessagePack->unpack >> should support
+        utf8 mode in a future.
+
 AUTHORS
     Tokuhiro Matsuno
 

From 4321b80999696a6d4e5340fec8f729e11c0ada06 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Wed, 6 Oct 2010 12:29:31 +0900
Subject: [PATCH 0838/1648] Checking in changes prior to tagging of version
 0.29.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index b506234..333a824 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.29
+
+    - add $unpacker->utf8 mode, decoding strings as UTF-8.
+
 0.28

     - added more tests(gfx)
---
 perl/Changes                 | 4 ++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index b506234..333a824 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,7 @@
+0.29
+
+    - add $unpacker->utf8 mode, decoding strings as UTF-8.
+
 0.28
 
     - added more tests(gfx)
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 3d2d695..01e0fc0 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.28';
+our $VERSION = '0.29';
 our $PreferInteger = 0;
 
 sub true () {

From 77a7d3d26aaf957c4f231fdc2090d1b8e842e78c Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Wed, 6 Oct 2010 17:52:32 +0900
Subject: [PATCH 0839/1648] perl: Fix utf8 mode not to be reseted by
 $unpacker->reset method

---
 perl/Changes         |  5 +++++
 perl/t/15_utf8.t     | 32 +++++++++++++++++++-------------
 perl/xs-src/unpack.c |  2 ++
 3 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/perl/Changes b/perl/Changes
index 333a824..091c875 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+
+0.30
+
+    - fix utf8 mode not to be reseted by $unpacker->reset method
+
 0.29
 
     - add $unpacker->utf8 mode, decoding strings as UTF-8.
diff --git a/perl/t/15_utf8.t b/perl/t/15_utf8.t
index d7d17b8..f3163df 100644
--- a/perl/t/15_utf8.t
+++ b/perl/t/15_utf8.t
@@ -5,23 +5,29 @@ use Data::MessagePack;
 use utf8;
 
 my $data = [42, undef, 'foo', "\x{99f1}\x{99dd}"];
-my $packed = Data::MessagePack->pack($data);
+my $packed = Data::MessagePack->pack($data) x 2;
 
 my $u = Data::MessagePack::Unpacker->new()->utf8();
-ok $u->get_utf8();
-$u->execute($packed);
-my $d = $u->data();
-$u->reset();
-is_deeply $d, $data, 'decoded';
+my $p = 0;
+for(1 .. 2) {
+    ok $u->get_utf8();
+    $p = $u->execute($packed, $p);
+    my $d = $u->data();
+    $u->reset();
+    is_deeply $d, $data, 'decoded';
+}
 
 is $u->utf8(0), $u, 'utf8(0)';
-ok !$u->get_utf8();
-$u->execute($packed);
-$d = $u->data();
-$u->reset();
-my $s = $data->[3];
-utf8::encode($s);
-is_deeply $d->[3], $s, 'not decoded';
+$p = 0;
+for(1 .. 2) {
+    ok !$u->get_utf8();
+    $p = $u->execute($packed, $p);
+    my $d = $u->data();
+    $u->reset();
+    my $s = $data->[3];
+    utf8::encode($s);
+    is_deeply $d->[3], $s, 'not decoded';
+}
 
 done_testing;
 
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index f39d8c1..caf8662 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -443,10 +443,12 @@ XS(xs_unpacker_reset) {
     }
 
     UNPACKER(ST(0), mp);
+    bool const utf8 = mp->user.utf8; // save
 
     SV* const data = template_data(mp);
     SvREFCNT_dec(data);
     _reset(ST(0));
+    mp->user.utf8 = utf8;
 
     XSRETURN(0);
 }

From 59f81b331c399f0d9975ee837be1ddbdeda82cf8 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Wed, 6 Oct 2010 18:00:00 +0900
Subject: [PATCH 0840/1648] Checking in changes prior to tagging of version
 0.30.

Changelog diff is:
---
 perl/.shipit                 | 1 +
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/perl/.shipit b/perl/.shipit
index f9997c8..7b4dbc0 100644
--- a/perl/.shipit
+++ b/perl/.shipit
@@ -1,2 +1,3 @@
 steps = FindVersion, ChangeVersion, CheckChangeLog, DistTest, Commit, Tag, MakeDist, UploadCPAN
+MakeDist.destination=~/.shipit-dist/
 git.tagpattern = perl-%v
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 01e0fc0..9f9dcf0 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.29';
+our $VERSION = '0.30';
 our $PreferInteger = 0;
 
 sub true () {

From 90da951a6f385b9860df1bdf4504e534e2a00397 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 7 Oct 2010 03:02:02 +0900
Subject: [PATCH 0841/1648] Use PyUnicode_AsUTF8String() instead of
 o.encode('utf-8').

---
 python/msgpack/_msgpack.pyx | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx
index c887127..66869c8 100644
--- a/python/msgpack/_msgpack.pyx
+++ b/python/msgpack/_msgpack.pyx
@@ -7,6 +7,7 @@ cdef extern from "Python.h":
     cdef object PyBytes_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
     cdef PyObject* Py_True
     cdef PyObject* Py_False
+    cdef object PyUnicode_AsUTF8String(object)
 
     cdef long long PyLong_AsLongLong(object o)
     cdef unsigned long long PyLong_AsUnsignedLongLong(object o)
@@ -105,7 +106,7 @@ cdef class Packer(object):
             if ret == 0:
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
         elif PyUnicode_Check(o):
-            o = o.encode('utf-8')
+            o = PyUnicode_AsUTF8String(o)
             rawval = o
             ret = msgpack_pack_raw(&self.pk, len(o))
             if ret == 0:
@@ -169,7 +170,7 @@ cdef extern from "unpack.h":
     object template_data(template_context* ctx)
 
 
-def unpackb(object packed_bytes):
+def unpackb(bytes packed_bytes):
     """Unpack packed_bytes to object. Returns an unpacked object."""
     cdef const_char_ptr p = packed_bytes
     cdef template_context ctx
@@ -232,7 +233,7 @@ cdef class Unpacker(object):
     cdef object file_like
     cdef int read_size
     cdef object waiting_bytes
-    cdef int use_list
+    cdef bint use_list
 
     def __cinit__(self):
         self.buf = NULL
@@ -241,7 +242,7 @@ cdef class Unpacker(object):
         if self.buf:
             free(self.buf);
 
-    def __init__(self, file_like=None, int read_size=0, use_list=0):
+    def __init__(self, file_like=None, int read_size=0, bint use_list=0):
         if read_size == 0:
             read_size = 1024*1024
         self.use_list = use_list

From 9db35d88316dc652d371de2f1fd3a82e9d285f70 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 7 Oct 2010 03:02:02 +0900
Subject: [PATCH 0842/1648] Use PyUnicode_AsUTF8String() instead of
 o.encode('utf-8').

---
 msgpack/_msgpack.pyx | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index c887127..66869c8 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -7,6 +7,7 @@ cdef extern from "Python.h":
     cdef object PyBytes_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
     cdef PyObject* Py_True
     cdef PyObject* Py_False
+    cdef object PyUnicode_AsUTF8String(object)
 
     cdef long long PyLong_AsLongLong(object o)
     cdef unsigned long long PyLong_AsUnsignedLongLong(object o)
@@ -105,7 +106,7 @@ cdef class Packer(object):
             if ret == 0:
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
         elif PyUnicode_Check(o):
-            o = o.encode('utf-8')
+            o = PyUnicode_AsUTF8String(o)
             rawval = o
             ret = msgpack_pack_raw(&self.pk, len(o))
             if ret == 0:
@@ -169,7 +170,7 @@ cdef extern from "unpack.h":
     object template_data(template_context* ctx)
 
 
-def unpackb(object packed_bytes):
+def unpackb(bytes packed_bytes):
     """Unpack packed_bytes to object. Returns an unpacked object."""
     cdef const_char_ptr p = packed_bytes
     cdef template_context ctx
@@ -232,7 +233,7 @@ cdef class Unpacker(object):
     cdef object file_like
     cdef int read_size
     cdef object waiting_bytes
-    cdef int use_list
+    cdef bint use_list
 
     def __cinit__(self):
         self.buf = NULL
@@ -241,7 +242,7 @@ cdef class Unpacker(object):
         if self.buf:
             free(self.buf);
 
-    def __init__(self, file_like=None, int read_size=0, use_list=0):
+    def __init__(self, file_like=None, int read_size=0, bint use_list=0):
         if read_size == 0:
             read_size = 1024*1024
         self.use_list = use_list

From 84673072396650520e24b5a770c11aa777f69557 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 7 Oct 2010 03:04:00 +0900
Subject: [PATCH 0843/1648] Fix testcase for unicode.

---
 python/test3/test_case.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/python/test3/test_case.py b/python/test3/test_case.py
index 53dfcaf..2f42316 100644
--- a/python/test3/test_case.py
+++ b/python/test3/test_case.py
@@ -98,5 +98,8 @@ def test_match():
     for v, p in cases:
         match(v, p)
 
+def test_unicode():
+    assert_equal(b'foobar', unpacks(packs('foobar')))
+
 if __name__ == '__main__':
     main()

From 4e3c620d367a99eabf2e28d8a6fd371fa33b5dcf Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 7 Oct 2010 03:04:00 +0900
Subject: [PATCH 0844/1648] Fix testcase for unicode.

---
 test3/test_case.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test3/test_case.py b/test3/test_case.py
index 53dfcaf..2f42316 100644
--- a/test3/test_case.py
+++ b/test3/test_case.py
@@ -98,5 +98,8 @@ def test_match():
     for v, p in cases:
         match(v, p)
 
+def test_unicode():
+    assert_equal(b'foobar', unpacks(packs('foobar')))
+
 if __name__ == '__main__':
     main()

From aa2a3e5b075f60c9c5143528ab6520c6231aec0a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 7 Oct 2010 03:07:52 +0900
Subject: [PATCH 0845/1648] Add unicode testcase for Python2.

---
 python/test/test_case.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/python/test/test_case.py b/python/test/test_case.py
index a08c6ce..1cbc494 100644
--- a/python/test/test_case.py
+++ b/python/test/test_case.py
@@ -98,5 +98,8 @@ def test_match():
     for v, p in cases:
         match(v, p)
 
+def test_unicode():
+    assert_equal('foobar', unpacks(packs(u'foobar')))
+
 if __name__ == '__main__':
     main()

From 367f15c247bf37beb37e2a81d1d707bc8ef2e085 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 7 Oct 2010 03:07:52 +0900
Subject: [PATCH 0846/1648] Add unicode testcase for Python2.

---
 test/test_case.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/test_case.py b/test/test_case.py
index a08c6ce..1cbc494 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -98,5 +98,8 @@ def test_match():
     for v, p in cases:
         match(v, p)
 
+def test_unicode():
+    assert_equal('foobar', unpacks(packs(u'foobar')))
+
 if __name__ == '__main__':
     main()

From f3ee5ab37266f61efb029365cf8689191ed2f756 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Thu, 7 Oct 2010 06:28:12 +0900
Subject: [PATCH 0847/1648] cpp: fixes ./bootstrap to surely run ./preprocess

---
 cpp/bootstrap | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cpp/bootstrap b/cpp/bootstrap
index 7f3a182..1ff6b76 100755
--- a/cpp/bootstrap
+++ b/cpp/bootstrap
@@ -38,7 +38,8 @@ test -f ChangeLog || touch ChangeLog
 test -f NEWS      || touch NEWS
 test -f README    || cp -f README.md README
 
-if test ! ./preprocess; then
+./preprocess
+if [ $? -ne 0 ]; then
 	exit 1
 fi
 

From 562d50df4dcc44c80b946279f3e0ba5c75affebf Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 7 Oct 2010 13:01:06 +0900
Subject: [PATCH 0848/1648] java: refactor several programs in a
 org.msgpack.util.codegen package

---
 .../msgpack/util/codegen/DynamicCodeGen.java  | 70 ++++++++-----------
 .../util/codegen/DynamicCodeGenBase.java      | 30 +++++++-
 .../util/codegen/DynamicCodeGenTemplate.java  |  2 +-
 3 files changed, 60 insertions(+), 42 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 1877607..1b4cb91 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -128,14 +128,15 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			checkClassValidation(origClass);
 			checkDefaultConstructorValidation(origClass);
 			CtClass tmplCtClass = pool.makeClass(tmplName);
+			CtClass acsCtClass = pool.get(TemplateAccessorImpl.class.getName());
 			setInterface(tmplCtClass, Template.class);
 			setInterface(tmplCtClass, DynamicCodeGenBase.TemplateAccessor.class);
 			addDefaultConstructor(tmplCtClass);
 			Field[] fields = getDeclaredFields(origClass);
 			Template[] tmpls = createTemplates(fields);
 			setTemplates(origClass, tmpls);
-			addTemplateArrayField(tmplCtClass, origClass);
-			addSetTemplatesMethod(tmplCtClass, origClass);
+			addTemplateArrayField(tmplCtClass, acsCtClass);
+			addSetTemplatesMethod(tmplCtClass, acsCtClass);
 			addUnpackMethod(tmplCtClass, origClass, fields);
 			addConvertMethod(tmplCtClass, origClass, fields);
 			return createClass(tmplCtClass);
@@ -153,9 +154,8 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				+ origClass.getName());
 		try {
 			String origName = origClass.getName();
-			String tmplName = origName + POSTFIX_TYPE_NAME_TEMPLATE + inc();
 			checkClassValidation(origClass);
-			CtClass tmplCtClass = pool.makeClass(tmplName);
+			CtClass tmplCtClass = makeClass(origName);
 			setInterface(tmplCtClass, Template.class);
 			addDefaultConstructor(tmplCtClass);
 			addUnpackMethodForOrdinalEnumTypes(tmplCtClass, origClass);
@@ -170,6 +170,17 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
+	private CtClass makeClass(String origName) throws NotFoundException {
+		StringBuilder sb = new StringBuilder();
+		sb.append(origName);
+		sb.append(POSTFIX_TYPE_NAME_TEMPLATE);
+		sb.append(inc());
+		String invokerName = sb.toString();
+		CtClass newCtClass = pool.makeClass(invokerName);
+		newCtClass.setModifiers(Modifier.PUBLIC);
+		return newCtClass;
+	}
+
 	private void checkClassValidation(Class origClass) {
 		// not public, abstract
 		int mod = origClass.getModifiers();
@@ -405,46 +416,25 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	private void addTemplateArrayField(CtClass newCtClass, Class origClass) {
-		StringBuilder sb = new StringBuilder();
-		addPublicFieldDecl(sb, Template.class, VARIABLE_NAME_TEMPLATES, 1);
-		insertSemicolon(sb);
-		LOG.trace("templates field src: " + sb.toString());
-		try {
-			CtField templatesCtField = CtField.make(sb.toString(), newCtClass);
-			newCtClass.addField(templatesCtField);
-		} catch (CannotCompileException e) {
-			DynamicCodeGenException ex = new DynamicCodeGenException(e
-					.getMessage()
-					+ ": " + sb.toString(), e);
-			LOG.error(ex.getMessage(), ex);
-			throw ex;
-		}
+	private void addTemplateArrayField(CtClass newCtClass, CtClass acsCtClass)
+			throws NotFoundException, CannotCompileException {
+		CtField tmplsField = acsCtClass
+				.getDeclaredField(VARIABLE_NAME_TEMPLATES);
+		CtField tmplsField2 = new CtField(tmplsField.getType(), tmplsField
+				.getName(), newCtClass);
+		newCtClass.addField(tmplsField2);
 	}
 
-	private void addSetTemplatesMethod(CtClass newCtClass, Class origClass) {
-		StringBuilder sb = new StringBuilder();
-		StringBuilder body = new StringBuilder();
-		body.append("_$$_templates = _$$_tmpls;");
-		addPublicMethodDecl(sb, METHOD_NAME_SETTEMPLATES, void.class,
-				new Class[] { Template.class }, new int[] { 1 },
-				new String[] { VARIABLE_NAME_TEMPLATES0 }, new Class[0],
-				body.toString());
-		LOG.trace("settemplates method src: " + sb.toString());
-		try {
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), newCtClass);
-			newCtClass.addMethod(newCtMethod);
-		} catch (CannotCompileException e) {
-			DynamicCodeGenException ex = new DynamicCodeGenException(e
-					.getMessage()
-					+ ": " + sb.toString(), e);
-			LOG.error(ex.getMessage(), ex);
-			throw ex;
-		}
+	private void addSetTemplatesMethod(CtClass newCtClass, CtClass acsCtClass)
+			throws NotFoundException, CannotCompileException {
+		CtMethod settmplsMethod = acsCtClass
+				.getDeclaredMethod(METHOD_NAME_SETTEMPLATES);
+		CtMethod settmplsMethod2 = CtNewMethod.copy(settmplsMethod, newCtClass,
+				null);
+		newCtClass.addMethod(settmplsMethod2);
 	}
 
-	private void addUnpackMethod(CtClass unpackerCtClass, Class c, Field[] fs)
-			throws CannotCompileException, NotFoundException {
+	private void addUnpackMethod(CtClass unpackerCtClass, Class c, Field[] fs) {
 		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
 		StringBuilder sb = new StringBuilder();
 		StringBuilder bsb = new StringBuilder();
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 2e5b204..533a8bc 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -21,6 +21,14 @@ public class DynamicCodeGenBase implements BasicConstants {
 	public static interface TemplateAccessor {
 		void setTemplates(Template[] templates);
 	}
+	
+	public static class TemplateAccessorImpl implements TemplateAccessor {
+		public Template[] _$$_templates;
+		
+		public void setTemplates(Template[] _$$_tmpls) {
+			_$$_templates = _$$_tmpls;
+		}
+	}
 
 	private static Logger LOG = LoggerFactory
 			.getLogger(DynamicCodeGenBase.class);
@@ -114,7 +122,11 @@ public class DynamicCodeGenBase implements BasicConstants {
 	public void insertLocalVariableDecl(StringBuilder sb, Class type,
 			String name, int dim) {
 		// int[] lv
-		sb.append(type.getName());
+		if (type.equals(byte[].class)) {
+			sb.append("byte[]");	
+		} else {
+			sb.append(type.getName());	
+		}
 		for (int i = 0; i < dim; ++i) {
 			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
 			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
@@ -122,6 +134,22 @@ public class DynamicCodeGenBase implements BasicConstants {
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(name);
 	}
+	
+	static int getArrayDim(Class type) {
+		if (type.isArray()) {
+			return 1 + getArrayDim(type.getComponentType());
+		} else {
+			return 0;
+		}
+	}
+
+	static Class getArrayBaseType(Class type) {
+		if (type.isArray()) {
+			return getArrayBaseType(type.getComponentType());
+		} else {
+			return type;
+		}
+	}
 
 	public void insertValueInsertion(StringBuilder sb, String expr) {
 		// = expr
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java
index 98a1f8d..2a9e584 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java
@@ -9,7 +9,7 @@ public class DynamicCodeGenTemplate {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
 			Class tmplClass = gen.generateTemplateClass(c);
 			Object obj = tmplClass.newInstance();
-			((DynamicCodeGenBase.TemplateAccessor)obj).setTemplates(gen.getTemplates(c));
+			((TemplateAccessor) obj).setTemplates(gen.getTemplates(c));
 			return (Template) obj;
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);

From 4bbfb3f9a398e569a560034e8d8eb58f7fb22c7f Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 7 Oct 2010 15:27:11 +0900
Subject: [PATCH 0849/1648] java: refactor DynamicCodeGen.java

---
 .../msgpack/util/codegen/BasicConstants.java  | 14 +++
 .../msgpack/util/codegen/DynamicCodeGen.java  | 84 +++--------------
 .../util/codegen/DynamicCodeGenBase.java      | 90 +++++++++++++------
 3 files changed, 88 insertions(+), 100 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
index 8448c02..f169bcd 100644
--- a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
@@ -88,6 +88,20 @@ public interface BasicConstants {
 
 	String METHOD_NAME_VALUEOF = "valueOf";
 
+	String METHOD_NAME_BOOLEANVALUE = "booleanValue";
+
+	String METHOD_NAME_BYTEVALUE = "byteValue";
+
+	String METHOD_NAME_SHORTVALUE = "shortValue";
+
+	String METHOD_NAME_INTVALUE = "intValue";
+
+	String METHOD_NAME_FLOATVALUE = "floatValue";
+
+	String METHOD_NAME_LONGVALUE = "longValue";
+
+	String METHOD_NAME_DOUBLEVALUE = "doubleValue";
+
 	String METHOD_NAME_ADD = "add";
 
 	String METHOD_NAME_PUT = "put";
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 1b4cb91..909bef6 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -254,18 +254,18 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		return allFields.toArray(new Field[0]);
 	}
 
-	private void checkFieldValidation(Field f, List fs) {
+	private void checkFieldValidation(Field field, List fields) {
 		// check that it has a public modifier
-		int mod = f.getModifiers();
+		int mod = field.getModifiers();
 		if ((!(Modifier.isPublic(mod))) || Modifier.isStatic(mod)
 				|| Modifier.isFinal(mod) || Modifier.isTransient(mod)
-				|| f.isSynthetic()) {
-			throwFieldValidationException(f);
+				|| field.isSynthetic()) {
+			throwFieldValidationException(field);
 		}
 		// check same name
-		for (Field f0 : fs) {
-			if (f0.getName().equals(f.getName())) {
-				throwFieldValidationException(f);
+		for (Field f : fields) {
+			if (f.getName().equals(field.getName())) {
+				throwFieldValidationException(field);
 			}
 		}
 	}
@@ -486,39 +486,10 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		String castType = null;
 		String rawValueGetter = null;
 		if (type.isPrimitive()) {
-			if (type.equals(byte.class)) {
-				castType = "(Byte)";
-				rawValueGetter = "byteValue";
-			} else if (type.equals(boolean.class)) {
-				castType = "(Boolean)";
-				rawValueGetter = "booleanValue";
-			} else if (type.equals(short.class)) {
-				castType = "(Short)";
-				rawValueGetter = "shortValue";
-			} else if (type.equals(int.class)) {
-				castType = "(Integer)";
-				rawValueGetter = "intValue";
-			} else if (type.equals(long.class)) {
-				castType = "(Long)";
-				rawValueGetter = "longValue";
-			} else if (type.equals(float.class)) {
-				castType = "(Float)";
-				rawValueGetter = "floatValue";
-			} else if (type.equals(double.class)) {
-				castType = "(Double)";
-				rawValueGetter = "doubleValue";
-			} else {
-				throw new DynamicCodeGenException("Fatal error: "
-						+ type.getName());
-			}
+			castType = "(" + primitiveTypeToWrapperType(type).getName() + ")";
+			rawValueGetter = getPrimTypeValueMethodName(type);
 		} else if (type.isArray()) {
-			Class ct = type.getComponentType();
-			if (ct.equals(byte.class)) {
-				castType = "(byte[])";
-			} else {
-				throw new UnsupportedOperationException("Not supported yet: "
-						+ type.getName());
-			}
+			castType = "(" + arrayTypeToString(type) + ")";
 		} else {
 			castType = "(" + type.getName() + ")";
 		}
@@ -696,39 +667,10 @@ public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		String castType = null;
 		String rawValueGetter = null;
 		if (type.isPrimitive()) {
-			if (type.equals(byte.class)) {
-				castType = "(Byte)";
-				rawValueGetter = "byteValue";
-			} else if (type.equals(boolean.class)) {
-				castType = "(Boolean)";
-				rawValueGetter = "booleanValue";
-			} else if (type.equals(short.class)) {
-				castType = "(Short)";
-				rawValueGetter = "shortValue";
-			} else if (type.equals(int.class)) {
-				castType = "(Integer)";
-				rawValueGetter = "intValue";
-			} else if (type.equals(long.class)) {
-				castType = "(Long)";
-				rawValueGetter = "longValue";
-			} else if (type.equals(float.class)) {
-				castType = "(Float)";
-				rawValueGetter = "floatValue";
-			} else if (type.equals(double.class)) {
-				castType = "(Double)";
-				rawValueGetter = "doubleValue";
-			} else {
-				throw new DynamicCodeGenException("Fatal error: "
-						+ type.getName());
-			}
+			castType = "(" + primitiveTypeToWrapperType(type).getName() + ")";
+			rawValueGetter = getPrimTypeValueMethodName(type);
 		} else if (type.isArray()) {
-			Class ct = type.getComponentType();
-			if (ct.equals(byte.class)) {
-				castType = "(byte[])";
-			} else {
-				throw new UnsupportedOperationException("Not supported yet: "
-						+ type.getName());
-			}
+			castType = "(" + arrayTypeToString(type) + ")";
 		} else {
 			castType = "(" + type.getName() + ")";
 		}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 533a8bc..af7c909 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -9,6 +9,7 @@ import java.util.Map;
 
 import org.msgpack.CustomConverter;
 import org.msgpack.CustomMessage;
+import org.msgpack.MessageTypeException;
 import org.msgpack.Template;
 import org.msgpack.Templates;
 import org.msgpack.annotation.MessagePackDelegate;
@@ -21,10 +22,10 @@ public class DynamicCodeGenBase implements BasicConstants {
 	public static interface TemplateAccessor {
 		void setTemplates(Template[] templates);
 	}
-	
+
 	public static class TemplateAccessorImpl implements TemplateAccessor {
 		public Template[] _$$_templates;
-		
+
 		public void setTemplates(Template[] _$$_tmpls) {
 			_$$_templates = _$$_tmpls;
 		}
@@ -122,19 +123,17 @@ public class DynamicCodeGenBase implements BasicConstants {
 	public void insertLocalVariableDecl(StringBuilder sb, Class type,
 			String name, int dim) {
 		// int[] lv
-		if (type.equals(byte[].class)) {
-			sb.append("byte[]");	
-		} else {
-			sb.append(type.getName());	
-		}
-		for (int i = 0; i < dim; ++i) {
+		int dim0 = dim + getArrayDim(type);
+		Class type0 = getArrayBaseType(type);
+		sb.append(type0.getName());
+		for (int i = 0; i < dim0; ++i) {
 			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
 			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
 		}
 		sb.append(CHAR_NAME_SPACE);
 		sb.append(name);
 	}
-	
+
 	static int getArrayDim(Class type) {
 		if (type.isArray()) {
 			return 1 + getArrayDim(type.getComponentType());
@@ -150,6 +149,18 @@ public class DynamicCodeGenBase implements BasicConstants {
 			return type;
 		}
 	}
+	
+	public String arrayTypeToString(Class type) {
+		StringBuilder sb = new StringBuilder();
+		int dim = getArrayDim(type);
+		Class t = getArrayBaseType(type);
+		sb.append(t.getName());
+		for (int i = 0; i < dim; ++i) {
+			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		}
+		return sb.toString();
+	}
 
 	public void insertValueInsertion(StringBuilder sb, String expr) {
 		// = expr
@@ -235,25 +246,7 @@ public class DynamicCodeGenBase implements BasicConstants {
 	public void insertTypeConvToObjectType(StringBuilder sb, Class type,
 			String expr) throws DynamicCodeGenException {
 		if (type.isPrimitive()) { // primitive type
-			if (type.equals(boolean.class)) {
-				// new Boolean(expr)
-				insertConsCall(sb, Boolean.class, expr);
-			} else if (type.equals(byte.class)) {
-				insertConsCall(sb, Byte.class, expr);
-			} else if (type.equals(short.class)) {
-				insertConsCall(sb, Short.class, expr);
-			} else if (type.equals(int.class)) {
-				insertConsCall(sb, Integer.class, expr);
-			} else if (type.equals(long.class)) {
-				insertConsCall(sb, Long.class, expr);
-			} else if (type.equals(float.class)) {
-				insertConsCall(sb, Float.class, expr);
-			} else if (type.equals(double.class)) {
-				insertConsCall(sb, Double.class, expr);
-			} else {
-				throw new DynamicCodeGenException("Type error: "
-						+ type.getName());
-			}
+			insertConsCall(sb, primitiveTypeToWrapperType(type), expr);
 		} else { // reference type
 			sb.append(expr);
 		}
@@ -285,7 +278,46 @@ public class DynamicCodeGenBase implements BasicConstants {
 			sb.append(CHAR_NAME_SPACE);
 		}
 	}
-
+	
+	public Class primitiveTypeToWrapperType(Class type) {
+		if (type.equals(boolean.class)) {
+			return Boolean.class;
+		} else if (type.equals(byte.class)) {
+			return Byte.class;
+		} else if (type.equals(short.class)) {
+			return Short.class;
+		} else if (type.equals(int.class)) {
+			return Integer.class;
+		} else if (type.equals(long.class)) {
+			return Long.class;
+		} else if (type.equals(float.class)) {
+			return Float.class;
+		} else if (type.equals(double.class)) {
+			return Double.class;
+		} else {
+			throw new MessageTypeException("Type error: " + type.getName());
+		}
+	}
+	
+	public String getPrimTypeValueMethodName(Class type) {
+		if (type.equals(boolean.class)) {
+			return METHOD_NAME_BOOLEANVALUE;
+		} else if (type.equals(byte.class)) {
+			return METHOD_NAME_BYTEVALUE;
+		} else if (type.equals(short.class)) {
+			return METHOD_NAME_SHORTVALUE;
+		} else if (type.equals(int.class)) {
+			return METHOD_NAME_INTVALUE;
+		} else if (type.equals(long.class)) {
+			return METHOD_NAME_LONGVALUE;
+		} else if (type.equals(float.class)) {
+			return METHOD_NAME_FLOATVALUE;
+		} else if (type.equals(double.class)) {
+			return METHOD_NAME_DOUBLEVALUE;
+		} else {
+			throw new MessageTypeException("Type error: " + type.getName());
+		}
+	}
 	public String getUnpackMethodName(Class c)
 			throws DynamicCodeGenException {
 		if (c.equals(boolean.class) || c.equals(Boolean.class)) {

From 8548b3645ad3d694d20483b20a1534a51e941897 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 7 Oct 2010 16:13:56 +0900
Subject: [PATCH 0850/1648] perl: cleanup

---
 perl/lib/Data/MessagePack.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 01e0fc0..9ce08c1 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -8,13 +8,13 @@ our $PreferInteger = 0;
 
 sub true () {
     require Data::MessagePack::Boolean;
-    no warnings 'once', 'redefine';
+    no warnings 'once';
     return $Data::MessagePack::Boolean::true;
 }
 
 sub false () {
     require Data::MessagePack::Boolean;
-    no warnings 'once', 'redefine';
+    no warnings 'once';
     return $Data::MessagePack::Boolean::false;
 }
 

From 91b1806e9d39cc8913b075d4c75a366759abfb21 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 7 Oct 2010 16:16:59 +0900
Subject: [PATCH 0851/1648] perl: add XS_VERSION_BOOTCHECK for safety

---
 perl/xs-src/MessagePack.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/xs-src/MessagePack.c b/perl/xs-src/MessagePack.c
index 0c3c0b1..82ad108 100644
--- a/perl/xs-src/MessagePack.c
+++ b/perl/xs-src/MessagePack.c
@@ -21,7 +21,7 @@ void init_Data__MessagePack_unpack(pTHX_ bool const cloning);
 
 XS(boot_Data__MessagePack) {
     dXSARGS;
-    PERL_UNUSED_VAR(items);
+    XS_VERSION_BOOTCHECK;
 
     init_Data__MessagePack_pack(aTHX_   false);
     init_Data__MessagePack_unpack(aTHX_ false);

From 2be98e84672027a1d89eddae90feaec1714ed8d3 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 7 Oct 2010 16:18:47 +0900
Subject: [PATCH 0852/1648] perl: changelogging

---
 perl/Changes | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/perl/Changes b/perl/Changes
index 091c875..76c2fd7 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,11 +1,15 @@
 
+0.31
+
+    - add version check at bootstrap in order to avoid load old .so (gfx)
+
 0.30
 
-    - fix utf8 mode not to be reseted by $unpacker->reset method
+    - fix utf8 mode not to be reseted by $unpacker->reset method (gfx)
 
 0.29
 
-    - add $unpacker->utf8 mode, decoding strings as UTF-8.
+    - add $unpacker->utf8 mode, decoding strings as UTF-8 (gfx)
 
 0.28
 

From f3fbb7e1f60f37bf283dc959141fa0a45ccaca43 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 7 Oct 2010 16:54:49 +0900
Subject: [PATCH 0853/1648] perl: Update Makefile.PL using M::I::XSUtil 0.36
 (support for ccache)

---
 perl/Makefile.PL | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/perl/Makefile.PL b/perl/Makefile.PL
index 219400f..111b705 100644
--- a/perl/Makefile.PL
+++ b/perl/Makefile.PL
@@ -1,8 +1,7 @@
 # Usage: Makefile.PL --pp # disable XS
 #        Makefile.PL -g   # add -g to the compiler and disable optimization flags
 use inc::Module::Install;
-use Module::Install::XSUtil 0.32;
-use Config;
+use Module::Install::XSUtil 0.36;
 
 name 'Data-MessagePack';
 all_from 'lib/Data/MessagePack.pm';

From 119a03dd5f2c2ae086b315ace39b8956b8aa6b59 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 7 Oct 2010 16:56:36 +0900
Subject: [PATCH 0854/1648] perl: changelogging

---
 perl/Changes | 1 +
 1 file changed, 1 insertion(+)

diff --git a/perl/Changes b/perl/Changes
index 76c2fd7..6e07966 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,6 +1,7 @@
 
 0.31
 
+    - update Module::Install::XSUtil for ccache support (gfx)
     - add version check at bootstrap in order to avoid load old .so (gfx)
 
 0.30

From b9483deea3417619fb02156d1146cd1a48a22ffb Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 7 Oct 2010 17:06:45 +0900
Subject: [PATCH 0855/1648] Checking in changes prior to tagging of version
 0.31.

Changelog diff is:
---
 perl/lib/Data/MessagePack.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 33edf34..6d37d8f 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.30';
+our $VERSION = '0.31';
 our $PreferInteger = 0;
 
 sub true () {

From a3d831b72342b1ae3fbab63213892651a4d21b6f Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 7 Oct 2010 23:53:01 +0900
Subject: [PATCH 0856/1648] java: refactor DynamicCodeGen.java

---
 .../main/java/org/msgpack/util/codegen/BasicConstants.java  | 6 ++++++
 .../main/java/org/msgpack/util/codegen/DynamicCodeGen.java  | 2 +-
 .../java/org/msgpack/util/codegen/DynamicCodeGenBase.java   | 6 +++---
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
index f169bcd..1e84843 100644
--- a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
@@ -85,6 +85,10 @@ public interface BasicConstants {
 	String VARIABLE_NAME_TEMPLATES = "_$$_templates";
 
 	String VARIABLE_NAME_TEMPLATES0 = "_$$_tmpls";
+	
+	String VARIABLE_NAME_CLIENT = "_$$_client";
+	
+	String VARIABLE_NAME_CLIENT0 = "_$$_c";
 
 	String METHOD_NAME_VALUEOF = "valueOf";
 
@@ -127,6 +131,8 @@ public interface BasicConstants {
 	String METHOD_NAME_MSGCONVERT = "messageConvert";
 
 	String METHOD_NAME_SETTEMPLATES = "setTemplates";
+	
+	String METHOD_NAME_SETCLIENT = "setClient";
 
 	String METHOD_NAME_PACK = "pack";
 
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 909bef6..ee5667d 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -38,7 +38,7 @@ import org.msgpack.annotation.MessagePackOrdinalEnum;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
+class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private static Logger LOG = LoggerFactory.getLogger(DynamicCodeGen.class);
 
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index af7c909..6fa6abc 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -134,7 +134,7 @@ public class DynamicCodeGenBase implements BasicConstants {
 		sb.append(name);
 	}
 
-	static int getArrayDim(Class type) {
+	protected int getArrayDim(Class type) {
 		if (type.isArray()) {
 			return 1 + getArrayDim(type.getComponentType());
 		} else {
@@ -142,7 +142,7 @@ public class DynamicCodeGenBase implements BasicConstants {
 		}
 	}
 
-	static Class getArrayBaseType(Class type) {
+	protected Class getArrayBaseType(Class type) {
 		if (type.isArray()) {
 			return getArrayBaseType(type.getComponentType());
 		} else {
@@ -150,7 +150,7 @@ public class DynamicCodeGenBase implements BasicConstants {
 		}
 	}
 	
-	public String arrayTypeToString(Class type) {
+	protected String arrayTypeToString(Class type) {
 		StringBuilder sb = new StringBuilder();
 		int dim = getArrayDim(type);
 		Class t = getArrayBaseType(type);

From 3f5ac54bf53f251493d475663c6a2a3de9df954e Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 9 Oct 2010 21:53:49 +0900
Subject: [PATCH 0857/1648] java: refactor the programs in a
 org.msgpack.util.codegen package

---
 .../msgpack/util/codegen/BasicConstants.java  | 194 -----
 .../org/msgpack/util/codegen/Constants.java   | 218 +++++-
 .../msgpack/util/codegen/DynamicCodeGen.java  | 665 +++++++-----------
 .../util/codegen/DynamicCodeGenBase.java      | 387 +++++-----
 ...enConverter.java => DynamicConverter.java} |   4 +-
 ....java => DynamicOrdinalEnumConverter.java} |   2 +-
 ...ker.java => DynamicOrdinalEnumPacker.java} |   2 +-
 ...e.java => DynamicOrdinalEnumTemplate.java} |   2 +-
 ...r.java => DynamicOrdinalEnumUnpacker.java} |   2 +-
 ...cCodeGenPacker.java => DynamicPacker.java} |   2 +-
 ...eGenTemplate.java => DynamicTemplate.java} |   2 +-
 ...eGenUnpacker.java => DynamicUnpacker.java} |   4 +-
 .../TestDynamicCodeGenPackerConverter.java    |  82 +--
 .../TestDynamicCodeGenPackerUnpacker.java     |  80 +--
 14 files changed, 729 insertions(+), 917 deletions(-)
 delete mode 100644 java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
 rename java/src/main/java/org/msgpack/util/codegen/{DynamicCodeGenConverter.java => DynamicConverter.java} (61%)
 rename java/src/main/java/org/msgpack/util/codegen/{DynamicCodeGenOrdinalEnumConverter.java => DynamicOrdinalEnumConverter.java} (91%)
 rename java/src/main/java/org/msgpack/util/codegen/{DynamicCodeGenOrdinalEnumPacker.java => DynamicOrdinalEnumPacker.java} (91%)
 rename java/src/main/java/org/msgpack/util/codegen/{DynamicCodeGenOrdinalEnumTemplate.java => DynamicOrdinalEnumTemplate.java} (90%)
 rename java/src/main/java/org/msgpack/util/codegen/{DynamicCodeGenOrdinalEnumUnpacker.java => DynamicOrdinalEnumUnpacker.java} (91%)
 rename java/src/main/java/org/msgpack/util/codegen/{DynamicCodeGenPacker.java => DynamicPacker.java} (93%)
 rename java/src/main/java/org/msgpack/util/codegen/{DynamicCodeGenTemplate.java => DynamicTemplate.java} (94%)
 rename java/src/main/java/org/msgpack/util/codegen/{DynamicCodeGenUnpacker.java => DynamicUnpacker.java} (61%)

diff --git a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java b/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
deleted file mode 100644
index 1e84843..0000000
--- a/java/src/main/java/org/msgpack/util/codegen/BasicConstants.java
+++ /dev/null
@@ -1,194 +0,0 @@
-package org.msgpack.util.codegen;
-
-public interface BasicConstants {
-
-	String KEYWORD_MODIFIER_PUBLIC = "public";
-
-	String KEYWORD_CATCH = "catch";
-
-	String KEYWORD_ELSE = "else";
-
-	String KEYWORD_ELSEIF = "else if";
-
-	String KEYWORD_FOR = "for";
-
-	String KEYWORD_IF = "if";
-
-	String KEYWORD_INSTANCEOF = "instanceof";
-
-	String KEYWORD_NEW = "new";
-
-	String KEYWORD_NULL = "null";
-
-	String KEYWORD_RETURN = "return";
-
-	String KEYWORD_THROW = "throw";
-
-	String KEYWORD_THROWS = "throws";
-
-	String KEYWORD_TRY = "try";
-
-	String CHAR_NAME_SPACE = " ";
-
-	String CHAR_NAME_COMMA = ",";
-
-	String CHAR_NAME_EQUAL = "=";
-
-	String CHAR_NAME_PLUS = "+";
-
-	String CHAR_NAME_MINUS = "-";
-
-	String CHAR_NAME_UNDERSCORE = "_";
-
-	String CHAR_NAME_LESSTHAN = "<";
-
-	String CHAR_NAME_RIGHT_PARENTHESIS = ")";
-
-	String CHAR_NAME_LEFT_PARENTHESIS = "(";
-
-	String CHAR_NAME_RIGHT_CURLY_BRACKET = "}";
-
-	String CHAR_NAME_LEFT_CURLY_BRACKET = "{";
-
-	String CHAR_NAME_RIGHT_SQUARE_BRACKET = "]";
-
-	String CHAR_NAME_LEFT_SQUARE_BRACKET = "[";
-
-	String CHAR_NAME_DOT = ".";
-
-	String CHAR_NAME_SEMICOLON = ";";
-
-	String VARIABLE_NAME_PK = "_$$_pk";
-
-	String VARIABLE_NAME_OBJECT = "_$$_obj";
-
-	String VARIABLE_NAME_TARGET = "_$$_t";
-
-	String VARIABLE_NAME_SIZE = "_$$_len";
-
-	String VARIABLE_NAME_ARRAY = "_$$_ary";
-
-	String VARIABLE_NAME_LIST = "_$$_list";
-
-	String VARIABLE_NAME_MAP = "_$$_map";
-
-	String VARIABLE_NAME_KEY = "_$$_key";
-
-	String VARIABLE_NAME_VAL = "_$$_val";
-
-	String VARIABLE_NAME_ITER = "_$$_iter";
-
-	String VARIABLE_NAME_MPO = "_$$_mpo";
-
-	String VARIABLE_NAME_I = "i";
-
-	String VARIABLE_NAME_TEMPLATES = "_$$_templates";
-
-	String VARIABLE_NAME_TEMPLATES0 = "_$$_tmpls";
-	
-	String VARIABLE_NAME_CLIENT = "_$$_client";
-	
-	String VARIABLE_NAME_CLIENT0 = "_$$_c";
-
-	String METHOD_NAME_VALUEOF = "valueOf";
-
-	String METHOD_NAME_BOOLEANVALUE = "booleanValue";
-
-	String METHOD_NAME_BYTEVALUE = "byteValue";
-
-	String METHOD_NAME_SHORTVALUE = "shortValue";
-
-	String METHOD_NAME_INTVALUE = "intValue";
-
-	String METHOD_NAME_FLOATVALUE = "floatValue";
-
-	String METHOD_NAME_LONGVALUE = "longValue";
-
-	String METHOD_NAME_DOUBLEVALUE = "doubleValue";
-
-	String METHOD_NAME_ADD = "add";
-
-	String METHOD_NAME_PUT = "put";
-
-	String METHOD_NAME_GET = "get";
-
-	String METHOD_NAME_SIZE = "size";
-
-	String METHOD_NAME_KEYSET = "keySet";
-
-	String METHOD_NAME_ITERATOR = "iterator";
-
-	String METHOD_NAME_HASNEXT = "hasNext";
-
-	String METHOD_NAME_NEXT = "next";
-
-	String METHOD_NAME_ORDINAL = "ordinal";
-
-	String METHOD_NAME_GETENUMCONSTANTS = "getEnumConstants";
-
-	String METHOD_NAME_CONVERT = "convert";
-
-	String METHOD_NAME_MSGCONVERT = "messageConvert";
-
-	String METHOD_NAME_SETTEMPLATES = "setTemplates";
-	
-	String METHOD_NAME_SETCLIENT = "setClient";
-
-	String METHOD_NAME_PACK = "pack";
-
-	String METHOD_NAME_PACKARRAY = "packArray";
-
-	String METHOD_NAME_UNPACK = "unpack";
-
-	String METHOD_NAME_UNPACKBOOLEAN = "unpackBoolean";
-
-	String METHOD_NAME_UNPACKBYTE = "unpackByte";
-
-	String METHOD_NAME_UNPACKDOUBLE = "unpackDouble";
-
-	String METHOD_NAME_UNPACKFLOAT = "unpackFloat";
-
-	String METHOD_NAME_UNPACKINT = "unpackInt";
-
-	String METHOD_NAME_UNPACKLONG = "unpackLong";
-
-	String METHOD_NAME_UNPACKSHORT = "unpackShort";
-
-	String METHOD_NAME_UNPACKSTRING = "unpackString";
-
-	String METHOD_NAME_UNPACKBIGINTEGER = "unpackBigInteger";
-
-	String METHOD_NAME_UNPACKOBJECT = "unpackObject";
-
-	String METHOD_NAME_UNPACKBYTEARRAY = "unpackByteArray";
-
-	String METHOD_NAME_UNPACKARRAY = "unpackArray";
-
-	String METHOD_NAME_UNPACKMAP = "unpackMap";
-
-	String METHOD_NAME_ASARRAY = "asArray";
-
-	String METHOD_NAME_ASBOOLEAN = "asBoolean";
-
-	String METHOD_NAME_ASBYTE = "asByte";
-
-	String METHOD_NAME_ASSHORT = "asShort";
-
-	String METHOD_NAME_ASINT = "asInt";
-
-	String METHOD_NAME_ASFLOAT = "asFloat";
-
-	String METHOD_NAME_ASLONG = "asLong";
-
-	String METHOD_NAME_ASDOUBLE = "asDouble";
-
-	String METHOD_NAME_ASSTRING = "asString";
-
-	String METHOD_NAME_ASBYTEARRAY = "asByteArray";
-
-	String METHOD_NAME_ASBIGINTEGER = "asBigInteger";
-
-	String METHOD_NAME_ASLIST = "asList";
-
-	String METHOD_NAME_ASMAP = "asMap";
-}
diff --git a/java/src/main/java/org/msgpack/util/codegen/Constants.java b/java/src/main/java/org/msgpack/util/codegen/Constants.java
index 91d584e..e61f039 100644
--- a/java/src/main/java/org/msgpack/util/codegen/Constants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/Constants.java
@@ -1,6 +1,6 @@
 package org.msgpack.util.codegen;
 
-public interface Constants extends BasicConstants {
+public interface Constants {
 	String POSTFIX_TYPE_NAME_PACKER = "_$$_Packer";
 
 	String POSTFIX_TYPE_NAME_UNPACKER = "_$$_Unpacker";
@@ -8,4 +8,220 @@ public interface Constants extends BasicConstants {
 	String POSTFIX_TYPE_NAME_CONVERTER = "_$$_Converter";
 
 	String POSTFIX_TYPE_NAME_TEMPLATE = "_$$_Template";
+
+	String KEYWORD_MODIFIER_PUBLIC = "public";
+
+	String KEYWORD_CATCH = "catch";
+
+	String KEYWORD_ELSE = "else";
+
+	String KEYWORD_ELSEIF = "else if";
+
+	String KEYWORD_FOR = "for";
+
+	String KEYWORD_IF = "if";
+
+	String KEYWORD_INSTANCEOF = "instanceof";
+
+	String KEYWORD_NEW = "new";
+
+	String KEYWORD_NULL = "null";
+
+	String KEYWORD_RETURN = "return";
+
+	String KEYWORD_THROW = "throw";
+
+	String KEYWORD_THROWS = "throws";
+
+	String KEYWORD_TRY = "try";
+
+	String CHAR_NAME_SPACE = " ";
+
+	String CHAR_NAME_COMMA = ",";
+
+	String CHAR_NAME_EQUAL = "=";
+
+	String CHAR_NAME_PLUS = "+";
+
+	String CHAR_NAME_MINUS = "-";
+
+	String CHAR_NAME_UNDERSCORE = "_";
+
+	String CHAR_NAME_LESSTHAN = "<";
+
+	String CHAR_NAME_RIGHT_PARENTHESIS = ")";
+
+	String CHAR_NAME_LEFT_PARENTHESIS = "(";
+
+	String CHAR_NAME_RIGHT_CURLY_BRACKET = "}";
+
+	String CHAR_NAME_LEFT_CURLY_BRACKET = "{";
+
+	String CHAR_NAME_RIGHT_SQUARE_BRACKET = "]";
+
+	String CHAR_NAME_LEFT_SQUARE_BRACKET = "[";
+
+	String CHAR_NAME_DOT = ".";
+
+	String CHAR_NAME_SEMICOLON = ";";
+
+	String VARIABLE_NAME_PK = "_$$_pk";
+
+	String VARIABLE_NAME_OBJECT = "_$$_obj";
+
+	String VARIABLE_NAME_TARGET = "_$$_t";
+
+	String VARIABLE_NAME_SIZE = "_$$_len";
+
+	String VARIABLE_NAME_ARRAY = "_$$_ary";
+
+	String VARIABLE_NAME_LIST = "_$$_list";
+
+	String VARIABLE_NAME_MAP = "_$$_map";
+
+	String VARIABLE_NAME_KEY = "_$$_key";
+
+	String VARIABLE_NAME_VAL = "_$$_val";
+
+	String VARIABLE_NAME_ITER = "_$$_iter";
+
+	String VARIABLE_NAME_MPO = "_$$_mpo";
+
+	String VARIABLE_NAME_I = "i";
+
+	String VARIABLE_NAME_TEMPLATES = "_$$_templates";
+
+	String VARIABLE_NAME_TEMPLATES0 = "_$$_tmpls";
+
+	String VARIABLE_NAME_CLIENT = "_$$_client";
+
+	String VARIABLE_NAME_CLIENT0 = "_$$_c";
+
+	String METHOD_NAME_VALUEOF = "valueOf";
+
+	String METHOD_NAME_BOOLEANVALUE = "booleanValue";
+
+	String METHOD_NAME_BYTEVALUE = "byteValue";
+
+	String METHOD_NAME_SHORTVALUE = "shortValue";
+
+	String METHOD_NAME_INTVALUE = "intValue";
+
+	String METHOD_NAME_FLOATVALUE = "floatValue";
+
+	String METHOD_NAME_LONGVALUE = "longValue";
+
+	String METHOD_NAME_DOUBLEVALUE = "doubleValue";
+
+	String METHOD_NAME_ADD = "add";
+
+	String METHOD_NAME_PUT = "put";
+
+	String METHOD_NAME_GET = "get";
+
+	String METHOD_NAME_SIZE = "size";
+
+	String METHOD_NAME_KEYSET = "keySet";
+
+	String METHOD_NAME_ITERATOR = "iterator";
+
+	String METHOD_NAME_HASNEXT = "hasNext";
+
+	String METHOD_NAME_NEXT = "next";
+
+	String METHOD_NAME_ORDINAL = "ordinal";
+
+	String METHOD_NAME_GETENUMCONSTANTS = "getEnumConstants";
+
+	String METHOD_NAME_CONVERT = "convert";
+
+	String METHOD_NAME_MSGCONVERT = "messageConvert";
+
+	String METHOD_NAME_SETTEMPLATES = "setTemplates";
+
+	String METHOD_NAME_SETCLIENT = "setClient";
+
+	String METHOD_NAME_PACK = "pack";
+
+	String METHOD_NAME_PACKARRAY = "packArray";
+
+	String METHOD_NAME_UNPACK = "unpack";
+
+	String METHOD_NAME_UNPACKBOOLEAN = "unpackBoolean";
+
+	String METHOD_NAME_UNPACKBYTE = "unpackByte";
+
+	String METHOD_NAME_UNPACKDOUBLE = "unpackDouble";
+
+	String METHOD_NAME_UNPACKFLOAT = "unpackFloat";
+
+	String METHOD_NAME_UNPACKINT = "unpackInt";
+
+	String METHOD_NAME_UNPACKLONG = "unpackLong";
+
+	String METHOD_NAME_UNPACKSHORT = "unpackShort";
+
+	String METHOD_NAME_UNPACKSTRING = "unpackString";
+
+	String METHOD_NAME_UNPACKBIGINTEGER = "unpackBigInteger";
+
+	String METHOD_NAME_UNPACKOBJECT = "unpackObject";
+
+	String METHOD_NAME_UNPACKBYTEARRAY = "unpackByteArray";
+
+	String METHOD_NAME_UNPACKARRAY = "unpackArray";
+
+	String METHOD_NAME_UNPACKMAP = "unpackMap";
+
+	String METHOD_NAME_ASARRAY = "asArray";
+
+	String METHOD_NAME_ASBOOLEAN = "asBoolean";
+
+	String METHOD_NAME_ASBYTE = "asByte";
+
+	String METHOD_NAME_ASSHORT = "asShort";
+
+	String METHOD_NAME_ASINT = "asInt";
+
+	String METHOD_NAME_ASFLOAT = "asFloat";
+
+	String METHOD_NAME_ASLONG = "asLong";
+
+	String METHOD_NAME_ASDOUBLE = "asDouble";
+
+	String METHOD_NAME_ASSTRING = "asString";
+
+	String METHOD_NAME_ASBYTEARRAY = "asByteArray";
+
+	String METHOD_NAME_ASBIGINTEGER = "asBigInteger";
+
+	String METHOD_NAME_ASLIST = "asList";
+
+	String METHOD_NAME_ASMAP = "asMap";
+
+	String STATEMENT_PACKER_PACKERMETHODBODY_01 = "%s _$$_t = (%s)$2; ";
+
+	String STATEMENT_PACKER_PACKERMETHODBODY_02 = "$1.packArray(%d); ";
+
+	String STATEMENT_PACKER_PACKERMETHODBODY_03 = "$1.pack(_$$_t.%s); ";
+
+	String STATEMENT_PACKER_PACKERMETHODBODY_04 = "$1.pack(((java.lang.Enum)_$$_t).ordinal()); ";
+
+	String STATEMENT_PACKER_UNPACKERMETHODBODY_01 = "%s _$$_t = new %s(); ";
+
+	String STATEMENT_PACKER_UNPACKERMETHODBODY_02 = "$1.unpackArray(); ";
+
+	String STATEMENT_PACKER_UNPACKERMETHODBODY_03 = "_$$_t.%s = %s(%s)_$$_templates[%d].unpack($1)%s; ";
+
+	String STATEMENT_PACKER_UNPACKERMETHODBODY_04 = "return _$$_t; ";
+
+	String STATEMENT_PACKER_UNPACKERMETHODBODY_05 = "int i = $1.unpackInt(); ";
+
+	String STATEMENT_PACKER_UNPACKERMETHODBODY_06 = "return %s.class.getEnumConstants()[i]; ";
+
+	String STATEMENT_PACKER_CONVERTMETHODBODY_01 = "%s _$$_ary = $1.asArray(); ";
+
+	String STATEMENT_PACKER_CONVERTMETHODBODY_02 = "_$$_t.%s = %s(%s)_$$_templates[%d].convert(_$$_ary[%d])%s; ";
+
+	String STATEMENT_PACKER_CONVERTMETHODBODY_03 = "int i = _$$_ary[0].asInt(); ";
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index ee5667d..3954f17 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -9,15 +9,10 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import javassist.CannotCompileException;
-import javassist.ClassPool;
 import javassist.CtClass;
-import javassist.CtConstructor;
-import javassist.CtField;
 import javassist.CtMethod;
-import javassist.CtNewConstructor;
 import javassist.CtNewMethod;
 import javassist.NotFoundException;
 
@@ -44,12 +39,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private static DynamicCodeGen INSTANCE;
 
-	private static AtomicInteger COUNTER = new AtomicInteger(0);
-
-	private static int inc() {
-		return COUNTER.addAndGet(1);
-	}
-
 	public static DynamicCodeGen getInstance() {
 		if (INSTANCE == null) {
 			LOG.info("create an instance of the type: "
@@ -59,37 +48,37 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		return INSTANCE;
 	}
 
-	private ClassPool pool;
-
-	private ConcurrentHashMap tmplMap;
+	private ConcurrentHashMap tmplCache;
 
 	DynamicCodeGen() {
-		pool = ClassPool.getDefault();
-		tmplMap = new ConcurrentHashMap();
+		super();
+		tmplCache = new ConcurrentHashMap();
 	}
 
-	public void setTemplates(Class origClass, Template[] tmpls) {
-		tmplMap.putIfAbsent(origClass.getName(), tmpls);
+	public void setTemplates(Class type, Template[] tmpls) {
+		tmplCache.putIfAbsent(type.getName(), tmpls);
 	}
 
-	public Template[] getTemplates(Class origClass) {
-		return tmplMap.get(origClass.getName());
+	public Template[] getTemplates(Class type) {
+		return tmplCache.get(type.getName());
 	}
 
 	public Class generateMessagePackerClass(Class origClass) {
-		LOG.debug("start generating a MessagePacker impl.: "
-				+ origClass.getName());
 		try {
+			LOG.debug("start generating a packer class for "
+					+ origClass.getName());
 			String origName = origClass.getName();
 			String packerName = origName + POSTFIX_TYPE_NAME_PACKER + inc();
-			checkClassValidation(origClass);
+			checkTypeValidation(origClass);
 			checkDefaultConstructorValidation(origClass);
 			CtClass packerCtClass = pool.makeClass(packerName);
 			setInterface(packerCtClass, MessagePacker.class);
 			addDefaultConstructor(packerCtClass);
 			Field[] fields = getDeclaredFields(origClass);
-			addPackMethod(packerCtClass, origClass, fields);
-			return createClass(packerCtClass);
+			addPackMethod(packerCtClass, origClass, fields, false);
+			Class packerClass = createClass(packerCtClass);
+			LOG.debug("generated a packer class for " + origClass.getName());
+			return packerClass;
 		} catch (NotFoundException e) {
 			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
@@ -100,17 +89,19 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	}
 
 	public Class generateOrdinalEnumPackerClass(Class origClass) {
-		LOG.debug("start generating a OrdinalEnumPacker impl.: "
-				+ origClass.getName());
 		try {
+			LOG.debug("start generating an enum packer for "
+					+ origClass.getName());
 			String origName = origClass.getName();
 			String packerName = origName + POSTFIX_TYPE_NAME_PACKER + inc();
-			checkClassValidation(origClass);
+			checkTypeValidation(origClass);
 			CtClass packerCtClass = pool.makeClass(packerName);
 			setInterface(packerCtClass, MessagePacker.class);
 			addDefaultConstructor(packerCtClass);
-			addPackMethodForOrdinalEnumTypes(packerCtClass, origClass);
-			return createClass(packerCtClass);
+			addPackMethod(packerCtClass, origClass, null, true);
+			Class packerClass = createClass(packerCtClass);
+			LOG.debug("generated an enum class for " + origClass.getName());
+			return packerClass;
 		} catch (NotFoundException e) {
 			LOG.error(e.getMessage(), e);
 			throw new DynamicCodeGenException(e.getMessage(), e);
@@ -121,123 +112,103 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	}
 
 	public Class generateTemplateClass(Class origClass) {
-		LOG.debug("start generating a Template impl.: " + origClass.getName());
 		try {
+			LOG.debug("start generating a template class for "
+					+ origClass.getName());
 			String origName = origClass.getName();
 			String tmplName = origName + POSTFIX_TYPE_NAME_TEMPLATE + inc();
-			checkClassValidation(origClass);
+			checkTypeValidation(origClass);
 			checkDefaultConstructorValidation(origClass);
 			CtClass tmplCtClass = pool.makeClass(tmplName);
-			CtClass acsCtClass = pool.get(TemplateAccessorImpl.class.getName());
 			setInterface(tmplCtClass, Template.class);
 			setInterface(tmplCtClass, DynamicCodeGenBase.TemplateAccessor.class);
 			addDefaultConstructor(tmplCtClass);
 			Field[] fields = getDeclaredFields(origClass);
 			Template[] tmpls = createTemplates(fields);
 			setTemplates(origClass, tmpls);
-			addTemplateArrayField(tmplCtClass, acsCtClass);
-			addSetTemplatesMethod(tmplCtClass, acsCtClass);
-			addUnpackMethod(tmplCtClass, origClass, fields);
-			addConvertMethod(tmplCtClass, origClass, fields);
-			return createClass(tmplCtClass);
+			addTemplateArrayField(tmplCtClass);
+			addSetTemplatesMethod(tmplCtClass);
+			addUnpackMethod(tmplCtClass, origClass, fields, false);
+			addConvertMethod(tmplCtClass, origClass, fields, false);
+			Class tmplClass = createClass(tmplCtClass);
+			LOG.debug("generated a template class for " + origClass.getName());
+			return tmplClass;
 		} catch (NotFoundException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
 		} catch (CannotCompileException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
 		}
 	}
 
 	public Class generateOrdinalEnumTemplateClass(Class origClass) {
-		LOG.debug("start generating a OrdinalEnumTemplate impl.: "
-				+ origClass.getName());
 		try {
+			LOG.debug("start generating a enum template class for "
+					+ origClass.getName());
 			String origName = origClass.getName();
-			checkClassValidation(origClass);
-			CtClass tmplCtClass = makeClass(origName);
+			checkTypeValidation(origClass);
+			String tmplName = origName + POSTFIX_TYPE_NAME_TEMPLATE + inc();
+			CtClass tmplCtClass = pool.makeClass(tmplName);
 			setInterface(tmplCtClass, Template.class);
 			addDefaultConstructor(tmplCtClass);
-			addUnpackMethodForOrdinalEnumTypes(tmplCtClass, origClass);
-			addConvertMethodForOrdinalEnumTypes(tmplCtClass, origClass);
-			return createClass(tmplCtClass);
+			addUnpackMethod(tmplCtClass, origClass, null, true);
+			addConvertMethod(tmplCtClass, origClass, null, true);
+			// addConvertMethodForOrdinalEnumTypes(tmplCtClass, origClass);
+			Class tmplClass = createClass(tmplCtClass);
+			LOG.debug("generated an enum template class for "
+					+ origClass.getName());
+			return tmplClass;
 		} catch (NotFoundException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
 		} catch (CannotCompileException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
 		}
 	}
 
-	private CtClass makeClass(String origName) throws NotFoundException {
-		StringBuilder sb = new StringBuilder();
-		sb.append(origName);
-		sb.append(POSTFIX_TYPE_NAME_TEMPLATE);
-		sb.append(inc());
-		String invokerName = sb.toString();
-		CtClass newCtClass = pool.makeClass(invokerName);
-		newCtClass.setModifiers(Modifier.PUBLIC);
-		return newCtClass;
-	}
-
-	private void checkClassValidation(Class origClass) {
+	@Override
+	public void checkTypeValidation(Class origClass) {
 		// not public, abstract
 		int mod = origClass.getModifiers();
 		if ((!Modifier.isPublic(mod)) || Modifier.isAbstract(mod)) {
-			throwClassValidationException(origClass,
+			throwTypeValidationException(origClass,
 					"a class must have a public modifier");
 		}
 		// interface
 		if (origClass.isInterface()) {
-			throwClassValidationException(origClass,
+			throwTypeValidationException(origClass,
 					"cannot generate packer and unpacker for an interface");
 		}
 	}
 
-	private static void throwClassValidationException(Class origClass,
-			String msg) {
-		DynamicCodeGenException e = new DynamicCodeGenException(msg + ": "
-				+ origClass.getName());
-		LOG.error(e.getMessage(), e);
-		throw e;
-	}
-
-	private void checkDefaultConstructorValidation(Class origClass) {
+	@Override
+	public void checkDefaultConstructorValidation(Class origClass) {
+		// check that the zero-argument constructor exists
 		Constructor cons = null;
 		try {
 			cons = origClass.getDeclaredConstructor(new Class[0]);
-		} catch (Exception e1) {
+		} catch (Exception e) {
 			throwConstructorValidationException(origClass);
 		}
+
+		// check the modifiers
 		int mod = cons.getModifiers();
 		if (!Modifier.isPublic(mod)) {
 			throwConstructorValidationException(origClass);
 		}
 	}
 
-	private static void throwConstructorValidationException(Class origClass) {
-		DynamicCodeGenException e = new DynamicCodeGenException(
-				"it must have a public zero-argument constructor: "
-						+ origClass.getName());
-		LOG.error(e.getMessage(), e);
-		throw e;
-	}
-
-	private void setInterface(CtClass packerCtClass, Class infClass)
-			throws NotFoundException {
-		CtClass infCtClass = pool.get(infClass.getName());
-		packerCtClass.addInterface(infCtClass);
-	}
-
-	private void addDefaultConstructor(CtClass dstCtClass)
-			throws CannotCompileException {
-		CtConstructor newCtCons = CtNewConstructor
-				.defaultConstructor(dstCtClass);
-		dstCtClass.addConstructor(newCtCons);
-	}
-
-	private Field[] getDeclaredFields(Class origClass) {
+	Field[] getDeclaredFields(Class origClass) {
 		ArrayList allFields = new ArrayList();
 		Class nextClass = origClass;
 		while (nextClass != Object.class) {
@@ -247,6 +218,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 					checkFieldValidation(field, allFields);
 					allFields.add(field);
 				} catch (DynamicCodeGenException e) { // ignore
+					LOG.error(e.getMessage(), e);
 				}
 			}
 			nextClass = nextClass.getSuperclass();
@@ -254,7 +226,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		return allFields.toArray(new Field[0]);
 	}
 
-	private void checkFieldValidation(Field field, List fields) {
+	void checkFieldValidation(Field field, List fields) {
 		// check that it has a public modifier
 		int mod = field.getModifiers();
 		if ((!(Modifier.isPublic(mod))) || Modifier.isStatic(mod)
@@ -270,13 +242,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	private static void throwFieldValidationException(Field f) {
-		DynamicCodeGenException e = new DynamicCodeGenException(
-				"it must be a public field: " + f.getName());
-		LOG.debug(e.getMessage(), e);
-		throw e;
-	}
-
 	Template[] createTemplates(Field[] fields) {
 		Template[] tmpls = new Template[fields.length];
 		for (int i = 0; i < tmpls.length; ++i) {
@@ -294,19 +259,25 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	private void addPackMethod(CtClass packerCtClass, Class c, Field[] fs) {
+	private void addPackMethod(CtClass packerCtClass, Class c, Field[] fs,
+			boolean isEnum) {
 		// void pack(Packer pk, Object target) throws IOException;
 		StringBuilder sb = new StringBuilder();
-		StringBuilder bsb = new StringBuilder();
-		insertPackMethodBody(bsb, c, fs);
-		addPublicMethodDecl(sb, METHOD_NAME_PACK, void.class, new Class[] {
-				Packer.class, Object.class }, new String[] { VARIABLE_NAME_PK,
-				VARIABLE_NAME_OBJECT }, new Class[] { IOException.class }, bsb
-				.toString());
-		LOG.trace("pack method src: " + sb.toString());
+		if (!isEnum) {
+			insertPackMethodBody(sb, c, fs);
+		} else {
+			insertOrdinalEnumPackMethodBody(sb, c);
+		}
 		try {
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(),
-					packerCtClass);
+			LOG.trace("pack method src: " + sb.toString());
+			int mod = javassist.Modifier.PUBLIC;
+			CtClass returnType = classToCtClass(void.class);
+			String mname = METHOD_NAME_PACK;
+			CtClass[] paramTypes = new CtClass[] {
+					classToCtClass(Packer.class), classToCtClass(Object.class) };
+			CtClass[] exceptTypes = new CtClass[] { classToCtClass(IOException.class) };
+			CtMethod newCtMethod = CtNewMethod.make(mod, returnType, mname,
+					paramTypes, exceptTypes, sb.toString(), packerCtClass);
 			packerCtClass.addMethod(newCtMethod);
 		} catch (CannotCompileException e) {
 			DynamicCodeGenException ex = new DynamicCodeGenException(e
@@ -314,21 +285,29 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 					+ ": " + sb.toString(), e);
 			LOG.error(ex.getMessage(), ex);
 			throw ex;
+		} catch (NotFoundException e) {
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage()
+					+ ": " + sb.toString(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
 		}
 	}
 
-	private void insertPackMethodBody(StringBuilder sb, Class c, Field[] fs) {
-		insertLocalVariableDecl(sb, c, VARIABLE_NAME_TARGET);
-		StringBuilder mc = new StringBuilder();
-		insertTypeCast(mc, c, VARIABLE_NAME_OBJECT);
-		insertValueInsertion(sb, mc.toString());
-		insertSemicolon(sb);
-		insertMethodCall(sb, VARIABLE_NAME_PK, METHOD_NAME_PACKARRAY,
-				new String[] { new Integer(fs.length).toString() });
-		insertSemicolon(sb);
-		for (Field f : fs) {
+	private void insertPackMethodBody(StringBuilder sb, Class type,
+			Field[] fields) {
+		// void pack(Packer packer, Object target) throws IOException;
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		String typeName = classToString(type);
+		Object[] args0 = new Object[] { typeName, typeName };
+		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_01, args0));
+		Object[] args1 = new Object[] { fields.length };
+		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_02, args1));
+		for (Field f : fields) {
 			insertCodeOfPackMethodCall(sb, f);
 		}
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
 	}
 
 	private void insertCodeOfPackMethodCall(StringBuilder sb, Field field) {
@@ -352,7 +331,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			; // ignore
 		} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
 			// @MessagePackMessage
-			MessagePacker packer = DynamicCodeGenPacker.create(c);
+			MessagePacker packer = DynamicPacker.create(c);
 			CustomMessage.registerPacker(c, packer);
 		} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
 			// FIXME DelegatePacker
@@ -362,7 +341,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			throw e;
 		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
 			// @MessagePackOrdinalEnum
-			MessagePacker packer = DynamicCodeGenOrdinalEnumPacker.create(c);
+			MessagePacker packer = DynamicOrdinalEnumPacker.create(c);
 			CustomMessage.registerPacker(c, packer);
 		} else {
 			MessageTypeException e = new MessageTypeException("unknown type: "
@@ -370,106 +349,77 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			LOG.error(e.getMessage(), e);
 			throw e;
 		}
-		StringBuilder fa = new StringBuilder();
-		insertFieldAccess(fa, VARIABLE_NAME_TARGET, field.getName());
-		insertMethodCall(sb, VARIABLE_NAME_PK, METHOD_NAME_PACK,
-				new String[] { fa.toString() });
-		insertSemicolon(sb);
+		Object[] args = new Object[] { field.getName() };
+		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_03, args));
 	}
 
-	private void addPackMethodForOrdinalEnumTypes(CtClass packerCtClass,
-			Class c) throws CannotCompileException, NotFoundException {
-		// void pack(Packer pk, Object target) throws IOException;
+	private void insertOrdinalEnumPackMethodBody(StringBuilder sb, Class c) {
+		// void pack(Packer packer, Object target) throws IOException;
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		String typeName = classToString(c);
+		Object[] args0 = new Object[] { typeName, typeName };
+		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_01, args0));
+		Object[] args1 = new Object[] { 1 };
+		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_02, args1));
+		Object[] args2 = new Object[0];
+		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_04, args2));
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+	}
+
+	private void addUnpackMethod(CtClass tmplCtClass, Class type,
+			Field[] fields, boolean isEnum) {
+		// Object unpack(Unpacker u) throws IOException, MessageTypeException;
 		StringBuilder sb = new StringBuilder();
-		StringBuilder bsb = new StringBuilder();
-		insertLocalVariableDecl(bsb, c, VARIABLE_NAME_TARGET);
-		StringBuilder fa = new StringBuilder();
-		insertTypeCast(fa, c, VARIABLE_NAME_OBJECT);
-		insertValueInsertion(bsb, fa.toString());
-		insertSemicolon(bsb);
-		insertMethodCall(bsb, VARIABLE_NAME_PK, METHOD_NAME_PACKARRAY,
-				new String[] { new Integer(1).toString() });
-		insertSemicolon(bsb);
-		fa = new StringBuilder();
-		insertTypeCast(fa, Enum.class, VARIABLE_NAME_TARGET);
-		String t = fa.toString();
-		fa = new StringBuilder();
-		insertMethodCall(fa, t, METHOD_NAME_ORDINAL, new String[0]);
-		insertMethodCall(bsb, VARIABLE_NAME_PK, METHOD_NAME_PACK,
-				new String[] { fa.toString() });
-		insertSemicolon(bsb);
-		addPublicMethodDecl(sb, METHOD_NAME_PACK, void.class, new Class[] {
-				Packer.class, Object.class }, new String[] { VARIABLE_NAME_PK,
-				VARIABLE_NAME_OBJECT }, new Class[] { IOException.class }, bsb
-				.toString());
-		LOG.trace("pack method src: " + sb.toString());
+		if (!isEnum) {
+			insertUnpackMethodBody(sb, type, fields);
+		} else {
+			insertOrdinalEnumUnpackMethodBody(sb, type);
+		}
 		try {
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(),
-					packerCtClass);
-			packerCtClass.addMethod(newCtMethod);
+			LOG.trace("unpack method src: " + sb.toString());
+			int mod = javassist.Modifier.PUBLIC;
+			CtClass returnType = classToCtClass(Object.class);
+			String mname = METHOD_NAME_UNPACK;
+			CtClass[] paramTypes = new CtClass[] { classToCtClass(Unpacker.class) };
+			CtClass[] exceptTypes = new CtClass[] {
+					classToCtClass(IOException.class),
+					classToCtClass(MessageTypeException.class) };
+			CtMethod newCtMethod = CtNewMethod.make(mod, returnType, mname,
+					paramTypes, exceptTypes, sb.toString(), tmplCtClass);
+			tmplCtClass.addMethod(newCtMethod);
 		} catch (CannotCompileException e) {
 			DynamicCodeGenException ex = new DynamicCodeGenException(e
 					.getMessage()
 					+ ": " + sb.toString(), e);
 			LOG.error(ex.getMessage(), ex);
 			throw ex;
+		} catch (NotFoundException e) {
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage()
+					+ ": " + sb.toString(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
 		}
 	}
 
-	private void addTemplateArrayField(CtClass newCtClass, CtClass acsCtClass)
-			throws NotFoundException, CannotCompileException {
-		CtField tmplsField = acsCtClass
-				.getDeclaredField(VARIABLE_NAME_TEMPLATES);
-		CtField tmplsField2 = new CtField(tmplsField.getType(), tmplsField
-				.getName(), newCtClass);
-		newCtClass.addField(tmplsField2);
-	}
-
-	private void addSetTemplatesMethod(CtClass newCtClass, CtClass acsCtClass)
-			throws NotFoundException, CannotCompileException {
-		CtMethod settmplsMethod = acsCtClass
-				.getDeclaredMethod(METHOD_NAME_SETTEMPLATES);
-		CtMethod settmplsMethod2 = CtNewMethod.copy(settmplsMethod, newCtClass,
-				null);
-		newCtClass.addMethod(settmplsMethod2);
-	}
-
-	private void addUnpackMethod(CtClass unpackerCtClass, Class c, Field[] fs) {
-		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
-		StringBuilder sb = new StringBuilder();
-		StringBuilder bsb = new StringBuilder();
-		insertUnpackMethodBody(bsb, c, fs);
-		addPublicMethodDecl(sb, METHOD_NAME_UNPACK, Object.class,
-				new Class[] { Unpacker.class },
-				new String[] { VARIABLE_NAME_PK }, new Class[] {
-						MessageTypeException.class, IOException.class }, bsb
-						.toString());
-		LOG.trace("unpack method src: " + sb.toString());
-		try {
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(),
-					unpackerCtClass);
-			unpackerCtClass.addMethod(newCtMethod);
-		} catch (CannotCompileException e) {
-			DynamicCodeGenException ex = new DynamicCodeGenException(e
-					.getMessage()
-					+ ": " + sb.toString(), e);
-			LOG.error(ex.getMessage(), ex);
-			throw ex;
-		}
-	}
-
-	private void insertUnpackMethodBody(StringBuilder sb, Class c, Field[] fs) {
-		insertLocalVariableDecl(sb, c, VARIABLE_NAME_TARGET);
-		StringBuilder mc = new StringBuilder();
-		insertDefaultConsCall(mc, c);
-		insertValueInsertion(sb, mc.toString());
-		insertSemicolon(sb);
-		insertMethodCall(sb, VARIABLE_NAME_PK, METHOD_NAME_UNPACKARRAY,
-				new String[0]);
-		insertSemicolon(sb);
-		insertCodeOfUnpackMethodCalls(sb, fs);
-		insertReturnStat(sb, VARIABLE_NAME_TARGET);
-		insertSemicolon(sb);
+	private void insertUnpackMethodBody(StringBuilder sb, Class type,
+			Field[] fields) {
+		// Object unpack(Unpacker u) throws IOException, MessageTypeException;
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		// Foo _$$_t = new Foo();
+		String typeName = classToString(type);
+		Object[] args0 = new Object[] { typeName, typeName };
+		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_01, args0));
+		// $1.unpackArray();
+		Object[] args1 = new Object[0];
+		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_02, args1));
+		insertCodeOfUnpackMethodCalls(sb, fields);
+		// return _$$_t;
+		Object[] args2 = new Object[0];
+		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_04, args2));
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
 	}
 
 	private void insertCodeOfUnpackMethodCalls(StringBuilder sb, Field[] fields) {
@@ -481,40 +431,17 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	private void insertCodeOfUnpackMethodCall(StringBuilder sb, Field field,
 			int i) {
 		// target.fi = ((Integer)_$$_tmpls[i].unpack(_$$_pk)).intValue();
-		Class type = field.getType();
-		insertFieldAccess(sb, VARIABLE_NAME_TARGET, field.getName());
-		String castType = null;
-		String rawValueGetter = null;
-		if (type.isPrimitive()) {
-			castType = "(" + primitiveTypeToWrapperType(type).getName() + ")";
-			rawValueGetter = getPrimTypeValueMethodName(type);
-		} else if (type.isArray()) {
-			castType = "(" + arrayTypeToString(type) + ")";
-		} else {
-			castType = "(" + type.getName() + ")";
-		}
-		StringBuilder mc = new StringBuilder();
-		mc.append(castType);
-		mc.append(VARIABLE_NAME_TEMPLATES);
-		mc.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-		mc.append(i);
-		mc.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		String tname = mc.toString();
-		mc = new StringBuilder();
-		insertMethodCall(mc, tname, METHOD_NAME_UNPACK,
-				new String[] { VARIABLE_NAME_PK });
-		if (type.isPrimitive()) {
-			tname = mc.toString();
-			mc = new StringBuilder();
-			mc.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			mc.append(tname);
-			mc.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			tname = mc.toString();
-			mc = new StringBuilder();
-			insertMethodCall(mc, tname, rawValueGetter, new String[0]);
-		}
-		insertValueInsertion(sb, mc.toString());
-		insertSemicolon(sb);
+		Class returnType = field.getType();
+		boolean isPrim = returnType.isPrimitive();
+		Object[] args = new Object[] {
+				field.getName(),
+				isPrim ? "(" : "",
+				isPrim ? getPrimToWrapperType(returnType).getName()
+						: classToString(returnType),
+				i,
+				isPrim ? ")." + getPrimTypeValueMethodName(returnType) + "()"
+						: "" };
+		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_03, args));
 	}
 
 	private void insertCodeOfUnpackMethodCallForMsgUnpackableType(
@@ -567,59 +494,41 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 	}
 
-	private void addUnpackMethodForOrdinalEnumTypes(CtClass unpackerCtClass,
-			Class c) {
-		// Object unpack(Unpacker pac) throws IOException, MessageTypeException;
-		StringBuilder sb = new StringBuilder();
-		StringBuilder bsb = new StringBuilder();
-		insertMethodCall(bsb, VARIABLE_NAME_PK, METHOD_NAME_UNPACKARRAY,
-				new String[0]);
-		insertSemicolon(bsb);
-		StringBuilder mc = new StringBuilder();
-		insertMethodCall(mc, VARIABLE_NAME_PK, METHOD_NAME_UNPACKINT,
-				new String[0]);
-		insertLocalVariableDecl(bsb, int.class, VARIABLE_NAME_I);
-		insertValueInsertion(bsb, mc.toString());
-		insertSemicolon(bsb);
-		mc = new StringBuilder();
-		insertMethodCall(mc, c.getName() + ".class",
-				METHOD_NAME_GETENUMCONSTANTS, new String[0]);
-		mc.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-		mc.append(VARIABLE_NAME_I);
-		mc.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		insertReturnStat(bsb, mc.toString());
-		insertSemicolon(bsb);
-		addPublicMethodDecl(sb, METHOD_NAME_UNPACK, Object.class,
-				new Class[] { Unpacker.class },
-				new String[] { VARIABLE_NAME_PK }, new Class[] {
-						MessageTypeException.class, IOException.class }, bsb
-						.toString());
-		LOG.trace("unpack method src: " + sb.toString());
-		try {
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(),
-					unpackerCtClass);
-			unpackerCtClass.addMethod(newCtMethod);
-		} catch (CannotCompileException e) {
-			DynamicCodeGenException ex = new DynamicCodeGenException(e
-					.getMessage()
-					+ ": " + sb.toString(), e);
-			LOG.error(ex.getMessage(), ex);
-			throw ex;
-		}
+	private void insertOrdinalEnumUnpackMethodBody(StringBuilder sb,
+			Class type) {
+		// Object unpack(Unpacker u) throws IOException, MessageTypeException;
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		// $1.unpackArray();
+		Object[] args0 = new Object[0];
+		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_02, args0));
+		// int i = $1.unapckInt();
+		Object[] args1 = new Object[0];
+		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_05, args1));
+		// return Foo.class.getEnumConstants()[i];
+		Object[] args2 = new Object[] { classToString(type) };
+		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_06, args2));
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
 	}
 
-	public void addConvertMethod(CtClass tmplCtClass, Class c, Field[] fs) {
-		// Object convert(MessagePackObject from) throws MessageTypeException;
+	public void addConvertMethod(CtClass tmplCtClass, Class type,
+			Field[] fields, boolean isEnum) {
+		// Object convert(MessagePackObject mpo) throws MessageTypeException;
 		StringBuilder sb = new StringBuilder();
-		StringBuilder bsb = new StringBuilder();
-		insertConvertMethodBody(bsb, c, fs);
-		addPublicMethodDecl(sb, METHOD_NAME_CONVERT, Object.class,
-				new Class[] { MessagePackObject.class },
-				new String[] { VARIABLE_NAME_MPO },
-				new Class[] { MessageTypeException.class }, bsb.toString());
-		LOG.trace("convert method src: " + sb.toString());
+		if (!isEnum) {
+			insertConvertMethodBody(sb, type, fields);
+		} else {
+			insertOrdinalEnumConvertMethodBody(sb, type);
+		}
 		try {
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
+			LOG.trace("convert method src: " + sb.toString());
+			int mod = javassist.Modifier.PUBLIC;
+			CtClass returnType = classToCtClass(Object.class);
+			String mname = METHOD_NAME_CONVERT;
+			CtClass[] paramTypes = new CtClass[] { classToCtClass(MessagePackObject.class) };
+			CtClass[] exceptTypes = new CtClass[] { classToCtClass(MessageTypeException.class) };
+			CtMethod newCtMethod = CtNewMethod.make(mod, returnType, mname,
+					paramTypes, exceptTypes, sb.toString(), tmplCtClass);
 			tmplCtClass.addMethod(newCtMethod);
 		} catch (CannotCompileException e) {
 			DynamicCodeGenException ex = new DynamicCodeGenException(e
@@ -627,31 +536,32 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 					+ ": " + sb.toString(), e);
 			LOG.error(ex.getMessage(), ex);
 			throw ex;
+		} catch (NotFoundException e) {
+			DynamicCodeGenException ex = new DynamicCodeGenException(e
+					.getMessage()
+					+ ": " + sb.toString(), e);
+			LOG.error(ex.getMessage(), ex);
+			throw ex;
 		}
 	}
 
-	private void insertConvertMethodBody(StringBuilder sb, Class c,
-			Field[] fs) {
-		insertLocalVariableDecl(sb, c, VARIABLE_NAME_TARGET);
-		StringBuilder mc = new StringBuilder();
-		insertDefaultConsCall(mc, c);
-		insertValueInsertion(sb, mc.toString());
-		insertSemicolon(sb);
-		insertCodeOfMessagePackObjectArrayGet(sb);
-		insertCodeOfConvertMethodCalls(sb, fs);
-		insertReturnStat(sb, VARIABLE_NAME_TARGET);
-		insertSemicolon(sb);
-	}
-
-	private void insertCodeOfMessagePackObjectArrayGet(StringBuilder sb) {
-		// MessagePackObject[] ary = obj.asArray();
-		insertLocalVariableDecl(sb, MessagePackObject.class,
-				VARIABLE_NAME_ARRAY, 1);
-		StringBuilder mc = new StringBuilder();
-		insertMethodCall(mc, VARIABLE_NAME_MPO, METHOD_NAME_ASARRAY,
-				new String[0]);
-		insertValueInsertion(sb, mc.toString());
-		insertSemicolon(sb);
+	private void insertConvertMethodBody(StringBuilder sb, Class type,
+			Field[] fields) {
+		// Object convert(MessagePackObject mpo) throws MessageTypeException;
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		// Foo _$$_t = new Foo();
+		String typeName = classToString(type);
+		Object[] args0 = new Object[] { typeName, typeName };
+		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_01, args0));
+		// MessagePackObject[] _$$_ary = $1.asArray();
+		Object[] args1 = new Object[] { classToString(MessagePackObject[].class) };
+		sb.append(String.format(STATEMENT_PACKER_CONVERTMETHODBODY_01, args1));
+		insertCodeOfConvertMethodCalls(sb, fields);
+		// return _$$_t;
+		Object[] args2 = new Object[0];
+		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_04, args2));
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
 	}
 
 	private void insertCodeOfConvertMethodCalls(StringBuilder sb, Field[] fields) {
@@ -661,46 +571,19 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	}
 
 	private void insertCodeOfConvMethodCall(StringBuilder sb, Field field, int i) {
-		// target.f0 = ((Integer)_$$_tmpls[i].convert(_$$_ary[i])).intValue();
-		Class type = field.getType();
-		insertFieldAccess(sb, VARIABLE_NAME_TARGET, field.getName());
-		String castType = null;
-		String rawValueGetter = null;
-		if (type.isPrimitive()) {
-			castType = "(" + primitiveTypeToWrapperType(type).getName() + ")";
-			rawValueGetter = getPrimTypeValueMethodName(type);
-		} else if (type.isArray()) {
-			castType = "(" + arrayTypeToString(type) + ")";
-		} else {
-			castType = "(" + type.getName() + ")";
-		}
-		StringBuilder mc = new StringBuilder();
-		mc.append(castType);
-		mc.append(VARIABLE_NAME_TEMPLATES);
-		mc.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-		mc.append(i);
-		mc.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		String tname = mc.toString();
-		mc = new StringBuilder();
-		mc.append(VARIABLE_NAME_ARRAY);
-		mc.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-		mc.append(i);
-		mc.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		String aname = mc.toString();
-		mc = new StringBuilder();
-		insertMethodCall(mc, tname, METHOD_NAME_CONVERT, new String[] { aname });
-		if (type.isPrimitive()) {
-			tname = mc.toString();
-			mc = new StringBuilder();
-			mc.append(Constants.CHAR_NAME_LEFT_PARENTHESIS);
-			mc.append(tname);
-			mc.append(Constants.CHAR_NAME_RIGHT_PARENTHESIS);
-			tname = mc.toString();
-			mc = new StringBuilder();
-			insertMethodCall(mc, tname, rawValueGetter, new String[0]);
-		}
-		insertValueInsertion(sb, mc.toString());
-		insertSemicolon(sb);
+		// target.fi = ((Object)_$$_tmpls[i].convert(_$$_ary[i])).intValue();
+		Class returnType = field.getType();
+		boolean isPrim = returnType.isPrimitive();
+		Object[] args = new Object[] {
+				field.getName(),
+				isPrim ? "(" : "",
+				isPrim ? getPrimToWrapperType(returnType).getName()
+						: classToString(returnType),
+				i,
+				i,
+				isPrim ? ")." + getPrimTypeValueMethodName(returnType) + "()"
+						: "" };
+		sb.append(String.format(STATEMENT_PACKER_CONVERTMETHODBODY_02, args));
 	}
 
 	private void insertCodeOfMessageConvertCallForMsgConvtblType(
@@ -752,50 +635,20 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 	}
 
-	public void addConvertMethodForOrdinalEnumTypes(CtClass tmplCtClass,
-			Class c) {
-		// Object convert(MessagePackObject from) throws MessageTypeException;
-		StringBuilder sb = new StringBuilder();
-		StringBuilder bsb = new StringBuilder();
-		insertCodeOfMessagePackObjectArrayGet(bsb);
-		StringBuilder mc = new StringBuilder();
-		insertMethodCall(mc, VARIABLE_NAME_ARRAY + "[0]", METHOD_NAME_ASINT,
-				new String[0]);
-		insertLocalVariableDecl(bsb, int.class, VARIABLE_NAME_I);
-		insertValueInsertion(bsb, mc.toString());
-		insertSemicolon(bsb);
-		mc = new StringBuilder();
-		insertMethodCall(mc, c.getName() + ".class",
-				METHOD_NAME_GETENUMCONSTANTS, new String[0]);
-		mc.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-		mc.append(VARIABLE_NAME_I);
-		mc.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		insertLocalVariableDecl(bsb, Object.class, VARIABLE_NAME_OBJECT);
-		insertValueInsertion(bsb, mc.toString());
-		insertSemicolon(bsb);
-		mc = new StringBuilder();
-		insertTypeCast(mc, c, VARIABLE_NAME_OBJECT);
-		insertReturnStat(bsb, mc.toString());
-		insertSemicolon(bsb);
-		addPublicMethodDecl(sb, METHOD_NAME_CONVERT, Object.class,
-				new Class[] { MessagePackObject.class },
-				new String[] { VARIABLE_NAME_MPO },
-				new Class[] { MessageTypeException.class }, bsb.toString());
-		LOG.trace("convert method src: " + sb.toString());
-		try {
-			CtMethod newCtMethod = CtNewMethod.make(sb.toString(), tmplCtClass);
-			tmplCtClass.addMethod(newCtMethod);
-		} catch (CannotCompileException e) {
-			DynamicCodeGenException ex = new DynamicCodeGenException(e
-					.getMessage()
-					+ ": " + sb.toString(), e);
-			LOG.error(ex.getMessage(), ex);
-			throw ex;
-		}
-	}
-
-	private Class createClass(CtClass packerCtClass)
-			throws CannotCompileException {
-		return packerCtClass.toClass(null, null);
+	private void insertOrdinalEnumConvertMethodBody(StringBuilder sb,
+			Class type) {
+		// Object convert(MessagePackObject mpo) throws MessageTypeException;
+		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+		sb.append(CHAR_NAME_SPACE);
+		// MessagePackObject[] _$$_ary = $1.asArray();
+		Object[] args0 = new Object[] { classToString(MessagePackObject[].class) };
+		sb.append(String.format(STATEMENT_PACKER_CONVERTMETHODBODY_01, args0));
+		// int i = _$$_ary[0].asInt();
+		Object[] args1 = new Object[0];
+		sb.append(String.format(STATEMENT_PACKER_CONVERTMETHODBODY_03, args1));
+		// return Foo.class.getEnumConstants()[i];
+		Object[] args2 = new Object[] { classToString(type) };
+		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_06, args2));
+		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 6fa6abc..4bfea8c 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -1,11 +1,24 @@
 package org.msgpack.util.codegen;
 
+import java.lang.reflect.Field;
 import java.lang.reflect.GenericArrayType;
+import java.lang.reflect.Method;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
 import java.math.BigInteger;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javassist.CannotCompileException;
+import javassist.ClassPool;
+import javassist.CtClass;
+import javassist.CtConstructor;
+import javassist.CtField;
+import javassist.CtMethod;
+import javassist.CtNewConstructor;
+import javassist.CtNewMethod;
+import javassist.NotFoundException;
 
 import org.msgpack.CustomConverter;
 import org.msgpack.CustomMessage;
@@ -18,7 +31,7 @@ import org.msgpack.annotation.MessagePackOrdinalEnum;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class DynamicCodeGenBase implements BasicConstants {
+public class DynamicCodeGenBase implements Constants {
 	public static interface TemplateAccessor {
 		void setTemplates(Template[] templates);
 	}
@@ -34,252 +47,104 @@ public class DynamicCodeGenBase implements BasicConstants {
 	private static Logger LOG = LoggerFactory
 			.getLogger(DynamicCodeGenBase.class);
 
+	private static AtomicInteger COUNTER = new AtomicInteger(0);
+
+	protected static int inc() {
+		return COUNTER.addAndGet(1);
+	}
+
+	protected ClassPool pool;
+
 	public DynamicCodeGenBase() {
+		pool = ClassPool.getDefault();
 	}
 
-	public void addPublicFieldDecl(StringBuilder sb, Class type, String name) {
-		addPublicFieldDecl(sb, type, name, 0);
+	protected void checkTypeValidation(Class type) {
+		DynamicCodeGenException e = new DynamicCodeGenException("Fatal error: "
+				+ type.getName());
+		LOG.error(e.getMessage(), e);
+		throw e;
 	}
 
-	public void addPublicFieldDecl(StringBuilder sb, Class type,
-			String name, int dim) {
-		sb.append(KEYWORD_MODIFIER_PUBLIC);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(type.getName());
-		for (int i = 0; i < dim; ++i) {
-			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		}
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(name);
+	protected void throwTypeValidationException(Class origClass,
+			String message) throws DynamicCodeGenException {
+		DynamicCodeGenException e = new DynamicCodeGenException(message + ": "
+				+ origClass.getName());
+		LOG.error(e.getMessage(), e);
+		throw e;
 	}
 
-	public void addPublicMethodDecl(StringBuilder sb, String mname,
-			Class returnType, Class[] paramTypes, String[] anames,
-			Class[] exceptTypes, String methodBody) {
-		int[] dims = new int[paramTypes.length];
-		for (int i = 0; i < paramTypes.length; ++i) {
-			dims[i] = 0;
-		}
-		addPublicMethodDecl(sb, mname, returnType, paramTypes, dims, anames,
-				exceptTypes, methodBody);
+	protected void checkDefaultConstructorValidation(Class type) {
+		DynamicCodeGenException e = new DynamicCodeGenException("Fatal error: "
+				+ type.getName());
+		LOG.error(e.getMessage(), e);
+		throw e;
 	}
 
-	public void addPublicMethodDecl(StringBuilder sb, String mname,
-			Class returnType, Class[] paramTypes, int[] dims,
-			String[] anames, Class[] exceptTypes, String methodBody) {
-		sb.append(KEYWORD_MODIFIER_PUBLIC);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(returnType.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(mname);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		for (int i = 0; i < paramTypes.length; ++i) {
-			sb.append(paramTypes[i].getName());
-			for (int j = 0; j < dims[i]; ++j) {
-				sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-				sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-			}
-			sb.append(CHAR_NAME_SPACE);
-			sb.append(anames[i]);
-			if (i + 1 != paramTypes.length) {
-				sb.append(CHAR_NAME_COMMA);
-				sb.append(CHAR_NAME_SPACE);
-			}
-		}
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SPACE);
-		if (exceptTypes.length != 0) {
-			sb.append(KEYWORD_THROWS);
-			sb.append(CHAR_NAME_SPACE);
-			for (int i = 0; i < exceptTypes.length; ++i) {
-				sb.append(exceptTypes[i].getName());
-				if (i + 1 != exceptTypes.length) {
-					sb.append(CHAR_NAME_COMMA);
-					sb.append(CHAR_NAME_SPACE);
-				}
-			}
-			sb.append(CHAR_NAME_SPACE);
-		}
-		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(methodBody);
-		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
+	protected void throwConstructorValidationException(Class origClass) {
+		DynamicCodeGenException e = new DynamicCodeGenException(
+				"it must have a public zero-argument constructor: "
+						+ origClass.getName());
+		LOG.error(e.getMessage(), e);
+		throw e;
 	}
 
-	public void insertSemicolon(StringBuilder sb) {
-		// ;
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
+	protected void throwFieldValidationException(Field f) {
+		DynamicCodeGenException e = new DynamicCodeGenException(
+				"it must be a public field: " + f.getName());
+		LOG.debug(e.getMessage(), e);
+		throw e;
 	}
 
-	public void insertLocalVariableDecl(StringBuilder sb, Class type,
-			String name) {
-		// int lv
-		insertLocalVariableDecl(sb, type, name, 0);
+	protected static void throwMethodValidationException(Method method,
+			String message) throws DynamicCodeGenException {
+		DynamicCodeGenException e = new DynamicCodeGenException(message + ": "
+				+ method.getName());
+		LOG.error(e.getMessage(), e);
+		throw e;
 	}
 
-	public void insertLocalVariableDecl(StringBuilder sb, Class type,
-			String name, int dim) {
-		// int[] lv
-		int dim0 = dim + getArrayDim(type);
-		Class type0 = getArrayBaseType(type);
-		sb.append(type0.getName());
-		for (int i = 0; i < dim0; ++i) {
-			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		}
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(name);
+	protected CtClass makeClass(String name) throws NotFoundException {
+		DynamicCodeGenException e = new DynamicCodeGenException("Fatal error: "
+				+ name);
+		LOG.error(e.getMessage(), e);
+		throw e;
 	}
 
-	protected int getArrayDim(Class type) {
-		if (type.isArray()) {
-			return 1 + getArrayDim(type.getComponentType());
-		} else {
-			return 0;
-		}
+	protected void setInterface(CtClass packerCtClass, Class infClass)
+			throws NotFoundException {
+		CtClass infCtClass = pool.get(infClass.getName());
+		packerCtClass.addInterface(infCtClass);
 	}
 
-	protected Class getArrayBaseType(Class type) {
-		if (type.isArray()) {
-			return getArrayBaseType(type.getComponentType());
-		} else {
-			return type;
-		}
-	}
-	
-	protected String arrayTypeToString(Class type) {
-		StringBuilder sb = new StringBuilder();
-		int dim = getArrayDim(type);
-		Class t = getArrayBaseType(type);
-		sb.append(t.getName());
-		for (int i = 0; i < dim; ++i) {
-			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		}
-		return sb.toString();
+	protected void addDefaultConstructor(CtClass enhancedCtClass)
+			throws CannotCompileException {
+		CtConstructor newCtCons = CtNewConstructor
+				.defaultConstructor(enhancedCtClass);
+		enhancedCtClass.addConstructor(newCtCons);
 	}
 
-	public void insertValueInsertion(StringBuilder sb, String expr) {
-		// = expr
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(expr);
+	protected void addTemplateArrayField(CtClass newCtClass)
+			throws NotFoundException, CannotCompileException {
+		CtClass acsCtClass = pool.get(TemplateAccessorImpl.class.getName());
+		CtField tmplsField = acsCtClass
+				.getDeclaredField(VARIABLE_NAME_TEMPLATES);
+		CtField tmplsField2 = new CtField(tmplsField.getType(), tmplsField
+				.getName(), newCtClass);
+		newCtClass.addField(tmplsField2);
 	}
 
-	public void insertInsertion(StringBuilder sb) {
-		// =
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
+	protected void addSetTemplatesMethod(CtClass newCtClass)
+			throws NotFoundException, CannotCompileException {
+		CtClass acsCtClass = pool.get(TemplateAccessorImpl.class.getName());
+		CtMethod settmplsMethod = acsCtClass
+				.getDeclaredMethod(METHOD_NAME_SETTEMPLATES);
+		CtMethod settmplsMethod2 = CtNewMethod.copy(settmplsMethod, newCtClass,
+				null);
+		newCtClass.addMethod(settmplsMethod2);
 	}
 
-	public void insertFieldAccess(StringBuilder sb, String target, String field) {
-		// target.field
-		sb.append(target);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(field);
-	}
-
-	public void insertDefaultConsCall(StringBuilder sb, Class type) {
-		// new tname()
-		insertConsCall(sb, type, null);
-	}
-
-	public void insertConsCall(StringBuilder sb, Class type, String expr) {
-		// new tname(expr)
-		sb.append(KEYWORD_NEW);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(type.getName());
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		if (expr != null) {
-			sb.append(expr);
-		}
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-	}
-
-	public void insertMethodCall(StringBuilder sb, String tname, String mname,
-			String[] anames) {
-		// tname.mname(anames[0], anames[1], ...)
-		int len = anames.length;
-		sb.append(tname);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(mname);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		for (int i = 0; i < len; ++i) {
-			sb.append(anames[i]);
-			if (i + 1 != len) {
-				sb.append(CHAR_NAME_COMMA);
-				sb.append(CHAR_NAME_SPACE);
-			}
-		}
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-	}
-
-	public void insertTypeCast(StringBuilder sb, Class type) {
-		// (type)
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(type.getName());
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-	}
-
-	public void insertTypeCast(StringBuilder sb, Class type, String varName) {
-		// ((type)var)
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(type.getName());
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(varName);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-	}
-
-	public void insertReturnStat(StringBuilder sb, String expr) {
-		// return expr
-		sb.append(KEYWORD_RETURN);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(expr);
-	}
-
-	public void insertTypeConvToObjectType(StringBuilder sb, Class type,
-			String expr) throws DynamicCodeGenException {
-		if (type.isPrimitive()) { // primitive type
-			insertConsCall(sb, primitiveTypeToWrapperType(type), expr);
-		} else { // reference type
-			sb.append(expr);
-		}
-	}
-
-	public void insertTryCatchBlocks(StringBuilder sb, String tryBody,
-			List> types, List names, List catchBodies) {
-		int len = types.size();
-		sb.append(KEYWORD_TRY);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(tryBody);
-		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-		for (int i = 0; i < len; ++i) {
-			sb.append(KEYWORD_CATCH);
-			sb.append(CHAR_NAME_SPACE);
-			sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-			sb.append(types.get(i).getName());
-			sb.append(CHAR_NAME_SPACE);
-			sb.append(names.get(i));
-			sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-			sb.append(CHAR_NAME_SPACE);
-			sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-			sb.append(CHAR_NAME_SPACE);
-			sb.append(catchBodies.get(i));
-			sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-			sb.append(CHAR_NAME_SPACE);
-		}
-	}
-	
-	public Class primitiveTypeToWrapperType(Class type) {
+	protected Class getPrimToWrapperType(Class type) {
 		if (type.equals(boolean.class)) {
 			return Boolean.class;
 		} else if (type.equals(byte.class)) {
@@ -298,7 +163,7 @@ public class DynamicCodeGenBase implements BasicConstants {
 			throw new MessageTypeException("Type error: " + type.getName());
 		}
 	}
-	
+
 	public String getPrimTypeValueMethodName(Class type) {
 		if (type.equals(boolean.class)) {
 			return METHOD_NAME_BOOLEANVALUE;
@@ -318,6 +183,7 @@ public class DynamicCodeGenBase implements BasicConstants {
 			throw new MessageTypeException("Type error: " + type.getName());
 		}
 	}
+
 	public String getUnpackMethodName(Class c)
 			throws DynamicCodeGenException {
 		if (c.equals(boolean.class) || c.equals(Boolean.class)) {
@@ -404,7 +270,7 @@ public class DynamicCodeGenBase implements BasicConstants {
 				return (Template) CustomConverter.get(c);
 			} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
 				// @MessagePackMessage
-				Template tmpl = DynamicCodeGenTemplate.create(c);
+				Template tmpl = DynamicTemplate.create(c);
 				CustomMessage.registerTemplate(c, tmpl);
 				return tmpl;
 			} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
@@ -416,7 +282,7 @@ public class DynamicCodeGenBase implements BasicConstants {
 			} else if (CustomMessage.isAnnotated(c,
 					MessagePackOrdinalEnum.class)) {
 				// @MessagePackOrdinalEnum
-				Template tmpl = DynamicCodeGenOrdinalEnumTemplate.create(c);
+				Template tmpl = DynamicOrdinalEnumTemplate.create(c);
 				CustomMessage.registerTemplate(c, tmpl);
 				return tmpl;
 			} else {
@@ -450,4 +316,75 @@ public class DynamicCodeGenBase implements BasicConstants {
 					+ t.getClass().getName());
 		}
 	}
+
+	protected int getArrayDim(Class type) {
+		if (type.isArray()) {
+			return 1 + getArrayDim(type.getComponentType());
+		} else {
+			return 0;
+		}
+	}
+
+	protected Class getArrayBaseType(Class type) {
+		if (type.isArray()) {
+			return getArrayBaseType(type.getComponentType());
+		} else {
+			return type;
+		}
+	}
+
+	protected String arrayTypeToString(Class type) {
+		StringBuilder sb = new StringBuilder();
+		int dim = getArrayDim(type);
+		Class t = getArrayBaseType(type);
+		sb.append(t.getName());
+		for (int i = 0; i < dim; ++i) {
+			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
+			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+		}
+		return sb.toString();
+	}
+
+	protected String classToString(Class type) {
+		if (type.isArray()) {
+			return arrayTypeToString(type);
+		} else {
+			return type.getName();
+		}
+	}
+
+	protected CtClass classToCtClass(Class type) throws NotFoundException {
+		if (type.equals(void.class)) {
+			return CtClass.voidType;
+		} else if (type.isPrimitive()) {
+			if (type.equals(boolean.class)) {
+				return CtClass.booleanType;
+			} else if (type.equals(byte.class)) {
+				return CtClass.byteType;
+			} else if (type.equals(char.class)) {
+				return CtClass.charType;
+			} else if (type.equals(short.class)) {
+				return CtClass.shortType;
+			} else if (type.equals(int.class)) {
+				return CtClass.intType;
+			} else if (type.equals(long.class)) {
+				return CtClass.longType;
+			} else if (type.equals(float.class)) {
+				return CtClass.floatType;
+			} else if (type.equals(double.class)) {
+				return CtClass.doubleType;
+			} else {
+				throw new MessageTypeException("Fatal error: " + type.getName());
+			}
+		} else if (type.isArray()) {
+			return pool.get(arrayTypeToString(type));
+		} else {
+			return pool.get(type.getName());
+		}
+	}
+
+	protected Class createClass(CtClass newCtClass)
+			throws CannotCompileException {
+		return newCtClass.toClass(null, null);
+	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenConverter.java b/java/src/main/java/org/msgpack/util/codegen/DynamicConverter.java
similarity index 61%
rename from java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenConverter.java
rename to java/src/main/java/org/msgpack/util/codegen/DynamicConverter.java
index 4e03714..ab599f1 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenConverter.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicConverter.java
@@ -2,8 +2,8 @@ package org.msgpack.util.codegen;
 
 import org.msgpack.MessageConverter;
 
-public class DynamicCodeGenConverter {
+public class DynamicConverter {
 	public static MessageConverter create(Class c) {
-		return DynamicCodeGenTemplate.create(c);
+		return DynamicTemplate.create(c);
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumConverter.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumConverter.java
similarity index 91%
rename from java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumConverter.java
rename to java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumConverter.java
index e06a5bb..648b4f8 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumConverter.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumConverter.java
@@ -2,7 +2,7 @@ package org.msgpack.util.codegen;
 
 import org.msgpack.MessageConverter;
 
-public class DynamicCodeGenOrdinalEnumConverter {
+public class DynamicOrdinalEnumConverter {
 	public static MessageConverter create(Class c) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
similarity index 91%
rename from java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumPacker.java
rename to java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
index 01da86a..98f4a39 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
@@ -2,7 +2,7 @@ package org.msgpack.util.codegen;
 
 import org.msgpack.MessagePacker;
 
-public class DynamicCodeGenOrdinalEnumPacker {
+public class DynamicOrdinalEnumPacker {
 	public static MessagePacker create(Class c) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumTemplate.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumTemplate.java
similarity index 90%
rename from java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumTemplate.java
rename to java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumTemplate.java
index 5b53d8e..c234683 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumTemplate.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumTemplate.java
@@ -2,7 +2,7 @@ package org.msgpack.util.codegen;
 
 import org.msgpack.Template;
 
-public class DynamicCodeGenOrdinalEnumTemplate {
+public class DynamicOrdinalEnumTemplate {
 	public static Template create(Class c) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumUnpacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumUnpacker.java
similarity index 91%
rename from java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumUnpacker.java
rename to java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumUnpacker.java
index 8f290bd..edad9a8 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenOrdinalEnumUnpacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumUnpacker.java
@@ -2,7 +2,7 @@ package org.msgpack.util.codegen;
 
 import org.msgpack.MessageUnpacker;
 
-public class DynamicCodeGenOrdinalEnumUnpacker {
+public class DynamicOrdinalEnumUnpacker {
 	public static MessageUnpacker create(Class c) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
similarity index 93%
rename from java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenPacker.java
rename to java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
index 4bd70f4..643ff52 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
@@ -2,7 +2,7 @@ package org.msgpack.util.codegen;
 
 import org.msgpack.MessagePacker;
 
-public class DynamicCodeGenPacker {
+public class DynamicPacker {
 
 	public static MessagePacker create(Class c) {
 		try {
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java b/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
similarity index 94%
rename from java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java
rename to java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
index 2a9e584..60a51e8 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenTemplate.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
@@ -3,7 +3,7 @@ package org.msgpack.util.codegen;
 import org.msgpack.Template;
 import org.msgpack.util.codegen.DynamicCodeGenBase.TemplateAccessor;
 
-public class DynamicCodeGenTemplate {
+public class DynamicTemplate {
 	public static Template create(Class c) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenUnpacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicUnpacker.java
similarity index 61%
rename from java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenUnpacker.java
rename to java/src/main/java/org/msgpack/util/codegen/DynamicUnpacker.java
index 3cb0fce..83590ce 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenUnpacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicUnpacker.java
@@ -2,8 +2,8 @@ package org.msgpack.util.codegen;
 
 import org.msgpack.MessageUnpacker;
 
-public class DynamicCodeGenUnpacker {
+public class DynamicUnpacker {
 	public static MessageUnpacker create(Class c) {
-		return DynamicCodeGenTemplate.create(c);
+		return DynamicTemplate.create(c);
 	}
 }
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
index 5514c27..436ea27 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
@@ -36,11 +36,11 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		src.f5 = 5;
 		src.f6 = false;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(PrimitiveTypeFieldsClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate
+		Template tmpl = DynamicTemplate
 				.create(PrimitiveTypeFieldsClass.class);
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
@@ -85,11 +85,11 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		src.f8 = "8";
 		src.f9 = new byte[] { 0x01, 0x02 };
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(GeneralReferenceTypeFieldsClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate
+		Template tmpl = DynamicTemplate
 				.create(GeneralReferenceTypeFieldsClass.class);
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
@@ -146,11 +146,11 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		slnt.f1 = "muga";
 		src.f4.add(slnt);
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(SampleListTypes.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate.create(SampleListTypes.class);
+		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -217,11 +217,11 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		src.f2.put("k2", 2);
 		src.f2.put("k3", 3);
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(SampleMapTypes.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate.create(SampleMapTypes.class);
+		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -261,28 +261,28 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	@Test
 	public void testDefaultConstructorModifiers01() throws Exception {
 		try {
-			DynamicCodeGenPacker.create(NoDefaultConstructorClass.class);
+			DynamicPacker.create(NoDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(PrivateDefaultConstructorClass.class);
+			DynamicPacker.create(PrivateDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(ProtectedDefaultConstructorClass.class);
+			DynamicPacker.create(ProtectedDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(PackageDefaultConstructorClass.class);
+			DynamicPacker.create(PackageDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -293,21 +293,21 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	@Test
 	public void testDefaultConstructorModifiers02() throws Exception {
 		try {
-			DynamicCodeGenTemplate.create(NoDefaultConstructorClass.class);
+			DynamicTemplate.create(NoDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenTemplate.create(PrivateDefaultConstructorClass.class);
+			DynamicTemplate.create(PrivateDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenTemplate
+			DynamicTemplate
 					.create(ProtectedDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
@@ -315,7 +315,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenTemplate.create(PackageDefaultConstructorClass.class);
+			DynamicTemplate.create(PackageDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -346,21 +346,21 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	@Test
 	public void testClassModifiers01() throws Exception {
 		try {
-			DynamicCodeGenPacker.create(PrivateModifierClass.class);
+			DynamicPacker.create(PrivateModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(ProtectedModifierClass.class);
+			DynamicPacker.create(ProtectedModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(PackageModifierClass.class);
+			DynamicPacker.create(PackageModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -371,21 +371,21 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	@Test
 	public void testClassModifiers02() throws Exception {
 		try {
-			DynamicCodeGenTemplate.create(PrivateModifierClass.class);
+			DynamicTemplate.create(PrivateModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenTemplate.create(ProtectedModifierClass.class);
+			DynamicTemplate.create(ProtectedModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenTemplate.create(PackageModifierClass.class);
+			DynamicTemplate.create(PackageModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -407,14 +407,14 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	@Test
 	public void testFinalClassAndAbstractClass01() throws Exception {
 		try {
-			DynamicCodeGenPacker.create(FinalModifierClass.class);
+			DynamicPacker.create(FinalModifierClass.class);
 			assertTrue(true);
 		} catch (DynamicCodeGenException e) {
 			fail();
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(AbstractModifierClass.class);
+			DynamicPacker.create(AbstractModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -425,14 +425,14 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	@Test
 	public void testFinalClassAndAbstractClass02() throws Exception {
 		try {
-			DynamicCodeGenTemplate.create(FinalModifierClass.class);
+			DynamicTemplate.create(FinalModifierClass.class);
 			assertTrue(true);
 		} catch (DynamicCodeGenException e) {
 			fail();
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenTemplate.create(AbstractModifierClass.class);
+			DynamicTemplate.create(AbstractModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -449,14 +449,14 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	@Test
 	public void testInterfaceAndEnumType01() throws Exception {
 		try {
-			DynamicCodeGenPacker.create(SampleInterface.class);
+			DynamicPacker.create(SampleInterface.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(SampleEnum.class);
+			DynamicPacker.create(SampleEnum.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -467,7 +467,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	@Test
 	public void testInterfaceType() throws Exception {
 		try {
-			DynamicCodeGenTemplate.create(SampleInterface.class);
+			DynamicTemplate.create(SampleInterface.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -484,11 +484,11 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		src.f0 = 0;
 		src.f1 = SampleEnum.ONE;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(SampleEnumFieldClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate
+		Template tmpl = DynamicTemplate
 				.create(SampleEnumFieldClass.class);
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
@@ -521,11 +521,11 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		src.f3 = 3;
 		src.f4 = 4;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(FieldModifiersClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate
+		Template tmpl = DynamicTemplate
 				.create(FieldModifiersClass.class);
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
@@ -553,14 +553,14 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 
 	@Test
 	public void testNestedFieldClass() throws Exception {
-		MessagePacker packer2 = DynamicCodeGenPacker.create(NestedClass.class);
+		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
 		CustomPacker.register(NestedClass.class, packer2);
-		MessagePacker packer1 = DynamicCodeGenPacker.create(BaseClass.class);
+		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
 		CustomPacker.register(BaseClass.class, packer1);
-		Template tmpl2 = DynamicCodeGenTemplate.create(NestedClass.class);
+		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
 		CustomUnpacker.register(NestedClass.class, tmpl2);
 		CustomConverter.register(NestedClass.class, tmpl2);
-		Template tmpl1 = DynamicCodeGenTemplate.create(BaseClass.class);
+		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
 		CustomUnpacker.register(BaseClass.class, tmpl1);
 		CustomConverter.register(BaseClass.class, tmpl1);
 		BaseClass src = new BaseClass();
@@ -604,14 +604,14 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		src2.f2 = 2;
 		src.f1 = src2;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker.create(BaseClass2.class);
+		MessagePacker packer = DynamicPacker.create(BaseClass2.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
 		MessagePackObject mpo = it.next();
-		Template tmpl = DynamicCodeGenTemplate.create(BaseClass2.class);
+		Template tmpl = DynamicTemplate.create(BaseClass2.class);
 		BaseClass2 dst = (BaseClass2) tmpl.convert(mpo);
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1.f2 == dst.f1.f2);
@@ -645,11 +645,11 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		src.f8 = 8;
 		src.f9 = 9;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(SampleSubClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate.create(SampleSubClass.class);
+		Template tmpl = DynamicTemplate.create(SampleSubClass.class);
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
index 7876c7f..f46d6a0 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
@@ -35,11 +35,11 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		src.f5 = 5;
 		src.f6 = false;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(PrimitiveTypeFieldsClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate
+		Template tmpl = DynamicTemplate
 				.create(PrimitiveTypeFieldsClass.class);
 		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
 				.unpack(new Unpacker(in));
@@ -79,11 +79,11 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		src.f8 = "8";
 		src.f9 = new byte[] { 0x01, 0x02 };
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(GeneralReferenceTypeFieldsClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate
+		Template tmpl = DynamicTemplate
 				.create(GeneralReferenceTypeFieldsClass.class);
 		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
 				.unpack(new Unpacker(in));
@@ -135,11 +135,11 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		slnt.f1 = "muga";
 		src.f4.add(slnt);
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(SampleListTypes.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate.create(SampleListTypes.class);
+		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
 		SampleListTypes dst = (SampleListTypes) tmpl.unpack(new Unpacker(in));
 		assertEquals(src.f0.size(), dst.f0.size());
 		assertEquals(src.f1.size(), dst.f1.size());
@@ -201,11 +201,11 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		src.f2.put("k2", 2);
 		src.f2.put("k3", 3);
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(SampleMapTypes.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate.create(SampleMapTypes.class);
+		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
 		SampleMapTypes dst = (SampleMapTypes) tmpl.unpack(new Unpacker(in));
 		assertEquals(src.f0.size(), dst.f0.size());
 		assertEquals(src.f1.size(), dst.f1.size());
@@ -240,28 +240,28 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	@Test
 	public void testDefaultConstructorModifiers01() throws Exception {
 		try {
-			DynamicCodeGenPacker.create(NoDefaultConstructorClass.class);
+			DynamicPacker.create(NoDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(PrivateDefaultConstructorClass.class);
+			DynamicPacker.create(PrivateDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(ProtectedDefaultConstructorClass.class);
+			DynamicPacker.create(ProtectedDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(PackageDefaultConstructorClass.class);
+			DynamicPacker.create(PackageDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -272,21 +272,21 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	@Test
 	public void testDefaultConstructorModifiers02() throws Exception {
 		try {
-			DynamicCodeGenUnpacker.create(NoDefaultConstructorClass.class);
+			DynamicUnpacker.create(NoDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenUnpacker.create(PrivateDefaultConstructorClass.class);
+			DynamicUnpacker.create(PrivateDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenUnpacker
+			DynamicUnpacker
 					.create(ProtectedDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
@@ -294,7 +294,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenUnpacker.create(PackageDefaultConstructorClass.class);
+			DynamicUnpacker.create(PackageDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -325,21 +325,21 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	@Test
 	public void testClassModifiers01() throws Exception {
 		try {
-			DynamicCodeGenPacker.create(PrivateModifierClass.class);
+			DynamicPacker.create(PrivateModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(ProtectedModifierClass.class);
+			DynamicPacker.create(ProtectedModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(PackageModifierClass.class);
+			DynamicPacker.create(PackageModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -350,21 +350,21 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	@Test
 	public void testClassModifiers02() throws Exception {
 		try {
-			DynamicCodeGenUnpacker.create(PrivateModifierClass.class);
+			DynamicUnpacker.create(PrivateModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenUnpacker.create(ProtectedModifierClass.class);
+			DynamicUnpacker.create(ProtectedModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenUnpacker.create(PackageModifierClass.class);
+			DynamicUnpacker.create(PackageModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -386,14 +386,14 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	@Test
 	public void testFinalClassAndAbstractClass01() throws Exception {
 		try {
-			DynamicCodeGenPacker.create(FinalModifierClass.class);
+			DynamicPacker.create(FinalModifierClass.class);
 			assertTrue(true);
 		} catch (DynamicCodeGenException e) {
 			fail();
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenPacker.create(AbstractModifierClass.class);
+			DynamicPacker.create(AbstractModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -404,14 +404,14 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	@Test
 	public void testFinalClassAndAbstractClass02() throws Exception {
 		try {
-			DynamicCodeGenUnpacker.create(FinalModifierClass.class);
+			DynamicUnpacker.create(FinalModifierClass.class);
 			assertTrue(true);
 		} catch (DynamicCodeGenException e) {
 			fail();
 		}
 		assertTrue(true);
 		try {
-			DynamicCodeGenUnpacker.create(AbstractModifierClass.class);
+			DynamicUnpacker.create(AbstractModifierClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -428,7 +428,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	@Test
 	public void testInterfaceType01() throws Exception {
 		try {
-			DynamicCodeGenPacker.create(SampleInterface.class);
+			DynamicPacker.create(SampleInterface.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -439,7 +439,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	@Test
 	public void testInterfaceType02() throws Exception {
 		try {
-			DynamicCodeGenUnpacker.create(SampleInterface.class);
+			DynamicUnpacker.create(SampleInterface.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -456,11 +456,11 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		src.f0 = 0;
 		src.f1 = SampleEnum.ONE;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(SampleEnumFieldClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate
+		Template tmpl = DynamicTemplate
 				.create(SampleEnumFieldClass.class);
 		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl
 				.unpack(new Unpacker(in));
@@ -490,11 +490,11 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		src.f3 = 3;
 		src.f4 = 4;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(FieldModifiersClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate
+		Template tmpl = DynamicTemplate
 				.create(FieldModifiersClass.class);
 		FieldModifiersClass dst = (FieldModifiersClass) tmpl
 				.unpack(new Unpacker(in));
@@ -518,14 +518,14 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 
 	@Test
 	public void testNestedFieldClass() throws Exception {
-		MessagePacker packer2 = DynamicCodeGenPacker.create(NestedClass.class);
+		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
 		CustomPacker.register(NestedClass.class, packer2);
-		MessagePacker packer1 = DynamicCodeGenPacker.create(BaseClass.class);
+		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
 		CustomPacker.register(BaseClass.class, packer1);
-		Template tmpl2 = DynamicCodeGenTemplate.create(NestedClass.class);
+		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
 		CustomUnpacker.register(NestedClass.class, tmpl2);
 		CustomConverter.register(NestedClass.class, tmpl2);
-		Template tmpl1 = DynamicCodeGenTemplate.create(BaseClass.class);
+		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
 		CustomUnpacker.register(BaseClass.class, tmpl1);
 		CustomConverter.register(BaseClass.class, tmpl1);
 		BaseClass src = new BaseClass();
@@ -564,10 +564,10 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		src2.f2 = 2;
 		src.f1 = src2;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker.create(BaseClass2.class);
+		MessagePacker packer = DynamicPacker.create(BaseClass2.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate.create(BaseClass2.class);
+		Template tmpl = DynamicTemplate.create(BaseClass2.class);
 		BaseClass2 dst = (BaseClass2) tmpl.unpack(new Unpacker(in));
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1.f2 == dst.f1.f2);
@@ -600,11 +600,11 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		src.f8 = 8;
 		src.f9 = 9;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicCodeGenPacker
+		MessagePacker packer = DynamicPacker
 				.create(SampleSubClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicCodeGenTemplate.create(SampleSubClass.class);
+		Template tmpl = DynamicTemplate.create(SampleSubClass.class);
 		SampleSubClass dst = (SampleSubClass) tmpl.unpack(new Unpacker(in));
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1 == dst.f1);

From c10eb2c17b46be4b8ce9e1014757f17a5d4a23b7 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Mon, 11 Oct 2010 01:30:42 +0900
Subject: [PATCH 0858/1648] java: add the code for checking a null pointer to
 DynamicCodeGen.java

---
 .../org/msgpack/util/codegen/Constants.java   |  24 +-
 .../msgpack/util/codegen/DynamicCodeGen.java  |  51 ++-
 .../util/codegen/DynamicCodeGenBase.java      |  65 ++-
 .../TestDynamicCodeGenPackerConverter.java    | 370 ++++++++++++++++--
 .../TestDynamicCodeGenPackerUnpacker.java     | 314 +++++++++++++--
 5 files changed, 730 insertions(+), 94 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/Constants.java b/java/src/main/java/org/msgpack/util/codegen/Constants.java
index e61f039..e32e6ad 100644
--- a/java/src/main/java/org/msgpack/util/codegen/Constants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/Constants.java
@@ -207,21 +207,27 @@ public interface Constants {
 
 	String STATEMENT_PACKER_PACKERMETHODBODY_04 = "$1.pack(((java.lang.Enum)_$$_t).ordinal()); ";
 
-	String STATEMENT_PACKER_UNPACKERMETHODBODY_01 = "%s _$$_t = new %s(); ";
+	String STATEMENT_PACKER_PACKERMETHODBODY_05 = "if (_$$_nullCheck && $2 == null) { $1.packNil(); return; } ";
 
-	String STATEMENT_PACKER_UNPACKERMETHODBODY_02 = "$1.unpackArray(); ";
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_01 = "%s _$$_t = new %s(); ";
 
-	String STATEMENT_PACKER_UNPACKERMETHODBODY_03 = "_$$_t.%s = %s(%s)_$$_templates[%d].unpack($1)%s; ";
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_02 = "$1.unpackArray(); ";
 
-	String STATEMENT_PACKER_UNPACKERMETHODBODY_04 = "return _$$_t; ";
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_03 = "if (!_$$_nullCheck || !$1.tryUnpackNull()) { _$$_t.%s = %s(%s)_$$_templates[%d].unpack($1)%s; } ";
 
-	String STATEMENT_PACKER_UNPACKERMETHODBODY_05 = "int i = $1.unpackInt(); ";
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_04 = "return _$$_t; ";
 
-	String STATEMENT_PACKER_UNPACKERMETHODBODY_06 = "return %s.class.getEnumConstants()[i]; ";
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_05 = "int i = $1.unpackInt(); ";
 
-	String STATEMENT_PACKER_CONVERTMETHODBODY_01 = "%s _$$_ary = $1.asArray(); ";
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_06 = "return %s.class.getEnumConstants()[i]; ";
 
-	String STATEMENT_PACKER_CONVERTMETHODBODY_02 = "_$$_t.%s = %s(%s)_$$_templates[%d].convert(_$$_ary[%d])%s; ";
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_07 = "if (_$$_nullCheck && $1.tryUnpackNull()) { return null; } ";
 
-	String STATEMENT_PACKER_CONVERTMETHODBODY_03 = "int i = _$$_ary[0].asInt(); ";
+	String STATEMENT_TMPL_CONVERTMETHODBODY_01 = "%s _$$_ary = $1.asArray(); ";
+
+	String STATEMENT_TMPL_CONVERTMETHODBODY_02 = "if (!_$$_nullCheck || !_$$_ary[%d].isNil()) { _$$_t.%s = %s(%s)_$$_templates[%d].convert(_$$_ary[%d])%s; } ";
+
+	String STATEMENT_TMPL_CONVERTMETHODBODY_03 = "int i = _$$_ary[0].asInt(); ";
+
+	String STATEMENT_TMPL_CONVERTMETHODBODY_04 = "if (_$$_nullCheck && $1.isNil()) { return null; } ";
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 3954f17..9ba7ec6 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -72,6 +72,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			checkTypeValidation(origClass);
 			checkDefaultConstructorValidation(origClass);
 			CtClass packerCtClass = pool.makeClass(packerName);
+			setSuperclass(packerCtClass, NullCheckerImpl.class);
 			setInterface(packerCtClass, MessagePacker.class);
 			addDefaultConstructor(packerCtClass);
 			Field[] fields = getDeclaredFields(origClass);
@@ -96,6 +97,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			String packerName = origName + POSTFIX_TYPE_NAME_PACKER + inc();
 			checkTypeValidation(origClass);
 			CtClass packerCtClass = pool.makeClass(packerName);
+			setSuperclass(packerCtClass, NullCheckerImpl.class);
 			setInterface(packerCtClass, MessagePacker.class);
 			addDefaultConstructor(packerCtClass);
 			addPackMethod(packerCtClass, origClass, null, true);
@@ -120,14 +122,12 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			checkTypeValidation(origClass);
 			checkDefaultConstructorValidation(origClass);
 			CtClass tmplCtClass = pool.makeClass(tmplName);
+			setSuperclass(tmplCtClass, TemplateTemplate.class);
 			setInterface(tmplCtClass, Template.class);
-			setInterface(tmplCtClass, DynamicCodeGenBase.TemplateAccessor.class);
 			addDefaultConstructor(tmplCtClass);
 			Field[] fields = getDeclaredFields(origClass);
 			Template[] tmpls = createTemplates(fields);
 			setTemplates(origClass, tmpls);
-			addTemplateArrayField(tmplCtClass);
-			addSetTemplatesMethod(tmplCtClass);
 			addUnpackMethod(tmplCtClass, origClass, fields, false);
 			addConvertMethod(tmplCtClass, origClass, fields, false);
 			Class tmplClass = createClass(tmplCtClass);
@@ -154,11 +154,11 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			checkTypeValidation(origClass);
 			String tmplName = origName + POSTFIX_TYPE_NAME_TEMPLATE + inc();
 			CtClass tmplCtClass = pool.makeClass(tmplName);
+			setSuperclass(tmplCtClass, TemplateTemplate.class);
 			setInterface(tmplCtClass, Template.class);
 			addDefaultConstructor(tmplCtClass);
 			addUnpackMethod(tmplCtClass, origClass, null, true);
 			addConvertMethod(tmplCtClass, origClass, null, true);
-			// addConvertMethodForOrdinalEnumTypes(tmplCtClass, origClass);
 			Class tmplClass = createClass(tmplCtClass);
 			LOG.debug("generated an enum template class for "
 					+ origClass.getName());
@@ -218,7 +218,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 					checkFieldValidation(field, allFields);
 					allFields.add(field);
 				} catch (DynamicCodeGenException e) { // ignore
-					LOG.error(e.getMessage(), e);
+					LOG.trace(e.getMessage(), e);
 				}
 			}
 			nextClass = nextClass.getSuperclass();
@@ -299,6 +299,8 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// void pack(Packer packer, Object target) throws IOException;
 		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
+		Object[] args2 = new Object[0];
+		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_05, args2));
 		String typeName = classToString(type);
 		Object[] args0 = new Object[] { typeName, typeName };
 		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_01, args0));
@@ -357,6 +359,8 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// void pack(Packer packer, Object target) throws IOException;
 		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
+		Object[] args3 = new Object[0];
+		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_05, args3));
 		String typeName = classToString(c);
 		Object[] args0 = new Object[] { typeName, typeName };
 		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_01, args0));
@@ -410,15 +414,17 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 		// Foo _$$_t = new Foo();
 		String typeName = classToString(type);
+		Object[] args3 = new Object[0];
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_07, args3));
 		Object[] args0 = new Object[] { typeName, typeName };
-		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_01, args0));
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_01, args0));
 		// $1.unpackArray();
 		Object[] args1 = new Object[0];
-		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_02, args1));
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_02, args1));
 		insertCodeOfUnpackMethodCalls(sb, fields);
 		// return _$$_t;
 		Object[] args2 = new Object[0];
-		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_04, args2));
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_04, args2));
 		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
 	}
 
@@ -441,7 +447,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				i,
 				isPrim ? ")." + getPrimTypeValueMethodName(returnType) + "()"
 						: "" };
-		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_03, args));
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_03, args));
 	}
 
 	private void insertCodeOfUnpackMethodCallForMsgUnpackableType(
@@ -499,15 +505,17 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// Object unpack(Unpacker u) throws IOException, MessageTypeException;
 		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
+		Object[] args3 = new Object[0];
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_07, args3));
 		// $1.unpackArray();
 		Object[] args0 = new Object[0];
-		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_02, args0));
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_02, args0));
 		// int i = $1.unapckInt();
 		Object[] args1 = new Object[0];
-		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_05, args1));
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_05, args1));
 		// return Foo.class.getEnumConstants()[i];
 		Object[] args2 = new Object[] { classToString(type) };
-		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_06, args2));
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_06, args2));
 		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
 	}
 
@@ -550,17 +558,19 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// Object convert(MessagePackObject mpo) throws MessageTypeException;
 		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
+		Object[] args3 = new Object[0];
+		sb.append(String.format(STATEMENT_TMPL_CONVERTMETHODBODY_04, args3));
 		// Foo _$$_t = new Foo();
 		String typeName = classToString(type);
 		Object[] args0 = new Object[] { typeName, typeName };
-		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_01, args0));
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_01, args0));
 		// MessagePackObject[] _$$_ary = $1.asArray();
 		Object[] args1 = new Object[] { classToString(MessagePackObject[].class) };
-		sb.append(String.format(STATEMENT_PACKER_CONVERTMETHODBODY_01, args1));
+		sb.append(String.format(STATEMENT_TMPL_CONVERTMETHODBODY_01, args1));
 		insertCodeOfConvertMethodCalls(sb, fields);
 		// return _$$_t;
 		Object[] args2 = new Object[0];
-		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_04, args2));
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_04, args2));
 		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
 	}
 
@@ -575,6 +585,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		Class returnType = field.getType();
 		boolean isPrim = returnType.isPrimitive();
 		Object[] args = new Object[] {
+				i,
 				field.getName(),
 				isPrim ? "(" : "",
 				isPrim ? getPrimToWrapperType(returnType).getName()
@@ -583,7 +594,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				i,
 				isPrim ? ")." + getPrimTypeValueMethodName(returnType) + "()"
 						: "" };
-		sb.append(String.format(STATEMENT_PACKER_CONVERTMETHODBODY_02, args));
+		sb.append(String.format(STATEMENT_TMPL_CONVERTMETHODBODY_02, args));
 	}
 
 	private void insertCodeOfMessageConvertCallForMsgConvtblType(
@@ -640,15 +651,17 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// Object convert(MessagePackObject mpo) throws MessageTypeException;
 		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
+		Object[] args3 = new Object[0];
+		sb.append(String.format(STATEMENT_TMPL_CONVERTMETHODBODY_04, args3));
 		// MessagePackObject[] _$$_ary = $1.asArray();
 		Object[] args0 = new Object[] { classToString(MessagePackObject[].class) };
-		sb.append(String.format(STATEMENT_PACKER_CONVERTMETHODBODY_01, args0));
+		sb.append(String.format(STATEMENT_TMPL_CONVERTMETHODBODY_01, args0));
 		// int i = _$$_ary[0].asInt();
 		Object[] args1 = new Object[0];
-		sb.append(String.format(STATEMENT_PACKER_CONVERTMETHODBODY_03, args1));
+		sb.append(String.format(STATEMENT_TMPL_CONVERTMETHODBODY_03, args1));
 		// return Foo.class.getEnumConstants()[i];
 		Object[] args2 = new Object[] { classToString(type) };
-		sb.append(String.format(STATEMENT_PACKER_UNPACKERMETHODBODY_06, args2));
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_06, args2));
 		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 4bfea8c..0f39427 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -32,11 +32,24 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class DynamicCodeGenBase implements Constants {
+
+	public static interface NullChecker {
+		void setNullCheck(boolean nullCheck);
+	}
+	
+	public static class NullCheckerImpl implements NullChecker {
+		public boolean _$$_nullCheck = true;
+
+		public void setNullCheck(boolean _$$_check) {
+			_$$_nullCheck = _$$_check;
+		}		
+	}
+	
 	public static interface TemplateAccessor {
 		void setTemplates(Template[] templates);
 	}
-
-	public static class TemplateAccessorImpl implements TemplateAccessor {
+	
+	public static class TemplateTemplate extends NullCheckerImpl implements TemplateAccessor {
 		public Template[] _$$_templates;
 
 		public void setTemplates(Template[] _$$_tmpls) {
@@ -111,10 +124,50 @@ public class DynamicCodeGenBase implements Constants {
 		throw e;
 	}
 
-	protected void setInterface(CtClass packerCtClass, Class infClass)
+	enum Color {
+		// TODO
+		RED, BLUE
+	}
+	
+	public static void main(String[] args) throws Exception {
+		// TODO
+		class Foo {
+		}
+		class Bar extends Foo {
+		}
+		Color c = Color.RED;
+		Color c1 = null;
+
+		ClassPool pool = ClassPool.getDefault();
+		CtClass barCtClass = pool.get(Bar.class.getName());
+		CtClass sbarCtClass = barCtClass.getSuperclass();
+		System.out.println("bar: " + sbarCtClass.getName());
+		CtClass fooCtClass = pool.get(Foo.class.getName());
+		CtClass sfooCtClass = fooCtClass.getSuperclass();
+		System.out.println("foo: " + sfooCtClass.getName());
+	}
+
+	protected void setSuperclass(CtClass newCtClass, Class superClass)
+			throws NotFoundException, CannotCompileException {
+		// check the specified super class
+		if (superClass.isInterface() || superClass.isEnum()
+				|| superClass.isAnnotation() || superClass.isArray()
+				|| superClass.isPrimitive()) {
+			throwTypeValidationException(superClass, "Fatal error");
+		}
+
+		// check the base class
+		if (!newCtClass.getSuperclass().equals(classToCtClass(Object.class))) {
+			throwTypeValidationException(superClass, "Fatal error");
+		}
+		CtClass superCtClass = pool.get(superClass.getName());
+		newCtClass.setSuperclass(superCtClass);
+	}
+
+	protected void setInterface(CtClass newCtClass, Class infClass)
 			throws NotFoundException {
 		CtClass infCtClass = pool.get(infClass.getName());
-		packerCtClass.addInterface(infCtClass);
+		newCtClass.addInterface(infCtClass);
 	}
 
 	protected void addDefaultConstructor(CtClass enhancedCtClass)
@@ -126,7 +179,7 @@ public class DynamicCodeGenBase implements Constants {
 
 	protected void addTemplateArrayField(CtClass newCtClass)
 			throws NotFoundException, CannotCompileException {
-		CtClass acsCtClass = pool.get(TemplateAccessorImpl.class.getName());
+		CtClass acsCtClass = pool.get(TemplateTemplate.class.getName());
 		CtField tmplsField = acsCtClass
 				.getDeclaredField(VARIABLE_NAME_TEMPLATES);
 		CtField tmplsField2 = new CtField(tmplsField.getType(), tmplsField
@@ -136,7 +189,7 @@ public class DynamicCodeGenBase implements Constants {
 
 	protected void addSetTemplatesMethod(CtClass newCtClass)
 			throws NotFoundException, CannotCompileException {
-		CtClass acsCtClass = pool.get(TemplateAccessorImpl.class.getName());
+		CtClass acsCtClass = pool.get(TemplateTemplate.class.getName());
 		CtMethod settmplsMethod = acsCtClass
 				.getDeclaredMethod(METHOD_NAME_SETTEMPLATES);
 		CtMethod settmplsMethod2 = CtNewMethod.copy(settmplsMethod, newCtClass,
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
index 436ea27..bad91ce 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
@@ -26,7 +26,7 @@ import org.msgpack.annotation.MessagePackOrdinalEnum;
 public class TestDynamicCodeGenPackerConverter extends TestCase {
 
 	@Test
-	public void testPrimitiveTypeFields() throws Exception {
+	public void testPrimitiveTypeField00s() throws Exception {
 		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
 		src.f0 = (byte) 0;
 		src.f1 = 1;
@@ -40,8 +40,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 				.create(PrimitiveTypeFieldsClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(PrimitiveTypeFieldsClass.class);
+		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class);
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -58,6 +57,50 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
+	@Test
+	public void testPrimitiveTypeFields01() throws Exception {
+		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(PrimitiveTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testPrimitiveTypeFields02() throws Exception {
+		PrimitiveTypeFieldsClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(PrimitiveTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
 	public static class PrimitiveTypeFieldsClass {
 		public byte f0;
 		public short f1;
@@ -72,7 +115,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	}
 
 	@Test
-	public void testGeneralReferenceTypeFieldsClass() throws Exception {
+	public void testGeneralReferenceTypeFieldsClass00() throws Exception {
 		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
 		src.f0 = 0;
 		src.f1 = 1;
@@ -111,6 +154,65 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
+	@Test
+	public void XtestGeneralReferenceTypeFieldsClass01() throws Exception {
+		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
+		src.f0 = null;
+		src.f1 = null;
+		src.f2 = null;
+		src.f3 = null;
+		src.f4 = null;
+		src.f5 = null;
+		src.f6 = null;
+		src.f7 = null;
+		src.f8 = null;
+		src.f9 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(GeneralReferenceTypeFieldsClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9, dst.f9);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void XtestGeneralReferenceTypeFieldsClass02() throws Exception {
+		GeneralReferenceTypeFieldsClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(GeneralReferenceTypeFieldsClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
 	public static class GeneralReferenceTypeFieldsClass {
 		public Byte f0;
 		public Short f1;
@@ -127,7 +229,8 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		}
 	}
 
-	public void testListTypes() throws Exception {
+	@Test
+	public void testListTypes00() throws Exception {
 		SampleListTypes src = new SampleListTypes();
 		src.f0 = new ArrayList();
 		src.f1 = new ArrayList();
@@ -146,8 +249,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		slnt.f1 = "muga";
 		src.f4.add(slnt);
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(SampleListTypes.class);
+		MessagePacker packer = DynamicPacker.create(SampleListTypes.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
@@ -185,6 +287,49 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
+	@Test
+	public void testListTypes01() throws Exception {
+		SampleListTypes src = new SampleListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = null;
+		src.f2 = new ArrayList();
+		src.f3 = null;
+		src.f4 = new ArrayList();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleListTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleListTypes dst = (SampleListTypes) tmpl.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4.size(), dst.f4.size());
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testListTypes02() throws Exception {
+		SampleListTypes src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleListTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleListTypes dst = (SampleListTypes) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
 	public static class SampleListTypes {
 		public List f0;
 		public List f1;
@@ -205,7 +350,8 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		}
 	}
 
-	public void testMapTypes() throws Exception {
+	@Test
+	public void testMapTypes00() throws Exception {
 		SampleMapTypes src = new SampleMapTypes();
 		src.f0 = new HashMap();
 		src.f1 = new HashMap();
@@ -217,8 +363,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		src.f2.put("k2", 2);
 		src.f2.put("k3", 3);
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(SampleMapTypes.class);
+		MessagePacker packer = DynamicPacker.create(SampleMapTypes.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
@@ -249,6 +394,45 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
+	@Test
+	public void testMapTypes01() throws Exception {
+		SampleMapTypes src = new SampleMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = null;
+		src.f2 = new HashMap();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleMapTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleMapTypes dst = (SampleMapTypes) tmpl.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testMapTypes02() throws Exception {
+		SampleMapTypes src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleMapTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleMapTypes dst = (SampleMapTypes) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
 	public static class SampleMapTypes {
 		public Map f0;
 		public Map f1;
@@ -307,8 +491,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		}
 		assertTrue(true);
 		try {
-			DynamicTemplate
-					.create(ProtectedDefaultConstructorClass.class);
+			DynamicTemplate.create(ProtectedDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -479,17 +662,15 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	}
 
 	@Test
-	public void testEnumTypeForOrdinal() throws Exception {
+	public void testEnumTypeForOrdinal00() throws Exception {
 		SampleEnumFieldClass src = new SampleEnumFieldClass();
 		src.f0 = 0;
 		src.f1 = SampleEnum.ONE;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(SampleEnumFieldClass.class);
+		MessagePacker packer = DynamicPacker.create(SampleEnumFieldClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(SampleEnumFieldClass.class);
+		Template tmpl = DynamicTemplate.create(SampleEnumFieldClass.class);
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -497,6 +678,43 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl.convert(mpo);
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1 == dst.f1);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testEnumTypeForOrdinal01() throws Exception {
+		SampleEnumFieldClass src = new SampleEnumFieldClass();
+		src.f1 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleEnumFieldClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleEnumFieldClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testEnumTypeForOrdinal02() throws Exception {
+		SampleEnumFieldClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleEnumFieldClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleEnumFieldClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
 	}
 
 	public static class SampleEnumFieldClass {
@@ -521,12 +739,10 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		src.f3 = 3;
 		src.f4 = 4;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(FieldModifiersClass.class);
+		MessagePacker packer = DynamicPacker.create(FieldModifiersClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(FieldModifiersClass.class);
+		Template tmpl = DynamicTemplate.create(FieldModifiersClass.class);
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -552,7 +768,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	}
 
 	@Test
-	public void testNestedFieldClass() throws Exception {
+	public void testNestedFieldClass00() throws Exception {
 		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
 		CustomPacker.register(NestedClass.class, packer2);
 		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
@@ -581,6 +797,57 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
+	@Test
+	public void testNestedFieldClass01() throws Exception {
+		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
+		CustomPacker.register(NestedClass.class, packer2);
+		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
+		CustomPacker.register(BaseClass.class, packer1);
+		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
+		CustomUnpacker.register(NestedClass.class, tmpl2);
+		CustomConverter.register(NestedClass.class, tmpl2);
+		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
+		CustomUnpacker.register(BaseClass.class, tmpl1);
+		CustomConverter.register(BaseClass.class, tmpl1);
+		BaseClass src = new BaseClass();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer1.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		BaseClass dst = (BaseClass) tmpl1.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testNestedFieldClass02() throws Exception {
+		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
+		CustomPacker.register(NestedClass.class, packer2);
+		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
+		CustomPacker.register(BaseClass.class, packer1);
+		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
+		CustomUnpacker.register(NestedClass.class, tmpl2);
+		CustomConverter.register(NestedClass.class, tmpl2);
+		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
+		CustomUnpacker.register(BaseClass.class, tmpl1);
+		CustomConverter.register(BaseClass.class, tmpl1);
+		BaseClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer1.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		BaseClass dst = (BaseClass) tmpl1.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
 	public static class BaseClass {
 		public int f0;
 		public NestedClass f1;
@@ -597,7 +864,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	}
 
 	@Test
-	public void testMessagePackMessageFieldClass() throws Exception {
+	public void testMessagePackMessageFieldClass00() throws Exception {
 		BaseClass2 src = new BaseClass2();
 		MessagePackMessageClass2 src2 = new MessagePackMessageClass2();
 		src.f0 = 0;
@@ -618,6 +885,41 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
+	@Test
+	public void testMessagePackMessageFieldClass01() throws Exception {
+		BaseClass2 src = new BaseClass2();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(BaseClass2.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		Template tmpl = DynamicTemplate.create(BaseClass2.class);
+		BaseClass2 dst = (BaseClass2) tmpl.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testMessagePackMessageFieldClass02() throws Exception {
+		BaseClass2 src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(BaseClass2.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		Template tmpl = DynamicTemplate.create(BaseClass2.class);
+		BaseClass2 dst = (BaseClass2) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
 	public static class BaseClass2 {
 		public int f0;
 		public MessagePackMessageClass2 f1;
@@ -635,7 +937,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 	}
 
 	@Test
-	public void testExtendedClass() throws Exception {
+	public void testExtendedClass00() throws Exception {
 		SampleSubClass src = new SampleSubClass();
 		src.f0 = 0;
 		src.f2 = 2;
@@ -645,8 +947,7 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		src.f8 = 8;
 		src.f9 = 9;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(SampleSubClass.class);
+		MessagePacker packer = DynamicPacker.create(SampleSubClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Template tmpl = DynamicTemplate.create(SampleSubClass.class);
@@ -667,6 +968,23 @@ public class TestDynamicCodeGenPackerConverter extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
+	@Test
+	public void testExtendedClass01() throws Exception {
+		SampleSubClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleSubClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleSubClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleSubClass dst = (SampleSubClass) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
 	public static class SampleSubClass extends SampleSuperClass {
 		public int f0;
 		public final int f1 = 1;
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
index f46d6a0..27f9adb 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
@@ -25,7 +25,7 @@ import junit.framework.TestCase;
 public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 
 	@Test
-	public void testPrimitiveTypeFields() throws Exception {
+	public void testPrimitiveTypeFields00() throws Exception {
 		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
 		src.f0 = (byte) 0;
 		src.f1 = 1;
@@ -39,8 +39,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 				.create(PrimitiveTypeFieldsClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(PrimitiveTypeFieldsClass.class);
+		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class);
 		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
 				.unpack(new Unpacker(in));
 		assertEquals(src.f0, dst.f0);
@@ -52,6 +51,40 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		assertEquals(src.f6, dst.f6);
 	}
 
+	@Test
+	public void testPrimitiveTypeFields01() throws Exception {
+		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(PrimitiveTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class);
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+	}
+
+	@Test
+	public void testPrimitiveTypeFields02() throws Exception {
+		PrimitiveTypeFieldsClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(PrimitiveTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class);
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
 	public static class PrimitiveTypeFieldsClass {
 		public byte f0;
 		public short f1;
@@ -66,7 +99,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testGeneralReferenceTypeFieldsClass() throws Exception {
+	public void testGeneralReferenceTypeFieldsClass00() throws Exception {
 		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
 		src.f0 = 0;
 		src.f1 = 1;
@@ -100,6 +133,55 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		assertEquals(src.f9[1], dst.f9[1]);
 	}
 
+	@Test
+	public void testGeneralReferenceTypeFieldsClass01() throws Exception {
+		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
+		src.f0 = null;
+		src.f1 = null;
+		src.f2 = null;
+		src.f3 = null;
+		src.f4 = null;
+		src.f5 = null;
+		src.f6 = null;
+		src.f7 = null;
+		src.f8 = null;
+		src.f9 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(GeneralReferenceTypeFieldsClass.class);
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9, dst.f9);
+	}
+
+	@Test
+	public void testGeneralReferenceTypeFieldsClass02() throws Exception {
+		GeneralReferenceTypeFieldsClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(GeneralReferenceTypeFieldsClass.class);
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
 	public static class GeneralReferenceTypeFieldsClass {
 		public Byte f0;
 		public Short f1;
@@ -116,7 +198,8 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		}
 	}
 
-	public void testListTypes() throws Exception {
+	@Test
+	public void testListTypes00() throws Exception {
 		SampleListTypes src = new SampleListTypes();
 		src.f0 = new ArrayList();
 		src.f1 = new ArrayList();
@@ -135,8 +218,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		slnt.f1 = "muga";
 		src.f4.add(slnt);
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(SampleListTypes.class);
+		MessagePacker packer = DynamicPacker.create(SampleListTypes.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
@@ -169,6 +251,39 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		}
 	}
 
+	@Test
+	public void testListTypes01() throws Exception {
+		SampleListTypes src = new SampleListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = null;
+		src.f2 = new ArrayList();
+		src.f3 = new ArrayList>();
+		src.f4 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleListTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
+		SampleListTypes dst = (SampleListTypes) tmpl.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertEquals(src.f3.size(), dst.f3.size());
+		assertEquals(src.f4, dst.f4);
+	}
+
+	@Test
+	public void testListTypes02() throws Exception {
+		SampleListTypes src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleListTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
+		SampleListTypes dst = (SampleListTypes) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
 	public static class SampleListTypes {
 		public List f0;
 		public List f1;
@@ -189,7 +304,8 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		}
 	}
 
-	public void testMapTypes() throws Exception {
+	@Test
+	public void testMapTypes00() throws Exception {
 		SampleMapTypes src = new SampleMapTypes();
 		src.f0 = new HashMap();
 		src.f1 = new HashMap();
@@ -201,8 +317,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		src.f2.put("k2", 2);
 		src.f2.put("k3", 3);
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(SampleMapTypes.class);
+		MessagePacker packer = DynamicPacker.create(SampleMapTypes.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
@@ -228,6 +343,35 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		}
 	}
 
+	@Test
+	public void testMapTypes01() throws Exception {
+		SampleMapTypes src = new SampleMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = null;
+		src.f2 = new HashMap();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleMapTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
+		SampleMapTypes dst = (SampleMapTypes) tmpl.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+	}
+
+	@Test
+	public void testMapTypes02() throws Exception {
+		SampleMapTypes src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleMapTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
+		SampleMapTypes dst = (SampleMapTypes) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
 	public static class SampleMapTypes {
 		public Map f0;
 		public Map f1;
@@ -238,7 +382,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testDefaultConstructorModifiers01() throws Exception {
+	public void testDefaultConstructorModifiers00() throws Exception {
 		try {
 			DynamicPacker.create(NoDefaultConstructorClass.class);
 			fail();
@@ -270,7 +414,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testDefaultConstructorModifiers02() throws Exception {
+	public void testDefaultConstructorModifiers01() throws Exception {
 		try {
 			DynamicUnpacker.create(NoDefaultConstructorClass.class);
 			fail();
@@ -286,8 +430,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		}
 		assertTrue(true);
 		try {
-			DynamicUnpacker
-					.create(ProtectedDefaultConstructorClass.class);
+			DynamicUnpacker.create(ProtectedDefaultConstructorClass.class);
 			fail();
 		} catch (DynamicCodeGenException e) {
 			assertTrue(true);
@@ -323,7 +466,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testClassModifiers01() throws Exception {
+	public void testClassModifiers00() throws Exception {
 		try {
 			DynamicPacker.create(PrivateModifierClass.class);
 			fail();
@@ -348,7 +491,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testClassModifiers02() throws Exception {
+	public void testClassModifiers01() throws Exception {
 		try {
 			DynamicUnpacker.create(PrivateModifierClass.class);
 			fail();
@@ -384,7 +527,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testFinalClassAndAbstractClass01() throws Exception {
+	public void testFinalClassAndAbstractClass00() throws Exception {
 		try {
 			DynamicPacker.create(FinalModifierClass.class);
 			assertTrue(true);
@@ -402,7 +545,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testFinalClassAndAbstractClass02() throws Exception {
+	public void testFinalClassAndAbstractClass01() throws Exception {
 		try {
 			DynamicUnpacker.create(FinalModifierClass.class);
 			assertTrue(true);
@@ -426,7 +569,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testInterfaceType01() throws Exception {
+	public void testInterfaceType00() throws Exception {
 		try {
 			DynamicPacker.create(SampleInterface.class);
 			fail();
@@ -437,7 +580,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testInterfaceType02() throws Exception {
+	public void testInterfaceType01() throws Exception {
 		try {
 			DynamicUnpacker.create(SampleInterface.class);
 			fail();
@@ -451,23 +594,49 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testEnumTypeForOrdinal() throws Exception {
+	public void testEnumTypeForOrdinal00() throws Exception {
 		SampleEnumFieldClass src = new SampleEnumFieldClass();
 		src.f0 = 0;
 		src.f1 = SampleEnum.ONE;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(SampleEnumFieldClass.class);
+		MessagePacker packer = DynamicPacker.create(SampleEnumFieldClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(SampleEnumFieldClass.class);
+		Template tmpl = DynamicTemplate.create(SampleEnumFieldClass.class);
 		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl
 				.unpack(new Unpacker(in));
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1 == dst.f1);
 	}
 
+	@Test
+	public void testEnumTypeForOrdinal01() throws Exception {
+		SampleEnumFieldClass src = new SampleEnumFieldClass();
+		src.f1 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleEnumFieldClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleEnumFieldClass.class);
+		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl
+				.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertEquals(src.f1, dst.f1);
+	}
+
+	@Test
+	public void testEnumTypeForOrdinal02() throws Exception {
+		SampleEnumFieldClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleEnumFieldClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleEnumFieldClass.class);
+		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
 	public static class SampleEnumFieldClass {
 		public int f0;
 
@@ -490,12 +659,10 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		src.f3 = 3;
 		src.f4 = 4;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(FieldModifiersClass.class);
+		MessagePacker packer = DynamicPacker.create(FieldModifiersClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(FieldModifiersClass.class);
+		Template tmpl = DynamicTemplate.create(FieldModifiersClass.class);
 		FieldModifiersClass dst = (FieldModifiersClass) tmpl
 				.unpack(new Unpacker(in));
 		assertTrue(src.f0 == dst.f0);
@@ -517,7 +684,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testNestedFieldClass() throws Exception {
+	public void testNestedFieldClass00() throws Exception {
 		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
 		CustomPacker.register(NestedClass.class, packer2);
 		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
@@ -541,6 +708,48 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		assertTrue(src.f1.f2 == dst.f1.f2);
 	}
 
+	@Test
+	public void testNestedFieldClass01() throws Exception {
+		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
+		CustomPacker.register(NestedClass.class, packer2);
+		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
+		CustomPacker.register(BaseClass.class, packer1);
+		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
+		CustomUnpacker.register(NestedClass.class, tmpl2);
+		CustomConverter.register(NestedClass.class, tmpl2);
+		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
+		CustomUnpacker.register(BaseClass.class, tmpl1);
+		CustomConverter.register(BaseClass.class, tmpl1);
+		BaseClass src = new BaseClass();
+		src.f1 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer1.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		BaseClass dst = (BaseClass) tmpl1.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+	}
+
+	@Test
+	public void testNestedFieldClass02() throws Exception {
+		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
+		CustomPacker.register(NestedClass.class, packer2);
+		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
+		CustomPacker.register(BaseClass.class, packer1);
+		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
+		CustomUnpacker.register(NestedClass.class, tmpl2);
+		CustomConverter.register(NestedClass.class, tmpl2);
+		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
+		CustomUnpacker.register(BaseClass.class, tmpl1);
+		CustomConverter.register(BaseClass.class, tmpl1);
+		BaseClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer1.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		BaseClass dst = (BaseClass) tmpl1.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
 	public static class BaseClass {
 		public int f0;
 		public NestedClass f1;
@@ -557,7 +766,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testMessagePackMessageFieldClass() throws Exception {
+	public void testMessagePackMessageFieldClass00() throws Exception {
 		BaseClass2 src = new BaseClass2();
 		MessagePackMessageClass2 src2 = new MessagePackMessageClass2();
 		src.f0 = 0;
@@ -573,6 +782,32 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		assertTrue(src.f1.f2 == dst.f1.f2);
 	}
 
+	@Test
+	public void testMessagePackMessageFieldClass01() throws Exception {
+		BaseClass2 src = new BaseClass2();
+		src.f1 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(BaseClass2.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(BaseClass2.class);
+		BaseClass2 dst = (BaseClass2) tmpl.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertEquals(src.f1, dst.f1);
+	}
+
+	@Test
+	public void testMessagePackMessageFieldClass02() throws Exception {
+		BaseClass2 src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(BaseClass2.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(BaseClass2.class);
+		BaseClass2 dst = (BaseClass2) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
 	public static class BaseClass2 {
 		public int f0;
 		public MessagePackMessageClass2 f1;
@@ -590,7 +825,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 	}
 
 	@Test
-	public void testExtendedClass() throws Exception {
+	public void testExtendedClass00() throws Exception {
 		SampleSubClass src = new SampleSubClass();
 		src.f0 = 0;
 		src.f2 = 2;
@@ -600,8 +835,7 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		src.f8 = 8;
 		src.f9 = 9;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(SampleSubClass.class);
+		MessagePacker packer = DynamicPacker.create(SampleSubClass.class);
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Template tmpl = DynamicTemplate.create(SampleSubClass.class);
@@ -617,6 +851,18 @@ public class TestDynamicCodeGenPackerUnpacker extends TestCase {
 		assertTrue(src.f9 != dst.f9);
 	}
 
+	@Test
+	public void testExtendedClass01() throws Exception {
+		SampleSubClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(SampleSubClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleSubClass.class);
+		SampleSubClass dst = (SampleSubClass) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
 	public static class SampleSubClass extends SampleSuperClass {
 		public int f0;
 		public final int f1 = 1;

From 2b65f81e23063cded71dda7b78cbab7f92671a8b Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Tue, 12 Oct 2010 22:58:53 +0900
Subject: [PATCH 0859/1648] Add tests

---
 perl/t/06_stream_unpack2.t | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t
index bb6fe93..11e3c61 100644
--- a/perl/t/06_stream_unpack2.t
+++ b/perl/t/06_stream_unpack2.t
@@ -1,15 +1,16 @@
 use strict;
 use warnings;
 use Data::MessagePack;
-use Test::More tests => 9;
+use Test::More tests => 61;
 use t::Util;
 
 my $input = [
     false,true,null,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,
     127,127,255,65535,4294967295,-32,-32,-128,-32768,
-    -2147483648,0.0,-0.0,1.0,-1.0,"a","a","a","","","",
+    -2147483648,0.0,-0.0, 3.14,-3.14,"a","a","a","","","",
     [0],[0],[0],[],[],[],{},{},{},
-    {"a" => 97},{"a" => 97},{"a" => 97},[[]],[["a"]]
+    {"a" => 97},{"abc" => 97},{"xyz" => 97},[[]], [["foo"], ["bar"]],
+    [["foo", true, false, null, 42]],
 ];
 
 my $packed = Data::MessagePack->pack($input);
@@ -40,4 +41,21 @@ is_deeply(Data::MessagePack->unpack($packed), $input);
     }
 }
 
+{
+    my $s    = '';
+    foreach my $datum(reverse @{$input}) {
+        $s .= Data::MessagePack->pack($datum);
+    }
+
+    my $up = Data::MessagePack::Unpacker->new();
+
+    my $offset = 0;
+    for my $datum(reverse @{$input}) {
+        note "offset: $offset/".length($s);
+
+        $offset = $up->execute($s, $offset);
+        is_deeply $up->data, $datum;
+        $up->reset();
+    }
+}
 

From 4f1207a38c39b9b7284f4d28012e4f76fa0b437e Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Tue, 12 Oct 2010 23:05:58 +0900
Subject: [PATCH 0860/1648] perl: add a strong assertion

---
 perl/xs-src/unpack.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index caf8662..e997af5 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -65,6 +65,9 @@ load_bool(pTHX_ const char* const name) {
     LEAVE;
     assert(sv);
     assert(sv_isobject(sv));
+    if(!SvOK(sv)) {
+        croak("Oops: Failed to load %"SVf, name);
+    }
     return sv;
 }
 

From 770542c8c7d3469af5e54b4036fba47e07e1dee0 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Tue, 12 Oct 2010 23:10:59 +0900
Subject: [PATCH 0861/1648] perl: add tests

---
 perl/t/12_stream_unpack4.t | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/perl/t/12_stream_unpack4.t b/perl/t/12_stream_unpack4.t
index de76e81..ef6fa39 100644
--- a/perl/t/12_stream_unpack4.t
+++ b/perl/t/12_stream_unpack4.t
@@ -5,11 +5,14 @@ use Test::More;
 use t::Util;
 
 my @input = (
-    +[[]],
+    [[]],
     [[],[]],
     [{"a" => 97},{"a" => 97}],
     [{"a" => 97},{"a" => 97},{"a" => 97}],
     [ map { +{ "foo $_" => "bar $_" } } 'aa' .. 'zz' ],
+    [42, null],
+    [42, true],
+    [42, false],
 );
 
 plan tests => @input * 2;

From 5b786f65a4ad4b2c04c6f792fd7526cfd63a0af7 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Tue, 12 Oct 2010 23:12:34 +0900
Subject: [PATCH 0862/1648] Checking in changes prior to tagging of version
 0.32.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index 6e07966..4657079 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,4 +1,8 @@

+0.32
+
+    - add tests to detect Alpha problems reported via CPAN testers (gfx)
+
 0.31

     - update Module::Install::XSUtil for ccache support (gfx)
---
 perl/Changes                 | 4 ++++
 perl/lib/Data/MessagePack.pm | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index 6e07966..4657079 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,4 +1,8 @@
 
+0.32
+
+    - add tests to detect Alpha problems reported via CPAN testers (gfx)
+
 0.31
 
     - update Module::Install::XSUtil for ccache support (gfx)
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 6d37d8f..d402167 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.31';
+our $VERSION = '0.32';
 our $PreferInteger = 0;
 
 sub true () {

From 0ced3ec2d29cd1df69f72349e5e1a60ef2bdc095 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Wed, 13 Oct 2010 09:38:40 +0900
Subject: [PATCH 0863/1648] perl: fix tests

---
 perl/t/06_stream_unpack2.t | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t
index 11e3c61..f50401f 100644
--- a/perl/t/06_stream_unpack2.t
+++ b/perl/t/06_stream_unpack2.t
@@ -7,7 +7,7 @@ use t::Util;
 my $input = [
     false,true,null,0,0,0,0,0,0,0,0,0,-1,-1,-1,-1,-1,
     127,127,255,65535,4294967295,-32,-32,-128,-32768,
-    -2147483648,0.0,-0.0, 3.14,-3.14,"a","a","a","","","",
+    -2147483648,0.0,-0.0, 3.0,-3.0,"a","a",("a" x 70000),"","","",
     [0],[0],[0],[],[],[],{},{},{},
     {"a" => 97},{"abc" => 97},{"xyz" => 97},[[]], [["foo"], ["bar"]],
     [["foo", true, false, null, 42]],
@@ -36,7 +36,7 @@ is_deeply(Data::MessagePack->unpack($packed), $input);
         $offset = $up->execute($packed, $offset);
         ok $up->is_finished, 'finished';
         my $data = $up->data;
-        is_deeply $data, $input;
+        is_deeply $data, $input, "block $i, offset $offset";
         $up->reset();
     }
 }
@@ -51,10 +51,8 @@ is_deeply(Data::MessagePack->unpack($packed), $input);
 
     my $offset = 0;
     for my $datum(reverse @{$input}) {
-        note "offset: $offset/".length($s);
-
         $offset = $up->execute($s, $offset);
-        is_deeply $up->data, $datum;
+        is_deeply $up->data, $datum, "offset $offset/" . length($s);
         $up->reset();
     }
 }

From 233f13aac5ed328c82749930a7c8c42f7081af6c Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Wed, 13 Oct 2010 10:03:56 +0900
Subject: [PATCH 0864/1648] perl: add tests for unpacking 'float'

---
 perl/t/02_unpack.t | 2 +-
 perl/t/data.pl     | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/perl/t/02_unpack.t b/perl/t/02_unpack.t
index 657387a..1087c40 100644
--- a/perl/t/02_unpack.t
+++ b/perl/t/02_unpack.t
@@ -15,7 +15,7 @@ sub pis ($$) {
         or diag( explain(unpackit($_[0])) );
 }
 
-my @dat = do 't/data.pl';
+my @dat = do 't/data.pl' or die $@;
 
 plan tests => 1*(scalar(@dat)/2);
 
diff --git a/perl/t/data.pl b/perl/t/data.pl
index 9bf07b7..300eec1 100644
--- a/perl/t/data.pl
+++ b/perl/t/data.pl
@@ -26,6 +26,9 @@ no warnings; # i need this, i need this.
     'aa 34 32 39 34 39 36 37 32 39 35' => ''.0xFFFFFFFF,
     'ab 36 38 37 31 39 34 37 36 37 33 35' => ''.0xFFFFFFFFF,
 
+    'ca 00 00 00 00' => 0.0, # float
+    'ca 40 2c cc cd' => unpack('f', pack 'f', 2.7),
+
     'd2 80 00 00 01' => '-2147483647', # int32_t
     'ce 80 00 00 01' => '2147483649',  # uint32_t
 

From 3761aacb1d4dbdaa922e024e5ae709a9037c06f3 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Wed, 13 Oct 2010 10:14:31 +0900
Subject: [PATCH 0865/1648] perl: cleanup PP

---
 perl/lib/Data/MessagePack/PP.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 00e58b9..15ba22f 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -265,7 +265,7 @@ sub _unpack {
     my ( $value ) = @_;
     my $byte = CORE::unpack( 'C', substr( $value, $p++, 1 ) ); # get header
 
-    die "invalid data" unless defined $byte;
+    Carp::croak("invalid data") unless defined $byte;
 
     if ( ( $byte >= 0x90 and $byte <= 0x9f ) or $byte == 0xdc or $byte == 0xdd ) {
         my $num;
@@ -312,7 +312,7 @@ sub _unpack {
         return $byte;
     }
     elsif ( $byte == 0xcc ) { # uint8
-        CORE::unpack( 'C', substr( $value, $p++, 1 ) );
+        return CORE::unpack( 'C', substr( $value, $p++, 1 ) );
     }
     elsif ( $byte == 0xcd ) { # uint16
         $p += 2;

From c506cd97e0032ec070a81b58a55f5089a4a1b60f Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Wed, 13 Oct 2010 10:43:56 +0900
Subject: [PATCH 0866/1648] perl: tests

---
 perl/t/01_pack.t | 2 ++
 perl/t/data.pl   | 1 +
 2 files changed, 3 insertions(+)

diff --git a/perl/t/01_pack.t b/perl/t/01_pack.t
index 50fd663..8c61980 100644
--- a/perl/t/01_pack.t
+++ b/perl/t/01_pack.t
@@ -55,6 +55,8 @@ my @dat = (
     [0, -128, -1, 0, -32768, -1, 0, -2147483648, -1], '99 00 d0 80 ff 00 d1 80 00 ff 00 d2 80 00 00 00 ff',
     2147483648, 'ce 80 00 00 00',
     -2147483648, 'd2 80 00 00 00',
+    'a' x 0x0100, 'da 01 00' . (' 61' x 0x0100),
+    [(undef) x 0x0100], 'dc 01 00' . (' c0' x 0x0100),
 );
 plan tests => 1*(scalar(@dat)/2);
 
diff --git a/perl/t/data.pl b/perl/t/data.pl
index 300eec1..b7bbaf1 100644
--- a/perl/t/data.pl
+++ b/perl/t/data.pl
@@ -22,6 +22,7 @@ no warnings; # i need this, i need this.
     '82 c2 81 c0 c0 c3 81 c0 80', {false,{undef,undef}, true,{undef,{}}},
     '96 de 00 00 de 00 01 c0 c2 de 00 02 c0 c2 c3 c2 df 00 00 00 00 df 00 00 00 01 c0 c2 df 00 00 00 02 c0 c2 c3 c2',
         [{}, {undef,false}, {true,false, undef,false}, {}, {undef,false}, {true,false, undef,false}],
+    'dc 01 00'       . (' c0' x 0x0100),     [(undef) x 0x0100],
     'ce 00 ff ff ff' => ''.0xFFFFFF,
     'aa 34 32 39 34 39 36 37 32 39 35' => ''.0xFFFFFFFF,
     'ab 36 38 37 31 39 34 37 36 37 33 35' => ''.0xFFFFFFFFF,

From 14aa1420f0142f49beb6114599c27f665b34a075 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Wed, 13 Oct 2010 10:54:37 +0900
Subject: [PATCH 0867/1648] perl: comments

---
 perl/lib/Data/MessagePack/PP.pm | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 15ba22f..3aedf4c 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -17,6 +17,8 @@ BEGIN {
     my $unpack_uint64_slow;
 
     if(!eval { pack 'Q', 1 }) { # don't have quad types
+        # emulates quad types with Math::BigInt.
+        # very slow but works well.
         $unpack_int64_slow = sub {
             require Math::BigInt;
             my $high = unpack_uint32( $_[0], $_[1] );
@@ -35,7 +37,7 @@ BEGIN {
         };
         $unpack_uint64_slow = sub {
             require Math::BigInt;
-            my $high = Math::BigInt->new( unpack_uint32(  $_[0], $_[1]) );
+            my $high = Math::BigInt->new( unpack_uint32( $_[0], $_[1]) );
             my $low  = Math::BigInt->new( unpack_uint32( $_[0], $_[1] + 4) );
             return +($high << 32 | $low)->bstr;
         };
@@ -104,7 +106,8 @@ BEGIN {
             *unpack_uint64 = $unpack_uint64_slow || sub { unpack 'Q', substr( $_[0], $_[1], 8 ); };
         }
     }
-    else {
+    else { # 5.10.0 or later
+        # pack_int64/uint64 are used only when the perl support quad types
         *pack_uint64   = sub { return pack 'CQ>', 0xcf, $_[0]; };
         *pack_int64    = sub { return pack 'Cq>', 0xd3, $_[0]; };
         *pack_double   = sub { return pack 'Cd>', 0xcb, $_[0]; };

From ef0874feba7ac35a5cd4a6fd8763abf2cb1de40e Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Wed, 13 Oct 2010 11:02:57 +0900
Subject: [PATCH 0868/1648] perl: tweaks for PreferInteger

---
 perl/lib/Data/MessagePack/PP.pm | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 3aedf4c..30b963b 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -214,15 +214,18 @@ sub _pack {
 
         if ( $Data::MessagePack::PreferInteger ) {
             if ( $value =~ /^-?[0-9]+$/ ) { # ok?
-                my $value2 = 0 + $value;
-                if ( $value > 0xFFFFFFFF or $value < '-'.0x80000000 or # <- needless but for XS compat
-                    0 + $value != B::svref_2object( \$value2 )->int_value
-                ) {
-                    local $Data::MessagePack::PreferInteger; # avoid for PV => NV
-                    return _pack( "$value" );
+                # checks whether $value is in (u)int32
+                my $ivalue = 0 + $value;
+                if (!(
+                       $ivalue > 0xFFFFFFFF
+                    or $ivalue < '-'.0x80000000 # for XS compat
+                    or $ivalue != B::svref_2object(\$ivalue)->int_value
+                )) {
+                    return _pack( $ivalue );
                 }
-                return _pack( $value + 0 );
+                # fallthrough
             }
+            # fallthrough
         }
 
         utf8::encode( $value ) if utf8::is_utf8( $value );

From 02f3dd947a02ef35b74583041a837d7f6af6a398 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Wed, 13 Oct 2010 12:46:11 +0900
Subject: [PATCH 0869/1648] perl: optimize PP

---
 perl/Changes                    |   5 +
 perl/lib/Data/MessagePack/PP.pm | 183 +++++++++++++++++---------------
 2 files changed, 100 insertions(+), 88 deletions(-)

diff --git a/perl/Changes b/perl/Changes
index 4657079..50177f4 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,4 +1,9 @@
 
+0.33
+
+    - fix tests (gfx)
+    - optimize unpacking routines in Data::MessagePack::PP (gfx)
+
 0.32
 
     - add tests to detect Alpha problems reported via CPAN testers (gfx)
diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 30b963b..5e64093 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -164,7 +164,7 @@ sub _pack {
 
     if ( ref($value) eq 'ARRAY' ) {
         my $num = @$value;
-        my $header = 
+        my $header =
               $num < 16          ? CORE::pack( 'C',  0x90 + $num )
             : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xdc,  $num )
             : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdd,  $num )
@@ -175,7 +175,7 @@ sub _pack {
 
     elsif ( ref($value) eq 'HASH' ) {
         my $num = keys %$value;
-        my $header = 
+        my $header =
               $num < 16          ? CORE::pack( 'C',  0x80 + $num )
             : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xde,  $num )
             : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdf,  $num )
@@ -231,11 +231,11 @@ sub _pack {
         utf8::encode( $value ) if utf8::is_utf8( $value );
 
         my $num = length $value;
-        my $header = 
+        my $header =
               $num < 32          ? CORE::pack( 'C',  0xa0 + $num )
             : $num < 2 ** 16 - 1 ? CORE::pack( 'Cn', 0xda, $num )
             : $num < 2 ** 32 - 1 ? CORE::pack( 'CN', 0xdb, $num )
-            : _unexpected_number($num)
+            : _unexpected('number %d', $num)
         ;
 
         return $header . $value;
@@ -266,14 +266,72 @@ sub unpack :method {
     return $data;
 }
 
+my $T_RAW             = 0x01;
+my $T_ARRAY           = 0x02;
+my $T_MAP             = 0x04;
+my $T_DIRECT          = 0x08; # direct mapping (e.g. 0xc0 <-> nil)
+
+my @typemap = ( (0x00) x 256 );
+
+$typemap[$_] |= $T_ARRAY for
+    0x90 .. 0x9f, # fix array
+    0xdc,         # array16
+    0xdd,         # array32
+;
+$typemap[$_] |= $T_MAP for
+    0x80 .. 0x8f, # fix map
+    0xde,         # map16
+    0xdf,         # map32
+;
+$typemap[$_] |= $T_RAW for
+    0xa0 .. 0xbf, # fix raw
+    0xda,         # raw16
+    0xdb,         # raw32
+;
+
+my @byte2value;
+foreach my $pair(
+    [0xc3, true],
+    [0xc2, false],
+    [0xc0, undef],
+
+    (map { [ $_, $_ ] }         0x00 .. 0x7f), # positive fixnum
+    (map { [ $_, $_ - 0x100 ] } 0xe0 .. 0xff), # negative fixnum
+) {
+    $typemap[    $pair->[0] ] |= $T_DIRECT;
+    $byte2value[ $pair->[0] ]  = $pair->[1];
+}
 
 sub _unpack {
     my ( $value ) = @_;
-    my $byte = CORE::unpack( 'C', substr( $value, $p++, 1 ) ); # get header
+    # get a header byte
+    my $byte = unpack "x$p C", $value; # "x$p" is faster than substr()
+    $p++;
 
     Carp::croak("invalid data") unless defined $byte;
 
-    if ( ( $byte >= 0x90 and $byte <= 0x9f ) or $byte == 0xdc or $byte == 0xdd ) {
+    # +/- fixnum, nil, true, false
+    return $byte2value[$byte] if $typemap[$byte] & $T_DIRECT;
+
+    if ( $typemap[$byte] & $T_RAW ) {
+        my $num;
+        if ( $byte == 0xda ) {
+            $num = CORE::unpack 'n', substr( $value, $p, 2 );
+            $p += 2 + $num;
+        }
+        elsif ( $byte == 0xdb ) {
+            $num = CORE::unpack 'N', substr( $value, $p, 4 );
+            $p += 4 + $num;
+        }
+        else { # fix raw
+            $num = $byte & ~0xa0;
+            $p += $num;
+        }
+        my $s = substr( $value, $p - $num, $num );
+        utf8::decode($s) if $_utf8;
+        return $s;
+    }
+    elsif ( $typemap[$byte] & $T_ARRAY ) {
         my $num;
         if ( $byte == 0xdc ) { # array 16
             $num = CORE::unpack 'n', substr( $value, $p, 2 );
@@ -287,11 +345,10 @@ sub _unpack {
             $num = $byte & ~0x90;
         }
         my @array;
-        push @array, _unpack( $value ) while $num-- > 0;
+        push @array, _unpack( $value ) while --$num >= 0;
         return \@array;
     }
-
-    elsif ( ( $byte >= 0x80 and $byte <= 0x8f ) or $byte == 0xde or $byte == 0xdf ) {
+    elsif ( $typemap[$byte] & $T_MAP ) {
         my $num;
         if ( $byte == 0xde ) { # map 16
             $num = CORE::unpack 'n', substr( $value, $p, 2 );
@@ -305,7 +362,7 @@ sub _unpack {
             $num = $byte & ~0x80;
         }
         my %map;
-        for ( 0 .. $num - 1 ) {
+        while ( --$num >= 0 ) {
             no warnings; # for undef key case
             my $key = _unpack( $value );
             my $val = _unpack( $value );
@@ -314,9 +371,6 @@ sub _unpack {
         return \%map;
     }
 
-    elsif ( $byte >= 0x00 and $byte <= 0x7f ) { # positive fixnum
-        return $byte;
-    }
     elsif ( $byte == 0xcc ) { # uint8
         return CORE::unpack( 'C', substr( $value, $p++, 1 ) );
     }
@@ -347,53 +401,17 @@ sub _unpack {
     elsif ( $byte == 0xd0 ) { # int8
         return CORE::unpack 'c',  substr( $value, $p++, 1 ); # c / C
     }
-    elsif ( $byte >= 0xe0 and $byte <= 0xff ) { # negative fixnum
-        return $byte - 256;
-    }
-
-    elsif ( ( $byte >= 0xa0 and $byte <= 0xbf ) or $byte == 0xda or $byte == 0xdb ) { # raw
-        my $num;
-        if ( $byte == 0xda ) {
-            $num = CORE::unpack 'n', substr( $value, $p, 2 );
-            $p += 2 + $num;
-        }
-        elsif ( $byte == 0xdb ) {
-            $num = CORE::unpack 'N', substr( $value, $p, 4 );
-            $p += 4 + $num;
-        }
-        else { # fix raw
-            $num = $byte & ~0xa0;
-            $p += $num;
-        }
-        my $s = substr( $value, $p - $num, $num );
-        utf8::decode($s) if $_utf8;
-        return $s;
-    }
-
-    elsif ( $byte == 0xc0 ) { # nil
-        return undef;
-    }
-    elsif ( $byte == 0xc2 ) { # boolean
-        return false;
-    }
-    elsif ( $byte == 0xc3 ) { # boolean
-        return true;
-    }
-
     elsif ( $byte == 0xcb ) { # double
         $p += 8;
         return unpack_double( $value, $p - 8 );
     }
-
     elsif ( $byte == 0xca ) { # float
         $p += 4;
         return unpack_float( $value, $p - 4 );
     }
-
     else {
         _unexpected("byte 0x%02x", $byte);
     }
-
 }
 
 
@@ -456,7 +474,28 @@ sub _count {
     my ( $self, $value ) = @_;
     my $byte = unpack( 'C', substr( $value, $p++, 1 ) ); # get header
 
-    if ( ( $byte >= 0x90 and $byte <= 0x9f ) or $byte == 0xdc or $byte == 0xdd ) {
+    Carp::croak('invalid data') unless defined $byte;
+
+    # +/- fixnum, nil, true, false
+    return 1 if $typemap[$byte] & $T_DIRECT;
+
+    if ( $typemap[$byte] & $T_RAW ) {
+        my $num;
+        if ( $byte == 0xda ) {
+            $num = unpack 'n', substr( $value, $p, 2 );
+            $p += 2;
+        }
+        elsif ( $byte == 0xdb ) {
+            $num = unpack 'N', substr( $value, $p, 4 );
+            $p += 4;
+        }
+        else { # fix raw
+            $num = $byte & ~0xa0;
+        }
+        $p += $num;
+        return 1;
+    }
+    elsif ( $typemap[$byte] & $T_ARRAY ) {
         my $num;
         if ( $byte == 0xdc ) { # array 16
             $num = unpack 'n', substr( $value, $p, 2 );
@@ -476,8 +515,7 @@ sub _count {
 
         return 1;
     }
-
-    elsif ( ( $byte >= 0x80 and $byte <= 0x8f ) or $byte == 0xde or $byte == 0xdf ) {
+    elsif ( $typemap[$byte] & $T_MAP ) {
         my $num;
         if ( $byte == 0xde ) { # map 16
             $num = unpack 'n', substr( $value, $p, 2 );
@@ -498,20 +536,12 @@ sub _count {
         return 1;
     }
 
-    elsif ( $byte == 0xc0 or $byte == 0xc2 or $byte == 0xc3 ) { # nil, false, true
-        return 1;
-    }
-
-    elsif ( $byte >= 0x00 and $byte <= 0x7f ) { # positive fixnum
-        return 1;
-    }
-
     elsif ( $byte >= 0xcc and $byte <= 0xcf ) { # uint
         $p += $byte == 0xcc ? 1
             : $byte == 0xcd ? 2
             : $byte == 0xce ? 4
             : $byte == 0xcf ? 8
-            : _unexpected("byte 0x%02x", $byte);
+            : Data::MessagePack::PP::_unexpected("byte 0x%02x", $byte);
         return 1;
     }
 
@@ -520,38 +550,15 @@ sub _count {
             : $byte == 0xd1 ? 2
             : $byte == 0xd2 ? 4
             : $byte == 0xd3 ? 8
-            : _unexpected("byte 0x%02x", $byte);
+            : Data::MessagePack::PP::_unexpected("byte 0x%02x", $byte);
         return 1;
     }
-
-    elsif ( $byte >= 0xe0 and $byte <= 0xff ) { # negative fixnum
-        return 1;
-    }
-
-    elsif ( $byte >= 0xca and $byte <= 0xcb ) { # float, double
+    elsif ( $byte == 0xca or $byte == 0xcb ) { # float, double
         $p += $byte == 0xca ? 4 : 8;
         return 1;
     }
-
-    elsif ( ( $byte >= 0xa0 and $byte <= 0xbf ) or $byte == 0xda or $byte == 0xdb ) {
-        my $num;
-        if ( $byte == 0xda ) {
-            $num = unpack 'n', substr( $value, $p, 2 );
-            $p += 2;
-        }
-        elsif ( $byte == 0xdb ) {
-            $num = unpack 'N', substr( $value, $p, 4 );
-            $p += 4;
-        }
-        else { # fix raw
-            $num = $byte & ~0xa0;
-        }
-        $p += $num;
-        return 1;
-    }
-
     else {
-        _unexpected("byte 0x%02x", $byte);
+        Data::MessagePack::PP::_unexpected("byte 0x%02x", $byte);
     }
 
     return 0;
@@ -602,6 +609,6 @@ makamaka
 =head1 COPYRIGHT AND LICENSE
 
 This library is free software; you can redistribute it and/or modify
-it under the same terms as Perl itself. 
+it under the same terms as Perl itself.
 
 =cut

From 0017f4fce8f85f5d189fcca808221da06631c264 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Wed, 13 Oct 2010 12:46:41 +0900
Subject: [PATCH 0870/1648] Checking in changes prior to tagging of version
 0.33.

Changelog diff is:
---
 perl/lib/Data/MessagePack.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index d402167..9111db0 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.32';
+our $VERSION = '0.33';
 our $PreferInteger = 0;
 
 sub true () {

From 2d0511023900dfc56efcf88341129a947232c5d3 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sun, 17 Oct 2010 14:37:13 +0900
Subject: [PATCH 0871/1648] java: refactor DynamicCodeGenBase.java

---
 .../msgpack/util/codegen/DynamicCodeGenBase.java  | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 0f39427..7cb9048 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -36,20 +36,21 @@ public class DynamicCodeGenBase implements Constants {
 	public static interface NullChecker {
 		void setNullCheck(boolean nullCheck);
 	}
-	
+
 	public static class NullCheckerImpl implements NullChecker {
 		public boolean _$$_nullCheck = true;
 
 		public void setNullCheck(boolean _$$_check) {
 			_$$_nullCheck = _$$_check;
-		}		
+		}
 	}
-	
+
 	public static interface TemplateAccessor {
 		void setTemplates(Template[] templates);
 	}
-	
-	public static class TemplateTemplate extends NullCheckerImpl implements TemplateAccessor {
+
+	public static class TemplateTemplate extends NullCheckerImpl implements
+			TemplateAccessor {
 		public Template[] _$$_templates;
 
 		public void setTemplates(Template[] _$$_tmpls) {
@@ -128,7 +129,7 @@ public class DynamicCodeGenBase implements Constants {
 		// TODO
 		RED, BLUE
 	}
-	
+
 	public static void main(String[] args) throws Exception {
 		// TODO
 		class Foo {
@@ -339,7 +340,7 @@ public class DynamicCodeGenBase implements Constants {
 				CustomMessage.registerTemplate(c, tmpl);
 				return tmpl;
 			} else {
-				throw new DynamicCodeGenException("Type error: "
+				throw new MessageTypeException("Type error: "
 						+ ((Class) t).getName());
 			}
 		} else if (t instanceof GenericArrayType) {

From 17def94ba4c2ee1a748b60fb48b1066f39286a67 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sun, 17 Oct 2010 18:24:08 +0900
Subject: [PATCH 0872/1648] java: describe test programs for
 org.msgpack.template.*.java

---
 java/src/test/java/org/msgpack/Util.java      |  19 +
 .../org/msgpack/template/TestPackConvert.java | 320 +++++++++++++++++
 .../org/msgpack/template/TestPackUnpack.java  | 326 ++++++++++++++++++
 ...kerConverter.java => TestPackConvert.java} |   2 +-
 ...ackerUnpacker.java => TestPackUnpack.java} |   2 +-
 5 files changed, 667 insertions(+), 2 deletions(-)
 create mode 100644 java/src/test/java/org/msgpack/Util.java
 create mode 100644 java/src/test/java/org/msgpack/template/TestPackConvert.java
 create mode 100644 java/src/test/java/org/msgpack/template/TestPackUnpack.java
 rename java/src/test/java/org/msgpack/util/codegen/{TestDynamicCodeGenPackerConverter.java => TestPackConvert.java} (99%)
 rename java/src/test/java/org/msgpack/util/codegen/{TestDynamicCodeGenPackerUnpacker.java => TestPackUnpack.java} (99%)

diff --git a/java/src/test/java/org/msgpack/Util.java b/java/src/test/java/org/msgpack/Util.java
new file mode 100644
index 0000000..442b4d6
--- /dev/null
+++ b/java/src/test/java/org/msgpack/Util.java
@@ -0,0 +1,19 @@
+package org.msgpack;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.util.Iterator;
+
+public class Util {
+
+	public static MessagePackObject unpackOne(byte[] bytes) {
+		ByteArrayInputStream in = new ByteArrayInputStream(bytes);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertEquals(true, it.hasNext());
+		MessagePackObject obj = it.next();
+		assertEquals(false, it.hasNext());
+		return obj;
+	}
+}
diff --git a/java/src/test/java/org/msgpack/template/TestPackConvert.java b/java/src/test/java/org/msgpack/template/TestPackConvert.java
new file mode 100644
index 0000000..15237b6
--- /dev/null
+++ b/java/src/test/java/org/msgpack/template/TestPackConvert.java
@@ -0,0 +1,320 @@
+package org.msgpack.template;
+
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.junit.Test;
+import org.msgpack.MessagePackObject;
+import org.msgpack.Packer;
+import org.msgpack.Template;
+import org.msgpack.Util;
+
+import junit.framework.TestCase;
+
+public class TestPackConvert extends TestCase {
+
+	@Test
+	public void testInteger() throws Exception {
+		// _testInteger(null); // FIXME
+		_testInteger(0);
+		_testInteger(-1);
+		_testInteger(1);
+		_testInteger(Integer.MIN_VALUE);
+		_testInteger(Integer.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testInteger(rand.nextInt());
+		}
+	}
+
+	static void _testInteger(Integer src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = IntegerTemplate.getInstance();
+		Integer dst = (Integer) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
+	@Test
+	public void testLong() throws Exception {
+		// _testLong(null); // FIXME
+		_testLong((long) 0);
+		_testLong((long) -1);
+		_testLong((long) 1);
+		_testLong((long) Integer.MIN_VALUE);
+		_testLong((long) Integer.MAX_VALUE);
+		_testLong(Long.MIN_VALUE);
+		_testLong(Long.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testLong(rand.nextLong());
+		}
+	}
+
+	public void _testLong(Long src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = LongTemplate.getInstance();
+		Long dst = (Long) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
+	@Test
+	public void testBiginteger() throws Exception {
+		// _testBigInteger(null); // FIXME
+		_testBigInteger(BigInteger.valueOf(0));
+		_testBigInteger(BigInteger.valueOf(-1));
+		_testBigInteger(BigInteger.valueOf(1));
+		_testBigInteger(BigInteger.valueOf(Integer.MIN_VALUE));
+		_testBigInteger(BigInteger.valueOf(Integer.MAX_VALUE));
+		_testBigInteger(BigInteger.valueOf(Long.MIN_VALUE));
+		_testBigInteger(BigInteger.valueOf(Long.MAX_VALUE));
+		BigInteger max = BigInteger.valueOf(Long.MAX_VALUE).setBit(63);
+		_testBigInteger(max);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testBigInteger(max.subtract(BigInteger.valueOf(Math.abs(rand
+					.nextLong()))));
+		}
+	}
+
+	static void _testBigInteger(BigInteger src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = BigIntegerTemplate.getInstance();
+		BigInteger dst = (BigInteger) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
+	@Test
+	public void testFloat() throws Exception {
+		// _testFloat(null); // FIXME
+		_testFloat((float) 0.0);
+		_testFloat((float) -0.0);
+		_testFloat((float) 1.0);
+		_testFloat((float) -1.0);
+		_testFloat((float) Float.MAX_VALUE);
+		_testFloat((float) Float.MIN_VALUE);
+		_testFloat((float) Float.NaN);
+		_testFloat((float) Float.NEGATIVE_INFINITY);
+		_testFloat((float) Float.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testFloat(rand.nextFloat());
+		}
+	}
+
+	static void _testFloat(Float src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = FloatTemplate.getInstance();
+		Float dst = (Float) tmpl.convert(obj);
+		assertEquals(src, dst, 10e-10);
+	}
+
+	@Test
+	public void testDouble() throws Exception {
+		// _testDouble(null); // FIXME
+		_testDouble((double) 0.0);
+		_testDouble((double) -0.0);
+		_testDouble((double) 1.0);
+		_testDouble((double) -1.0);
+		_testDouble((double) Double.MAX_VALUE);
+		_testDouble((double) Double.MIN_VALUE);
+		_testDouble((double) Double.NaN);
+		_testDouble((double) Double.NEGATIVE_INFINITY);
+		_testDouble((double) Double.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testDouble(rand.nextDouble());
+		}
+	}
+
+	static void _testDouble(Double src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = DoubleTemplate.getInstance();
+		Double dst = (Double) tmpl.convert(obj);
+		assertEquals(src, dst, 10e-10);
+	}
+
+	@Test
+	public void testBoolean() throws Exception {
+		// _testBoolean(null); // FIXME
+		_testBoolean(false);
+		_testBoolean(true);
+	}
+
+	static void _testBoolean(Boolean src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = BooleanTemplate.getInstance();
+		Boolean dst = (Boolean) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
+	@Test
+	public void testString() throws Exception {
+		// _testString(null); // FIXME
+		_testString("");
+		_testString("a");
+		_testString("ab");
+		_testString("abc");
+
+		// small size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 31 + 1;
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+
+		// medium size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 100 + (1 << 15);
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+
+		// large size string
+		for (int i = 0; i < 10; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 100 + (1 << 31);
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+	}
+
+	static void _testString(String src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = StringTemplate.getInstance();
+		String dst = (String) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testList() throws Exception {
+		// nullList // FIXME
+		List emptyList = new ArrayList();
+		{
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(emptyList);
+			MessagePackObject obj = Util.unpackOne(out.toByteArray());
+			Template tmpl = new ListTemplate(IntegerTemplate.getInstance());
+			List dst = (List) tmpl.convert(obj);
+			assertEquals(emptyList, dst);
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			List l = new ArrayList();
+			int len = (int) Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++) {
+				l.add(j);
+			}
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(l);
+			MessagePackObject obj = Util.unpackOne(out.toByteArray());
+			Template tmpl = new ListTemplate(IntegerTemplate.getInstance());
+			List dst = (List) tmpl.convert(obj);
+			assertEquals(l.size(), dst.size());
+			for (int j = 0; j < len; j++) {
+				assertEquals(l.get(j), dst.get(j));
+			}
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			List l = new ArrayList();
+			int len = (int) Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++) {
+				l.add(Integer.toString(j));
+			}
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(l);
+			MessagePackObject obj = Util.unpackOne(out.toByteArray());
+			Template tmpl = new ListTemplate(StringTemplate.getInstance());
+			List dst = (List) tmpl.convert(obj);
+			assertEquals(l.size(), dst.size());
+			for (int j = 0; j < len; j++) {
+				assertEquals(l.get(j), dst.get(j));
+			}
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testMap() throws Exception {
+		// nullMap // FIXME
+		Map emptyMap = new HashMap();
+		{
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(emptyMap);
+			MessagePackObject obj = Util.unpackOne(out.toByteArray());
+			Template tmpl = new MapTemplate(IntegerTemplate.getInstance(),
+					IntegerTemplate.getInstance());
+			Map dst = (Map) tmpl
+					.convert(obj);
+			assertEquals(emptyMap, dst);
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			Map m = new HashMap();
+			int len = (int) Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++) {
+				m.put(j, j);
+			}
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(m);
+			MessagePackObject obj = Util.unpackOne(out.toByteArray());
+			Template tmpl = new MapTemplate(IntegerTemplate.getInstance(),
+					IntegerTemplate.getInstance());
+			Map map = (Map) tmpl
+					.convert(obj);
+			assertEquals(m.size(), map.size());
+			for (Map.Entry pair : map.entrySet()) {
+				Integer val = m.get(pair.getKey());
+				assertNotNull(val);
+				assertEquals(val, pair.getValue());
+			}
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			Map m = new HashMap();
+			int len = (int) Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++)
+				m.put(Integer.toString(j), j);
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(m);
+			MessagePackObject obj = Util.unpackOne(out.toByteArray());
+			Template tmpl = new MapTemplate(StringTemplate.getInstance(),
+					IntegerTemplate.getInstance());
+			Map map = (Map) tmpl.convert(obj);
+			assertEquals(m.size(), map.size());
+			for (Map.Entry pair : map.entrySet()) {
+				Integer val = m.get(pair.getKey());
+				assertNotNull(val);
+				assertEquals(val, pair.getValue());
+			}
+		}
+	}
+}
diff --git a/java/src/test/java/org/msgpack/template/TestPackUnpack.java b/java/src/test/java/org/msgpack/template/TestPackUnpack.java
new file mode 100644
index 0000000..fbd510f
--- /dev/null
+++ b/java/src/test/java/org/msgpack/template/TestPackUnpack.java
@@ -0,0 +1,326 @@
+package org.msgpack.template;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.junit.Test;
+import org.msgpack.Packer;
+import org.msgpack.Template;
+import org.msgpack.Unpacker;
+
+import junit.framework.TestCase;
+
+public class TestPackUnpack extends TestCase {
+	@Test
+	public void testInteger() throws Exception {
+		// _testInteger(null); // FIXME
+		_testInteger(0);
+		_testInteger(-1);
+		_testInteger(1);
+		_testInteger(Integer.MIN_VALUE);
+		_testInteger(Integer.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testInteger(rand.nextInt());
+		}
+	}
+
+	static void _testInteger(Integer src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = IntegerTemplate.getInstance();
+		Integer dst = (Integer) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	@Test
+	public void testLong() throws Exception {
+		// _testLong(null); // FIXME
+		_testLong((long) 0);
+		_testLong((long) -1);
+		_testLong((long) 1);
+		_testLong((long) Integer.MIN_VALUE);
+		_testLong((long) Integer.MAX_VALUE);
+		_testLong(Long.MIN_VALUE);
+		_testLong(Long.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testLong(rand.nextLong());
+		}
+	}
+
+	static void _testLong(Long src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = LongTemplate.getInstance();
+		Long dst = (Long) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	@Test
+	public void testBigInteger() throws Exception {
+		// _testBigInteger(null); // FIXME
+		_testBigInteger(BigInteger.valueOf(0));
+		_testBigInteger(BigInteger.valueOf(-1));
+		_testBigInteger(BigInteger.valueOf(1));
+		_testBigInteger(BigInteger.valueOf(Integer.MIN_VALUE));
+		_testBigInteger(BigInteger.valueOf(Integer.MAX_VALUE));
+		_testBigInteger(BigInteger.valueOf(Long.MIN_VALUE));
+		_testBigInteger(BigInteger.valueOf(Long.MAX_VALUE));
+		BigInteger max = BigInteger.valueOf(Long.MAX_VALUE).setBit(63);
+		_testBigInteger(max);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testBigInteger(max.subtract(BigInteger.valueOf(Math.abs(rand
+					.nextLong()))));
+		}
+	}
+
+	static void _testBigInteger(BigInteger src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack((Object) src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = BigIntegerTemplate.getInstance();
+		BigInteger dst = (BigInteger) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	@Test
+	public void testFloat() throws Exception {
+		// _testFloat(null); // FIXME
+		_testFloat((float) 0.0);
+		_testFloat((float) -0.0);
+		_testFloat((float) 1.0);
+		_testFloat((float) -1.0);
+		_testFloat((float) Float.MAX_VALUE);
+		_testFloat((float) Float.MIN_VALUE);
+		_testFloat((float) Float.NaN);
+		_testFloat((float) Float.NEGATIVE_INFINITY);
+		_testFloat((float) Float.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testFloat(rand.nextFloat());
+		}
+	}
+
+	static void _testFloat(Float src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = FloatTemplate.getInstance();
+		Float dst = (Float) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst, 10e-10);
+	}
+
+	@Test
+	public void testDouble() throws Exception {
+		// _testDouble(null); // FIXME
+		_testDouble((double) 0.0);
+		_testDouble((double) -0.0);
+		_testDouble((double) 1.0);
+		_testDouble((double) -1.0);
+		_testDouble((double) Double.MAX_VALUE);
+		_testDouble((double) Double.MIN_VALUE);
+		_testDouble((double) Double.NaN);
+		_testDouble((double) Double.NEGATIVE_INFINITY);
+		_testDouble((double) Double.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testDouble(rand.nextDouble());
+		}
+	}
+
+	static void _testDouble(Double src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DoubleTemplate.getInstance();
+		Double dst = (Double) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst, 10e-10);
+	}
+
+	@Test
+	public void testBoolean() throws Exception {
+		// _testBoolean(null); // FIXME
+		_testBoolean(false);
+		_testBoolean(true);
+	}
+
+	static void _testBoolean(Boolean src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = BooleanTemplate.getInstance();
+		Boolean dst = (Boolean) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	@Test
+	public void testString() throws Exception {
+		// _testString(null); // FIXME
+		_testString("");
+		_testString("a");
+		_testString("ab");
+		_testString("abc");
+
+		// small size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 31 + 1;
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+
+		// medium size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 100 + (1 << 15);
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+
+		// large size string
+		for (int i = 0; i < 10; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 100 + (1 << 31);
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+	}
+
+	static void _testString(String src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = StringTemplate.getInstance();
+		String dst = (String) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testList() throws Exception {
+		// nullList // FIXME
+		List emptyList = new ArrayList();
+		{
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(emptyList);
+			ByteArrayInputStream in = new ByteArrayInputStream(out
+					.toByteArray());
+			Template tmpl = new ListTemplate(IntegerTemplate.getInstance());
+			List dst = (List) tmpl.unpack(new Unpacker(in));
+			assertEquals(emptyList, dst);
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			List l = new ArrayList();
+			int len = (int) Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++) {
+				l.add(j);
+			}
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(l);
+			ByteArrayInputStream in = new ByteArrayInputStream(out
+					.toByteArray());
+			Template tmpl = new ListTemplate(IntegerTemplate.getInstance());
+			List dst = (List) tmpl.unpack(new Unpacker(in));
+			assertEquals(len, dst.size());
+			for (int j = 0; j < len; j++) {
+				assertEquals(l.get(j), dst.get(j));
+			}
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			List l = new ArrayList();
+			int len = (int) Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++)
+				l.add(Integer.toString(j));
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(l);
+			ByteArrayInputStream in = new ByteArrayInputStream(out
+					.toByteArray());
+			Template tmpl = new ListTemplate(StringTemplate.getInstance());
+			List dst = (List) tmpl.unpack(new Unpacker(in));
+			assertEquals(len, dst.size());
+			for (int j = 0; j < len; j++) {
+				assertEquals(l.get(j), dst.get(j));
+			}
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testMap() throws Exception {
+		// nullMap // FIXME
+		Map emptyMap = new HashMap();
+		{
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(emptyMap);
+			ByteArrayInputStream in = new ByteArrayInputStream(out
+					.toByteArray());
+			Template tmpl = new MapTemplate(IntegerTemplate.getInstance(),
+					IntegerTemplate.getInstance());
+			Map dst = (Map) tmpl
+					.unpack(new Unpacker(in));
+			assertEquals(emptyMap, dst);
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			Map m = new HashMap();
+			int len = (int) Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++) {
+				m.put(j, j);
+			}
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(m);
+			ByteArrayInputStream in = new ByteArrayInputStream(out
+					.toByteArray());
+			Template tmpl = new MapTemplate(IntegerTemplate.getInstance(),
+					IntegerTemplate.getInstance());
+			Map map = (Map) tmpl
+					.unpack(new Unpacker(in));
+			assertEquals(len, map.size());
+			for (Map.Entry pair : map.entrySet()) {
+				Integer val = m.get(pair.getKey());
+				assertNotNull(val);
+				assertEquals(val, pair.getValue());
+			}
+		}
+
+		for (int i = 0; i < 1000; i++) {
+			Map m = new HashMap();
+			int len = (int) Math.random() % 1000 + 1;
+			for (int j = 0; j < len; j++) {
+				m.put(Integer.toString(j), j);
+			}
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			new Packer(out).pack(m);
+			ByteArrayInputStream in = new ByteArrayInputStream(out
+					.toByteArray());
+			Template tmpl = new MapTemplate(StringTemplate.getInstance(),
+					IntegerTemplate.getInstance());
+			Map map = (Map) tmpl
+					.unpack(new Unpacker(in));
+			assertEquals(m.size(), map.size());
+			for (Map.Entry pair : map.entrySet()) {
+				Integer val = m.get(pair.getKey());
+				assertNotNull(val);
+				assertEquals(val, pair.getValue());
+			}
+		}
+	}
+}
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java b/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
similarity index 99%
rename from java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
rename to java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
index bad91ce..40fed22 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerConverter.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
@@ -23,7 +23,7 @@ import org.msgpack.Unpacker;
 import org.msgpack.annotation.MessagePackMessage;
 import org.msgpack.annotation.MessagePackOrdinalEnum;
 
-public class TestDynamicCodeGenPackerConverter extends TestCase {
+public class TestPackConvert extends TestCase {
 
 	@Test
 	public void testPrimitiveTypeField00s() throws Exception {
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
similarity index 99%
rename from java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
rename to java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
index 27f9adb..3222bec 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestDynamicCodeGenPackerUnpacker.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
@@ -22,7 +22,7 @@ import org.msgpack.annotation.MessagePackOrdinalEnum;
 
 import junit.framework.TestCase;
 
-public class TestDynamicCodeGenPackerUnpacker extends TestCase {
+public class TestPackUnpack extends TestCase {
 
 	@Test
 	public void testPrimitiveTypeFields00() throws Exception {

From d6c2a97931b5c71142b9923fe5b8359f32707485 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sun, 17 Oct 2010 18:57:00 +0900
Subject: [PATCH 0873/1648] java: write test programs for
 org.msgpack.packer.*.java

---
 .../org/msgpack/packer/TestPackConvert.java   | 201 +++++++++++++++++
 .../org/msgpack/packer/TestPackUnpack.java    | 208 ++++++++++++++++++
 2 files changed, 409 insertions(+)
 create mode 100644 java/src/test/java/org/msgpack/packer/TestPackConvert.java
 create mode 100644 java/src/test/java/org/msgpack/packer/TestPackUnpack.java

diff --git a/java/src/test/java/org/msgpack/packer/TestPackConvert.java b/java/src/test/java/org/msgpack/packer/TestPackConvert.java
new file mode 100644
index 0000000..98caf82
--- /dev/null
+++ b/java/src/test/java/org/msgpack/packer/TestPackConvert.java
@@ -0,0 +1,201 @@
+package org.msgpack.packer;
+
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+import org.msgpack.MessagePackObject;
+import org.msgpack.MessagePacker;
+import org.msgpack.Packer;
+import org.msgpack.Util;
+
+public class TestPackConvert extends TestCase {
+	@Test
+	public void testInteger() throws Exception {
+		// _testInteger(null); // FIXME
+		_testInteger(0);
+		_testInteger(-1);
+		_testInteger(1);
+		_testInteger(Integer.MIN_VALUE);
+		_testInteger(Integer.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testInteger(rand.nextInt());
+		}
+	}
+
+	static void _testInteger(Integer src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = IntegerPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		assertEquals(src.intValue(), obj.asInt());
+	}
+
+	@Test
+	public void testLong() throws Exception {
+		// _testLong((null); // FIXME
+		_testLong((long) 0);
+		_testLong((long) -1);
+		_testLong((long) 1);
+		_testLong((long) Integer.MIN_VALUE);
+		_testLong((long) Integer.MAX_VALUE);
+		_testLong(Long.MIN_VALUE);
+		_testLong(Long.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testLong(rand.nextLong());
+		}
+	}
+
+	static void _testLong(Long src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = LongPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		assertEquals(src.longValue(), obj.asLong());
+	}
+
+	@Test
+	public void testBigInteger() throws Exception {
+		// _testBigInteger(null); // FIXME
+		_testBigInteger(BigInteger.valueOf(0));
+		_testBigInteger(BigInteger.valueOf(-1));
+		_testBigInteger(BigInteger.valueOf(1));
+		_testBigInteger(BigInteger.valueOf(Integer.MIN_VALUE));
+		_testBigInteger(BigInteger.valueOf(Integer.MAX_VALUE));
+		_testBigInteger(BigInteger.valueOf(Long.MIN_VALUE));
+		_testBigInteger(BigInteger.valueOf(Long.MAX_VALUE));
+		BigInteger max = BigInteger.valueOf(Long.MAX_VALUE).setBit(63);
+		_testBigInteger(max);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testBigInteger(max.subtract(BigInteger.valueOf(Math.abs(rand
+					.nextLong()))));
+		}
+	}
+
+	static void _testBigInteger(BigInteger src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = BigIntegerPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		assertEquals(src, obj.asBigInteger());
+	}
+
+	@Test
+	public void testFloat() throws Exception {
+		// _testFloat(null); // FIXME
+		_testFloat((float) 0.0);
+		_testFloat((float) -0.0);
+		_testFloat((float) 1.0);
+		_testFloat((float) -1.0);
+		_testFloat((float) Float.MAX_VALUE);
+		_testFloat((float) Float.MIN_VALUE);
+		_testFloat((float) Float.NaN);
+		_testFloat((float) Float.NEGATIVE_INFINITY);
+		_testFloat((float) Float.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testFloat(rand.nextFloat());
+		}
+	}
+
+	static void _testFloat(Float src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = FloatPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		assertEquals(src.floatValue(), obj.asFloat(), 10e-10);
+	}
+
+	@Test
+	public void testDouble() throws Exception {
+		// _testDouble(null); // FIXME
+		_testDouble((double) 0.0);
+		_testDouble((double) -0.0);
+		_testDouble((double) 1.0);
+		_testDouble((double) -1.0);
+		_testDouble((double) Double.MAX_VALUE);
+		_testDouble((double) Double.MIN_VALUE);
+		_testDouble((double) Double.NaN);
+		_testDouble((double) Double.NEGATIVE_INFINITY);
+		_testDouble((double) Double.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			_testDouble(rand.nextDouble());
+	}
+
+	static void _testDouble(Double src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DoublePacker.getInstance();
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		assertEquals(src.doubleValue(), obj.asDouble(), 10e-10);
+	}
+
+	@Test
+	public void testBoolean() throws Exception {
+		// _testBoolean(null); // FIXME
+		_testBoolean(false);
+		_testBoolean(true);
+	}
+
+	static void _testBoolean(Boolean src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = BooleanPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		assertEquals(src.booleanValue(), obj.asBoolean());
+	}
+
+	@Test
+	public void testString() throws Exception {
+		// _testString(null); // FIXME
+		_testString("");
+		_testString("a");
+		_testString("ab");
+		_testString("abc");
+
+		// small size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 31 + 1;
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+
+		// medium size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 100 + (1 << 15);
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+
+		// large size string
+		for (int i = 0; i < 10; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 100 + (1 << 31);
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+	}
+
+	static void _testString(String src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = StringPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		assertEquals(src, obj.asString());
+	}
+}
diff --git a/java/src/test/java/org/msgpack/packer/TestPackUnpack.java b/java/src/test/java/org/msgpack/packer/TestPackUnpack.java
new file mode 100644
index 0000000..f1d7842
--- /dev/null
+++ b/java/src/test/java/org/msgpack/packer/TestPackUnpack.java
@@ -0,0 +1,208 @@
+package org.msgpack.packer;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+import org.msgpack.MessagePacker;
+import org.msgpack.Packer;
+import org.msgpack.Unpacker;
+
+public class TestPackUnpack extends TestCase {
+	@Test
+	public void testInteger() throws Exception {
+		// _testInteger(null); // FIXME
+		_testInteger(0);
+		_testInteger(-1);
+		_testInteger(1);
+		_testInteger(Integer.MIN_VALUE);
+		_testInteger(Integer.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testInteger(rand.nextInt());
+		}
+	}
+
+	static void _testInteger(Integer src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = IntegerPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker unpacker = new Unpacker(in);
+		assertEquals(src.intValue(), unpacker.unpackInt());
+	}
+
+	@Test
+	public void testLong() throws Exception {
+		// _testLong((null); // FIXME
+		_testLong((long) 0);
+		_testLong((long) -1);
+		_testLong((long) 1);
+		_testLong((long) Integer.MIN_VALUE);
+		_testLong((long) Integer.MAX_VALUE);
+		_testLong(Long.MIN_VALUE);
+		_testLong(Long.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testLong(rand.nextLong());
+		}
+	}
+
+	static void _testLong(Long src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = LongPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker unpacker = new Unpacker(in);
+		assertEquals(src.longValue(), unpacker.unpackLong());
+	}
+
+	@Test
+	public void testBigInteger() throws Exception {
+		// _testBigInteger(null); // FIXME
+		_testBigInteger(BigInteger.valueOf(0));
+		_testBigInteger(BigInteger.valueOf(-1));
+		_testBigInteger(BigInteger.valueOf(1));
+		_testBigInteger(BigInteger.valueOf(Integer.MIN_VALUE));
+		_testBigInteger(BigInteger.valueOf(Integer.MAX_VALUE));
+		_testBigInteger(BigInteger.valueOf(Long.MIN_VALUE));
+		_testBigInteger(BigInteger.valueOf(Long.MAX_VALUE));
+		BigInteger max = BigInteger.valueOf(Long.MAX_VALUE).setBit(63);
+		_testBigInteger(max);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testBigInteger(max.subtract(BigInteger.valueOf(Math.abs(rand
+					.nextLong()))));
+		}
+	}
+
+	static void _testBigInteger(BigInteger src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = BigIntegerPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker unpacker = new Unpacker(in);
+		assertEquals(src, unpacker.unpackBigInteger());
+	}
+
+	@Test
+	public void testFloat() throws Exception {
+		// _testFloat(null); // FIXME
+		_testFloat((float) 0.0);
+		_testFloat((float) -0.0);
+		_testFloat((float) 1.0);
+		_testFloat((float) -1.0);
+		_testFloat((float) Float.MAX_VALUE);
+		_testFloat((float) Float.MIN_VALUE);
+		_testFloat((float) Float.NaN);
+		_testFloat((float) Float.NEGATIVE_INFINITY);
+		_testFloat((float) Float.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testFloat(rand.nextFloat());
+		}
+	}
+
+	static void _testFloat(Float src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = FloatPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker unpacker = new Unpacker(in);
+		assertEquals(src.floatValue(), unpacker.unpackFloat(), 10e-10);
+	}
+
+	@Test
+	public void testDouble() throws Exception {
+		// _testDouble(null); // FIXME
+		_testDouble((double) 0.0);
+		_testDouble((double) -0.0);
+		_testDouble((double) 1.0);
+		_testDouble((double) -1.0);
+		_testDouble((double) Double.MAX_VALUE);
+		_testDouble((double) Double.MIN_VALUE);
+		_testDouble((double) Double.NaN);
+		_testDouble((double) Double.NEGATIVE_INFINITY);
+		_testDouble((double) Double.POSITIVE_INFINITY);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++)
+			_testDouble(rand.nextDouble());
+	}
+
+	static void _testDouble(Double src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DoublePacker.getInstance();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker unpacker = new Unpacker(in);
+		assertEquals(src.doubleValue(), unpacker.unpackDouble(), 10e-10);
+	}
+
+	@Test
+	public void testBoolean() throws Exception {
+		// _testBoolean(null); // FIXME
+		_testBoolean(false);
+		_testBoolean(true);
+	}
+
+	static void _testBoolean(Boolean src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = BooleanPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker unpacker = new Unpacker(in);
+		assertEquals(src.booleanValue(), unpacker.unpackBoolean());
+	}
+
+	@Test
+	public void testString() throws Exception {
+		// _testString(null); // FIXME
+		_testString("");
+		_testString("a");
+		_testString("ab");
+		_testString("abc");
+
+		// small size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 31 + 1;
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+
+		// medium size string
+		for (int i = 0; i < 100; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 100 + (1 << 15);
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+
+		// large size string
+		for (int i = 0; i < 10; i++) {
+			StringBuilder sb = new StringBuilder();
+			int len = (int) Math.random() % 100 + (1 << 31);
+			for (int j = 0; j < len; j++) {
+				sb.append('a' + ((int) Math.random()) & 26);
+			}
+			_testString(sb.toString());
+		}
+	}
+
+	static void _testString(String src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = StringPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker unpacker = new Unpacker(in);
+		assertEquals(src, unpacker.unpackString());
+	}
+}

From ad855339759b4cc5a3bf2cb77bda475b5029103f Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sun, 17 Oct 2010 19:23:42 +0900
Subject: [PATCH 0874/1648] java: add a log4j.properties file for unit testing

---
 java/src/test/resources/log4j.properties | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 java/src/test/resources/log4j.properties

diff --git a/java/src/test/resources/log4j.properties b/java/src/test/resources/log4j.properties
new file mode 100644
index 0000000..4e8e88d
--- /dev/null
+++ b/java/src/test/resources/log4j.properties
@@ -0,0 +1,14 @@
+### direct log messages to stdout ###
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d %5p %c{1} - %m%n
+
+### direct messages to file mylog.log ###
+log4j.appender.file=org.apache.log4j.FileAppender
+log4j.appender.file.File=mylog.log
+log4j.appender.file.Append=true
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d %5p %c{1} - %m%n
+
+log4j.rootLogger=off, stdout

From 3e164eab26a985e990d0772ec306aff8451dfbe7 Mon Sep 17 00:00:00 2001
From: Kazuho Oku 
Date: Mon, 18 Oct 2010 19:22:28 +0900
Subject: [PATCH 0875/1648] do not use the corrupt my_snprintf(%ll[du]) on
 win32

---
 perl/xs-src/unpack.c | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index e997af5..fa09480 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -121,12 +121,41 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const
     return 0;
 }
 
+static const char* str_from_uint64(char* buf_end, uint64_t v)
+{
+  char *p = buf_end;
+  *--p = '\0';
+  if (v == 0) {
+    *--p = '0';
+  } else {
+    do {
+      *--p = '0' + v % 10;
+      v /= 10;
+    } while (v != 0);
+  }
+  return p;
+}
+
+static const char* str_from_int64(char* buf_end, int64_t v)
+{
+  int minus = v < 0;
+  char* p = str_from_uint64(buf_end, minus ? -v : v);
+  if (minus)
+    *--p = '-';
+  return p;
+}
+
 static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
 {
     dTHX;
     char tbuf[64];
+#if 1 /* workaround for win32-32bit (my_snprintf(%%d) returns are incorrect) */
+    char* s = str_from_uint64(tbuf + sizeof(tbuf), d);
+    *o = newSVpvn(s, tbuf + sizeof(tbuf) - 1 - s);
+#else
     STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d);
     *o = newSVpvn(tbuf, len);
+#endif
     return 0;
 }
 
@@ -134,8 +163,13 @@ static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t cons
 {
     dTHX;
     char tbuf[64];
+#if 1 /* workaround for win32-32bit (my_snprintf(%%d) returns are incorrect) */
+    char* s = str_from_int64(tbuf + sizeof(tbuf), d);
+    *o = newSVpvn(s, tbuf + sizeof(tbuf) - 1 - s);
+#else
     STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d);
     *o = newSVpvn(tbuf, len);
+#endif
     return 0;
 }
 

From 5395b62f2ffb4e9ecf96fa8ef737ab53a885fd0c Mon Sep 17 00:00:00 2001
From: Kazuho Oku 
Date: Mon, 18 Oct 2010 19:26:24 +0900
Subject: [PATCH 0876/1648] shorter code, fix comment

---
 perl/xs-src/unpack.c | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index fa09480..60ffd87 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -125,14 +125,9 @@ static const char* str_from_uint64(char* buf_end, uint64_t v)
 {
   char *p = buf_end;
   *--p = '\0';
-  if (v == 0) {
-    *--p = '0';
-  } else {
-    do {
-      *--p = '0' + v % 10;
-      v /= 10;
-    } while (v != 0);
-  }
+  do {
+    *--p = '0' + v % 10;
+  } while ((v /= 10) != 0);
   return p;
 }
 
@@ -149,7 +144,7 @@ static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t co
 {
     dTHX;
     char tbuf[64];
-#if 1 /* workaround for win32-32bit (my_snprintf(%%d) returns are incorrect) */
+#if 1 /* workaround for win32-32bit (my_snprintf(%llu) returns are incorrect) */
     char* s = str_from_uint64(tbuf + sizeof(tbuf), d);
     *o = newSVpvn(s, tbuf + sizeof(tbuf) - 1 - s);
 #else
@@ -163,7 +158,7 @@ static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t cons
 {
     dTHX;
     char tbuf[64];
-#if 1 /* workaround for win32-32bit (my_snprintf(%%d) returns are incorrect) */
+#if 1 /* workaround for win32-32bit (my_snprintf(%lld) returns are incorrect) */
     char* s = str_from_int64(tbuf + sizeof(tbuf), d);
     *o = newSVpvn(s, tbuf + sizeof(tbuf) - 1 - s);
 #else

From 371ba3ffe3cc84c471468c1b91195908944a6c80 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Mon, 18 Oct 2010 19:30:08 +0900
Subject: [PATCH 0877/1648] Checking in changes prior to tagging of version
 0.34.

Changelog diff is:

diff --git a/perl/Changes b/perl/Changes
index 50177f4..486f1c3 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,6 @@
+0.34
+
+    - do not use the corrupt my_snprintf(%ll[du]) on win32(kazuho)

 0.33
---
 perl/Changes                 | 3 +++
 perl/README                  | 2 ++
 perl/lib/Data/MessagePack.pm | 4 +++-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/perl/Changes b/perl/Changes
index 50177f4..486f1c3 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,6 @@
+0.34
+
+    - do not use the corrupt my_snprintf(%ll[du]) on win32(kazuho)
 
 0.33
 
diff --git a/perl/README b/perl/README
index ae924ee..3f25f70 100644
--- a/perl/README
+++ b/perl/README
@@ -123,6 +123,8 @@ THANKS TO
 
     hanekomu
 
+    Kazuho Oku
+
 LICENSE
     This library is free software; you can redistribute it and/or modify it
     under the same terms as Perl itself.
diff --git a/perl/lib/Data/MessagePack.pm b/perl/lib/Data/MessagePack.pm
index 9111db0..afe37af 100644
--- a/perl/lib/Data/MessagePack.pm
+++ b/perl/lib/Data/MessagePack.pm
@@ -3,7 +3,7 @@ use strict;
 use warnings;
 use 5.008001;
 
-our $VERSION = '0.33';
+our $VERSION = '0.34';
 our $PreferInteger = 0;
 
 sub true () {
@@ -191,6 +191,8 @@ FURUHASHI Sadayuki
 
 hanekomu
 
+Kazuho Oku
+
 =head1 LICENSE
 
 This library is free software; you can redistribute it and/or modify

From 65515638aae2d9ab678a67d9f8edd0ea8ffe4457 Mon Sep 17 00:00:00 2001
From: tokuhirom 
Date: Tue, 19 Oct 2010 13:45:44 +0900
Subject: [PATCH 0878/1648] perl: oops! added settings for shipit

---
 perl/.shipit | 1 +
 1 file changed, 1 insertion(+)

diff --git a/perl/.shipit b/perl/.shipit
index 7b4dbc0..3a66936 100644
--- a/perl/.shipit
+++ b/perl/.shipit
@@ -1,3 +1,4 @@
 steps = FindVersion, ChangeVersion, CheckChangeLog, DistTest, Commit, Tag, MakeDist, UploadCPAN
 MakeDist.destination=~/.shipit-dist/
 git.tagpattern = perl-%v
+git.push_to = origin

From 4067c56b5dde019d945c4d16acebf033f3c97403 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Wed, 20 Oct 2010 16:32:01 +0900
Subject: [PATCH 0879/1648] java: supports Collection to pack

---
 java/src/main/java/org/msgpack/Packer.java    | 6 ++++++
 java/src/main/java/org/msgpack/Templates.java | 5 +++++
 2 files changed, 11 insertions(+)

diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index 704481d..a10bf19 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
 import java.util.List;
 import java.util.Set;
 import java.util.Map;
+import java.util.Collection;
 import java.math.BigInteger;
 
 import org.msgpack.annotation.MessagePackDelegate;
@@ -480,6 +481,11 @@ public class Packer {
 				pack(e.getValue());
 			}
 			return this;
+		} else if(o instanceof Collection) {
+			Collection l = (Collection)o;
+			packArray(l.size());
+			for(Object i : l) { pack(i); }
+			return this;
 		//} else if(o instanceof Boolean) {
 		//	if((Boolean)o) {
 		//		return packTrue();
diff --git a/java/src/main/java/org/msgpack/Templates.java b/java/src/main/java/org/msgpack/Templates.java
index b693840..270f59b 100644
--- a/java/src/main/java/org/msgpack/Templates.java
+++ b/java/src/main/java/org/msgpack/Templates.java
@@ -30,6 +30,10 @@ public class Templates {
 		return new MapTemplate(keyTemplate, valueTemplate);
 	}
 
+	public static Template tCollection(Template elementTemplate) {
+		return new CollectionTemplate(elementTemplate);
+	}
+
 	public static Template tClass(Class target) {
 		return new ClassTemplate(target);
 	}
@@ -84,5 +88,6 @@ public class Templates {
 	public static Template tByteArray() {
 		return TByteArray;
 	}
+
 }
 

From 2065affd45fbf74bdbeccee4cbd86aad4b321c7e Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Wed, 20 Oct 2010 16:40:25 +0900
Subject: [PATCH 0880/1648] java: adds CollectionTemplate

---
 .../msgpack/template/CollectionTemplate.java  | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 java/src/main/java/org/msgpack/template/CollectionTemplate.java

diff --git a/java/src/main/java/org/msgpack/template/CollectionTemplate.java b/java/src/main/java/org/msgpack/template/CollectionTemplate.java
new file mode 100644
index 0000000..0c9ff17
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/CollectionTemplate.java
@@ -0,0 +1,50 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.io.IOException;
+import org.msgpack.*;
+
+public class CollectionTemplate implements Template {
+	private Template elementTemplate;
+
+	public CollectionTemplate(Template elementTemplate) {
+		this.elementTemplate = elementTemplate;
+	}
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		int length = pac.unpackArray();
+		List list = new LinkedList();
+		for(; length > 0; length--) {
+			list.add( elementTemplate.unpack(pac) );
+		}
+		return list;
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		MessagePackObject[] array = from.asArray();
+		List list = new LinkedList();
+		for(MessagePackObject element : array) {
+			list.add( elementTemplate.convert(element) );
+		}
+		return list;
+	}
+}
+

From 64711e615e78328cee3250e83db0527f1c33c0f7 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Wed, 20 Oct 2010 21:04:37 +0900
Subject: [PATCH 0881/1648] java: write test programs for
 org.msgpack.util.codegen.*.java

---
 .../util/codegen/DynamicCodeGenBase.java      |  66 ++++------
 .../org/msgpack/util/codegen/FieldOption.java |  10 ++
 .../msgpack/util/codegen/TestPackConvert.java | 121 ++++++++++++++++++
 .../msgpack/util/codegen/TestPackUnpack.java  | 104 +++++++++++++++
 4 files changed, 257 insertions(+), 44 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/util/codegen/FieldOption.java

diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 7cb9048..f1415d5 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -33,6 +33,9 @@ import org.slf4j.LoggerFactory;
 
 public class DynamicCodeGenBase implements Constants {
 
+	private static Logger LOG = LoggerFactory
+			.getLogger(DynamicCodeGenBase.class);
+
 	public static interface NullChecker {
 		void setNullCheck(boolean nullCheck);
 	}
@@ -58,9 +61,6 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	private static Logger LOG = LoggerFactory
-			.getLogger(DynamicCodeGenBase.class);
-
 	private static AtomicInteger COUNTER = new AtomicInteger(0);
 
 	protected static int inc() {
@@ -74,80 +74,58 @@ public class DynamicCodeGenBase implements Constants {
 	}
 
 	protected void checkTypeValidation(Class type) {
-		DynamicCodeGenException e = new DynamicCodeGenException("Fatal error: "
-				+ type.getName());
+		DynamicCodeGenException e = new DynamicCodeGenException(String.format(
+				"Fatal error: %s", new Object[] { type.getName() }));
 		LOG.error(e.getMessage(), e);
 		throw e;
 	}
 
-	protected void throwTypeValidationException(Class origClass,
-			String message) throws DynamicCodeGenException {
-		DynamicCodeGenException e = new DynamicCodeGenException(message + ": "
-				+ origClass.getName());
+	protected void throwTypeValidationException(Class type, String message)
+			throws DynamicCodeGenException {
+		DynamicCodeGenException e = new DynamicCodeGenException(String.format(
+				"%s: %s", new Object[] { message, type.getName() }));
 		LOG.error(e.getMessage(), e);
 		throw e;
 	}
 
 	protected void checkDefaultConstructorValidation(Class type) {
-		DynamicCodeGenException e = new DynamicCodeGenException("Fatal error: "
-				+ type.getName());
+		DynamicCodeGenException e = new DynamicCodeGenException(String.format(
+				"Fatal error: %s", new Object[] { type.getName() }));
 		LOG.error(e.getMessage(), e);
 		throw e;
 	}
 
 	protected void throwConstructorValidationException(Class origClass) {
-		DynamicCodeGenException e = new DynamicCodeGenException(
-				"it must have a public zero-argument constructor: "
-						+ origClass.getName());
+		DynamicCodeGenException e = new DynamicCodeGenException(String.format(
+				"it must have a public zero-argument constructor: %s",
+				new Object[] { origClass.getName() }));
 		LOG.error(e.getMessage(), e);
 		throw e;
 	}
 
-	protected void throwFieldValidationException(Field f) {
-		DynamicCodeGenException e = new DynamicCodeGenException(
-				"it must be a public field: " + f.getName());
+	protected void throwFieldValidationException(Field field) {
+		DynamicCodeGenException e = new DynamicCodeGenException(String.format(
+				"it must be a public field: %s",
+				new Object[] { field.getName() }));
 		LOG.debug(e.getMessage(), e);
 		throw e;
 	}
 
 	protected static void throwMethodValidationException(Method method,
 			String message) throws DynamicCodeGenException {
-		DynamicCodeGenException e = new DynamicCodeGenException(message + ": "
-				+ method.getName());
+		DynamicCodeGenException e = new DynamicCodeGenException(String.format(
+				"%s: %s", new Object[] { message, method.getName() }));
 		LOG.error(e.getMessage(), e);
 		throw e;
 	}
 
 	protected CtClass makeClass(String name) throws NotFoundException {
-		DynamicCodeGenException e = new DynamicCodeGenException("Fatal error: "
-				+ name);
+		DynamicCodeGenException e = new DynamicCodeGenException(String.format(
+				"Fatal error: %s", new Object[] { name }));
 		LOG.error(e.getMessage(), e);
 		throw e;
 	}
 
-	enum Color {
-		// TODO
-		RED, BLUE
-	}
-
-	public static void main(String[] args) throws Exception {
-		// TODO
-		class Foo {
-		}
-		class Bar extends Foo {
-		}
-		Color c = Color.RED;
-		Color c1 = null;
-
-		ClassPool pool = ClassPool.getDefault();
-		CtClass barCtClass = pool.get(Bar.class.getName());
-		CtClass sbarCtClass = barCtClass.getSuperclass();
-		System.out.println("bar: " + sbarCtClass.getName());
-		CtClass fooCtClass = pool.get(Foo.class.getName());
-		CtClass sfooCtClass = fooCtClass.getSuperclass();
-		System.out.println("foo: " + sfooCtClass.getName());
-	}
-
 	protected void setSuperclass(CtClass newCtClass, Class superClass)
 			throws NotFoundException, CannotCompileException {
 		// check the specified super class
diff --git a/java/src/main/java/org/msgpack/util/codegen/FieldOption.java b/java/src/main/java/org/msgpack/util/codegen/FieldOption.java
new file mode 100644
index 0000000..723a765
--- /dev/null
+++ b/java/src/main/java/org/msgpack/util/codegen/FieldOption.java
@@ -0,0 +1,10 @@
+package org.msgpack.util.codegen;
+
+import org.msgpack.Template;
+
+public class FieldOption {
+
+    public String name;
+    
+    public Template tmpl;
+}
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java b/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
index 40fed22..2e6ea58 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
@@ -2,6 +2,7 @@ package org.msgpack.util.codegen;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -15,8 +16,11 @@ import org.junit.Test;
 import org.msgpack.CustomConverter;
 import org.msgpack.CustomPacker;
 import org.msgpack.CustomUnpacker;
+import org.msgpack.MessageConvertable;
 import org.msgpack.MessagePackObject;
+import org.msgpack.MessagePackable;
 import org.msgpack.MessagePacker;
+import org.msgpack.MessageTypeException;
 import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
@@ -1007,4 +1011,121 @@ public class TestPackConvert extends TestCase {
 		public SampleSuperClass() {
 		}
 	}
+
+	//@Test
+	public void XtestMessagePackableUnpackableClass00() throws Exception {
+		BaseMessagePackableConvertableClass src = new BaseMessagePackableConvertableClass();
+		MessagePackableConvertableClass src1 = new MessagePackableConvertableClass();
+		List src2 = new ArrayList();
+		src1.f0 = 0;
+		src1.f1 = 1;
+		src.f0 = src1;
+		src.f1 = 1;
+		src2.add(src1);
+		src.f2 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(BaseMessagePackableConvertableClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(BaseMessagePackableConvertableClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		BaseMessagePackableConvertableClass dst = (BaseMessagePackableConvertableClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0.f0, dst.f0.f0);
+		assertEquals(src.f0.f1, dst.f0.f1);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertEquals(src.f2.get(0).f0, dst.f2.get(0).f0);
+		assertEquals(src.f2.get(0).f1, dst.f2.get(0).f1);
+		assertFalse(it.hasNext());
+	}
+
+	//@Test
+	public void XtestMessagePackableUnpackableClass01() throws Exception {
+		BaseMessagePackableConvertableClass src = new BaseMessagePackableConvertableClass();
+		src.f0 = null;
+		src.f1 = 1;
+		src.f2 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(BaseMessagePackableConvertableClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(BaseMessagePackableConvertableClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		BaseMessagePackableConvertableClass dst = (BaseMessagePackableConvertableClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertFalse(it.hasNext());
+	}
+
+	//@Test
+	public void XtestMessagePackableUnpackableClass02() throws Exception {
+		BaseMessagePackableConvertableClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(BaseMessagePackableConvertableClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(BaseMessagePackableConvertableClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		BaseMessagePackableConvertableClass dst = (BaseMessagePackableConvertableClass) tmpl
+				.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class BaseMessagePackableConvertableClass {
+		public MessagePackableConvertableClass f0;
+
+		public int f1;
+
+		public List f2;
+
+		public BaseMessagePackableConvertableClass() {
+		}
+	}
+
+	public static class MessagePackableConvertableClass implements
+			MessagePackable, MessageConvertable {
+
+		public int f0;
+
+		public int f1;
+
+		public MessagePackableConvertableClass() {
+		}
+
+		@Override
+		public void messagePack(Packer packer) throws IOException {
+			packer.pack(f0);
+			packer.pack(f1);
+		}
+
+		@Override
+		public void messageConvert(MessagePackObject obj)
+				throws MessageTypeException {
+			if (obj.isNil()) {
+				return;
+			}
+			MessagePackObject[] objs = obj.asArray();
+			f0 = objs[0].asInt();
+			f1 = objs[1].asInt();
+		}
+	}
 }
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
index 3222bec..dc56b6c 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
@@ -2,6 +2,7 @@ package org.msgpack.util.codegen;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -13,7 +14,10 @@ import org.junit.Test;
 import org.msgpack.CustomConverter;
 import org.msgpack.CustomPacker;
 import org.msgpack.CustomUnpacker;
+import org.msgpack.MessagePackable;
 import org.msgpack.MessagePacker;
+import org.msgpack.MessageTypeException;
+import org.msgpack.MessageUnpackable;
 import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
@@ -885,4 +889,104 @@ public class TestPackUnpack extends TestCase {
 		public SampleSuperClass() {
 		}
 	}
+
+	//@Test
+	public void XtestMessagePackableUnpackableClass00() throws Exception {
+		BaseMessagePackableUnpackableClass src = new BaseMessagePackableUnpackableClass();
+		MessagePackableUnpackableClass src1 = new MessagePackableUnpackableClass();
+		List src2 = new ArrayList();
+		src1.f0 = 0;
+		src1.f1 = 1;
+		src.f0 = src1;
+		src.f1 = 1;
+		src2.add(src1);
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(BaseMessagePackableUnpackableClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(BaseMessagePackableUnpackableClass.class);
+		BaseMessagePackableUnpackableClass dst = (BaseMessagePackableUnpackableClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0.f0, dst.f0.f0);
+		assertEquals(src.f0.f1, dst.f0.f1);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertEquals(src.f2.get(0).f0, dst.f2.get(0).f0);
+		assertEquals(src.f2.get(0).f1, dst.f2.get(0).f1);
+	}
+
+	//@Test
+	public void XtestMessagePackableUnpackableClass01() throws Exception {
+		BaseMessagePackableUnpackableClass src = new BaseMessagePackableUnpackableClass();
+		src.f0 = null;
+		src.f1 = 1;
+		src.f2 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(BaseMessagePackableUnpackableClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(BaseMessagePackableUnpackableClass.class);
+		BaseMessagePackableUnpackableClass dst = (BaseMessagePackableUnpackableClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+	}
+
+	//@Test
+	public void XtestMessagePackableUnpackableClass02() throws Exception {
+		BaseMessagePackableUnpackableClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(BaseMessagePackableUnpackableClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(BaseMessagePackableUnpackableClass.class);
+		BaseMessagePackableUnpackableClass dst = (BaseMessagePackableUnpackableClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class BaseMessagePackableUnpackableClass {
+		public MessagePackableUnpackableClass f0;
+
+		public int f1;
+
+		public List f2;
+
+		public BaseMessagePackableUnpackableClass() {
+		}
+	}
+
+	public static class MessagePackableUnpackableClass implements
+			MessagePackable, MessageUnpackable {
+
+		public int f0;
+
+		public int f1;
+
+		public MessagePackableUnpackableClass() {
+		}
+
+		@Override
+		public void messagePack(Packer packer) throws IOException {
+			packer.pack(f0);
+			packer.pack(f1);
+		}
+
+		@Override
+		public void messageUnpack(Unpacker unpacker) throws IOException,
+				MessageTypeException {
+			if (unpacker.tryUnpackNull()) {
+				return;
+			}
+			f0 = unpacker.unpackInt();
+			f1 = unpacker.unpackInt();
+		}
+	}
 }

From c790735b9ff97535ab5d2b776ae2ccc4ef1a5fc0 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Thu, 21 Oct 2010 02:02:23 +0900
Subject: [PATCH 0882/1648] java: add test programs for
 org.msgpack.util.codegen.*.java

---
 .../msgpack/util/codegen/DynamicCodeGen.java  | 101 ------------------
 .../util/codegen/DynamicCodeGenBase.java      |  54 ++++++++++
 .../msgpack/util/codegen/TestPackConvert.java |  19 ++--
 .../msgpack/util/codegen/TestPackUnpack.java  |  15 +--
 4 files changed, 73 insertions(+), 116 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 9ba7ec6..677ace9 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -18,12 +18,10 @@ import javassist.NotFoundException;
 
 import org.msgpack.CustomMessage;
 import org.msgpack.CustomPacker;
-import org.msgpack.MessageConvertable;
 import org.msgpack.MessagePackObject;
 import org.msgpack.MessagePackable;
 import org.msgpack.MessagePacker;
 import org.msgpack.MessageTypeException;
-import org.msgpack.MessageUnpackable;
 import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
@@ -450,56 +448,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_03, args));
 	}
 
-	private void insertCodeOfUnpackMethodCallForMsgUnpackableType(
-			StringBuilder sb, Field f, Class c) {
-		// if (t.fi == null) { t.fi = new Foo(); }
-		sb.append(KEYWORD_IF);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(f.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(KEYWORD_NULL);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(f.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(KEYWORD_NEW);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(c.getName());
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-
-		// insert a right variable // ignore
-		sb.append(VARIABLE_NAME_PK);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_UNPACK);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(MessageUnpackable.class.getName());
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(VARIABLE_NAME_TARGET);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(f.getName());
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-	}
-
 	private void insertOrdinalEnumUnpackMethodBody(StringBuilder sb,
 			Class type) {
 		// Object unpack(Unpacker u) throws IOException, MessageTypeException;
@@ -597,55 +545,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(String.format(STATEMENT_TMPL_CONVERTMETHODBODY_02, args));
 	}
 
-	private void insertCodeOfMessageConvertCallForMsgConvtblType(
-			StringBuilder sb, Field f, Class c, int i) {
-		// if (fi == null) { fi = new Foo_$$_Enhanced(); }
-		sb.append(KEYWORD_IF);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(f.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(KEYWORD_NULL);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(f.getName());
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_EQUAL);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(KEYWORD_NEW);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(c.getName());
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
-		sb.append(CHAR_NAME_SPACE);
-
-		// ((MessageConvertable)f_i).messageConvert(ary[i]);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(MessageConvertable.class.getName());
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(f.getName());
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_DOT);
-		sb.append(METHOD_NAME_MSGCONVERT);
-		sb.append(CHAR_NAME_LEFT_PARENTHESIS);
-		sb.append(VARIABLE_NAME_ARRAY);
-		sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-		sb.append(i);
-		sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
-		sb.append(CHAR_NAME_RIGHT_PARENTHESIS);
-		sb.append(CHAR_NAME_SEMICOLON);
-		sb.append(CHAR_NAME_SPACE);
-	}
-
 	private void insertOrdinalEnumConvertMethodBody(StringBuilder sb,
 			Class type) {
 		// Object convert(MessagePackObject mpo) throws MessageTypeException;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index f1415d5..0f8cace 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -1,5 +1,6 @@
 package org.msgpack.util.codegen;
 
+import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.GenericArrayType;
 import java.lang.reflect.Method;
@@ -22,9 +23,13 @@ import javassist.NotFoundException;
 
 import org.msgpack.CustomConverter;
 import org.msgpack.CustomMessage;
+import org.msgpack.MessageConvertable;
+import org.msgpack.MessagePackObject;
 import org.msgpack.MessageTypeException;
+import org.msgpack.MessageUnpackable;
 import org.msgpack.Template;
 import org.msgpack.Templates;
+import org.msgpack.Unpacker;
 import org.msgpack.annotation.MessagePackDelegate;
 import org.msgpack.annotation.MessagePackMessage;
 import org.msgpack.annotation.MessagePackOrdinalEnum;
@@ -40,6 +45,50 @@ public class DynamicCodeGenBase implements Constants {
 		void setNullCheck(boolean nullCheck);
 	}
 
+	public static class MessageUnpackableConvertableTemplate implements
+			Template {
+
+		private Class type;
+
+		public MessageUnpackableConvertableTemplate(Class type) {
+			this.type = type;
+		}
+
+		@Override
+		public Object unpack(Unpacker unpacker) throws IOException,
+				MessageTypeException {
+			try {
+				MessageUnpackable obj = (MessageUnpackable) type.newInstance();
+				obj.messageUnpack(unpacker);
+				return obj;
+			} catch (ClassCastException e) {
+				throw new MessageTypeException(e.getMessage(), e);
+			} catch (InstantiationException e) {
+				throw new MessageTypeException(e.getMessage(), e);
+			} catch (IllegalAccessException e) {
+				throw new MessageTypeException(e.getMessage(), e);
+			}
+		}
+
+		@Override
+		public Object convert(MessagePackObject from)
+				throws MessageTypeException {
+			try {
+				MessageConvertable obj = (MessageConvertable) type
+						.newInstance();
+				obj.messageConvert(from);
+				return obj;
+			} catch (ClassCastException e) {
+				throw new MessageTypeException(e.getMessage(), e);
+			} catch (InstantiationException e) {
+				throw new MessageTypeException(e.getMessage(), e);
+			} catch (IllegalAccessException e) {
+				throw new MessageTypeException(e.getMessage(), e);
+			}
+		}
+
+	}
+
 	public static class NullCheckerImpl implements NullChecker {
 		public boolean _$$_nullCheck = true;
 
@@ -317,6 +366,11 @@ public class DynamicCodeGenBase implements Constants {
 				Template tmpl = DynamicOrdinalEnumTemplate.create(c);
 				CustomMessage.registerTemplate(c, tmpl);
 				return tmpl;
+			} else if (MessageConvertable.class.isAssignableFrom(c)
+					|| MessageUnpackable.class.isAssignableFrom(c)) {
+				Template tmpl = new MessageUnpackableConvertableTemplate(c);
+				CustomMessage.registerTemplate(c, tmpl);
+				return tmpl;
 			} else {
 				throw new MessageTypeException("Type error: "
 						+ ((Class) t).getName());
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java b/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
index 2e6ea58..d2cf35c 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
@@ -1012,8 +1012,8 @@ public class TestPackConvert extends TestCase {
 		}
 	}
 
-	//@Test
-	public void XtestMessagePackableUnpackableClass00() throws Exception {
+	@Test
+	public void testMessagePackableUnpackableClass00() throws Exception {
 		BaseMessagePackableConvertableClass src = new BaseMessagePackableConvertableClass();
 		MessagePackableConvertableClass src1 = new MessagePackableConvertableClass();
 		List src2 = new ArrayList();
@@ -1045,8 +1045,8 @@ public class TestPackConvert extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
-	//@Test
-	public void XtestMessagePackableUnpackableClass01() throws Exception {
+	@Test
+	public void testMessagePackableUnpackableClass01() throws Exception {
 		BaseMessagePackableConvertableClass src = new BaseMessagePackableConvertableClass();
 		src.f0 = null;
 		src.f1 = 1;
@@ -1070,8 +1070,8 @@ public class TestPackConvert extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
-	//@Test
-	public void XtestMessagePackableUnpackableClass02() throws Exception {
+	@Test
+	public void testMessagePackableUnpackableClass02() throws Exception {
 		BaseMessagePackableConvertableClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		MessagePacker packer = DynamicPacker
@@ -1113,17 +1113,18 @@ public class TestPackConvert extends TestCase {
 
 		@Override
 		public void messagePack(Packer packer) throws IOException {
+			packer.packArray(2);
 			packer.pack(f0);
 			packer.pack(f1);
 		}
 
 		@Override
-		public void messageConvert(MessagePackObject obj)
+		public void messageConvert(MessagePackObject from)
 				throws MessageTypeException {
-			if (obj.isNil()) {
+			if (from.isNil()) {
 				return;
 			}
-			MessagePackObject[] objs = obj.asArray();
+			MessagePackObject[] objs = from.asArray();
 			f0 = objs[0].asInt();
 			f1 = objs[1].asInt();
 		}
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
index dc56b6c..b7ac7eb 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
@@ -890,8 +890,8 @@ public class TestPackUnpack extends TestCase {
 		}
 	}
 
-	//@Test
-	public void XtestMessagePackableUnpackableClass00() throws Exception {
+	@Test
+	public void testMessagePackableUnpackableClass00() throws Exception {
 		BaseMessagePackableUnpackableClass src = new BaseMessagePackableUnpackableClass();
 		MessagePackableUnpackableClass src1 = new MessagePackableUnpackableClass();
 		List src2 = new ArrayList();
@@ -900,6 +900,7 @@ public class TestPackUnpack extends TestCase {
 		src.f0 = src1;
 		src.f1 = 1;
 		src2.add(src1);
+		src.f2 = src2;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		MessagePacker packer = DynamicPacker
 				.create(BaseMessagePackableUnpackableClass.class);
@@ -917,8 +918,8 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src.f2.get(0).f1, dst.f2.get(0).f1);
 	}
 
-	//@Test
-	public void XtestMessagePackableUnpackableClass01() throws Exception {
+	@Test
+	public void testMessagePackableUnpackableClass01() throws Exception {
 		BaseMessagePackableUnpackableClass src = new BaseMessagePackableUnpackableClass();
 		src.f0 = null;
 		src.f1 = 1;
@@ -937,8 +938,8 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src.f2, dst.f2);
 	}
 
-	//@Test
-	public void XtestMessagePackableUnpackableClass02() throws Exception {
+	@Test
+	public void testMessagePackableUnpackableClass02() throws Exception {
 		BaseMessagePackableUnpackableClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		MessagePacker packer = DynamicPacker
@@ -975,6 +976,7 @@ public class TestPackUnpack extends TestCase {
 
 		@Override
 		public void messagePack(Packer packer) throws IOException {
+			packer.packArray(2);
 			packer.pack(f0);
 			packer.pack(f1);
 		}
@@ -985,6 +987,7 @@ public class TestPackUnpack extends TestCase {
 			if (unpacker.tryUnpackNull()) {
 				return;
 			}
+			unpacker.unpackArray();
 			f0 = unpacker.unpackInt();
 			f1 = unpacker.unpackInt();
 		}

From 92b346efa79fc66eeeed64cf833ef654abe1de21 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 21 Oct 2010 23:26:03 +0900
Subject: [PATCH 0883/1648] perl: clean up

---
 perl/xs-src/unpack.c | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 60ffd87..db7e394 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -1,6 +1,5 @@
 #define NEED_newRV_noinc
 #define NEED_sv_2pv_flags
-#define NEED_my_snprintf
 #include "xshelper.h"
 
 #define MY_CXT_KEY "Data::MessagePack::_unpack_guts" XS_VERSION
@@ -121,7 +120,8 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const
     return 0;
 }
 
-static const char* str_from_uint64(char* buf_end, uint64_t v)
+/* workaround win32 problems (my_snprintf(%llu) returns incorrect values ) */
+static char* str_from_uint64(char* buf_end, uint64_t v)
 {
   char *p = buf_end;
   *--p = '\0';
@@ -131,9 +131,8 @@ static const char* str_from_uint64(char* buf_end, uint64_t v)
   return p;
 }
 
-static const char* str_from_int64(char* buf_end, int64_t v)
-{
-  int minus = v < 0;
+static const char* str_from_int64(char* buf_end, int64_t const v) {
+  bool const minus = v < 0;
   char* p = str_from_uint64(buf_end, minus ? -v : v);
   if (minus)
     *--p = '-';
@@ -144,13 +143,8 @@ static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t co
 {
     dTHX;
     char tbuf[64];
-#if 1 /* workaround for win32-32bit (my_snprintf(%llu) returns are incorrect) */
-    char* s = str_from_uint64(tbuf + sizeof(tbuf), d);
+    const char* const s = str_from_uint64(tbuf + sizeof(tbuf), d);
     *o = newSVpvn(s, tbuf + sizeof(tbuf) - 1 - s);
-#else
-    STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%llu", d);
-    *o = newSVpvn(tbuf, len);
-#endif
     return 0;
 }
 
@@ -158,13 +152,8 @@ static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t cons
 {
     dTHX;
     char tbuf[64];
-#if 1 /* workaround for win32-32bit (my_snprintf(%lld) returns are incorrect) */
-    char* s = str_from_int64(tbuf + sizeof(tbuf), d);
+    const char* const s = str_from_int64(tbuf + sizeof(tbuf), d);
     *o = newSVpvn(s, tbuf + sizeof(tbuf) - 1 - s);
-#else
-    STRLEN const len = my_snprintf(tbuf, sizeof(tbuf), "%lld", d);
-    *o = newSVpvn(tbuf, len);
-#endif
     return 0;
 }
 

From 7fc34b63698660fa72aeb41e61d2e60f165eb719 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 21 Oct 2010 23:27:26 +0900
Subject: [PATCH 0884/1648] perl: tweaks

---
 perl/t/00_compile.t | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/perl/t/00_compile.t b/perl/t/00_compile.t
index d465f8d..29c6ca0 100644
--- a/perl/t/00_compile.t
+++ b/perl/t/00_compile.t
@@ -1,7 +1,9 @@
 use strict;
 use warnings;
 use Test::More tests => 1;
-
+use Config;
 use_ok 'Data::MessagePack';
 diag ( "Testing Data::MessagePack/$Data::MessagePack::VERSION (",
     $INC{'Data/MessagePack/PP.pm'} ? 'PP' : 'XS', ")" );
+
+diag "byteoder: $Config{byteorder}, ivsize=$Config{ivsize}";

From d8aaef4f04e3e22c028a7d6a10755f2a084f3cd2 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 21 Oct 2010 23:35:07 +0900
Subject: [PATCH 0885/1648] perl: tests

---
 perl/t/06_stream_unpack2.t | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/perl/t/06_stream_unpack2.t b/perl/t/06_stream_unpack2.t
index f50401f..0872437 100644
--- a/perl/t/06_stream_unpack2.t
+++ b/perl/t/06_stream_unpack2.t
@@ -1,7 +1,7 @@
 use strict;
 use warnings;
 use Data::MessagePack;
-use Test::More tests => 61;
+use Test::More tests => 64;
 use t::Util;
 
 my $input = [
@@ -36,6 +36,7 @@ is_deeply(Data::MessagePack->unpack($packed), $input);
         $offset = $up->execute($packed, $offset);
         ok $up->is_finished, 'finished';
         my $data = $up->data;
+        is scalar(@{$data}), scalar(@{$input}), 'size of @{$data}';
         is_deeply $data, $input, "block $i, offset $offset";
         $up->reset();
     }

From 86043fd87e14690e48fca25d53cfb8be652d009c Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Fri, 22 Oct 2010 08:31:03 +0900
Subject: [PATCH 0886/1648] java: adds OptionalTemplate

---
 java/pom.xml                                  |  2 +-
 .../msgpack/template/OptionalTemplate.java    | 52 +++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100644 java/src/main/java/org/msgpack/template/OptionalTemplate.java

diff --git a/java/pom.xml b/java/pom.xml
index beec503..aa3e3ce 100755
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -3,7 +3,7 @@
   4.0.0
   org.msgpack
   msgpack
-  0.4
+  0.4.0-devel
   MessagePack for Java
 
   MessagePack for Java
diff --git a/java/src/main/java/org/msgpack/template/OptionalTemplate.java b/java/src/main/java/org/msgpack/template/OptionalTemplate.java
new file mode 100644
index 0000000..6be16c1
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/OptionalTemplate.java
@@ -0,0 +1,52 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.io.IOException;
+import org.msgpack.*;
+
+public class OptionalTemplate implements Template {
+	private Template elementTemplate;
+	private Object defaultObject;
+
+	public OptionalTemplate(Template elementTemplate) {
+		this(elementTemplate, null);
+	}
+
+	public OptionalTemplate(Template elementTemplate, Object defaultObject) {
+		this.elementTemplate = elementTemplate;
+		this.defaultObject = defaultObject;
+	}
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		if(pac.tryUnpackNull()) {
+			return defaultObject;
+		}
+		return elementTemplate.unpack(pac);
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		if(from.isNil()) {
+			return defaultObject;
+		}
+		return elementTemplate.convert(from);
+	}
+}
+

From 71ae75a5bff920d886395275fdc6f991207d8000 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Fri, 22 Oct 2010 16:18:07 +0900
Subject: [PATCH 0887/1648] java: adds OptionalPacker

---
 java/src/main/java/org/msgpack/Packer.java    | 10 +++++
 .../org/msgpack/packer/BigIntegerPacker.java  |  2 +-
 .../org/msgpack/packer/BooleanPacker.java     |  2 +-
 .../org/msgpack/packer/ByteArrayPacker.java   |  2 +-
 .../java/org/msgpack/packer/BytePacker.java   |  2 +-
 .../java/org/msgpack/packer/DoublePacker.java |  2 +-
 .../java/org/msgpack/packer/FloatPacker.java  |  2 +-
 .../org/msgpack/packer/IntegerPacker.java     |  2 +-
 .../java/org/msgpack/packer/LongPacker.java   |  2 +-
 .../org/msgpack/packer/OptionalPacker.java    | 38 +++++++++++++++++++
 .../java/org/msgpack/packer/ShortPacker.java  |  2 +-
 .../java/org/msgpack/packer/StringPacker.java |  2 +-
 12 files changed, 58 insertions(+), 10 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/packer/OptionalPacker.java

diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index a10bf19..1475ac6 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -331,6 +331,16 @@ public class Packer {
 	}
 
 
+	public Packer packByteArray(byte[] b) throws IOException {
+		packRaw(b.length);
+		return packRawBody(b, 0, b.length);
+	}
+
+	public Packer packByteArray(byte[] b, int off, int length) throws IOException {
+		packRaw(length);
+		return packRawBody(b, off, length);
+	}
+
 	public Packer packString(String s) throws IOException {
 		byte[] b = ((String)s).getBytes("UTF-8");
 		packRaw(b.length);
diff --git a/java/src/main/java/org/msgpack/packer/BigIntegerPacker.java b/java/src/main/java/org/msgpack/packer/BigIntegerPacker.java
index 60bbab9..7d4c0a6 100644
--- a/java/src/main/java/org/msgpack/packer/BigIntegerPacker.java
+++ b/java/src/main/java/org/msgpack/packer/BigIntegerPacker.java
@@ -25,7 +25,7 @@ public class BigIntegerPacker implements MessagePacker {
 	private BigIntegerPacker() { }
 
 	public void pack(Packer pk, Object target) throws IOException {
-		pk.pack((BigInteger)target);
+		pk.packBigInteger((BigInteger)target);
 	}
 
 	static public BigIntegerPacker getInstance() {
diff --git a/java/src/main/java/org/msgpack/packer/BooleanPacker.java b/java/src/main/java/org/msgpack/packer/BooleanPacker.java
index 6f300c9..5392494 100644
--- a/java/src/main/java/org/msgpack/packer/BooleanPacker.java
+++ b/java/src/main/java/org/msgpack/packer/BooleanPacker.java
@@ -24,7 +24,7 @@ public class BooleanPacker implements MessagePacker {
 	private BooleanPacker() { }
 
 	public void pack(Packer pk, Object target) throws IOException {
-		pk.pack((Boolean)target);
+		pk.packBoolean((Boolean)target);
 	}
 
 	static public BooleanPacker getInstance() {
diff --git a/java/src/main/java/org/msgpack/packer/ByteArrayPacker.java b/java/src/main/java/org/msgpack/packer/ByteArrayPacker.java
index f5f38f8..1c590fb 100644
--- a/java/src/main/java/org/msgpack/packer/ByteArrayPacker.java
+++ b/java/src/main/java/org/msgpack/packer/ByteArrayPacker.java
@@ -24,7 +24,7 @@ public class ByteArrayPacker implements MessagePacker {
 	private ByteArrayPacker() { }
 
 	public void pack(Packer pk, Object target) throws IOException {
-		pk.pack((byte[])target);
+		pk.packByteArray((byte[])target);
 	}
 
 	static public ByteArrayPacker getInstance() {
diff --git a/java/src/main/java/org/msgpack/packer/BytePacker.java b/java/src/main/java/org/msgpack/packer/BytePacker.java
index a005f76..79d5989 100644
--- a/java/src/main/java/org/msgpack/packer/BytePacker.java
+++ b/java/src/main/java/org/msgpack/packer/BytePacker.java
@@ -24,7 +24,7 @@ public class BytePacker implements MessagePacker {
 	private BytePacker() { }
 
 	public void pack(Packer pk, Object target) throws IOException {
-		pk.pack(((Byte)target).byteValue());
+		pk.packByte((Byte)target);
 	}
 
 	static public BytePacker getInstance() {
diff --git a/java/src/main/java/org/msgpack/packer/DoublePacker.java b/java/src/main/java/org/msgpack/packer/DoublePacker.java
index df53e33..c555869 100644
--- a/java/src/main/java/org/msgpack/packer/DoublePacker.java
+++ b/java/src/main/java/org/msgpack/packer/DoublePacker.java
@@ -24,7 +24,7 @@ public class DoublePacker implements MessagePacker {
 	private DoublePacker() { }
 
 	public void pack(Packer pk, Object target) throws IOException {
-		pk.pack(((Double)target).doubleValue());
+		pk.packDouble(((Double)target));
 	}
 
 	static public DoublePacker getInstance() {
diff --git a/java/src/main/java/org/msgpack/packer/FloatPacker.java b/java/src/main/java/org/msgpack/packer/FloatPacker.java
index 3aa47e1..d9dbc71 100644
--- a/java/src/main/java/org/msgpack/packer/FloatPacker.java
+++ b/java/src/main/java/org/msgpack/packer/FloatPacker.java
@@ -24,7 +24,7 @@ public class FloatPacker implements MessagePacker {
 	private FloatPacker() { }
 
 	public void pack(Packer pk, Object target) throws IOException {
-		pk.pack(((Float)target).floatValue());
+		pk.packFloat((Float)target);
 	}
 
 	static public FloatPacker getInstance() {
diff --git a/java/src/main/java/org/msgpack/packer/IntegerPacker.java b/java/src/main/java/org/msgpack/packer/IntegerPacker.java
index 1e64429..4f9cde1 100644
--- a/java/src/main/java/org/msgpack/packer/IntegerPacker.java
+++ b/java/src/main/java/org/msgpack/packer/IntegerPacker.java
@@ -24,7 +24,7 @@ public class IntegerPacker implements MessagePacker {
 	private IntegerPacker() { }
 
 	public void pack(Packer pk, Object target) throws IOException {
-		pk.pack(((Integer)target).intValue());
+		pk.packInt((Integer)target);
 	}
 
 	static public IntegerPacker getInstance() {
diff --git a/java/src/main/java/org/msgpack/packer/LongPacker.java b/java/src/main/java/org/msgpack/packer/LongPacker.java
index 146e42d..bc7da8d 100644
--- a/java/src/main/java/org/msgpack/packer/LongPacker.java
+++ b/java/src/main/java/org/msgpack/packer/LongPacker.java
@@ -24,7 +24,7 @@ public class LongPacker implements MessagePacker {
 	private LongPacker() { }
 
 	public void pack(Packer pk, Object target) throws IOException {
-		pk.pack(((Long)target).longValue());
+		pk.packLong((Long)target);
 	}
 
 	static public LongPacker getInstance() {
diff --git a/java/src/main/java/org/msgpack/packer/OptionalPacker.java b/java/src/main/java/org/msgpack/packer/OptionalPacker.java
new file mode 100644
index 0000000..e9550a4
--- /dev/null
+++ b/java/src/main/java/org/msgpack/packer/OptionalPacker.java
@@ -0,0 +1,38 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.packer;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class OptionalPacker implements MessagePacker {
+	private MessagePacker elementPacker;
+
+	public OptionalPacker(MessagePacker elementPacker) {
+		this.elementPacker = elementPacker;
+	}
+
+	public void pack(Packer pk, Object target) throws IOException {
+		if(target == null) {
+			pk.packNil();
+		} else {
+			elementPacker.pack(pk, target);
+		}
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/packer/ShortPacker.java b/java/src/main/java/org/msgpack/packer/ShortPacker.java
index e4be9ec..4c55cc7 100644
--- a/java/src/main/java/org/msgpack/packer/ShortPacker.java
+++ b/java/src/main/java/org/msgpack/packer/ShortPacker.java
@@ -24,7 +24,7 @@ public class ShortPacker implements MessagePacker {
 	private ShortPacker() { }
 
 	public void pack(Packer pk, Object target) throws IOException {
-		pk.pack(((Short)target).shortValue());
+		pk.packShort((Short)target);
 	}
 
 	static public ShortPacker getInstance() {
diff --git a/java/src/main/java/org/msgpack/packer/StringPacker.java b/java/src/main/java/org/msgpack/packer/StringPacker.java
index 17a0500..dfea620 100644
--- a/java/src/main/java/org/msgpack/packer/StringPacker.java
+++ b/java/src/main/java/org/msgpack/packer/StringPacker.java
@@ -24,7 +24,7 @@ public class StringPacker implements MessagePacker {
 	private StringPacker() { }
 
 	public void pack(Packer pk, Object target) throws IOException {
-		pk.pack((String)target);
+		pk.packString((String)target);
 	}
 
 	static public StringPacker getInstance() {

From 69e32d264ce1d15ea979d0f258fb14bdc738cdf1 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Fri, 22 Oct 2010 23:23:53 +0900
Subject: [PATCH 0888/1648] java: write test programs for OptionalTemplate.java

---
 .../annotation/MessagePackOptional.java       |    3 +-
 .../annotation/MessagePackRequired.java       |   12 -
 .../org/msgpack/template/MapTemplate.java     |    1 +
 .../msgpack/template/OptionalTemplate.java    |    2 -
 .../org/msgpack/util/codegen/Constants.java   |  114 +-
 .../msgpack/util/codegen/DynamicCodeGen.java  |   89 +-
 .../util/codegen/DynamicCodeGenBase.java      |   29 +-
 .../util/codegen/DynamicCodeGenException.java |    1 +
 .../util/codegen/DynamicConverter.java        |    9 +-
 .../msgpack/util/codegen/DynamicPacker.java   |    8 +-
 .../msgpack/util/codegen/DynamicTemplate.java |    8 +-
 .../msgpack/util/codegen/DynamicUnpacker.java |    8 +-
 .../org/msgpack/util/codegen/FieldOption.java |   21 +-
 .../org/msgpack/packer/TestPackConvert.java   |  261 +++-
 .../org/msgpack/packer/TestPackUnpack.java    |  281 +++-
 .../org/msgpack/template/TestPackConvert.java |  195 ++-
 .../org/msgpack/template/TestPackUnpack.java  |  199 ++-
 .../msgpack/util/codegen/TestPackConvert.java | 1239 +++++++++++++----
 .../msgpack/util/codegen/TestPackUnpack.java  | 1068 +++++++++++---
 19 files changed, 2925 insertions(+), 623 deletions(-)
 delete mode 100644 java/src/main/java/org/msgpack/annotation/MessagePackRequired.java

diff --git a/java/src/main/java/org/msgpack/annotation/MessagePackOptional.java b/java/src/main/java/org/msgpack/annotation/MessagePackOptional.java
index 081b28c..f9eb227 100644
--- a/java/src/main/java/org/msgpack/annotation/MessagePackOptional.java
+++ b/java/src/main/java/org/msgpack/annotation/MessagePackOptional.java
@@ -6,7 +6,6 @@ import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 
 @Target(ElementType.FIELD)
-@Retention(RetentionPolicy.CLASS)
+@Retention(RetentionPolicy.RUNTIME)
 public @interface MessagePackOptional {
-	int value() default -1;
 }
\ No newline at end of file
diff --git a/java/src/main/java/org/msgpack/annotation/MessagePackRequired.java b/java/src/main/java/org/msgpack/annotation/MessagePackRequired.java
deleted file mode 100644
index e268c4f..0000000
--- a/java/src/main/java/org/msgpack/annotation/MessagePackRequired.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.msgpack.annotation;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target(ElementType.FIELD)
-@Retention(RetentionPolicy.CLASS)
-public @interface MessagePackRequired {
-	int value() default -1;
-}
\ No newline at end of file
diff --git a/java/src/main/java/org/msgpack/template/MapTemplate.java b/java/src/main/java/org/msgpack/template/MapTemplate.java
index d2b4eff..6b8457d 100644
--- a/java/src/main/java/org/msgpack/template/MapTemplate.java
+++ b/java/src/main/java/org/msgpack/template/MapTemplate.java
@@ -42,6 +42,7 @@ public class MapTemplate implements Template {
 		return map;
 	}
 
+	@SuppressWarnings("unchecked")
 	public Object convert(MessagePackObject from) throws MessageTypeException {
 		Map src = from.asMap();
 		Map map = new HashMap();
diff --git a/java/src/main/java/org/msgpack/template/OptionalTemplate.java b/java/src/main/java/org/msgpack/template/OptionalTemplate.java
index 6be16c1..f288fc4 100644
--- a/java/src/main/java/org/msgpack/template/OptionalTemplate.java
+++ b/java/src/main/java/org/msgpack/template/OptionalTemplate.java
@@ -17,8 +17,6 @@
 //
 package org.msgpack.template;
 
-import java.util.List;
-import java.util.ArrayList;
 import java.io.IOException;
 import org.msgpack.*;
 
diff --git a/java/src/main/java/org/msgpack/util/codegen/Constants.java b/java/src/main/java/org/msgpack/util/codegen/Constants.java
index e32e6ad..8a4b708 100644
--- a/java/src/main/java/org/msgpack/util/codegen/Constants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/Constants.java
@@ -9,96 +9,20 @@ public interface Constants {
 
 	String POSTFIX_TYPE_NAME_TEMPLATE = "_$$_Template";
 
-	String KEYWORD_MODIFIER_PUBLIC = "public";
+	String STRING_NAME_COMMA_SPACE = ", ";
 
-	String KEYWORD_CATCH = "catch";
-
-	String KEYWORD_ELSE = "else";
-
-	String KEYWORD_ELSEIF = "else if";
-
-	String KEYWORD_FOR = "for";
-
-	String KEYWORD_IF = "if";
-
-	String KEYWORD_INSTANCEOF = "instanceof";
-
-	String KEYWORD_NEW = "new";
-
-	String KEYWORD_NULL = "null";
-
-	String KEYWORD_RETURN = "return";
-
-	String KEYWORD_THROW = "throw";
-
-	String KEYWORD_THROWS = "throws";
-
-	String KEYWORD_TRY = "try";
+	String STRING_NAME_LEFT_RIGHT_SQUARE_BRACKET = "[]";
 
 	String CHAR_NAME_SPACE = " ";
 
-	String CHAR_NAME_COMMA = ",";
-
-	String CHAR_NAME_EQUAL = "=";
-
-	String CHAR_NAME_PLUS = "+";
-
-	String CHAR_NAME_MINUS = "-";
-
-	String CHAR_NAME_UNDERSCORE = "_";
-
-	String CHAR_NAME_LESSTHAN = "<";
-
-	String CHAR_NAME_RIGHT_PARENTHESIS = ")";
-
-	String CHAR_NAME_LEFT_PARENTHESIS = "(";
-
 	String CHAR_NAME_RIGHT_CURLY_BRACKET = "}";
 
 	String CHAR_NAME_LEFT_CURLY_BRACKET = "{";
 
-	String CHAR_NAME_RIGHT_SQUARE_BRACKET = "]";
-
-	String CHAR_NAME_LEFT_SQUARE_BRACKET = "[";
-
-	String CHAR_NAME_DOT = ".";
-
-	String CHAR_NAME_SEMICOLON = ";";
-
-	String VARIABLE_NAME_PK = "_$$_pk";
-
-	String VARIABLE_NAME_OBJECT = "_$$_obj";
-
-	String VARIABLE_NAME_TARGET = "_$$_t";
-
-	String VARIABLE_NAME_SIZE = "_$$_len";
-
-	String VARIABLE_NAME_ARRAY = "_$$_ary";
-
-	String VARIABLE_NAME_LIST = "_$$_list";
-
-	String VARIABLE_NAME_MAP = "_$$_map";
-
-	String VARIABLE_NAME_KEY = "_$$_key";
-
-	String VARIABLE_NAME_VAL = "_$$_val";
-
-	String VARIABLE_NAME_ITER = "_$$_iter";
-
-	String VARIABLE_NAME_MPO = "_$$_mpo";
-
-	String VARIABLE_NAME_I = "i";
-
 	String VARIABLE_NAME_TEMPLATES = "_$$_templates";
 
-	String VARIABLE_NAME_TEMPLATES0 = "_$$_tmpls";
-
 	String VARIABLE_NAME_CLIENT = "_$$_client";
 
-	String VARIABLE_NAME_CLIENT0 = "_$$_c";
-
-	String METHOD_NAME_VALUEOF = "valueOf";
-
 	String METHOD_NAME_BOOLEANVALUE = "booleanValue";
 
 	String METHOD_NAME_BYTEVALUE = "byteValue";
@@ -113,38 +37,14 @@ public interface Constants {
 
 	String METHOD_NAME_DOUBLEVALUE = "doubleValue";
 
-	String METHOD_NAME_ADD = "add";
-
-	String METHOD_NAME_PUT = "put";
-
-	String METHOD_NAME_GET = "get";
-
-	String METHOD_NAME_SIZE = "size";
-
-	String METHOD_NAME_KEYSET = "keySet";
-
-	String METHOD_NAME_ITERATOR = "iterator";
-
-	String METHOD_NAME_HASNEXT = "hasNext";
-
-	String METHOD_NAME_NEXT = "next";
-
-	String METHOD_NAME_ORDINAL = "ordinal";
-
 	String METHOD_NAME_GETENUMCONSTANTS = "getEnumConstants";
 
 	String METHOD_NAME_CONVERT = "convert";
 
-	String METHOD_NAME_MSGCONVERT = "messageConvert";
-
 	String METHOD_NAME_SETTEMPLATES = "setTemplates";
 
-	String METHOD_NAME_SETCLIENT = "setClient";
-
 	String METHOD_NAME_PACK = "pack";
 
-	String METHOD_NAME_PACKARRAY = "packArray";
-
 	String METHOD_NAME_UNPACK = "unpack";
 
 	String METHOD_NAME_UNPACKBOOLEAN = "unpackBoolean";
@@ -207,13 +107,11 @@ public interface Constants {
 
 	String STATEMENT_PACKER_PACKERMETHODBODY_04 = "$1.pack(((java.lang.Enum)_$$_t).ordinal()); ";
 
-	String STATEMENT_PACKER_PACKERMETHODBODY_05 = "if (_$$_nullCheck && $2 == null) { $1.packNil(); return; } ";
-
 	String STATEMENT_TMPL_UNPACKERMETHODBODY_01 = "%s _$$_t = new %s(); ";
 
 	String STATEMENT_TMPL_UNPACKERMETHODBODY_02 = "$1.unpackArray(); ";
 
-	String STATEMENT_TMPL_UNPACKERMETHODBODY_03 = "if (!_$$_nullCheck || !$1.tryUnpackNull()) { _$$_t.%s = %s(%s)_$$_templates[%d].unpack($1)%s; } ";
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_03 = "_$$_t.%s = %s(%s)_$$_templates[%d].unpack($1)%s; ";
 
 	String STATEMENT_TMPL_UNPACKERMETHODBODY_04 = "return _$$_t; ";
 
@@ -221,13 +119,9 @@ public interface Constants {
 
 	String STATEMENT_TMPL_UNPACKERMETHODBODY_06 = "return %s.class.getEnumConstants()[i]; ";
 
-	String STATEMENT_TMPL_UNPACKERMETHODBODY_07 = "if (_$$_nullCheck && $1.tryUnpackNull()) { return null; } ";
-
 	String STATEMENT_TMPL_CONVERTMETHODBODY_01 = "%s _$$_ary = $1.asArray(); ";
 
-	String STATEMENT_TMPL_CONVERTMETHODBODY_02 = "if (!_$$_nullCheck || !_$$_ary[%d].isNil()) { _$$_t.%s = %s(%s)_$$_templates[%d].convert(_$$_ary[%d])%s; } ";
+	String STATEMENT_TMPL_CONVERTMETHODBODY_02 = "_$$_t.%s = %s(%s)_$$_templates[%d].convert(_$$_ary[%d])%s; ";
 
 	String STATEMENT_TMPL_CONVERTMETHODBODY_03 = "int i = _$$_ary[0].asInt(); ";
-
-	String STATEMENT_TMPL_CONVERTMETHODBODY_04 = "if (_$$_nullCheck && $1.isNil()) { return null; } ";
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 677ace9..bb9cdd5 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -1,6 +1,7 @@
 package org.msgpack.util.codegen;
 
 import java.io.IOException;
+import java.lang.annotation.Annotation;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
@@ -27,7 +28,9 @@ import org.msgpack.Template;
 import org.msgpack.Unpacker;
 import org.msgpack.annotation.MessagePackDelegate;
 import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.annotation.MessagePackOptional;
 import org.msgpack.annotation.MessagePackOrdinalEnum;
+import org.msgpack.template.OptionalTemplate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,7 +64,8 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		return tmplCache.get(type.getName());
 	}
 
-	public Class generateMessagePackerClass(Class origClass) {
+	public Class generateMessagePackerClass(Class origClass,
+			List fieldOpts) {
 		try {
 			LOG.debug("start generating a packer class for "
 					+ origClass.getName());
@@ -70,10 +74,12 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			checkTypeValidation(origClass);
 			checkDefaultConstructorValidation(origClass);
 			CtClass packerCtClass = pool.makeClass(packerName);
-			setSuperclass(packerCtClass, NullCheckerImpl.class);
 			setInterface(packerCtClass, MessagePacker.class);
 			addDefaultConstructor(packerCtClass);
 			Field[] fields = getDeclaredFields(origClass);
+			if (fieldOpts != null) {
+				fields = sortFields(fields, fieldOpts);
+			}
 			addPackMethod(packerCtClass, origClass, fields, false);
 			Class packerClass = createClass(packerCtClass);
 			LOG.debug("generated a packer class for " + origClass.getName());
@@ -95,7 +101,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			String packerName = origName + POSTFIX_TYPE_NAME_PACKER + inc();
 			checkTypeValidation(origClass);
 			CtClass packerCtClass = pool.makeClass(packerName);
-			setSuperclass(packerCtClass, NullCheckerImpl.class);
 			setInterface(packerCtClass, MessagePacker.class);
 			addDefaultConstructor(packerCtClass);
 			addPackMethod(packerCtClass, origClass, null, true);
@@ -111,7 +116,8 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	public Class generateTemplateClass(Class origClass) {
+	public Class generateTemplateClass(Class origClass,
+			List fieldOpts) {
 		try {
 			LOG.debug("start generating a template class for "
 					+ origClass.getName());
@@ -120,11 +126,17 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			checkTypeValidation(origClass);
 			checkDefaultConstructorValidation(origClass);
 			CtClass tmplCtClass = pool.makeClass(tmplName);
-			setSuperclass(tmplCtClass, TemplateTemplate.class);
+			setSuperclass(tmplCtClass, TemplateAccessorImpl.class);
 			setInterface(tmplCtClass, Template.class);
 			addDefaultConstructor(tmplCtClass);
 			Field[] fields = getDeclaredFields(origClass);
-			Template[] tmpls = createTemplates(fields);
+			Template[] tmpls = null;
+			if (fieldOpts != null) {
+				fields = sortFields(fields, fieldOpts);
+				tmpls = createTemplates(fieldOpts);
+			} else {
+				tmpls = createTemplates(fields);
+			}
 			setTemplates(origClass, tmpls);
 			addUnpackMethod(tmplCtClass, origClass, fields, false);
 			addConvertMethod(tmplCtClass, origClass, fields, false);
@@ -152,7 +164,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			checkTypeValidation(origClass);
 			String tmplName = origName + POSTFIX_TYPE_NAME_TEMPLATE + inc();
 			CtClass tmplCtClass = pool.makeClass(tmplName);
-			setSuperclass(tmplCtClass, TemplateTemplate.class);
+			setSuperclass(tmplCtClass, TemplateAccessorImpl.class);
 			setInterface(tmplCtClass, Template.class);
 			addDefaultConstructor(tmplCtClass);
 			addUnpackMethod(tmplCtClass, origClass, null, true);
@@ -240,6 +252,41 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		}
 	}
 
+	Field[] sortFields(Field[] fields, List fieldOpts) {
+		if (fields.length != fieldOpts.size()) {
+			throwFieldSortingException(String.format(
+					"Mismatch: public field num: %d, option num: %d",
+					new Object[] { fields.length, fieldOpts.size() }));
+		}
+		Field[] sorted = new Field[fields.length];
+		for (int i = 0; i < sorted.length; ++i) {
+			FieldOption opt = fieldOpts.get(i);
+			Field match = null;
+			for (Field f : fields) {
+				if (opt.name.equals(f.getName())) {
+					match = f;
+					break;
+				}
+			}
+			if (match != null) {
+				sorted[i] = match;
+			} else {
+				throwFieldSortingException(String.format(
+						"Mismatch: a %s field option is not declared",
+						new Object[] { opt.name }));
+			}
+		}
+		return sorted;
+	}
+
+	Template[] createTemplates(List fieldOpts) {
+		Template[] tmpls = new Template[fieldOpts.size()];
+		for (int i = 0; i < tmpls.length; ++i) {
+			tmpls[i] = fieldOpts.get(i).tmpl;
+		}
+		return tmpls;
+	}
+
 	Template[] createTemplates(Field[] fields) {
 		Template[] tmpls = new Template[fields.length];
 		for (int i = 0; i < tmpls.length; ++i) {
@@ -249,12 +296,23 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	}
 
 	Template createTemplate(Field field) {
+		boolean isOptional = isAnnotated(field, MessagePackOptional.class);
 		Class c = field.getType();
+		Template tmpl = null;
 		if (List.class.isAssignableFrom(c) || Map.class.isAssignableFrom(c)) {
-			return createTemplate(field.getGenericType());
+			tmpl = createTemplate(field.getGenericType());
 		} else {
-			return createTemplate(c);
+			tmpl = createTemplate(c);
 		}
+		if (isOptional) {
+			return new OptionalTemplate(tmpl);
+		} else {
+			return tmpl;
+		}
+	}
+	
+	private boolean isAnnotated(Field field, Class with) {
+		return field.getAnnotation(with) != null;
 	}
 
 	private void addPackMethod(CtClass packerCtClass, Class c, Field[] fs,
@@ -297,8 +355,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// void pack(Packer packer, Object target) throws IOException;
 		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
-		Object[] args2 = new Object[0];
-		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_05, args2));
 		String typeName = classToString(type);
 		Object[] args0 = new Object[] { typeName, typeName };
 		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_01, args0));
@@ -357,8 +413,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// void pack(Packer packer, Object target) throws IOException;
 		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
-		Object[] args3 = new Object[0];
-		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_05, args3));
 		String typeName = classToString(c);
 		Object[] args0 = new Object[] { typeName, typeName };
 		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_01, args0));
@@ -412,8 +466,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(CHAR_NAME_SPACE);
 		// Foo _$$_t = new Foo();
 		String typeName = classToString(type);
-		Object[] args3 = new Object[0];
-		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_07, args3));
 		Object[] args0 = new Object[] { typeName, typeName };
 		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_01, args0));
 		// $1.unpackArray();
@@ -453,8 +505,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// Object unpack(Unpacker u) throws IOException, MessageTypeException;
 		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
-		Object[] args3 = new Object[0];
-		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_07, args3));
 		// $1.unpackArray();
 		Object[] args0 = new Object[0];
 		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_02, args0));
@@ -506,8 +556,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// Object convert(MessagePackObject mpo) throws MessageTypeException;
 		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
-		Object[] args3 = new Object[0];
-		sb.append(String.format(STATEMENT_TMPL_CONVERTMETHODBODY_04, args3));
 		// Foo _$$_t = new Foo();
 		String typeName = classToString(type);
 		Object[] args0 = new Object[] { typeName, typeName };
@@ -533,7 +581,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		Class returnType = field.getType();
 		boolean isPrim = returnType.isPrimitive();
 		Object[] args = new Object[] {
-				i,
 				field.getName(),
 				isPrim ? "(" : "",
 				isPrim ? getPrimToWrapperType(returnType).getName()
@@ -550,8 +597,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		// Object convert(MessagePackObject mpo) throws MessageTypeException;
 		sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
 		sb.append(CHAR_NAME_SPACE);
-		Object[] args3 = new Object[0];
-		sb.append(String.format(STATEMENT_TMPL_CONVERTMETHODBODY_04, args3));
 		// MessagePackObject[] _$$_ary = $1.asArray();
 		Object[] args0 = new Object[] { classToString(MessagePackObject[].class) };
 		sb.append(String.format(STATEMENT_TMPL_CONVERTMETHODBODY_01, args0));
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 0f8cace..95c3748 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -41,10 +41,6 @@ public class DynamicCodeGenBase implements Constants {
 	private static Logger LOG = LoggerFactory
 			.getLogger(DynamicCodeGenBase.class);
 
-	public static interface NullChecker {
-		void setNullCheck(boolean nullCheck);
-	}
-
 	public static class MessageUnpackableConvertableTemplate implements
 			Template {
 
@@ -86,23 +82,13 @@ public class DynamicCodeGenBase implements Constants {
 				throw new MessageTypeException(e.getMessage(), e);
 			}
 		}
-
-	}
-
-	public static class NullCheckerImpl implements NullChecker {
-		public boolean _$$_nullCheck = true;
-
-		public void setNullCheck(boolean _$$_check) {
-			_$$_nullCheck = _$$_check;
-		}
 	}
 
 	public static interface TemplateAccessor {
 		void setTemplates(Template[] templates);
 	}
 
-	public static class TemplateTemplate extends NullCheckerImpl implements
-			TemplateAccessor {
+	public static class TemplateAccessorImpl implements TemplateAccessor {
 		public Template[] _$$_templates;
 
 		public void setTemplates(Template[] _$$_tmpls) {
@@ -160,6 +146,12 @@ public class DynamicCodeGenBase implements Constants {
 		throw e;
 	}
 
+	protected void throwFieldSortingException(String message) {
+		DynamicCodeGenException e = new DynamicCodeGenException(message);
+		LOG.debug(e.getMessage(), e);
+		throw e;
+	}
+
 	protected static void throwMethodValidationException(Method method,
 			String message) throws DynamicCodeGenException {
 		DynamicCodeGenException e = new DynamicCodeGenException(String.format(
@@ -207,7 +199,7 @@ public class DynamicCodeGenBase implements Constants {
 
 	protected void addTemplateArrayField(CtClass newCtClass)
 			throws NotFoundException, CannotCompileException {
-		CtClass acsCtClass = pool.get(TemplateTemplate.class.getName());
+		CtClass acsCtClass = pool.get(TemplateAccessorImpl.class.getName());
 		CtField tmplsField = acsCtClass
 				.getDeclaredField(VARIABLE_NAME_TEMPLATES);
 		CtField tmplsField2 = new CtField(tmplsField.getType(), tmplsField
@@ -217,7 +209,7 @@ public class DynamicCodeGenBase implements Constants {
 
 	protected void addSetTemplatesMethod(CtClass newCtClass)
 			throws NotFoundException, CannotCompileException {
-		CtClass acsCtClass = pool.get(TemplateTemplate.class.getName());
+		CtClass acsCtClass = pool.get(TemplateAccessorImpl.class.getName());
 		CtMethod settmplsMethod = acsCtClass
 				.getDeclaredMethod(METHOD_NAME_SETTEMPLATES);
 		CtMethod settmplsMethod2 = CtNewMethod.copy(settmplsMethod, newCtClass,
@@ -425,8 +417,7 @@ public class DynamicCodeGenBase implements Constants {
 		Class t = getArrayBaseType(type);
 		sb.append(t.getName());
 		for (int i = 0; i < dim; ++i) {
-			sb.append(CHAR_NAME_LEFT_SQUARE_BRACKET);
-			sb.append(CHAR_NAME_RIGHT_SQUARE_BRACKET);
+			sb.append(STRING_NAME_LEFT_RIGHT_SQUARE_BRACKET);
 		}
 		return sb.toString();
 	}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
index defb3c9..df1f2e7 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
@@ -1,5 +1,6 @@
 package org.msgpack.util.codegen;
 
+@SuppressWarnings("serial")
 public class DynamicCodeGenException extends RuntimeException {
 
 	public DynamicCodeGenException(String reason) {
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicConverter.java b/java/src/main/java/org/msgpack/util/codegen/DynamicConverter.java
index ab599f1..8919a67 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicConverter.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicConverter.java
@@ -1,9 +1,16 @@
 package org.msgpack.util.codegen;
 
+import java.util.List;
+
 import org.msgpack.MessageConverter;
 
 public class DynamicConverter {
 	public static MessageConverter create(Class c) {
-		return DynamicTemplate.create(c);
+		return create(c, null);
+	}
+
+	public static MessageConverter create(Class c,
+			List fieldOpts) {
+		return DynamicTemplate.create(c, fieldOpts);
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
index 643ff52..fbd019d 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
@@ -1,13 +1,19 @@
 package org.msgpack.util.codegen;
 
+import java.util.List;
+
 import org.msgpack.MessagePacker;
 
 public class DynamicPacker {
 
 	public static MessagePacker create(Class c) {
+		return create(c, null);
+	}
+
+	public static MessagePacker create(Class c, List fieldOpts) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
-			Class packerClass = gen.generateMessagePackerClass(c);
+			Class packerClass = gen.generateMessagePackerClass(c, fieldOpts);
 			return (MessagePacker) packerClass.newInstance();
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java b/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
index 60a51e8..673c344 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
@@ -1,13 +1,19 @@
 package org.msgpack.util.codegen;
 
+import java.util.List;
+
 import org.msgpack.Template;
 import org.msgpack.util.codegen.DynamicCodeGenBase.TemplateAccessor;
 
 public class DynamicTemplate {
 	public static Template create(Class c) {
+		return create(c, null);
+	}
+
+	public static Template create(Class c, List fieldOpts) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
-			Class tmplClass = gen.generateTemplateClass(c);
+			Class tmplClass = gen.generateTemplateClass(c, fieldOpts);
 			Object obj = tmplClass.newInstance();
 			((TemplateAccessor) obj).setTemplates(gen.getTemplates(c));
 			return (Template) obj;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicUnpacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicUnpacker.java
index 83590ce..8b927cb 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicUnpacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicUnpacker.java
@@ -1,9 +1,15 @@
 package org.msgpack.util.codegen;
 
+import java.util.List;
+
 import org.msgpack.MessageUnpacker;
 
 public class DynamicUnpacker {
 	public static MessageUnpacker create(Class c) {
-		return DynamicTemplate.create(c);
+		return create(c, null);
+	}
+
+	public static MessageUnpacker create(Class c, List fieldOpts) {
+		return DynamicTemplate.create(c, fieldOpts);
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/FieldOption.java b/java/src/main/java/org/msgpack/util/codegen/FieldOption.java
index 723a765..127688d 100644
--- a/java/src/main/java/org/msgpack/util/codegen/FieldOption.java
+++ b/java/src/main/java/org/msgpack/util/codegen/FieldOption.java
@@ -4,7 +4,22 @@ import org.msgpack.Template;
 
 public class FieldOption {
 
-    public String name;
-    
-    public Template tmpl;
+	private static final String NULL_ERR_MSG = "param is FieldOption is null.";
+
+	String name;
+
+	Template tmpl;
+
+	public FieldOption(final String name, final Template tmpl) {
+		if (name == null) {
+			throw new NullPointerException(String.format("%s %s", new Object[] {
+					"1st", NULL_ERR_MSG }));
+		}
+		if (tmpl == null) {
+			throw new NullPointerException(String.format("%s %s", new Object[] {
+					"2nd", NULL_ERR_MSG }));
+		}
+		this.name = name;
+		this.tmpl = tmpl;
+	}
 }
diff --git a/java/src/test/java/org/msgpack/packer/TestPackConvert.java b/java/src/test/java/org/msgpack/packer/TestPackConvert.java
index 98caf82..26f3313 100644
--- a/java/src/test/java/org/msgpack/packer/TestPackConvert.java
+++ b/java/src/test/java/org/msgpack/packer/TestPackConvert.java
@@ -9,13 +9,111 @@ import junit.framework.TestCase;
 import org.junit.Test;
 import org.msgpack.MessagePackObject;
 import org.msgpack.MessagePacker;
+import org.msgpack.MessageTypeException;
 import org.msgpack.Packer;
+import org.msgpack.Template;
 import org.msgpack.Util;
+import org.msgpack.template.BigIntegerTemplate;
+import org.msgpack.template.BooleanTemplate;
+import org.msgpack.template.ByteTemplate;
+import org.msgpack.template.DoubleTemplate;
+import org.msgpack.template.FloatTemplate;
+import org.msgpack.template.IntegerTemplate;
+import org.msgpack.template.LongTemplate;
+import org.msgpack.template.OptionalTemplate;
+import org.msgpack.template.ShortTemplate;
+import org.msgpack.template.StringTemplate;
 
 public class TestPackConvert extends TestCase {
+
+	@Test
+	public void testByte() throws Exception {
+		_testByte((byte) 0);
+		_testByte((byte) -1);
+		_testByte((byte) 1);
+		_testByte(Byte.MIN_VALUE);
+		_testByte(Byte.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testByte((byte) rand.nextInt());
+		}
+	}
+
+	static void _testByte(Byte src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = BytePacker.getInstance();
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		assertEquals(src.byteValue(), obj.asByte());
+	}
+
+	@Test
+	public void testNullByte() throws Exception {
+		Byte src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(BytePacker.getInstance());
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = null;
+		Byte dst = null;
+		try {
+			tmpl = ByteTemplate.getInstance();
+			dst = (Byte) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(ByteTemplate.getInstance());
+		dst = (Byte) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
+	@Test
+	public void testShort() throws Exception {
+		_testShort((short) 0);
+		_testShort((short) -1);
+		_testShort((short) 1);
+		_testShort(Short.MIN_VALUE);
+		_testShort(Short.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testShort((short) rand.nextInt());
+		}
+	}
+
+	static void _testShort(Short src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = ShortPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		assertEquals(src.shortValue(), obj.asShort());
+	}
+
+	@Test
+	public void testNullShort() throws Exception {
+		Short src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(ShortPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = null;
+		Short dst = null;
+		try {
+			tmpl = ShortTemplate.getInstance();
+			dst = (Short) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(ShortTemplate.getInstance());
+		dst = (Short) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testInteger() throws Exception {
-		// _testInteger(null); // FIXME
 		_testInteger(0);
 		_testInteger(-1);
 		_testInteger(1);
@@ -35,9 +133,30 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src.intValue(), obj.asInt());
 	}
 
+	@Test
+	public void testNullInteger() throws Exception {
+		Integer src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(IntegerPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = null;
+		Integer dst = null;
+		try {
+			tmpl = IntegerTemplate.getInstance();
+			dst = (Integer) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(IntegerTemplate.getInstance());
+		dst = (Integer) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testLong() throws Exception {
-		// _testLong((null); // FIXME
 		_testLong((long) 0);
 		_testLong((long) -1);
 		_testLong((long) 1);
@@ -59,9 +178,30 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src.longValue(), obj.asLong());
 	}
 
+	@Test
+	public void testNullLong() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(LongPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = null;
+		Long dst = null;
+		try {
+			tmpl = LongTemplate.getInstance();
+			dst = (Long) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(LongTemplate.getInstance());
+		dst = (Long) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testBigInteger() throws Exception {
-		// _testBigInteger(null); // FIXME
 		_testBigInteger(BigInteger.valueOf(0));
 		_testBigInteger(BigInteger.valueOf(-1));
 		_testBigInteger(BigInteger.valueOf(1));
@@ -86,9 +226,31 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src, obj.asBigInteger());
 	}
 
+	@Test
+	public void testNullBigInteger() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(BigIntegerPacker
+				.getInstance());
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = null;
+		BigInteger dst = null;
+		try {
+			tmpl = BigIntegerTemplate.getInstance();
+			dst = (BigInteger) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(BigIntegerTemplate.getInstance());
+		dst = (BigInteger) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testFloat() throws Exception {
-		// _testFloat(null); // FIXME
 		_testFloat((float) 0.0);
 		_testFloat((float) -0.0);
 		_testFloat((float) 1.0);
@@ -112,9 +274,30 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src.floatValue(), obj.asFloat(), 10e-10);
 	}
 
+	@Test
+	public void testNullFloat() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(FloatPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = null;
+		Float dst = null;
+		try {
+			tmpl = FloatTemplate.getInstance();
+			dst = (Float) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(FloatTemplate.getInstance());
+		dst = (Float) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testDouble() throws Exception {
-		// _testDouble(null); // FIXME
 		_testDouble((double) 0.0);
 		_testDouble((double) -0.0);
 		_testDouble((double) 1.0);
@@ -137,9 +320,30 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src.doubleValue(), obj.asDouble(), 10e-10);
 	}
 
+	@Test
+	public void testNullDouble() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DoublePacker.getInstance());
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = null;
+		Double dst = null;
+		try {
+			tmpl = DoubleTemplate.getInstance();
+			dst = (Double) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(DoubleTemplate.getInstance());
+		dst = (Double) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testBoolean() throws Exception {
-		// _testBoolean(null); // FIXME
 		_testBoolean(false);
 		_testBoolean(true);
 	}
@@ -152,9 +356,30 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src.booleanValue(), obj.asBoolean());
 	}
 
+	@Test
+	public void testNullBoolean() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(BooleanPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = null;
+		Boolean dst = null;
+		try {
+			tmpl = BooleanTemplate.getInstance();
+			dst = (Boolean) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(BooleanTemplate.getInstance());
+		dst = (Boolean) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testString() throws Exception {
-		// _testString(null); // FIXME
 		_testString("");
 		_testString("a");
 		_testString("ab");
@@ -198,4 +423,26 @@ public class TestPackConvert extends TestCase {
 		MessagePackObject obj = Util.unpackOne(out.toByteArray());
 		assertEquals(src, obj.asString());
 	}
+
+	@Test
+	public void testNullString() throws Exception {
+		String src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(StringPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = null;
+		String dst = null;
+		try {
+			tmpl = StringTemplate.getInstance();
+			dst = (String) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(StringTemplate.getInstance());
+		dst = (String) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
 }
diff --git a/java/src/test/java/org/msgpack/packer/TestPackUnpack.java b/java/src/test/java/org/msgpack/packer/TestPackUnpack.java
index f1d7842..2dd631d 100644
--- a/java/src/test/java/org/msgpack/packer/TestPackUnpack.java
+++ b/java/src/test/java/org/msgpack/packer/TestPackUnpack.java
@@ -9,13 +9,117 @@ import junit.framework.TestCase;
 
 import org.junit.Test;
 import org.msgpack.MessagePacker;
+import org.msgpack.MessageTypeException;
 import org.msgpack.Packer;
+import org.msgpack.Template;
 import org.msgpack.Unpacker;
+import org.msgpack.template.BigIntegerTemplate;
+import org.msgpack.template.BooleanTemplate;
+import org.msgpack.template.ByteTemplate;
+import org.msgpack.template.DoubleTemplate;
+import org.msgpack.template.FloatTemplate;
+import org.msgpack.template.IntegerTemplate;
+import org.msgpack.template.LongTemplate;
+import org.msgpack.template.OptionalTemplate;
+import org.msgpack.template.ShortTemplate;
+import org.msgpack.template.StringTemplate;
 
 public class TestPackUnpack extends TestCase {
+
+	@Test
+	public void testByte() throws Exception {
+		_testByte((byte) 0);
+		_testByte((byte) -1);
+		_testByte((byte) 1);
+		_testByte(Byte.MIN_VALUE);
+		_testByte(Byte.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testByte((byte) rand.nextInt());
+		}
+	}
+
+	static void _testByte(Byte src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = BytePacker.getInstance();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker unpacker = new Unpacker(in);
+		assertEquals(src.byteValue(), unpacker.unpackByte());
+	}
+
+	@Test
+	public void testNullByte() throws Exception {
+		Byte src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(BytePacker.getInstance());
+		packer.pack(new Packer(out), src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Byte dst = null;
+		try {
+			tmpl = ByteTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (Byte) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(ByteTemplate.getInstance());
+		dst = (Byte) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
+	@Test
+	public void testSort() throws Exception {
+		_testShort((short) 0);
+		_testShort((short) -1);
+		_testShort((short) 1);
+		_testShort(Short.MIN_VALUE);
+		_testShort(Short.MAX_VALUE);
+		Random rand = new Random();
+		for (int i = 0; i < 1000; i++) {
+			_testShort((short) rand.nextInt());
+		}
+	}
+
+	static void _testShort(Short src) throws Exception {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = ShortPacker.getInstance();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker unpacker = new Unpacker(in);
+		assertEquals(src.shortValue(), unpacker.unpackShort());
+	}
+
+	@Test
+	public void testNullShort() throws Exception {
+		Short src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(ShortPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Short dst = null;
+		try {
+			tmpl = ShortTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (Short) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(ShortTemplate.getInstance());
+		dst = (Short) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testInteger() throws Exception {
-		// _testInteger(null); // FIXME
 		_testInteger(0);
 		_testInteger(-1);
 		_testInteger(1);
@@ -36,9 +140,32 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src.intValue(), unpacker.unpackInt());
 	}
 
+	@Test
+	public void testNullInteger() throws Exception {
+		Integer src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(IntegerPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Integer dst = null;
+		try {
+			tmpl = IntegerTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (Integer) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(IntegerTemplate.getInstance());
+		dst = (Integer) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testLong() throws Exception {
-		// _testLong((null); // FIXME
 		_testLong((long) 0);
 		_testLong((long) -1);
 		_testLong((long) 1);
@@ -61,9 +188,32 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src.longValue(), unpacker.unpackLong());
 	}
 
+	@Test
+	public void testNullLong() throws Exception {
+		Integer src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(LongPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Long dst = null;
+		try {
+			tmpl = LongTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (Long) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(LongTemplate.getInstance());
+		dst = (Long) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testBigInteger() throws Exception {
-		// _testBigInteger(null); // FIXME
 		_testBigInteger(BigInteger.valueOf(0));
 		_testBigInteger(BigInteger.valueOf(-1));
 		_testBigInteger(BigInteger.valueOf(1));
@@ -89,9 +239,33 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src, unpacker.unpackBigInteger());
 	}
 
+	@Test
+	public void testNullBigInteger() throws Exception {
+		BigInteger src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(BigIntegerPacker
+				.getInstance());
+		packer.pack(new Packer(out), src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		BigInteger dst = null;
+		try {
+			tmpl = BigIntegerTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (BigInteger) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(BigIntegerTemplate.getInstance());
+		dst = (BigInteger) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testFloat() throws Exception {
-		// _testFloat(null); // FIXME
 		_testFloat((float) 0.0);
 		_testFloat((float) -0.0);
 		_testFloat((float) 1.0);
@@ -116,9 +290,32 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src.floatValue(), unpacker.unpackFloat(), 10e-10);
 	}
 
+	@Test
+	public void testNullFloat() throws Exception {
+		Float src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(FloatPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Float dst = null;
+		try {
+			tmpl = FloatTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (Float) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(FloatTemplate.getInstance());
+		dst = (Float) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testDouble() throws Exception {
-		// _testDouble(null); // FIXME
 		_testDouble((double) 0.0);
 		_testDouble((double) -0.0);
 		_testDouble((double) 1.0);
@@ -142,9 +339,32 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src.doubleValue(), unpacker.unpackDouble(), 10e-10);
 	}
 
+	@Test
+	public void testNullDouble() throws Exception {
+		Double src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DoublePacker.getInstance());
+		packer.pack(new Packer(out), src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Double dst = null;
+		try {
+			tmpl = DoubleTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (Double) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(DoubleTemplate.getInstance());
+		dst = (Double) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testBoolean() throws Exception {
-		// _testBoolean(null); // FIXME
 		_testBoolean(false);
 		_testBoolean(true);
 	}
@@ -158,9 +378,32 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src.booleanValue(), unpacker.unpackBoolean());
 	}
 
+	@Test
+	public void testNullBoolean() throws Exception {
+		Boolean src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(BooleanPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Boolean dst = null;
+		try {
+			tmpl = BooleanTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (Boolean) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(BooleanTemplate.getInstance());
+		dst = (Boolean) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testString() throws Exception {
-		// _testString(null); // FIXME
 		_testString("");
 		_testString("a");
 		_testString("ab");
@@ -205,4 +448,28 @@ public class TestPackUnpack extends TestCase {
 		Unpacker unpacker = new Unpacker(in);
 		assertEquals(src, unpacker.unpackString());
 	}
+
+	@Test
+	public void testNullString() throws Exception {
+		String src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(StringPacker.getInstance());
+		packer.pack(new Packer(out), src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		String dst = null;
+		try {
+			tmpl = StringTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (String) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(StringTemplate.getInstance());
+		dst = (String) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
 }
diff --git a/java/src/test/java/org/msgpack/template/TestPackConvert.java b/java/src/test/java/org/msgpack/template/TestPackConvert.java
index 15237b6..01063a5 100644
--- a/java/src/test/java/org/msgpack/template/TestPackConvert.java
+++ b/java/src/test/java/org/msgpack/template/TestPackConvert.java
@@ -10,6 +10,7 @@ import java.util.Random;
 
 import org.junit.Test;
 import org.msgpack.MessagePackObject;
+import org.msgpack.MessageTypeException;
 import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Util;
@@ -20,7 +21,6 @@ public class TestPackConvert extends TestCase {
 
 	@Test
 	public void testInteger() throws Exception {
-		// _testInteger(null); // FIXME
 		_testInteger(0);
 		_testInteger(-1);
 		_testInteger(1);
@@ -41,9 +41,28 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src, dst);
 	}
 
+	@Test
+	public void testNullInteger() throws Exception {
+		Integer src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = IntegerTemplate.getInstance();
+		Integer dst = null;
+		try {
+			dst = (Integer) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(IntegerTemplate.getInstance());
+		dst = (Integer) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testLong() throws Exception {
-		// _testLong(null); // FIXME
 		_testLong((long) 0);
 		_testLong((long) -1);
 		_testLong((long) 1);
@@ -66,9 +85,28 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src, dst);
 	}
 
+	@Test
+	public void testNullLong() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = LongTemplate.getInstance();
+		Long dst = null;
+		try {
+			dst = (Long) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(LongTemplate.getInstance());
+		dst = (Long) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testBiginteger() throws Exception {
-		// _testBigInteger(null); // FIXME
 		_testBigInteger(BigInteger.valueOf(0));
 		_testBigInteger(BigInteger.valueOf(-1));
 		_testBigInteger(BigInteger.valueOf(1));
@@ -94,9 +132,28 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src, dst);
 	}
 
+	@Test
+	public void testNullBigInteger() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = BigIntegerTemplate.getInstance();
+		BigInteger dst = null;
+		try {
+			dst = (BigInteger) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(BigIntegerTemplate.getInstance());
+		dst = (BigInteger) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testFloat() throws Exception {
-		// _testFloat(null); // FIXME
 		_testFloat((float) 0.0);
 		_testFloat((float) -0.0);
 		_testFloat((float) 1.0);
@@ -121,9 +178,28 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src, dst, 10e-10);
 	}
 
+	@Test
+	public void testNullFloat() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = FloatTemplate.getInstance();
+		Float dst = null;
+		try {
+			dst = (Float) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(FloatTemplate.getInstance());
+		dst = (Float) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testDouble() throws Exception {
-		// _testDouble(null); // FIXME
 		_testDouble((double) 0.0);
 		_testDouble((double) -0.0);
 		_testDouble((double) 1.0);
@@ -148,9 +224,28 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src, dst, 10e-10);
 	}
 
+	@Test
+	public void testNullDouble() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = DoubleTemplate.getInstance();
+		Double dst = null;
+		try {
+			dst = (Double) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(DoubleTemplate.getInstance());
+		dst = (Double) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testBoolean() throws Exception {
-		// _testBoolean(null); // FIXME
 		_testBoolean(false);
 		_testBoolean(true);
 	}
@@ -164,9 +259,28 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src, dst);
 	}
 
+	@Test
+	public void testNullBoolean() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = BooleanTemplate.getInstance();
+		Boolean dst = null;
+		try {
+			dst = (Boolean) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(BooleanTemplate.getInstance());
+		dst = (Boolean) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testString() throws Exception {
-		// _testString(null); // FIXME
 		_testString("");
 		_testString("a");
 		_testString("ab");
@@ -212,10 +326,29 @@ public class TestPackConvert extends TestCase {
 		assertEquals(src, dst);
 	}
 
+	@Test
+	public void testNullString() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = StringTemplate.getInstance();
+		String dst = null;
+		try {
+			dst = (String) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(StringTemplate.getInstance());
+		dst = (String) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@SuppressWarnings("unchecked")
 	@Test
 	public void testList() throws Exception {
-		// nullList // FIXME
 		List emptyList = new ArrayList();
 		{
 			ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -261,10 +394,31 @@ public class TestPackConvert extends TestCase {
 		}
 	}
 
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testNullList() throws Exception {
+		List src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = new ListTemplate(StringTemplate.getInstance());
+		List dst = null;
+		try {
+			dst = (List) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(new ListTemplate(StringTemplate
+				.getInstance()));
+		dst = (List) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
+
 	@SuppressWarnings("unchecked")
 	@Test
 	public void testMap() throws Exception {
-		// nullMap // FIXME
 		Map emptyMap = new HashMap();
 		{
 			ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -317,4 +471,27 @@ public class TestPackConvert extends TestCase {
 			}
 		}
 	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testNullMap() throws Exception {
+		Map src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		MessagePackObject obj = Util.unpackOne(out.toByteArray());
+		Template tmpl = new MapTemplate(StringTemplate.getInstance(),
+				StringTemplate.getInstance());
+		Map dst = null;
+		try {
+			dst = (Map) tmpl.convert(obj);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		obj = Util.unpackOne(out.toByteArray());
+		tmpl = new OptionalTemplate(new MapTemplate(StringTemplate
+				.getInstance(), StringTemplate.getInstance()));
+		dst = (Map) tmpl.convert(obj);
+		assertEquals(src, dst);
+	}
 }
diff --git a/java/src/test/java/org/msgpack/template/TestPackUnpack.java b/java/src/test/java/org/msgpack/template/TestPackUnpack.java
index fbd510f..29ee78d 100644
--- a/java/src/test/java/org/msgpack/template/TestPackUnpack.java
+++ b/java/src/test/java/org/msgpack/template/TestPackUnpack.java
@@ -10,6 +10,7 @@ import java.util.Map;
 import java.util.Random;
 
 import org.junit.Test;
+import org.msgpack.MessageTypeException;
 import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
@@ -19,7 +20,6 @@ import junit.framework.TestCase;
 public class TestPackUnpack extends TestCase {
 	@Test
 	public void testInteger() throws Exception {
-		// _testInteger(null); // FIXME
 		_testInteger(0);
 		_testInteger(-1);
 		_testInteger(1);
@@ -40,9 +40,31 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src, dst);
 	}
 
+	@Test
+	public void testNullInteger() throws Exception {
+		Integer src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Integer dst = null;
+		try {
+			tmpl = IntegerTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (Integer) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(IntegerTemplate.getInstance());
+		dst = (Integer) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testLong() throws Exception {
-		// _testLong(null); // FIXME
 		_testLong((long) 0);
 		_testLong((long) -1);
 		_testLong((long) 1);
@@ -65,9 +87,31 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src, dst);
 	}
 
+	@Test
+	public void testNullLong() throws Exception {
+		Long src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Long dst = null;
+		try {
+			tmpl = LongTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (Long) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(LongTemplate.getInstance());
+		dst = (Long) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testBigInteger() throws Exception {
-		// _testBigInteger(null); // FIXME
 		_testBigInteger(BigInteger.valueOf(0));
 		_testBigInteger(BigInteger.valueOf(-1));
 		_testBigInteger(BigInteger.valueOf(1));
@@ -93,9 +137,31 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src, dst);
 	}
 
+	@Test
+	public void testNullBigInteger() throws Exception {
+		BigInteger src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		BigInteger dst = null;
+		try {
+			tmpl = BigIntegerTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (BigInteger) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(BigIntegerTemplate.getInstance());
+		dst = (BigInteger) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testFloat() throws Exception {
-		// _testFloat(null); // FIXME
 		_testFloat((float) 0.0);
 		_testFloat((float) -0.0);
 		_testFloat((float) 1.0);
@@ -122,7 +188,6 @@ public class TestPackUnpack extends TestCase {
 
 	@Test
 	public void testDouble() throws Exception {
-		// _testDouble(null); // FIXME
 		_testDouble((double) 0.0);
 		_testDouble((double) -0.0);
 		_testDouble((double) 1.0);
@@ -147,9 +212,31 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src, dst, 10e-10);
 	}
 
+	@Test
+	public void testNullDouble() throws Exception {
+		Double src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Double dst = null;
+		try {
+			tmpl = DoubleTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (Double) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(DoubleTemplate.getInstance());
+		dst = (Double) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testBoolean() throws Exception {
-		// _testBoolean(null); // FIXME
 		_testBoolean(false);
 		_testBoolean(true);
 	}
@@ -163,9 +250,31 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src, dst);
 	}
 
+	@Test
+	public void testNullBoolean() throws Exception {
+		Boolean src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Boolean dst = null;
+		try {
+			tmpl = BooleanTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (Boolean) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(BooleanTemplate.getInstance());
+		dst = (Boolean) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@Test
 	public void testString() throws Exception {
-		// _testString(null); // FIXME
 		_testString("");
 		_testString("a");
 		_testString("ab");
@@ -211,10 +320,32 @@ public class TestPackUnpack extends TestCase {
 		assertEquals(src, dst);
 	}
 
+	@Test
+	public void testNullString() throws Exception {
+		String src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		String dst = null;
+		try {
+			tmpl = StringTemplate.getInstance();
+			unpacker.wrap(bytes);
+			dst = (String) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(StringTemplate.getInstance());
+		dst = (String) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@SuppressWarnings("unchecked")
 	@Test
 	public void testList() throws Exception {
-		// nullList // FIXME
 		List emptyList = new ArrayList();
 		{
 			ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -262,10 +393,34 @@ public class TestPackUnpack extends TestCase {
 		}
 	}
 
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testNullList() throws Exception {
+		List src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		List dst = null;
+		try {
+			tmpl = new ListTemplate(StringTemplate.getInstance());
+			unpacker.wrap(bytes);
+			dst = (List) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(new ListTemplate(StringTemplate
+				.getInstance()));
+		dst = (List) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
+
 	@SuppressWarnings("unchecked")
 	@Test
 	public void testMap() throws Exception {
-		// nullMap // FIXME
 		Map emptyMap = new HashMap();
 		{
 			ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -323,4 +478,30 @@ public class TestPackUnpack extends TestCase {
 			}
 		}
 	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testNullMap() throws Exception {
+		Map src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		new Packer(out).pack(src);
+		byte[] bytes = out.toByteArray();
+		Template tmpl = null;
+		Unpacker unpacker = new Unpacker();
+		Map dst = null;
+		try {
+			tmpl = new MapTemplate(StringTemplate.getInstance(), StringTemplate
+					.getInstance());
+			unpacker.wrap(bytes);
+			dst = (Map) tmpl.unpack(unpacker);
+			fail();
+		} catch (Exception e) {
+			assertTrue(e instanceof MessageTypeException);
+		}
+		unpacker.wrap(bytes);
+		tmpl = new OptionalTemplate(new MapTemplate(StringTemplate
+				.getInstance(), StringTemplate.getInstance()));
+		dst = (Map) tmpl.unpack(unpacker);
+		assertEquals(src, dst);
+	}
 }
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java b/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
index d2cf35c..63532a0 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
@@ -25,12 +25,15 @@ import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
 import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.annotation.MessagePackOptional;
 import org.msgpack.annotation.MessagePackOrdinalEnum;
+import org.msgpack.packer.OptionalPacker;
+import org.msgpack.template.OptionalTemplate;
 
 public class TestPackConvert extends TestCase {
 
 	@Test
-	public void testPrimitiveTypeField00s() throws Exception {
+	public void testPrimitiveTypeFields00() throws Exception {
 		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
 		src.f0 = (byte) 0;
 		src.f1 = 1;
@@ -63,38 +66,14 @@ public class TestPackConvert extends TestCase {
 
 	@Test
 	public void testPrimitiveTypeFields01() throws Exception {
-		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(PrimitiveTypeFieldsClass.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class);
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
-				.convert(mpo);
-		assertEquals(src.f0, dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2, dst.f2);
-		assertEquals(src.f3, dst.f3);
-		assertEquals(src.f4, dst.f4);
-		assertEquals(src.f5, dst.f5);
-		assertEquals(src.f6, dst.f6);
-		assertFalse(it.hasNext());
-	}
-
-	@Test
-	public void testPrimitiveTypeFields02() throws Exception {
 		PrimitiveTypeFieldsClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(PrimitiveTypeFieldsClass.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(PrimitiveTypeFieldsClass.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(PrimitiveTypeFieldsClass.class));
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -118,6 +97,105 @@ public class TestPackConvert extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalPrimitiveTypeFields00() throws Exception {
+		OptionalPrimitiveTypeFieldsClass src = new OptionalPrimitiveTypeFieldsClass();
+		src.f0 = (byte) 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f6 = false;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalPrimitiveTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalPrimitiveTypeFieldsClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalPrimitiveTypeFieldsClass dst = (OptionalPrimitiveTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalPrimitiveTypeFields01() throws Exception {
+		OptionalPrimitiveTypeFieldsClass src = new OptionalPrimitiveTypeFieldsClass();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalPrimitiveTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalPrimitiveTypeFieldsClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalPrimitiveTypeFieldsClass dst = (OptionalPrimitiveTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalPrimitiveTypeFields02() throws Exception {
+		OptionalPrimitiveTypeFieldsClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(OptionalPrimitiveTypeFieldsClass.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(OptionalPrimitiveTypeFieldsClass.class));
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalPrimitiveTypeFieldsClass dst = (OptionalPrimitiveTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class OptionalPrimitiveTypeFieldsClass {
+		@MessagePackOptional
+		public byte f0;
+		@MessagePackOptional
+		public short f1;
+		@MessagePackOptional
+		public int f2;
+		@MessagePackOptional
+		public long f3;
+		@MessagePackOptional
+		public float f4;
+		@MessagePackOptional
+		public double f5;
+		@MessagePackOptional
+		public boolean f6;
+
+		public OptionalPrimitiveTypeFieldsClass() {
+		}
+	}
+
 	@Test
 	public void testGeneralReferenceTypeFieldsClass00() throws Exception {
 		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
@@ -159,54 +237,15 @@ public class TestPackConvert extends TestCase {
 	}
 
 	@Test
-	public void XtestGeneralReferenceTypeFieldsClass01() throws Exception {
-		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
-		src.f0 = null;
-		src.f1 = null;
-		src.f2 = null;
-		src.f3 = null;
-		src.f4 = null;
-		src.f5 = null;
-		src.f6 = null;
-		src.f7 = null;
-		src.f8 = null;
-		src.f9 = null;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(GeneralReferenceTypeFieldsClass.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(GeneralReferenceTypeFieldsClass.class);
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
-				.convert(mpo);
-		assertEquals(src.f0, dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2, dst.f2);
-		assertEquals(src.f3, dst.f3);
-		assertEquals(src.f4, dst.f4);
-		assertEquals(src.f5, dst.f5);
-		assertEquals(src.f6, dst.f6);
-		assertEquals(src.f7, dst.f7);
-		assertEquals(src.f8, dst.f8);
-		assertEquals(src.f9, dst.f9);
-		assertFalse(it.hasNext());
-	}
-
-	@Test
-	public void XtestGeneralReferenceTypeFieldsClass02() throws Exception {
+	public void testGeneralReferenceTypeFieldsClass01() throws Exception {
 		GeneralReferenceTypeFieldsClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(GeneralReferenceTypeFieldsClass.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(GeneralReferenceTypeFieldsClass.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(GeneralReferenceTypeFieldsClass.class));
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -233,6 +272,134 @@ public class TestPackConvert extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalGeneralReferenceTypeFieldsClass00()
+			throws Exception {
+		OptionalGeneralReferenceTypeFieldsClass src = new OptionalGeneralReferenceTypeFieldsClass();
+		src.f0 = 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = (long) 3;
+		src.f4 = (float) 4;
+		src.f5 = (double) 5;
+		src.f6 = false;
+		src.f7 = new BigInteger("7");
+		src.f8 = "8";
+		src.f9 = new byte[] { 0x01, 0x02 };
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalGeneralReferenceTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalGeneralReferenceTypeFieldsClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalGeneralReferenceTypeFieldsClass dst = (OptionalGeneralReferenceTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9[0], dst.f9[0]);
+		assertEquals(src.f9[1], dst.f9[1]);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalGeneralReferenceTypeFieldsClass01()
+			throws Exception {
+		OptionalGeneralReferenceTypeFieldsClass src = new OptionalGeneralReferenceTypeFieldsClass();
+		src.f0 = null;
+		src.f1 = null;
+		src.f2 = null;
+		src.f3 = null;
+		src.f4 = null;
+		src.f5 = null;
+		src.f6 = null;
+		src.f7 = null;
+		src.f8 = null;
+		src.f9 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalGeneralReferenceTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalGeneralReferenceTypeFieldsClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalGeneralReferenceTypeFieldsClass dst = (OptionalGeneralReferenceTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9, dst.f9);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalGeneralReferenceTypeFieldsClass02()
+			throws Exception {
+		OptionalGeneralReferenceTypeFieldsClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(OptionalGeneralReferenceTypeFieldsClass.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(OptionalGeneralReferenceTypeFieldsClass.class));
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalGeneralReferenceTypeFieldsClass dst = (OptionalGeneralReferenceTypeFieldsClass) tmpl
+				.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class OptionalGeneralReferenceTypeFieldsClass {
+		@MessagePackOptional
+		public Byte f0;
+		@MessagePackOptional
+		public Short f1;
+		@MessagePackOptional
+		public Integer f2;
+		@MessagePackOptional
+		public Long f3;
+		@MessagePackOptional
+		public Float f4;
+		@MessagePackOptional
+		public Double f5;
+		@MessagePackOptional
+		public Boolean f6;
+		@MessagePackOptional
+		public BigInteger f7;
+		@MessagePackOptional
+		public String f8;
+		@MessagePackOptional
+		public byte[] f9;
+
+		public OptionalGeneralReferenceTypeFieldsClass() {
+		}
+	}
+
 	@Test
 	public void testListTypes00() throws Exception {
 		SampleListTypes src = new SampleListTypes();
@@ -293,38 +460,14 @@ public class TestPackConvert extends TestCase {
 
 	@Test
 	public void testListTypes01() throws Exception {
-		SampleListTypes src = new SampleListTypes();
-		src.f0 = new ArrayList();
-		src.f1 = null;
-		src.f2 = new ArrayList();
-		src.f3 = null;
-		src.f4 = new ArrayList();
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleListTypes.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		SampleListTypes dst = (SampleListTypes) tmpl.convert(mpo);
-		assertEquals(src.f0.size(), dst.f0.size());
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2.size(), dst.f2.size());
-		assertEquals(src.f3, dst.f3);
-		assertEquals(src.f4.size(), dst.f4.size());
-		assertFalse(it.hasNext());
-	}
-
-	@Test
-	public void testListTypes02() throws Exception {
 		SampleListTypes src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleListTypes.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleListTypes.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleListTypes.class));
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -354,6 +497,141 @@ public class TestPackConvert extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalListTypes00() throws Exception {
+		SampleOptionalListTypes src = new SampleOptionalListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = new ArrayList();
+		src.f1.add(1);
+		src.f1.add(2);
+		src.f1.add(3);
+		src.f2 = new ArrayList();
+		src.f2.add("e1");
+		src.f2.add("e2");
+		src.f2.add("e3");
+		src.f3 = new ArrayList>();
+		src.f3.add(src.f2);
+		src.f4 = new ArrayList();
+		SampleOptionalListNestedType slnt = new SampleOptionalListNestedType();
+		slnt.f0 = new byte[] { 0x01, 0x02 };
+		slnt.f1 = "muga";
+		src.f4.add(slnt);
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalListTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleOptionalListTypes.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleOptionalListTypes dst = (SampleOptionalListTypes) tmpl
+				.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		for (int i = 0; i < src.f1.size(); ++i) {
+			assertEquals(src.f1.get(i), dst.f1.get(i));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		for (int i = 0; i < src.f2.size(); ++i) {
+			assertEquals(src.f2.get(i), dst.f2.get(i));
+		}
+		assertEquals(src.f3.size(), dst.f3.size());
+		for (int i = 0; i < src.f3.size(); ++i) {
+			List srclist = src.f3.get(i);
+			List dstlist = dst.f3.get(i);
+			assertEquals(srclist.size(), dstlist.size());
+			for (int j = 0; j < srclist.size(); ++j) {
+				assertEquals(srclist.get(j), dstlist.get(j));
+			}
+		}
+		assertEquals(src.f4.size(), dst.f4.size());
+		for (int i = 0; i < src.f4.size(); ++i) {
+			SampleOptionalListNestedType s = src.f4.get(i);
+			SampleOptionalListNestedType d = dst.f4.get(i);
+			assertEquals(s.f0[0], d.f0[0]);
+			assertEquals(s.f0[1], d.f0[1]);
+			assertEquals(s.f1, d.f1);
+		}
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalListTypes01() throws Exception {
+		SampleOptionalListTypes src = new SampleOptionalListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = null;
+		src.f2 = new ArrayList();
+		src.f3 = null;
+		src.f4 = new ArrayList();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalListTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleOptionalListTypes.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleOptionalListTypes dst = (SampleOptionalListTypes) tmpl
+				.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4.size(), dst.f4.size());
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalListTypes02() throws Exception {
+		SampleOptionalListTypes src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleOptionalListTypes.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleOptionalListTypes.class));
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleOptionalListTypes dst = (SampleOptionalListTypes) tmpl
+				.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class SampleOptionalListTypes {
+		@MessagePackOptional
+		public List f0;
+		@MessagePackOptional
+		public List f1;
+		@MessagePackOptional
+		public List f2;
+		@MessagePackOptional
+		public List> f3;
+		@MessagePackOptional
+		public List f4;
+
+		public SampleOptionalListTypes() {
+		}
+	}
+
+	@MessagePackMessage
+	public static class SampleOptionalListNestedType {
+		@MessagePackOptional
+		public byte[] f0;
+		@MessagePackOptional
+		public String f1;
+
+		public SampleOptionalListNestedType() {
+		}
+	}
+
 	@Test
 	public void testMapTypes00() throws Exception {
 		SampleMapTypes src = new SampleMapTypes();
@@ -398,36 +676,16 @@ public class TestPackConvert extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
-	@Test
-	public void testMapTypes01() throws Exception {
-		SampleMapTypes src = new SampleMapTypes();
-		src.f0 = new HashMap();
-		src.f1 = null;
-		src.f2 = new HashMap();
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleMapTypes.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		SampleMapTypes dst = (SampleMapTypes) tmpl.convert(mpo);
-		assertEquals(src.f0.size(), dst.f0.size());
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2.size(), dst.f2.size());
-		assertFalse(it.hasNext());
-	}
-
 	@Test
 	public void testMapTypes02() throws Exception {
 		SampleMapTypes src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleMapTypes.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleMapTypes.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleMapTypes.class));
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -446,6 +704,105 @@ public class TestPackConvert extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalMapTypes00() throws Exception {
+		SampleOptionalMapTypes src = new SampleOptionalMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = new HashMap();
+		src.f1.put(1, 1);
+		src.f1.put(2, 2);
+		src.f1.put(3, 3);
+		src.f2 = new HashMap();
+		src.f2.put("k1", 1);
+		src.f2.put("k2", 2);
+		src.f2.put("k3", 3);
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalMapTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleOptionalMapTypes.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleOptionalMapTypes dst = (SampleOptionalMapTypes) tmpl.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		Iterator srcf1 = src.f1.keySet().iterator();
+		Iterator dstf1 = dst.f1.keySet().iterator();
+		while (srcf1.hasNext()) {
+			Integer s1 = srcf1.next();
+			Integer d1 = dstf1.next();
+			assertEquals(s1, d1);
+			assertEquals(src.f1.get(s1), dst.f1.get(d1));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		Iterator srcf2 = src.f2.keySet().iterator();
+		Iterator dstf2 = dst.f2.keySet().iterator();
+		while (srcf2.hasNext()) {
+			String s2 = srcf2.next();
+			String d2 = dstf2.next();
+			assertEquals(s2, d2);
+			assertEquals(src.f2.get(s2), dst.f2.get(d2));
+		}
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalMapTypes01() throws Exception {
+		SampleOptionalMapTypes src = new SampleOptionalMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = null;
+		src.f2 = new HashMap();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalMapTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleOptionalMapTypes.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleOptionalMapTypes dst = (SampleOptionalMapTypes) tmpl.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalMapTypes02() throws Exception {
+		SampleOptionalMapTypes src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleOptionalMapTypes.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleOptionalMapTypes.class));
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleOptionalMapTypes dst = (SampleOptionalMapTypes) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class SampleOptionalMapTypes {
+		@MessagePackOptional
+		public Map f0;
+		@MessagePackOptional
+		public Map f1;
+		@MessagePackOptional
+		public Map f2;
+
+		public SampleOptionalMapTypes() {
+		}
+	}
+
 	@Test
 	public void testDefaultConstructorModifiers01() throws Exception {
 		try {
@@ -687,31 +1044,14 @@ public class TestPackConvert extends TestCase {
 
 	@Test
 	public void testEnumTypeForOrdinal01() throws Exception {
-		SampleEnumFieldClass src = new SampleEnumFieldClass();
-		src.f1 = null;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleEnumFieldClass.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleEnumFieldClass.class);
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl.convert(mpo);
-		assertEquals(src.f0, dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertFalse(it.hasNext());
-	}
-
-	@Test
-	public void testEnumTypeForOrdinal02() throws Exception {
 		SampleEnumFieldClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleEnumFieldClass.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleEnumFieldClass.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleEnumFieldClass.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleEnumFieldClass.class));
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -723,7 +1063,6 @@ public class TestPackConvert extends TestCase {
 
 	public static class SampleEnumFieldClass {
 		public int f0;
-
 		public SampleEnum f1;
 
 		public SampleEnumFieldClass() {
@@ -735,6 +1074,86 @@ public class TestPackConvert extends TestCase {
 		ONE, TWO, THREE;
 	}
 
+	@Test
+	public void testOptionalEnumTypeForOrdinal00() throws Exception {
+		SampleOptionalEnumFieldClass src = new SampleOptionalEnumFieldClass();
+		src.f0 = 0;
+		src.f1 = SampleOptionalEnum.ONE;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalEnumFieldClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(SampleOptionalEnumFieldClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleOptionalEnumFieldClass dst = (SampleOptionalEnumFieldClass) tmpl
+				.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalEnumTypeForOrdinal01() throws Exception {
+		SampleOptionalEnumFieldClass src = new SampleOptionalEnumFieldClass();
+		src.f1 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalEnumFieldClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(SampleOptionalEnumFieldClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleOptionalEnumFieldClass dst = (SampleOptionalEnumFieldClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalEnumTypeForOrdinal02() throws Exception {
+		SampleOptionalEnumFieldClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleOptionalEnumFieldClass.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleOptionalEnumFieldClass.class));
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleOptionalEnumFieldClass dst = (SampleOptionalEnumFieldClass) tmpl
+				.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class SampleOptionalEnumFieldClass {
+		@MessagePackOptional
+		public int f0;
+		@MessagePackOptional
+		public SampleOptionalEnum f1;
+
+		public SampleOptionalEnumFieldClass() {
+		}
+	}
+
+	@MessagePackOrdinalEnum
+	public enum SampleOptionalEnum {
+		ONE, TWO, THREE;
+	}
+
 	@Test
 	public void testFieldModifiers() throws Exception {
 		FieldModifiersClass src = new FieldModifiersClass();
@@ -771,6 +1190,47 @@ public class TestPackConvert extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalFieldModifiers() throws Exception {
+		OptionalFieldModifiersClass src = new OptionalFieldModifiersClass();
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalFieldModifiersClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalFieldModifiersClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalFieldModifiersClass dst = (OptionalFieldModifiersClass) tmpl
+				.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 != dst.f3);
+		assertTrue(src.f4 != dst.f4);
+		assertFalse(it.hasNext());
+	}
+
+	public static class OptionalFieldModifiersClass {
+		@MessagePackOptional
+		public int f0;
+		@MessagePackOptional
+		public final int f1 = 1;
+		private int f2;
+		protected int f3;
+		int f4;
+
+		public OptionalFieldModifiersClass() {
+		}
+	}
+
 	@Test
 	public void testNestedFieldClass00() throws Exception {
 		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
@@ -801,53 +1261,29 @@ public class TestPackConvert extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
-	@Test
-	public void testNestedFieldClass01() throws Exception {
-		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
-		CustomPacker.register(NestedClass.class, packer2);
-		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
-		CustomPacker.register(BaseClass.class, packer1);
-		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
-		CustomUnpacker.register(NestedClass.class, tmpl2);
-		CustomConverter.register(NestedClass.class, tmpl2);
-		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
-		CustomUnpacker.register(BaseClass.class, tmpl1);
-		CustomConverter.register(BaseClass.class, tmpl1);
-		BaseClass src = new BaseClass();
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer1.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		BaseClass dst = (BaseClass) tmpl1.convert(mpo);
-		assertTrue(src.f0 == dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertFalse(it.hasNext());
-	}
-
 	@Test
 	public void testNestedFieldClass02() throws Exception {
 		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
 		CustomPacker.register(NestedClass.class, packer2);
-		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
-		CustomPacker.register(BaseClass.class, packer1);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(BaseClass.class));
+		CustomPacker.register(BaseClass.class, packer);
 		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
 		CustomUnpacker.register(NestedClass.class, tmpl2);
 		CustomConverter.register(NestedClass.class, tmpl2);
-		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
-		CustomUnpacker.register(BaseClass.class, tmpl1);
-		CustomConverter.register(BaseClass.class, tmpl1);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(BaseClass.class));
+		CustomUnpacker.register(BaseClass.class, tmpl);
+		CustomConverter.register(BaseClass.class, tmpl);
 		BaseClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer1.pack(new Packer(out), src);
+		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
 		MessagePackObject mpo = it.next();
-		BaseClass dst = (BaseClass) tmpl1.convert(mpo);
+		BaseClass dst = (BaseClass) tmpl.convert(mpo);
 		assertEquals(src, dst);
 		assertFalse(it.hasNext());
 	}
@@ -867,6 +1303,107 @@ public class TestPackConvert extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalNestedFieldClass00() throws Exception {
+		MessagePacker packer2 = DynamicPacker.create(OptionalNestedClass.class);
+		CustomPacker.register(OptionalNestedClass.class, packer2);
+		MessagePacker packer = DynamicPacker.create(OptionalBaseClass.class);
+		CustomPacker.register(OptionalBaseClass.class, packer);
+		Template tmpl2 = DynamicTemplate.create(OptionalNestedClass.class);
+		CustomUnpacker.register(OptionalNestedClass.class, tmpl2);
+		CustomConverter.register(OptionalNestedClass.class, tmpl2);
+		Template tmpl = DynamicTemplate.create(OptionalBaseClass.class);
+		CustomUnpacker.register(OptionalBaseClass.class, tmpl);
+		CustomConverter.register(OptionalBaseClass.class, tmpl);
+		OptionalBaseClass src = new OptionalBaseClass();
+		OptionalNestedClass src2 = new OptionalNestedClass();
+		src.f0 = 0;
+		src2.f2 = 2;
+		src.f1 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalBaseClass dst = (OptionalBaseClass) tmpl.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1.f2 == dst.f1.f2);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalNestedFieldClass01() throws Exception {
+		MessagePacker packer2 = DynamicPacker.create(OptionalNestedClass.class);
+		CustomPacker.register(OptionalNestedClass.class, packer2);
+		MessagePacker packer = DynamicPacker.create(OptionalBaseClass.class);
+		CustomPacker.register(OptionalBaseClass.class, packer);
+		Template tmpl2 = DynamicTemplate.create(OptionalNestedClass.class);
+		CustomUnpacker.register(OptionalNestedClass.class, tmpl2);
+		CustomConverter.register(OptionalNestedClass.class, tmpl2);
+		Template tmpl = DynamicTemplate.create(OptionalBaseClass.class);
+		CustomUnpacker.register(OptionalBaseClass.class, tmpl);
+		CustomConverter.register(OptionalBaseClass.class, tmpl);
+		OptionalBaseClass src = new OptionalBaseClass();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalBaseClass dst = (OptionalBaseClass) tmpl.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalNestedFieldClass02() throws Exception {
+		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
+		CustomPacker.register(NestedClass.class, packer2);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(BaseClass.class));
+		CustomPacker.register(BaseClass.class, packer);
+		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
+		CustomUnpacker.register(NestedClass.class, tmpl2);
+		CustomConverter.register(NestedClass.class, tmpl2);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(BaseClass.class));
+		CustomUnpacker.register(BaseClass.class, tmpl);
+		CustomConverter.register(BaseClass.class, tmpl);
+		BaseClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		BaseClass dst = (BaseClass) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class OptionalBaseClass {
+		@MessagePackOptional
+		public int f0;
+		@MessagePackOptional
+		public OptionalNestedClass f1;
+
+		public OptionalBaseClass() {
+		}
+	}
+
+	public static class OptionalNestedClass {
+		@MessagePackOptional
+		public int f2;
+
+		public OptionalNestedClass() {
+		}
+	}
+
 	@Test
 	public void testMessagePackMessageFieldClass00() throws Exception {
 		BaseClass2 src = new BaseClass2();
@@ -889,36 +1426,20 @@ public class TestPackConvert extends TestCase {
 		assertFalse(it.hasNext());
 	}
 
-	@Test
-	public void testMessagePackMessageFieldClass01() throws Exception {
-		BaseClass2 src = new BaseClass2();
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(BaseClass2.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		Template tmpl = DynamicTemplate.create(BaseClass2.class);
-		BaseClass2 dst = (BaseClass2) tmpl.convert(mpo);
-		assertTrue(src.f0 == dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertFalse(it.hasNext());
-	}
-
 	@Test
 	public void testMessagePackMessageFieldClass02() throws Exception {
 		BaseClass2 src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(BaseClass2.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(BaseClass2.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
 		MessagePackObject mpo = it.next();
-		Template tmpl = DynamicTemplate.create(BaseClass2.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(BaseClass2.class));
 		BaseClass2 dst = (BaseClass2) tmpl.convert(mpo);
 		assertEquals(src, dst);
 		assertFalse(it.hasNext());
@@ -940,6 +1461,84 @@ public class TestPackConvert extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalMessagePackMessageFieldClass00() throws Exception {
+		OptionalBaseClass2 src = new OptionalBaseClass2();
+		OptionalMessagePackMessageClass2 src2 = new OptionalMessagePackMessageClass2();
+		src.f0 = 0;
+		src2.f2 = 2;
+		src.f1 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(OptionalBaseClass2.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		Template tmpl = DynamicTemplate.create(OptionalBaseClass2.class);
+		OptionalBaseClass2 dst = (OptionalBaseClass2) tmpl.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1.f2 == dst.f1.f2);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalMessagePackMessageFieldClass01() throws Exception {
+		OptionalBaseClass2 src = new OptionalBaseClass2();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(OptionalBaseClass2.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		Template tmpl = DynamicTemplate.create(OptionalBaseClass2.class);
+		OptionalBaseClass2 dst = (OptionalBaseClass2) tmpl.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalMessagePackMessageFieldClass02() throws Exception {
+		OptionalBaseClass2 src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(OptionalBaseClass2.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(OptionalBaseClass2.class));
+		OptionalBaseClass2 dst = (OptionalBaseClass2) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class OptionalBaseClass2 {
+		@MessagePackOptional
+		public int f0;
+		@MessagePackOptional
+		public OptionalMessagePackMessageClass2 f1;
+
+		public OptionalBaseClass2() {
+		}
+	}
+
+	@MessagePackMessage
+	public static class OptionalMessagePackMessageClass2 {
+		@MessagePackOptional
+		public int f2;
+
+		public OptionalMessagePackMessageClass2() {
+		}
+	}
+
 	@Test
 	public void testExtendedClass00() throws Exception {
 		SampleSubClass src = new SampleSubClass();
@@ -976,10 +1575,12 @@ public class TestPackConvert extends TestCase {
 	public void testExtendedClass01() throws Exception {
 		SampleSubClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleSubClass.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleSubClass.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleSubClass.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleSubClass.class));
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -1012,6 +1613,83 @@ public class TestPackConvert extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalExtendedClass00() throws Exception {
+		SampleOptionalSubClass src = new SampleOptionalSubClass();
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f8 = 8;
+		src.f9 = 9;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalSubClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleOptionalSubClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleOptionalSubClass dst = (SampleOptionalSubClass) tmpl.convert(mpo);
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 != dst.f3);
+		assertTrue(src.f4 != dst.f4);
+		assertTrue(src.f5 == dst.f5);
+		assertTrue(src.f6 == dst.f6);
+		assertTrue(src.f8 != dst.f8);
+		assertTrue(src.f9 != dst.f9);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalExtendedClass01() throws Exception {
+		SampleOptionalSubClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleOptionalSubClass.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleOptionalSubClass.class));
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleOptionalSubClass dst = (SampleOptionalSubClass) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class SampleOptionalSubClass extends SampleOptionalSuperClass {
+		@MessagePackOptional
+		public int f0;
+		public final int f1 = 1;
+		private int f2;
+		protected int f3;
+		int f4;
+
+		public SampleOptionalSubClass() {
+		}
+	}
+
+	public static class SampleOptionalSuperClass {
+		@MessagePackOptional
+		public int f5;
+		public final int f6 = 2;
+		@SuppressWarnings("unused")
+		private int f7;
+		protected int f8;
+		int f9;
+
+		public SampleOptionalSuperClass() {
+		}
+	}
+
 	@Test
 	public void testMessagePackableUnpackableClass00() throws Exception {
 		BaseMessagePackableConvertableClass src = new BaseMessagePackableConvertableClass();
@@ -1047,39 +1725,14 @@ public class TestPackConvert extends TestCase {
 
 	@Test
 	public void testMessagePackableUnpackableClass01() throws Exception {
-		BaseMessagePackableConvertableClass src = new BaseMessagePackableConvertableClass();
-		src.f0 = null;
-		src.f1 = 1;
-		src.f2 = null;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(BaseMessagePackableConvertableClass.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(BaseMessagePackableConvertableClass.class);
-		Unpacker pac = new Unpacker(in);
-		Iterator it = pac.iterator();
-		assertTrue(it.hasNext());
-		MessagePackObject mpo = it.next();
-		BaseMessagePackableConvertableClass dst = (BaseMessagePackableConvertableClass) tmpl
-				.convert(mpo);
-		assertEquals(src.f0, dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2, dst.f2);
-		assertFalse(it.hasNext());
-	}
-
-	@Test
-	public void testMessagePackableUnpackableClass02() throws Exception {
 		BaseMessagePackableConvertableClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(BaseMessagePackableConvertableClass.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(BaseMessagePackableConvertableClass.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(BaseMessagePackableConvertableClass.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(BaseMessagePackableConvertableClass.class));
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
@@ -1092,9 +1745,7 @@ public class TestPackConvert extends TestCase {
 
 	public static class BaseMessagePackableConvertableClass {
 		public MessagePackableConvertableClass f0;
-
 		public int f1;
-
 		public List f2;
 
 		public BaseMessagePackableConvertableClass() {
@@ -1103,9 +1754,7 @@ public class TestPackConvert extends TestCase {
 
 	public static class MessagePackableConvertableClass implements
 			MessagePackable, MessageConvertable {
-
 		public int f0;
-
 		public int f1;
 
 		public MessagePackableConvertableClass() {
@@ -1129,4 +1778,124 @@ public class TestPackConvert extends TestCase {
 			f1 = objs[1].asInt();
 		}
 	}
+
+	@Test
+	public void testOptionalMessagePackableUnpackableClass00() throws Exception {
+		OptionalBaseMessagePackableConvertableClass src = new OptionalBaseMessagePackableConvertableClass();
+		OptionalMessagePackableConvertableClass src1 = new OptionalMessagePackableConvertableClass();
+		List src2 = new ArrayList();
+		src1.f0 = 0;
+		src1.f1 = 1;
+		src.f0 = src1;
+		src.f1 = 1;
+		src2.add(src1);
+		src.f2 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalBaseMessagePackableConvertableClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalBaseMessagePackableConvertableClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalBaseMessagePackableConvertableClass dst = (OptionalBaseMessagePackableConvertableClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0.f0, dst.f0.f0);
+		assertEquals(src.f0.f1, dst.f0.f1);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertEquals(src.f2.get(0).f0, dst.f2.get(0).f0);
+		assertEquals(src.f2.get(0).f1, dst.f2.get(0).f1);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalMessagePackableUnpackableClass01() throws Exception {
+		OptionalBaseMessagePackableConvertableClass src = new OptionalBaseMessagePackableConvertableClass();
+		src.f0 = null;
+		src.f1 = 1;
+		src.f2 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalBaseMessagePackableConvertableClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalBaseMessagePackableConvertableClass.class);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalBaseMessagePackableConvertableClass dst = (OptionalBaseMessagePackableConvertableClass) tmpl
+				.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testOptionalMessagePackableUnpackableClass02() throws Exception {
+		OptionalBaseMessagePackableConvertableClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(OptionalBaseMessagePackableConvertableClass.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(OptionalBaseMessagePackableConvertableClass.class));
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		OptionalBaseMessagePackableConvertableClass dst = (OptionalBaseMessagePackableConvertableClass) tmpl
+				.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class OptionalBaseMessagePackableConvertableClass {
+		@MessagePackOptional
+		public OptionalMessagePackableConvertableClass f0;
+		@MessagePackOptional
+		public int f1;
+
+		@MessagePackOptional
+		public List f2;
+
+		public OptionalBaseMessagePackableConvertableClass() {
+		}
+	}
+
+	public static class OptionalMessagePackableConvertableClass implements
+			MessagePackable, MessageConvertable {
+		@MessagePackOptional
+		public int f0;
+		@MessagePackOptional
+		public int f1;
+
+		public OptionalMessagePackableConvertableClass() {
+		}
+
+		@Override
+		public void messagePack(Packer packer) throws IOException {
+			packer.packArray(2);
+			packer.pack(f0);
+			packer.pack(f1);
+		}
+
+		@Override
+		public void messageConvert(MessagePackObject from)
+				throws MessageTypeException {
+			if (from.isNil()) {
+				return;
+			}
+			MessagePackObject[] objs = from.asArray();
+			f0 = objs[0].asInt();
+			f1 = objs[1].asInt();
+		}
+	}
 }
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
index b7ac7eb..9496604 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
@@ -22,7 +22,10 @@ import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
 import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.annotation.MessagePackOptional;
 import org.msgpack.annotation.MessagePackOrdinalEnum;
+import org.msgpack.packer.OptionalPacker;
+import org.msgpack.template.OptionalTemplate;
 
 import junit.framework.TestCase;
 
@@ -79,11 +82,12 @@ public class TestPackUnpack extends TestCase {
 	public void testPrimitiveTypeFields02() throws Exception {
 		PrimitiveTypeFieldsClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(PrimitiveTypeFieldsClass.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(PrimitiveTypeFieldsClass.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(PrimitiveTypeFieldsClass.class));
 		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
 				.unpack(new Unpacker(in));
 		assertEquals(src, dst);
@@ -102,6 +106,90 @@ public class TestPackUnpack extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalPrimitiveTypeFields00() throws Exception {
+		OptionalPrimitiveTypeFieldsClass src = new OptionalPrimitiveTypeFieldsClass();
+		src.f0 = (byte) 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f6 = false;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalPrimitiveTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalPrimitiveTypeFieldsClass.class);
+		OptionalPrimitiveTypeFieldsClass dst = (OptionalPrimitiveTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+	}
+
+	@Test
+	public void testOptionalPrimitiveTypeFields01() throws Exception {
+		OptionalPrimitiveTypeFieldsClass src = new OptionalPrimitiveTypeFieldsClass();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalPrimitiveTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalPrimitiveTypeFieldsClass.class);
+		OptionalPrimitiveTypeFieldsClass dst = (OptionalPrimitiveTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+	}
+
+	@Test
+	public void testOptionalPrimitiveTypeFields02() throws Exception {
+		OptionalPrimitiveTypeFieldsClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(OptionalPrimitiveTypeFieldsClass.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(OptionalPrimitiveTypeFieldsClass.class));
+		OptionalPrimitiveTypeFieldsClass dst = (OptionalPrimitiveTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class OptionalPrimitiveTypeFieldsClass {
+		@MessagePackOptional
+		public byte f0;
+		@MessagePackOptional
+		public short f1;
+		@MessagePackOptional
+		public int f2;
+		@MessagePackOptional
+		public long f3;
+		@MessagePackOptional
+		public float f4;
+		@MessagePackOptional
+		public double f5;
+		@MessagePackOptional
+		public boolean f6;
+
+		public OptionalPrimitiveTypeFieldsClass() {
+		}
+	}
+
 	@Test
 	public void testGeneralReferenceTypeFieldsClass00() throws Exception {
 		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
@@ -139,48 +227,14 @@ public class TestPackUnpack extends TestCase {
 
 	@Test
 	public void testGeneralReferenceTypeFieldsClass01() throws Exception {
-		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
-		src.f0 = null;
-		src.f1 = null;
-		src.f2 = null;
-		src.f3 = null;
-		src.f4 = null;
-		src.f5 = null;
-		src.f6 = null;
-		src.f7 = null;
-		src.f8 = null;
-		src.f9 = null;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(GeneralReferenceTypeFieldsClass.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(GeneralReferenceTypeFieldsClass.class);
-		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
-				.unpack(new Unpacker(in));
-		assertEquals(src.f0, dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2, dst.f2);
-		assertEquals(src.f3, dst.f3);
-		assertEquals(src.f4, dst.f4);
-		assertEquals(src.f5, dst.f5);
-		assertEquals(src.f6, dst.f6);
-		assertEquals(src.f7, dst.f7);
-		assertEquals(src.f8, dst.f8);
-		assertEquals(src.f9, dst.f9);
-	}
-
-	@Test
-	public void testGeneralReferenceTypeFieldsClass02() throws Exception {
 		GeneralReferenceTypeFieldsClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(GeneralReferenceTypeFieldsClass.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(GeneralReferenceTypeFieldsClass.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(GeneralReferenceTypeFieldsClass.class));
 		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
 				.unpack(new Unpacker(in));
 		assertEquals(src, dst);
@@ -202,6 +256,119 @@ public class TestPackUnpack extends TestCase {
 		}
 	}
 
+	@Test
+	public void testGeneralOptionalReferenceTypeFieldsClass00()
+			throws Exception {
+		GeneralOptionalReferenceTypeFieldsClass src = new GeneralOptionalReferenceTypeFieldsClass();
+		src.f0 = 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = (long) 3;
+		src.f4 = (float) 4;
+		src.f5 = (double) 5;
+		src.f6 = false;
+		src.f7 = new BigInteger("7");
+		src.f8 = "8";
+		src.f9 = new byte[] { 0x01, 0x02 };
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(GeneralOptionalReferenceTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(GeneralOptionalReferenceTypeFieldsClass.class);
+		GeneralOptionalReferenceTypeFieldsClass dst = (GeneralOptionalReferenceTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9[0], dst.f9[0]);
+		assertEquals(src.f9[1], dst.f9[1]);
+	}
+
+	@Test
+	public void testGeneralOptionalReferenceTypeFieldsClass01()
+			throws Exception {
+		GeneralOptionalReferenceTypeFieldsClass src = new GeneralOptionalReferenceTypeFieldsClass();
+		src.f0 = null;
+		src.f1 = null;
+		src.f2 = null;
+		src.f3 = null;
+		src.f4 = null;
+		src.f5 = null;
+		src.f6 = null;
+		src.f7 = null;
+		src.f8 = null;
+		src.f9 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(GeneralOptionalReferenceTypeFieldsClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(GeneralOptionalReferenceTypeFieldsClass.class);
+		GeneralOptionalReferenceTypeFieldsClass dst = (GeneralOptionalReferenceTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9, dst.f9);
+	}
+
+	@Test
+	public void testGeneralOptionalReferenceTypeFieldsClass02()
+			throws Exception {
+		GeneralOptionalReferenceTypeFieldsClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(GeneralOptionalReferenceTypeFieldsClass.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(GeneralOptionalReferenceTypeFieldsClass.class));
+		GeneralOptionalReferenceTypeFieldsClass dst = (GeneralOptionalReferenceTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class GeneralOptionalReferenceTypeFieldsClass {
+		@MessagePackOptional
+		public Byte f0;
+		@MessagePackOptional
+		public Short f1;
+		@MessagePackOptional
+		public Integer f2;
+		@MessagePackOptional
+		public Long f3;
+		@MessagePackOptional
+		public Float f4;
+		@MessagePackOptional
+		public Double f5;
+		@MessagePackOptional
+		public Boolean f6;
+		@MessagePackOptional
+		public BigInteger f7;
+		@MessagePackOptional
+		public String f8;
+		@MessagePackOptional
+		public byte[] f9;
+
+		public GeneralOptionalReferenceTypeFieldsClass() {
+		}
+	}
+
 	@Test
 	public void testListTypes00() throws Exception {
 		SampleListTypes src = new SampleListTypes();
@@ -257,33 +424,14 @@ public class TestPackUnpack extends TestCase {
 
 	@Test
 	public void testListTypes01() throws Exception {
-		SampleListTypes src = new SampleListTypes();
-		src.f0 = new ArrayList();
-		src.f1 = null;
-		src.f2 = new ArrayList();
-		src.f3 = new ArrayList>();
-		src.f4 = null;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleListTypes.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
-		SampleListTypes dst = (SampleListTypes) tmpl.unpack(new Unpacker(in));
-		assertEquals(src.f0.size(), dst.f0.size());
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2.size(), dst.f2.size());
-		assertEquals(src.f3.size(), dst.f3.size());
-		assertEquals(src.f4, dst.f4);
-	}
-
-	@Test
-	public void testListTypes02() throws Exception {
 		SampleListTypes src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleListTypes.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleListTypes.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleListTypes.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleListTypes.class));
 		SampleListTypes dst = (SampleListTypes) tmpl.unpack(new Unpacker(in));
 		assertEquals(src, dst);
 	}
@@ -308,6 +456,125 @@ public class TestPackUnpack extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalListTypes00() throws Exception {
+		SampleOptionalListTypes src = new SampleOptionalListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = new ArrayList();
+		src.f1.add(1);
+		src.f1.add(2);
+		src.f1.add(3);
+		src.f2 = new ArrayList();
+		src.f2.add("e1");
+		src.f2.add("e2");
+		src.f2.add("e3");
+		src.f3 = new ArrayList>();
+		src.f3.add(src.f2);
+		src.f4 = new ArrayList();
+		SampleOptionalListNestedType slnt = new SampleOptionalListNestedType();
+		slnt.f0 = new byte[] { 0x01, 0x02 };
+		slnt.f1 = "muga";
+		src.f4.add(slnt);
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalListTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleOptionalListTypes.class);
+		SampleOptionalListTypes dst = (SampleOptionalListTypes) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		for (int i = 0; i < src.f1.size(); ++i) {
+			assertEquals(src.f1.get(i), dst.f1.get(i));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		for (int i = 0; i < src.f2.size(); ++i) {
+			assertEquals(src.f2.get(i), dst.f2.get(i));
+		}
+		assertEquals(src.f3.size(), dst.f3.size());
+		for (int i = 0; i < src.f3.size(); ++i) {
+			List srclist = src.f3.get(i);
+			List dstlist = dst.f3.get(i);
+			assertEquals(srclist.size(), dstlist.size());
+			for (int j = 0; j < srclist.size(); ++j) {
+				assertEquals(srclist.get(j), dstlist.get(j));
+			}
+		}
+		assertEquals(src.f4.size(), dst.f4.size());
+		for (int i = 0; i < src.f4.size(); ++i) {
+			SampleOptionalListNestedType s = src.f4.get(i);
+			SampleOptionalListNestedType d = dst.f4.get(i);
+			assertEquals(s.f0[0], d.f0[0]);
+			assertEquals(s.f0[1], d.f0[1]);
+			assertEquals(s.f1, d.f1);
+		}
+	}
+
+	@Test
+	public void testOptionalListTypes01() throws Exception {
+		SampleOptionalListTypes src = new SampleOptionalListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = null;
+		src.f2 = new ArrayList();
+		src.f3 = new ArrayList>();
+		src.f4 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalListTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleOptionalListTypes.class);
+		SampleOptionalListTypes dst = (SampleOptionalListTypes) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertEquals(src.f3.size(), dst.f3.size());
+		assertEquals(src.f4, dst.f4);
+	}
+
+	@Test
+	public void testOptionalListTypes02() throws Exception {
+		SampleListTypes src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleOptionalListTypes.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleOptionalListTypes.class));
+		SampleListTypes dst = (SampleListTypes) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class SampleOptionalListTypes {
+		@MessagePackOptional
+		public List f0;
+		@MessagePackOptional
+		public List f1;
+		@MessagePackOptional
+		public List f2;
+		@MessagePackOptional
+		public List> f3;
+		@MessagePackOptional
+		public List f4;
+
+		public SampleOptionalListTypes() {
+		}
+	}
+
+	@MessagePackMessage
+	public static class SampleOptionalListNestedType {
+		@MessagePackOptional
+		public byte[] f0;
+		@MessagePackOptional
+		public String f1;
+
+		public SampleOptionalListNestedType() {
+		}
+	}
+
 	@Test
 	public void testMapTypes00() throws Exception {
 		SampleMapTypes src = new SampleMapTypes();
@@ -349,29 +616,14 @@ public class TestPackUnpack extends TestCase {
 
 	@Test
 	public void testMapTypes01() throws Exception {
-		SampleMapTypes src = new SampleMapTypes();
-		src.f0 = new HashMap();
-		src.f1 = null;
-		src.f2 = new HashMap();
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleMapTypes.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
-		SampleMapTypes dst = (SampleMapTypes) tmpl.unpack(new Unpacker(in));
-		assertEquals(src.f0.size(), dst.f0.size());
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2.size(), dst.f2.size());
-	}
-
-	@Test
-	public void testMapTypes02() throws Exception {
 		SampleMapTypes src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleMapTypes.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleMapTypes.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleMapTypes.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleMapTypes.class));
 		SampleMapTypes dst = (SampleMapTypes) tmpl.unpack(new Unpacker(in));
 		assertEquals(src, dst);
 	}
@@ -385,6 +637,93 @@ public class TestPackUnpack extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalMapTypes00() throws Exception {
+		SampleOptionalMapTypes src = new SampleOptionalMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = new HashMap();
+		src.f1.put(1, 1);
+		src.f1.put(2, 2);
+		src.f1.put(3, 3);
+		src.f2 = new HashMap();
+		src.f2.put("k1", 1);
+		src.f2.put("k2", 2);
+		src.f2.put("k3", 3);
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalMapTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleOptionalMapTypes.class);
+		SampleOptionalMapTypes dst = (SampleOptionalMapTypes) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		Iterator srcf1 = src.f1.keySet().iterator();
+		Iterator dstf1 = dst.f1.keySet().iterator();
+		while (srcf1.hasNext()) {
+			Integer s1 = srcf1.next();
+			Integer d1 = dstf1.next();
+			assertEquals(s1, d1);
+			assertEquals(src.f1.get(s1), dst.f1.get(d1));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		Iterator srcf2 = src.f2.keySet().iterator();
+		Iterator dstf2 = dst.f2.keySet().iterator();
+		while (srcf2.hasNext()) {
+			String s2 = srcf2.next();
+			String d2 = dstf2.next();
+			assertEquals(s2, d2);
+			assertEquals(src.f2.get(s2), dst.f2.get(d2));
+		}
+	}
+
+	@Test
+	public void testOptionalMapTypes01() throws Exception {
+		SampleOptionalMapTypes src = new SampleOptionalMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = null;
+		src.f2 = new HashMap();
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalMapTypes.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleOptionalMapTypes.class);
+		SampleOptionalMapTypes dst = (SampleOptionalMapTypes) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+	}
+
+	@Test
+	public void testOptionalMapTypes02() throws Exception {
+		SampleOptionalMapTypes src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleOptionalMapTypes.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleOptionalMapTypes.class));
+		SampleOptionalMapTypes dst = (SampleOptionalMapTypes) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class SampleOptionalMapTypes {
+		@MessagePackOptional
+		public Map f0;
+		@MessagePackOptional
+		public Map f1;
+		@MessagePackOptional
+		public Map f2;
+
+		public SampleOptionalMapTypes() {
+		}
+	}
+
 	@Test
 	public void testDefaultConstructorModifiers00() throws Exception {
 		try {
@@ -615,27 +954,14 @@ public class TestPackUnpack extends TestCase {
 
 	@Test
 	public void testEnumTypeForOrdinal01() throws Exception {
-		SampleEnumFieldClass src = new SampleEnumFieldClass();
-		src.f1 = null;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleEnumFieldClass.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleEnumFieldClass.class);
-		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl
-				.unpack(new Unpacker(in));
-		assertTrue(src.f0 == dst.f0);
-		assertEquals(src.f1, dst.f1);
-	}
-
-	@Test
-	public void testEnumTypeForOrdinal02() throws Exception {
 		SampleEnumFieldClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleEnumFieldClass.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleEnumFieldClass.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleEnumFieldClass.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleEnumFieldClass.class));
 		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl
 				.unpack(new Unpacker(in));
 		assertEquals(src, dst);
@@ -655,6 +981,72 @@ public class TestPackUnpack extends TestCase {
 		ONE, TWO, THREE;
 	}
 
+	@Test
+	public void testOptionalEnumTypeForOrdinal00() throws Exception {
+		SampleOptionalEnumFieldClass src = new SampleOptionalEnumFieldClass();
+		src.f0 = 0;
+		src.f1 = SampleOptionalEnum.ONE;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalEnumFieldClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(SampleOptionalEnumFieldClass.class);
+		SampleOptionalEnumFieldClass dst = (SampleOptionalEnumFieldClass) tmpl
+				.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+	}
+
+	@Test
+	public void testOptionalEnumTypeForOrdinal01() throws Exception {
+		SampleOptionalEnumFieldClass src = new SampleOptionalEnumFieldClass();
+		src.f1 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalEnumFieldClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(SampleOptionalEnumFieldClass.class);
+		SampleOptionalEnumFieldClass dst = (SampleOptionalEnumFieldClass) tmpl
+				.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertEquals(src.f1, dst.f1);
+	}
+
+	@Test
+	public void testOptionalEnumTypeForOrdinal02() throws Exception {
+		SampleEnumFieldClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleEnumFieldClass.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleEnumFieldClass.class));
+		SampleEnumFieldClass dst = (SampleEnumFieldClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class SampleOptionalEnumFieldClass {
+		@MessagePackOptional
+		public int f0;
+
+		@MessagePackOptional
+		public SampleOptionalEnum f1;
+
+		public SampleOptionalEnumFieldClass() {
+		}
+	}
+
+	@MessagePackOrdinalEnum
+	public enum SampleOptionalEnum {
+		ONE, TWO, THREE;
+	}
+
 	@Test
 	public void testFieldModifiers() throws Exception {
 		FieldModifiersClass src = new FieldModifiersClass();
@@ -687,27 +1079,63 @@ public class TestPackUnpack extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalFieldModifiers() throws Exception {
+		OptionalFieldModifiersClass src = new OptionalFieldModifiersClass();
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalFieldModifiersClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalFieldModifiersClass.class);
+		OptionalFieldModifiersClass dst = (OptionalFieldModifiersClass) tmpl
+				.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 != dst.f3);
+		assertTrue(src.f4 != dst.f4);
+	}
+
+	public static class OptionalFieldModifiersClass {
+		@MessagePackOptional
+		public int f0;
+		@MessagePackOptional
+		public final int f1 = 1;
+		private int f2;
+		protected int f3;
+		int f4;
+
+		public OptionalFieldModifiersClass() {
+		}
+	}
+
 	@Test
 	public void testNestedFieldClass00() throws Exception {
 		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
 		CustomPacker.register(NestedClass.class, packer2);
-		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
-		CustomPacker.register(BaseClass.class, packer1);
+		MessagePacker packer = DynamicPacker.create(BaseClass.class);
+		CustomPacker.register(BaseClass.class, packer);
 		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
 		CustomUnpacker.register(NestedClass.class, tmpl2);
 		CustomConverter.register(NestedClass.class, tmpl2);
-		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
-		CustomUnpacker.register(BaseClass.class, tmpl1);
-		CustomConverter.register(BaseClass.class, tmpl1);
+		Template tmpl = DynamicTemplate.create(BaseClass.class);
+		CustomUnpacker.register(BaseClass.class, tmpl);
+		CustomConverter.register(BaseClass.class, tmpl);
 		BaseClass src = new BaseClass();
 		NestedClass src2 = new NestedClass();
 		src.f0 = 0;
 		src2.f2 = 2;
 		src.f1 = src2;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer1.pack(new Packer(out), src);
+		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		BaseClass dst = (BaseClass) tmpl1.unpack(new Unpacker(in));
+		BaseClass dst = (BaseClass) tmpl.unpack(new Unpacker(in));
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1.f2 == dst.f1.f2);
 	}
@@ -716,41 +1144,21 @@ public class TestPackUnpack extends TestCase {
 	public void testNestedFieldClass01() throws Exception {
 		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
 		CustomPacker.register(NestedClass.class, packer2);
-		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
-		CustomPacker.register(BaseClass.class, packer1);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(BaseClass.class));
+		CustomPacker.register(BaseClass.class, packer);
 		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
 		CustomUnpacker.register(NestedClass.class, tmpl2);
 		CustomConverter.register(NestedClass.class, tmpl2);
-		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
-		CustomUnpacker.register(BaseClass.class, tmpl1);
-		CustomConverter.register(BaseClass.class, tmpl1);
-		BaseClass src = new BaseClass();
-		src.f1 = null;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer1.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		BaseClass dst = (BaseClass) tmpl1.unpack(new Unpacker(in));
-		assertTrue(src.f0 == dst.f0);
-		assertTrue(src.f1 == dst.f1);
-	}
-
-	@Test
-	public void testNestedFieldClass02() throws Exception {
-		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
-		CustomPacker.register(NestedClass.class, packer2);
-		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
-		CustomPacker.register(BaseClass.class, packer1);
-		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
-		CustomUnpacker.register(NestedClass.class, tmpl2);
-		CustomConverter.register(NestedClass.class, tmpl2);
-		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
-		CustomUnpacker.register(BaseClass.class, tmpl1);
-		CustomConverter.register(BaseClass.class, tmpl1);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(BaseClass.class));
+		CustomUnpacker.register(BaseClass.class, tmpl);
+		CustomConverter.register(BaseClass.class, tmpl);
 		BaseClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer1.pack(new Packer(out), src);
+		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		BaseClass dst = (BaseClass) tmpl1.unpack(new Unpacker(in));
+		BaseClass dst = (BaseClass) tmpl.unpack(new Unpacker(in));
 		assertEquals(src, dst);
 	}
 
@@ -769,6 +1177,96 @@ public class TestPackUnpack extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalNestedFieldClass00() throws Exception {
+		MessagePacker packer2 = DynamicPacker.create(OptionalNestedClass.class);
+		CustomPacker.register(OptionalNestedClass.class, packer2);
+		MessagePacker packer = DynamicPacker.create(OptionalBaseClass.class);
+		CustomPacker.register(OptionalBaseClass.class, packer);
+		Template tmpl2 = DynamicTemplate.create(OptionalNestedClass.class);
+		CustomUnpacker.register(OptionalNestedClass.class, tmpl2);
+		CustomConverter.register(OptionalNestedClass.class, tmpl2);
+		Template tmpl = DynamicTemplate.create(OptionalBaseClass.class);
+		CustomUnpacker.register(OptionalBaseClass.class, tmpl);
+		CustomConverter.register(OptionalBaseClass.class, tmpl);
+		OptionalBaseClass src = new OptionalBaseClass();
+		OptionalNestedClass src2 = new OptionalNestedClass();
+		src.f0 = 0;
+		src2.f2 = 2;
+		src.f1 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		OptionalBaseClass dst = (OptionalBaseClass) tmpl
+				.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1.f2 == dst.f1.f2);
+	}
+
+	@Test
+	public void testOptionalNestedFieldClass01() throws Exception {
+		MessagePacker packer2 = DynamicPacker.create(OptionalNestedClass.class);
+		CustomPacker.register(OptionalNestedClass.class, packer2);
+		MessagePacker packer = DynamicPacker.create(OptionalBaseClass.class);
+		CustomPacker.register(OptionalBaseClass.class, packer);
+		Template tmpl2 = DynamicTemplate.create(OptionalNestedClass.class);
+		CustomUnpacker.register(OptionalNestedClass.class, tmpl2);
+		CustomConverter.register(OptionalNestedClass.class, tmpl2);
+		Template tmpl = DynamicTemplate.create(OptionalBaseClass.class);
+		CustomUnpacker.register(OptionalBaseClass.class, tmpl);
+		CustomConverter.register(OptionalBaseClass.class, tmpl);
+		OptionalBaseClass src = new OptionalBaseClass();
+		src.f1 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		OptionalBaseClass dst = (OptionalBaseClass) tmpl
+				.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+	}
+
+	@Test
+	public void testOptionalNestedFieldClass02() throws Exception {
+		MessagePacker packer2 = DynamicPacker.create(OptionalNestedClass.class);
+		CustomPacker.register(OptionalNestedClass.class, packer2);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(OptionalBaseClass.class));
+		CustomPacker.register(OptionalBaseClass.class, packer);
+		Template tmpl2 = DynamicTemplate.create(OptionalNestedClass.class);
+		CustomUnpacker.register(OptionalNestedClass.class, tmpl2);
+		CustomConverter.register(OptionalNestedClass.class, tmpl2);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(OptionalBaseClass.class));
+		CustomUnpacker.register(OptionalBaseClass.class, tmpl);
+		CustomConverter.register(OptionalBaseClass.class, tmpl);
+		OptionalBaseClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		OptionalBaseClass dst = (OptionalBaseClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class OptionalBaseClass {
+		@MessagePackOptional
+		public int f0;
+		@MessagePackOptional
+		public OptionalNestedClass f1;
+
+		public OptionalBaseClass() {
+		}
+	}
+
+	public static class OptionalNestedClass {
+		@MessagePackOptional
+		public int f2;
+
+		public OptionalNestedClass() {
+		}
+	}
+
 	@Test
 	public void testMessagePackMessageFieldClass00() throws Exception {
 		BaseClass2 src = new BaseClass2();
@@ -788,26 +1286,14 @@ public class TestPackUnpack extends TestCase {
 
 	@Test
 	public void testMessagePackMessageFieldClass01() throws Exception {
-		BaseClass2 src = new BaseClass2();
-		src.f1 = null;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(BaseClass2.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(BaseClass2.class);
-		BaseClass2 dst = (BaseClass2) tmpl.unpack(new Unpacker(in));
-		assertTrue(src.f0 == dst.f0);
-		assertEquals(src.f1, dst.f1);
-	}
-
-	@Test
-	public void testMessagePackMessageFieldClass02() throws Exception {
 		BaseClass2 src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(BaseClass2.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(BaseClass2.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(BaseClass2.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(BaseClass2.class));
 		BaseClass2 dst = (BaseClass2) tmpl.unpack(new Unpacker(in));
 		assertEquals(src, dst);
 	}
@@ -828,6 +1314,73 @@ public class TestPackUnpack extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalMessagePackMessageFieldClass00() throws Exception {
+		OptionalBaseClass2 src = new OptionalBaseClass2();
+		OptionalMessagePackMessageClass2 src2 = new OptionalMessagePackMessageClass2();
+		src.f0 = 0;
+		src2.f2 = 2;
+		src.f1 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(OptionalBaseClass2.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(OptionalBaseClass2.class);
+		OptionalBaseClass2 dst = (OptionalBaseClass2) tmpl.unpack(new Unpacker(
+				in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1.f2 == dst.f1.f2);
+	}
+
+	@Test
+	public void testOptionalMessagePackMessageFieldClass01() throws Exception {
+		OptionalBaseClass2 src = new OptionalBaseClass2();
+		src.f1 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(OptionalBaseClass2.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(OptionalBaseClass2.class);
+		OptionalBaseClass2 dst = (OptionalBaseClass2) tmpl.unpack(new Unpacker(
+				in));
+		assertTrue(src.f0 == dst.f0);
+		assertEquals(src.f1, dst.f1);
+	}
+
+	@Test
+	public void testOptionalMessagePackMessageFieldClass02() throws Exception {
+		OptionalBaseClass2 src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(OptionalBaseClass2.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(OptionalBaseClass2.class));
+		OptionalBaseClass2 dst = (OptionalBaseClass2) tmpl.unpack(new Unpacker(
+				in));
+		assertEquals(src, dst);
+	}
+
+	public static class OptionalBaseClass2 {
+		@MessagePackOptional
+		public int f0;
+		@MessagePackOptional
+		public OptionalMessagePackMessageClass2 f1;
+
+		public OptionalBaseClass2() {
+		}
+	}
+
+	@MessagePackMessage
+	public static class OptionalMessagePackMessageClass2 {
+		@MessagePackOptional
+		public int f2;
+
+		public OptionalMessagePackMessageClass2() {
+		}
+	}
+
 	@Test
 	public void testExtendedClass00() throws Exception {
 		SampleSubClass src = new SampleSubClass();
@@ -859,10 +1412,12 @@ public class TestPackUnpack extends TestCase {
 	public void testExtendedClass01() throws Exception {
 		SampleSubClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker.create(SampleSubClass.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleSubClass.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate.create(SampleSubClass.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleSubClass.class));
 		SampleSubClass dst = (SampleSubClass) tmpl.unpack(new Unpacker(in));
 		assertEquals(src, dst);
 	}
@@ -890,6 +1445,75 @@ public class TestPackUnpack extends TestCase {
 		}
 	}
 
+	@Test
+	public void testOptionalExtendedClass00() throws Exception {
+		SampleOptionalSubClass src = new SampleOptionalSubClass();
+		src.f0 = 0;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f8 = 8;
+		src.f9 = 9;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleOptionalSubClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleOptionalSubClass.class);
+		SampleOptionalSubClass dst = (SampleOptionalSubClass) tmpl
+				.unpack(new Unpacker(in));
+		assertTrue(src.f0 == dst.f0);
+		assertTrue(src.f1 == dst.f1);
+		assertTrue(src.f2 != dst.f2);
+		assertTrue(src.f3 != dst.f3);
+		assertTrue(src.f4 != dst.f4);
+		assertTrue(src.f5 == dst.f5);
+		assertTrue(src.f6 == dst.f6);
+		assertTrue(src.f8 != dst.f8);
+		assertTrue(src.f9 != dst.f9);
+	}
+
+	@Test
+	public void testOptionalExtendedClass01() throws Exception {
+		SampleOptionalSubClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleOptionalSubClass.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleOptionalSubClass.class));
+		SampleOptionalSubClass dst = (SampleOptionalSubClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class SampleOptionalSubClass extends SampleOptionalSuperClass {
+		@MessagePackOptional
+		public int f0;
+		public final int f1 = 1;
+		private int f2;
+		protected int f3;
+		int f4;
+
+		public SampleOptionalSubClass() {
+		}
+	}
+
+	public static class SampleOptionalSuperClass {
+		@MessagePackOptional
+		public int f5;
+		public final int f6 = 2;
+		@SuppressWarnings("unused")
+		private int f7;
+		protected int f8;
+		int f9;
+
+		public SampleOptionalSuperClass() {
+		}
+	}
+
 	@Test
 	public void testMessagePackableUnpackableClass00() throws Exception {
 		BaseMessagePackableUnpackableClass src = new BaseMessagePackableUnpackableClass();
@@ -920,34 +1544,14 @@ public class TestPackUnpack extends TestCase {
 
 	@Test
 	public void testMessagePackableUnpackableClass01() throws Exception {
-		BaseMessagePackableUnpackableClass src = new BaseMessagePackableUnpackableClass();
-		src.f0 = null;
-		src.f1 = 1;
-		src.f2 = null;
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(BaseMessagePackableUnpackableClass.class);
-		packer.pack(new Packer(out), src);
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(BaseMessagePackableUnpackableClass.class);
-		BaseMessagePackableUnpackableClass dst = (BaseMessagePackableUnpackableClass) tmpl
-				.unpack(new Unpacker(in));
-		assertEquals(src.f0, dst.f0);
-		assertEquals(src.f1, dst.f1);
-		assertEquals(src.f2, dst.f2);
-	}
-
-	@Test
-	public void testMessagePackableUnpackableClass02() throws Exception {
 		BaseMessagePackableUnpackableClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		MessagePacker packer = DynamicPacker
-				.create(BaseMessagePackableUnpackableClass.class);
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(BaseMessagePackableUnpackableClass.class));
 		packer.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		Template tmpl = DynamicTemplate
-				.create(BaseMessagePackableUnpackableClass.class);
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(BaseMessagePackableUnpackableClass.class));
 		BaseMessagePackableUnpackableClass dst = (BaseMessagePackableUnpackableClass) tmpl
 				.unpack(new Unpacker(in));
 		assertEquals(src, dst);
@@ -955,9 +1559,7 @@ public class TestPackUnpack extends TestCase {
 
 	public static class BaseMessagePackableUnpackableClass {
 		public MessagePackableUnpackableClass f0;
-
 		public int f1;
-
 		public List f2;
 
 		public BaseMessagePackableUnpackableClass() {
@@ -966,9 +1568,7 @@ public class TestPackUnpack extends TestCase {
 
 	public static class MessagePackableUnpackableClass implements
 			MessagePackable, MessageUnpackable {
-
 		public int f0;
-
 		public int f1;
 
 		public MessagePackableUnpackableClass() {
@@ -992,4 +1592,108 @@ public class TestPackUnpack extends TestCase {
 			f1 = unpacker.unpackInt();
 		}
 	}
+
+	@Test
+	public void testOptionalMessagePackableUnpackableClass00() throws Exception {
+		OptionalBaseMessagePackableUnpackableClass src = new OptionalBaseMessagePackableUnpackableClass();
+		OptionalMessagePackableUnpackableClass src1 = new OptionalMessagePackableUnpackableClass();
+		List src2 = new ArrayList();
+		src1.f0 = 0;
+		src1.f1 = 1;
+		src.f0 = src1;
+		src.f1 = 1;
+		src2.add(src1);
+		src.f2 = src2;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalBaseMessagePackableUnpackableClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalBaseMessagePackableUnpackableClass.class);
+		OptionalBaseMessagePackableUnpackableClass dst = (OptionalBaseMessagePackableUnpackableClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0.f0, dst.f0.f0);
+		assertEquals(src.f0.f1, dst.f0.f1);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertEquals(src.f2.get(0).f0, dst.f2.get(0).f0);
+		assertEquals(src.f2.get(0).f1, dst.f2.get(0).f1);
+	}
+
+	@Test
+	public void testOptionalMessagePackableUnpackableClass01() throws Exception {
+		OptionalBaseMessagePackableUnpackableClass src = new OptionalBaseMessagePackableUnpackableClass();
+		src.f0 = null;
+		src.f1 = 1;
+		src.f2 = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(OptionalBaseMessagePackableUnpackableClass.class);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(OptionalBaseMessagePackableUnpackableClass.class);
+		OptionalBaseMessagePackableUnpackableClass dst = (OptionalBaseMessagePackableUnpackableClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+	}
+
+	@Test
+	public void testOptionalMessagePackableUnpackableClass02() throws Exception {
+		OptionalBaseMessagePackableUnpackableClass src = null;
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(OptionalBaseMessagePackableUnpackableClass.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(OptionalBaseMessagePackableUnpackableClass.class));
+		OptionalBaseMessagePackableUnpackableClass dst = (OptionalBaseMessagePackableUnpackableClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class OptionalBaseMessagePackableUnpackableClass {
+		@MessagePackOptional
+		public OptionalMessagePackableUnpackableClass f0;
+		@MessagePackOptional
+		public int f1;
+		@MessagePackOptional
+		public List f2;
+
+		public OptionalBaseMessagePackableUnpackableClass() {
+		}
+	}
+
+	public static class OptionalMessagePackableUnpackableClass implements
+			MessagePackable, MessageUnpackable {
+		@MessagePackOptional
+		public int f0;
+		@MessagePackOptional
+		public int f1;
+
+		public OptionalMessagePackableUnpackableClass() {
+		}
+
+		@Override
+		public void messagePack(Packer packer) throws IOException {
+			packer.packArray(2);
+			packer.pack(f0);
+			packer.pack(f1);
+		}
+
+		@Override
+		public void messageUnpack(Unpacker unpacker) throws IOException,
+				MessageTypeException {
+			if (unpacker.tryUnpackNull()) {
+				return;
+			}
+			unpacker.unpackArray();
+			f0 = unpacker.unpackInt();
+			f1 = unpacker.unpackInt();
+		}
+	}
 }

From 2aac51dd20a18a02bbbe958d6462ad9e47060de2 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 23 Oct 2010 14:37:24 +0900
Subject: [PATCH 0889/1648] java: append getter methods for element templates
 in org.msgpack.template.ListTemplate.java, MapTemplate.java and
 OptionalTemplate.java

---
 java/src/main/java/org/msgpack/template/ListTemplate.java | 4 ++++
 java/src/main/java/org/msgpack/template/MapTemplate.java  | 8 ++++++++
 .../main/java/org/msgpack/template/OptionalTemplate.java  | 4 ++++
 3 files changed, 16 insertions(+)

diff --git a/java/src/main/java/org/msgpack/template/ListTemplate.java b/java/src/main/java/org/msgpack/template/ListTemplate.java
index 54975f8..4ea7a9b 100644
--- a/java/src/main/java/org/msgpack/template/ListTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ListTemplate.java
@@ -29,6 +29,10 @@ public class ListTemplate implements Template {
 		this.elementTemplate = elementTemplate;
 	}
 
+	public Template getElementTemplate() {
+		return elementTemplate;
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		int length = pac.unpackArray();
 		List list = new ArrayList(length);
diff --git a/java/src/main/java/org/msgpack/template/MapTemplate.java b/java/src/main/java/org/msgpack/template/MapTemplate.java
index 6b8457d..d23282e 100644
--- a/java/src/main/java/org/msgpack/template/MapTemplate.java
+++ b/java/src/main/java/org/msgpack/template/MapTemplate.java
@@ -31,6 +31,14 @@ public class MapTemplate implements Template {
 		this.valueTemplate = valueTemplate;
 	}
 
+	public Template getKeyTemplate() {
+		return keyTemplate;
+	}
+
+	public Template getValueTemplate() {
+		return valueTemplate;
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		int length = pac.unpackMap();
 		Map map = new HashMap(length);
diff --git a/java/src/main/java/org/msgpack/template/OptionalTemplate.java b/java/src/main/java/org/msgpack/template/OptionalTemplate.java
index f288fc4..cc5ee60 100644
--- a/java/src/main/java/org/msgpack/template/OptionalTemplate.java
+++ b/java/src/main/java/org/msgpack/template/OptionalTemplate.java
@@ -28,6 +28,10 @@ public class OptionalTemplate implements Template {
 		this(elementTemplate, null);
 	}
 
+	public Template getElementTemplate() {
+		return elementTemplate;
+	}
+
 	public OptionalTemplate(Template elementTemplate, Object defaultObject) {
 		this.elementTemplate = elementTemplate;
 		this.defaultObject = defaultObject;

From 1b8979f28578c739143f89f0ccb251d53076df1b Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 23 Oct 2010 14:39:01 +0900
Subject: [PATCH 0890/1648] java: refactor org.msgpack.util.codegen.*.java

---
 .../org/msgpack/util/codegen/Constants.java   |   6 +-
 .../msgpack/util/codegen/DynamicCodeGen.java  | 126 ++++----
 .../util/codegen/DynamicCodeGenBase.java      | 293 +++++++++++++++++-
 .../codegen/DynamicOrdinalEnumConverter.java  |  10 +-
 .../codegen/DynamicOrdinalEnumPacker.java     |  19 +-
 .../codegen/DynamicOrdinalEnumTemplate.java   |  18 +-
 .../codegen/DynamicOrdinalEnumUnpacker.java   |  10 +-
 .../msgpack/util/codegen/DynamicPacker.java   |  18 +-
 .../msgpack/util/codegen/DynamicTemplate.java |  14 +-
 9 files changed, 425 insertions(+), 89 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/Constants.java b/java/src/main/java/org/msgpack/util/codegen/Constants.java
index 8a4b708..b85c94d 100644
--- a/java/src/main/java/org/msgpack/util/codegen/Constants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/Constants.java
@@ -21,6 +21,8 @@ public interface Constants {
 
 	String VARIABLE_NAME_TEMPLATES = "_$$_templates";
 
+	String VARIABLE_NAME_PACKERS = "_$$_packers";
+
 	String VARIABLE_NAME_CLIENT = "_$$_client";
 
 	String METHOD_NAME_BOOLEANVALUE = "booleanValue";
@@ -43,6 +45,8 @@ public interface Constants {
 
 	String METHOD_NAME_SETTEMPLATES = "setTemplates";
 
+	String METHOD_NAME_SETMESSAGEPACKERS = "setMessagePackers";
+
 	String METHOD_NAME_PACK = "pack";
 
 	String METHOD_NAME_UNPACK = "unpack";
@@ -103,7 +107,7 @@ public interface Constants {
 
 	String STATEMENT_PACKER_PACKERMETHODBODY_02 = "$1.packArray(%d); ";
 
-	String STATEMENT_PACKER_PACKERMETHODBODY_03 = "$1.pack(_$$_t.%s); ";
+	String STATEMENT_PACKER_PACKERMETHODBODY_03 = "_$$_packers[%d].pack($1, %s_$$_t.%s%s); ";
 
 	String STATEMENT_PACKER_PACKERMETHODBODY_04 = "$1.pack(((java.lang.Enum)_$$_t).ordinal()); ";
 
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index bb9cdd5..3cf8043 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -5,7 +5,6 @@ import java.lang.annotation.Annotation;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
-import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -17,19 +16,14 @@ import javassist.CtMethod;
 import javassist.CtNewMethod;
 import javassist.NotFoundException;
 
-import org.msgpack.CustomMessage;
-import org.msgpack.CustomPacker;
 import org.msgpack.MessagePackObject;
-import org.msgpack.MessagePackable;
 import org.msgpack.MessagePacker;
 import org.msgpack.MessageTypeException;
 import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
-import org.msgpack.annotation.MessagePackDelegate;
-import org.msgpack.annotation.MessagePackMessage;
 import org.msgpack.annotation.MessagePackOptional;
-import org.msgpack.annotation.MessagePackOrdinalEnum;
+import org.msgpack.packer.OptionalPacker;
 import org.msgpack.template.OptionalTemplate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -51,9 +45,12 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private ConcurrentHashMap tmplCache;
 
+	private ConcurrentHashMap pkCache;
+
 	DynamicCodeGen() {
 		super();
 		tmplCache = new ConcurrentHashMap();
+		pkCache = new ConcurrentHashMap();
 	}
 
 	public void setTemplates(Class type, Template[] tmpls) {
@@ -64,6 +61,14 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		return tmplCache.get(type.getName());
 	}
 
+	public void setMessagePackers(Class type, MessagePacker[] pks) {
+		pkCache.putIfAbsent(type.getName(), pks);
+	}
+
+	public MessagePacker[] getMessagePackers(Class type) {
+		return pkCache.get(type.getName());
+	}
+
 	public Class generateMessagePackerClass(Class origClass,
 			List fieldOpts) {
 		try {
@@ -74,12 +79,18 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			checkTypeValidation(origClass);
 			checkDefaultConstructorValidation(origClass);
 			CtClass packerCtClass = pool.makeClass(packerName);
+			setSuperclass(packerCtClass, MessagePackerAccessorImpl.class);
 			setInterface(packerCtClass, MessagePacker.class);
-			addDefaultConstructor(packerCtClass);
+			addClassTypeConstructor(packerCtClass);
 			Field[] fields = getDeclaredFields(origClass);
+			MessagePacker[] packers = null;
 			if (fieldOpts != null) {
 				fields = sortFields(fields, fieldOpts);
+				packers = createMessagePackers(fieldOpts);
+			} else {
+				packers = createMessagePackers(fields);
 			}
+			setMessagePackers(origClass, packers);
 			addPackMethod(packerCtClass, origClass, fields, false);
 			Class packerClass = createClass(packerCtClass);
 			LOG.debug("generated a packer class for " + origClass.getName());
@@ -101,8 +112,9 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			String packerName = origName + POSTFIX_TYPE_NAME_PACKER + inc();
 			checkTypeValidation(origClass);
 			CtClass packerCtClass = pool.makeClass(packerName);
+			setSuperclass(packerCtClass, MessagePackerAccessorImpl.class);
 			setInterface(packerCtClass, MessagePacker.class);
-			addDefaultConstructor(packerCtClass);
+			addClassTypeConstructor(packerCtClass);
 			addPackMethod(packerCtClass, origClass, null, true);
 			Class packerClass = createClass(packerCtClass);
 			LOG.debug("generated an enum class for " + origClass.getName());
@@ -128,7 +140,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			CtClass tmplCtClass = pool.makeClass(tmplName);
 			setSuperclass(tmplCtClass, TemplateAccessorImpl.class);
 			setInterface(tmplCtClass, Template.class);
-			addDefaultConstructor(tmplCtClass);
+			addClassTypeConstructor(tmplCtClass);
 			Field[] fields = getDeclaredFields(origClass);
 			Template[] tmpls = null;
 			if (fieldOpts != null) {
@@ -166,7 +178,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			CtClass tmplCtClass = pool.makeClass(tmplName);
 			setSuperclass(tmplCtClass, TemplateAccessorImpl.class);
 			setInterface(tmplCtClass, Template.class);
-			addDefaultConstructor(tmplCtClass);
+			addClassTypeConstructor(tmplCtClass);
 			addUnpackMethod(tmplCtClass, origClass, null, true);
 			addConvertMethod(tmplCtClass, origClass, null, true);
 			Class tmplClass = createClass(tmplCtClass);
@@ -279,6 +291,38 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		return sorted;
 	}
 
+	MessagePacker[] createMessagePackers(List fieldOpts) {
+		MessagePacker[] pks = new MessagePacker[fieldOpts.size()];
+		for (int i = 0; i < pks.length; ++i) {
+			pks[i] = toMessagePacker(fieldOpts.get(i).tmpl);
+		}
+		return pks;
+	}
+
+	MessagePacker[] createMessagePackers(Field[] fields) {
+		MessagePacker[] pks = new MessagePacker[fields.length];
+		for (int i = 0; i < pks.length; ++i) {
+			pks[i] = createMessagePacker(fields[i]);
+		}
+		return pks;
+	}
+
+	MessagePacker createMessagePacker(Field field) {
+		boolean isOptional = isAnnotated(field, MessagePackOptional.class);
+		Class c = field.getType();
+		MessagePacker pk = null;
+		if (List.class.isAssignableFrom(c) || Map.class.isAssignableFrom(c)) {
+			pk = createMessagePacker(field.getGenericType());
+		} else {
+			pk = createMessagePacker(c);
+		}
+		if (isOptional) {
+			return new OptionalPacker(pk);
+		} else {
+			return pk;
+		}
+	}
+
 	Template[] createTemplates(List fieldOpts) {
 		Template[] tmpls = new Template[fieldOpts.size()];
 		for (int i = 0; i < tmpls.length; ++i) {
@@ -310,17 +354,17 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			return tmpl;
 		}
 	}
-	
+
 	private boolean isAnnotated(Field field, Class with) {
 		return field.getAnnotation(with) != null;
 	}
 
-	private void addPackMethod(CtClass packerCtClass, Class c, Field[] fs,
-			boolean isEnum) {
+	private void addPackMethod(CtClass packerCtClass, Class c,
+			Field[] fields, boolean isEnum) {
 		// void pack(Packer pk, Object target) throws IOException;
 		StringBuilder sb = new StringBuilder();
 		if (!isEnum) {
-			insertPackMethodBody(sb, c, fs);
+			insertPackMethodBody(sb, c, fields);
 		} else {
 			insertOrdinalEnumPackMethodBody(sb, c);
 		}
@@ -360,52 +404,20 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_01, args0));
 		Object[] args1 = new Object[] { fields.length };
 		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_02, args1));
-		for (Field f : fields) {
-			insertCodeOfPackMethodCall(sb, f);
+		for (int i = 0; i < fields.length; ++i) {
+			insertCodeOfPackMethodCall(sb, fields[i], i);
 		}
 		sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
 	}
 
-	private void insertCodeOfPackMethodCall(StringBuilder sb, Field field) {
-		Class c = field.getType();
-		if (c.isPrimitive()) {
-			; // ignore
-		} else if (c.equals(Boolean.class) || c.equals(Byte.class)
-				|| c.equals(Double.class) || c.equals(Float.class)
-				|| c.equals(Integer.class) || c.equals(Long.class)
-				|| c.equals(Short.class)) {
-			; // ignore
-		} else if (c.equals(String.class) || c.equals(BigInteger.class)
-				|| c.equals(byte[].class)) {
-			; // ignore
-		} else if (List.class.isAssignableFrom(c)
-				|| Map.class.isAssignableFrom(c)) {
-			; // ignore
-		} else if (CustomPacker.isRegistered(c)) {
-			; // ignore
-		} else if (MessagePackable.class.isAssignableFrom(c)) {
-			; // ignore
-		} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
-			// @MessagePackMessage
-			MessagePacker packer = DynamicPacker.create(c);
-			CustomMessage.registerPacker(c, packer);
-		} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
-			// FIXME DelegatePacker
-			UnsupportedOperationException e = new UnsupportedOperationException(
-					"not supported yet. : " + c.getName());
-			LOG.error(e.getMessage(), e);
-			throw e;
-		} else if (CustomMessage.isAnnotated(c, MessagePackOrdinalEnum.class)) {
-			// @MessagePackOrdinalEnum
-			MessagePacker packer = DynamicOrdinalEnumPacker.create(c);
-			CustomMessage.registerPacker(c, packer);
-		} else {
-			MessageTypeException e = new MessageTypeException("unknown type: "
-					+ c.getName());
-			LOG.error(e.getMessage(), e);
-			throw e;
-		}
-		Object[] args = new Object[] { field.getName() };
+	private void insertCodeOfPackMethodCall(StringBuilder sb, Field field, int i) {
+		// _$$_packers[i].pack($1, new Integer(target.fi));
+		Class type = field.getType();
+		boolean isPrim = type.isPrimitive();
+		Object[] args = new Object[] {
+				i,
+				isPrim ? "new " + getPrimToWrapperType(type).getName() + "("
+						: "", field.getName(), isPrim ? ")" : "" };
 		sb.append(String.format(STATEMENT_PACKER_PACKERMETHODBODY_03, args));
 	}
 
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 95c3748..9b51c94 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -7,6 +7,7 @@ import java.lang.reflect.Method;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
 import java.math.BigInteger;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -23,16 +24,46 @@ import javassist.NotFoundException;
 
 import org.msgpack.CustomConverter;
 import org.msgpack.CustomMessage;
+import org.msgpack.CustomPacker;
 import org.msgpack.MessageConvertable;
 import org.msgpack.MessagePackObject;
+import org.msgpack.MessagePackable;
+import org.msgpack.MessagePacker;
 import org.msgpack.MessageTypeException;
 import org.msgpack.MessageUnpackable;
+import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Templates;
 import org.msgpack.Unpacker;
 import org.msgpack.annotation.MessagePackDelegate;
 import org.msgpack.annotation.MessagePackMessage;
 import org.msgpack.annotation.MessagePackOrdinalEnum;
+import org.msgpack.packer.BigIntegerPacker;
+import org.msgpack.packer.BooleanPacker;
+import org.msgpack.packer.ByteArrayPacker;
+import org.msgpack.packer.BytePacker;
+import org.msgpack.packer.DoublePacker;
+import org.msgpack.packer.FloatPacker;
+import org.msgpack.packer.IntegerPacker;
+import org.msgpack.packer.LongPacker;
+import org.msgpack.packer.OptionalPacker;
+import org.msgpack.packer.ShortPacker;
+import org.msgpack.packer.StringPacker;
+import org.msgpack.template.BigIntegerTemplate;
+import org.msgpack.template.BooleanTemplate;
+import org.msgpack.template.ByteArrayTemplate;
+import org.msgpack.template.ByteTemplate;
+import org.msgpack.template.ClassTemplate;
+import org.msgpack.template.CollectionTemplate;
+import org.msgpack.template.DoubleTemplate;
+import org.msgpack.template.FloatTemplate;
+import org.msgpack.template.IntegerTemplate;
+import org.msgpack.template.ListTemplate;
+import org.msgpack.template.LongTemplate;
+import org.msgpack.template.MapTemplate;
+import org.msgpack.template.OptionalTemplate;
+import org.msgpack.template.ShortTemplate;
+import org.msgpack.template.StringTemplate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,12 +72,66 @@ public class DynamicCodeGenBase implements Constants {
 	private static Logger LOG = LoggerFactory
 			.getLogger(DynamicCodeGenBase.class);
 
-	public static class MessageUnpackableConvertableTemplate implements
-			Template {
-
+	static class MessagePackablePacker implements MessagePacker {
+		@SuppressWarnings("unused")
 		private Class type;
 
-		public MessageUnpackableConvertableTemplate(Class type) {
+		MessagePackablePacker(Class type) {
+			this.type = type;
+		}
+
+		@Override
+		public void pack(Packer packer, Object target) throws IOException {
+			MessagePackable mp = MessagePackable.class.cast(target);
+			mp.messagePack(packer);
+		}
+	}
+
+	static class ListPacker implements MessagePacker {
+		private MessagePacker elementPacker;
+
+		ListPacker(MessagePacker elementPacker) {
+			this.elementPacker = elementPacker;
+		}
+
+		@SuppressWarnings("unchecked")
+		@Override
+		public void pack(Packer packer, Object target) throws IOException {
+			List list = (List) target;
+			packer.packArray(list.size());
+			for (Iterator iter = list.iterator(); iter.hasNext();) {
+				elementPacker.pack(packer, iter.next());
+			}
+		}
+	}
+
+	static class MapPacker implements MessagePacker {
+		private MessagePacker keyPacker;
+		private MessagePacker valPacker;
+
+		MapPacker(MessagePacker keyPacker, MessagePacker valPacker) {
+			this.keyPacker = keyPacker;
+			this.valPacker = valPacker;
+		}
+
+		@SuppressWarnings("unchecked")
+		@Override
+		public void pack(Packer packer, Object target) throws IOException {
+			Map map = (Map) target;
+			packer.packMap(map.size());
+			for (Map.Entry e : ((Map) map)
+					.entrySet()) {
+				keyPacker.pack(packer, e.getKey());
+				valPacker.pack(packer, e.getValue());
+			}
+		}
+
+	}
+
+	static class MessageUnpackableConvertableTemplate implements Template {
+		private Class type;
+
+		MessageUnpackableConvertableTemplate(Class type) {
 			this.type = type;
 		}
 
@@ -84,13 +169,43 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	public static interface TemplateAccessor {
+	static interface MessagePackerAccessor {
+		void setMessagePackers(MessagePacker[] packers);
+	}
+
+	static class MessagePackerAccessorImpl implements MessagePackerAccessor {
+		public Class type;
+
+		public MessagePacker[] _$$_packers;
+
+		public MessagePackerAccessorImpl() {
+		}
+
+		public MessagePackerAccessorImpl(Class type) {
+			this.type = type;
+		}
+
+		public void setMessagePackers(MessagePacker[] _$$_pks) {
+			_$$_packers = _$$_pks;
+		}
+	}
+
+	static interface TemplateAccessor {
 		void setTemplates(Template[] templates);
 	}
 
-	public static class TemplateAccessorImpl implements TemplateAccessor {
+	static class TemplateAccessorImpl implements TemplateAccessor {
+		public Class type;
+
 		public Template[] _$$_templates;
 
+		public TemplateAccessorImpl() {
+		}
+
+		public TemplateAccessorImpl(Class type) {
+			this.type = type;
+		}
+
 		public void setTemplates(Template[] _$$_tmpls) {
 			_$$_templates = _$$_tmpls;
 		}
@@ -190,11 +305,38 @@ public class DynamicCodeGenBase implements Constants {
 		newCtClass.addInterface(infCtClass);
 	}
 
-	protected void addDefaultConstructor(CtClass enhancedCtClass)
+	protected void addClassTypeConstructor(CtClass newCtClass)
+			throws CannotCompileException, NotFoundException {
+		CtConstructor newCtCons = CtNewConstructor.make(new CtClass[] { pool
+				.get(Class.class.getName()) }, new CtClass[0], newCtClass);
+		newCtClass.addConstructor(newCtCons);
+	}
+
+	protected void addDefaultConstructor(CtClass newCtClass)
 			throws CannotCompileException {
 		CtConstructor newCtCons = CtNewConstructor
-				.defaultConstructor(enhancedCtClass);
-		enhancedCtClass.addConstructor(newCtCons);
+				.defaultConstructor(newCtClass);
+		newCtClass.addConstructor(newCtCons);
+	}
+
+	protected void addMessagePackerArrayField(CtClass newCtClass)
+			throws NotFoundException, CannotCompileException {
+		CtClass acsCtClass = pool
+				.get(MessagePackerAccessorImpl.class.getName());
+		CtField pksField = acsCtClass.getDeclaredField(VARIABLE_NAME_PACKERS);
+		CtField pksField2 = new CtField(pksField.getType(), pksField.getName(),
+				newCtClass);
+		newCtClass.addField(pksField2);
+	}
+
+	protected void addSetMessagePackersMethod(CtClass newCtClass)
+			throws NotFoundException, CannotCompileException {
+		CtClass acsCtClass = pool.get(TemplateAccessorImpl.class.getName());
+		CtMethod setPksMethod = acsCtClass
+				.getDeclaredMethod(METHOD_NAME_SETMESSAGEPACKERS);
+		CtMethod setPksMethod2 = CtNewMethod.copy(setPksMethod, newCtClass,
+				null);
+		newCtClass.addMethod(setPksMethod2);
 	}
 
 	protected void addTemplateArrayField(CtClass newCtClass)
@@ -318,6 +460,139 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
+	public static MessagePacker toMessagePacker(Template tmpl) {
+		if (tmpl instanceof BigIntegerTemplate) {
+			return BigIntegerPacker.getInstance();
+		} else if (tmpl instanceof BooleanTemplate) {
+			return BooleanPacker.getInstance();
+		} else if (tmpl instanceof ByteArrayTemplate) {
+			return ByteArrayPacker.getInstance();
+		} else if (tmpl instanceof ByteTemplate) {
+			return BytePacker.getInstance();
+		} else if (tmpl instanceof ClassTemplate) {
+			UnsupportedOperationException e = new UnsupportedOperationException(
+					"not supported yet.");
+			LOG.error(e.getMessage(), e);
+			throw e;
+		} else if (tmpl instanceof CollectionTemplate) {
+			UnsupportedOperationException e = new UnsupportedOperationException(
+					"not supported yet.");
+			LOG.error(e.getMessage(), e);
+			throw e;
+		} else if (tmpl instanceof DoubleTemplate) {
+			return DoublePacker.getInstance();
+		} else if (tmpl instanceof FloatTemplate) {
+			return FloatPacker.getInstance();
+		} else if (tmpl instanceof IntegerTemplate) {
+			return IntegerPacker.getInstance();
+		} else if (tmpl instanceof ListTemplate) {
+			ListTemplate t = (ListTemplate) tmpl;
+			return new ListPacker(toMessagePacker(t.getElementTemplate()));
+		} else if (tmpl instanceof LongTemplate) {
+			return LongPacker.getInstance();
+		} else if (tmpl instanceof MapTemplate) {
+			MapTemplate t = (MapTemplate) tmpl;
+			return new MapPacker(toMessagePacker(t.getKeyTemplate()),
+					toMessagePacker(t.getValueTemplate()));
+		} else if (tmpl instanceof OptionalTemplate) {
+			OptionalTemplate t = (OptionalTemplate) tmpl;
+			return new OptionalPacker(toMessagePacker(t.getElementTemplate()));
+		} else if (tmpl instanceof ShortTemplate) {
+			return ShortPacker.getInstance();
+		} else if (tmpl instanceof StringTemplate) {
+			return StringPacker.getInstance();
+		} else if (tmpl instanceof TemplateAccessorImpl) {
+			Class c = ((TemplateAccessorImpl) tmpl).type;
+			if (CustomPacker.isRegistered(c)) {
+				return CustomPacker.get(c);
+			} else {
+				MessagePacker packer = DynamicPacker.create(c);
+				CustomMessage.registerPacker(c, packer);
+				return packer;
+			}
+		}
+		UnsupportedOperationException e = new UnsupportedOperationException(
+				"not supported yet.");
+		LOG.error(e.getMessage(), e);
+		throw e;
+	}
+
+	public MessagePacker createMessagePacker(Type t) {
+		if (t.getClass().equals(Class.class)) {
+			Class c = (Class) t;
+			if (c.equals(boolean.class) || c.equals(Boolean.class)) {
+				return BooleanPacker.getInstance();
+			} else if (c.equals(byte.class) || c.equals(Byte.class)) {
+				return BytePacker.getInstance();
+			} else if (c.equals(short.class) || c.equals(Short.class)) {
+				return ShortPacker.getInstance();
+			} else if (c.equals(int.class) || c.equals(Integer.class)) {
+				return IntegerPacker.getInstance();
+			} else if (c.equals(float.class) || c.equals(Float.class)) {
+				return FloatPacker.getInstance();
+			} else if (c.equals(long.class) || c.equals(Long.class)) {
+				return LongPacker.getInstance();
+			} else if (c.equals(double.class) || c.equals(Double.class)) {
+				return DoublePacker.getInstance();
+			} else if (c.equals(String.class)) {
+				return StringPacker.getInstance();
+			} else if (c.equals(BigInteger.class)) {
+				return BigIntegerPacker.getInstance();
+			} else if (CustomPacker.isRegistered(c)) {
+				return CustomPacker.get(c);
+			} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
+				// @MessagePackMessage
+				MessagePacker packer = DynamicPacker.create(c);
+				CustomMessage.registerPacker(c, packer);
+				return packer;
+			} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
+				// FIXME DelegatePacker
+				UnsupportedOperationException e = new UnsupportedOperationException(
+						"not supported yet. : " + c.getName());
+				LOG.error(e.getMessage(), e);
+				throw e;
+			} else if (CustomMessage.isAnnotated(c,
+					MessagePackOrdinalEnum.class)) {
+				// @MessagePackOrdinalEnum
+				MessagePacker packer = DynamicOrdinalEnumPacker.create(c);
+				CustomMessage.registerPacker(c, packer);
+				return packer;
+			} else if (MessagePackable.class.isAssignableFrom(c)) {
+				MessagePacker packer = new MessagePackablePacker(c);
+				CustomMessage.registerPacker(c, packer);
+				return packer;
+			} else {
+				throw new MessageTypeException("Type error: "
+						+ ((Class) t).getName());
+			}
+		} else if (t instanceof GenericArrayType) {
+			GenericArrayType gat = (GenericArrayType) t;
+			Type gct = gat.getGenericComponentType();
+			if (gct.equals(byte.class)) {
+				return ByteArrayPacker.getInstance();
+			} else {
+				throw new DynamicCodeGenException("Not supported yet: " + gat);
+			}
+		} else if (t instanceof ParameterizedType) {
+			ParameterizedType pt = (ParameterizedType) t;
+			Class rawType = (Class) pt.getRawType();
+			if (rawType.equals(List.class)) {
+				Type[] ats = pt.getActualTypeArguments();
+				return new ListPacker(createMessagePacker(ats[0]));
+			} else if (rawType.equals(Map.class)) {
+				Type[] ats = pt.getActualTypeArguments();
+				return new MapPacker(createMessagePacker(ats[0]),
+						createMessagePacker(ats[1]));
+			} else {
+				throw new DynamicCodeGenException("Type error: "
+						+ t.getClass().getName());
+			}
+		} else {
+			throw new DynamicCodeGenException("Type error: "
+					+ t.getClass().getName());
+		}
+	}
+
 	public Template createTemplate(Type t) {
 		if (t.getClass().equals(Class.class)) {
 			Class c = (Class) t;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumConverter.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumConverter.java
index 648b4f8..7f5db66 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumConverter.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumConverter.java
@@ -4,14 +4,6 @@ import org.msgpack.MessageConverter;
 
 public class DynamicOrdinalEnumConverter {
 	public static MessageConverter create(Class c) {
-		try {
-			DynamicCodeGen gen = DynamicCodeGen.getInstance();
-			Class unpackerClass = gen.generateOrdinalEnumTemplateClass(c);
-			return (MessageConverter) unpackerClass.newInstance();
-		} catch (InstantiationException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (IllegalAccessException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
+		return DynamicOrdinalEnumTemplate.create(c);
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
index 98f4a39..101b3f1 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
@@ -1,17 +1,34 @@
 package org.msgpack.util.codegen;
 
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
 import org.msgpack.MessagePacker;
+import org.msgpack.util.codegen.DynamicCodeGenBase.MessagePackerAccessor;
 
 public class DynamicOrdinalEnumPacker {
 	public static MessagePacker create(Class c) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
 			Class packerClass = gen.generateOrdinalEnumPackerClass(c);
-			return (MessagePacker) packerClass.newInstance();
+			Constructor cons = packerClass
+					.getDeclaredConstructor(new Class[] { Class.class });
+			Object obj = cons.newInstance(new Object[] { c });
+			((MessagePackerAccessor) obj).setMessagePackers(gen
+					.getMessagePackers(c));
+			return (MessagePacker) obj;
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (IllegalAccessException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (SecurityException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (NoSuchMethodException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalArgumentException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (InvocationTargetException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumTemplate.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumTemplate.java
index c234683..6a0287a 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumTemplate.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumTemplate.java
@@ -1,17 +1,33 @@
 package org.msgpack.util.codegen;
 
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
 import org.msgpack.Template;
+import org.msgpack.util.codegen.DynamicCodeGenBase.TemplateAccessor;
 
 public class DynamicOrdinalEnumTemplate {
 	public static Template create(Class c) {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
 			Class tmplClass = gen.generateOrdinalEnumTemplateClass(c);
-			return (Template) tmplClass.newInstance();
+			Constructor cons = tmplClass
+					.getDeclaredConstructor(new Class[] { Class.class });
+			Object obj = cons.newInstance(new Object[] { c });
+			((TemplateAccessor) obj).setTemplates(gen.getTemplates(c));
+			return (Template) obj;
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (IllegalAccessException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (SecurityException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (NoSuchMethodException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalArgumentException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (InvocationTargetException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumUnpacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumUnpacker.java
index edad9a8..7b83900 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumUnpacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumUnpacker.java
@@ -4,14 +4,6 @@ import org.msgpack.MessageUnpacker;
 
 public class DynamicOrdinalEnumUnpacker {
 	public static MessageUnpacker create(Class c) {
-		try {
-			DynamicCodeGen gen = DynamicCodeGen.getInstance();
-			Class unpackerClass = gen.generateOrdinalEnumTemplateClass(c);
-			return (MessageUnpacker) unpackerClass.newInstance();
-		} catch (InstantiationException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (IllegalAccessException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
+		return DynamicOrdinalEnumTemplate.create(c);
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
index fbd019d..65bcc96 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
@@ -1,8 +1,11 @@
 package org.msgpack.util.codegen;
 
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 
 import org.msgpack.MessagePacker;
+import org.msgpack.util.codegen.DynamicCodeGenBase.MessagePackerAccessor;
 
 public class DynamicPacker {
 
@@ -14,11 +17,24 @@ public class DynamicPacker {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
 			Class packerClass = gen.generateMessagePackerClass(c, fieldOpts);
-			return (MessagePacker) packerClass.newInstance();
+			Constructor cons = packerClass
+					.getDeclaredConstructor(new Class[] { Class.class });
+			Object obj = cons.newInstance(new Object[] { c });
+			((MessagePackerAccessor) obj).setMessagePackers(gen
+					.getMessagePackers(c));
+			return (MessagePacker) obj;
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (IllegalAccessException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (SecurityException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (NoSuchMethodException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalArgumentException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (InvocationTargetException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java b/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
index 673c344..7aeffa9 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
@@ -1,5 +1,7 @@
 package org.msgpack.util.codegen;
 
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 
 import org.msgpack.Template;
@@ -14,13 +16,23 @@ public class DynamicTemplate {
 		try {
 			DynamicCodeGen gen = DynamicCodeGen.getInstance();
 			Class tmplClass = gen.generateTemplateClass(c, fieldOpts);
-			Object obj = tmplClass.newInstance();
+			Constructor cons = tmplClass
+					.getDeclaredConstructor(new Class[] { Class.class });
+			Object obj = cons.newInstance(new Object[] { c });
 			((TemplateAccessor) obj).setTemplates(gen.getTemplates(c));
 			return (Template) obj;
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
 		} catch (IllegalAccessException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (SecurityException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (NoSuchMethodException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (IllegalArgumentException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
+		} catch (InvocationTargetException e) {
+			throw new DynamicCodeGenException(e.getMessage(), e);
 		}
 	}
 }

From 7e5c5153a8fdaeda80f147d0b76804530d08f9e6 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 23 Oct 2010 14:50:32 +0900
Subject: [PATCH 0891/1648] java: edit copyright notions in
 org.msgpack.annotation.*.java and org.msgpack.util.codegen.*.java

---
 .../annotation/MessagePackDelegate.java        | 18 +++++++++++++++++-
 .../msgpack/annotation/MessagePackMessage.java | 18 +++++++++++++++++-
 .../annotation/MessagePackOptional.java        | 17 +++++++++++++++++
 .../annotation/MessagePackOrdinalEnum.java     | 18 +++++++++++++++++-
 .../org/msgpack/util/codegen/Constants.java    | 17 +++++++++++++++++
 .../msgpack/util/codegen/DynamicCodeGen.java   | 17 +++++++++++++++++
 .../util/codegen/DynamicCodeGenBase.java       | 17 +++++++++++++++++
 .../util/codegen/DynamicCodeGenException.java  | 17 +++++++++++++++++
 .../msgpack/util/codegen/DynamicConverter.java | 17 +++++++++++++++++
 .../codegen/DynamicOrdinalEnumConverter.java   | 17 +++++++++++++++++
 .../util/codegen/DynamicOrdinalEnumPacker.java | 17 +++++++++++++++++
 .../codegen/DynamicOrdinalEnumTemplate.java    | 17 +++++++++++++++++
 .../codegen/DynamicOrdinalEnumUnpacker.java    | 17 +++++++++++++++++
 .../msgpack/util/codegen/DynamicPacker.java    | 17 +++++++++++++++++
 .../msgpack/util/codegen/DynamicTemplate.java  | 17 +++++++++++++++++
 .../msgpack/util/codegen/DynamicUnpacker.java  | 17 +++++++++++++++++
 .../org/msgpack/util/codegen/FieldOption.java  | 17 +++++++++++++++++
 17 files changed, 289 insertions(+), 3 deletions(-)

diff --git a/java/src/main/java/org/msgpack/annotation/MessagePackDelegate.java b/java/src/main/java/org/msgpack/annotation/MessagePackDelegate.java
index 2a72d73..88c6f8c 100644
--- a/java/src/main/java/org/msgpack/annotation/MessagePackDelegate.java
+++ b/java/src/main/java/org/msgpack/annotation/MessagePackDelegate.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.annotation;
 
 import java.lang.annotation.ElementType;
@@ -8,5 +25,4 @@ import java.lang.annotation.Target;
 @Target(ElementType.TYPE)
 @Retention(RetentionPolicy.RUNTIME)
 public @interface MessagePackDelegate {
-
 }
diff --git a/java/src/main/java/org/msgpack/annotation/MessagePackMessage.java b/java/src/main/java/org/msgpack/annotation/MessagePackMessage.java
index 6efeb9d..5f781e0 100644
--- a/java/src/main/java/org/msgpack/annotation/MessagePackMessage.java
+++ b/java/src/main/java/org/msgpack/annotation/MessagePackMessage.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.annotation;
 
 import java.lang.annotation.ElementType;
@@ -8,5 +25,4 @@ import java.lang.annotation.Target;
 @Target(ElementType.TYPE)
 @Retention(RetentionPolicy.RUNTIME)
 public @interface MessagePackMessage {
-
 }
diff --git a/java/src/main/java/org/msgpack/annotation/MessagePackOptional.java b/java/src/main/java/org/msgpack/annotation/MessagePackOptional.java
index f9eb227..9c1a0bb 100644
--- a/java/src/main/java/org/msgpack/annotation/MessagePackOptional.java
+++ b/java/src/main/java/org/msgpack/annotation/MessagePackOptional.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.annotation;
 
 import java.lang.annotation.ElementType;
diff --git a/java/src/main/java/org/msgpack/annotation/MessagePackOrdinalEnum.java b/java/src/main/java/org/msgpack/annotation/MessagePackOrdinalEnum.java
index 4b0d9bb..eac4767 100644
--- a/java/src/main/java/org/msgpack/annotation/MessagePackOrdinalEnum.java
+++ b/java/src/main/java/org/msgpack/annotation/MessagePackOrdinalEnum.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.annotation;
 
 import java.lang.annotation.ElementType;
@@ -8,5 +25,4 @@ import java.lang.annotation.Target;
 @Target(ElementType.TYPE)
 @Retention(RetentionPolicy.RUNTIME)
 public @interface MessagePackOrdinalEnum {
-
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/Constants.java b/java/src/main/java/org/msgpack/util/codegen/Constants.java
index b85c94d..7474bd3 100644
--- a/java/src/main/java/org/msgpack/util/codegen/Constants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/Constants.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 public interface Constants {
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 3cf8043..0a01f62 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 import java.io.IOException;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 9b51c94..43db5c6 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 import java.io.IOException;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
index df1f2e7..d4a2906 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenException.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 @SuppressWarnings("serial")
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicConverter.java b/java/src/main/java/org/msgpack/util/codegen/DynamicConverter.java
index 8919a67..346de2d 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicConverter.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicConverter.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 import java.util.List;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumConverter.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumConverter.java
index 7f5db66..598a878 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumConverter.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumConverter.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 import org.msgpack.MessageConverter;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
index 101b3f1..ddb0990 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 import java.lang.reflect.Constructor;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumTemplate.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumTemplate.java
index 6a0287a..65adcd8 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumTemplate.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumTemplate.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 import java.lang.reflect.Constructor;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumUnpacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumUnpacker.java
index 7b83900..4c00386 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumUnpacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumUnpacker.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 import org.msgpack.MessageUnpacker;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
index 65bcc96..a483c02 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 import java.lang.reflect.Constructor;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java b/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
index 7aeffa9..9bd6aef 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicTemplate.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 import java.lang.reflect.Constructor;
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicUnpacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicUnpacker.java
index 8b927cb..ea198f0 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicUnpacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicUnpacker.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 import java.util.List;
diff --git a/java/src/main/java/org/msgpack/util/codegen/FieldOption.java b/java/src/main/java/org/msgpack/util/codegen/FieldOption.java
index 127688d..9fb3e3b 100644
--- a/java/src/main/java/org/msgpack/util/codegen/FieldOption.java
+++ b/java/src/main/java/org/msgpack/util/codegen/FieldOption.java
@@ -1,3 +1,20 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
 package org.msgpack.util.codegen;
 
 import org.msgpack.Template;

From 3473800ab652683f02f2ac6709ae88419e016ff8 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 23 Oct 2010 23:11:52 +0900
Subject: [PATCH 0892/1648] java: not use a putIfAbsent method in
 ConcurrentHashMap class

---
 java/src/main/java/org/msgpack/CustomConverter.java           | 2 +-
 java/src/main/java/org/msgpack/CustomPacker.java              | 2 +-
 java/src/main/java/org/msgpack/CustomUnpacker.java            | 2 +-
 .../main/java/org/msgpack/util/codegen/DynamicCodeGen.java    | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/java/src/main/java/org/msgpack/CustomConverter.java b/java/src/main/java/org/msgpack/CustomConverter.java
index a82e3e1..b9fb0b3 100644
--- a/java/src/main/java/org/msgpack/CustomConverter.java
+++ b/java/src/main/java/org/msgpack/CustomConverter.java
@@ -30,7 +30,7 @@ public class CustomConverter {
 	public static void register(Class target, MessageConverter converter) {
 		LOG.debug("register a MessageConverter object for the type: "
 				+ target.getName());
-		map.putIfAbsent(target, converter);
+		map.put(target, converter);
 	}
 
 	public static MessageConverter get(Class target) {
diff --git a/java/src/main/java/org/msgpack/CustomPacker.java b/java/src/main/java/org/msgpack/CustomPacker.java
index 6a65fc3..0c128b8 100644
--- a/java/src/main/java/org/msgpack/CustomPacker.java
+++ b/java/src/main/java/org/msgpack/CustomPacker.java
@@ -30,7 +30,7 @@ public class CustomPacker {
 	public static void register(Class target, MessagePacker packer) {
 		LOG.debug("register a MessagePacker object for the type: "
 				+ target.getName());
-		map.putIfAbsent(target, packer);
+		map.put(target, packer);
 	}
 
 	public static MessagePacker get(Class target) {
diff --git a/java/src/main/java/org/msgpack/CustomUnpacker.java b/java/src/main/java/org/msgpack/CustomUnpacker.java
index 8e80115..fbf64b7 100644
--- a/java/src/main/java/org/msgpack/CustomUnpacker.java
+++ b/java/src/main/java/org/msgpack/CustomUnpacker.java
@@ -30,7 +30,7 @@ public class CustomUnpacker {
 	public static void register(Class target, MessageUnpacker converter) {
 		LOG.debug("register a MessageUnpacker object for the type: "
 				+ target.getName());
-		map.putIfAbsent(target, converter);
+		map.put(target, converter);
 	}
 
 	public static MessageUnpacker get(Class target) {
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 0a01f62..43f5b3d 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -71,7 +71,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	}
 
 	public void setTemplates(Class type, Template[] tmpls) {
-		tmplCache.putIfAbsent(type.getName(), tmpls);
+		tmplCache.put(type.getName(), tmpls);
 	}
 
 	public Template[] getTemplates(Class type) {
@@ -79,7 +79,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 	}
 
 	public void setMessagePackers(Class type, MessagePacker[] pks) {
-		pkCache.putIfAbsent(type.getName(), pks);
+		pkCache.put(type.getName(), pks);
 	}
 
 	public MessagePacker[] getMessagePackers(Class type) {

From 57446de875ab7f7d7515923f41d833cd5655eacb Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 23 Oct 2010 23:13:17 +0900
Subject: [PATCH 0893/1648] java: refactor org.msgpack.util.codegen.*.java

---
 .../org/msgpack/util/codegen/Constants.java   | 52 ---------------
 .../util/codegen/DynamicCodeGenBase.java      | 65 +------------------
 .../msgpack/util/codegen/DynamicPacker.java   |  4 +-
 3 files changed, 4 insertions(+), 117 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/Constants.java b/java/src/main/java/org/msgpack/util/codegen/Constants.java
index 7474bd3..1932739 100644
--- a/java/src/main/java/org/msgpack/util/codegen/Constants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/Constants.java
@@ -68,58 +68,6 @@ public interface Constants {
 
 	String METHOD_NAME_UNPACK = "unpack";
 
-	String METHOD_NAME_UNPACKBOOLEAN = "unpackBoolean";
-
-	String METHOD_NAME_UNPACKBYTE = "unpackByte";
-
-	String METHOD_NAME_UNPACKDOUBLE = "unpackDouble";
-
-	String METHOD_NAME_UNPACKFLOAT = "unpackFloat";
-
-	String METHOD_NAME_UNPACKINT = "unpackInt";
-
-	String METHOD_NAME_UNPACKLONG = "unpackLong";
-
-	String METHOD_NAME_UNPACKSHORT = "unpackShort";
-
-	String METHOD_NAME_UNPACKSTRING = "unpackString";
-
-	String METHOD_NAME_UNPACKBIGINTEGER = "unpackBigInteger";
-
-	String METHOD_NAME_UNPACKOBJECT = "unpackObject";
-
-	String METHOD_NAME_UNPACKBYTEARRAY = "unpackByteArray";
-
-	String METHOD_NAME_UNPACKARRAY = "unpackArray";
-
-	String METHOD_NAME_UNPACKMAP = "unpackMap";
-
-	String METHOD_NAME_ASARRAY = "asArray";
-
-	String METHOD_NAME_ASBOOLEAN = "asBoolean";
-
-	String METHOD_NAME_ASBYTE = "asByte";
-
-	String METHOD_NAME_ASSHORT = "asShort";
-
-	String METHOD_NAME_ASINT = "asInt";
-
-	String METHOD_NAME_ASFLOAT = "asFloat";
-
-	String METHOD_NAME_ASLONG = "asLong";
-
-	String METHOD_NAME_ASDOUBLE = "asDouble";
-
-	String METHOD_NAME_ASSTRING = "asString";
-
-	String METHOD_NAME_ASBYTEARRAY = "asByteArray";
-
-	String METHOD_NAME_ASBIGINTEGER = "asBigInteger";
-
-	String METHOD_NAME_ASLIST = "asList";
-
-	String METHOD_NAME_ASMAP = "asMap";
-
 	String STATEMENT_PACKER_PACKERMETHODBODY_01 = "%s _$$_t = (%s)$2; ";
 
 	String STATEMENT_PACKER_PACKERMETHODBODY_02 = "$1.packArray(%d); ";
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 43db5c6..f45764f 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -207,11 +207,11 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	static interface TemplateAccessor {
+	public static interface TemplateAccessor {
 		void setTemplates(Template[] templates);
 	}
 
-	static class TemplateAccessorImpl implements TemplateAccessor {
+	protected static class TemplateAccessorImpl implements TemplateAccessor {
 		public Class type;
 
 		public Template[] _$$_templates;
@@ -416,67 +416,6 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	public String getUnpackMethodName(Class c)
-			throws DynamicCodeGenException {
-		if (c.equals(boolean.class) || c.equals(Boolean.class)) {
-			return METHOD_NAME_UNPACKBOOLEAN;
-		} else if (c.equals(byte.class) || c.equals(Byte.class)) {
-			return METHOD_NAME_UNPACKBYTE;
-		} else if (c.equals(short.class) || c.equals(Short.class)) {
-			return METHOD_NAME_UNPACKSHORT;
-		} else if (c.equals(int.class) || c.equals(Integer.class)) {
-			return METHOD_NAME_UNPACKINT;
-		} else if (c.equals(float.class) || c.equals(Float.class)) {
-			return METHOD_NAME_UNPACKFLOAT;
-		} else if (c.equals(long.class) || c.equals(Long.class)) {
-			return METHOD_NAME_UNPACKLONG;
-		} else if (c.equals(double.class) || c.equals(Double.class)) {
-			return METHOD_NAME_UNPACKDOUBLE;
-		} else if (c.equals(String.class)) {
-			return METHOD_NAME_UNPACKSTRING;
-		} else if (c.equals(byte[].class)) {
-			return METHOD_NAME_UNPACKBYTEARRAY;
-		} else if (c.equals(BigInteger.class)) {
-			return METHOD_NAME_UNPACKBIGINTEGER;
-		} else if (List.class.isAssignableFrom(c)) {
-			return METHOD_NAME_UNPACK;
-		} else if (Map.class.isAssignableFrom(c)) {
-			return METHOD_NAME_UNPACK;
-		} else {
-			throw new DynamicCodeGenException("Type error: " + c.getName());
-		}
-	}
-
-	public String getAsMethodName(Class c) {
-		if (c.equals(boolean.class) || c.equals(Boolean.class)) {
-			return METHOD_NAME_ASBOOLEAN;
-		} else if (c.equals(byte.class) || c.equals(Byte.class)) {
-			return METHOD_NAME_ASBYTE;
-		} else if (c.equals(short.class) || c.equals(Short.class)) {
-			return METHOD_NAME_ASSHORT;
-		} else if (c.equals(int.class) || c.equals(Integer.class)) {
-			return METHOD_NAME_ASINT;
-		} else if (c.equals(float.class) || c.equals(Float.class)) {
-			return METHOD_NAME_ASFLOAT;
-		} else if (c.equals(long.class) || c.equals(Long.class)) {
-			return METHOD_NAME_ASLONG;
-		} else if (c.equals(double.class) || c.equals(Double.class)) {
-			return METHOD_NAME_ASDOUBLE;
-		} else if (c.equals(String.class)) {
-			return METHOD_NAME_ASSTRING;
-		} else if (c.equals(byte[].class)) {
-			return METHOD_NAME_ASBYTEARRAY;
-		} else if (c.equals(BigInteger.class)) {
-			return METHOD_NAME_ASBIGINTEGER;
-		} else if (List.class.isAssignableFrom(c)) {
-			return METHOD_NAME_ASLIST;
-		} else if (Map.class.isAssignableFrom(c)) {
-			return METHOD_NAME_ASMAP;
-		} else {
-			throw new DynamicCodeGenException("Type error: " + c.getName());
-		}
-	}
-
 	public static MessagePacker toMessagePacker(Template tmpl) {
 		if (tmpl instanceof BigIntegerTemplate) {
 			return BigIntegerPacker.getInstance();
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
index a483c02..fe0095c 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
@@ -37,8 +37,8 @@ public class DynamicPacker {
 			Constructor cons = packerClass
 					.getDeclaredConstructor(new Class[] { Class.class });
 			Object obj = cons.newInstance(new Object[] { c });
-			((MessagePackerAccessor) obj).setMessagePackers(gen
-					.getMessagePackers(c));
+			MessagePacker[] packers = gen.getMessagePackers(c);
+			((MessagePackerAccessor) obj).setMessagePackers(packers);
 			return (MessagePacker) obj;
 		} catch (InstantiationException e) {
 			throw new DynamicCodeGenException(e.getMessage(), e);

From 645d29684140ccfacbe8aff06931169f3c6f99f5 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sat, 23 Oct 2010 23:13:54 +0900
Subject: [PATCH 0894/1648] java: add test programs for
 org.msgpack.util.codegen.FieldOption.java

---
 .../TestPackConvertWithFieldOption.java       | 553 ++++++++++++++++++
 .../TestPackUnpackWithFieldOption.java        | 509 ++++++++++++++++
 2 files changed, 1062 insertions(+)
 create mode 100644 java/src/test/java/org/msgpack/util/codegen/TestPackConvertWithFieldOption.java
 create mode 100644 java/src/test/java/org/msgpack/util/codegen/TestPackUnpackWithFieldOption.java

diff --git a/java/src/test/java/org/msgpack/util/codegen/TestPackConvertWithFieldOption.java b/java/src/test/java/org/msgpack/util/codegen/TestPackConvertWithFieldOption.java
new file mode 100644
index 0000000..a6110c5
--- /dev/null
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackConvertWithFieldOption.java
@@ -0,0 +1,553 @@
+package org.msgpack.util.codegen;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+import org.msgpack.MessagePackObject;
+import org.msgpack.MessagePacker;
+import org.msgpack.Packer;
+import org.msgpack.Template;
+import org.msgpack.Unpacker;
+import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.packer.OptionalPacker;
+import org.msgpack.template.ListTemplate;
+import org.msgpack.template.MapTemplate;
+import org.msgpack.template.OptionalTemplate;
+import static org.msgpack.Templates.tBigInteger;
+import static org.msgpack.Templates.tBoolean;
+import static org.msgpack.Templates.tByte;
+import static org.msgpack.Templates.tByteArray;
+import static org.msgpack.Templates.tDouble;
+import static org.msgpack.Templates.tFloat;
+import static org.msgpack.Templates.tInteger;
+import static org.msgpack.Templates.tLong;
+import static org.msgpack.Templates.tShort;
+import static org.msgpack.Templates.tString;
+
+import junit.framework.TestCase;
+
+public class TestPackConvertWithFieldOption extends TestCase {
+
+	@Test
+	public void testPrimitiveTypeFieldsClass00() throws Exception {
+		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
+		src.f0 = (byte) 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f6 = false;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", tByte()));
+		opts.add(new FieldOption("f1", tShort()));
+		opts.add(new FieldOption("f2", tInteger()));
+		opts.add(new FieldOption("f3", tLong()));
+		opts.add(new FieldOption("f4", tFloat()));
+		opts.add(new FieldOption("f5", tDouble()));
+		opts.add(new FieldOption("f6", tBoolean()));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(
+				PrimitiveTypeFieldsClass.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class, opts);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testPrimitiveTypeFieldsClass01() throws Exception {
+		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(tByte())));
+		opts.add(new FieldOption("f1", new OptionalTemplate(tShort())));
+		opts.add(new FieldOption("f2", new OptionalTemplate(tInteger())));
+		opts.add(new FieldOption("f3", new OptionalTemplate(tLong())));
+		opts.add(new FieldOption("f4", new OptionalTemplate(tFloat())));
+		opts.add(new FieldOption("f5", new OptionalTemplate(tDouble())));
+		opts.add(new FieldOption("f6", new OptionalTemplate(tBoolean())));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(
+				PrimitiveTypeFieldsClass.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class, opts);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testPrimitiveTypeFieldsClass02() throws Exception {
+		PrimitiveTypeFieldsClass src = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", tByte()));
+		opts.add(new FieldOption("f1", tShort()));
+		opts.add(new FieldOption("f2", tInteger()));
+		opts.add(new FieldOption("f3", tLong()));
+		opts.add(new FieldOption("f4", tFloat()));
+		opts.add(new FieldOption("f5", tDouble()));
+		opts.add(new FieldOption("f6", tBoolean()));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker.create(
+				PrimitiveTypeFieldsClass.class, opts));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate.create(PrimitiveTypeFieldsClass.class, opts));
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class PrimitiveTypeFieldsClass {
+		public byte f0;
+		public short f1;
+		public int f2;
+		public long f3;
+		public float f4;
+		public double f5;
+		public boolean f6;
+
+		public PrimitiveTypeFieldsClass() {
+		}
+	}
+
+	@Test
+	public void testGeneralReferenceTypeFieldsClass00() throws Exception {
+		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
+		src.f0 = 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = (long) 3;
+		src.f4 = (float) 4;
+		src.f5 = (double) 5;
+		src.f6 = false;
+		src.f7 = new BigInteger("7");
+		src.f8 = "8";
+		src.f9 = new byte[] { 0x01, 0x02 };
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", tByte()));
+		opts.add(new FieldOption("f1", tShort()));
+		opts.add(new FieldOption("f2", tInteger()));
+		opts.add(new FieldOption("f3", tLong()));
+		opts.add(new FieldOption("f4", tFloat()));
+		opts.add(new FieldOption("f5", tDouble()));
+		opts.add(new FieldOption("f6", tBoolean()));
+		opts.add(new FieldOption("f7", tBigInteger()));
+		opts.add(new FieldOption("f8", tString()));
+		opts.add(new FieldOption("f9", tByteArray()));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(GeneralReferenceTypeFieldsClass.class, opts);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9[0], dst.f9[0]);
+		assertEquals(src.f9[1], dst.f9[1]);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testGeneralReferenceTypeFieldsClass01() throws Exception {
+		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
+		src.f0 = null;
+		src.f1 = null;
+		src.f2 = null;
+		src.f3 = null;
+		src.f4 = null;
+		src.f5 = null;
+		src.f6 = null;
+		src.f7 = null;
+		src.f8 = null;
+		src.f9 = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(tByte())));
+		opts.add(new FieldOption("f1", new OptionalTemplate(tShort())));
+		opts.add(new FieldOption("f2", new OptionalTemplate(tInteger())));
+		opts.add(new FieldOption("f3", new OptionalTemplate(tLong())));
+		opts.add(new FieldOption("f4", new OptionalTemplate(tFloat())));
+		opts.add(new FieldOption("f5", new OptionalTemplate(tDouble())));
+		opts.add(new FieldOption("f6", new OptionalTemplate(tBoolean())));
+		opts.add(new FieldOption("f7", new OptionalTemplate(tBigInteger())));
+		opts.add(new FieldOption("f8", new OptionalTemplate(tString())));
+		opts.add(new FieldOption("f9", new OptionalTemplate(tByteArray())));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(GeneralReferenceTypeFieldsClass.class, opts);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl.convert(mpo);
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9, dst.f9);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testGeneralReferenceTypeFieldsClass02()
+			throws Exception {
+		GeneralReferenceTypeFieldsClass src = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(tByte())));
+		opts.add(new FieldOption("f1", new OptionalTemplate(tShort())));
+		opts.add(new FieldOption("f2", new OptionalTemplate(tInteger())));
+		opts.add(new FieldOption("f3", new OptionalTemplate(tLong())));
+		opts.add(new FieldOption("f4", new OptionalTemplate(tFloat())));
+		opts.add(new FieldOption("f5", new OptionalTemplate(tDouble())));
+		opts.add(new FieldOption("f6", new OptionalTemplate(tBoolean())));
+		opts.add(new FieldOption("f7", new OptionalTemplate(tBigInteger())));
+		opts.add(new FieldOption("f8", new OptionalTemplate(tString())));
+		opts.add(new FieldOption("f9", new OptionalTemplate(tByteArray())));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class, opts));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(GeneralReferenceTypeFieldsClass.class, opts));
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class GeneralReferenceTypeFieldsClass {
+		public Byte f0;
+		public Short f1;
+		public Integer f2;
+		public Long f3;
+		public Float f4;
+		public Double f5;
+		public Boolean f6;
+		public BigInteger f7;
+		public String f8;
+		public byte[] f9;
+
+		public GeneralReferenceTypeFieldsClass() {
+		}
+	}
+
+	@Test
+	public void testListTypes00() throws Exception {
+		SampleListTypes src = new SampleListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = new ArrayList();
+		src.f1.add(1);
+		src.f1.add(2);
+		src.f1.add(3);
+		src.f2 = new ArrayList();
+		src.f2.add("e1");
+		src.f2.add("e2");
+		src.f2.add("e3");
+		src.f3 = new ArrayList>();
+		src.f3.add(src.f2);
+		src.f4 = new ArrayList();
+		SampleListNestedType slnt = new SampleListNestedType();
+		slnt.f0 = new byte[] { 0x01, 0x02 };
+		slnt.f1 = "muga";
+		src.f4.add(slnt);
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new ListTemplate(tInteger())));
+		opts.add(new FieldOption("f1", new ListTemplate(tInteger())));
+		opts.add(new FieldOption("f2", new ListTemplate(tString())));
+		opts.add(new FieldOption("f3", new ListTemplate(new ListTemplate(tString()))));
+		opts.add(new FieldOption("f4", new ListTemplate(DynamicTemplate.create(SampleListNestedType.class))));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleListTypes.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleListTypes.class, opts);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleListTypes dst = (SampleListTypes) tmpl.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		for (int i = 0; i < src.f1.size(); ++i) {
+			assertEquals(src.f1.get(i), dst.f1.get(i));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		for (int i = 0; i < src.f2.size(); ++i) {
+			assertEquals(src.f2.get(i), dst.f2.get(i));
+		}
+		assertEquals(src.f3.size(), dst.f3.size());
+		for (int i = 0; i < src.f3.size(); ++i) {
+			List srclist = src.f3.get(i);
+			List dstlist = dst.f3.get(i);
+			assertEquals(srclist.size(), dstlist.size());
+			for (int j = 0; j < srclist.size(); ++j) {
+				assertEquals(srclist.get(j), dstlist.get(j));
+			}
+		}
+		assertEquals(src.f4.size(), dst.f4.size());
+		for (int i = 0; i < src.f4.size(); ++i) {
+			SampleListNestedType s = src.f4.get(i);
+			SampleListNestedType d = dst.f4.get(i);
+			assertEquals(s.f0[0], d.f0[0]);
+			assertEquals(s.f0[1], d.f0[1]);
+			assertEquals(s.f1, d.f1);
+		}
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testListTypes01() throws Exception {
+		SampleListTypes src = new SampleListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = null;
+		src.f2 = new ArrayList();
+		src.f3 = new ArrayList>();
+		src.f4 = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(new ListTemplate(tInteger()))));
+		opts.add(new FieldOption("f1", new OptionalTemplate(new ListTemplate(tInteger()))));
+		opts.add(new FieldOption("f2", new OptionalTemplate(new ListTemplate(tString()))));
+		opts.add(new FieldOption("f3", new OptionalTemplate(new ListTemplate(new ListTemplate(tString())))));
+		opts.add(new FieldOption("f4", new OptionalTemplate(new ListTemplate(DynamicTemplate.create(SampleListNestedType.class)))));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleListTypes.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleListTypes.class, opts);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleListTypes dst = (SampleListTypes) tmpl.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertEquals(src.f3.size(), dst.f3.size());
+		assertEquals(src.f4, dst.f4);
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testListTypes02() throws Exception {
+		SampleListTypes src = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(new ListTemplate(tInteger()))));
+		opts.add(new FieldOption("f1", new OptionalTemplate(new ListTemplate(tInteger()))));
+		opts.add(new FieldOption("f2", new OptionalTemplate(new ListTemplate(tString()))));
+		opts.add(new FieldOption("f3", new OptionalTemplate(new ListTemplate(new ListTemplate(tString())))));
+		opts.add(new FieldOption("f4", new OptionalTemplate(new ListTemplate(DynamicTemplate.create(SampleListNestedType.class)))));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleListTypes.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleListTypes.class, opts));
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleListTypes dst = (SampleListTypes) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class SampleListTypes {
+		public List f0;
+		public List f1;
+		public List f2;
+		public List> f3;
+		public List f4;
+
+		public SampleListTypes() {
+		}
+	}
+
+	@MessagePackMessage
+	public static class SampleListNestedType {
+		public byte[] f0;
+		public String f1;
+
+		public SampleListNestedType() {
+		}
+	}
+
+	@Test
+	public void testMapTypes00() throws Exception {
+		SampleMapTypes src = new SampleMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = new HashMap();
+		src.f1.put(1, 1);
+		src.f1.put(2, 2);
+		src.f1.put(3, 3);
+		src.f2 = new HashMap();
+		src.f2.put("k1", 1);
+		src.f2.put("k2", 2);
+		src.f2.put("k3", 3);
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new MapTemplate(tInteger(), tInteger())));
+		opts.add(new FieldOption("f1", new MapTemplate(tInteger(), tInteger())));
+		opts.add(new FieldOption("f2", new MapTemplate(tString(), tInteger())));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleMapTypes.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleMapTypes.class, opts);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleMapTypes dst = (SampleMapTypes) tmpl.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		Iterator srcf1 = src.f1.keySet().iterator();
+		Iterator dstf1 = dst.f1.keySet().iterator();
+		while (srcf1.hasNext()) {
+			Integer s1 = srcf1.next();
+			Integer d1 = dstf1.next();
+			assertEquals(s1, d1);
+			assertEquals(src.f1.get(s1), dst.f1.get(d1));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		Iterator srcf2 = src.f2.keySet().iterator();
+		Iterator dstf2 = dst.f2.keySet().iterator();
+		while (srcf2.hasNext()) {
+			String s2 = srcf2.next();
+			String d2 = dstf2.next();
+			assertEquals(s2, d2);
+			assertEquals(src.f2.get(s2), dst.f2.get(d2));
+		}
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testMapTypes01() throws Exception {
+		SampleMapTypes src = new SampleMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = null;
+		src.f2 = new HashMap();
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(new MapTemplate(tInteger(), tInteger()))));
+		opts.add(new FieldOption("f1", new OptionalTemplate(new MapTemplate(tInteger(), tInteger()))));
+		opts.add(new FieldOption("f2", new OptionalTemplate(new MapTemplate(tString(), tInteger()))));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleMapTypes.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleMapTypes.class, opts);
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleMapTypes dst = (SampleMapTypes) tmpl.convert(mpo);
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertFalse(it.hasNext());
+	}
+
+	@Test
+	public void testMapTypes02() throws Exception {
+		SampleMapTypes src = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(new MapTemplate(tInteger(), tInteger()))));
+		opts.add(new FieldOption("f1", new OptionalTemplate(new MapTemplate(tInteger(), tInteger()))));
+		opts.add(new FieldOption("f2", new OptionalTemplate(new MapTemplate(tString(), tInteger()))));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleMapTypes.class, opts));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate.create(SampleMapTypes.class, opts));
+		Unpacker pac = new Unpacker(in);
+		Iterator it = pac.iterator();
+		assertTrue(it.hasNext());
+		MessagePackObject mpo = it.next();
+		SampleMapTypes dst = (SampleMapTypes) tmpl.convert(mpo);
+		assertEquals(src, dst);
+		assertFalse(it.hasNext());
+	}
+
+	public static class SampleMapTypes {
+		public Map f0;
+		public Map f1;
+		public Map f2;
+
+		public SampleMapTypes() {
+		}
+	}
+}
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestPackUnpackWithFieldOption.java b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpackWithFieldOption.java
new file mode 100644
index 0000000..62ad7ed
--- /dev/null
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpackWithFieldOption.java
@@ -0,0 +1,509 @@
+package org.msgpack.util.codegen;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Test;
+import org.msgpack.MessagePacker;
+import org.msgpack.Packer;
+import org.msgpack.Template;
+import org.msgpack.Unpacker;
+import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.packer.OptionalPacker;
+import org.msgpack.template.ListTemplate;
+import org.msgpack.template.MapTemplate;
+import org.msgpack.template.OptionalTemplate;
+import static org.msgpack.Templates.tBigInteger;
+import static org.msgpack.Templates.tBoolean;
+import static org.msgpack.Templates.tByte;
+import static org.msgpack.Templates.tByteArray;
+import static org.msgpack.Templates.tDouble;
+import static org.msgpack.Templates.tFloat;
+import static org.msgpack.Templates.tInteger;
+import static org.msgpack.Templates.tLong;
+import static org.msgpack.Templates.tShort;
+import static org.msgpack.Templates.tString;
+
+import junit.framework.TestCase;
+
+public class TestPackUnpackWithFieldOption extends TestCase {
+
+	@Test
+	public void testPrimitiveTypeFieldsClass00() throws Exception {
+		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
+		src.f0 = (byte) 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = 3;
+		src.f4 = 4;
+		src.f5 = 5;
+		src.f6 = false;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", tByte()));
+		opts.add(new FieldOption("f1", tShort()));
+		opts.add(new FieldOption("f2", tInteger()));
+		opts.add(new FieldOption("f3", tLong()));
+		opts.add(new FieldOption("f4", tFloat()));
+		opts.add(new FieldOption("f5", tDouble()));
+		opts.add(new FieldOption("f6", tBoolean()));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(
+				PrimitiveTypeFieldsClass.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class,
+				opts);
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+	}
+
+	@Test
+	public void testPrimitiveTypeFieldsClass01() throws Exception {
+		PrimitiveTypeFieldsClass src = new PrimitiveTypeFieldsClass();
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(tByte())));
+		opts.add(new FieldOption("f1", new OptionalTemplate(tShort())));
+		opts.add(new FieldOption("f2", new OptionalTemplate(tInteger())));
+		opts.add(new FieldOption("f3", new OptionalTemplate(tLong())));
+		opts.add(new FieldOption("f4", new OptionalTemplate(tFloat())));
+		opts.add(new FieldOption("f5", new OptionalTemplate(tDouble())));
+		opts.add(new FieldOption("f6", new OptionalTemplate(tBoolean())));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker.create(
+				PrimitiveTypeFieldsClass.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(PrimitiveTypeFieldsClass.class,
+				opts);
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+	}
+
+	@Test
+	public void testPrimitiveTypeFieldsClass02() throws Exception {
+		PrimitiveTypeFieldsClass src = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", tByte()));
+		opts.add(new FieldOption("f1", tShort()));
+		opts.add(new FieldOption("f2", tInteger()));
+		opts.add(new FieldOption("f3", tLong()));
+		opts.add(new FieldOption("f4", tFloat()));
+		opts.add(new FieldOption("f5", tDouble()));
+		opts.add(new FieldOption("f6", tBoolean()));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker.create(
+				PrimitiveTypeFieldsClass.class, opts));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate.create(
+				PrimitiveTypeFieldsClass.class, opts));
+		PrimitiveTypeFieldsClass dst = (PrimitiveTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class PrimitiveTypeFieldsClass {
+		public byte f0;
+		public short f1;
+		public int f2;
+		public long f3;
+		public float f4;
+		public double f5;
+		public boolean f6;
+
+		public PrimitiveTypeFieldsClass() {
+		}
+	}
+
+	@Test
+	public void testGeneralReferenceTypeFieldsClass00() throws Exception {
+		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
+		src.f0 = 0;
+		src.f1 = 1;
+		src.f2 = 2;
+		src.f3 = (long) 3;
+		src.f4 = (float) 4;
+		src.f5 = (double) 5;
+		src.f6 = false;
+		src.f7 = new BigInteger("7");
+		src.f8 = "8";
+		src.f9 = new byte[] { 0x01, 0x02 };
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", tByte()));
+		opts.add(new FieldOption("f1", tShort()));
+		opts.add(new FieldOption("f2", tInteger()));
+		opts.add(new FieldOption("f3", tLong()));
+		opts.add(new FieldOption("f4", tFloat()));
+		opts.add(new FieldOption("f5", tDouble()));
+		opts.add(new FieldOption("f6", tBoolean()));
+		opts.add(new FieldOption("f7", tBigInteger()));
+		opts.add(new FieldOption("f8", tString()));
+		opts.add(new FieldOption("f9", tByteArray()));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(GeneralReferenceTypeFieldsClass.class, opts);
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9[0], dst.f9[0]);
+		assertEquals(src.f9[1], dst.f9[1]);
+	}
+
+	@Test
+	public void testGeneralReferenceTypeFieldsClass01() throws Exception {
+		GeneralReferenceTypeFieldsClass src = new GeneralReferenceTypeFieldsClass();
+		src.f0 = null;
+		src.f1 = null;
+		src.f2 = null;
+		src.f3 = null;
+		src.f4 = null;
+		src.f5 = null;
+		src.f6 = null;
+		src.f7 = null;
+		src.f8 = null;
+		src.f9 = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(tByte())));
+		opts.add(new FieldOption("f1", new OptionalTemplate(tShort())));
+		opts.add(new FieldOption("f2", new OptionalTemplate(tInteger())));
+		opts.add(new FieldOption("f3", new OptionalTemplate(tLong())));
+		opts.add(new FieldOption("f4", new OptionalTemplate(tFloat())));
+		opts.add(new FieldOption("f5", new OptionalTemplate(tDouble())));
+		opts.add(new FieldOption("f6", new OptionalTemplate(tBoolean())));
+		opts.add(new FieldOption("f7", new OptionalTemplate(tBigInteger())));
+		opts.add(new FieldOption("f8", new OptionalTemplate(tString())));
+		opts.add(new FieldOption("f9", new OptionalTemplate(tByteArray())));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate
+				.create(GeneralReferenceTypeFieldsClass.class, opts);
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0, dst.f0);
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2, dst.f2);
+		assertEquals(src.f3, dst.f3);
+		assertEquals(src.f4, dst.f4);
+		assertEquals(src.f5, dst.f5);
+		assertEquals(src.f6, dst.f6);
+		assertEquals(src.f7, dst.f7);
+		assertEquals(src.f8, dst.f8);
+		assertEquals(src.f9, dst.f9);
+	}
+
+	@Test
+	public void testGeneralReferenceTypeFieldsClass02()
+			throws Exception {
+		GeneralReferenceTypeFieldsClass src = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(tByte())));
+		opts.add(new FieldOption("f1", new OptionalTemplate(tShort())));
+		opts.add(new FieldOption("f2", new OptionalTemplate(tInteger())));
+		opts.add(new FieldOption("f3", new OptionalTemplate(tLong())));
+		opts.add(new FieldOption("f4", new OptionalTemplate(tFloat())));
+		opts.add(new FieldOption("f5", new OptionalTemplate(tDouble())));
+		opts.add(new FieldOption("f6", new OptionalTemplate(tBoolean())));
+		opts.add(new FieldOption("f7", new OptionalTemplate(tBigInteger())));
+		opts.add(new FieldOption("f8", new OptionalTemplate(tString())));
+		opts.add(new FieldOption("f9", new OptionalTemplate(tByteArray())));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(GeneralReferenceTypeFieldsClass.class, opts));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(GeneralReferenceTypeFieldsClass.class, opts));
+		GeneralReferenceTypeFieldsClass dst = (GeneralReferenceTypeFieldsClass) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class GeneralReferenceTypeFieldsClass {
+		public Byte f0;
+		public Short f1;
+		public Integer f2;
+		public Long f3;
+		public Float f4;
+		public Double f5;
+		public Boolean f6;
+		public BigInteger f7;
+		public String f8;
+		public byte[] f9;
+
+		public GeneralReferenceTypeFieldsClass() {
+		}
+	}
+
+	@Test
+	public void testListTypes00() throws Exception {
+		SampleListTypes src = new SampleListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = new ArrayList();
+		src.f1.add(1);
+		src.f1.add(2);
+		src.f1.add(3);
+		src.f2 = new ArrayList();
+		src.f2.add("e1");
+		src.f2.add("e2");
+		src.f2.add("e3");
+		src.f3 = new ArrayList>();
+		src.f3.add(src.f2);
+		src.f4 = new ArrayList();
+		SampleListNestedType slnt = new SampleListNestedType();
+		slnt.f0 = new byte[] { 0x01, 0x02 };
+		slnt.f1 = "muga";
+		src.f4.add(slnt);
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new ListTemplate(tInteger())));
+		opts.add(new FieldOption("f1", new ListTemplate(tInteger())));
+		opts.add(new FieldOption("f2", new ListTemplate(tString())));
+		opts.add(new FieldOption("f3", new ListTemplate(new ListTemplate(tString()))));
+		opts.add(new FieldOption("f4", new ListTemplate(DynamicTemplate.create(SampleListNestedType.class))));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleListTypes.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleListTypes.class, opts);
+		SampleListTypes dst = (SampleListTypes) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		for (int i = 0; i < src.f1.size(); ++i) {
+			assertEquals(src.f1.get(i), dst.f1.get(i));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		for (int i = 0; i < src.f2.size(); ++i) {
+			assertEquals(src.f2.get(i), dst.f2.get(i));
+		}
+		assertEquals(src.f3.size(), dst.f3.size());
+		for (int i = 0; i < src.f3.size(); ++i) {
+			List srclist = src.f3.get(i);
+			List dstlist = dst.f3.get(i);
+			assertEquals(srclist.size(), dstlist.size());
+			for (int j = 0; j < srclist.size(); ++j) {
+				assertEquals(srclist.get(j), dstlist.get(j));
+			}
+		}
+		assertEquals(src.f4.size(), dst.f4.size());
+		for (int i = 0; i < src.f4.size(); ++i) {
+			SampleListNestedType s = src.f4.get(i);
+			SampleListNestedType d = dst.f4.get(i);
+			assertEquals(s.f0[0], d.f0[0]);
+			assertEquals(s.f0[1], d.f0[1]);
+			assertEquals(s.f1, d.f1);
+		}
+	}
+
+	@Test
+	public void testListTypes01() throws Exception {
+		SampleListTypes src = new SampleListTypes();
+		src.f0 = new ArrayList();
+		src.f1 = null;
+		src.f2 = new ArrayList();
+		src.f3 = new ArrayList>();
+		src.f4 = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(new ListTemplate(tInteger()))));
+		opts.add(new FieldOption("f1", new OptionalTemplate(new ListTemplate(tInteger()))));
+		opts.add(new FieldOption("f2", new OptionalTemplate(new ListTemplate(tString()))));
+		opts.add(new FieldOption("f3", new OptionalTemplate(new ListTemplate(new ListTemplate(tString())))));
+		opts.add(new FieldOption("f4", new OptionalTemplate(new ListTemplate(DynamicTemplate.create(SampleListNestedType.class)))));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleListTypes.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleListTypes.class, opts);
+		SampleListTypes dst = (SampleListTypes) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+		assertEquals(src.f3.size(), dst.f3.size());
+		assertEquals(src.f4, dst.f4);
+	}
+
+	@Test
+	public void testListTypes02() throws Exception {
+		SampleListTypes src = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(new ListTemplate(tInteger()))));
+		opts.add(new FieldOption("f1", new OptionalTemplate(new ListTemplate(tInteger()))));
+		opts.add(new FieldOption("f2", new OptionalTemplate(new ListTemplate(tString()))));
+		opts.add(new FieldOption("f3", new OptionalTemplate(new ListTemplate(new ListTemplate(tString())))));
+		opts.add(new FieldOption("f4", new OptionalTemplate(new ListTemplate(DynamicTemplate.create(SampleListNestedType.class)))));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleListTypes.class));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleListTypes.class));
+		SampleListTypes dst = (SampleListTypes) tmpl.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class SampleListTypes {
+		public List f0;
+		public List f1;
+		public List f2;
+		public List> f3;
+		public List f4;
+
+		public SampleListTypes() {
+		}
+	}
+
+	@MessagePackMessage
+	public static class SampleListNestedType {
+		public byte[] f0;
+		public String f1;
+
+		public SampleListNestedType() {
+		}
+	}
+
+	@Test
+	public void testMapTypes00() throws Exception {
+		SampleMapTypes src = new SampleMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = new HashMap();
+		src.f1.put(1, 1);
+		src.f1.put(2, 2);
+		src.f1.put(3, 3);
+		src.f2 = new HashMap();
+		src.f2.put("k1", 1);
+		src.f2.put("k2", 2);
+		src.f2.put("k3", 3);
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new MapTemplate(tInteger(), tInteger())));
+		opts.add(new FieldOption("f1", new MapTemplate(tInteger(), tInteger())));
+		opts.add(new FieldOption("f2", new MapTemplate(tString(), tInteger())));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleMapTypes.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleMapTypes.class, opts);
+		SampleMapTypes dst = (SampleMapTypes) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1.size(), dst.f1.size());
+		Iterator srcf1 = src.f1.keySet().iterator();
+		Iterator dstf1 = dst.f1.keySet().iterator();
+		while (srcf1.hasNext()) {
+			Integer s1 = srcf1.next();
+			Integer d1 = dstf1.next();
+			assertEquals(s1, d1);
+			assertEquals(src.f1.get(s1), dst.f1.get(d1));
+		}
+		assertEquals(src.f2.size(), dst.f2.size());
+		Iterator srcf2 = src.f2.keySet().iterator();
+		Iterator dstf2 = dst.f2.keySet().iterator();
+		while (srcf2.hasNext()) {
+			String s2 = srcf2.next();
+			String d2 = dstf2.next();
+			assertEquals(s2, d2);
+			assertEquals(src.f2.get(s2), dst.f2.get(d2));
+		}
+	}
+
+	@Test
+	public void testMapTypes01() throws Exception {
+		SampleMapTypes src = new SampleMapTypes();
+		src.f0 = new HashMap();
+		src.f1 = null;
+		src.f2 = new HashMap();
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(new MapTemplate(tInteger(), tInteger()))));
+		opts.add(new FieldOption("f1", new OptionalTemplate(new MapTemplate(tInteger(), tInteger()))));
+		opts.add(new FieldOption("f2", new OptionalTemplate(new MapTemplate(tString(), tInteger()))));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = DynamicPacker
+				.create(SampleMapTypes.class, opts);
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = DynamicTemplate.create(SampleMapTypes.class, opts);
+		SampleMapTypes dst = (SampleMapTypes) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src.f0.size(), dst.f0.size());
+		assertEquals(src.f1, dst.f1);
+		assertEquals(src.f2.size(), dst.f2.size());
+	}
+
+	@Test
+	public void testMapTypes02() throws Exception {
+		SampleMapTypes src = null;
+
+		List opts = new ArrayList();
+		opts.add(new FieldOption("f0", new OptionalTemplate(new MapTemplate(tInteger(), tInteger()))));
+		opts.add(new FieldOption("f1", new OptionalTemplate(new MapTemplate(tInteger(), tInteger()))));
+		opts.add(new FieldOption("f2", new OptionalTemplate(new MapTemplate(tString(), tInteger()))));
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		MessagePacker packer = new OptionalPacker(DynamicPacker
+				.create(SampleMapTypes.class, opts));
+		packer.pack(new Packer(out), src);
+		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
+		Template tmpl = new OptionalTemplate(DynamicTemplate
+				.create(SampleMapTypes.class, opts));
+		SampleMapTypes dst = (SampleMapTypes) tmpl
+				.unpack(new Unpacker(in));
+		assertEquals(src, dst);
+	}
+
+	public static class SampleMapTypes {
+		public Map f0;
+		public Map f1;
+		public Map f2;
+
+		public SampleMapTypes() {
+		}
+	}
+}

From 945d279f28a61ebc2193b7d952affb17c996f0bb Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sun, 24 Oct 2010 00:40:58 +0900
Subject: [PATCH 0895/1648] java: fix a bug within a code of
 DynamicCodeGenBase.java

---
 .../java/org/msgpack/util/codegen/DynamicCodeGenBase.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index f45764f..dee6589 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -186,11 +186,11 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	static interface MessagePackerAccessor {
+	public static interface MessagePackerAccessor {
 		void setMessagePackers(MessagePacker[] packers);
 	}
 
-	static class MessagePackerAccessorImpl implements MessagePackerAccessor {
+	protected static class MessagePackerAccessorImpl implements MessagePackerAccessor {
 		public Class type;
 
 		public MessagePacker[] _$$_packers;

From 77698cd924b1e8fc5f3033e4e04d7b1da95563f6 Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sun, 24 Oct 2010 00:59:14 +0900
Subject: [PATCH 0896/1648] java: change modifiers specified by several methods

---
 .../util/codegen/DynamicCodeGenBase.java       | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index dee6589..3222b47 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -236,7 +236,7 @@ public class DynamicCodeGenBase implements Constants {
 
 	protected ClassPool pool;
 
-	public DynamicCodeGenBase() {
+	protected DynamicCodeGenBase() {
 		pool = ClassPool.getDefault();
 	}
 
@@ -396,7 +396,7 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	public String getPrimTypeValueMethodName(Class type) {
+	public static String getPrimTypeValueMethodName(Class type) {
 		if (type.equals(boolean.class)) {
 			return METHOD_NAME_BOOLEANVALUE;
 		} else if (type.equals(byte.class)) {
@@ -473,7 +473,7 @@ public class DynamicCodeGenBase implements Constants {
 		throw e;
 	}
 
-	public MessagePacker createMessagePacker(Type t) {
+	public static MessagePacker createMessagePacker(Type t) {
 		if (t.getClass().equals(Class.class)) {
 			Class c = (Class) t;
 			if (c.equals(boolean.class) || c.equals(Boolean.class)) {
@@ -549,7 +549,7 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	public Template createTemplate(Type t) {
+	public static Template createTemplate(Type t) {
 		if (t.getClass().equals(Class.class)) {
 			Class c = (Class) t;
 			if (c.equals(boolean.class) || c.equals(Boolean.class)) {
@@ -626,7 +626,7 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	protected int getArrayDim(Class type) {
+	static int getArrayDim(Class type) {
 		if (type.isArray()) {
 			return 1 + getArrayDim(type.getComponentType());
 		} else {
@@ -634,7 +634,7 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	protected Class getArrayBaseType(Class type) {
+	static Class getArrayBaseType(Class type) {
 		if (type.isArray()) {
 			return getArrayBaseType(type.getComponentType());
 		} else {
@@ -642,7 +642,7 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	protected String arrayTypeToString(Class type) {
+	static String arrayTypeToString(Class type) {
 		StringBuilder sb = new StringBuilder();
 		int dim = getArrayDim(type);
 		Class t = getArrayBaseType(type);
@@ -653,7 +653,7 @@ public class DynamicCodeGenBase implements Constants {
 		return sb.toString();
 	}
 
-	protected String classToString(Class type) {
+	protected static String classToString(Class type) {
 		if (type.isArray()) {
 			return arrayTypeToString(type);
 		} else {
@@ -691,7 +691,7 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	protected Class createClass(CtClass newCtClass)
+	protected static Class createClass(CtClass newCtClass)
 			throws CannotCompileException {
 		return newCtClass.toClass(null, null);
 	}

From 19fd4e755cd88fc0b108e2c860ebf88394d98f03 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 24 Oct 2010 18:45:58 +0900
Subject: [PATCH 0897/1648] java: removes ReflectionPacker and
 ReflectionTemplate (replaced by DynamicCodeGen)

---
 .../main/java/org/msgpack/ReflectionBase.java | 26 -------
 .../java/org/msgpack/ReflectionPacker.java    | 49 ------------
 .../java/org/msgpack/ReflectionTemplate.java  | 74 -------------------
 .../msgpack/TestReflectionPackerTemplate.java | 70 ------------------
 4 files changed, 219 deletions(-)
 delete mode 100644 java/src/main/java/org/msgpack/ReflectionBase.java
 delete mode 100644 java/src/main/java/org/msgpack/ReflectionPacker.java
 delete mode 100644 java/src/main/java/org/msgpack/ReflectionTemplate.java
 delete mode 100644 java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java

diff --git a/java/src/main/java/org/msgpack/ReflectionBase.java b/java/src/main/java/org/msgpack/ReflectionBase.java
deleted file mode 100644
index 66ec12a..0000000
--- a/java/src/main/java/org/msgpack/ReflectionBase.java
+++ /dev/null
@@ -1,26 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack;
-
-import java.io.IOException;
-import java.lang.reflect.*;
-
-// FIXME mock-up
-abstract class ReflectionBase {
-}
-
diff --git a/java/src/main/java/org/msgpack/ReflectionPacker.java b/java/src/main/java/org/msgpack/ReflectionPacker.java
deleted file mode 100644
index 72406aa..0000000
--- a/java/src/main/java/org/msgpack/ReflectionPacker.java
+++ /dev/null
@@ -1,49 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack;
-
-import java.io.IOException;
-import java.lang.reflect.*;
-
-// FIXME mock-up
-public class ReflectionPacker extends ReflectionBase implements MessagePacker {
-	private Class klass;
-
-	private ReflectionPacker(Class klass) {
-		this.klass = klass;
-	}
-
-	static public ReflectionPacker create(Class klass) {
-		// FIXME code generation: generates subclass of ReflectionPacker
-		//       returned instance will be cached by Packer into CustomPacker
-		return new ReflectionPacker(klass);
-	}
-
-	public void pack(Packer pk, Object target) throws IOException {
-		Field[] fields = klass.getDeclaredFields();
-		pk.packArray(fields.length);
-		try {
-			for(int i=0; i < fields.length; i++) {
-				pk.pack(fields[i].get(target));
-			}
-		} catch(IllegalAccessException e) {
-			throw new MessageTypeException(e.getMessage());  // FIXME
-		}
-	}
-}
-
diff --git a/java/src/main/java/org/msgpack/ReflectionTemplate.java b/java/src/main/java/org/msgpack/ReflectionTemplate.java
deleted file mode 100644
index 5b49078..0000000
--- a/java/src/main/java/org/msgpack/ReflectionTemplate.java
+++ /dev/null
@@ -1,74 +0,0 @@
-//
-// MessagePack for Java
-//
-// Copyright (C) 2009-2010 FURUHASHI Sadayuki
-//
-//    Licensed under the Apache License, Version 2.0 (the "License");
-//    you may not use this file except in compliance with the License.
-//    You may obtain a copy of the License at
-//
-//        http://www.apache.org/licenses/LICENSE-2.0
-//
-//    Unless required by applicable law or agreed to in writing, software
-//    distributed under the License is distributed on an "AS IS" BASIS,
-//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//    See the License for the specific language governing permissions and
-//    limitations under the License.
-//
-package org.msgpack;
-
-import java.io.IOException;
-import java.lang.reflect.*;
-import org.msgpack.template.ClassTemplate;
-
-// FIXME mock-up
-public class ReflectionTemplate extends ReflectionBase implements Template {
-	private Class klass;
-
-	private ReflectionTemplate(Class klass) {
-		this.klass = klass;
-	}
-
-	static public ReflectionTemplate create(Class klass) {
-		// FIXME code generation: generates subclass of ReflectionPacker
-		//       returned instance will be cached by ClassTemplate into CustomUnpacker/CustomConverter
-		return new ReflectionTemplate(klass);
-	}
-
-	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
-		// FIXME optimize it
-		return convert(pac.unpackObject());
-	}
-
-	public Object convert(MessagePackObject from) throws MessageTypeException {
-		Object obj;
-		try {
-			obj = klass.newInstance();
-		} catch (IllegalAccessException e) {
-			throw new MessageTypeException(e.getMessage());  // FIXME
-		} catch (InstantiationException e) {
-			throw new MessageTypeException(e.getMessage());  // FIXME
-		}
-
-		// FIXME check Requred/Optional
-
-		Field[] fields = klass.getDeclaredFields();
-		MessagePackObject[] array = from.asArray();
-		if(fields.length < array.length) {
-			throw new MessageTypeException();
-		}
-
-		try {
-			for(int i=0; i < fields.length; i++) {
-				// FIXME generics getDeclaringClass
-				Object value = new ClassTemplate(fields[i].getType()).convert(array[i]);
-				fields[i].set(obj, value);
-			}
-		} catch(IllegalAccessException e) {
-			throw new MessageTypeException(e.getMessage());  // FIXME
-		}
-
-		return obj;
-	}
-}
-
diff --git a/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java b/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
deleted file mode 100644
index 1f0016d..0000000
--- a/java/src/test/java/org/msgpack/TestReflectionPackerTemplate.java
+++ /dev/null
@@ -1,70 +0,0 @@
-package org.msgpack;
-
-import static org.msgpack.Templates.*;
-
-import java.util.*;
-import java.io.*;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-public class TestReflectionPackerTemplate {
-
-	public static class StringFieldClass {
-		public String s1;
-		public String s2;
-		public StringFieldClass() { }
-	}
-
-	@Test
-	public void testPackConvert() throws Exception {
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-
-		MessagePacker packer = ReflectionPacker.create(StringFieldClass.class);
-
-		StringFieldClass src = new StringFieldClass();
-
-		src.s1 = "kumofs";
-		src.s2 = "frsyuki";
-
-		packer.pack(new Packer(out), src);
-
-		Template tmpl = ReflectionTemplate.create(StringFieldClass.class);
-
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-
-		Object obj = tmpl.unpack(new Unpacker(in));
-		assertEquals(obj.getClass(), StringFieldClass.class);
-
-		StringFieldClass dst = (StringFieldClass)obj;
-		assertEquals(src.s1, dst.s1);
-		assertEquals(src.s2, dst.s2);
-	}
-
-	@Test
-	public void testPackConvert02() throws Exception {
-		ByteArrayOutputStream out = new ByteArrayOutputStream();
-
-		CustomPacker.register(StringFieldClass.class, ReflectionPacker.create(StringFieldClass.class));
-				
-
-		StringFieldClass src = new StringFieldClass();
-
-		src.s1 = "kumofs";
-		src.s2 = "frsyuki";
-
-		new Packer(out).pack(src);
-
-		Template tmpl = ReflectionTemplate.create(StringFieldClass.class);
-
-		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-
-		Object obj = tmpl.unpack(new Unpacker(in));
-		assertEquals(obj.getClass(), StringFieldClass.class);
-
-		StringFieldClass dst = (StringFieldClass)obj;
-		assertEquals(src.s1, dst.s1);
-		assertEquals(src.s2, dst.s2);
-	}
-}
-

From dbb28d9a8ffd259d9a9d977892d71434eebffb5a Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 24 Oct 2010 18:46:48 +0900
Subject: [PATCH 0898/1648] java: Template extends MessagePacker

---
 java/src/main/java/org/msgpack/Packer.java    | 108 +--------
 java/src/main/java/org/msgpack/Template.java  |   2 +-
 java/src/main/java/org/msgpack/Templates.java |  15 ++
 java/src/main/java/org/msgpack/Unpacker.java  |  48 +---
 .../org/msgpack/template/AnyTemplate.java     |  52 ++++
 .../msgpack/template/BigIntegerTemplate.java  |   4 +
 .../org/msgpack/template/BooleanTemplate.java |   4 +
 .../msgpack/template/ByteArrayTemplate.java   |   4 +
 .../org/msgpack/template/ByteTemplate.java    |   4 +
 .../org/msgpack/template/ClassTemplate.java   | 222 ++++++++++++++++--
 .../msgpack/template/CollectionTemplate.java  |  11 +
 .../org/msgpack/template/DoubleTemplate.java  |   4 +
 .../org/msgpack/template/FloatTemplate.java   |   4 +
 .../org/msgpack/template/IntegerTemplate.java |   4 +
 .../org/msgpack/template/ListTemplate.java    |  10 +
 .../org/msgpack/template/LongTemplate.java    |   4 +
 .../org/msgpack/template/MapTemplate.java     |  11 +
 .../msgpack/template/OptionalTemplate.java    |  14 +-
 .../org/msgpack/template/ShortTemplate.java   |   4 +
 .../org/msgpack/template/StringTemplate.java  |   4 +
 .../util/codegen/DynamicCodeGenBase.java      |   6 +
 21 files changed, 379 insertions(+), 160 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/template/AnyTemplate.java

diff --git a/java/src/main/java/org/msgpack/Packer.java b/java/src/main/java/org/msgpack/Packer.java
index 1475ac6..8349a30 100644
--- a/java/src/main/java/org/msgpack/Packer.java
+++ b/java/src/main/java/org/msgpack/Packer.java
@@ -29,7 +29,7 @@ import java.math.BigInteger;
 import org.msgpack.annotation.MessagePackDelegate;
 import org.msgpack.annotation.MessagePackMessage;
 import org.msgpack.annotation.MessagePackOrdinalEnum;
-import org.msgpack.packer.*;
+import org.msgpack.util.codegen.DynamicTemplate;
 
 /**
  * Packer enables you to serialize objects into OutputStream.
@@ -49,17 +49,7 @@ import org.msgpack.packer.*;
  */
 public class Packer {
 	static {
-		// final classes
-		BooleanPacker.getInstance();
-		ByteArrayPacker.getInstance();
-		BytePacker.getInstance();
-		DoublePacker.getInstance();
-		FloatPacker.getInstance();
-		IntegerPacker.getInstance();
-		LongPacker.getInstance();
-		ShortPacker.getInstance();
-		StringPacker.getInstance();
-		//BigIntegerPacker.getInstance();  // BigInteger is not final
+		Templates.load();
 	}
 
 	public static void load() { }
@@ -429,12 +419,6 @@ public class Packer {
 		return packString(o);
 	}
 
-	public Packer pack(MessagePackable o) throws IOException {
-		if(o == null) { return packNil(); }
-		o.messagePack(this);
-		return this;
-	}
-
 	public Packer pack(byte[] o) throws IOException {
 		if(o == null) { return packNil(); }
 		packRaw(o.length);
@@ -458,86 +442,20 @@ public class Packer {
 		return this;
 	}
 
+	public Packer pack(MessagePackable o) throws IOException {
+		if(o == null) { return packNil(); }
+		o.messagePack(this);
+		return this;
+	}
 
 	public Packer pack(Object o) throws IOException {
-		if(o == null) {
-			return packNil();
-		//} else if(o instanceof String) {
-		//	byte[] b = ((String)o).getBytes("UTF-8");
-		//	packRaw(b.length);
-		//	return packRawBody(b);
-		} else if(o instanceof MessagePackable) {
-			((MessagePackable)o).messagePack(this);
-			return this;
-		//} else if(o instanceof byte[]) {
-		//	byte[] b = (byte[])o;
-		//	packRaw(b.length);
-		//	return packRawBody(b);
-		} else if(o instanceof List) {
-			List l = (List)o;
-			packArray(l.size());
-			for(Object i : l) { pack(i); }
-			return this;
-		} else if(o instanceof Set) {
-			Set l = (Set)o;
-			packArray(l.size());
-			for(Object i : l) { pack(i); }
-			return this;
-		} else if(o instanceof Map) {
-			Map m = (Map)o;
-			packMap(m.size());
-			for(Map.Entry e : m.entrySet()) {
-				pack(e.getKey());
-				pack(e.getValue());
-			}
-			return this;
-		} else if(o instanceof Collection) {
-			Collection l = (Collection)o;
-			packArray(l.size());
-			for(Object i : l) { pack(i); }
-			return this;
-		//} else if(o instanceof Boolean) {
-		//	if((Boolean)o) {
-		//		return packTrue();
-		//	} else {
-		//		return packFalse();
-		//	}
-		//} else if(o instanceof Integer) {
-		//	return packInt((Integer)o);
-		//} else if(o instanceof Long) {
-		//	return packLong((Long)o);
-		//} else if(o instanceof Short) {
-		//	return packShort((Short)o);
-		//} else if(o instanceof Byte) {
-		//	return packByte((Byte)o);
-		//} else if(o instanceof Float) {
-		//	return packFloat((Float)o);
-		//} else if(o instanceof Double) {
-		//	return packDouble((Double)o);
-		} else if(o instanceof BigInteger) {
-			return packBigInteger((BigInteger)o);
-		}
+		Templates.TAny.pack(this, o);
+		return this;
+	}
 
-		Class klass = o.getClass();
-		MessagePacker packer = CustomPacker.get(klass);
-		if(packer != null) {
-			packer.pack(this, o);
-			return this;
-		} else if (CustomMessage.isAnnotated(klass, MessagePackMessage.class)) {
-			packer = ReflectionPacker.create(klass);
-			packer.pack(this, o);
-			return this;
-		} else if (CustomMessage.isAnnotated(klass, MessagePackDelegate.class)) {
-			// FIXME DelegatePacker
-			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
-		} else if (CustomMessage.isAnnotated(klass, MessagePackOrdinalEnum.class)) {
-			// FIXME OrdinalEnumPacker
-			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
-		}
-		if (packer != null) {
-			CustomMessage.registerPacker(klass, packer);
-		}
-		throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")");
+	public Packer pack(Object o, Template tmpl) throws IOException {
+		tmpl.pack(this, o);
+		return this;
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/Template.java b/java/src/main/java/org/msgpack/Template.java
index 71e64e0..19808af 100644
--- a/java/src/main/java/org/msgpack/Template.java
+++ b/java/src/main/java/org/msgpack/Template.java
@@ -17,6 +17,6 @@
 //
 package org.msgpack;
 
-public interface Template extends MessageUnpacker, MessageConverter {
+public interface Template extends MessagePacker, MessageUnpacker, MessageConverter {
 }
 
diff --git a/java/src/main/java/org/msgpack/Templates.java b/java/src/main/java/org/msgpack/Templates.java
index 270f59b..3d7ccc5 100644
--- a/java/src/main/java/org/msgpack/Templates.java
+++ b/java/src/main/java/org/msgpack/Templates.java
@@ -22,6 +22,21 @@ import org.msgpack.template.*;
 public class Templates {
 	public static void load() { }
 
+
+	public static final Template TAny = AnyTemplate.getInstance();
+	public static Template tAny() {
+		return TAny;
+	}
+
+	public static Template tOptional(Template elementTemplate) {
+		return new OptionalTemplate(elementTemplate);
+	}
+
+	public static Template tOptional(Template elementTemplate, Object defaultObject) {
+		return new OptionalTemplate(elementTemplate, defaultObject);
+	}
+
+
 	public static Template tList(Template elementTemplate) {
 		return new ListTemplate(elementTemplate);
 	}
diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index 8bbafb3..e458b62 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -24,10 +24,6 @@ import java.util.Iterator;
 import java.nio.ByteBuffer;
 import java.math.BigInteger;
 
-import org.msgpack.annotation.MessagePackDelegate;
-import org.msgpack.annotation.MessagePackMessage;
-import org.msgpack.annotation.MessagePackOrdinalEnum;
-
 /**
  * Unpacker enables you to deserialize objects from stream.
  *
@@ -572,46 +568,20 @@ public class Unpacker implements Iterable {
 		return impl.tryUnpackNull();
 	}
 
-	final public Object unpack(MessageUnpacker unpacker) throws IOException, MessageTypeException {
-		return unpacker.unpack(this);
-	}
-
 	final public void unpack(MessageUnpackable obj) throws IOException, MessageTypeException {
 		obj.messageUnpack(this);
 	}
 
+	//final public MessagePackObject unpack() throws IOException {
+	//	return unpackObject();
+	//}
+
+	final public Object unpack(Template tmpl) throws IOException, MessageTypeException {
+		return tmpl.unpack(this);
+	}
+
 	final public Object unpack(Class klass) throws IOException, MessageTypeException, InstantiationException, IllegalAccessException {
-		if(MessageUnpackable.class.isAssignableFrom(klass)) {
-			Object obj = klass.newInstance();
-			((MessageUnpackable)obj).messageUnpack(this);
-			return obj;
-		}
-
-		MessageUnpacker unpacker = CustomUnpacker.get(klass);
-
-		if(unpacker != null) {
-			return unpacker.unpack(this);
-		}
-
-		Template tmpl = null;
-		if (CustomMessage.isAnnotated(klass, MessagePackMessage.class)) {
-			tmpl = ReflectionTemplate.create(klass);
-			return tmpl.unpack(this);
-		} else if (CustomMessage.isAnnotated(klass, MessagePackDelegate.class)) {
-			// FIXME DelegateTemplate
-			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
-		} else if (CustomMessage.isAnnotated(klass, MessagePackOrdinalEnum.class)) {
-			// FIXME OrdinalEnumTemplate
-			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
-		}
-		if (tmpl != null) {
-			CustomMessage.registerTemplate(klass, tmpl);
-		}
-		MessageConverter converter = CustomConverter.get(klass);
-		if(converter != null) {
-			return converter.convert(unpackObject());
-		}
-		throw new MessageTypeException();
+		return unpack(Templates.tClass(klass));
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/AnyTemplate.java b/java/src/main/java/org/msgpack/template/AnyTemplate.java
new file mode 100644
index 0000000..fb2e1ee
--- /dev/null
+++ b/java/src/main/java/org/msgpack/template/AnyTemplate.java
@@ -0,0 +1,52 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack.template;
+
+import java.io.IOException;
+import org.msgpack.*;
+
+public class AnyTemplate implements Template {
+	private AnyTemplate() { }
+
+	public void pack(Packer pk, Object target) throws IOException {
+		if(target == null) {
+			pk.packNil();
+		} else {
+			new ClassTemplate(target.getClass()).pack(pk, target);
+		}
+	}
+
+	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
+		return pac.unpackObject();
+	}
+
+	public Object convert(MessagePackObject from) throws MessageTypeException {
+		return from;
+	}
+
+	static public AnyTemplate getInstance() {
+		return instance;
+	}
+
+	static final AnyTemplate instance = new AnyTemplate();
+
+	static {
+		CustomMessage.registerTemplate(MessagePackObject.class, instance);
+	}
+}
+
diff --git a/java/src/main/java/org/msgpack/template/BigIntegerTemplate.java b/java/src/main/java/org/msgpack/template/BigIntegerTemplate.java
index e8a2993..66e142b 100644
--- a/java/src/main/java/org/msgpack/template/BigIntegerTemplate.java
+++ b/java/src/main/java/org/msgpack/template/BigIntegerTemplate.java
@@ -24,6 +24,10 @@ import org.msgpack.*;
 public class BigIntegerTemplate implements Template {
 	private BigIntegerTemplate() { }
 
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.packBigInteger((BigInteger)target);
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		return pac.unpackBigInteger();
 	}
diff --git a/java/src/main/java/org/msgpack/template/BooleanTemplate.java b/java/src/main/java/org/msgpack/template/BooleanTemplate.java
index 0d64ecc..e9fd33c 100644
--- a/java/src/main/java/org/msgpack/template/BooleanTemplate.java
+++ b/java/src/main/java/org/msgpack/template/BooleanTemplate.java
@@ -23,6 +23,10 @@ import org.msgpack.*;
 public class BooleanTemplate implements Template {
 	private BooleanTemplate() { }
 
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.packBoolean((Boolean)target);
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		return pac.unpackBoolean();
 	}
diff --git a/java/src/main/java/org/msgpack/template/ByteArrayTemplate.java b/java/src/main/java/org/msgpack/template/ByteArrayTemplate.java
index fe26369..45a006f 100644
--- a/java/src/main/java/org/msgpack/template/ByteArrayTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ByteArrayTemplate.java
@@ -23,6 +23,10 @@ import org.msgpack.*;
 public class ByteArrayTemplate implements Template {
 	private ByteArrayTemplate() { }
 
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.packByteArray((byte[])target);
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		return pac.unpackByteArray();
 	}
diff --git a/java/src/main/java/org/msgpack/template/ByteTemplate.java b/java/src/main/java/org/msgpack/template/ByteTemplate.java
index 8d0e6e6..d58e8b0 100644
--- a/java/src/main/java/org/msgpack/template/ByteTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ByteTemplate.java
@@ -23,6 +23,10 @@ import org.msgpack.*;
 public class ByteTemplate implements Template {
 	private ByteTemplate() { }
 
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.packByte((Byte)target);
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		return pac.unpackByte();
 	}
diff --git a/java/src/main/java/org/msgpack/template/ClassTemplate.java b/java/src/main/java/org/msgpack/template/ClassTemplate.java
index 9bb957c..86e8fe8 100644
--- a/java/src/main/java/org/msgpack/template/ClassTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ClassTemplate.java
@@ -19,21 +19,186 @@ package org.msgpack.template;
 
 import java.io.IOException;
 import org.msgpack.*;
+import org.msgpack.annotation.MessagePackDelegate;
+import org.msgpack.annotation.MessagePackMessage;
+import org.msgpack.annotation.MessagePackOrdinalEnum;
+import org.msgpack.util.codegen.DynamicPacker;
+import org.msgpack.util.codegen.DynamicConverter;
+import org.msgpack.util.codegen.DynamicUnpacker;
+
+import java.util.*;
+import java.math.BigInteger;
 
 public class ClassTemplate implements Template {
 	static {
 		Templates.load();
 	}
 
-	private Class klass;
+	private Class klass;
 
-	public ClassTemplate(Class klass) {
+	public ClassTemplate(Class klass) {
 		this.klass = klass;
 	}
 
+	public void pack(Packer pk, Object o) throws IOException {
+		// FIXME
+		if(o == null) {
+			pk.packNil();
+			return;
+		}
+		//if(o instanceof String) {
+		//	pk.packString((String)o);
+		//	return;
+		//}
+		if(o instanceof MessagePackable) {
+			((MessagePackable)o).messagePack(pk);
+			return;
+		}
+		//if(o instanceof byte[]) {
+		//	pk.packByteArray((byte[])o);
+		//	return;
+		//}
+		if(o instanceof List) {
+			List l = (List)o;
+			pk.packArray(l.size());
+			for(Object i : l) {
+				pk.pack(i);
+			}
+			return;
+		}
+		if(o instanceof Set) {
+			Set l = (Set)o;
+			pk.packArray(l.size());
+			for(Object i : l) {
+				pk.pack(i);
+			}
+			return;
+		}
+		if(o instanceof Map) {
+			Map m = (Map)o;
+			pk.packMap(m.size());
+			for(Map.Entry e : m.entrySet()) {
+				pk.pack(e.getKey());
+				pk.pack(e.getValue());
+			}
+			return;
+		}
+		if(o instanceof Collection) {
+			Collection l = (Collection)o;
+			pk.packArray(l.size());
+			for(Object i : l) {
+				pk.pack(i);
+			}
+			return;
+		}
+		//if(o instanceof Boolean) {
+		//	pk.packBoolean((Boolean)o);
+		//	return;
+		//}
+		//if(o instanceof Integer) {
+		//	pk.packInt((Integer)o);
+		//	return;
+		//}
+		//if(o instanceof Long) {
+		//	pk.packLong((Long)o);
+		//	return;
+		//}
+		//if(o instanceof Short) {
+		//	pk.packShort((Short)o);
+		//	return;
+		//}
+		//if(o instanceof Byte) {
+		//	pk.packByte((Byte)o);
+		//	return;
+		//}
+		//if(o instanceof Float) {
+		//	pk.packFloat((Float)o);
+		//	return;
+		//}
+		//if(o instanceof Double) {
+		//	pk.packDouble((Double)o);
+		//	return;
+		//}
+		if(o instanceof BigInteger) {
+			pk.packBigInteger((BigInteger)o);
+			return;
+		}
+
+		MessagePacker packer = CustomPacker.get(klass);
+		if(packer != null) {
+			packer.pack(pk, o);
+			return;
+		}
+
+		if (CustomMessage.isAnnotated(klass, MessagePackMessage.class)) {
+			packer = DynamicPacker.create(klass);
+		} else if (CustomMessage.isAnnotated(klass, MessagePackDelegate.class)) {
+			// FIXME DelegatePacker
+			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
+		} else if (CustomMessage.isAnnotated(klass, MessagePackOrdinalEnum.class)) {
+			// FIXME OrdinalEnumPacker
+			throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
+		}
+
+		if (packer != null) {
+			CustomPacker.register(klass, packer);
+			packer.pack(pk, o);
+			return;
+		}
+
+		throw new MessageTypeException("unknown object "+o+" ("+o.getClass()+")");
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		try {
-			return pac.unpack(klass);
+			MessageUnpacker unpacker = CustomUnpacker.get(klass);
+			if(unpacker != null) {
+				return unpacker.unpack(pac);
+			}
+
+			if(MessageUnpackable.class.isAssignableFrom(klass)) {
+				Object obj = klass.newInstance();
+				((MessageUnpackable)obj).messageUnpack(pac);
+				return obj;
+			}
+
+			if (CustomMessage.isAnnotated(klass, MessagePackMessage.class)) {
+				unpacker = DynamicUnpacker.create(klass);
+			} else if (CustomMessage.isAnnotated(klass, MessagePackDelegate.class)) {
+				// TODO DelegateUnpacker
+				throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
+			} else if (CustomMessage.isAnnotated(klass, MessagePackOrdinalEnum.class)) {
+				// TODO OrdinalEnumUnpacker
+				throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
+			}
+
+			if (unpacker != null) {
+				CustomUnpacker.register(klass, unpacker);
+				return unpacker.unpack(pac);
+			}
+
+			// fallback
+			{
+				MessageConverter converter = null;
+
+				if (CustomMessage.isAnnotated(klass, MessagePackMessage.class)) {
+					converter = DynamicConverter.create(klass);
+				} else if (CustomMessage.isAnnotated(klass, MessagePackDelegate.class)) {
+					// TODO DelegateConverter
+					throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
+				} else if (CustomMessage.isAnnotated(klass, MessagePackOrdinalEnum.class)) {
+					// TODO OrdinalEnumConverter
+					throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
+				}
+
+				if (converter != null) {
+					CustomConverter.register(klass, converter);
+					return converter.convert(pac.unpackObject());
+				}
+			}
+
+			throw new MessageTypeException();
+
 		} catch (IllegalAccessException e) {
 			throw new MessageTypeException(e.getMessage());  // FIXME
 		} catch (InstantiationException e) {
@@ -42,27 +207,40 @@ public class ClassTemplate implements Template {
 	}
 
 	public Object convert(MessagePackObject from) throws MessageTypeException {
-		if(MessageConvertable.class.isAssignableFrom(klass)) {
-			Object obj;
-			try {
-				obj = klass.newInstance();
-			} catch (IllegalAccessException e) {
-				throw new MessageTypeException(e.getMessage());  // FIXME
-			} catch (InstantiationException e) {
-				throw new MessageTypeException(e.getMessage());  // FIXME
+		try {
+			MessageConverter converter = CustomConverter.get(klass);
+			if(converter != null) {
+				return converter.convert(from);
 			}
-			((MessageConvertable)obj).messageConvert(from);
-			return obj;
+
+			if(MessageConvertable.class.isAssignableFrom(klass)) {
+				Object obj = klass.newInstance();
+				((MessageConvertable)obj).messageConvert(from);
+				return obj;
+			}
+
+			if (CustomMessage.isAnnotated(klass, MessagePackMessage.class)) {
+				converter = DynamicConverter.create(klass);
+			} else if (CustomMessage.isAnnotated(klass, MessagePackDelegate.class)) {
+				// TODO DelegateConverter
+				throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
+			} else if (CustomMessage.isAnnotated(klass, MessagePackOrdinalEnum.class)) {
+				// TODO OrdinalEnumConverter
+				throw new UnsupportedOperationException("not supported yet. : " + klass.getName());
+			}
+
+			if (converter != null) {
+				CustomConverter.register(klass, converter);
+				return converter.convert(from);
+			}
+
+			throw new MessageTypeException();
+
+		} catch (IllegalAccessException e) {
+			throw new MessageTypeException(e.getMessage());  // FIXME
+		} catch (InstantiationException e) {
+			throw new MessageTypeException(e.getMessage());  // FIXME
 		}
-
-		MessageConverter converter = CustomConverter.get(klass);
-		if(converter != null) {
-			return converter.convert(from);
-		}
-
-		// FIXME check annotations -> code generation -> CustomMessage.registerTemplate
-
-		throw new MessageTypeException();
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/CollectionTemplate.java b/java/src/main/java/org/msgpack/template/CollectionTemplate.java
index 0c9ff17..0269d67 100644
--- a/java/src/main/java/org/msgpack/template/CollectionTemplate.java
+++ b/java/src/main/java/org/msgpack/template/CollectionTemplate.java
@@ -17,6 +17,7 @@
 //
 package org.msgpack.template;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.LinkedList;
 import java.io.IOException;
@@ -29,6 +30,16 @@ public class CollectionTemplate implements Template {
 		this.elementTemplate = elementTemplate;
 	}
 
+	public void pack(Packer pk, Object target) throws IOException {
+		if(target instanceof Collection) {
+			throw new MessageTypeException();
+		}
+		Collection collection = (Collection)target;
+		for(Object element : collection) {
+			elementTemplate.pack(pk, element);
+		}
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		int length = pac.unpackArray();
 		List list = new LinkedList();
diff --git a/java/src/main/java/org/msgpack/template/DoubleTemplate.java b/java/src/main/java/org/msgpack/template/DoubleTemplate.java
index 2e26f50..c899e73 100644
--- a/java/src/main/java/org/msgpack/template/DoubleTemplate.java
+++ b/java/src/main/java/org/msgpack/template/DoubleTemplate.java
@@ -23,6 +23,10 @@ import org.msgpack.*;
 public class DoubleTemplate implements Template {
 	private DoubleTemplate() { }
 
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.packDouble(((Double)target));
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		return pac.unpackDouble();
 	}
diff --git a/java/src/main/java/org/msgpack/template/FloatTemplate.java b/java/src/main/java/org/msgpack/template/FloatTemplate.java
index 8730172..354e3f8 100644
--- a/java/src/main/java/org/msgpack/template/FloatTemplate.java
+++ b/java/src/main/java/org/msgpack/template/FloatTemplate.java
@@ -23,6 +23,10 @@ import org.msgpack.*;
 public class FloatTemplate implements Template {
 	private FloatTemplate() { }
 
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.packFloat((Float)target);
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		return pac.unpackFloat();
 	}
diff --git a/java/src/main/java/org/msgpack/template/IntegerTemplate.java b/java/src/main/java/org/msgpack/template/IntegerTemplate.java
index c56c044..21ff0af 100644
--- a/java/src/main/java/org/msgpack/template/IntegerTemplate.java
+++ b/java/src/main/java/org/msgpack/template/IntegerTemplate.java
@@ -23,6 +23,10 @@ import org.msgpack.*;
 public class IntegerTemplate implements Template {
 	private IntegerTemplate() { }
 
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.packInt((Integer)target);
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		return pac.unpackInt();
 	}
diff --git a/java/src/main/java/org/msgpack/template/ListTemplate.java b/java/src/main/java/org/msgpack/template/ListTemplate.java
index 4ea7a9b..6703df4 100644
--- a/java/src/main/java/org/msgpack/template/ListTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ListTemplate.java
@@ -33,6 +33,16 @@ public class ListTemplate implements Template {
 		return elementTemplate;
 	}
 
+	public void pack(Packer pk, Object target) throws IOException {
+		if(target instanceof List) {
+			throw new MessageTypeException();
+		}
+		List list = (List)target;
+		for(Object element : list) {
+			elementTemplate.pack(pk, element);
+		}
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		int length = pac.unpackArray();
 		List list = new ArrayList(length);
diff --git a/java/src/main/java/org/msgpack/template/LongTemplate.java b/java/src/main/java/org/msgpack/template/LongTemplate.java
index a0c8210..d8990af 100644
--- a/java/src/main/java/org/msgpack/template/LongTemplate.java
+++ b/java/src/main/java/org/msgpack/template/LongTemplate.java
@@ -23,6 +23,10 @@ import org.msgpack.*;
 public class LongTemplate implements Template {
 	private LongTemplate() { }
 
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.packLong((Long)target);
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		return pac.unpackLong();
 	}
diff --git a/java/src/main/java/org/msgpack/template/MapTemplate.java b/java/src/main/java/org/msgpack/template/MapTemplate.java
index d23282e..6925982 100644
--- a/java/src/main/java/org/msgpack/template/MapTemplate.java
+++ b/java/src/main/java/org/msgpack/template/MapTemplate.java
@@ -39,6 +39,17 @@ public class MapTemplate implements Template {
 		return valueTemplate;
 	}
 
+	public void pack(Packer pk, Object target) throws IOException {
+		if(target instanceof Map) {
+			throw new MessageTypeException();
+		}
+		Map map = (Map)target;
+		for(Map.Entry pair : map.entrySet()) {
+			keyTemplate.pack(pk, pair.getKey());
+			valueTemplate.pack(pk, pair.getValue());
+		}
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		int length = pac.unpackMap();
 		Map map = new HashMap(length);
diff --git a/java/src/main/java/org/msgpack/template/OptionalTemplate.java b/java/src/main/java/org/msgpack/template/OptionalTemplate.java
index cc5ee60..6358d7d 100644
--- a/java/src/main/java/org/msgpack/template/OptionalTemplate.java
+++ b/java/src/main/java/org/msgpack/template/OptionalTemplate.java
@@ -28,13 +28,21 @@ public class OptionalTemplate implements Template {
 		this(elementTemplate, null);
 	}
 
+	public OptionalTemplate(Template elementTemplate, Object defaultObject) {
+		this.elementTemplate = elementTemplate;
+		this.defaultObject = defaultObject;
+	}
+
 	public Template getElementTemplate() {
 		return elementTemplate;
 	}
 
-	public OptionalTemplate(Template elementTemplate, Object defaultObject) {
-		this.elementTemplate = elementTemplate;
-		this.defaultObject = defaultObject;
+	public void pack(Packer pk, Object target) throws IOException {
+		if(target == null) {
+			pk.pack(defaultObject);
+		} else {
+			elementTemplate.pack(pk, target);
+		}
 	}
 
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
diff --git a/java/src/main/java/org/msgpack/template/ShortTemplate.java b/java/src/main/java/org/msgpack/template/ShortTemplate.java
index b3bf43b..0268797 100644
--- a/java/src/main/java/org/msgpack/template/ShortTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ShortTemplate.java
@@ -23,6 +23,10 @@ import org.msgpack.*;
 public class ShortTemplate implements Template {
 	private ShortTemplate() { }
 
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.packShort((Short)target);
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		return pac.unpackShort();
 	}
diff --git a/java/src/main/java/org/msgpack/template/StringTemplate.java b/java/src/main/java/org/msgpack/template/StringTemplate.java
index 563112d..4a4be3e 100644
--- a/java/src/main/java/org/msgpack/template/StringTemplate.java
+++ b/java/src/main/java/org/msgpack/template/StringTemplate.java
@@ -23,6 +23,10 @@ import org.msgpack.*;
 public class StringTemplate implements Template {
 	private StringTemplate() { }
 
+	public void pack(Packer pk, Object target) throws IOException {
+		pk.packString((String)target);
+	}
+
 	public Object unpack(Unpacker pac) throws IOException, MessageTypeException {
 		return pac.unpackString();
 	}
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index 3222b47..c80d354 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -152,6 +152,12 @@ public class DynamicCodeGenBase implements Constants {
 			this.type = type;
 		}
 
+		@Override
+		public void pack(Packer pk, Object target) throws IOException {
+			// FIXME
+			pk.pack(target);
+		}
+
 		@Override
 		public Object unpack(Unpacker unpacker) throws IOException,
 				MessageTypeException {

From e3bf8a404b993da62676d1d435e49a37c404fe3b Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 24 Oct 2010 19:32:45 +0900
Subject: [PATCH 0899/1648] java: adds MessagePack class

---
 .../main/java/org/msgpack/MessagePack.java    | 123 ++++++++++++++++++
 java/src/main/java/org/msgpack/Unpacker.java  |   5 +-
 .../msgpack/TestMessagePackStaticMethods.java | 121 +++++++++++++++++
 3 files changed, 247 insertions(+), 2 deletions(-)
 create mode 100644 java/src/main/java/org/msgpack/MessagePack.java
 create mode 100644 java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java

diff --git a/java/src/main/java/org/msgpack/MessagePack.java b/java/src/main/java/org/msgpack/MessagePack.java
new file mode 100644
index 0000000..f9dcce8
--- /dev/null
+++ b/java/src/main/java/org/msgpack/MessagePack.java
@@ -0,0 +1,123 @@
+//
+// MessagePack for Java
+//
+// Copyright (C) 2009-2010 FURUHASHI Sadayuki
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//        http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+//
+package org.msgpack;
+
+import java.io.OutputStream;
+import java.io.InputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+public class MessagePack {
+	public static byte[] pack(Object obj) {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		try {
+			new Packer(out).pack(obj);
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+		return out.toByteArray();
+	}
+
+	public static void pack(OutputStream out, Object obj) throws IOException {
+		new Packer(out).pack(obj);
+	}
+
+	public static byte[] pack(Object obj, Template tmpl) throws MessageTypeException {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		try {
+			new Packer(out).pack(obj, tmpl);
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+		return out.toByteArray();
+	}
+
+	public static void pack(OutputStream out, Object obj, Template tmpl) throws IOException {
+		new Packer(out).pack(obj, tmpl);
+	}
+
+
+	public static MessagePackObject unpack(byte[] buffer) throws IOException {
+		Unpacker pac = new Unpacker();
+		pac.wrap(buffer);
+		try {
+			return pac.unpackObject();
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	public static Object unpack(byte[] buffer, Template tmpl) throws MessageTypeException {
+		Unpacker pac = new Unpacker();
+		pac.wrap(buffer);
+		try {
+			return pac.unpack(tmpl);
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	public static  T unpack(byte[] buffer, Class klass) throws MessageTypeException {
+		Unpacker pac = new Unpacker();
+		pac.wrap(buffer);
+		try {
+			return pac.unpack(klass);
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	public static MessagePackObject unpack(InputStream in) {
+		Unpacker pac = new Unpacker(in);
+		try {
+			return pac.unpackObject();
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	public static Object unpack(InputStream in, Template tmpl) throws IOException, MessageTypeException {
+		Unpacker pac = new Unpacker(in);
+		try {
+			return pac.unpack(tmpl);
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	public static  T unpack(InputStream in, Class klass) throws IOException, MessageTypeException {
+		Unpacker pac = new Unpacker(in);
+		try {
+			return pac.unpack(klass);
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+
+	}
+
+	//public static void register(Class target);  // TODO: auto-detect
+
+	//public static void register(Class target, Template tmpl); // TODO
+
+	//public static void registerPacker(Class target, MessagePacker packer); // TODO
+
+	//public static void registerConverter(Class target, MessageConverter converter); // TODO
+
+	//public static void registerUnpacker(Class target, MessageUnpacker unpacker); // TODO
+}
+
diff --git a/java/src/main/java/org/msgpack/Unpacker.java b/java/src/main/java/org/msgpack/Unpacker.java
index e458b62..d07de1e 100644
--- a/java/src/main/java/org/msgpack/Unpacker.java
+++ b/java/src/main/java/org/msgpack/Unpacker.java
@@ -580,8 +580,9 @@ public class Unpacker implements Iterable {
 		return tmpl.unpack(this);
 	}
 
-	final public Object unpack(Class klass) throws IOException, MessageTypeException, InstantiationException, IllegalAccessException {
-		return unpack(Templates.tClass(klass));
+	final public  T unpack(Class klass) throws IOException, MessageTypeException {
+		// FIXME optional?
+		return (T)unpack(Templates.tOptional(Templates.tClass(klass)));
 	}
 }
 
diff --git a/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
new file mode 100644
index 0000000..dea2a37
--- /dev/null
+++ b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
@@ -0,0 +1,121 @@
+package org.msgpack;
+
+import org.msgpack.*;
+import org.msgpack.object.*;
+import static org.msgpack.Templates.*;
+
+import java.io.*;
+import java.util.*;
+import java.math.BigInteger;
+
+import org.junit.Test;
+import junit.framework.TestCase;
+
+public class TestMessagePackStaticMethods extends TestCase {
+	@Test
+	public void testPackToByteArray() throws Exception {
+		byte[] a = MessagePack.pack("msgpack");
+		byte[] b = MessagePack.pack((Object)1);
+		byte[] c = MessagePack.pack((Object)null);
+
+		{
+			MessagePackObject aobj = MessagePack.unpack(a);
+			MessagePackObject bobj = MessagePack.unpack(b);
+			MessagePackObject cobj = MessagePack.unpack(c);
+
+			assertEquals(aobj, RawType.create("msgpack"));
+			assertEquals(bobj, IntegerType.create(1));
+			assertEquals(cobj, NilType.create());
+		}
+	}
+
+	@Test
+	public void testPackToStream() throws Exception {
+		ByteArrayOutputStream aout = new ByteArrayOutputStream();
+		MessagePack.pack(aout, "msgpack");
+		ByteArrayOutputStream bout = new ByteArrayOutputStream();
+		MessagePack.pack(bout, (Object)1);
+		ByteArrayOutputStream cout = new ByteArrayOutputStream();
+		MessagePack.pack(cout, (Object)null);
+
+		{
+			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
+			MessagePackObject aobj = MessagePack.unpack(ain);
+			InputStream bin = new ByteArrayInputStream(bout.toByteArray());
+			MessagePackObject bobj = MessagePack.unpack(bin);
+			InputStream cin = new ByteArrayInputStream(cout.toByteArray());
+			MessagePackObject cobj = MessagePack.unpack(cin);
+
+			assertEquals(aobj, RawType.create("msgpack"));
+			assertEquals(bobj, IntegerType.create(1));
+			assertEquals(cobj, NilType.create());
+		}
+	}
+
+	@Test
+	public void testCheckedPackToByteArray() throws Exception {
+		byte[] a = MessagePack.pack("msgpack", TString);
+		byte[] b = MessagePack.pack((Object)1, TInteger);
+		byte[] c = MessagePack.pack((Object)null, TAny);
+
+		{
+			Object aobj = MessagePack.unpack(a, TString);
+			Object bobj = MessagePack.unpack(b, TInteger);
+			Object cobj_any = MessagePack.unpack(c, TAny);
+			Object cobj_obj = MessagePack.unpack(c, tOptional(TAny));
+			assertEquals(aobj, "msgpack");
+			assertEquals(bobj, 1);
+			assertEquals(cobj_any, NilType.create());
+			assertEquals(cobj_obj, null);
+		}
+
+		{
+			String  aobj = MessagePack.unpack(a, String.class);
+			Integer bobj = MessagePack.unpack(b, Integer.class);
+			Object  cobj = MessagePack.unpack(c, Object.class);
+			assertEquals(aobj, "msgpack");
+			assertEquals(bobj, (Integer)1);
+			assertEquals(cobj, null);
+		}
+	}
+
+	@Test
+	public void testCheckedPackToStream() throws Exception {
+		ByteArrayOutputStream aout = new ByteArrayOutputStream();
+		MessagePack.pack(aout, "msgpack");
+		ByteArrayOutputStream bout = new ByteArrayOutputStream();
+		MessagePack.pack(bout, (Object)1);
+		ByteArrayOutputStream cout = new ByteArrayOutputStream();
+		MessagePack.pack(cout, (Object)null);
+
+		{
+			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
+			Object aobj = MessagePack.unpack(ain, TString);
+			InputStream bin = new ByteArrayInputStream(bout.toByteArray());
+			Object bobj = MessagePack.unpack(bin, TInteger);
+			InputStream cin_any = new ByteArrayInputStream(cout.toByteArray());
+			Object cobj_any = MessagePack.unpack(cin_any, TAny);
+			InputStream cin_obj = new ByteArrayInputStream(cout.toByteArray());
+			Object cobj_obj = MessagePack.unpack(cin_obj, tOptional(TAny));
+
+			assertEquals(aobj, "msgpack");
+			assertEquals(bobj, 1);
+			assertEquals(cobj_any, NilType.create());
+			assertEquals(cobj_obj, null);
+		}
+
+		{
+			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
+			String  aobj = MessagePack.unpack(ain, String.class);
+			InputStream bin = new ByteArrayInputStream(bout.toByteArray());
+			Integer bobj = MessagePack.unpack(bin, Integer.class);
+			InputStream cin = new ByteArrayInputStream(cout.toByteArray());
+			Object  cobj = MessagePack.unpack(cin, Object.class);
+
+			assertEquals(aobj, "msgpack");
+			assertEquals(bobj, (Integer)1);
+			assertEquals(cobj, null);
+		}
+	}
+}
+

From 19ff0dd17fee50f2ea18b6f941c9f7068260bcb7 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 24 Oct 2010 20:11:39 +0900
Subject: [PATCH 0900/1648] java: fixes ListTemplate and MapTemplate

---
 .../org/msgpack/template/ListTemplate.java    |  3 +-
 .../org/msgpack/template/MapTemplate.java     |  3 +-
 .../msgpack/TestMessagePackStaticMethods.java | 32 +++++++++++++++++++
 3 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/java/src/main/java/org/msgpack/template/ListTemplate.java b/java/src/main/java/org/msgpack/template/ListTemplate.java
index 6703df4..f15ae52 100644
--- a/java/src/main/java/org/msgpack/template/ListTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ListTemplate.java
@@ -34,10 +34,11 @@ public class ListTemplate implements Template {
 	}
 
 	public void pack(Packer pk, Object target) throws IOException {
-		if(target instanceof List) {
+		if(!(target instanceof List)) {
 			throw new MessageTypeException();
 		}
 		List list = (List)target;
+		pk.packArray(list.size());
 		for(Object element : list) {
 			elementTemplate.pack(pk, element);
 		}
diff --git a/java/src/main/java/org/msgpack/template/MapTemplate.java b/java/src/main/java/org/msgpack/template/MapTemplate.java
index 6925982..9b7b250 100644
--- a/java/src/main/java/org/msgpack/template/MapTemplate.java
+++ b/java/src/main/java/org/msgpack/template/MapTemplate.java
@@ -40,10 +40,11 @@ public class MapTemplate implements Template {
 	}
 
 	public void pack(Packer pk, Object target) throws IOException {
-		if(target instanceof Map) {
+		if(!(target instanceof Map)) {
 			throw new MessageTypeException();
 		}
 		Map map = (Map)target;
+		pk.packMap(map.size());
 		for(Map.Entry pair : map.entrySet()) {
 			keyTemplate.pack(pk, pair.getKey());
 			valueTemplate.pack(pk, pair.getValue());
diff --git a/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
index dea2a37..b91615b 100644
--- a/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
+++ b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
@@ -17,15 +17,18 @@ public class TestMessagePackStaticMethods extends TestCase {
 		byte[] a = MessagePack.pack("msgpack");
 		byte[] b = MessagePack.pack((Object)1);
 		byte[] c = MessagePack.pack((Object)null);
+		byte[] d = MessagePack.pack(createStringList());
 
 		{
 			MessagePackObject aobj = MessagePack.unpack(a);
 			MessagePackObject bobj = MessagePack.unpack(b);
 			MessagePackObject cobj = MessagePack.unpack(c);
+			MessagePackObject dobj = MessagePack.unpack(d);
 
 			assertEquals(aobj, RawType.create("msgpack"));
 			assertEquals(bobj, IntegerType.create(1));
 			assertEquals(cobj, NilType.create());
+			assertEquals(dobj, createStringList_dynamic());
 		}
 	}
 
@@ -37,6 +40,8 @@ public class TestMessagePackStaticMethods extends TestCase {
 		MessagePack.pack(bout, (Object)1);
 		ByteArrayOutputStream cout = new ByteArrayOutputStream();
 		MessagePack.pack(cout, (Object)null);
+		ByteArrayOutputStream dout = new ByteArrayOutputStream();
+		MessagePack.pack(dout, createStringList());
 
 		{
 			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
@@ -45,10 +50,13 @@ public class TestMessagePackStaticMethods extends TestCase {
 			MessagePackObject bobj = MessagePack.unpack(bin);
 			InputStream cin = new ByteArrayInputStream(cout.toByteArray());
 			MessagePackObject cobj = MessagePack.unpack(cin);
+			InputStream din = new ByteArrayInputStream(dout.toByteArray());
+			MessagePackObject dobj = MessagePack.unpack(din);
 
 			assertEquals(aobj, RawType.create("msgpack"));
 			assertEquals(bobj, IntegerType.create(1));
 			assertEquals(cobj, NilType.create());
+			assertEquals(dobj, createStringList_dynamic());
 		}
 	}
 
@@ -57,16 +65,19 @@ public class TestMessagePackStaticMethods extends TestCase {
 		byte[] a = MessagePack.pack("msgpack", TString);
 		byte[] b = MessagePack.pack((Object)1, TInteger);
 		byte[] c = MessagePack.pack((Object)null, TAny);
+		byte[] d = MessagePack.pack(createStringList(), tList(TString));
 
 		{
 			Object aobj = MessagePack.unpack(a, TString);
 			Object bobj = MessagePack.unpack(b, TInteger);
 			Object cobj_any = MessagePack.unpack(c, TAny);
 			Object cobj_obj = MessagePack.unpack(c, tOptional(TAny));
+			Object dobj = MessagePack.unpack(d, tList(TString));
 			assertEquals(aobj, "msgpack");
 			assertEquals(bobj, 1);
 			assertEquals(cobj_any, NilType.create());
 			assertEquals(cobj_obj, null);
+			assertEquals(dobj, createStringList());
 		}
 
 		{
@@ -87,6 +98,8 @@ public class TestMessagePackStaticMethods extends TestCase {
 		MessagePack.pack(bout, (Object)1);
 		ByteArrayOutputStream cout = new ByteArrayOutputStream();
 		MessagePack.pack(cout, (Object)null);
+		ByteArrayOutputStream dout = new ByteArrayOutputStream();
+		MessagePack.pack(dout, createStringList());
 
 		{
 			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
@@ -97,11 +110,14 @@ public class TestMessagePackStaticMethods extends TestCase {
 			Object cobj_any = MessagePack.unpack(cin_any, TAny);
 			InputStream cin_obj = new ByteArrayInputStream(cout.toByteArray());
 			Object cobj_obj = MessagePack.unpack(cin_obj, tOptional(TAny));
+			InputStream din = new ByteArrayInputStream(dout.toByteArray());
+			Object dobj = MessagePack.unpack(din, tList(TString));
 
 			assertEquals(aobj, "msgpack");
 			assertEquals(bobj, 1);
 			assertEquals(cobj_any, NilType.create());
 			assertEquals(cobj_obj, null);
+			assertEquals(dobj, createStringList());
 		}
 
 		{
@@ -117,5 +133,21 @@ public class TestMessagePackStaticMethods extends TestCase {
 			assertEquals(cobj, null);
 		}
 	}
+
+	private List createStringList() {
+		List list = new ArrayList();
+		list.add("frsyuki");
+		list.add("kumofs");
+		list.add("gem-compile");
+		return list;
+	}
+
+	private MessagePackObject createStringList_dynamic() {
+		MessagePackObject[] array = new MessagePackObject[3];
+		array[0] = RawType.create("frsyuki");
+		array[1] = RawType.create("kumofs");
+		array[2] = RawType.create("gem-compile");
+		return ArrayType.create(array);
+	}
 }
 

From 0a345cb12bb77c7afc5b29cc4568f717e7862ff6 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 24 Oct 2010 20:16:37 +0900
Subject: [PATCH 0901/1648] java: fixes CollectionTemplate

---
 .../src/main/java/org/msgpack/template/CollectionTemplate.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/java/src/main/java/org/msgpack/template/CollectionTemplate.java b/java/src/main/java/org/msgpack/template/CollectionTemplate.java
index 0269d67..50e6b13 100644
--- a/java/src/main/java/org/msgpack/template/CollectionTemplate.java
+++ b/java/src/main/java/org/msgpack/template/CollectionTemplate.java
@@ -31,10 +31,11 @@ public class CollectionTemplate implements Template {
 	}
 
 	public void pack(Packer pk, Object target) throws IOException {
-		if(target instanceof Collection) {
+		if(!(target instanceof Collection)) {
 			throw new MessageTypeException();
 		}
 		Collection collection = (Collection)target;
+		pk.packArray(collection.size());
 		for(Object element : collection) {
 			elementTemplate.pack(pk, element);
 		}

From 7ac4ad3e38d5537e83a36f2d9fb9a5ce872c0c57 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 24 Oct 2010 20:46:22 +0900
Subject: [PATCH 0902/1648] java: adds MessagePack.register methods

---
 .../main/java/org/msgpack/MessagePack.java    | 43 +++++++++--
 .../msgpack/TestMessagePackStaticMethods.java | 73 +++++++++++++++++++
 2 files changed, 111 insertions(+), 5 deletions(-)

diff --git a/java/src/main/java/org/msgpack/MessagePack.java b/java/src/main/java/org/msgpack/MessagePack.java
index f9dcce8..449f8a0 100644
--- a/java/src/main/java/org/msgpack/MessagePack.java
+++ b/java/src/main/java/org/msgpack/MessagePack.java
@@ -21,6 +21,10 @@ import java.io.OutputStream;
 import java.io.InputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+//import org.msgpack.util.codegen.DynamicTemplate;  // FIXME
+import org.msgpack.util.codegen.DynamicPacker;
+import org.msgpack.util.codegen.DynamicConverter;
+import org.msgpack.util.codegen.DynamicUnpacker;
 
 public class MessagePack {
 	public static byte[] pack(Object obj) {
@@ -110,14 +114,43 @@ public class MessagePack {
 
 	}
 
-	//public static void register(Class target);  // TODO: auto-detect
+	public static void register(Class target) {  // auto-detect
+		// FIXME
+		//Template tmpl;
+		//if(List.isAssignableFrom(target)) {
+		//} else if(Set.isAssignableFrom(target)) {
+		//} else if(Map.isAssignableFrom(target)) {
+		//} else if(Collection.isAssignableFrom(target)) {
+		//} else if(BigInteger.isAssignableFrom(target)) {
+		//} else {
+		//}
 
-	//public static void register(Class target, Template tmpl); // TODO
+		// FIXME
+		//Template tmpl = DynamicTemplate.create(target);
+		//register(target, tmpl);
 
-	//public static void registerPacker(Class target, MessagePacker packer); // TODO
+		// FIXME
+		CustomPacker.register(target, DynamicPacker.create(target));
+		CustomConverter.register(target, DynamicConverter.create(target));
+		CustomUnpacker.register(target, DynamicUnpacker.create(target));
+	}
 
-	//public static void registerConverter(Class target, MessageConverter converter); // TODO
+	public static void register(Class target, Template tmpl) {
+		CustomPacker.register(target, tmpl);
+		CustomConverter.register(target, tmpl);
+		CustomUnpacker.register(target, tmpl);
+	}
 
-	//public static void registerUnpacker(Class target, MessageUnpacker unpacker); // TODO
+	public static void registerPacker(Class target, MessagePacker packer) {
+		CustomPacker.register(target, packer);
+	}
+
+	public static void registerConverter(Class target, MessageConverter converter) {
+		CustomConverter.register(target, converter);
+	}
+
+	public static void registerUnpacker(Class target, MessageUnpacker unpacker) {
+		CustomUnpacker.register(target, unpacker);
+	}
 }
 
diff --git a/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
index b91615b..f2aad15 100644
--- a/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
+++ b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
@@ -12,23 +12,51 @@ import org.junit.Test;
 import junit.framework.TestCase;
 
 public class TestMessagePackStaticMethods extends TestCase {
+	public static class CustomClass {
+		public boolean bool;
+		public String str;
+		public List list;
+
+		public boolean equals(Object obj) {
+			if (obj == this) {
+				return true;
+			}
+			if (!(obj instanceof CustomClass)) {
+				return false;
+			}
+			CustomClass o = (CustomClass)obj;
+			return bool == o.bool && str.equals(o.str) && list.equals(o.list);
+		}
+
+		public String toString() {
+			return "CustomClass";
+		}
+	}
+
+	static {
+		MessagePack.register(CustomClass.class);
+	}
+
 	@Test
 	public void testPackToByteArray() throws Exception {
 		byte[] a = MessagePack.pack("msgpack");
 		byte[] b = MessagePack.pack((Object)1);
 		byte[] c = MessagePack.pack((Object)null);
 		byte[] d = MessagePack.pack(createStringList());
+		byte[] e = MessagePack.pack(createCustomClass());
 
 		{
 			MessagePackObject aobj = MessagePack.unpack(a);
 			MessagePackObject bobj = MessagePack.unpack(b);
 			MessagePackObject cobj = MessagePack.unpack(c);
 			MessagePackObject dobj = MessagePack.unpack(d);
+			MessagePackObject eobj = MessagePack.unpack(e);
 
 			assertEquals(aobj, RawType.create("msgpack"));
 			assertEquals(bobj, IntegerType.create(1));
 			assertEquals(cobj, NilType.create());
 			assertEquals(dobj, createStringList_dynamic());
+			assertEquals(eobj, createCustomClass_dynamic());
 		}
 	}
 
@@ -42,6 +70,8 @@ public class TestMessagePackStaticMethods extends TestCase {
 		MessagePack.pack(cout, (Object)null);
 		ByteArrayOutputStream dout = new ByteArrayOutputStream();
 		MessagePack.pack(dout, createStringList());
+		ByteArrayOutputStream eout = new ByteArrayOutputStream();
+		MessagePack.pack(eout, createCustomClass());
 
 		{
 			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
@@ -52,11 +82,14 @@ public class TestMessagePackStaticMethods extends TestCase {
 			MessagePackObject cobj = MessagePack.unpack(cin);
 			InputStream din = new ByteArrayInputStream(dout.toByteArray());
 			MessagePackObject dobj = MessagePack.unpack(din);
+			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
+			MessagePackObject eobj = MessagePack.unpack(ein);
 
 			assertEquals(aobj, RawType.create("msgpack"));
 			assertEquals(bobj, IntegerType.create(1));
 			assertEquals(cobj, NilType.create());
 			assertEquals(dobj, createStringList_dynamic());
+			assertEquals(eobj, createCustomClass_dynamic());
 		}
 	}
 
@@ -66,6 +99,7 @@ public class TestMessagePackStaticMethods extends TestCase {
 		byte[] b = MessagePack.pack((Object)1, TInteger);
 		byte[] c = MessagePack.pack((Object)null, TAny);
 		byte[] d = MessagePack.pack(createStringList(), tList(TString));
+		byte[] e = MessagePack.pack(createCustomClass(), tClass(CustomClass.class));
 
 		{
 			Object aobj = MessagePack.unpack(a, TString);
@@ -73,20 +107,26 @@ public class TestMessagePackStaticMethods extends TestCase {
 			Object cobj_any = MessagePack.unpack(c, TAny);
 			Object cobj_obj = MessagePack.unpack(c, tOptional(TAny));
 			Object dobj = MessagePack.unpack(d, tList(TString));
+			Object eobj = MessagePack.unpack(e, tClass(CustomClass.class));
+
 			assertEquals(aobj, "msgpack");
 			assertEquals(bobj, 1);
 			assertEquals(cobj_any, NilType.create());
 			assertEquals(cobj_obj, null);
 			assertEquals(dobj, createStringList());
+			assertEquals(eobj, createCustomClass());
 		}
 
 		{
 			String  aobj = MessagePack.unpack(a, String.class);
 			Integer bobj = MessagePack.unpack(b, Integer.class);
 			Object  cobj = MessagePack.unpack(c, Object.class);
+			CustomClass eobj = MessagePack.unpack(e, CustomClass.class);
+
 			assertEquals(aobj, "msgpack");
 			assertEquals(bobj, (Integer)1);
 			assertEquals(cobj, null);
+			assertEquals(eobj, createCustomClass());
 		}
 	}
 
@@ -100,6 +140,8 @@ public class TestMessagePackStaticMethods extends TestCase {
 		MessagePack.pack(cout, (Object)null);
 		ByteArrayOutputStream dout = new ByteArrayOutputStream();
 		MessagePack.pack(dout, createStringList());
+		ByteArrayOutputStream eout = new ByteArrayOutputStream();
+		MessagePack.pack(eout, createCustomClass());
 
 		{
 			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
@@ -112,12 +154,15 @@ public class TestMessagePackStaticMethods extends TestCase {
 			Object cobj_obj = MessagePack.unpack(cin_obj, tOptional(TAny));
 			InputStream din = new ByteArrayInputStream(dout.toByteArray());
 			Object dobj = MessagePack.unpack(din, tList(TString));
+			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
+			Object eobj = MessagePack.unpack(ein, tClass(CustomClass.class));
 
 			assertEquals(aobj, "msgpack");
 			assertEquals(bobj, 1);
 			assertEquals(cobj_any, NilType.create());
 			assertEquals(cobj_obj, null);
 			assertEquals(dobj, createStringList());
+			assertEquals(eobj, createCustomClass());
 		}
 
 		{
@@ -127,13 +172,17 @@ public class TestMessagePackStaticMethods extends TestCase {
 			Integer bobj = MessagePack.unpack(bin, Integer.class);
 			InputStream cin = new ByteArrayInputStream(cout.toByteArray());
 			Object  cobj = MessagePack.unpack(cin, Object.class);
+			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
+			Object  eobj = MessagePack.unpack(ein, CustomClass.class);
 
 			assertEquals(aobj, "msgpack");
 			assertEquals(bobj, (Integer)1);
 			assertEquals(cobj, null);
+			assertEquals(eobj, createCustomClass());
 		}
 	}
 
+
 	private List createStringList() {
 		List list = new ArrayList();
 		list.add("frsyuki");
@@ -149,5 +198,29 @@ public class TestMessagePackStaticMethods extends TestCase {
 		array[2] = RawType.create("gem-compile");
 		return ArrayType.create(array);
 	}
+
+
+	private CustomClass createCustomClass() {
+		CustomClass obj = new CustomClass();
+		obj.bool = true;
+		obj.str = "viver";
+		obj.list = new ArrayList();
+		obj.list.add(1);
+		obj.list.add(2);
+		obj.list.add(3);
+		return obj;
+	}
+
+	private MessagePackObject createCustomClass_dynamic() {
+		MessagePackObject[] obj = new MessagePackObject[3];
+		obj[0] = BooleanType.create(true);
+		obj[1] = RawType.create("viver");
+		MessagePackObject[] list = new MessagePackObject[3];
+		list[0] = IntegerType.create(1);
+		list[1] = IntegerType.create(2);
+		list[2] = IntegerType.create(3);
+		obj[2] = ArrayType.create(list);
+		return ArrayType.create(obj);
+	}
 }
 

From 1b3231e6174e3ea908a5fd3ae1437e43b1a2c320 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 24 Oct 2010 20:59:19 +0900
Subject: [PATCH 0903/1648] java: improves test case of the MessagePack class

---
 .../msgpack/TestMessagePackStaticMethods.java | 114 ++++++++++++++----
 1 file changed, 92 insertions(+), 22 deletions(-)

diff --git a/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
index f2aad15..5d97953 100644
--- a/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
+++ b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
@@ -2,6 +2,7 @@ package org.msgpack;
 
 import org.msgpack.*;
 import org.msgpack.object.*;
+import org.msgpack.annotation.*;
 import static org.msgpack.Templates.*;
 
 import java.io.*;
@@ -12,7 +13,7 @@ import org.junit.Test;
 import junit.framework.TestCase;
 
 public class TestMessagePackStaticMethods extends TestCase {
-	public static class CustomClass {
+	public static class ProvidedClass {
 		public boolean bool;
 		public String str;
 		public List list;
@@ -21,20 +22,44 @@ public class TestMessagePackStaticMethods extends TestCase {
 			if (obj == this) {
 				return true;
 			}
-			if (!(obj instanceof CustomClass)) {
+			if (!(obj instanceof ProvidedClass)) {
 				return false;
 			}
-			CustomClass o = (CustomClass)obj;
+			ProvidedClass o = (ProvidedClass)obj;
 			return bool == o.bool && str.equals(o.str) && list.equals(o.list);
 		}
 
 		public String toString() {
-			return "CustomClass";
+			return "ProvidedClass";
+		}
+	}
+
+	@MessagePackMessage
+	public static class UserDefinedClass {
+		public boolean bool;
+		public String str;
+		public List list;
+
+		public boolean equals(Object obj) {
+			if (obj == this) {
+				return true;
+			}
+			if (!(obj instanceof UserDefinedClass)) {
+				return false;
+			}
+			UserDefinedClass o = (UserDefinedClass)obj;
+			return bool == o.bool && str.equals(o.str) && list.equals(o.list);
+		}
+
+		public String toString() {
+			return "UserDefinedClass";
 		}
 	}
 
 	static {
-		MessagePack.register(CustomClass.class);
+		// provided classes needs registration
+		MessagePack.register(ProvidedClass.class);
+		// annotated classes doesn't need registration
 	}
 
 	@Test
@@ -43,7 +68,8 @@ public class TestMessagePackStaticMethods extends TestCase {
 		byte[] b = MessagePack.pack((Object)1);
 		byte[] c = MessagePack.pack((Object)null);
 		byte[] d = MessagePack.pack(createStringList());
-		byte[] e = MessagePack.pack(createCustomClass());
+		byte[] e = MessagePack.pack(createProvidedClass());
+		byte[] f = MessagePack.pack(createUserDefinedClass_dynamic());
 
 		{
 			MessagePackObject aobj = MessagePack.unpack(a);
@@ -51,12 +77,14 @@ public class TestMessagePackStaticMethods extends TestCase {
 			MessagePackObject cobj = MessagePack.unpack(c);
 			MessagePackObject dobj = MessagePack.unpack(d);
 			MessagePackObject eobj = MessagePack.unpack(e);
+			MessagePackObject fobj = MessagePack.unpack(f);
 
 			assertEquals(aobj, RawType.create("msgpack"));
 			assertEquals(bobj, IntegerType.create(1));
 			assertEquals(cobj, NilType.create());
 			assertEquals(dobj, createStringList_dynamic());
-			assertEquals(eobj, createCustomClass_dynamic());
+			assertEquals(eobj, createProvidedClass_dynamic());
+			assertEquals(fobj, createUserDefinedClass_dynamic());
 		}
 	}
 
@@ -71,7 +99,9 @@ public class TestMessagePackStaticMethods extends TestCase {
 		ByteArrayOutputStream dout = new ByteArrayOutputStream();
 		MessagePack.pack(dout, createStringList());
 		ByteArrayOutputStream eout = new ByteArrayOutputStream();
-		MessagePack.pack(eout, createCustomClass());
+		MessagePack.pack(eout, createProvidedClass());
+		ByteArrayOutputStream fout = new ByteArrayOutputStream();
+		MessagePack.pack(fout, createUserDefinedClass());
 
 		{
 			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
@@ -84,12 +114,15 @@ public class TestMessagePackStaticMethods extends TestCase {
 			MessagePackObject dobj = MessagePack.unpack(din);
 			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
 			MessagePackObject eobj = MessagePack.unpack(ein);
+			InputStream fin = new ByteArrayInputStream(fout.toByteArray());
+			MessagePackObject fobj = MessagePack.unpack(fin);
 
 			assertEquals(aobj, RawType.create("msgpack"));
 			assertEquals(bobj, IntegerType.create(1));
 			assertEquals(cobj, NilType.create());
 			assertEquals(dobj, createStringList_dynamic());
-			assertEquals(eobj, createCustomClass_dynamic());
+			assertEquals(eobj, createProvidedClass_dynamic());
+			assertEquals(fobj, createUserDefinedClass_dynamic());
 		}
 	}
 
@@ -99,7 +132,8 @@ public class TestMessagePackStaticMethods extends TestCase {
 		byte[] b = MessagePack.pack((Object)1, TInteger);
 		byte[] c = MessagePack.pack((Object)null, TAny);
 		byte[] d = MessagePack.pack(createStringList(), tList(TString));
-		byte[] e = MessagePack.pack(createCustomClass(), tClass(CustomClass.class));
+		byte[] e = MessagePack.pack(createProvidedClass(), tClass(ProvidedClass.class));
+		byte[] f = MessagePack.pack(createUserDefinedClass(), tClass(UserDefinedClass.class));
 
 		{
 			Object aobj = MessagePack.unpack(a, TString);
@@ -107,26 +141,30 @@ public class TestMessagePackStaticMethods extends TestCase {
 			Object cobj_any = MessagePack.unpack(c, TAny);
 			Object cobj_obj = MessagePack.unpack(c, tOptional(TAny));
 			Object dobj = MessagePack.unpack(d, tList(TString));
-			Object eobj = MessagePack.unpack(e, tClass(CustomClass.class));
+			Object eobj = MessagePack.unpack(e, tClass(ProvidedClass.class));
+			Object fobj = MessagePack.unpack(f, tClass(UserDefinedClass.class));
 
 			assertEquals(aobj, "msgpack");
 			assertEquals(bobj, 1);
 			assertEquals(cobj_any, NilType.create());
 			assertEquals(cobj_obj, null);
 			assertEquals(dobj, createStringList());
-			assertEquals(eobj, createCustomClass());
+			assertEquals(eobj, createProvidedClass());
+			assertEquals(fobj, createUserDefinedClass());
 		}
 
 		{
 			String  aobj = MessagePack.unpack(a, String.class);
 			Integer bobj = MessagePack.unpack(b, Integer.class);
 			Object  cobj = MessagePack.unpack(c, Object.class);
-			CustomClass eobj = MessagePack.unpack(e, CustomClass.class);
+			ProvidedClass eobj = MessagePack.unpack(e, ProvidedClass.class);
+			UserDefinedClass fobj = MessagePack.unpack(f, UserDefinedClass.class);
 
 			assertEquals(aobj, "msgpack");
 			assertEquals(bobj, (Integer)1);
 			assertEquals(cobj, null);
-			assertEquals(eobj, createCustomClass());
+			assertEquals(eobj, createProvidedClass());
+			assertEquals(fobj, createUserDefinedClass());
 		}
 	}
 
@@ -141,7 +179,9 @@ public class TestMessagePackStaticMethods extends TestCase {
 		ByteArrayOutputStream dout = new ByteArrayOutputStream();
 		MessagePack.pack(dout, createStringList());
 		ByteArrayOutputStream eout = new ByteArrayOutputStream();
-		MessagePack.pack(eout, createCustomClass());
+		MessagePack.pack(eout, createProvidedClass());
+		ByteArrayOutputStream fout = new ByteArrayOutputStream();
+		MessagePack.pack(fout, createUserDefinedClass());
 
 		{
 			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
@@ -155,14 +195,17 @@ public class TestMessagePackStaticMethods extends TestCase {
 			InputStream din = new ByteArrayInputStream(dout.toByteArray());
 			Object dobj = MessagePack.unpack(din, tList(TString));
 			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
-			Object eobj = MessagePack.unpack(ein, tClass(CustomClass.class));
+			Object eobj = MessagePack.unpack(ein, tClass(ProvidedClass.class));
+			InputStream fin = new ByteArrayInputStream(fout.toByteArray());
+			Object fobj = MessagePack.unpack(fin, tClass(UserDefinedClass.class));
 
 			assertEquals(aobj, "msgpack");
 			assertEquals(bobj, 1);
 			assertEquals(cobj_any, NilType.create());
 			assertEquals(cobj_obj, null);
 			assertEquals(dobj, createStringList());
-			assertEquals(eobj, createCustomClass());
+			assertEquals(eobj, createProvidedClass());
+			assertEquals(fobj, createUserDefinedClass());
 		}
 
 		{
@@ -173,12 +216,15 @@ public class TestMessagePackStaticMethods extends TestCase {
 			InputStream cin = new ByteArrayInputStream(cout.toByteArray());
 			Object  cobj = MessagePack.unpack(cin, Object.class);
 			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
-			Object  eobj = MessagePack.unpack(ein, CustomClass.class);
+			ProvidedClass eobj = MessagePack.unpack(ein, ProvidedClass.class);
+			InputStream fin = new ByteArrayInputStream(fout.toByteArray());
+			UserDefinedClass fobj = MessagePack.unpack(fin, UserDefinedClass.class);
 
 			assertEquals(aobj, "msgpack");
 			assertEquals(bobj, (Integer)1);
 			assertEquals(cobj, null);
-			assertEquals(eobj, createCustomClass());
+			assertEquals(eobj, createProvidedClass());
+			assertEquals(fobj, createUserDefinedClass());
 		}
 	}
 
@@ -200,8 +246,8 @@ public class TestMessagePackStaticMethods extends TestCase {
 	}
 
 
-	private CustomClass createCustomClass() {
-		CustomClass obj = new CustomClass();
+	private ProvidedClass createProvidedClass() {
+		ProvidedClass obj = new ProvidedClass();
 		obj.bool = true;
 		obj.str = "viver";
 		obj.list = new ArrayList();
@@ -211,7 +257,7 @@ public class TestMessagePackStaticMethods extends TestCase {
 		return obj;
 	}
 
-	private MessagePackObject createCustomClass_dynamic() {
+	private MessagePackObject createProvidedClass_dynamic() {
 		MessagePackObject[] obj = new MessagePackObject[3];
 		obj[0] = BooleanType.create(true);
 		obj[1] = RawType.create("viver");
@@ -222,5 +268,29 @@ public class TestMessagePackStaticMethods extends TestCase {
 		obj[2] = ArrayType.create(list);
 		return ArrayType.create(obj);
 	}
+
+
+	private UserDefinedClass createUserDefinedClass() {
+		UserDefinedClass obj = new UserDefinedClass();
+		obj.bool = false;
+		obj.str = "muga";
+		obj.list = new ArrayList();
+		obj.list.add(9);
+		obj.list.add(10);
+		obj.list.add(11);
+		return obj;
+	}
+
+	private MessagePackObject createUserDefinedClass_dynamic() {
+		MessagePackObject[] obj = new MessagePackObject[3];
+		obj[0] = BooleanType.create(false);
+		obj[1] = RawType.create("muga");
+		MessagePackObject[] list = new MessagePackObject[3];
+		list[0] = IntegerType.create(9);
+		list[1] = IntegerType.create(10);
+		list[2] = IntegerType.create(11);
+		obj[2] = ArrayType.create(list);
+		return ArrayType.create(obj);
+	}
 }
 

From 147056073d71a845fdbcd7562db344e9eb0c69c9 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 24 Oct 2010 21:04:31 +0900
Subject: [PATCH 0904/1648] java: improves test case of the MessagePack class

---
 .../msgpack/TestMessagePackStaticMethods.java | 209 +++++++++---------
 1 file changed, 105 insertions(+), 104 deletions(-)

diff --git a/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
index 5d97953..e7b8d84 100644
--- a/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
+++ b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
@@ -57,9 +57,112 @@ public class TestMessagePackStaticMethods extends TestCase {
 	}
 
 	static {
-		// provided classes needs registration
+		// provided classes need registration
 		MessagePack.register(ProvidedClass.class);
-		// annotated classes doesn't need registration
+		// annotated classes don't need registration
+	}
+
+
+	@Test
+	public void testCheckedPackToByteArray() throws Exception {
+		byte[] a = MessagePack.pack("msgpack", TString);
+		byte[] b = MessagePack.pack((Object)1, TInteger);
+		byte[] c = MessagePack.pack((Object)null, TAny);
+		byte[] d = MessagePack.pack(createStringList(), tList(TString));
+		byte[] e = MessagePack.pack(createProvidedClass(), tClass(ProvidedClass.class));
+		byte[] f = MessagePack.pack(createUserDefinedClass(), tClass(UserDefinedClass.class));
+
+		{
+			String  aobj = MessagePack.unpack(a, String.class);
+			Integer bobj = MessagePack.unpack(b, Integer.class);
+			Object  cobj = MessagePack.unpack(c, Object.class);
+			ProvidedClass eobj = MessagePack.unpack(e, ProvidedClass.class);
+			UserDefinedClass fobj = MessagePack.unpack(f, UserDefinedClass.class);
+
+			assertEquals(aobj, "msgpack");
+			assertEquals(bobj, (Integer)1);
+			assertEquals(cobj, null);
+			assertEquals(eobj, createProvidedClass());
+			assertEquals(fobj, createUserDefinedClass());
+		}
+
+		{
+			Object aobj = MessagePack.unpack(a, TString);
+			Object bobj = MessagePack.unpack(b, TInteger);
+			Object cobj_any = MessagePack.unpack(c, TAny);
+			Object cobj_obj = MessagePack.unpack(c, tOptional(TAny));
+			Object dobj = MessagePack.unpack(d, tList(TString));
+			Object eobj = MessagePack.unpack(e, tClass(ProvidedClass.class));
+			Object fobj = MessagePack.unpack(f, tClass(UserDefinedClass.class));
+
+			assertEquals(aobj, "msgpack");
+			assertEquals(bobj, 1);
+			assertEquals(cobj_any, NilType.create());
+			assertEquals(cobj_obj, null);
+			assertEquals(dobj, createStringList());
+			assertEquals(eobj, createProvidedClass());
+			assertEquals(fobj, createUserDefinedClass());
+		}
+	}
+
+	@Test
+	public void testCheckedPackToStream() throws Exception {
+		ByteArrayOutputStream aout = new ByteArrayOutputStream();
+		MessagePack.pack(aout, "msgpack");
+		ByteArrayOutputStream bout = new ByteArrayOutputStream();
+		MessagePack.pack(bout, (Object)1);
+		ByteArrayOutputStream cout = new ByteArrayOutputStream();
+		MessagePack.pack(cout, (Object)null);
+		ByteArrayOutputStream dout = new ByteArrayOutputStream();
+		MessagePack.pack(dout, createStringList());
+		ByteArrayOutputStream eout = new ByteArrayOutputStream();
+		MessagePack.pack(eout, createProvidedClass());
+		ByteArrayOutputStream fout = new ByteArrayOutputStream();
+		MessagePack.pack(fout, createUserDefinedClass());
+
+		{
+			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
+			String  aobj = MessagePack.unpack(ain, String.class);
+			InputStream bin = new ByteArrayInputStream(bout.toByteArray());
+			Integer bobj = MessagePack.unpack(bin, Integer.class);
+			InputStream cin = new ByteArrayInputStream(cout.toByteArray());
+			Object  cobj = MessagePack.unpack(cin, Object.class);
+			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
+			ProvidedClass eobj = MessagePack.unpack(ein, ProvidedClass.class);
+			InputStream fin = new ByteArrayInputStream(fout.toByteArray());
+			UserDefinedClass fobj = MessagePack.unpack(fin, UserDefinedClass.class);
+
+			assertEquals(aobj, "msgpack");
+			assertEquals(bobj, (Integer)1);
+			assertEquals(cobj, null);
+			assertEquals(eobj, createProvidedClass());
+			assertEquals(fobj, createUserDefinedClass());
+		}
+
+		{
+			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
+			Object aobj = MessagePack.unpack(ain, TString);
+			InputStream bin = new ByteArrayInputStream(bout.toByteArray());
+			Object bobj = MessagePack.unpack(bin, TInteger);
+			InputStream cin_any = new ByteArrayInputStream(cout.toByteArray());
+			Object cobj_any = MessagePack.unpack(cin_any, TAny);
+			InputStream cin_obj = new ByteArrayInputStream(cout.toByteArray());
+			Object cobj_obj = MessagePack.unpack(cin_obj, tOptional(TAny));
+			InputStream din = new ByteArrayInputStream(dout.toByteArray());
+			Object dobj = MessagePack.unpack(din, tList(TString));
+			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
+			Object eobj = MessagePack.unpack(ein, tClass(ProvidedClass.class));
+			InputStream fin = new ByteArrayInputStream(fout.toByteArray());
+			Object fobj = MessagePack.unpack(fin, tClass(UserDefinedClass.class));
+
+			assertEquals(aobj, "msgpack");
+			assertEquals(bobj, 1);
+			assertEquals(cobj_any, NilType.create());
+			assertEquals(cobj_obj, null);
+			assertEquals(dobj, createStringList());
+			assertEquals(eobj, createProvidedClass());
+			assertEquals(fobj, createUserDefinedClass());
+		}
 	}
 
 	@Test
@@ -126,108 +229,6 @@ public class TestMessagePackStaticMethods extends TestCase {
 		}
 	}
 
-	@Test
-	public void testCheckedPackToByteArray() throws Exception {
-		byte[] a = MessagePack.pack("msgpack", TString);
-		byte[] b = MessagePack.pack((Object)1, TInteger);
-		byte[] c = MessagePack.pack((Object)null, TAny);
-		byte[] d = MessagePack.pack(createStringList(), tList(TString));
-		byte[] e = MessagePack.pack(createProvidedClass(), tClass(ProvidedClass.class));
-		byte[] f = MessagePack.pack(createUserDefinedClass(), tClass(UserDefinedClass.class));
-
-		{
-			Object aobj = MessagePack.unpack(a, TString);
-			Object bobj = MessagePack.unpack(b, TInteger);
-			Object cobj_any = MessagePack.unpack(c, TAny);
-			Object cobj_obj = MessagePack.unpack(c, tOptional(TAny));
-			Object dobj = MessagePack.unpack(d, tList(TString));
-			Object eobj = MessagePack.unpack(e, tClass(ProvidedClass.class));
-			Object fobj = MessagePack.unpack(f, tClass(UserDefinedClass.class));
-
-			assertEquals(aobj, "msgpack");
-			assertEquals(bobj, 1);
-			assertEquals(cobj_any, NilType.create());
-			assertEquals(cobj_obj, null);
-			assertEquals(dobj, createStringList());
-			assertEquals(eobj, createProvidedClass());
-			assertEquals(fobj, createUserDefinedClass());
-		}
-
-		{
-			String  aobj = MessagePack.unpack(a, String.class);
-			Integer bobj = MessagePack.unpack(b, Integer.class);
-			Object  cobj = MessagePack.unpack(c, Object.class);
-			ProvidedClass eobj = MessagePack.unpack(e, ProvidedClass.class);
-			UserDefinedClass fobj = MessagePack.unpack(f, UserDefinedClass.class);
-
-			assertEquals(aobj, "msgpack");
-			assertEquals(bobj, (Integer)1);
-			assertEquals(cobj, null);
-			assertEquals(eobj, createProvidedClass());
-			assertEquals(fobj, createUserDefinedClass());
-		}
-	}
-
-	@Test
-	public void testCheckedPackToStream() throws Exception {
-		ByteArrayOutputStream aout = new ByteArrayOutputStream();
-		MessagePack.pack(aout, "msgpack");
-		ByteArrayOutputStream bout = new ByteArrayOutputStream();
-		MessagePack.pack(bout, (Object)1);
-		ByteArrayOutputStream cout = new ByteArrayOutputStream();
-		MessagePack.pack(cout, (Object)null);
-		ByteArrayOutputStream dout = new ByteArrayOutputStream();
-		MessagePack.pack(dout, createStringList());
-		ByteArrayOutputStream eout = new ByteArrayOutputStream();
-		MessagePack.pack(eout, createProvidedClass());
-		ByteArrayOutputStream fout = new ByteArrayOutputStream();
-		MessagePack.pack(fout, createUserDefinedClass());
-
-		{
-			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
-			Object aobj = MessagePack.unpack(ain, TString);
-			InputStream bin = new ByteArrayInputStream(bout.toByteArray());
-			Object bobj = MessagePack.unpack(bin, TInteger);
-			InputStream cin_any = new ByteArrayInputStream(cout.toByteArray());
-			Object cobj_any = MessagePack.unpack(cin_any, TAny);
-			InputStream cin_obj = new ByteArrayInputStream(cout.toByteArray());
-			Object cobj_obj = MessagePack.unpack(cin_obj, tOptional(TAny));
-			InputStream din = new ByteArrayInputStream(dout.toByteArray());
-			Object dobj = MessagePack.unpack(din, tList(TString));
-			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
-			Object eobj = MessagePack.unpack(ein, tClass(ProvidedClass.class));
-			InputStream fin = new ByteArrayInputStream(fout.toByteArray());
-			Object fobj = MessagePack.unpack(fin, tClass(UserDefinedClass.class));
-
-			assertEquals(aobj, "msgpack");
-			assertEquals(bobj, 1);
-			assertEquals(cobj_any, NilType.create());
-			assertEquals(cobj_obj, null);
-			assertEquals(dobj, createStringList());
-			assertEquals(eobj, createProvidedClass());
-			assertEquals(fobj, createUserDefinedClass());
-		}
-
-		{
-			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
-			String  aobj = MessagePack.unpack(ain, String.class);
-			InputStream bin = new ByteArrayInputStream(bout.toByteArray());
-			Integer bobj = MessagePack.unpack(bin, Integer.class);
-			InputStream cin = new ByteArrayInputStream(cout.toByteArray());
-			Object  cobj = MessagePack.unpack(cin, Object.class);
-			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
-			ProvidedClass eobj = MessagePack.unpack(ein, ProvidedClass.class);
-			InputStream fin = new ByteArrayInputStream(fout.toByteArray());
-			UserDefinedClass fobj = MessagePack.unpack(fin, UserDefinedClass.class);
-
-			assertEquals(aobj, "msgpack");
-			assertEquals(bobj, (Integer)1);
-			assertEquals(cobj, null);
-			assertEquals(eobj, createProvidedClass());
-			assertEquals(fobj, createUserDefinedClass());
-		}
-	}
-
 
 	private List createStringList() {
 		List list = new ArrayList();

From 1bd347d9970ba8d00b757ece940b8a32564821e4 Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Sun, 24 Oct 2010 21:17:19 +0900
Subject: [PATCH 0905/1648] java: fixes CustomMessage class

---
 .../main/java/org/msgpack/CustomMessage.java  | 13 +++-
 .../org/msgpack/template/AnyTemplate.java     |  2 +-
 .../msgpack/template/BigIntegerTemplate.java  |  2 +-
 .../org/msgpack/template/BooleanTemplate.java |  2 +-
 .../msgpack/template/ByteArrayTemplate.java   |  2 +-
 .../org/msgpack/template/ByteTemplate.java    |  2 +-
 .../org/msgpack/template/DoubleTemplate.java  |  2 +-
 .../org/msgpack/template/FloatTemplate.java   |  2 +-
 .../org/msgpack/template/IntegerTemplate.java |  2 +-
 .../org/msgpack/template/LongTemplate.java    |  2 +-
 .../org/msgpack/template/ShortTemplate.java   |  2 +-
 .../org/msgpack/template/StringTemplate.java  |  2 +-
 .../util/codegen/DynamicCodeGenBase.java      |  6 +-
 .../msgpack/TestMessagePackStaticMethods.java | 66 +++++++++----------
 14 files changed, 58 insertions(+), 49 deletions(-)

diff --git a/java/src/main/java/org/msgpack/CustomMessage.java b/java/src/main/java/org/msgpack/CustomMessage.java
index 959c9cd..832aa59 100644
--- a/java/src/main/java/org/msgpack/CustomMessage.java
+++ b/java/src/main/java/org/msgpack/CustomMessage.java
@@ -24,9 +24,18 @@ public class CustomMessage {
 		CustomPacker.register(target, packer);
 	}
 
-	public static void registerTemplate(Class target, Template tmpl) {
-		CustomUnpacker.register(target, tmpl);
+	public static void registerConverter(Class target, MessageConverter converter) {
+		CustomConverter.register(target, converter);
+	}
+
+	public static void registerUnpacker(Class target, MessageUnpacker unpacker) {
+		CustomUnpacker.register(target, unpacker);
+	}
+
+	public static void register(Class target, Template tmpl) {
+		CustomPacker.register(target, tmpl);
 		CustomConverter.register(target, tmpl);
+		CustomUnpacker.register(target, tmpl);
 	}
 
 	public static boolean isAnnotated(Class target, Class with) {
diff --git a/java/src/main/java/org/msgpack/template/AnyTemplate.java b/java/src/main/java/org/msgpack/template/AnyTemplate.java
index fb2e1ee..91eab90 100644
--- a/java/src/main/java/org/msgpack/template/AnyTemplate.java
+++ b/java/src/main/java/org/msgpack/template/AnyTemplate.java
@@ -46,7 +46,7 @@ public class AnyTemplate implements Template {
 	static final AnyTemplate instance = new AnyTemplate();
 
 	static {
-		CustomMessage.registerTemplate(MessagePackObject.class, instance);
+		CustomMessage.register(MessagePackObject.class, instance);
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/BigIntegerTemplate.java b/java/src/main/java/org/msgpack/template/BigIntegerTemplate.java
index 66e142b..79b5c7d 100644
--- a/java/src/main/java/org/msgpack/template/BigIntegerTemplate.java
+++ b/java/src/main/java/org/msgpack/template/BigIntegerTemplate.java
@@ -43,7 +43,7 @@ public class BigIntegerTemplate implements Template {
 	static final BigIntegerTemplate instance = new BigIntegerTemplate();
 
 	static {
-		CustomMessage.registerTemplate(BigInteger.class, instance);
+		CustomMessage.register(BigInteger.class, instance);
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/BooleanTemplate.java b/java/src/main/java/org/msgpack/template/BooleanTemplate.java
index e9fd33c..dd3367f 100644
--- a/java/src/main/java/org/msgpack/template/BooleanTemplate.java
+++ b/java/src/main/java/org/msgpack/template/BooleanTemplate.java
@@ -42,7 +42,7 @@ public class BooleanTemplate implements Template {
 	static final BooleanTemplate instance = new BooleanTemplate();
 
 	static {
-		CustomMessage.registerTemplate(Boolean.class, instance);
+		CustomMessage.register(Boolean.class, instance);
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/ByteArrayTemplate.java b/java/src/main/java/org/msgpack/template/ByteArrayTemplate.java
index 45a006f..2008b7c 100644
--- a/java/src/main/java/org/msgpack/template/ByteArrayTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ByteArrayTemplate.java
@@ -42,7 +42,7 @@ public class ByteArrayTemplate implements Template {
 	static final ByteArrayTemplate instance = new ByteArrayTemplate();
 
 	static {
-		CustomMessage.registerTemplate(byte[].class, instance);
+		CustomMessage.register(byte[].class, instance);
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/ByteTemplate.java b/java/src/main/java/org/msgpack/template/ByteTemplate.java
index d58e8b0..0c8a31b 100644
--- a/java/src/main/java/org/msgpack/template/ByteTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ByteTemplate.java
@@ -42,7 +42,7 @@ public class ByteTemplate implements Template {
 	static final ByteTemplate instance = new ByteTemplate();
 
 	static {
-		CustomMessage.registerTemplate(Byte.class, instance);
+		CustomMessage.register(Byte.class, instance);
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/DoubleTemplate.java b/java/src/main/java/org/msgpack/template/DoubleTemplate.java
index c899e73..94550eb 100644
--- a/java/src/main/java/org/msgpack/template/DoubleTemplate.java
+++ b/java/src/main/java/org/msgpack/template/DoubleTemplate.java
@@ -42,7 +42,7 @@ public class DoubleTemplate implements Template {
 	static final DoubleTemplate instance = new DoubleTemplate();
 
 	static {
-		CustomMessage.registerTemplate(Double.class, instance);
+		CustomMessage.register(Double.class, instance);
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/FloatTemplate.java b/java/src/main/java/org/msgpack/template/FloatTemplate.java
index 354e3f8..c247e29 100644
--- a/java/src/main/java/org/msgpack/template/FloatTemplate.java
+++ b/java/src/main/java/org/msgpack/template/FloatTemplate.java
@@ -42,7 +42,7 @@ public class FloatTemplate implements Template {
 	static final FloatTemplate instance = new FloatTemplate();
 
 	static {
-		CustomMessage.registerTemplate(Float.class, instance);
+		CustomMessage.register(Float.class, instance);
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/IntegerTemplate.java b/java/src/main/java/org/msgpack/template/IntegerTemplate.java
index 21ff0af..2dee8e0 100644
--- a/java/src/main/java/org/msgpack/template/IntegerTemplate.java
+++ b/java/src/main/java/org/msgpack/template/IntegerTemplate.java
@@ -42,7 +42,7 @@ public class IntegerTemplate implements Template {
 	static final IntegerTemplate instance = new IntegerTemplate();
 
 	static {
-		CustomMessage.registerTemplate(Integer.class, instance);
+		CustomMessage.register(Integer.class, instance);
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/LongTemplate.java b/java/src/main/java/org/msgpack/template/LongTemplate.java
index d8990af..930b7d0 100644
--- a/java/src/main/java/org/msgpack/template/LongTemplate.java
+++ b/java/src/main/java/org/msgpack/template/LongTemplate.java
@@ -42,7 +42,7 @@ public class LongTemplate implements Template {
 	static final LongTemplate instance = new LongTemplate();
 
 	static {
-		CustomMessage.registerTemplate(Long.class, instance);
+		CustomMessage.register(Long.class, instance);
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/ShortTemplate.java b/java/src/main/java/org/msgpack/template/ShortTemplate.java
index 0268797..10ac43b 100644
--- a/java/src/main/java/org/msgpack/template/ShortTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ShortTemplate.java
@@ -42,7 +42,7 @@ public class ShortTemplate implements Template {
 	static final ShortTemplate instance = new ShortTemplate();
 
 	static {
-		CustomMessage.registerTemplate(Short.class, instance);
+		CustomMessage.register(Short.class, instance);
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/template/StringTemplate.java b/java/src/main/java/org/msgpack/template/StringTemplate.java
index 4a4be3e..dd31d9e 100644
--- a/java/src/main/java/org/msgpack/template/StringTemplate.java
+++ b/java/src/main/java/org/msgpack/template/StringTemplate.java
@@ -42,7 +42,7 @@ public class StringTemplate implements Template {
 	static final StringTemplate instance = new StringTemplate();
 
 	static {
-		CustomMessage.registerTemplate(String.class, instance);
+		CustomMessage.register(String.class, instance);
 	}
 }
 
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index c80d354..c8ab3c7 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -581,7 +581,7 @@ public class DynamicCodeGenBase implements Constants {
 			} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
 				// @MessagePackMessage
 				Template tmpl = DynamicTemplate.create(c);
-				CustomMessage.registerTemplate(c, tmpl);
+				CustomMessage.register(c, tmpl);
 				return tmpl;
 			} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
 				// FIXME DelegatePacker
@@ -593,12 +593,12 @@ public class DynamicCodeGenBase implements Constants {
 					MessagePackOrdinalEnum.class)) {
 				// @MessagePackOrdinalEnum
 				Template tmpl = DynamicOrdinalEnumTemplate.create(c);
-				CustomMessage.registerTemplate(c, tmpl);
+				CustomMessage.register(c, tmpl);
 				return tmpl;
 			} else if (MessageConvertable.class.isAssignableFrom(c)
 					|| MessageUnpackable.class.isAssignableFrom(c)) {
 				Template tmpl = new MessageUnpackableConvertableTemplate(c);
-				CustomMessage.registerTemplate(c, tmpl);
+				CustomMessage.register(c, tmpl);
 				return tmpl;
 			} else {
 				throw new MessageTypeException("Type error: "
diff --git a/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
index e7b8d84..f08176e 100644
--- a/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
+++ b/java/src/test/java/org/msgpack/TestMessagePackStaticMethods.java
@@ -72,20 +72,6 @@ public class TestMessagePackStaticMethods extends TestCase {
 		byte[] e = MessagePack.pack(createProvidedClass(), tClass(ProvidedClass.class));
 		byte[] f = MessagePack.pack(createUserDefinedClass(), tClass(UserDefinedClass.class));
 
-		{
-			String  aobj = MessagePack.unpack(a, String.class);
-			Integer bobj = MessagePack.unpack(b, Integer.class);
-			Object  cobj = MessagePack.unpack(c, Object.class);
-			ProvidedClass eobj = MessagePack.unpack(e, ProvidedClass.class);
-			UserDefinedClass fobj = MessagePack.unpack(f, UserDefinedClass.class);
-
-			assertEquals(aobj, "msgpack");
-			assertEquals(bobj, (Integer)1);
-			assertEquals(cobj, null);
-			assertEquals(eobj, createProvidedClass());
-			assertEquals(fobj, createUserDefinedClass());
-		}
-
 		{
 			Object aobj = MessagePack.unpack(a, TString);
 			Object bobj = MessagePack.unpack(b, TInteger);
@@ -103,6 +89,20 @@ public class TestMessagePackStaticMethods extends TestCase {
 			assertEquals(eobj, createProvidedClass());
 			assertEquals(fobj, createUserDefinedClass());
 		}
+
+		{
+			String  aobj = MessagePack.unpack(a, String.class);
+			Integer bobj = MessagePack.unpack(b, Integer.class);
+			Object  cobj = MessagePack.unpack(c, Object.class);
+			ProvidedClass eobj = MessagePack.unpack(e, ProvidedClass.class);
+			UserDefinedClass fobj = MessagePack.unpack(f, UserDefinedClass.class);
+
+			assertEquals(aobj, "msgpack");
+			assertEquals(bobj, (Integer)1);
+			assertEquals(cobj, null);
+			assertEquals(eobj, createProvidedClass());
+			assertEquals(fobj, createUserDefinedClass());
+		}
 	}
 
 	@Test
@@ -120,25 +120,6 @@ public class TestMessagePackStaticMethods extends TestCase {
 		ByteArrayOutputStream fout = new ByteArrayOutputStream();
 		MessagePack.pack(fout, createUserDefinedClass());
 
-		{
-			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
-			String  aobj = MessagePack.unpack(ain, String.class);
-			InputStream bin = new ByteArrayInputStream(bout.toByteArray());
-			Integer bobj = MessagePack.unpack(bin, Integer.class);
-			InputStream cin = new ByteArrayInputStream(cout.toByteArray());
-			Object  cobj = MessagePack.unpack(cin, Object.class);
-			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
-			ProvidedClass eobj = MessagePack.unpack(ein, ProvidedClass.class);
-			InputStream fin = new ByteArrayInputStream(fout.toByteArray());
-			UserDefinedClass fobj = MessagePack.unpack(fin, UserDefinedClass.class);
-
-			assertEquals(aobj, "msgpack");
-			assertEquals(bobj, (Integer)1);
-			assertEquals(cobj, null);
-			assertEquals(eobj, createProvidedClass());
-			assertEquals(fobj, createUserDefinedClass());
-		}
-
 		{
 			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
 			Object aobj = MessagePack.unpack(ain, TString);
@@ -163,6 +144,25 @@ public class TestMessagePackStaticMethods extends TestCase {
 			assertEquals(eobj, createProvidedClass());
 			assertEquals(fobj, createUserDefinedClass());
 		}
+
+		{
+			InputStream ain = new ByteArrayInputStream(aout.toByteArray());
+			String  aobj = MessagePack.unpack(ain, String.class);
+			InputStream bin = new ByteArrayInputStream(bout.toByteArray());
+			Integer bobj = MessagePack.unpack(bin, Integer.class);
+			InputStream cin = new ByteArrayInputStream(cout.toByteArray());
+			Object  cobj = MessagePack.unpack(cin, Object.class);
+			InputStream ein = new ByteArrayInputStream(eout.toByteArray());
+			ProvidedClass eobj = MessagePack.unpack(ein, ProvidedClass.class);
+			InputStream fin = new ByteArrayInputStream(fout.toByteArray());
+			UserDefinedClass fobj = MessagePack.unpack(fin, UserDefinedClass.class);
+
+			assertEquals(aobj, "msgpack");
+			assertEquals(bobj, (Integer)1);
+			assertEquals(cobj, null);
+			assertEquals(eobj, createProvidedClass());
+			assertEquals(fobj, createUserDefinedClass());
+		}
 	}
 
 	@Test

From ad5ebd007ef6f1ebef3b60f35708846a3752210b Mon Sep 17 00:00:00 2001
From: Muga Nishizawa 
Date: Sun, 24 Oct 2010 23:10:07 +0900
Subject: [PATCH 0906/1648] java: refactor org.msgpack.util.codegen.*.java

---
 .../main/java/org/msgpack/CustomMessage.java  |   1 +
 .../org/msgpack/template/ListTemplate.java    |   4 +-
 .../org/msgpack/template/MapTemplate.java     |   4 +-
 .../org/msgpack/util/codegen/Constants.java   |   2 +-
 .../msgpack/util/codegen/DynamicCodeGen.java  | 112 +-------
 .../util/codegen/DynamicCodeGenBase.java      | 267 +-----------------
 .../codegen/DynamicOrdinalEnumPacker.java     |  27 +-
 .../msgpack/util/codegen/DynamicPacker.java   |  26 +-
 .../msgpack/util/codegen/TestPackConvert.java |  57 ++--
 .../msgpack/util/codegen/TestPackUnpack.java  |  76 ++---
 10 files changed, 52 insertions(+), 524 deletions(-)

diff --git a/java/src/main/java/org/msgpack/CustomMessage.java b/java/src/main/java/org/msgpack/CustomMessage.java
index 959c9cd..e7408bb 100644
--- a/java/src/main/java/org/msgpack/CustomMessage.java
+++ b/java/src/main/java/org/msgpack/CustomMessage.java
@@ -25,6 +25,7 @@ public class CustomMessage {
 	}
 
 	public static void registerTemplate(Class target, Template tmpl) {
+		CustomPacker.register(target, tmpl);
 		CustomUnpacker.register(target, tmpl);
 		CustomConverter.register(target, tmpl);
 	}
diff --git a/java/src/main/java/org/msgpack/template/ListTemplate.java b/java/src/main/java/org/msgpack/template/ListTemplate.java
index 6703df4..34cbe35 100644
--- a/java/src/main/java/org/msgpack/template/ListTemplate.java
+++ b/java/src/main/java/org/msgpack/template/ListTemplate.java
@@ -33,11 +33,13 @@ public class ListTemplate implements Template {
 		return elementTemplate;
 	}
 
+	@SuppressWarnings("unchecked")
 	public void pack(Packer pk, Object target) throws IOException {
-		if(target instanceof List) {
+		if(!(target instanceof List)) {
 			throw new MessageTypeException();
 		}
 		List list = (List)target;
+		pk.packArray(list.size());
 		for(Object element : list) {
 			elementTemplate.pack(pk, element);
 		}
diff --git a/java/src/main/java/org/msgpack/template/MapTemplate.java b/java/src/main/java/org/msgpack/template/MapTemplate.java
index 6925982..3720292 100644
--- a/java/src/main/java/org/msgpack/template/MapTemplate.java
+++ b/java/src/main/java/org/msgpack/template/MapTemplate.java
@@ -39,11 +39,13 @@ public class MapTemplate implements Template {
 		return valueTemplate;
 	}
 
+	@SuppressWarnings("unchecked")
 	public void pack(Packer pk, Object target) throws IOException {
-		if(target instanceof Map) {
+		if(!(target instanceof Map)) {
 			throw new MessageTypeException();
 		}
 		Map map = (Map)target;
+		pk.packMap(map.size());
 		for(Map.Entry pair : map.entrySet()) {
 			keyTemplate.pack(pk, pair.getKey());
 			valueTemplate.pack(pk, pair.getValue());
diff --git a/java/src/main/java/org/msgpack/util/codegen/Constants.java b/java/src/main/java/org/msgpack/util/codegen/Constants.java
index 1932739..e0cfc1e 100644
--- a/java/src/main/java/org/msgpack/util/codegen/Constants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/Constants.java
@@ -72,7 +72,7 @@ public interface Constants {
 
 	String STATEMENT_PACKER_PACKERMETHODBODY_02 = "$1.packArray(%d); ";
 
-	String STATEMENT_PACKER_PACKERMETHODBODY_03 = "_$$_packers[%d].pack($1, %s_$$_t.%s%s); ";
+	String STATEMENT_PACKER_PACKERMETHODBODY_03 = "_$$_templates[%d].pack($1, %s_$$_t.%s%s); ";
 
 	String STATEMENT_PACKER_PACKERMETHODBODY_04 = "$1.pack(((java.lang.Enum)_$$_t).ordinal()); ";
 
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index 43f5b3d..f54f809 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -34,13 +34,11 @@ import javassist.CtNewMethod;
 import javassist.NotFoundException;
 
 import org.msgpack.MessagePackObject;
-import org.msgpack.MessagePacker;
 import org.msgpack.MessageTypeException;
 import org.msgpack.Packer;
 import org.msgpack.Template;
 import org.msgpack.Unpacker;
 import org.msgpack.annotation.MessagePackOptional;
-import org.msgpack.packer.OptionalPacker;
 import org.msgpack.template.OptionalTemplate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -62,12 +60,9 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 
 	private ConcurrentHashMap tmplCache;
 
-	private ConcurrentHashMap pkCache;
-
 	DynamicCodeGen() {
 		super();
 		tmplCache = new ConcurrentHashMap();
-		pkCache = new ConcurrentHashMap();
 	}
 
 	public void setTemplates(Class type, Template[] tmpls) {
@@ -78,73 +73,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		return tmplCache.get(type.getName());
 	}
 
-	public void setMessagePackers(Class type, MessagePacker[] pks) {
-		pkCache.put(type.getName(), pks);
-	}
-
-	public MessagePacker[] getMessagePackers(Class type) {
-		return pkCache.get(type.getName());
-	}
-
-	public Class generateMessagePackerClass(Class origClass,
-			List fieldOpts) {
-		try {
-			LOG.debug("start generating a packer class for "
-					+ origClass.getName());
-			String origName = origClass.getName();
-			String packerName = origName + POSTFIX_TYPE_NAME_PACKER + inc();
-			checkTypeValidation(origClass);
-			checkDefaultConstructorValidation(origClass);
-			CtClass packerCtClass = pool.makeClass(packerName);
-			setSuperclass(packerCtClass, MessagePackerAccessorImpl.class);
-			setInterface(packerCtClass, MessagePacker.class);
-			addClassTypeConstructor(packerCtClass);
-			Field[] fields = getDeclaredFields(origClass);
-			MessagePacker[] packers = null;
-			if (fieldOpts != null) {
-				fields = sortFields(fields, fieldOpts);
-				packers = createMessagePackers(fieldOpts);
-			} else {
-				packers = createMessagePackers(fields);
-			}
-			setMessagePackers(origClass, packers);
-			addPackMethod(packerCtClass, origClass, fields, false);
-			Class packerClass = createClass(packerCtClass);
-			LOG.debug("generated a packer class for " + origClass.getName());
-			return packerClass;
-		} catch (NotFoundException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (CannotCompileException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
-	}
-
-	public Class generateOrdinalEnumPackerClass(Class origClass) {
-		try {
-			LOG.debug("start generating an enum packer for "
-					+ origClass.getName());
-			String origName = origClass.getName();
-			String packerName = origName + POSTFIX_TYPE_NAME_PACKER + inc();
-			checkTypeValidation(origClass);
-			CtClass packerCtClass = pool.makeClass(packerName);
-			setSuperclass(packerCtClass, MessagePackerAccessorImpl.class);
-			setInterface(packerCtClass, MessagePacker.class);
-			addClassTypeConstructor(packerCtClass);
-			addPackMethod(packerCtClass, origClass, null, true);
-			Class packerClass = createClass(packerCtClass);
-			LOG.debug("generated an enum class for " + origClass.getName());
-			return packerClass;
-		} catch (NotFoundException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (CannotCompileException e) {
-			LOG.error(e.getMessage(), e);
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
-	}
-
 	public Class generateTemplateClass(Class origClass,
 			List fieldOpts) {
 		try {
@@ -167,19 +95,18 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				tmpls = createTemplates(fields);
 			}
 			setTemplates(origClass, tmpls);
+			addPackMethod(tmplCtClass, origClass, fields, false);
 			addUnpackMethod(tmplCtClass, origClass, fields, false);
 			addConvertMethod(tmplCtClass, origClass, fields, false);
 			Class tmplClass = createClass(tmplCtClass);
 			LOG.debug("generated a template class for " + origClass.getName());
 			return tmplClass;
 		} catch (NotFoundException e) {
-			DynamicCodeGenException ex = new DynamicCodeGenException(e
-					.getMessage(), e);
+			DynamicCodeGenException ex = new DynamicCodeGenException(e.getMessage(), e);
 			LOG.error(ex.getMessage(), ex);
 			throw ex;
 		} catch (CannotCompileException e) {
-			DynamicCodeGenException ex = new DynamicCodeGenException(e
-					.getMessage(), e);
+			DynamicCodeGenException ex = new DynamicCodeGenException(e.getMessage(), e);
 			LOG.error(ex.getMessage(), ex);
 			throw ex;
 		}
@@ -196,6 +123,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			setSuperclass(tmplCtClass, TemplateAccessorImpl.class);
 			setInterface(tmplCtClass, Template.class);
 			addClassTypeConstructor(tmplCtClass);
+			addPackMethod(tmplCtClass, origClass, null, true);
 			addUnpackMethod(tmplCtClass, origClass, null, true);
 			addConvertMethod(tmplCtClass, origClass, null, true);
 			Class tmplClass = createClass(tmplCtClass);
@@ -308,38 +236,6 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		return sorted;
 	}
 
-	MessagePacker[] createMessagePackers(List fieldOpts) {
-		MessagePacker[] pks = new MessagePacker[fieldOpts.size()];
-		for (int i = 0; i < pks.length; ++i) {
-			pks[i] = toMessagePacker(fieldOpts.get(i).tmpl);
-		}
-		return pks;
-	}
-
-	MessagePacker[] createMessagePackers(Field[] fields) {
-		MessagePacker[] pks = new MessagePacker[fields.length];
-		for (int i = 0; i < pks.length; ++i) {
-			pks[i] = createMessagePacker(fields[i]);
-		}
-		return pks;
-	}
-
-	MessagePacker createMessagePacker(Field field) {
-		boolean isOptional = isAnnotated(field, MessagePackOptional.class);
-		Class c = field.getType();
-		MessagePacker pk = null;
-		if (List.class.isAssignableFrom(c) || Map.class.isAssignableFrom(c)) {
-			pk = createMessagePacker(field.getGenericType());
-		} else {
-			pk = createMessagePacker(c);
-		}
-		if (isOptional) {
-			return new OptionalPacker(pk);
-		} else {
-			return pk;
-		}
-	}
-
 	Template[] createTemplates(List fieldOpts) {
 		Template[] tmpls = new Template[fieldOpts.size()];
 		for (int i = 0; i < tmpls.length; ++i) {
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
index c80d354..c9ff471 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGenBase.java
@@ -24,7 +24,6 @@ import java.lang.reflect.Method;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
 import java.math.BigInteger;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -41,11 +40,9 @@ import javassist.NotFoundException;
 
 import org.msgpack.CustomConverter;
 import org.msgpack.CustomMessage;
-import org.msgpack.CustomPacker;
 import org.msgpack.MessageConvertable;
 import org.msgpack.MessagePackObject;
 import org.msgpack.MessagePackable;
-import org.msgpack.MessagePacker;
 import org.msgpack.MessageTypeException;
 import org.msgpack.MessageUnpackable;
 import org.msgpack.Packer;
@@ -55,32 +52,6 @@ import org.msgpack.Unpacker;
 import org.msgpack.annotation.MessagePackDelegate;
 import org.msgpack.annotation.MessagePackMessage;
 import org.msgpack.annotation.MessagePackOrdinalEnum;
-import org.msgpack.packer.BigIntegerPacker;
-import org.msgpack.packer.BooleanPacker;
-import org.msgpack.packer.ByteArrayPacker;
-import org.msgpack.packer.BytePacker;
-import org.msgpack.packer.DoublePacker;
-import org.msgpack.packer.FloatPacker;
-import org.msgpack.packer.IntegerPacker;
-import org.msgpack.packer.LongPacker;
-import org.msgpack.packer.OptionalPacker;
-import org.msgpack.packer.ShortPacker;
-import org.msgpack.packer.StringPacker;
-import org.msgpack.template.BigIntegerTemplate;
-import org.msgpack.template.BooleanTemplate;
-import org.msgpack.template.ByteArrayTemplate;
-import org.msgpack.template.ByteTemplate;
-import org.msgpack.template.ClassTemplate;
-import org.msgpack.template.CollectionTemplate;
-import org.msgpack.template.DoubleTemplate;
-import org.msgpack.template.FloatTemplate;
-import org.msgpack.template.IntegerTemplate;
-import org.msgpack.template.ListTemplate;
-import org.msgpack.template.LongTemplate;
-import org.msgpack.template.MapTemplate;
-import org.msgpack.template.OptionalTemplate;
-import org.msgpack.template.ShortTemplate;
-import org.msgpack.template.StringTemplate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -89,11 +60,10 @@ public class DynamicCodeGenBase implements Constants {
 	private static Logger LOG = LoggerFactory
 			.getLogger(DynamicCodeGenBase.class);
 
-	static class MessagePackablePacker implements MessagePacker {
-		@SuppressWarnings("unused")
+	static class MessagePackUnpackConvertableTemplate implements Template {
 		private Class type;
 
-		MessagePackablePacker(Class type) {
+		MessagePackUnpackConvertableTemplate(Class type) {
 			this.type = type;
 		}
 
@@ -102,61 +72,6 @@ public class DynamicCodeGenBase implements Constants {
 			MessagePackable mp = MessagePackable.class.cast(target);
 			mp.messagePack(packer);
 		}
-	}
-
-	static class ListPacker implements MessagePacker {
-		private MessagePacker elementPacker;
-
-		ListPacker(MessagePacker elementPacker) {
-			this.elementPacker = elementPacker;
-		}
-
-		@SuppressWarnings("unchecked")
-		@Override
-		public void pack(Packer packer, Object target) throws IOException {
-			List list = (List) target;
-			packer.packArray(list.size());
-			for (Iterator iter = list.iterator(); iter.hasNext();) {
-				elementPacker.pack(packer, iter.next());
-			}
-		}
-	}
-
-	static class MapPacker implements MessagePacker {
-		private MessagePacker keyPacker;
-		private MessagePacker valPacker;
-
-		MapPacker(MessagePacker keyPacker, MessagePacker valPacker) {
-			this.keyPacker = keyPacker;
-			this.valPacker = valPacker;
-		}
-
-		@SuppressWarnings("unchecked")
-		@Override
-		public void pack(Packer packer, Object target) throws IOException {
-			Map map = (Map) target;
-			packer.packMap(map.size());
-			for (Map.Entry e : ((Map) map)
-					.entrySet()) {
-				keyPacker.pack(packer, e.getKey());
-				valPacker.pack(packer, e.getValue());
-			}
-		}
-
-	}
-
-	static class MessageUnpackableConvertableTemplate implements Template {
-		private Class type;
-
-		MessageUnpackableConvertableTemplate(Class type) {
-			this.type = type;
-		}
-
-		@Override
-		public void pack(Packer pk, Object target) throws IOException {
-			// FIXME
-			pk.pack(target);
-		}
 
 		@Override
 		public Object unpack(Unpacker unpacker) throws IOException,
@@ -192,27 +107,6 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	public static interface MessagePackerAccessor {
-		void setMessagePackers(MessagePacker[] packers);
-	}
-
-	protected static class MessagePackerAccessorImpl implements MessagePackerAccessor {
-		public Class type;
-
-		public MessagePacker[] _$$_packers;
-
-		public MessagePackerAccessorImpl() {
-		}
-
-		public MessagePackerAccessorImpl(Class type) {
-			this.type = type;
-		}
-
-		public void setMessagePackers(MessagePacker[] _$$_pks) {
-			_$$_packers = _$$_pks;
-		}
-	}
-
 	public static interface TemplateAccessor {
 		void setTemplates(Template[] templates);
 	}
@@ -342,26 +236,6 @@ public class DynamicCodeGenBase implements Constants {
 		newCtClass.addConstructor(newCtCons);
 	}
 
-	protected void addMessagePackerArrayField(CtClass newCtClass)
-			throws NotFoundException, CannotCompileException {
-		CtClass acsCtClass = pool
-				.get(MessagePackerAccessorImpl.class.getName());
-		CtField pksField = acsCtClass.getDeclaredField(VARIABLE_NAME_PACKERS);
-		CtField pksField2 = new CtField(pksField.getType(), pksField.getName(),
-				newCtClass);
-		newCtClass.addField(pksField2);
-	}
-
-	protected void addSetMessagePackersMethod(CtClass newCtClass)
-			throws NotFoundException, CannotCompileException {
-		CtClass acsCtClass = pool.get(TemplateAccessorImpl.class.getName());
-		CtMethod setPksMethod = acsCtClass
-				.getDeclaredMethod(METHOD_NAME_SETMESSAGEPACKERS);
-		CtMethod setPksMethod2 = CtNewMethod.copy(setPksMethod, newCtClass,
-				null);
-		newCtClass.addMethod(setPksMethod2);
-	}
-
 	protected void addTemplateArrayField(CtClass newCtClass)
 			throws NotFoundException, CannotCompileException {
 		CtClass acsCtClass = pool.get(TemplateAccessorImpl.class.getName());
@@ -422,139 +296,6 @@ public class DynamicCodeGenBase implements Constants {
 		}
 	}
 
-	public static MessagePacker toMessagePacker(Template tmpl) {
-		if (tmpl instanceof BigIntegerTemplate) {
-			return BigIntegerPacker.getInstance();
-		} else if (tmpl instanceof BooleanTemplate) {
-			return BooleanPacker.getInstance();
-		} else if (tmpl instanceof ByteArrayTemplate) {
-			return ByteArrayPacker.getInstance();
-		} else if (tmpl instanceof ByteTemplate) {
-			return BytePacker.getInstance();
-		} else if (tmpl instanceof ClassTemplate) {
-			UnsupportedOperationException e = new UnsupportedOperationException(
-					"not supported yet.");
-			LOG.error(e.getMessage(), e);
-			throw e;
-		} else if (tmpl instanceof CollectionTemplate) {
-			UnsupportedOperationException e = new UnsupportedOperationException(
-					"not supported yet.");
-			LOG.error(e.getMessage(), e);
-			throw e;
-		} else if (tmpl instanceof DoubleTemplate) {
-			return DoublePacker.getInstance();
-		} else if (tmpl instanceof FloatTemplate) {
-			return FloatPacker.getInstance();
-		} else if (tmpl instanceof IntegerTemplate) {
-			return IntegerPacker.getInstance();
-		} else if (tmpl instanceof ListTemplate) {
-			ListTemplate t = (ListTemplate) tmpl;
-			return new ListPacker(toMessagePacker(t.getElementTemplate()));
-		} else if (tmpl instanceof LongTemplate) {
-			return LongPacker.getInstance();
-		} else if (tmpl instanceof MapTemplate) {
-			MapTemplate t = (MapTemplate) tmpl;
-			return new MapPacker(toMessagePacker(t.getKeyTemplate()),
-					toMessagePacker(t.getValueTemplate()));
-		} else if (tmpl instanceof OptionalTemplate) {
-			OptionalTemplate t = (OptionalTemplate) tmpl;
-			return new OptionalPacker(toMessagePacker(t.getElementTemplate()));
-		} else if (tmpl instanceof ShortTemplate) {
-			return ShortPacker.getInstance();
-		} else if (tmpl instanceof StringTemplate) {
-			return StringPacker.getInstance();
-		} else if (tmpl instanceof TemplateAccessorImpl) {
-			Class c = ((TemplateAccessorImpl) tmpl).type;
-			if (CustomPacker.isRegistered(c)) {
-				return CustomPacker.get(c);
-			} else {
-				MessagePacker packer = DynamicPacker.create(c);
-				CustomMessage.registerPacker(c, packer);
-				return packer;
-			}
-		}
-		UnsupportedOperationException e = new UnsupportedOperationException(
-				"not supported yet.");
-		LOG.error(e.getMessage(), e);
-		throw e;
-	}
-
-	public static MessagePacker createMessagePacker(Type t) {
-		if (t.getClass().equals(Class.class)) {
-			Class c = (Class) t;
-			if (c.equals(boolean.class) || c.equals(Boolean.class)) {
-				return BooleanPacker.getInstance();
-			} else if (c.equals(byte.class) || c.equals(Byte.class)) {
-				return BytePacker.getInstance();
-			} else if (c.equals(short.class) || c.equals(Short.class)) {
-				return ShortPacker.getInstance();
-			} else if (c.equals(int.class) || c.equals(Integer.class)) {
-				return IntegerPacker.getInstance();
-			} else if (c.equals(float.class) || c.equals(Float.class)) {
-				return FloatPacker.getInstance();
-			} else if (c.equals(long.class) || c.equals(Long.class)) {
-				return LongPacker.getInstance();
-			} else if (c.equals(double.class) || c.equals(Double.class)) {
-				return DoublePacker.getInstance();
-			} else if (c.equals(String.class)) {
-				return StringPacker.getInstance();
-			} else if (c.equals(BigInteger.class)) {
-				return BigIntegerPacker.getInstance();
-			} else if (CustomPacker.isRegistered(c)) {
-				return CustomPacker.get(c);
-			} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
-				// @MessagePackMessage
-				MessagePacker packer = DynamicPacker.create(c);
-				CustomMessage.registerPacker(c, packer);
-				return packer;
-			} else if (CustomMessage.isAnnotated(c, MessagePackDelegate.class)) {
-				// FIXME DelegatePacker
-				UnsupportedOperationException e = new UnsupportedOperationException(
-						"not supported yet. : " + c.getName());
-				LOG.error(e.getMessage(), e);
-				throw e;
-			} else if (CustomMessage.isAnnotated(c,
-					MessagePackOrdinalEnum.class)) {
-				// @MessagePackOrdinalEnum
-				MessagePacker packer = DynamicOrdinalEnumPacker.create(c);
-				CustomMessage.registerPacker(c, packer);
-				return packer;
-			} else if (MessagePackable.class.isAssignableFrom(c)) {
-				MessagePacker packer = new MessagePackablePacker(c);
-				CustomMessage.registerPacker(c, packer);
-				return packer;
-			} else {
-				throw new MessageTypeException("Type error: "
-						+ ((Class) t).getName());
-			}
-		} else if (t instanceof GenericArrayType) {
-			GenericArrayType gat = (GenericArrayType) t;
-			Type gct = gat.getGenericComponentType();
-			if (gct.equals(byte.class)) {
-				return ByteArrayPacker.getInstance();
-			} else {
-				throw new DynamicCodeGenException("Not supported yet: " + gat);
-			}
-		} else if (t instanceof ParameterizedType) {
-			ParameterizedType pt = (ParameterizedType) t;
-			Class rawType = (Class) pt.getRawType();
-			if (rawType.equals(List.class)) {
-				Type[] ats = pt.getActualTypeArguments();
-				return new ListPacker(createMessagePacker(ats[0]));
-			} else if (rawType.equals(Map.class)) {
-				Type[] ats = pt.getActualTypeArguments();
-				return new MapPacker(createMessagePacker(ats[0]),
-						createMessagePacker(ats[1]));
-			} else {
-				throw new DynamicCodeGenException("Type error: "
-						+ t.getClass().getName());
-			}
-		} else {
-			throw new DynamicCodeGenException("Type error: "
-					+ t.getClass().getName());
-		}
-	}
-
 	public static Template createTemplate(Type t) {
 		if (t.getClass().equals(Class.class)) {
 			Class c = (Class) t;
@@ -576,7 +317,7 @@ public class DynamicCodeGenBase implements Constants {
 				return Templates.tString();
 			} else if (c.equals(BigInteger.class)) {
 				return Templates.tBigInteger();
-			} else if (CustomConverter.isRegistered(c)) {
+			} else if (CustomConverter.isRegistered(c)) {// FIXME
 				return (Template) CustomConverter.get(c);
 			} else if (CustomMessage.isAnnotated(c, MessagePackMessage.class)) {
 				// @MessagePackMessage
@@ -597,7 +338,7 @@ public class DynamicCodeGenBase implements Constants {
 				return tmpl;
 			} else if (MessageConvertable.class.isAssignableFrom(c)
 					|| MessageUnpackable.class.isAssignableFrom(c)) {
-				Template tmpl = new MessageUnpackableConvertableTemplate(c);
+				Template tmpl = new MessagePackUnpackConvertableTemplate(c);
 				CustomMessage.registerTemplate(c, tmpl);
 				return tmpl;
 			} else {
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
index ddb0990..d1fdbc6 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicOrdinalEnumPacker.java
@@ -17,35 +17,10 @@
 //
 package org.msgpack.util.codegen;
 
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-
 import org.msgpack.MessagePacker;
-import org.msgpack.util.codegen.DynamicCodeGenBase.MessagePackerAccessor;
 
 public class DynamicOrdinalEnumPacker {
 	public static MessagePacker create(Class c) {
-		try {
-			DynamicCodeGen gen = DynamicCodeGen.getInstance();
-			Class packerClass = gen.generateOrdinalEnumPackerClass(c);
-			Constructor cons = packerClass
-					.getDeclaredConstructor(new Class[] { Class.class });
-			Object obj = cons.newInstance(new Object[] { c });
-			((MessagePackerAccessor) obj).setMessagePackers(gen
-					.getMessagePackers(c));
-			return (MessagePacker) obj;
-		} catch (InstantiationException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (IllegalAccessException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (SecurityException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (NoSuchMethodException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (IllegalArgumentException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (InvocationTargetException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
+		return DynamicOrdinalEnumTemplate.create(c);
 	}
 }
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
index fe0095c..5796b71 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicPacker.java
@@ -17,12 +17,9 @@
 //
 package org.msgpack.util.codegen;
 
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 
 import org.msgpack.MessagePacker;
-import org.msgpack.util.codegen.DynamicCodeGenBase.MessagePackerAccessor;
 
 public class DynamicPacker {
 
@@ -31,27 +28,6 @@ public class DynamicPacker {
 	}
 
 	public static MessagePacker create(Class c, List fieldOpts) {
-		try {
-			DynamicCodeGen gen = DynamicCodeGen.getInstance();
-			Class packerClass = gen.generateMessagePackerClass(c, fieldOpts);
-			Constructor cons = packerClass
-					.getDeclaredConstructor(new Class[] { Class.class });
-			Object obj = cons.newInstance(new Object[] { c });
-			MessagePacker[] packers = gen.getMessagePackers(c);
-			((MessagePackerAccessor) obj).setMessagePackers(packers);
-			return (MessagePacker) obj;
-		} catch (InstantiationException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (IllegalAccessException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (SecurityException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (NoSuchMethodException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (IllegalArgumentException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		} catch (InvocationTargetException e) {
-			throw new DynamicCodeGenException(e.getMessage(), e);
-		}
+		return DynamicTemplate.create(c, fieldOpts);
 	}
 }
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java b/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
index 63532a0..1d34079 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackConvert.java
@@ -14,6 +14,7 @@ import junit.framework.TestCase;
 
 import org.junit.Test;
 import org.msgpack.CustomConverter;
+import org.msgpack.CustomMessage;
 import org.msgpack.CustomPacker;
 import org.msgpack.CustomUnpacker;
 import org.msgpack.MessageConvertable;
@@ -1233,29 +1234,23 @@ public class TestPackConvert extends TestCase {
 
 	@Test
 	public void testNestedFieldClass00() throws Exception {
-		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
-		CustomPacker.register(NestedClass.class, packer2);
-		MessagePacker packer1 = DynamicPacker.create(BaseClass.class);
-		CustomPacker.register(BaseClass.class, packer1);
 		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
-		CustomUnpacker.register(NestedClass.class, tmpl2);
-		CustomConverter.register(NestedClass.class, tmpl2);
-		Template tmpl1 = DynamicTemplate.create(BaseClass.class);
-		CustomUnpacker.register(BaseClass.class, tmpl1);
-		CustomConverter.register(BaseClass.class, tmpl1);
+		CustomMessage.registerTemplate(NestedClass.class, tmpl2);
+		Template tmpl = DynamicTemplate.create(BaseClass.class);
+		CustomMessage.registerTemplate(BaseClass.class, tmpl);
 		BaseClass src = new BaseClass();
 		NestedClass src2 = new NestedClass();
 		src.f0 = 0;
 		src2.f2 = 2;
 		src.f1 = src2;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer1.pack(new Packer(out), src);
+		tmpl.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
 		assertTrue(it.hasNext());
 		MessagePackObject mpo = it.next();
-		BaseClass dst = (BaseClass) tmpl1.convert(mpo);
+		BaseClass dst = (BaseClass) tmpl.convert(mpo);
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1.f2 == dst.f1.f2);
 		assertFalse(it.hasNext());
@@ -1263,21 +1258,13 @@ public class TestPackConvert extends TestCase {
 
 	@Test
 	public void testNestedFieldClass02() throws Exception {
-		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
-		CustomPacker.register(NestedClass.class, packer2);
-		MessagePacker packer = new OptionalPacker(DynamicPacker
-				.create(BaseClass.class));
-		CustomPacker.register(BaseClass.class, packer);
 		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
-		CustomUnpacker.register(NestedClass.class, tmpl2);
-		CustomConverter.register(NestedClass.class, tmpl2);
-		Template tmpl = new OptionalTemplate(DynamicTemplate
-				.create(BaseClass.class));
-		CustomUnpacker.register(BaseClass.class, tmpl);
-		CustomConverter.register(BaseClass.class, tmpl);
+		CustomMessage.registerTemplate(NestedClass.class, tmpl2);
+		Template tmpl = new OptionalTemplate(DynamicTemplate.create(BaseClass.class));
+		CustomMessage.registerTemplate(BaseClass.class, tmpl);
 		BaseClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer.pack(new Packer(out), src);
+		tmpl.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
@@ -1305,23 +1292,17 @@ public class TestPackConvert extends TestCase {
 
 	@Test
 	public void testOptionalNestedFieldClass00() throws Exception {
-		MessagePacker packer2 = DynamicPacker.create(OptionalNestedClass.class);
-		CustomPacker.register(OptionalNestedClass.class, packer2);
-		MessagePacker packer = DynamicPacker.create(OptionalBaseClass.class);
-		CustomPacker.register(OptionalBaseClass.class, packer);
 		Template tmpl2 = DynamicTemplate.create(OptionalNestedClass.class);
-		CustomUnpacker.register(OptionalNestedClass.class, tmpl2);
-		CustomConverter.register(OptionalNestedClass.class, tmpl2);
+		CustomMessage.registerTemplate(OptionalNestedClass.class, tmpl2);
 		Template tmpl = DynamicTemplate.create(OptionalBaseClass.class);
-		CustomUnpacker.register(OptionalBaseClass.class, tmpl);
-		CustomConverter.register(OptionalBaseClass.class, tmpl);
+		CustomMessage.registerTemplate(OptionalBaseClass.class, tmpl);
 		OptionalBaseClass src = new OptionalBaseClass();
 		OptionalNestedClass src2 = new OptionalNestedClass();
 		src.f0 = 0;
 		src2.f2 = 2;
 		src.f1 = src2;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer.pack(new Packer(out), src);
+		tmpl.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
@@ -1335,19 +1316,13 @@ public class TestPackConvert extends TestCase {
 
 	@Test
 	public void testOptionalNestedFieldClass01() throws Exception {
-		MessagePacker packer2 = DynamicPacker.create(OptionalNestedClass.class);
-		CustomPacker.register(OptionalNestedClass.class, packer2);
-		MessagePacker packer = DynamicPacker.create(OptionalBaseClass.class);
-		CustomPacker.register(OptionalBaseClass.class, packer);
 		Template tmpl2 = DynamicTemplate.create(OptionalNestedClass.class);
-		CustomUnpacker.register(OptionalNestedClass.class, tmpl2);
-		CustomConverter.register(OptionalNestedClass.class, tmpl2);
+		CustomMessage.registerTemplate(OptionalNestedClass.class, tmpl2);
 		Template tmpl = DynamicTemplate.create(OptionalBaseClass.class);
-		CustomUnpacker.register(OptionalBaseClass.class, tmpl);
-		CustomConverter.register(OptionalBaseClass.class, tmpl);
+		CustomMessage.registerTemplate(OptionalBaseClass.class, tmpl);
 		OptionalBaseClass src = new OptionalBaseClass();
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer.pack(new Packer(out), src);
+		tmpl.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		Unpacker pac = new Unpacker(in);
 		Iterator it = pac.iterator();
diff --git a/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
index 9496604..95c4b6b 100644
--- a/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
+++ b/java/src/test/java/org/msgpack/util/codegen/TestPackUnpack.java
@@ -11,9 +11,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.junit.Test;
-import org.msgpack.CustomConverter;
-import org.msgpack.CustomPacker;
-import org.msgpack.CustomUnpacker;
+import org.msgpack.CustomMessage;
 import org.msgpack.MessagePackable;
 import org.msgpack.MessagePacker;
 import org.msgpack.MessageTypeException;
@@ -1117,23 +1115,17 @@ public class TestPackUnpack extends TestCase {
 
 	@Test
 	public void testNestedFieldClass00() throws Exception {
-		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
-		CustomPacker.register(NestedClass.class, packer2);
-		MessagePacker packer = DynamicPacker.create(BaseClass.class);
-		CustomPacker.register(BaseClass.class, packer);
 		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
-		CustomUnpacker.register(NestedClass.class, tmpl2);
-		CustomConverter.register(NestedClass.class, tmpl2);
+		CustomMessage.registerTemplate(NestedClass.class, tmpl2);
 		Template tmpl = DynamicTemplate.create(BaseClass.class);
-		CustomUnpacker.register(BaseClass.class, tmpl);
-		CustomConverter.register(BaseClass.class, tmpl);
+		CustomMessage.registerTemplate(BaseClass.class, tmpl);
 		BaseClass src = new BaseClass();
 		NestedClass src2 = new NestedClass();
 		src.f0 = 0;
 		src2.f2 = 2;
 		src.f1 = src2;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer.pack(new Packer(out), src);
+		tmpl.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		BaseClass dst = (BaseClass) tmpl.unpack(new Unpacker(in));
 		assertTrue(src.f0 == dst.f0);
@@ -1142,21 +1134,12 @@ public class TestPackUnpack extends TestCase {
 
 	@Test
 	public void testNestedFieldClass01() throws Exception {
-		MessagePacker packer2 = DynamicPacker.create(NestedClass.class);
-		CustomPacker.register(NestedClass.class, packer2);
-		MessagePacker packer = new OptionalPacker(DynamicPacker
-				.create(BaseClass.class));
-		CustomPacker.register(BaseClass.class, packer);
 		Template tmpl2 = DynamicTemplate.create(NestedClass.class);
-		CustomUnpacker.register(NestedClass.class, tmpl2);
-		CustomConverter.register(NestedClass.class, tmpl2);
-		Template tmpl = new OptionalTemplate(DynamicTemplate
-				.create(BaseClass.class));
-		CustomUnpacker.register(BaseClass.class, tmpl);
-		CustomConverter.register(BaseClass.class, tmpl);
+		CustomMessage.registerTemplate(NestedClass.class, tmpl2);
+		Template tmpl = new OptionalTemplate(DynamicTemplate.create(BaseClass.class));
 		BaseClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer.pack(new Packer(out), src);
+		tmpl.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		BaseClass dst = (BaseClass) tmpl.unpack(new Unpacker(in));
 		assertEquals(src, dst);
@@ -1179,70 +1162,47 @@ public class TestPackUnpack extends TestCase {
 
 	@Test
 	public void testOptionalNestedFieldClass00() throws Exception {
-		MessagePacker packer2 = DynamicPacker.create(OptionalNestedClass.class);
-		CustomPacker.register(OptionalNestedClass.class, packer2);
-		MessagePacker packer = DynamicPacker.create(OptionalBaseClass.class);
-		CustomPacker.register(OptionalBaseClass.class, packer);
 		Template tmpl2 = DynamicTemplate.create(OptionalNestedClass.class);
-		CustomUnpacker.register(OptionalNestedClass.class, tmpl2);
-		CustomConverter.register(OptionalNestedClass.class, tmpl2);
+		CustomMessage.registerTemplate(OptionalNestedClass.class, tmpl2);
 		Template tmpl = DynamicTemplate.create(OptionalBaseClass.class);
-		CustomUnpacker.register(OptionalBaseClass.class, tmpl);
-		CustomConverter.register(OptionalBaseClass.class, tmpl);
 		OptionalBaseClass src = new OptionalBaseClass();
 		OptionalNestedClass src2 = new OptionalNestedClass();
 		src.f0 = 0;
 		src2.f2 = 2;
 		src.f1 = src2;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer.pack(new Packer(out), src);
+		tmpl.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		OptionalBaseClass dst = (OptionalBaseClass) tmpl
-				.unpack(new Unpacker(in));
+		OptionalBaseClass dst = (OptionalBaseClass) tmpl.unpack(new Unpacker(in));
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1.f2 == dst.f1.f2);
 	}
 
 	@Test
 	public void testOptionalNestedFieldClass01() throws Exception {
-		MessagePacker packer2 = DynamicPacker.create(OptionalNestedClass.class);
-		CustomPacker.register(OptionalNestedClass.class, packer2);
-		MessagePacker packer = DynamicPacker.create(OptionalBaseClass.class);
-		CustomPacker.register(OptionalBaseClass.class, packer);
 		Template tmpl2 = DynamicTemplate.create(OptionalNestedClass.class);
-		CustomUnpacker.register(OptionalNestedClass.class, tmpl2);
-		CustomConverter.register(OptionalNestedClass.class, tmpl2);
+		CustomMessage.registerTemplate(OptionalNestedClass.class, tmpl2);
 		Template tmpl = DynamicTemplate.create(OptionalBaseClass.class);
-		CustomUnpacker.register(OptionalBaseClass.class, tmpl);
-		CustomConverter.register(OptionalBaseClass.class, tmpl);
+		CustomMessage.registerTemplate(OptionalBaseClass.class, tmpl);
 		OptionalBaseClass src = new OptionalBaseClass();
 		src.f1 = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer.pack(new Packer(out), src);
+		tmpl.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
-		OptionalBaseClass dst = (OptionalBaseClass) tmpl
-				.unpack(new Unpacker(in));
+		OptionalBaseClass dst = (OptionalBaseClass) tmpl.unpack(new Unpacker(in));
 		assertTrue(src.f0 == dst.f0);
 		assertTrue(src.f1 == dst.f1);
 	}
 
 	@Test
 	public void testOptionalNestedFieldClass02() throws Exception {
-		MessagePacker packer2 = DynamicPacker.create(OptionalNestedClass.class);
-		CustomPacker.register(OptionalNestedClass.class, packer2);
-		MessagePacker packer = new OptionalPacker(DynamicPacker
-				.create(OptionalBaseClass.class));
-		CustomPacker.register(OptionalBaseClass.class, packer);
 		Template tmpl2 = DynamicTemplate.create(OptionalNestedClass.class);
-		CustomUnpacker.register(OptionalNestedClass.class, tmpl2);
-		CustomConverter.register(OptionalNestedClass.class, tmpl2);
-		Template tmpl = new OptionalTemplate(DynamicTemplate
-				.create(OptionalBaseClass.class));
-		CustomUnpacker.register(OptionalBaseClass.class, tmpl);
-		CustomConverter.register(OptionalBaseClass.class, tmpl);
+		CustomMessage.registerTemplate(OptionalNestedClass.class, tmpl2);
+		Template tmpl = new OptionalTemplate(DynamicTemplate.create(OptionalBaseClass.class));
+		CustomMessage.registerTemplate(OptionalBaseClass.class, tmpl);
 		OptionalBaseClass src = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
-		packer.pack(new Packer(out), src);
+		tmpl.pack(new Packer(out), src);
 		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
 		OptionalBaseClass dst = (OptionalBaseClass) tmpl
 				.unpack(new Unpacker(in));

From 59610e81de83c3d2a269ce70434f5f434b76852f Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Mon, 25 Oct 2010 14:18:41 +0900
Subject: [PATCH 0907/1648] java: DynamicCodeGen::insertCodeOfUnpackMethodBody
 unpacks last optional field of classes correctly

---
 .../org/msgpack/util/codegen/Constants.java   |  8 ++++-
 .../msgpack/util/codegen/DynamicCodeGen.java  | 33 ++++++++++++++++++-
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/java/src/main/java/org/msgpack/util/codegen/Constants.java b/java/src/main/java/org/msgpack/util/codegen/Constants.java
index e0cfc1e..fc6650f 100644
--- a/java/src/main/java/org/msgpack/util/codegen/Constants.java
+++ b/java/src/main/java/org/msgpack/util/codegen/Constants.java
@@ -78,7 +78,7 @@ public interface Constants {
 
 	String STATEMENT_TMPL_UNPACKERMETHODBODY_01 = "%s _$$_t = new %s(); ";
 
-	String STATEMENT_TMPL_UNPACKERMETHODBODY_02 = "$1.unpackArray(); ";
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_02 = "int _$$_L = $1.unpackArray(); ";
 
 	String STATEMENT_TMPL_UNPACKERMETHODBODY_03 = "_$$_t.%s = %s(%s)_$$_templates[%d].unpack($1)%s; ";
 
@@ -88,6 +88,12 @@ public interface Constants {
 
 	String STATEMENT_TMPL_UNPACKERMETHODBODY_06 = "return %s.class.getEnumConstants()[i]; ";
 
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_07 = "if(_$$_L <= %d) { throw new org.msgpack.MessageTypeException(); } ";
+
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_08 = "if(_$$_L > %d && !$1.tryUnpackNull()) ";
+
+	String STATEMENT_TMPL_UNPACKERMETHODBODY_09 = "for(int _$$_n = %d; _$$_n < _$$_L; _$$_n++) { $1.unpackObject(); } ";
+
 	String STATEMENT_TMPL_CONVERTMETHODBODY_01 = "%s _$$_ary = $1.asArray(); ";
 
 	String STATEMENT_TMPL_CONVERTMETHODBODY_02 = "_$$_t.%s = %s(%s)_$$_templates[%d].convert(_$$_ary[%d])%s; ";
diff --git a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
index f54f809..edd23d8 100644
--- a/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
+++ b/java/src/main/java/org/msgpack/util/codegen/DynamicCodeGen.java
@@ -262,6 +262,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 			tmpl = createTemplate(c);
 		}
 		if (isOptional) {
+			// for pack
 			return new OptionalTemplate(tmpl);
 		} else {
 			return tmpl;
@@ -393,7 +394,7 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		String typeName = classToString(type);
 		Object[] args0 = new Object[] { typeName, typeName };
 		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_01, args0));
-		// $1.unpackArray();
+		// int _$$_L = $1.unpackArray();
 		Object[] args1 = new Object[0];
 		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_02, args1));
 		insertCodeOfUnpackMethodCalls(sb, fields);
@@ -407,10 +408,27 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 		for (int i = 0; i < fields.length; ++i) {
 			insertCodeOfUnpackMethodCall(sb, fields[i], i);
 		}
+		insertCodeOfUnpackTrails(sb, fields.length);
 	}
 
 	private void insertCodeOfUnpackMethodCall(StringBuilder sb, Field field,
 			int i) {
+		boolean isOptional = isAnnotated(field, MessagePackOptional.class);
+
+		if(isOptional) {
+			// if(_$$_L > i && !$1.tryUnpackNull()) {
+			Object[] args0 = new Object[] { i };
+			sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_08, args0));
+			sb.append(CHAR_NAME_LEFT_CURLY_BRACKET);
+
+		} else {
+			// if(_$$_L <= i) {
+			// 	throw new MessageTypeException();
+			// }
+			Object[] args0 = new Object[] { i };
+			sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_07, args0));
+		}
+
 		// target.fi = ((Integer)_$$_tmpls[i].unpack(_$$_pk)).intValue();
 		Class returnType = field.getType();
 		boolean isPrim = returnType.isPrimitive();
@@ -423,6 +441,19 @@ class DynamicCodeGen extends DynamicCodeGenBase implements Constants {
 				isPrim ? ")." + getPrimTypeValueMethodName(returnType) + "()"
 						: "" };
 		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_03, args));
+
+		if(isOptional) {
+			// }
+			sb.append(CHAR_NAME_RIGHT_CURLY_BRACKET);
+		}
+	}
+
+	private void insertCodeOfUnpackTrails(StringBuilder sb, int length) {
+		// for(int _$$_n = length; _$$_n < _$$_L; _$$_n++) {
+		//     $1.unpackObject();
+		// }
+		Object[] args0 = new Object[] { length };
+		sb.append(String.format(STATEMENT_TMPL_UNPACKERMETHODBODY_09, args0));
 	}
 
 	private void insertOrdinalEnumUnpackMethodBody(StringBuilder sb,

From 2af7df58657b6e85c5d69f8127972cf254a064ec Mon Sep 17 00:00:00 2001
From: frsyuki 
Date: Mon, 25 Oct 2010 14:49:14 +0900
Subject: [PATCH 0908/1648] java: fixes MessagePack.unpack method

---
 java/src/main/java/org/msgpack/MessagePack.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/src/main/java/org/msgpack/MessagePack.java b/java/src/main/java/org/msgpack/MessagePack.java
index 449f8a0..e4bc627 100644
--- a/java/src/main/java/org/msgpack/MessagePack.java
+++ b/java/src/main/java/org/msgpack/MessagePack.java
@@ -56,7 +56,7 @@ public class MessagePack {
 	}
 
 
-	public static MessagePackObject unpack(byte[] buffer) throws IOException {
+	public static MessagePackObject unpack(byte[] buffer) throws MessageTypeException {
 		Unpacker pac = new Unpacker();
 		pac.wrap(buffer);
 		try {

From 70982e204cb53b996e4f7bd4fd078b6fa91307df Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Oct 2010 01:26:06 +0900
Subject: [PATCH 0909/1648] Add `object_hook` option to unpack and `default`
 option to pack. (see simplejson for how to use).

---
 python/msgpack/_msgpack.pyx      | 56 ++++++++++++++++++++++++++------
 python/msgpack/unpack.h          | 14 ++++++++
 python/msgpack/unpack_template.h |  1 +
 python/test/test_obj.py          | 31 ++++++++++++++++++
 4 files changed, 92 insertions(+), 10 deletions(-)
 create mode 100644 python/test/test_obj.py

diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx
index 66869c8..fb7f0c1 100644
--- a/python/msgpack/_msgpack.pyx
+++ b/python/msgpack/_msgpack.pyx
@@ -20,6 +20,9 @@ cdef extern from "Python.h":
     cdef bint PyFloat_Check(object o)
     cdef bint PyBytes_Check(object o)
     cdef bint PyUnicode_Check(object o)
+    cdef bint PyCallable_Check(object o)
+    cdef void Py_INCREF(object o)
+    cdef void Py_DECREF(object o)
 
 cdef extern from "stdlib.h":
     void* malloc(size_t)
@@ -60,6 +63,7 @@ cdef class Packer(object):
         astream.write(packer.pack(b))
     """
     cdef msgpack_packer pk
+    cdef object default
 
     def __cinit__(self):
         cdef int buf_size = 1024*1024
@@ -67,6 +71,12 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
+    def __init__(self, default=None):
+        if default is not None:
+            if not PyCallable_Check(default):
+                raise TypeError("default must be a callable.")
+        self.default = default
+
     def __dealloc__(self):
         free(self.pk.buf);
 
@@ -126,9 +136,18 @@ cdef class Packer(object):
                 for v in o:
                     ret = self._pack(v)
                     if ret != 0: break
+        elif self.default is not None:
+            o = self.default(o)
+            d = o
+            ret = msgpack_pack_map(&self.pk, len(d))
+            if ret == 0:
+                for k,v in d.items():
+                    ret = self._pack(k)
+                    if ret != 0: break
+                    ret = self._pack(v)
+                    if ret != 0: break
         else:
-            # TODO: Serialize with defalt() like simplejson.
-            raise TypeError, "can't serialize %r" % (o,)
+            raise TypeError("can't serialize %r" % (o,))
         return ret
 
     def pack(self, object obj):
@@ -141,14 +160,14 @@ cdef class Packer(object):
         return buf
 
 
-def pack(object o, object stream):
+def pack(object o, object stream, default=None):
     """pack an object `o` and write it to stream)."""
-    packer = Packer()
+    packer = Packer(default)
     stream.write(packer.pack(o))
 
-def packb(object o):
+def packb(object o, default=None):
     """pack o and return packed bytes."""
-    packer = Packer()
+    packer = Packer(default=default)
     return packer.pack(o)
 
 packs = packb
@@ -156,6 +175,7 @@ packs = packb
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
         int use_list
+        PyObject* object_hook
 
     ctypedef struct template_context:
         msgpack_user user
@@ -170,7 +190,7 @@ cdef extern from "unpack.h":
     object template_data(template_context* ctx)
 
 
-def unpackb(bytes packed_bytes):
+def unpackb(bytes packed_bytes, object object_hook=None):
     """Unpack packed_bytes to object. Returns an unpacked object."""
     cdef const_char_ptr p = packed_bytes
     cdef template_context ctx
@@ -178,7 +198,16 @@ def unpackb(bytes packed_bytes):
     cdef int ret
     template_init(&ctx)
     ctx.user.use_list = 0
+    ctx.user.object_hook = NULL
+    if object_hook is not None:
+        if not PyCallable_Check(object_hook):
+            raise TypeError("object_hook must be a callable.")
+        Py_INCREF(object_hook)
+        ctx.user.object_hook = object_hook
     ret = template_execute(&ctx, p, len(packed_bytes), &off)
+    if object_hook is not None:
+        pass
+        #Py_DECREF(object_hook)
     if ret == 1:
         return template_data(&ctx)
     else:
@@ -186,10 +215,10 @@ def unpackb(bytes packed_bytes):
 
 unpacks = unpackb
 
-def unpack(object stream):
+def unpack(object stream, object object_hook=None):
     """unpack an object from stream."""
     packed = stream.read()
-    return unpackb(packed)
+    return unpackb(packed, object_hook=object_hook)
 
 cdef class UnpackIterator(object):
     cdef object unpacker
@@ -234,6 +263,7 @@ cdef class Unpacker(object):
     cdef int read_size
     cdef object waiting_bytes
     cdef bint use_list
+    cdef object object_hook
 
     def __cinit__(self):
         self.buf = NULL
@@ -242,7 +272,8 @@ cdef class Unpacker(object):
         if self.buf:
             free(self.buf);
 
-    def __init__(self, file_like=None, int read_size=0, bint use_list=0):
+    def __init__(self, file_like=None, int read_size=0, bint use_list=0,
+                 object object_hook=None):
         if read_size == 0:
             read_size = 1024*1024
         self.use_list = use_list
@@ -255,6 +286,11 @@ cdef class Unpacker(object):
         self.buf_tail = 0
         template_init(&self.ctx)
         self.ctx.user.use_list = use_list
+        self.ctx.user.object_hook = NULL
+        if object_hook is not None:
+            if not PyCallable_Check(object_hook):
+                raise TypeError("object_hook must be a callable.")
+            self.ctx.user.object_hook = object_hook
 
     def feed(self, bytes next_bytes):
         self.waiting_bytes.append(next_bytes)
diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h
index 9eb8ce7..e4c03bd 100644
--- a/python/msgpack/unpack.h
+++ b/python/msgpack/unpack.h
@@ -21,6 +21,7 @@
 
 typedef struct unpack_user {
     int use_list;
+    PyObject *object_hook;
 } unpack_user;
 
 
@@ -172,6 +173,19 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje
     return -1;
 }
 
+//static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
+int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
+{
+    if (u->object_hook) {
+        PyObject *arglist = Py_BuildValue("(O)", *c);
+        Py_INCREF(*c);
+        *c = PyEval_CallObject(u->object_hook, arglist);
+        Py_DECREF(arglist);
+        return 0;
+    }
+    return -1;
+}
+
 static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
 {
     PyObject *py;
diff --git a/python/msgpack/unpack_template.h b/python/msgpack/unpack_template.h
index ca6e1f3..1fdedd7 100644
--- a/python/msgpack/unpack_template.h
+++ b/python/msgpack/unpack_template.h
@@ -317,6 +317,7 @@ _push:
 	case CT_MAP_VALUE:
 		if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
 		if(--c->count == 0) {
+            msgpack_unpack_callback(_map_end)(user, &c->obj);
 			obj = c->obj;
 			--top;
 			/*printf("stack pop %d\n", top);*/
diff --git a/python/test/test_obj.py b/python/test/test_obj.py
new file mode 100644
index 0000000..64a6390
--- /dev/null
+++ b/python/test/test_obj.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+
+from msgpack import packs, unpacks
+
+def _decode_complex(obj):
+    if '__complex__' in obj:
+        return complex(obj['real'], obj['imag'])
+    return obj
+
+def _encode_complex(obj):
+    if isinstance(obj, complex):
+        return {'__complex__': True, 'real': 1, 'imag': 2}
+    return obj
+
+def test_encode_hook():
+    packed = packs([3, 1+2j], default=_encode_complex)
+    unpacked = unpacks(packed)
+    eq_(unpacked[1], {'__complex__': True, 'real': 1, 'imag': 2})
+
+def test_decode_hook():
+    packed = packs([3, {'__complex__': True, 'real': 1, 'imag': 2}])
+    unpacked = unpacks(packed, object_hook=_decode_complex)
+    eq_(unpacked[1], 1+2j)
+
+if __name__ == '__main__':
+    #main()
+    test_decode_hook()

From fa157082ac8db71e3312ca97fe1ceb7f56546fcb Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Oct 2010 01:26:06 +0900
Subject: [PATCH 0910/1648] Add `object_hook` option to unpack and `default`
 option to pack. (see simplejson for how to use).

---
 msgpack/_msgpack.pyx      | 56 ++++++++++++++++++++++++++++++++-------
 msgpack/unpack.h          | 14 ++++++++++
 msgpack/unpack_template.h |  1 +
 test/test_obj.py          | 31 ++++++++++++++++++++++
 4 files changed, 92 insertions(+), 10 deletions(-)
 create mode 100644 test/test_obj.py

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 66869c8..fb7f0c1 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -20,6 +20,9 @@ cdef extern from "Python.h":
     cdef bint PyFloat_Check(object o)
     cdef bint PyBytes_Check(object o)
     cdef bint PyUnicode_Check(object o)
+    cdef bint PyCallable_Check(object o)
+    cdef void Py_INCREF(object o)
+    cdef void Py_DECREF(object o)
 
 cdef extern from "stdlib.h":
     void* malloc(size_t)
@@ -60,6 +63,7 @@ cdef class Packer(object):
         astream.write(packer.pack(b))
     """
     cdef msgpack_packer pk
+    cdef object default
 
     def __cinit__(self):
         cdef int buf_size = 1024*1024
@@ -67,6 +71,12 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
+    def __init__(self, default=None):
+        if default is not None:
+            if not PyCallable_Check(default):
+                raise TypeError("default must be a callable.")
+        self.default = default
+
     def __dealloc__(self):
         free(self.pk.buf);
 
@@ -126,9 +136,18 @@ cdef class Packer(object):
                 for v in o:
                     ret = self._pack(v)
                     if ret != 0: break
+        elif self.default is not None:
+            o = self.default(o)
+            d = o
+            ret = msgpack_pack_map(&self.pk, len(d))
+            if ret == 0:
+                for k,v in d.items():
+                    ret = self._pack(k)
+                    if ret != 0: break
+                    ret = self._pack(v)
+                    if ret != 0: break
         else:
-            # TODO: Serialize with defalt() like simplejson.
-            raise TypeError, "can't serialize %r" % (o,)
+            raise TypeError("can't serialize %r" % (o,))
         return ret
 
     def pack(self, object obj):
@@ -141,14 +160,14 @@ cdef class Packer(object):
         return buf
 
 
-def pack(object o, object stream):
+def pack(object o, object stream, default=None):
     """pack an object `o` and write it to stream)."""
-    packer = Packer()
+    packer = Packer(default)
     stream.write(packer.pack(o))
 
-def packb(object o):
+def packb(object o, default=None):
     """pack o and return packed bytes."""
-    packer = Packer()
+    packer = Packer(default=default)
     return packer.pack(o)
 
 packs = packb
@@ -156,6 +175,7 @@ packs = packb
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
         int use_list
+        PyObject* object_hook
 
     ctypedef struct template_context:
         msgpack_user user
@@ -170,7 +190,7 @@ cdef extern from "unpack.h":
     object template_data(template_context* ctx)
 
 
-def unpackb(bytes packed_bytes):
+def unpackb(bytes packed_bytes, object object_hook=None):
     """Unpack packed_bytes to object. Returns an unpacked object."""
     cdef const_char_ptr p = packed_bytes
     cdef template_context ctx
@@ -178,7 +198,16 @@ def unpackb(bytes packed_bytes):
     cdef int ret
     template_init(&ctx)
     ctx.user.use_list = 0
+    ctx.user.object_hook = NULL
+    if object_hook is not None:
+        if not PyCallable_Check(object_hook):
+            raise TypeError("object_hook must be a callable.")
+        Py_INCREF(object_hook)
+        ctx.user.object_hook = object_hook
     ret = template_execute(&ctx, p, len(packed_bytes), &off)
+    if object_hook is not None:
+        pass
+        #Py_DECREF(object_hook)
     if ret == 1:
         return template_data(&ctx)
     else:
@@ -186,10 +215,10 @@ def unpackb(bytes packed_bytes):
 
 unpacks = unpackb
 
-def unpack(object stream):
+def unpack(object stream, object object_hook=None):
     """unpack an object from stream."""
     packed = stream.read()
-    return unpackb(packed)
+    return unpackb(packed, object_hook=object_hook)
 
 cdef class UnpackIterator(object):
     cdef object unpacker
@@ -234,6 +263,7 @@ cdef class Unpacker(object):
     cdef int read_size
     cdef object waiting_bytes
     cdef bint use_list
+    cdef object object_hook
 
     def __cinit__(self):
         self.buf = NULL
@@ -242,7 +272,8 @@ cdef class Unpacker(object):
         if self.buf:
             free(self.buf);
 
-    def __init__(self, file_like=None, int read_size=0, bint use_list=0):
+    def __init__(self, file_like=None, int read_size=0, bint use_list=0,
+                 object object_hook=None):
         if read_size == 0:
             read_size = 1024*1024
         self.use_list = use_list
@@ -255,6 +286,11 @@ cdef class Unpacker(object):
         self.buf_tail = 0
         template_init(&self.ctx)
         self.ctx.user.use_list = use_list
+        self.ctx.user.object_hook = NULL
+        if object_hook is not None:
+            if not PyCallable_Check(object_hook):
+                raise TypeError("object_hook must be a callable.")
+            self.ctx.user.object_hook = object_hook
 
     def feed(self, bytes next_bytes):
         self.waiting_bytes.append(next_bytes)
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 9eb8ce7..e4c03bd 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -21,6 +21,7 @@
 
 typedef struct unpack_user {
     int use_list;
+    PyObject *object_hook;
 } unpack_user;
 
 
@@ -172,6 +173,19 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje
     return -1;
 }
 
+//static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
+int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
+{
+    if (u->object_hook) {
+        PyObject *arglist = Py_BuildValue("(O)", *c);
+        Py_INCREF(*c);
+        *c = PyEval_CallObject(u->object_hook, arglist);
+        Py_DECREF(arglist);
+        return 0;
+    }
+    return -1;
+}
+
 static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
 {
     PyObject *py;
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index ca6e1f3..1fdedd7 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -317,6 +317,7 @@ _push:
 	case CT_MAP_VALUE:
 		if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
 		if(--c->count == 0) {
+            msgpack_unpack_callback(_map_end)(user, &c->obj);
 			obj = c->obj;
 			--top;
 			/*printf("stack pop %d\n", top);*/
diff --git a/test/test_obj.py b/test/test_obj.py
new file mode 100644
index 0000000..64a6390
--- /dev/null
+++ b/test/test_obj.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+
+from msgpack import packs, unpacks
+
+def _decode_complex(obj):
+    if '__complex__' in obj:
+        return complex(obj['real'], obj['imag'])
+    return obj
+
+def _encode_complex(obj):
+    if isinstance(obj, complex):
+        return {'__complex__': True, 'real': 1, 'imag': 2}
+    return obj
+
+def test_encode_hook():
+    packed = packs([3, 1+2j], default=_encode_complex)
+    unpacked = unpacks(packed)
+    eq_(unpacked[1], {'__complex__': True, 'real': 1, 'imag': 2})
+
+def test_decode_hook():
+    packed = packs([3, {'__complex__': True, 'real': 1, 'imag': 2}])
+    unpacked = unpacks(packed, object_hook=_decode_complex)
+    eq_(unpacked[1], 1+2j)
+
+if __name__ == '__main__':
+    #main()
+    test_decode_hook()

From 6fa609be3fc6cf771bb880bf8c844f874c7acffb Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Oct 2010 01:31:27 +0900
Subject: [PATCH 0911/1648] Add test for Python3.

---
 python/test3/test_obj.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 python/test3/test_obj.py

diff --git a/python/test3/test_obj.py b/python/test3/test_obj.py
new file mode 100644
index 0000000..972d1ca
--- /dev/null
+++ b/python/test3/test_obj.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+
+from msgpack import packs, unpacks
+
+def _decode_complex(obj):
+    if b'__complex__' in obj:
+        return complex(obj[b'real'], obj[b'imag'])
+    return obj
+
+def _encode_complex(obj):
+    if isinstance(obj, complex):
+        return {b'__complex__': True, b'real': 1, b'imag': 2}
+    return obj
+
+def test_encode_hook():
+    packed = packs([3, 1+2j], default=_encode_complex)
+    unpacked = unpacks(packed)
+    eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2})
+
+def test_decode_hook():
+    packed = packs([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
+    unpacked = unpacks(packed, object_hook=_decode_complex)
+    eq_(unpacked[1], 1+2j)
+
+if __name__ == '__main__':
+    #main()
+    test_decode_hook()

From c355f224dc4293683a426e8620a07724ab0e75ac Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Oct 2010 01:31:27 +0900
Subject: [PATCH 0912/1648] Add test for Python3.

---
 test3/test_obj.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 test3/test_obj.py

diff --git a/test3/test_obj.py b/test3/test_obj.py
new file mode 100644
index 0000000..972d1ca
--- /dev/null
+++ b/test3/test_obj.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+
+from msgpack import packs, unpacks
+
+def _decode_complex(obj):
+    if b'__complex__' in obj:
+        return complex(obj[b'real'], obj[b'imag'])
+    return obj
+
+def _encode_complex(obj):
+    if isinstance(obj, complex):
+        return {b'__complex__': True, b'real': 1, b'imag': 2}
+    return obj
+
+def test_encode_hook():
+    packed = packs([3, 1+2j], default=_encode_complex)
+    unpacked = unpacks(packed)
+    eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2})
+
+def test_decode_hook():
+    packed = packs([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
+    unpacked = unpacks(packed, object_hook=_decode_complex)
+    eq_(unpacked[1], 1+2j)
+
+if __name__ == '__main__':
+    #main()
+    test_decode_hook()

From d8e3575a465d28c91efb285035fb8e4aa15b27da Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Oct 2010 01:32:08 +0900
Subject: [PATCH 0913/1648] Remove unnecessary refcount manipulation.

---
 python/msgpack/_msgpack.pyx | 4 ----
 python/msgpack/unpack.h     | 1 -
 2 files changed, 5 deletions(-)

diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx
index fb7f0c1..24e4f8b 100644
--- a/python/msgpack/_msgpack.pyx
+++ b/python/msgpack/_msgpack.pyx
@@ -202,12 +202,8 @@ def unpackb(bytes packed_bytes, object object_hook=None):
     if object_hook is not None:
         if not PyCallable_Check(object_hook):
             raise TypeError("object_hook must be a callable.")
-        Py_INCREF(object_hook)
         ctx.user.object_hook = object_hook
     ret = template_execute(&ctx, p, len(packed_bytes), &off)
-    if object_hook is not None:
-        pass
-        #Py_DECREF(object_hook)
     if ret == 1:
         return template_data(&ctx)
     else:
diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h
index e4c03bd..404ee5a 100644
--- a/python/msgpack/unpack.h
+++ b/python/msgpack/unpack.h
@@ -178,7 +178,6 @@ int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->object_hook) {
         PyObject *arglist = Py_BuildValue("(O)", *c);
-        Py_INCREF(*c);
         *c = PyEval_CallObject(u->object_hook, arglist);
         Py_DECREF(arglist);
         return 0;

From 3980d381f7ad44b14483dc90c0f6f0a36804c290 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Oct 2010 01:32:08 +0900
Subject: [PATCH 0914/1648] Remove unnecessary refcount manipulation.

---
 msgpack/_msgpack.pyx | 4 ----
 msgpack/unpack.h     | 1 -
 2 files changed, 5 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index fb7f0c1..24e4f8b 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -202,12 +202,8 @@ def unpackb(bytes packed_bytes, object object_hook=None):
     if object_hook is not None:
         if not PyCallable_Check(object_hook):
             raise TypeError("object_hook must be a callable.")
-        Py_INCREF(object_hook)
         ctx.user.object_hook = object_hook
     ret = template_execute(&ctx, p, len(packed_bytes), &off)
-    if object_hook is not None:
-        pass
-        #Py_DECREF(object_hook)
     if ret == 1:
         return template_data(&ctx)
     else:
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index e4c03bd..404ee5a 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -178,7 +178,6 @@ int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->object_hook) {
         PyObject *arglist = Py_BuildValue("(O)", *c);
-        Py_INCREF(*c);
         *c = PyEval_CallObject(u->object_hook, arglist);
         Py_DECREF(arglist);
         return 0;

From 063d51c6625ba6384cfb644e55ac729dbda2681b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Oct 2010 01:49:00 +0900
Subject: [PATCH 0915/1648] Add check for recursion limit and default hook
 result.

---
 python/msgpack/_msgpack.pyx | 24 ++++++++++--------------
 python/test/test_obj.py     |  8 +++++++-
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx
index 24e4f8b..0abdd51 100644
--- a/python/msgpack/_msgpack.pyx
+++ b/python/msgpack/_msgpack.pyx
@@ -80,7 +80,7 @@ cdef class Packer(object):
     def __dealloc__(self):
         free(self.pk.buf);
 
-    cdef int _pack(self, object o) except -1:
+    cdef int _pack(self, object o, int nest_limit=511, default=None) except -1:
         cdef long long llval
         cdef unsigned long long ullval
         cdef long longval
@@ -89,6 +89,9 @@ cdef class Packer(object):
         cdef int ret
         cdef dict d
 
+        if nest_limit < 0:
+            raise ValueError("Too deep.")
+
         if o is None:
             ret = msgpack_pack_nil(&self.pk)
             #elif PyBool_Check(o):
@@ -126,33 +129,26 @@ cdef class Packer(object):
             ret = msgpack_pack_map(&self.pk, len(d))
             if ret == 0:
                 for k,v in d.items():
-                    ret = self._pack(k)
+                    ret = self._pack(k, nest_limit-1, default)
                     if ret != 0: break
-                    ret = self._pack(v)
+                    ret = self._pack(v, nest_limit-1, default)
                     if ret != 0: break
         elif PySequence_Check(o):
             ret = msgpack_pack_array(&self.pk, len(o))
             if ret == 0:
                 for v in o:
-                    ret = self._pack(v)
+                    ret = self._pack(v, nest_limit-1, default)
                     if ret != 0: break
-        elif self.default is not None:
+        elif default is not None:
             o = self.default(o)
-            d = o
-            ret = msgpack_pack_map(&self.pk, len(d))
-            if ret == 0:
-                for k,v in d.items():
-                    ret = self._pack(k)
-                    if ret != 0: break
-                    ret = self._pack(v)
-                    if ret != 0: break
+            ret = self._pack(o, nest_limit)
         else:
             raise TypeError("can't serialize %r" % (o,))
         return ret
 
     def pack(self, object obj):
         cdef int ret
-        ret = self._pack(obj)
+        ret = self._pack(obj, self.default)
         if ret:
             raise TypeError
         buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
diff --git a/python/test/test_obj.py b/python/test/test_obj.py
index 64a6390..28edacb 100644
--- a/python/test/test_obj.py
+++ b/python/test/test_obj.py
@@ -26,6 +26,12 @@ def test_decode_hook():
     unpacked = unpacks(packed, object_hook=_decode_complex)
     eq_(unpacked[1], 1+2j)
 
+@raises(TypeError)
+def test_bad_hook():
+    packed = packs([3, 1+2j], default=lambda o: o)
+    unpacked = unpacks(packed)
+
 if __name__ == '__main__':
-    #main()
     test_decode_hook()
+    test_encode_hook()
+    test_bad_hook()

From 0076d42a0ddeb3601f1a0c806c7874da2110a986 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Oct 2010 01:49:00 +0900
Subject: [PATCH 0916/1648] Add check for recursion limit and default hook
 result.

---
 msgpack/_msgpack.pyx | 24 ++++++++++--------------
 test/test_obj.py     |  8 +++++++-
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 24e4f8b..0abdd51 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -80,7 +80,7 @@ cdef class Packer(object):
     def __dealloc__(self):
         free(self.pk.buf);
 
-    cdef int _pack(self, object o) except -1:
+    cdef int _pack(self, object o, int nest_limit=511, default=None) except -1:
         cdef long long llval
         cdef unsigned long long ullval
         cdef long longval
@@ -89,6 +89,9 @@ cdef class Packer(object):
         cdef int ret
         cdef dict d
 
+        if nest_limit < 0:
+            raise ValueError("Too deep.")
+
         if o is None:
             ret = msgpack_pack_nil(&self.pk)
             #elif PyBool_Check(o):
@@ -126,33 +129,26 @@ cdef class Packer(object):
             ret = msgpack_pack_map(&self.pk, len(d))
             if ret == 0:
                 for k,v in d.items():
-                    ret = self._pack(k)
+                    ret = self._pack(k, nest_limit-1, default)
                     if ret != 0: break
-                    ret = self._pack(v)
+                    ret = self._pack(v, nest_limit-1, default)
                     if ret != 0: break
         elif PySequence_Check(o):
             ret = msgpack_pack_array(&self.pk, len(o))
             if ret == 0:
                 for v in o:
-                    ret = self._pack(v)
+                    ret = self._pack(v, nest_limit-1, default)
                     if ret != 0: break
-        elif self.default is not None:
+        elif default is not None:
             o = self.default(o)
-            d = o
-            ret = msgpack_pack_map(&self.pk, len(d))
-            if ret == 0:
-                for k,v in d.items():
-                    ret = self._pack(k)
-                    if ret != 0: break
-                    ret = self._pack(v)
-                    if ret != 0: break
+            ret = self._pack(o, nest_limit)
         else:
             raise TypeError("can't serialize %r" % (o,))
         return ret
 
     def pack(self, object obj):
         cdef int ret
-        ret = self._pack(obj)
+        ret = self._pack(obj, self.default)
         if ret:
             raise TypeError
         buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
diff --git a/test/test_obj.py b/test/test_obj.py
index 64a6390..28edacb 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -26,6 +26,12 @@ def test_decode_hook():
     unpacked = unpacks(packed, object_hook=_decode_complex)
     eq_(unpacked[1], 1+2j)
 
+@raises(TypeError)
+def test_bad_hook():
+    packed = packs([3, 1+2j], default=lambda o: o)
+    unpacked = unpacks(packed)
+
 if __name__ == '__main__':
-    #main()
     test_decode_hook()
+    test_encode_hook()
+    test_bad_hook()

From e1711ffcf203d43ee71cfb5eea8e65f3e50b145b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Oct 2010 02:09:52 +0900
Subject: [PATCH 0917/1648] Add list_hook option to unpacker.

---
 python/msgpack/_msgpack.pyx      | 27 +++++++++++++++++++--------
 python/msgpack/unpack.h          | 17 +++++++++++++----
 python/msgpack/unpack_template.h |  3 ++-
 python/test/test_obj.py          |  9 +++++++++
 4 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx
index 0abdd51..e9d6c7b 100644
--- a/python/msgpack/_msgpack.pyx
+++ b/python/msgpack/_msgpack.pyx
@@ -52,6 +52,7 @@ cdef extern from "pack.h":
     int msgpack_pack_raw(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
 
+cdef int DEFAULT_RECURSE_LIMIT=511
 
 cdef class Packer(object):
     """MessagePack Packer
@@ -80,7 +81,8 @@ cdef class Packer(object):
     def __dealloc__(self):
         free(self.pk.buf);
 
-    cdef int _pack(self, object o, int nest_limit=511, default=None) except -1:
+    cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT,
+                   default=None) except -1:
         cdef long long llval
         cdef unsigned long long ullval
         cdef long longval
@@ -148,7 +150,7 @@ cdef class Packer(object):
 
     def pack(self, object obj):
         cdef int ret
-        ret = self._pack(obj, self.default)
+        ret = self._pack(obj, DEFAULT_RECURSE_LIMIT, self.default)
         if ret:
             raise TypeError
         buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
@@ -172,6 +174,7 @@ cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
         int use_list
         PyObject* object_hook
+        PyObject* list_hook
 
     ctypedef struct template_context:
         msgpack_user user
@@ -186,7 +189,7 @@ cdef extern from "unpack.h":
     object template_data(template_context* ctx)
 
 
-def unpackb(bytes packed_bytes, object object_hook=None):
+def unpackb(bytes packed_bytes, object object_hook=None, object list_hook=None):
     """Unpack packed_bytes to object. Returns an unpacked object."""
     cdef const_char_ptr p = packed_bytes
     cdef template_context ctx
@@ -194,11 +197,15 @@ def unpackb(bytes packed_bytes, object object_hook=None):
     cdef int ret
     template_init(&ctx)
     ctx.user.use_list = 0
-    ctx.user.object_hook = NULL
+    ctx.user.object_hook = ctx.user.list_hook = NULL
     if object_hook is not None:
         if not PyCallable_Check(object_hook):
             raise TypeError("object_hook must be a callable.")
         ctx.user.object_hook = object_hook
+    if list_hook is not None:
+        if not PyCallable_Check(list_hook):
+            raise TypeError("list_hook must be a callable.")
+        ctx.user.list_hook = list_hook
     ret = template_execute(&ctx, p, len(packed_bytes), &off)
     if ret == 1:
         return template_data(&ctx)
@@ -207,10 +214,10 @@ def unpackb(bytes packed_bytes, object object_hook=None):
 
 unpacks = unpackb
 
-def unpack(object stream, object object_hook=None):
+def unpack(object stream, object object_hook=None, object list_hook=None):
     """unpack an object from stream."""
     packed = stream.read()
-    return unpackb(packed, object_hook=object_hook)
+    return unpackb(packed, object_hook=object_hook, list_hook=list_hook)
 
 cdef class UnpackIterator(object):
     cdef object unpacker
@@ -265,7 +272,7 @@ cdef class Unpacker(object):
             free(self.buf);
 
     def __init__(self, file_like=None, int read_size=0, bint use_list=0,
-                 object object_hook=None):
+                 object object_hook=None, object list_hook=None):
         if read_size == 0:
             read_size = 1024*1024
         self.use_list = use_list
@@ -278,11 +285,15 @@ cdef class Unpacker(object):
         self.buf_tail = 0
         template_init(&self.ctx)
         self.ctx.user.use_list = use_list
-        self.ctx.user.object_hook = NULL
+        self.ctx.user.object_hook = self.ctx.user.list_hook = NULL
         if object_hook is not None:
             if not PyCallable_Check(object_hook):
                 raise TypeError("object_hook must be a callable.")
             self.ctx.user.object_hook = object_hook
+        if list_hook is not None:
+            if not PyCallable_Check(list_hook):
+                raise TypeError("object_hook must be a callable.")
+            self.ctx.user.list_hook = list_hook
 
     def feed(self, bytes next_bytes):
         self.waiting_bytes.append(next_bytes)
diff --git a/python/msgpack/unpack.h b/python/msgpack/unpack.h
index 404ee5a..453ec2b 100644
--- a/python/msgpack/unpack.h
+++ b/python/msgpack/unpack.h
@@ -22,6 +22,7 @@
 typedef struct unpack_user {
     int use_list;
     PyObject *object_hook;
+    PyObject *list_hook;
 } unpack_user;
 
 
@@ -154,6 +155,16 @@ static inline int template_callback_array_item(unpack_user* u, unsigned int curr
     return 0;
 }
 
+static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
+{
+    if (u->list_hook) {
+        PyObject *arglist = Py_BuildValue("(O)", *c);
+        *c = PyEval_CallObject(u->list_hook, arglist);
+        Py_DECREF(arglist);
+    }
+    return 0;
+}
+
 static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
 {
     PyObject *p = PyDict_New();
@@ -173,16 +184,14 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje
     return -1;
 }
 
-//static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
-int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
+static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->object_hook) {
         PyObject *arglist = Py_BuildValue("(O)", *c);
         *c = PyEval_CallObject(u->object_hook, arglist);
         Py_DECREF(arglist);
-        return 0;
     }
-    return -1;
+    return 0;
 }
 
 static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
diff --git a/python/msgpack/unpack_template.h b/python/msgpack/unpack_template.h
index 1fdedd7..7a2288f 100644
--- a/python/msgpack/unpack_template.h
+++ b/python/msgpack/unpack_template.h
@@ -304,6 +304,7 @@ _push:
 	case CT_ARRAY_ITEM:
 		if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; }
 		if(++c->curr == c->count) {
+			msgpack_unpack_callback(_array_end)(user, &c->obj);
 			obj = c->obj;
 			--top;
 			/*printf("stack pop %d\n", top);*/
@@ -317,7 +318,7 @@ _push:
 	case CT_MAP_VALUE:
 		if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
 		if(--c->count == 0) {
-            msgpack_unpack_callback(_map_end)(user, &c->obj);
+			msgpack_unpack_callback(_map_end)(user, &c->obj);
 			obj = c->obj;
 			--top;
 			/*printf("stack pop %d\n", top);*/
diff --git a/python/test/test_obj.py b/python/test/test_obj.py
index 28edacb..bc85736 100644
--- a/python/test/test_obj.py
+++ b/python/test/test_obj.py
@@ -31,7 +31,16 @@ def test_bad_hook():
     packed = packs([3, 1+2j], default=lambda o: o)
     unpacked = unpacks(packed)
 
+def _arr_to_str(arr):
+    return ''.join(str(c) for c in arr)
+
+def test_array_hook():
+    packed = packs([1,2,3])
+    unpacked = unpacks(packed, list_hook=_arr_to_str)
+    eq_(unpacked, '123')
+
 if __name__ == '__main__':
     test_decode_hook()
     test_encode_hook()
     test_bad_hook()
+    test_array_hook()

From 3d8978417a177c280b837d72054848e8dd4bc649 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Oct 2010 02:09:52 +0900
Subject: [PATCH 0918/1648] Add list_hook option to unpacker.

---
 msgpack/_msgpack.pyx      | 27 +++++++++++++++++++--------
 msgpack/unpack.h          | 17 +++++++++++++----
 msgpack/unpack_template.h |  3 ++-
 test/test_obj.py          |  9 +++++++++
 4 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 0abdd51..e9d6c7b 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -52,6 +52,7 @@ cdef extern from "pack.h":
     int msgpack_pack_raw(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
 
+cdef int DEFAULT_RECURSE_LIMIT=511
 
 cdef class Packer(object):
     """MessagePack Packer
@@ -80,7 +81,8 @@ cdef class Packer(object):
     def __dealloc__(self):
         free(self.pk.buf);
 
-    cdef int _pack(self, object o, int nest_limit=511, default=None) except -1:
+    cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT,
+                   default=None) except -1:
         cdef long long llval
         cdef unsigned long long ullval
         cdef long longval
@@ -148,7 +150,7 @@ cdef class Packer(object):
 
     def pack(self, object obj):
         cdef int ret
-        ret = self._pack(obj, self.default)
+        ret = self._pack(obj, DEFAULT_RECURSE_LIMIT, self.default)
         if ret:
             raise TypeError
         buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
@@ -172,6 +174,7 @@ cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
         int use_list
         PyObject* object_hook
+        PyObject* list_hook
 
     ctypedef struct template_context:
         msgpack_user user
@@ -186,7 +189,7 @@ cdef extern from "unpack.h":
     object template_data(template_context* ctx)
 
 
-def unpackb(bytes packed_bytes, object object_hook=None):
+def unpackb(bytes packed_bytes, object object_hook=None, object list_hook=None):
     """Unpack packed_bytes to object. Returns an unpacked object."""
     cdef const_char_ptr p = packed_bytes
     cdef template_context ctx
@@ -194,11 +197,15 @@ def unpackb(bytes packed_bytes, object object_hook=None):
     cdef int ret
     template_init(&ctx)
     ctx.user.use_list = 0
-    ctx.user.object_hook = NULL
+    ctx.user.object_hook = ctx.user.list_hook = NULL
     if object_hook is not None:
         if not PyCallable_Check(object_hook):
             raise TypeError("object_hook must be a callable.")
         ctx.user.object_hook = object_hook
+    if list_hook is not None:
+        if not PyCallable_Check(list_hook):
+            raise TypeError("list_hook must be a callable.")
+        ctx.user.list_hook = list_hook
     ret = template_execute(&ctx, p, len(packed_bytes), &off)
     if ret == 1:
         return template_data(&ctx)
@@ -207,10 +214,10 @@ def unpackb(bytes packed_bytes, object object_hook=None):
 
 unpacks = unpackb
 
-def unpack(object stream, object object_hook=None):
+def unpack(object stream, object object_hook=None, object list_hook=None):
     """unpack an object from stream."""
     packed = stream.read()
-    return unpackb(packed, object_hook=object_hook)
+    return unpackb(packed, object_hook=object_hook, list_hook=list_hook)
 
 cdef class UnpackIterator(object):
     cdef object unpacker
@@ -265,7 +272,7 @@ cdef class Unpacker(object):
             free(self.buf);
 
     def __init__(self, file_like=None, int read_size=0, bint use_list=0,
-                 object object_hook=None):
+                 object object_hook=None, object list_hook=None):
         if read_size == 0:
             read_size = 1024*1024
         self.use_list = use_list
@@ -278,11 +285,15 @@ cdef class Unpacker(object):
         self.buf_tail = 0
         template_init(&self.ctx)
         self.ctx.user.use_list = use_list
-        self.ctx.user.object_hook = NULL
+        self.ctx.user.object_hook = self.ctx.user.list_hook = NULL
         if object_hook is not None:
             if not PyCallable_Check(object_hook):
                 raise TypeError("object_hook must be a callable.")
             self.ctx.user.object_hook = object_hook
+        if list_hook is not None:
+            if not PyCallable_Check(list_hook):
+                raise TypeError("object_hook must be a callable.")
+            self.ctx.user.list_hook = list_hook
 
     def feed(self, bytes next_bytes):
         self.waiting_bytes.append(next_bytes)
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 404ee5a..453ec2b 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -22,6 +22,7 @@
 typedef struct unpack_user {
     int use_list;
     PyObject *object_hook;
+    PyObject *list_hook;
 } unpack_user;
 
 
@@ -154,6 +155,16 @@ static inline int template_callback_array_item(unpack_user* u, unsigned int curr
     return 0;
 }
 
+static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
+{
+    if (u->list_hook) {
+        PyObject *arglist = Py_BuildValue("(O)", *c);
+        *c = PyEval_CallObject(u->list_hook, arglist);
+        Py_DECREF(arglist);
+    }
+    return 0;
+}
+
 static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
 {
     PyObject *p = PyDict_New();
@@ -173,16 +184,14 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje
     return -1;
 }
 
-//static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
-int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
+static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->object_hook) {
         PyObject *arglist = Py_BuildValue("(O)", *c);
         *c = PyEval_CallObject(u->object_hook, arglist);
         Py_DECREF(arglist);
-        return 0;
     }
-    return -1;
+    return 0;
 }
 
 static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 1fdedd7..7a2288f 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -304,6 +304,7 @@ _push:
 	case CT_ARRAY_ITEM:
 		if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; }
 		if(++c->curr == c->count) {
+			msgpack_unpack_callback(_array_end)(user, &c->obj);
 			obj = c->obj;
 			--top;
 			/*printf("stack pop %d\n", top);*/
@@ -317,7 +318,7 @@ _push:
 	case CT_MAP_VALUE:
 		if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
 		if(--c->count == 0) {
-            msgpack_unpack_callback(_map_end)(user, &c->obj);
+			msgpack_unpack_callback(_map_end)(user, &c->obj);
 			obj = c->obj;
 			--top;
 			/*printf("stack pop %d\n", top);*/
diff --git a/test/test_obj.py b/test/test_obj.py
index 28edacb..bc85736 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -31,7 +31,16 @@ def test_bad_hook():
     packed = packs([3, 1+2j], default=lambda o: o)
     unpacked = unpacks(packed)
 
+def _arr_to_str(arr):
+    return ''.join(str(c) for c in arr)
+
+def test_array_hook():
+    packed = packs([1,2,3])
+    unpacked = unpacks(packed, list_hook=_arr_to_str)
+    eq_(unpacked, '123')
+
 if __name__ == '__main__':
     test_decode_hook()
     test_encode_hook()
     test_bad_hook()
+    test_array_hook()

From 8a629ad6fba97a8a1fde77616d21ce53e2add4ca Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 28 Oct 2010 15:48:28 +0900
Subject: [PATCH 0919/1648] Fix streaming unpacking for splitted packed data

---
 perl/t/10_splitted_bytes.t      |  4 +--
 perl/t/16_unpacker_for_larges.t | 22 ++++++++++++
 perl/xs-src/unpack.c            | 59 +++++++++++++++++++++++----------
 3 files changed, 64 insertions(+), 21 deletions(-)
 create mode 100644 perl/t/16_unpacker_for_larges.t

diff --git a/perl/t/10_splitted_bytes.t b/perl/t/10_splitted_bytes.t
index 15598f4..d94472d 100644
--- a/perl/t/10_splitted_bytes.t
+++ b/perl/t/10_splitted_bytes.t
@@ -20,8 +20,6 @@ my $input = [
 my $packed = Data::MessagePack->pack($input);
 
 foreach my $size(1 .. 16) {
-    local $TODO = "Splitted byte streaming is not yet supported (bufer size: $size)";
-
     my $up = Data::MessagePack::Unpacker->new();
 
     open my $stream, '<:bytes :scalar', \$packed;
@@ -29,7 +27,7 @@ foreach my $size(1 .. 16) {
     my $buff;
     my $done = 0;
     while( read($stream, $buff, $size) ) {
-        #note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff;
+        note "buff: ", join " ", map { unpack 'H2', $_ } split //, $buff;
 
         $done = $up->execute($buff);
     }
diff --git a/perl/t/16_unpacker_for_larges.t b/perl/t/16_unpacker_for_larges.t
new file mode 100644
index 0000000..81e7c6a
--- /dev/null
+++ b/perl/t/16_unpacker_for_larges.t
@@ -0,0 +1,22 @@
+use strict;
+use Test::More;
+use Data::MessagePack;
+
+foreach my $data("abc") {
+    my $packed = Data::MessagePack->pack($data);
+
+    my $unpacker = Data::MessagePack::Unpacker->new;
+    note "buff: ", join " ", map { unpack 'H2', $_ } split //, $packed;
+
+    my $offset = 0;
+    foreach my $byte(split //, $packed) {
+        note "offset: $offset";
+        $offset += $unpacker->execute($byte);
+    }
+
+    ok $unpacker->is_finished, 'finished';
+    is_deeply $unpacker->data, $data, 'data';
+}
+
+done_testing;
+
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index db7e394..e3f62c6 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -9,11 +9,13 @@ typedef struct {
 } my_cxt_t;
 START_MY_CXT
 
+// context data for execute_template()
 typedef struct {
     bool finished;
-    bool incremented;
     bool utf8;
+    SV*  buffer;
 } unpack_user;
+#define UNPACK_USER_INIT { false, false, NULL }
 
 #include "msgpack/unpack_define.h"
 
@@ -301,7 +303,7 @@ XS(xs_unpack) {
     msgpack_unpack_t mp;
     template_init(&mp);
 
-    unpack_user const u = {false, false, false};
+    unpack_user const u = UNPACK_USER_INIT;
     mp.user = u;
 
     size_t from = 0;
@@ -326,14 +328,6 @@ XS(xs_unpack) {
 /* ------------------------------ stream -- */
 /* http://twitter.com/frsyuki/status/13249304748 */
 
-STATIC_INLINE void _reset(SV* const self) {
-    dTHX;
-	unpack_user const u = {false, false, false};
-
-	UNPACKER(self, mp);
-	template_init(mp);
-	mp->user = u;
-}
 
 XS(xs_unpacker_new) {
     dXSARGS;
@@ -345,9 +339,14 @@ XS(xs_unpacker_new) {
     msgpack_unpack_t *mp;
 
     Newxz(mp, 1, msgpack_unpack_t);
+    template_init(mp);
+    unpack_user const u = UNPACK_USER_INIT;
+    mp->user = u;
+
+    mp->user.buffer = newSV(80);
+    sv_setpvs(mp->user.buffer, "");
 
     sv_setref_pv(self, "Data::MessagePack::Unpacker", mp);
-    _reset(self);
 
     ST(0) = self;
     XSRETURN(1);
@@ -378,21 +377,44 @@ _execute_impl(SV* const self, SV* const data, UV const offset, UV const limit) {
     dTHX;
 
     if(offset >= limit) {
-        Perl_croak(aTHX_ "offset (%"UVuf") is bigger than data buffer size (%"UVuf")",
+        Perl_croak(aTHX_
+            "offset (%"UVuf") is bigger than data buffer size (%"UVuf")",
             offset, limit);
     }
 
     UNPACKER(self, mp);
 
     size_t from = offset;
-    const char* const dptr = SvPV_nolen_const(data);
+    const char* dptr = SvPV_nolen_const(data);
+    STRLEN      dlen = limit;
 
-    int const ret = template_execute(mp, dptr, limit, &from);
+    if(SvCUR(mp->user.buffer) != 0) {
+        sv_catpvn(mp->user.buffer, dptr, dlen);
+        dptr = SvPV_const(mp->user.buffer, dlen);
+        from = 0;
+    }
+
+    int const ret = template_execute(mp, dptr, dlen, &from);
+    // ret <  0 : error
+    // ret == 0 : insufficient
+    // ret >  0 : success
 
     if(ret < 0) {
-        Perl_croak(aTHX_ "Data::MessagePack::Unpacker: parse error while executing");
+        Perl_croak(aTHX_
+            "Data::MessagePack::Unpacker: parse error while executing");
     }
+
     mp->user.finished = (ret > 0) ? true : false;
+    if(!mp->user.finished) {
+        template_init(mp); // reset the state
+        sv_setpvn(mp->user.buffer, dptr, dlen);
+        from = 0;
+    }
+    else {
+        sv_setpvs(mp->user.buffer, "");
+    }
+    //warn(">> (%d) dlen=%d, from=%d, rest=%d",
+    //    (int)ret, (int)dlen, (int)from, dlen - from);
     return from;
 }
 
@@ -464,12 +486,12 @@ XS(xs_unpacker_reset) {
     }
 
     UNPACKER(ST(0), mp);
-    bool const utf8 = mp->user.utf8; // save
 
     SV* const data = template_data(mp);
     SvREFCNT_dec(data);
-    _reset(ST(0));
-    mp->user.utf8 = utf8;
+
+    template_init(mp);
+    sv_setpvs(mp->user.buffer, "");
 
     XSRETURN(0);
 }
@@ -484,6 +506,7 @@ XS(xs_unpacker_destroy) {
 
     SV* const data = template_data(mp);
     SvREFCNT_dec(data);
+    SvREFCNT_dec(mp->user.buffer);
     Safefree(mp);
 
     XSRETURN(0);

From 82d33944e6f0ee1f98791e41feb06d0d665abfb6 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 28 Oct 2010 16:08:20 +0900
Subject: [PATCH 0920/1648] perl: cleanup

---
 perl/xs-src/pack.c   | 38 ++++++++++++++++++++-------------
 perl/xs-src/unpack.c | 51 +++++++++++++++++++++++++++++---------------
 2 files changed, 57 insertions(+), 32 deletions(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 862808e..fe646f1 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -47,7 +47,8 @@ STATIC_INLINE void need(enc_t* const enc, STRLEN const len);
 #define ERR_NESTING_EXCEEDED "perl structure exceeds maximum nesting level (max_depth set too low?)"
 
 
-STATIC_INLINE void need(enc_t* const enc, STRLEN const len)
+STATIC_INLINE
+void need(enc_t* const enc, STRLEN const len)
 {
     if (enc->cur + len >= enc->end) {
         dTHX;
@@ -61,7 +62,8 @@ STATIC_INLINE void need(enc_t* const enc, STRLEN const len)
 
 static int s_pref_int = 0;
 
-STATIC_INLINE int pref_int_set(pTHX_ SV* sv, MAGIC* mg PERL_UNUSED_DECL) {
+STATIC_INLINE
+int pref_int_set(pTHX_ SV* sv, MAGIC* mg PERL_UNUSED_DECL) {
     if (SvTRUE(sv)) {
         s_pref_int = 1;
     } else {
@@ -90,7 +92,8 @@ void init_Data__MessagePack_pack(pTHX_ bool const cloning) {
 }
 
 
-STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) {
+STATIC_INLINE
+int try_int(enc_t* enc, const char *p, size_t len) {
     int negative = 0;
     const char* pe = p + len;
     uint64_t num = 0;
@@ -144,9 +147,11 @@ STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) {
 }
 
 
-STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
+STATIC_INLINE
+void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
 
-STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
+STATIC_INLINE
+void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
     dTHX;
     assert(sv);
     if (UNLIKELY(depth <= 0)) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
@@ -185,12 +190,13 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const de
     }
 }
 
-STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
-    svtype svt;
+STATIC_INLINE
+void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
     dTHX;
+
     assert(sv);
     SvGETMAGIC(sv);
-    svt = SvTYPE(sv);
+    svtype const svt = SvTYPE(sv);
 
     if (SvOBJECT (sv)) {
         HV *stash = gv_stashpv ("Data::MessagePack::Boolean", 1); // TODO: cache?
@@ -201,8 +207,8 @@ STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
                 msgpack_pack_false(enc);
             }
         } else {
-            croak ("encountered object '%s', Data::MessagePack doesn't allow the object",
-                           SvPV_nolen(sv_2mortal(newRV_inc(sv))));
+            croak ("encountered object '%"SVf"', Data::MessagePack doesn't allow the object",
+                           sv_2mortal(newRV_inc(sv)));
         }
     } else if (svt == SVt_PVHV) {
         HV* hval = (HV*)sv;
@@ -238,12 +244,12 @@ STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
             msgpack_pack_false(enc); 
         else {
             //sv_dump(sv);
-            croak("cannot encode reference to scalar '%s' unless the scalar is 0 or 1",
-                    SvPV_nolen (sv_2mortal (newRV_inc (sv))));
+            croak("cannot encode reference to scalar '%"SVf"' unless the scalar is 0 or 1",
+                    sv_2mortal(newRV_inc(sv)));
         }
     } else {
-        croak ("encountered %s, but msgpack can only represent references to arrays or hashes",
-                   SvPV_nolen (sv_2mortal (newRV_inc (sv))));
+        croak ("encountered %"SVf", but msgpack can only represent references to arrays or hashes",
+                   sv_2mortal(newRV_inc(sv)));
     }
 }
 
@@ -266,8 +272,10 @@ XS(xs_pack) {
     _msgpack_pack_sv(&enc, val, depth);
 
     SvCUR_set(enc.sv, enc.cur - SvPVX (enc.sv));
-    *SvEND (enc.sv) = 0; /* many xs functions expect a trailing 0 for text strings */
+    /* many C functions expect a trailing NUL for strings */
+    *SvEND(enc.sv) = '\0';
 
     ST(0) = enc.sv;
     XSRETURN(1);
 }
+
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index e3f62c6..2294446 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -101,21 +101,24 @@ static SV* template_data(msgpack_unpack_t* u);
 static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL,
     const char* data, size_t len, size_t* off);
 
-STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
+STATIC_INLINE
+SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
 {
     return NULL;
 }
 
 #if IVSIZE == 4
 
-STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
+STATIC_INLINE
+int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
 {
     dTHX;
     *o = newSVuv(d);
     return 0;
 }
 
-STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
+STATIC_INLINE
+int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
 {
     dTHX;
     *o = newSViv(d);
@@ -123,7 +126,8 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const
 }
 
 /* workaround win32 problems (my_snprintf(%llu) returns incorrect values ) */
-static char* str_from_uint64(char* buf_end, uint64_t v)
+static
+char* str_from_uint64(char* buf_end, uint64_t v)
 {
   char *p = buf_end;
   *--p = '\0';
@@ -133,7 +137,8 @@ static char* str_from_uint64(char* buf_end, uint64_t v)
   return p;
 }
 
-static const char* str_from_int64(char* buf_end, int64_t const v) {
+static
+const char* str_from_int64(char* buf_end, int64_t const v) {
   bool const minus = v < 0;
   char* p = str_from_uint64(buf_end, minus ? -v : v);
   if (minus)
@@ -141,7 +146,8 @@ static const char* str_from_int64(char* buf_end, int64_t const v) {
   return p;
 }
 
-static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
+static
+int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
 {
     dTHX;
     char tbuf[64];
@@ -150,7 +156,8 @@ static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t co
     return 0;
 }
 
-static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
+static
+int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
 {
     dTHX;
     char tbuf[64];
@@ -162,7 +169,8 @@ static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t cons
 #else /* IVSIZE == 8 */
 
 
-STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
+STATIC_INLINE
+int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
 {
     dTHX;
     *o = newSVuv(d);
@@ -192,7 +200,8 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const
 
 #define template_callback_float template_callback_double
 
-STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
+STATIC_INLINE
+int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
 {
     dTHX;
     *o = newSVnv(d);
@@ -200,26 +209,30 @@ STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, doub
 }
 
 /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */
-STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE
+int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     dTHX;
     *o = newSV(0);
     return 0;
 }
 
-STATIC_INLINE int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE
+int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     *o = get_bool(true);
     return 0;
 }
 
-STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE
+int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     *o = get_bool(false);
     return 0;
 }
 
-STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
+STATIC_INLINE
+int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
 {
     dTHX;
     AV* const a = newAV();
@@ -228,7 +241,8 @@ STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsig
     return 0;
 }
 
-STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
+STATIC_INLINE
+int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
 {
     dTHX;
     AV* const a = (AV*)SvRV(*c);
@@ -237,7 +251,8 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL,
     return 0;
 }
 
-STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
+STATIC_INLINE
+int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
 {
     dTHX;
     HV* const h = newHV();
@@ -246,7 +261,8 @@ STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigne
     return 0;
 }
 
-STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v)
+STATIC_INLINE
+int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v)
 {
     dTHX;
     HV* const h = (HV*)SvRV(*c);
@@ -256,7 +272,8 @@ STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV
     return 0;
 }
 
-STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o)
+STATIC_INLINE int
+template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o)
 {
     dTHX;
     /*  newSVpvn(p, l) returns an undef if p == NULL */

From a7a23d3bc8bb7a5c156becb3b8a43e296986f42e Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 28 Oct 2010 16:16:12 +0900
Subject: [PATCH 0921/1648] Revert "perl: cleanup"

This reverts commit 82d33944e6f0ee1f98791e41feb06d0d665abfb6.
---
 perl/xs-src/pack.c   | 38 +++++++++++++--------------------
 perl/xs-src/unpack.c | 51 +++++++++++++++-----------------------------
 2 files changed, 32 insertions(+), 57 deletions(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index fe646f1..862808e 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -47,8 +47,7 @@ STATIC_INLINE void need(enc_t* const enc, STRLEN const len);
 #define ERR_NESTING_EXCEEDED "perl structure exceeds maximum nesting level (max_depth set too low?)"
 
 
-STATIC_INLINE
-void need(enc_t* const enc, STRLEN const len)
+STATIC_INLINE void need(enc_t* const enc, STRLEN const len)
 {
     if (enc->cur + len >= enc->end) {
         dTHX;
@@ -62,8 +61,7 @@ void need(enc_t* const enc, STRLEN const len)
 
 static int s_pref_int = 0;
 
-STATIC_INLINE
-int pref_int_set(pTHX_ SV* sv, MAGIC* mg PERL_UNUSED_DECL) {
+STATIC_INLINE int pref_int_set(pTHX_ SV* sv, MAGIC* mg PERL_UNUSED_DECL) {
     if (SvTRUE(sv)) {
         s_pref_int = 1;
     } else {
@@ -92,8 +90,7 @@ void init_Data__MessagePack_pack(pTHX_ bool const cloning) {
 }
 
 
-STATIC_INLINE
-int try_int(enc_t* enc, const char *p, size_t len) {
+STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) {
     int negative = 0;
     const char* pe = p + len;
     uint64_t num = 0;
@@ -147,11 +144,9 @@ int try_int(enc_t* enc, const char *p, size_t len) {
 }
 
 
-STATIC_INLINE
-void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
+STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
 
-STATIC_INLINE
-void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
+STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
     dTHX;
     assert(sv);
     if (UNLIKELY(depth <= 0)) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
@@ -190,13 +185,12 @@ void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
     }
 }
 
-STATIC_INLINE
-void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
+STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
+    svtype svt;
     dTHX;
-
     assert(sv);
     SvGETMAGIC(sv);
-    svtype const svt = SvTYPE(sv);
+    svt = SvTYPE(sv);
 
     if (SvOBJECT (sv)) {
         HV *stash = gv_stashpv ("Data::MessagePack::Boolean", 1); // TODO: cache?
@@ -207,8 +201,8 @@ void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
                 msgpack_pack_false(enc);
             }
         } else {
-            croak ("encountered object '%"SVf"', Data::MessagePack doesn't allow the object",
-                           sv_2mortal(newRV_inc(sv)));
+            croak ("encountered object '%s', Data::MessagePack doesn't allow the object",
+                           SvPV_nolen(sv_2mortal(newRV_inc(sv))));
         }
     } else if (svt == SVt_PVHV) {
         HV* hval = (HV*)sv;
@@ -244,12 +238,12 @@ void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
             msgpack_pack_false(enc); 
         else {
             //sv_dump(sv);
-            croak("cannot encode reference to scalar '%"SVf"' unless the scalar is 0 or 1",
-                    sv_2mortal(newRV_inc(sv)));
+            croak("cannot encode reference to scalar '%s' unless the scalar is 0 or 1",
+                    SvPV_nolen (sv_2mortal (newRV_inc (sv))));
         }
     } else {
-        croak ("encountered %"SVf", but msgpack can only represent references to arrays or hashes",
-                   sv_2mortal(newRV_inc(sv)));
+        croak ("encountered %s, but msgpack can only represent references to arrays or hashes",
+                   SvPV_nolen (sv_2mortal (newRV_inc (sv))));
     }
 }
 
@@ -272,10 +266,8 @@ XS(xs_pack) {
     _msgpack_pack_sv(&enc, val, depth);
 
     SvCUR_set(enc.sv, enc.cur - SvPVX (enc.sv));
-    /* many C functions expect a trailing NUL for strings */
-    *SvEND(enc.sv) = '\0';
+    *SvEND (enc.sv) = 0; /* many xs functions expect a trailing 0 for text strings */
 
     ST(0) = enc.sv;
     XSRETURN(1);
 }
-
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 2294446..e3f62c6 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -101,24 +101,21 @@ static SV* template_data(msgpack_unpack_t* u);
 static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL,
     const char* data, size_t len, size_t* off);
 
-STATIC_INLINE
-SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
+STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
 {
     return NULL;
 }
 
 #if IVSIZE == 4
 
-STATIC_INLINE
-int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
+STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
 {
     dTHX;
     *o = newSVuv(d);
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
+STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
 {
     dTHX;
     *o = newSViv(d);
@@ -126,8 +123,7 @@ int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
 }
 
 /* workaround win32 problems (my_snprintf(%llu) returns incorrect values ) */
-static
-char* str_from_uint64(char* buf_end, uint64_t v)
+static char* str_from_uint64(char* buf_end, uint64_t v)
 {
   char *p = buf_end;
   *--p = '\0';
@@ -137,8 +133,7 @@ char* str_from_uint64(char* buf_end, uint64_t v)
   return p;
 }
 
-static
-const char* str_from_int64(char* buf_end, int64_t const v) {
+static const char* str_from_int64(char* buf_end, int64_t const v) {
   bool const minus = v < 0;
   char* p = str_from_uint64(buf_end, minus ? -v : v);
   if (minus)
@@ -146,8 +141,7 @@ const char* str_from_int64(char* buf_end, int64_t const v) {
   return p;
 }
 
-static
-int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
+static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
 {
     dTHX;
     char tbuf[64];
@@ -156,8 +150,7 @@ int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d,
     return 0;
 }
 
-static
-int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
+static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
 {
     dTHX;
     char tbuf[64];
@@ -169,8 +162,7 @@ int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV
 #else /* IVSIZE == 8 */
 
 
-STATIC_INLINE
-int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
+STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
 {
     dTHX;
     *o = newSVuv(d);
@@ -200,8 +192,7 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const
 
 #define template_callback_float template_callback_double
 
-STATIC_INLINE
-int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
+STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
 {
     dTHX;
     *o = newSVnv(d);
@@ -209,30 +200,26 @@ int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
 }
 
 /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */
-STATIC_INLINE
-int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     dTHX;
     *o = newSV(0);
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     *o = get_bool(true);
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     *o = get_bool(false);
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
+STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
 {
     dTHX;
     AV* const a = newAV();
@@ -241,8 +228,7 @@ int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV*
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
+STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
 {
     dTHX;
     AV* const a = (AV*)SvRV(*c);
@@ -251,8 +237,7 @@ int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
+STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
 {
     dTHX;
     HV* const h = newHV();
@@ -261,8 +246,7 @@ int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV**
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v)
+STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v)
 {
     dTHX;
     HV* const h = (HV*)SvRV(*c);
@@ -272,8 +256,7 @@ int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, S
     return 0;
 }
 
-STATIC_INLINE int
-template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o)
+STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o)
 {
     dTHX;
     /*  newSVpvn(p, l) returns an undef if p == NULL */

From bc8d8ab65a0f0cecfa5af03b4c3b39a13da22a7b Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 28 Oct 2010 16:17:53 +0900
Subject: [PATCH 0922/1648] Revert "Revert "perl: cleanup""

This reverts commit a7a23d3bc8bb7a5c156becb3b8a43e296986f42e.
---
 perl/xs-src/pack.c   | 38 ++++++++++++++++++++-------------
 perl/xs-src/unpack.c | 51 +++++++++++++++++++++++++++++---------------
 2 files changed, 57 insertions(+), 32 deletions(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index 862808e..fe646f1 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -47,7 +47,8 @@ STATIC_INLINE void need(enc_t* const enc, STRLEN const len);
 #define ERR_NESTING_EXCEEDED "perl structure exceeds maximum nesting level (max_depth set too low?)"
 
 
-STATIC_INLINE void need(enc_t* const enc, STRLEN const len)
+STATIC_INLINE
+void need(enc_t* const enc, STRLEN const len)
 {
     if (enc->cur + len >= enc->end) {
         dTHX;
@@ -61,7 +62,8 @@ STATIC_INLINE void need(enc_t* const enc, STRLEN const len)
 
 static int s_pref_int = 0;
 
-STATIC_INLINE int pref_int_set(pTHX_ SV* sv, MAGIC* mg PERL_UNUSED_DECL) {
+STATIC_INLINE
+int pref_int_set(pTHX_ SV* sv, MAGIC* mg PERL_UNUSED_DECL) {
     if (SvTRUE(sv)) {
         s_pref_int = 1;
     } else {
@@ -90,7 +92,8 @@ void init_Data__MessagePack_pack(pTHX_ bool const cloning) {
 }
 
 
-STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) {
+STATIC_INLINE
+int try_int(enc_t* enc, const char *p, size_t len) {
     int negative = 0;
     const char* pe = p + len;
     uint64_t num = 0;
@@ -144,9 +147,11 @@ STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) {
 }
 
 
-STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
+STATIC_INLINE
+void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
 
-STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
+STATIC_INLINE
+void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
     dTHX;
     assert(sv);
     if (UNLIKELY(depth <= 0)) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
@@ -185,12 +190,13 @@ STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const de
     }
 }
 
-STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
-    svtype svt;
+STATIC_INLINE
+void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
     dTHX;
+
     assert(sv);
     SvGETMAGIC(sv);
-    svt = SvTYPE(sv);
+    svtype const svt = SvTYPE(sv);
 
     if (SvOBJECT (sv)) {
         HV *stash = gv_stashpv ("Data::MessagePack::Boolean", 1); // TODO: cache?
@@ -201,8 +207,8 @@ STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
                 msgpack_pack_false(enc);
             }
         } else {
-            croak ("encountered object '%s', Data::MessagePack doesn't allow the object",
-                           SvPV_nolen(sv_2mortal(newRV_inc(sv))));
+            croak ("encountered object '%"SVf"', Data::MessagePack doesn't allow the object",
+                           sv_2mortal(newRV_inc(sv)));
         }
     } else if (svt == SVt_PVHV) {
         HV* hval = (HV*)sv;
@@ -238,12 +244,12 @@ STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
             msgpack_pack_false(enc); 
         else {
             //sv_dump(sv);
-            croak("cannot encode reference to scalar '%s' unless the scalar is 0 or 1",
-                    SvPV_nolen (sv_2mortal (newRV_inc (sv))));
+            croak("cannot encode reference to scalar '%"SVf"' unless the scalar is 0 or 1",
+                    sv_2mortal(newRV_inc(sv)));
         }
     } else {
-        croak ("encountered %s, but msgpack can only represent references to arrays or hashes",
-                   SvPV_nolen (sv_2mortal (newRV_inc (sv))));
+        croak ("encountered %"SVf", but msgpack can only represent references to arrays or hashes",
+                   sv_2mortal(newRV_inc(sv)));
     }
 }
 
@@ -266,8 +272,10 @@ XS(xs_pack) {
     _msgpack_pack_sv(&enc, val, depth);
 
     SvCUR_set(enc.sv, enc.cur - SvPVX (enc.sv));
-    *SvEND (enc.sv) = 0; /* many xs functions expect a trailing 0 for text strings */
+    /* many C functions expect a trailing NUL for strings */
+    *SvEND(enc.sv) = '\0';
 
     ST(0) = enc.sv;
     XSRETURN(1);
 }
+
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index e3f62c6..2294446 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -101,21 +101,24 @@ static SV* template_data(msgpack_unpack_t* u);
 static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL,
     const char* data, size_t len, size_t* off);
 
-STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
+STATIC_INLINE
+SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
 {
     return NULL;
 }
 
 #if IVSIZE == 4
 
-STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
+STATIC_INLINE
+int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
 {
     dTHX;
     *o = newSVuv(d);
     return 0;
 }
 
-STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
+STATIC_INLINE
+int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
 {
     dTHX;
     *o = newSViv(d);
@@ -123,7 +126,8 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const
 }
 
 /* workaround win32 problems (my_snprintf(%llu) returns incorrect values ) */
-static char* str_from_uint64(char* buf_end, uint64_t v)
+static
+char* str_from_uint64(char* buf_end, uint64_t v)
 {
   char *p = buf_end;
   *--p = '\0';
@@ -133,7 +137,8 @@ static char* str_from_uint64(char* buf_end, uint64_t v)
   return p;
 }
 
-static const char* str_from_int64(char* buf_end, int64_t const v) {
+static
+const char* str_from_int64(char* buf_end, int64_t const v) {
   bool const minus = v < 0;
   char* p = str_from_uint64(buf_end, minus ? -v : v);
   if (minus)
@@ -141,7 +146,8 @@ static const char* str_from_int64(char* buf_end, int64_t const v) {
   return p;
 }
 
-static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
+static
+int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
 {
     dTHX;
     char tbuf[64];
@@ -150,7 +156,8 @@ static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t co
     return 0;
 }
 
-static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
+static
+int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
 {
     dTHX;
     char tbuf[64];
@@ -162,7 +169,8 @@ static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t cons
 #else /* IVSIZE == 8 */
 
 
-STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
+STATIC_INLINE
+int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
 {
     dTHX;
     *o = newSVuv(d);
@@ -192,7 +200,8 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const
 
 #define template_callback_float template_callback_double
 
-STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
+STATIC_INLINE
+int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
 {
     dTHX;
     *o = newSVnv(d);
@@ -200,26 +209,30 @@ STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, doub
 }
 
 /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */
-STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE
+int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     dTHX;
     *o = newSV(0);
     return 0;
 }
 
-STATIC_INLINE int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE
+int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     *o = get_bool(true);
     return 0;
 }
 
-STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE
+int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     *o = get_bool(false);
     return 0;
 }
 
-STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
+STATIC_INLINE
+int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
 {
     dTHX;
     AV* const a = newAV();
@@ -228,7 +241,8 @@ STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsig
     return 0;
 }
 
-STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
+STATIC_INLINE
+int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
 {
     dTHX;
     AV* const a = (AV*)SvRV(*c);
@@ -237,7 +251,8 @@ STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL,
     return 0;
 }
 
-STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
+STATIC_INLINE
+int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
 {
     dTHX;
     HV* const h = newHV();
@@ -246,7 +261,8 @@ STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigne
     return 0;
 }
 
-STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v)
+STATIC_INLINE
+int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v)
 {
     dTHX;
     HV* const h = (HV*)SvRV(*c);
@@ -256,7 +272,8 @@ STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV
     return 0;
 }
 
-STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o)
+STATIC_INLINE int
+template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o)
 {
     dTHX;
     /*  newSVpvn(p, l) returns an undef if p == NULL */

From 86ccfcc03c4346a44d7d64b4269cba5488634aaa Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 28 Oct 2010 16:25:19 +0900
Subject: [PATCH 0923/1648] perl: changelogging

---
 perl/Changes | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/perl/Changes b/perl/Changes
index 486f1c3..d219b4a 100644
--- a/perl/Changes
+++ b/perl/Changes
@@ -1,3 +1,8 @@
+0.35
+
+    - address issue/20 (cho45): Data::MessagePack did not finish correctly
+      when was given devided packed data
+
 0.34
 
     - do not use the corrupt my_snprintf(%ll[du]) on win32(kazuho)

From fedc37d079bade6135cbeac7f3b157dfcabe8093 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 28 Oct 2010 16:31:19 +0900
Subject: [PATCH 0924/1648] Revert "perl: cleanup"

This reverts commit bc8d8ab65a0f0cecfa5af03b4c3b39a13da22a7b.
---
 perl/xs-src/pack.c   | 38 +++++++++++++--------------------
 perl/xs-src/unpack.c | 51 +++++++++++++++-----------------------------
 2 files changed, 32 insertions(+), 57 deletions(-)

diff --git a/perl/xs-src/pack.c b/perl/xs-src/pack.c
index fe646f1..862808e 100644
--- a/perl/xs-src/pack.c
+++ b/perl/xs-src/pack.c
@@ -47,8 +47,7 @@ STATIC_INLINE void need(enc_t* const enc, STRLEN const len);
 #define ERR_NESTING_EXCEEDED "perl structure exceeds maximum nesting level (max_depth set too low?)"
 
 
-STATIC_INLINE
-void need(enc_t* const enc, STRLEN const len)
+STATIC_INLINE void need(enc_t* const enc, STRLEN const len)
 {
     if (enc->cur + len >= enc->end) {
         dTHX;
@@ -62,8 +61,7 @@ void need(enc_t* const enc, STRLEN const len)
 
 static int s_pref_int = 0;
 
-STATIC_INLINE
-int pref_int_set(pTHX_ SV* sv, MAGIC* mg PERL_UNUSED_DECL) {
+STATIC_INLINE int pref_int_set(pTHX_ SV* sv, MAGIC* mg PERL_UNUSED_DECL) {
     if (SvTRUE(sv)) {
         s_pref_int = 1;
     } else {
@@ -92,8 +90,7 @@ void init_Data__MessagePack_pack(pTHX_ bool const cloning) {
 }
 
 
-STATIC_INLINE
-int try_int(enc_t* enc, const char *p, size_t len) {
+STATIC_INLINE int try_int(enc_t* enc, const char *p, size_t len) {
     int negative = 0;
     const char* pe = p + len;
     uint64_t num = 0;
@@ -147,11 +144,9 @@ int try_int(enc_t* enc, const char *p, size_t len) {
 }
 
 
-STATIC_INLINE
-void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
+STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth);
 
-STATIC_INLINE
-void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
+STATIC_INLINE void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
     dTHX;
     assert(sv);
     if (UNLIKELY(depth <= 0)) Perl_croak(aTHX_ ERR_NESTING_EXCEEDED);
@@ -190,13 +185,12 @@ void _msgpack_pack_sv(enc_t* const enc, SV* const sv, int const depth) {
     }
 }
 
-STATIC_INLINE
-void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
+STATIC_INLINE void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
+    svtype svt;
     dTHX;
-
     assert(sv);
     SvGETMAGIC(sv);
-    svtype const svt = SvTYPE(sv);
+    svt = SvTYPE(sv);
 
     if (SvOBJECT (sv)) {
         HV *stash = gv_stashpv ("Data::MessagePack::Boolean", 1); // TODO: cache?
@@ -207,8 +201,8 @@ void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
                 msgpack_pack_false(enc);
             }
         } else {
-            croak ("encountered object '%"SVf"', Data::MessagePack doesn't allow the object",
-                           sv_2mortal(newRV_inc(sv)));
+            croak ("encountered object '%s', Data::MessagePack doesn't allow the object",
+                           SvPV_nolen(sv_2mortal(newRV_inc(sv))));
         }
     } else if (svt == SVt_PVHV) {
         HV* hval = (HV*)sv;
@@ -244,12 +238,12 @@ void _msgpack_pack_rv(enc_t *enc, SV* sv, int depth) {
             msgpack_pack_false(enc); 
         else {
             //sv_dump(sv);
-            croak("cannot encode reference to scalar '%"SVf"' unless the scalar is 0 or 1",
-                    sv_2mortal(newRV_inc(sv)));
+            croak("cannot encode reference to scalar '%s' unless the scalar is 0 or 1",
+                    SvPV_nolen (sv_2mortal (newRV_inc (sv))));
         }
     } else {
-        croak ("encountered %"SVf", but msgpack can only represent references to arrays or hashes",
-                   sv_2mortal(newRV_inc(sv)));
+        croak ("encountered %s, but msgpack can only represent references to arrays or hashes",
+                   SvPV_nolen (sv_2mortal (newRV_inc (sv))));
     }
 }
 
@@ -272,10 +266,8 @@ XS(xs_pack) {
     _msgpack_pack_sv(&enc, val, depth);
 
     SvCUR_set(enc.sv, enc.cur - SvPVX (enc.sv));
-    /* many C functions expect a trailing NUL for strings */
-    *SvEND(enc.sv) = '\0';
+    *SvEND (enc.sv) = 0; /* many xs functions expect a trailing 0 for text strings */
 
     ST(0) = enc.sv;
     XSRETURN(1);
 }
-
diff --git a/perl/xs-src/unpack.c b/perl/xs-src/unpack.c
index 2294446..e3f62c6 100644
--- a/perl/xs-src/unpack.c
+++ b/perl/xs-src/unpack.c
@@ -101,24 +101,21 @@ static SV* template_data(msgpack_unpack_t* u);
 static int template_execute(msgpack_unpack_t* u PERL_UNUSED_DECL,
     const char* data, size_t len, size_t* off);
 
-STATIC_INLINE
-SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
+STATIC_INLINE SV* template_callback_root(unpack_user* u PERL_UNUSED_DECL)
 {
     return NULL;
 }
 
 #if IVSIZE == 4
 
-STATIC_INLINE
-int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
+STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
 {
     dTHX;
     *o = newSVuv(d);
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
+STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
 {
     dTHX;
     *o = newSViv(d);
@@ -126,8 +123,7 @@ int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const d, SV** o)
 }
 
 /* workaround win32 problems (my_snprintf(%llu) returns incorrect values ) */
-static
-char* str_from_uint64(char* buf_end, uint64_t v)
+static char* str_from_uint64(char* buf_end, uint64_t v)
 {
   char *p = buf_end;
   *--p = '\0';
@@ -137,8 +133,7 @@ char* str_from_uint64(char* buf_end, uint64_t v)
   return p;
 }
 
-static
-const char* str_from_int64(char* buf_end, int64_t const v) {
+static const char* str_from_int64(char* buf_end, int64_t const v) {
   bool const minus = v < 0;
   char* p = str_from_uint64(buf_end, minus ? -v : v);
   if (minus)
@@ -146,8 +141,7 @@ const char* str_from_int64(char* buf_end, int64_t const v) {
   return p;
 }
 
-static
-int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
+static int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d, SV** o)
 {
     dTHX;
     char tbuf[64];
@@ -156,8 +150,7 @@ int template_callback_uint64(unpack_user* u PERL_UNUSED_DECL, uint64_t const d,
     return 0;
 }
 
-static
-int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
+static int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV** o)
 {
     dTHX;
     char tbuf[64];
@@ -169,8 +162,7 @@ int template_callback_int64(unpack_user* u PERL_UNUSED_DECL, int64_t const d, SV
 #else /* IVSIZE == 8 */
 
 
-STATIC_INLINE
-int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
+STATIC_INLINE int template_callback_UV(unpack_user* u PERL_UNUSED_DECL, UV const d, SV** o)
 {
     dTHX;
     *o = newSVuv(d);
@@ -200,8 +192,7 @@ STATIC_INLINE int template_callback_IV(unpack_user* u PERL_UNUSED_DECL, IV const
 
 #define template_callback_float template_callback_double
 
-STATIC_INLINE
-int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
+STATIC_INLINE int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
 {
     dTHX;
     *o = newSVnv(d);
@@ -209,30 +200,26 @@ int template_callback_double(unpack_user* u PERL_UNUSED_DECL, double d, SV** o)
 }
 
 /* &PL_sv_undef is not so good. see http://gist.github.com/387743 */
-STATIC_INLINE
-int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE int template_callback_nil(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     dTHX;
     *o = newSV(0);
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE int template_callback_true(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     *o = get_bool(true);
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
+STATIC_INLINE int template_callback_false(unpack_user* u PERL_UNUSED_DECL, SV** o)
 {
     *o = get_bool(false);
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
+STATIC_INLINE int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
 {
     dTHX;
     AV* const a = newAV();
@@ -241,8 +228,7 @@ int template_callback_array(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV*
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
+STATIC_INLINE int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
 {
     dTHX;
     AV* const a = (AV*)SvRV(*c);
@@ -251,8 +237,7 @@ int template_callback_array_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* o)
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
+STATIC_INLINE int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV** o)
 {
     dTHX;
     HV* const h = newHV();
@@ -261,8 +246,7 @@ int template_callback_map(unpack_user* u PERL_UNUSED_DECL, unsigned int n, SV**
     return 0;
 }
 
-STATIC_INLINE
-int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v)
+STATIC_INLINE int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, SV* v)
 {
     dTHX;
     HV* const h = (HV*)SvRV(*c);
@@ -272,8 +256,7 @@ int template_callback_map_item(unpack_user* u PERL_UNUSED_DECL, SV** c, SV* k, S
     return 0;
 }
 
-STATIC_INLINE int
-template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o)
+STATIC_INLINE int template_callback_raw(unpack_user* u PERL_UNUSED_DECL, const char* b PERL_UNUSED_DECL, const char* p, unsigned int l, SV** o)
 {
     dTHX;
     /*  newSVpvn(p, l) returns an undef if p == NULL */

From ea36ef310783b502af7125848ab9679a8fb3b293 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 28 Oct 2010 17:05:41 +0900
Subject: [PATCH 0925/1648] perl: tests

---
 perl/t/16_unpacker_for_larges.t | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/perl/t/16_unpacker_for_larges.t b/perl/t/16_unpacker_for_larges.t
index 81e7c6a..8d0b5d7 100644
--- a/perl/t/16_unpacker_for_larges.t
+++ b/perl/t/16_unpacker_for_larges.t
@@ -2,16 +2,14 @@ use strict;
 use Test::More;
 use Data::MessagePack;
 
-foreach my $data("abc") {
+foreach my $data("abc", [42]) {
     my $packed = Data::MessagePack->pack($data);
 
     my $unpacker = Data::MessagePack::Unpacker->new;
     note "buff: ", join " ", map { unpack 'H2', $_ } split //, $packed;
 
-    my $offset = 0;
     foreach my $byte(split //, $packed) {
-        note "offset: $offset";
-        $offset += $unpacker->execute($byte);
+        $unpacker->execute($byte);
     }
 
     ok $unpacker->is_finished, 'finished';

From c320e44a236762d73677bbf5780f8c3a42fd98ff Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Thu, 28 Oct 2010 17:26:04 +0900
Subject: [PATCH 0926/1648] perl: update Unpacker.pod

---
 perl/lib/Data/MessagePack/Unpacker.pod | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/perl/lib/Data/MessagePack/Unpacker.pod b/perl/lib/Data/MessagePack/Unpacker.pod
index 24dafd0..04cb0a4 100644
--- a/perl/lib/Data/MessagePack/Unpacker.pod
+++ b/perl/lib/Data/MessagePack/Unpacker.pod
@@ -6,11 +6,16 @@ Data::MessagePack::Unpacker - messagepack streaming deserializer
 
     use Data::Dumper;
     my $up = Data::MessagePack::Unpacker->new;
-    my $ret = $up->execute($v, 0);
-    if ($ret != length($v)) {
-        fail "extra bytes";
+
+    open my $fh, $data or die $!;
+
+    my $offset = 0;
+    while( read($fh, my $buf, 1024) ) {
+        $offset = $up->execute($buf, $offset);
+        if($up->is_finished) {
+            print Dumper($up->data);
+        }
     }
-    return Dumper($up->data);
 
 =head1 DESCRIPTION
 
@@ -22,7 +27,7 @@ This is a streaming deserializer for messagepack.
 
 =item my $up = Data::MessagePack::Unpacker->new()
 
-creates a new instance of stream deserializer.
+creates a new instance of the stream deserializer.
 
 =item $up->utf8([$bool])
 
@@ -37,14 +42,15 @@ The utf8 mode is disabled by default.
 
 returns the utf8 mode flag of I<$up>.
 
-=item my $ret = $up->execute($data, $offset);
+=item $offset = $up->execute($data, $offset);
 
-=item my $ret = $up->execute_limit($data, $offset, $limit)
+=item $offset = $up->execute_limit($data, $offset, $limit)
 
-    $up->execute(substr($data, 0, 3), 0);
-    $up->execute($data,               3);
+parses unpacked I<$data> from I<$offset> to I<$limit>.
+returns a new offset of I<$data>, which is for the next .
 
-$offset is the offset of $data.
+If I<$data> is insufficient, I<$offset> does not change, saving
+I<$data> in internal buffers.
 
 =item my $bool = $up->is_finished();
 

From f1c294ca5012329c3d550d7b32a34a5d8039de7c Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Sat, 30 Oct 2010 00:28:53 +0900
Subject: [PATCH 0927/1648] perl: make error messages compatible with XS

---
 perl/lib/Data/MessagePack/PP.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 5e64093..8d148d2 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -305,10 +305,10 @@ foreach my $pair(
 sub _unpack {
     my ( $value ) = @_;
     # get a header byte
-    my $byte = unpack "x$p C", $value; # "x$p" is faster than substr()
+    defined(my $byte = unpack "x$p C", $value)
+        or Carp::confess("Data::MessagePack->unpack: insufficient bytes");
     $p++;
 
-    Carp::croak("invalid data") unless defined $byte;
 
     # +/- fixnum, nil, true, false
     return $byte2value[$byte] if $typemap[$byte] & $T_DIRECT;

From 6a9cb5182882076429e2c6ed22ccaa6bfda94022 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Sat, 30 Oct 2010 00:31:41 +0900
Subject: [PATCH 0928/1648] perl: modify internal names for the next
 refactoring

---
 perl/lib/Data/MessagePack/PP.pm | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 8d148d2..8daf248 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -423,7 +423,11 @@ package
     Data::MessagePack::PP::Unpacker;
 
 sub new {
-    bless { pos => 0, utf8 => 0 }, shift;
+    bless {
+        pos  => 0,
+        utf8 => 0,
+        buff => '',
+    }, shift;
 }
 
 sub utf8 {
@@ -447,13 +451,13 @@ sub execute {
     my $value = substr( $data, $offset, $limit ? $limit : length $data );
     my $len   = length $value;
 
-    $self->{data} .= $value;
+    $self->{buff} .= $value;
     local $self->{stack} = [];
 
     $p = 0;
 
-    while ( length($self->{data}) > $p ) {
-        _count( $self, $self->{data} ) or last;
+    while ( length($self->{buff}) > $p ) {
+        _count( $self, $self->{buff} ) or last;
 
         while ( @{ $self->{stack} } > 0 && --$self->{stack}->[-1] == 0) {
             pop @{ $self->{stack} };
@@ -568,7 +572,7 @@ sub _count {
 sub data {
     my($self) = @_;
     local $Data::MessagePack::PP::_utf8 = $self->{utf8};
-    return Data::MessagePack->unpack( substr($self->{ data }, 0, $self->{pos}) );
+    return Data::MessagePack->unpack( substr($self->{buff}, 0, $self->{pos}) );
 }
 
 
@@ -578,9 +582,9 @@ sub is_finished {
 }
 
 sub reset :method {
-    $_[0]->{ data }   = undef;
-    $_[0]->{ pos }    = 0;
-    $_[0]->{ is_finished } = 0;
+    $_[0]->{buff}        = '';
+    $_[0]->{pos}         = 0;
+    $_[0]->{is_finished} = 0;
 }
 
 1;

From 5de2b974fb1709ee776a49ce4cecbddd6f54893b Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Sat, 30 Oct 2010 00:42:00 +0900
Subject: [PATCH 0929/1648] perl: ord(substr(...)) *is* faster than unpack()

---
 perl/lib/Data/MessagePack/PP.pm | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 8daf248..0c19cb1 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -304,12 +304,12 @@ foreach my $pair(
 
 sub _unpack {
     my ( $value ) = @_;
-    # get a header byte
-    defined(my $byte = unpack "x$p C", $value)
+    $p < length($value)
         or Carp::confess("Data::MessagePack->unpack: insufficient bytes");
+    # get a header byte
+    my $byte = ord( substr $value, $p, 1 );
     $p++;
 
-
     # +/- fixnum, nil, true, false
     return $byte2value[$byte] if $typemap[$byte] & $T_DIRECT;
 
@@ -454,6 +454,9 @@ sub execute {
     $self->{buff} .= $value;
     local $self->{stack} = [];
 
+    #$p = 0;
+    #eval { Data::MessagePack::PP::_unpack($self->{buff}) };
+    #warn "[$p][$@]";
     $p = 0;
 
     while ( length($self->{buff}) > $p ) {

From 3f16f080aca89758f8d8e4cc5c6000718598ce4f Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Sat, 30 Oct 2010 01:09:12 +0900
Subject: [PATCH 0930/1648] perl: add failing tests for PP

---
 perl/t/16_unpacker_for_larges.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perl/t/16_unpacker_for_larges.t b/perl/t/16_unpacker_for_larges.t
index 8d0b5d7..26894a2 100644
--- a/perl/t/16_unpacker_for_larges.t
+++ b/perl/t/16_unpacker_for_larges.t
@@ -2,7 +2,7 @@ use strict;
 use Test::More;
 use Data::MessagePack;
 
-foreach my $data("abc", [42]) {
+foreach my $data("abc", [ 'x' x 1024 ], [0xFFFF42]) {
     my $packed = Data::MessagePack->pack($data);
 
     my $unpacker = Data::MessagePack::Unpacker->new;

From eac0f838645f4f1bedf0a66c92cd19339a5490b8 Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Sat, 30 Oct 2010 12:38:32 +0900
Subject: [PATCH 0931/1648] perl: check data strictly; which is slow, but
 required

---
 perl/lib/Data/MessagePack/PP.pm | 89 +++++++++++++++++----------------
 perl/t/02_unpack.t              |  5 +-
 2 files changed, 49 insertions(+), 45 deletions(-)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index 0c19cb1..fca73b8 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -138,6 +138,7 @@ sub _unexpected {
     Carp::confess("Unexpected " . sprintf(shift, @_) . " found");
 }
 
+
 #
 # PACK
 #
@@ -257,6 +258,10 @@ sub _pack {
 our $_utf8 = 0;
 my $p; # position variables for speed.
 
+sub _insufficient {
+    Carp::confess("Insufficient bytes (pos=$p, type=@_)");
+}
+
 sub unpack :method {
     $p = 0; # init
     my $data = _unpack( $_[1] );
@@ -302,10 +307,26 @@ foreach my $pair(
     $byte2value[ $pair->[0] ]  = $pair->[1];
 }
 
+sub _fetch_size {
+    my($value_ref, $byte, $x16, $x32, $x_fixbits) = @_;
+    if ( $byte == $x16 ) {
+        $p += 2;
+        $p <= length(${$value_ref}) or _insufficient('x/16');
+        return unpack 'n', substr( ${$value_ref}, $p - 2, 2 );
+    }
+    elsif ( $byte == $x32 ) {
+        $p += 4;
+        $p <= length(${$value_ref}) or _insufficient('x/32');
+        return unpack 'N', substr( ${$value_ref}, $p - 4, 4 );
+    }
+    else { # fix raw
+        return $byte & ~$x_fixbits;
+    }
+}
+
 sub _unpack {
     my ( $value ) = @_;
-    $p < length($value)
-        or Carp::confess("Data::MessagePack->unpack: insufficient bytes");
+    $p < length($value) or _insufficient('header byte');
     # get a header byte
     my $byte = ord( substr $value, $p, 1 );
     $p++;
@@ -314,55 +335,23 @@ sub _unpack {
     return $byte2value[$byte] if $typemap[$byte] & $T_DIRECT;
 
     if ( $typemap[$byte] & $T_RAW ) {
-        my $num;
-        if ( $byte == 0xda ) {
-            $num = CORE::unpack 'n', substr( $value, $p, 2 );
-            $p += 2 + $num;
-        }
-        elsif ( $byte == 0xdb ) {
-            $num = CORE::unpack 'N', substr( $value, $p, 4 );
-            $p += 4 + $num;
-        }
-        else { # fix raw
-            $num = $byte & ~0xa0;
-            $p += $num;
-        }
-        my $s = substr( $value, $p - $num, $num );
+        my $size = _fetch_size(\$value, $byte, 0xda, 0xdb, 0xa0);
+        my $s    = substr( $value, $p, $size );
+        length($s) == $size or _insufficient('raw');
+        $p      += $size;
         utf8::decode($s) if $_utf8;
         return $s;
     }
     elsif ( $typemap[$byte] & $T_ARRAY ) {
-        my $num;
-        if ( $byte == 0xdc ) { # array 16
-            $num = CORE::unpack 'n', substr( $value, $p, 2 );
-            $p += 2;
-        }
-        elsif ( $byte == 0xdd ) { # array 32
-            $num = CORE::unpack 'N', substr( $value, $p, 4 );
-            $p += 4;
-        }
-        else { # fix array
-            $num = $byte & ~0x90;
-        }
+        my $size = _fetch_size(\$value, $byte, 0xdc, 0xdd, 0x90);
         my @array;
-        push @array, _unpack( $value ) while --$num >= 0;
+        push @array, _unpack( $value ) while --$size >= 0;
         return \@array;
     }
     elsif ( $typemap[$byte] & $T_MAP ) {
-        my $num;
-        if ( $byte == 0xde ) { # map 16
-            $num = CORE::unpack 'n', substr( $value, $p, 2 );
-            $p += 2;
-        }
-        elsif ( $byte == 0xdf ) { # map 32
-            $num = CORE::unpack 'N', substr( $value, $p, 4 );
-            $p += 4;
-        }
-        else { # fix map
-            $num = $byte & ~0x80;
-        }
+        my $size = _fetch_size(\$value, $byte, 0xde, 0xdf, 0x80);
         my %map;
-        while ( --$num >= 0 ) {
+        while(--$size >= 0) {
             no warnings; # for undef key case
             my $key = _unpack( $value );
             my $val = _unpack( $value );
@@ -372,41 +361,53 @@ sub _unpack {
     }
 
     elsif ( $byte == 0xcc ) { # uint8
-        return CORE::unpack( 'C', substr( $value, $p++, 1 ) );
+        $p++;
+        $p <= length($value) or _insufficient('uint8');
+        return CORE::unpack( 'C', substr( $value, $p - 1, 1 ) );
     }
     elsif ( $byte == 0xcd ) { # uint16
         $p += 2;
+        $p <= length($value) or _insufficient('uint16');
         return unpack_uint16( $value, $p - 2 );
     }
     elsif ( $byte == 0xce ) { # unit32
         $p += 4;
+        $p <= length($value) or _insufficient('uint32');
         return unpack_uint32( $value, $p - 4 );
     }
     elsif ( $byte == 0xcf ) { # unit64
         $p += 8;
+        $p <= length($value) or _insufficient('uint64');
         return unpack_uint64( $value, $p - 8 );
     }
     elsif ( $byte == 0xd3 ) { # int64
         $p += 8;
+        $p <= length($value) or _insufficient('int64');
         return unpack_int64( $value, $p - 8 );
     }
     elsif ( $byte == 0xd2 ) { # int32
         $p += 4;
+        $p <= length($value) or _insufficient('int32');
         return unpack_int32( $value, $p - 4 );
     }
     elsif ( $byte == 0xd1 ) { # int16
         $p += 2;
+        $p <= length($value) or _insufficient('int16');
         return unpack_int16( $value, $p - 2 );
     }
     elsif ( $byte == 0xd0 ) { # int8
-        return CORE::unpack 'c',  substr( $value, $p++, 1 ); # c / C
+        $p++;
+        $p <= length($value) or _insufficient('int8');
+        return CORE::unpack 'c',  substr( $value, $p - 1, 1 );
     }
     elsif ( $byte == 0xcb ) { # double
         $p += 8;
+        $p <= length($value) or _insufficient('double');
         return unpack_double( $value, $p - 8 );
     }
     elsif ( $byte == 0xca ) { # float
         $p += 4;
+        $p <= length($value) or _insufficient('float');
         return unpack_float( $value, $p - 4 );
     }
     else {
diff --git a/perl/t/02_unpack.t b/perl/t/02_unpack.t
index 1087c40..9e471d1 100644
--- a/perl/t/02_unpack.t
+++ b/perl/t/02_unpack.t
@@ -12,7 +12,10 @@ sub unpackit {
 
 sub pis ($$) {
     is_deeply unpackit($_[0]), $_[1], 'dump ' . $_[0]
-        or diag( explain(unpackit($_[0])) );
+        or do {
+            diag( 'got:', explain(unpackit($_[0])) );
+            diag( 'expected:', explain($_[1]) );
+        };
 }
 
 my @dat = do 't/data.pl' or die $@;

From b4ae6bf82c8bcd6012c805a2206fcb81ad39dd0b Mon Sep 17 00:00:00 2001
From: "Fuji, Goro" 
Date: Sat, 30 Oct 2010 13:04:30 +0900
Subject: [PATCH 0932/1648] perl: disable warnings

---
 perl/lib/Data/MessagePack/PP.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/perl/lib/Data/MessagePack/PP.pm b/perl/lib/Data/MessagePack/PP.pm
index fca73b8..f179ad7 100644
--- a/perl/lib/Data/MessagePack/PP.pm
+++ b/perl/lib/Data/MessagePack/PP.pm
@@ -480,6 +480,7 @@ sub execute {
 
 sub _count {
     my ( $self, $value ) = @_;
+    no warnings; # FIXME
     my $byte = unpack( 'C', substr( $value, $p++, 1 ) ); # get header
 
     Carp::croak('invalid data') unless defined $byte;

From bad69fd3977fca94d643a88d819badaca4839fd5 Mon Sep 17 00:00:00 2001
From: advect 
Date: Sat, 30 Oct 2010 16:06:50 +0900
Subject: [PATCH 0933/1648] php: fiexed unpacker

---
 php/ChangeLog        |   4 +
 php/msgpack.c        |   2 +-
 php/msgpack_class.c  |  35 ++-
 php/package.xml      |  62 ++---
 php/php-msgpack.spec |   1 -
 php/php_msgpack.h    |   2 +-
 php/tests/009.phpt   |  28 +--
 php/tests/009b.phpt  |  28 ++-
 php/tests/024.phpt   |  11 +-
 php/tests/024b.phpt  |  11 +-
 php/tests/026.phpt   |  52 +----
 php/tests/026b.phpt  |  52 ++++-
 php/tests/028.phpt   | 544 ++-----------------------------------------
 php/tests/028b.phpt  | 544 +++++++++++++++++++++++++++++++++++++++++--
 php/tests/042.phpt   |   2 +-
 php/tests/060.phpt   |  12 +-
 php/tests/060b.phpt  |  12 +-
 php/tests/061.phpt   |  12 +-
 php/tests/061b.phpt  |  12 +-
 php/tests/064.phpt   | 315 +++++++++++++++++++++++++
 php/tests/064b.phpt  | 321 +++++++++++++++++++++++++
 php/tests/065.phpt   | 320 +++++++++++++++++++++++++
 php/tests/065b.phpt  | 326 ++++++++++++++++++++++++++
 php/tests/066.phpt   |  70 ++++++
 php/tests/067.phpt   |  74 ++++++
 php/tests/070.phpt   |  12 +-
 php/tests/070b.phpt  |  12 +-
 php/tests/071.phpt   |  12 +-
 php/tests/071b.phpt  |  12 +-
 php/tests/072.phpt   |  12 +-
 php/tests/072b.phpt  |  12 +-
 php/tests/073.phpt   |  12 +-
 php/tests/073b.phpt  |  12 +-
 33 files changed, 2206 insertions(+), 742 deletions(-)
 create mode 100644 php/tests/064.phpt
 create mode 100644 php/tests/064b.phpt
 create mode 100644 php/tests/065.phpt
 create mode 100644 php/tests/065b.phpt
 create mode 100644 php/tests/066.phpt
 create mode 100644 php/tests/067.phpt

diff --git a/php/ChangeLog b/php/ChangeLog
index 3cee3b6..3eb64e7 100644
--- a/php/ChangeLog
+++ b/php/ChangeLog
@@ -1,5 +1,9 @@
 msgpack extension changelog
 
+Version 0.3.1
+-------------
+	* Fix class MessagePackUnpacker.
+
 Version 0.3.0
 -------------
 	* Change msgpack_unpack.c (used template)
diff --git a/php/msgpack.c b/php/msgpack.c
index 5d4f926..3b375ba 100644
--- a/php/msgpack.c
+++ b/php/msgpack.c
@@ -113,7 +113,7 @@ zend_module_entry msgpack_module_entry = {
     NULL,
     ZEND_MINFO(msgpack),
 #if ZEND_MODULE_API_NO >= 20010901
-    MSGPACK_VERSION,
+    MSGPACK_EXTENSION_VERSION,
 #endif
     STANDARD_MODULE_PROPERTIES
 };
diff --git a/php/msgpack_class.c b/php/msgpack_class.c
index 5cfff8d..ce008d9 100644
--- a/php/msgpack_class.c
+++ b/php/msgpack_class.c
@@ -19,6 +19,7 @@ typedef struct {
     msgpack_unpack_t mp;
     php_unserialize_data_t var_hash;
     long php_only;
+    zend_bool finished;
 } php_msgpack_unpacker_t;
 
 #if ZEND_MODULE_API_NO >= 20060613
@@ -348,6 +349,7 @@ static ZEND_METHOD(msgpack_unpacker, __construct)
     unpacker->buffer.a = 0;
     unpacker->retval = NULL;
     unpacker->offset = 0;
+    unpacker->finished = 0;
 
     template_init(&unpacker->mp);
 
@@ -460,6 +462,22 @@ static ZEND_METHOD(msgpack_unpacker, execute)
     {
         ALLOC_INIT_ZVAL(unpacker->retval);
     }
+    else if (unpacker->finished)
+    {
+        zval_ptr_dtor(&unpacker->retval);
+
+        msgpack_unserialize_var_destroy(&unpacker->var_hash);
+
+
+        ALLOC_INIT_ZVAL(unpacker->retval);
+
+        template_init(&unpacker->mp);
+
+        msgpack_unserialize_var_init(&unpacker->var_hash);
+
+        (&unpacker->mp)->user.var_hash =
+            (php_unserialize_data_t *)&unpacker->var_hash;
+    }
     (&unpacker->mp)->user.retval = (zval *)unpacker->retval;
 
     MSGPACK_G(error_display) = 0;
@@ -483,6 +501,7 @@ static ZEND_METHOD(msgpack_unpacker, execute)
     {
         case MSGPACK_UNPACK_EXTRA_BYTES:
         case MSGPACK_UNPACK_SUCCESS:
+            unpacker->finished = 1;
             RETURN_TRUE;
         default:
             RETURN_FALSE;
@@ -493,7 +512,16 @@ static ZEND_METHOD(msgpack_unpacker, data)
 {
     MSGPACK_UNPACKER_OBJECT;
 
-    RETURN_ZVAL(unpacker->retval, 1, 1);
+    if (unpacker->retval != NULL)
+    {
+        ZVAL_ZVAL(return_value, unpacker->retval, 1, 0);
+
+        MSGPACK_METHOD(msgpack_unpacker, reset, NULL, getThis());
+
+        return;
+    }
+
+    RETURN_FALSE;
 }
 
 static ZEND_METHOD(msgpack_unpacker, reset)
@@ -513,6 +541,7 @@ static ZEND_METHOD(msgpack_unpacker, reset)
     unpacker->buffer.len = 0;
     unpacker->buffer.a = 0;
     unpacker->offset = 0;
+    unpacker->finished = 0;
 
     if (buffer.len > 0)
     {
@@ -530,12 +559,12 @@ static ZEND_METHOD(msgpack_unpacker, reset)
     msgpack_unserialize_var_destroy(&unpacker->var_hash);
 
 
+    template_init(&unpacker->mp);
+
     msgpack_unserialize_var_init(&unpacker->var_hash);
 
     (&unpacker->mp)->user.var_hash =
         (php_unserialize_data_t *)&unpacker->var_hash;
-
-    msgpack_unserialize_init(&((&unpacker->mp)->user));
 }
 
 void msgpack_init_class()
diff --git a/php/package.xml b/php/package.xml
index 6172a4c..803aa97 100644
--- a/php/package.xml
+++ b/php/package.xml
@@ -10,11 +10,11 @@
     advect@gmail.com
     yes
   
-  2010-09-28
-  
+  2010-10-26
+  
   
-    0.3.0
-    0.3.0
+    0.3.1
+    0.3.1
   
   
     beta
@@ -27,14 +27,14 @@
       
       
       
-      
-      
+      
+      
       
       
       
       
       
-      
+      
       
       
       
@@ -49,8 +49,8 @@
       
       
       
-      
-      
+      
+      
       
       
       
@@ -65,14 +65,14 @@
       
       
       
-      
-      
+      
+      
       
-      
-      
+      
+      
       
-      
-      
+      
+      
       
       
       
@@ -82,22 +82,28 @@
       
       
       
-      
+      
       
-      
-      
-      
-      
+      
+      
+      
+      
       
       
-      
-      
-      
-      
-      
-      
-      
-      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
+      
       
       
       
diff --git a/php/php-msgpack.spec b/php/php-msgpack.spec
index 6a98288..7609ca4 100644
--- a/php/php-msgpack.spec
+++ b/php/php-msgpack.spec
@@ -12,7 +12,6 @@ Packager: advect 
 Provides: php-pecl-msgpack
 BuildRoot: %{_tmppath}/%{name}-%{version}-root
 BuildRequires: php-devel
-Requires: msgpack
 %if 0%{?php_zend_api}
 Requires: php(zend-abi) = %{php_zend_api}
 Requires: php(api) = %{php_core_api}
diff --git a/php/php_msgpack.h b/php/php_msgpack.h
index 61badea..0791deb 100644
--- a/php/php_msgpack.h
+++ b/php/php_msgpack.h
@@ -2,7 +2,7 @@
 #ifndef PHP_MSGPACK_H
 #define PHP_MSGPACK_H
 
-#define MSGPACK_EXTENSION_VERSION "0.3.0"
+#define MSGPACK_EXTENSION_VERSION "0.3.1"
 
 #include "ext/standard/php_smart_str.h"
 
diff --git a/php/tests/009.phpt b/php/tests/009.phpt
index 9992988..a1534c9 100644
--- a/php/tests/009.phpt
+++ b/php/tests/009.phpt
@@ -2,8 +2,8 @@
 Check for reference serialization
 --SKIPIF--
 = 0) {
-    echo "skip tests in PHP 5.3.2 and lower";
+if (version_compare(PHP_VERSION, '5.3.2') <= 0) {
+    echo "skip tests in PHP 5.3.3 or newer";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      &array(1) {
-        [0]=>
-        &array(1) {
-          [0]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
@@ -91,13 +85,7 @@ array(1) {
     [0]=>
     &array(1) {
       [0]=>
-      &array(1) {
-        [0]=>
-        &array(1) {
-          [0]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
@@ -107,13 +95,7 @@ array(1) {
     [0]=>
     &array(1) {
       [0]=>
-      &array(1) {
-        [0]=>
-        &array(1) {
-          [0]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
diff --git a/php/tests/009b.phpt b/php/tests/009b.phpt
index 7765d62..10e7259 100644
--- a/php/tests/009b.phpt
+++ b/php/tests/009b.phpt
@@ -2,8 +2,8 @@
 Check for reference serialization
 --SKIPIF--
 = 0) {
+    echo "skip tests in PHP 5.3.2 or older";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      *RECURSION*
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }
@@ -85,7 +91,13 @@ array(1) {
     [0]=>
     &array(1) {
       [0]=>
-      *RECURSION*
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }
@@ -95,7 +107,13 @@ array(1) {
     [0]=>
     &array(1) {
       [0]=>
-      *RECURSION*
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }
diff --git a/php/tests/024.phpt b/php/tests/024.phpt
index 9b185f7..97db2a7 100644
--- a/php/tests/024.phpt
+++ b/php/tests/024.phpt
@@ -2,8 +2,8 @@
 Recursive objects
 --SKIPIF--
 = 0) {
-    echo "skip tests in PHP 5.3.2 and lower";
+if (version_compare(PHP_VERSION, '5.3.2') <= 0) {
+    echo "skip tests in PHP 5.3.3 or newer";
 }
 --FILE--
 
   int(100)
   [%r"?obj"?:("Obj4":)?private"?%r]=>
-  object(Obj4)#%d (2) {
-    [%r"?a"?:("Obj4":)?private"?%r]=>
-    int(100)
-    [%r"?obj"?:("Obj4":)?private"?%r]=>
-    *RECURSION*
-  }
+  *RECURSION*
 }
 OK
diff --git a/php/tests/024b.phpt b/php/tests/024b.phpt
index 7c691be..c7612d7 100644
--- a/php/tests/024b.phpt
+++ b/php/tests/024b.phpt
@@ -2,8 +2,8 @@
 Recursive objects
 --SKIPIF--
 = 0) {
+    echo "skip tests in PHP 5.3.2 or older";
 }
 --FILE--
 
   int(100)
   [%r"?obj"?:("Obj4":)?private"?%r]=>
-  *RECURSION*
+  object(Obj4)#%d (2) {
+    [%r"?a"?:("Obj4":)?private"?%r]=>
+    int(100)
+    [%r"?obj"?:("Obj4":)?private"?%r]=>
+    *RECURSION*
+  }
 }
 OK
diff --git a/php/tests/026.phpt b/php/tests/026.phpt
index c243ef5..383289d 100644
--- a/php/tests/026.phpt
+++ b/php/tests/026.phpt
@@ -3,8 +3,8 @@ Cyclic array test
 --INI--
 --SKIPIF--
 = 0) {
-    echo "skip tests in PHP 5.3.2 and lower";
+if (version_compare(PHP_VERSION, '5.3.2') <= 0) {
+    echo "skip tests in PHP 5.3.3 or newer";
 }
 --FILE--
 
-    &array(2) {
-      ["a"]=>
-      array(2) {
-        ["b"]=>
-        string(1) "c"
-        ["d"]=>
-        string(1) "e"
-      }
-      ["f"]=>
-      *RECURSION*
-    }
+    *RECURSION*
   }
 }
 OK
@@ -84,17 +74,7 @@ array(1) {
     [1]=>
     int(2)
     [2]=>
-    array(1) {
-      ["foo"]=>
-      &array(3) {
-        [0]=>
-        int(1)
-        [1]=>
-        int(2)
-        [2]=>
-        *RECURSION*
-      }
-    }
+    *RECURSION*
   }
 }
 array(1) {
@@ -107,17 +87,7 @@ array(1) {
     [2]=>
     array(1) {
       ["foo"]=>
-      &array(3) {
-        [0]=>
-        int(1)
-        [1]=>
-        int(2)
-        [2]=>
-        array(1) {
-          ["foo"]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
@@ -131,17 +101,7 @@ array(1) {
     [2]=>
     array(1) {
       ["foo"]=>
-      &array(3) {
-        [0]=>
-        int(1)
-        [1]=>
-        string(1) "b"
-        [2]=>
-        array(1) {
-          ["foo"]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
diff --git a/php/tests/026b.phpt b/php/tests/026b.phpt
index f7a3381..9eef7cb 100644
--- a/php/tests/026b.phpt
+++ b/php/tests/026b.phpt
@@ -3,8 +3,8 @@ Cyclic array test
 --INI--
 --SKIPIF--
 = 0) {
+    echo "skip tests in PHP 5.3.2 or older";
 }
 --FILE--
 
-    *RECURSION*
+    &array(2) {
+      ["a"]=>
+      array(2) {
+        ["b"]=>
+        string(1) "c"
+        ["d"]=>
+        string(1) "e"
+      }
+      ["f"]=>
+      *RECURSION*
+    }
   }
 }
 OK
@@ -74,7 +84,17 @@ array(1) {
     [1]=>
     int(2)
     [2]=>
-    *RECURSION*
+    array(1) {
+      ["foo"]=>
+      &array(3) {
+        [0]=>
+        int(1)
+        [1]=>
+        int(2)
+        [2]=>
+        *RECURSION*
+      }
+    }
   }
 }
 array(1) {
@@ -87,7 +107,17 @@ array(1) {
     [2]=>
     array(1) {
       ["foo"]=>
-      *RECURSION*
+      &array(3) {
+        [0]=>
+        int(1)
+        [1]=>
+        int(2)
+        [2]=>
+        array(1) {
+          ["foo"]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }
@@ -101,7 +131,17 @@ array(1) {
     [2]=>
     array(1) {
       ["foo"]=>
-      *RECURSION*
+      &array(3) {
+        [0]=>
+        int(1)
+        [1]=>
+        string(1) "b"
+        [2]=>
+        array(1) {
+          ["foo"]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }
diff --git a/php/tests/028.phpt b/php/tests/028.phpt
index 00db675..1d326ce 100644
--- a/php/tests/028.phpt
+++ b/php/tests/028.phpt
@@ -2,8 +2,8 @@
 Serialize object into session, full set
 --SKIPIF--
 = 0) {
-    echo "skip tests in PHP 5.3.2 and lower";
+if (version_compare(PHP_VERSION, '5.3.2') <= 0) {
+    echo "skip tests in PHP 5.3.3 or newer";
 }
 --FILE--
 
     object(Bar)#4 (3) {
       ["d1"]=>
-      object(Foo)#3 (3) {
-        [%r"?d1"?:("Foo":)?private"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:protected"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        ["d3"]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       [%r"?d2"?:("Bar":)?private"?%r]=>
-      object(Foo)#3 (3) {
-        [%r"?d1"?:("Foo":)?private"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:protected"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        ["d3"]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       [%r"?d3"?:protected"?%r]=>
-      object(Foo)#3 (3) {
-        [%r"?d1"?:("Foo":)?private"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:protected"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        ["d3"]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
     [%r"?d2"?:protected"?%r]=>
     object(Bar)#4 (3) {
       ["d1"]=>
-      object(Foo)#3 (3) {
-        [%r"?d1"?:("Foo":)?private"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:protected"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        ["d3"]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       [%r"?d2"?:("Bar":)?private"?%r]=>
-      object(Foo)#3 (3) {
-        [%r"?d1"?:("Foo":)?private"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:protected"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        ["d3"]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       [%r"?d3"?:protected"?%r]=>
-      object(Foo)#3 (3) {
-        [%r"?d1"?:("Foo":)?private"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:protected"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        ["d3"]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
     ["d3"]=>
     object(Bar)#4 (3) {
       ["d1"]=>
-      object(Foo)#3 (3) {
-        [%r"?d1"?:("Foo":)?private"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:protected"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        ["d3"]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       [%r"?d2"?:("Bar":)?private"?%r]=>
-      object(Foo)#3 (3) {
-        [%r"?d1"?:("Foo":)?private"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:protected"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        ["d3"]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       [%r"?d3"?:protected"?%r]=>
-      object(Foo)#3 (3) {
-        [%r"?d1"?:("Foo":)?private"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:protected"?%r]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-        ["d3"]=>
-        object(Bar)#4 (3) {
-          ["d1"]=>
-          *RECURSION*
-          [%r"?d2"?:("Bar":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d3"?:protected"?%r]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
   ["test"]=>
@@ -391,281 +139,29 @@ array(3) {
     ["d1"]=>
     object(Foo)#6 (3) {
       [%r"?d1"?:("Foo":)?private"?%r]=>
-      object(Bar)#5 (3) {
-        ["d1"]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:("Bar":)?private"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d3"?:protected"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       [%r"?d2"?:protected"?%r]=>
-      object(Bar)#5 (3) {
-        ["d1"]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:("Bar":)?private"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d3"?:protected"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       ["d3"]=>
-      object(Bar)#5 (3) {
-        ["d1"]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:("Bar":)?private"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d3"?:protected"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
     [%r"?d2"?:("Bar":)?private"?%r]=>
     object(Foo)#6 (3) {
       [%r"?d1"?:("Foo":)?private"?%r]=>
-      object(Bar)#5 (3) {
-        ["d1"]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:("Bar":)?private"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d3"?:protected"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       [%r"?d2"?:protected"?%r]=>
-      object(Bar)#5 (3) {
-        ["d1"]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:("Bar":)?private"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d3"?:protected"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       ["d3"]=>
-      object(Bar)#5 (3) {
-        ["d1"]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:("Bar":)?private"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d3"?:protected"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
     [%r"?d3"?:protected"?%r]=>
     object(Foo)#6 (3) {
       [%r"?d1"?:("Foo":)?private"?%r]=>
-      object(Bar)#5 (3) {
-        ["d1"]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:("Bar":)?private"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d3"?:protected"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       [%r"?d2"?:protected"?%r]=>
-      object(Bar)#5 (3) {
-        ["d1"]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:("Bar":)?private"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d3"?:protected"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
       ["d3"]=>
-      object(Bar)#5 (3) {
-        ["d1"]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d2"?:("Bar":)?private"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-        [%r"?d3"?:protected"?%r]=>
-        object(Foo)#6 (3) {
-          [%r"?d1"?:("Foo":)?private"?%r]=>
-          *RECURSION*
-          [%r"?d2"?:protected"?%r]=>
-          *RECURSION*
-          ["d3"]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
diff --git a/php/tests/028b.phpt b/php/tests/028b.phpt
index 7331a57..0efba18 100644
--- a/php/tests/028b.phpt
+++ b/php/tests/028b.phpt
@@ -2,8 +2,8 @@
 Serialize object into session, full set
 --SKIPIF--
 = 0) {
+    echo "skip tests in PHP 5.3.2 or older";
 }
 --FILE--
 
     object(Bar)#4 (3) {
       ["d1"]=>
-      *RECURSION*
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
       [%r"?d2"?:("Bar":)?private"?%r]=>
-      *RECURSION*
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
       [%r"?d3"?:protected"?%r]=>
-      *RECURSION*
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
     }
     [%r"?d2"?:protected"?%r]=>
     object(Bar)#4 (3) {
       ["d1"]=>
-      *RECURSION*
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
       [%r"?d2"?:("Bar":)?private"?%r]=>
-      *RECURSION*
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
       [%r"?d3"?:protected"?%r]=>
-      *RECURSION*
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
     }
     ["d3"]=>
     object(Bar)#4 (3) {
       ["d1"]=>
-      *RECURSION*
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
       [%r"?d2"?:("Bar":)?private"?%r]=>
-      *RECURSION*
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
       [%r"?d3"?:protected"?%r]=>
-      *RECURSION*
+      object(Foo)#3 (3) {
+        [%r"?d1"?:("Foo":)?private"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:protected"?%r]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+        ["d3"]=>
+        object(Bar)#4 (3) {
+          ["d1"]=>
+          *RECURSION*
+          [%r"?d2"?:("Bar":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d3"?:protected"?%r]=>
+          *RECURSION*
+        }
+      }
     }
   }
   ["test"]=>
@@ -139,29 +391,281 @@ array(3) {
     ["d1"]=>
     object(Foo)#6 (3) {
       [%r"?d1"?:("Foo":)?private"?%r]=>
-      *RECURSION*
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
       [%r"?d2"?:protected"?%r]=>
-      *RECURSION*
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
       ["d3"]=>
-      *RECURSION*
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
     }
     [%r"?d2"?:("Bar":)?private"?%r]=>
     object(Foo)#6 (3) {
       [%r"?d1"?:("Foo":)?private"?%r]=>
-      *RECURSION*
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
       [%r"?d2"?:protected"?%r]=>
-      *RECURSION*
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
       ["d3"]=>
-      *RECURSION*
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
     }
     [%r"?d3"?:protected"?%r]=>
     object(Foo)#6 (3) {
       [%r"?d1"?:("Foo":)?private"?%r]=>
-      *RECURSION*
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
       [%r"?d2"?:protected"?%r]=>
-      *RECURSION*
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
       ["d3"]=>
-      *RECURSION*
+      object(Bar)#5 (3) {
+        ["d1"]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d2"?:("Bar":)?private"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+        [%r"?d3"?:protected"?%r]=>
+        object(Foo)#6 (3) {
+          [%r"?d1"?:("Foo":)?private"?%r]=>
+          *RECURSION*
+          [%r"?d2"?:protected"?%r]=>
+          *RECURSION*
+          ["d3"]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }
diff --git a/php/tests/042.phpt b/php/tests/042.phpt
index 2ba73ac..83e8bc7 100644
--- a/php/tests/042.phpt
+++ b/php/tests/042.phpt
@@ -3,7 +3,7 @@ Closure
 --SKIPIF--
 = 0) {
-    echo "skip tests in PHP 5.3.2 and lower";
+if (version_compare(PHP_VERSION, '5.3.2') <= 0) {
+    echo "skip tests in PHP 5.3.3 or newer";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      &array(1) {
-        [0]=>
-        &array(1) {
-          [0]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
diff --git a/php/tests/060b.phpt b/php/tests/060b.phpt
index 0b947c8..a2e50bc 100644
--- a/php/tests/060b.phpt
+++ b/php/tests/060b.phpt
@@ -2,8 +2,8 @@
 Check for buffered streaming unserialization
 --SKIPIF--
 = 0) {
+    echo "skip tests in PHP 5.3.2 or older";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      *RECURSION*
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }
diff --git a/php/tests/061.phpt b/php/tests/061.phpt
index 7ecc0c9..2b42d31 100644
--- a/php/tests/061.phpt
+++ b/php/tests/061.phpt
@@ -2,8 +2,8 @@
 Check for unbuffered streaming unserialization
 --SKIPIF--
 = 0) {
-    echo "skip tests in PHP 5.3.2 and lower";
+if (version_compare(PHP_VERSION, '5.3.2') <= 0) {
+    echo "skip tests in PHP 5.3.3 or newer";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      &array(1) {
-        [0]=>
-        &array(1) {
-          [0]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
diff --git a/php/tests/061b.phpt b/php/tests/061b.phpt
index efb3a9b..21706db 100644
--- a/php/tests/061b.phpt
+++ b/php/tests/061b.phpt
@@ -2,8 +2,8 @@
 Check for unbuffered streaming unserialization
 --SKIPIF--
 = 0) {
+    echo "skip tests in PHP 5.3.2 or older";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      *RECURSION*
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }
diff --git a/php/tests/064.phpt b/php/tests/064.phpt
new file mode 100644
index 0000000..a7a2f3b
--- /dev/null
+++ b/php/tests/064.phpt
@@ -0,0 +1,315 @@
+--TEST--
+Check for buffered streaming unserialization (single)
+--SKIPIF--
+feed($str);
+        if ($unpacker->execute())
+        {
+            $unserialized = $unpacker->data();
+            var_dump($unserialized);
+            $unpacker->reset();
+        }
+
+        $i += $len;
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('bool: true', true);
+test('bool: false', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('array', array(), false);
+test('array(1, 2, 3)', array(1, 2, 3), false);
+test('array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      *RECURSION*
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/064b.phpt b/php/tests/064b.phpt
new file mode 100644
index 0000000..aba7a2d
--- /dev/null
+++ b/php/tests/064b.phpt
@@ -0,0 +1,321 @@
+--TEST--
+Check for buffered streaming unserialization (single)
+--SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 or older";
+}
+--FILE--
+feed($str);
+        if ($unpacker->execute())
+        {
+            $unserialized = $unpacker->data();
+            var_dump($unserialized);
+            $unpacker->reset();
+        }
+
+        $i += $len;
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('bool: true', true);
+test('bool: false', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('array', array(), false);
+test('array(1, 2, 3)', array(1, 2, 3), false);
+test('array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/065.phpt b/php/tests/065.phpt
new file mode 100644
index 0000000..c37ca12
--- /dev/null
+++ b/php/tests/065.phpt
@@ -0,0 +1,320 @@
+--TEST--
+Check for unbuffered streaming unserialization (single)
+--SKIPIF--
+execute($str, $offset))
+        {
+            $unserialized = $unpacker->data();
+            var_dump($unserialized);
+
+            $unpacker->reset();
+            $str = "";
+            $offset = 0;
+        }
+
+        $i += $len;
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      *RECURSION*
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/065b.phpt b/php/tests/065b.phpt
new file mode 100644
index 0000000..c623d9a
--- /dev/null
+++ b/php/tests/065b.phpt
@@ -0,0 +1,326 @@
+--TEST--
+Check for unbuffered streaming unserialization (single)
+--SKIPIF--
+= 0) {
+    echo "skip tests in PHP 5.3.2 or older";
+}
+--FILE--
+execute($str, $offset))
+        {
+            $unserialized = $unpacker->data();
+            var_dump($unserialized);
+
+            $unpacker->reset();
+            $str = "";
+            $offset = 0;
+        }
+
+        $i += $len;
+    }
+
+    if (!is_bool($test))
+    {
+        echo $unserialized === $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+    else
+    {
+        echo $test || $unserialized == $variable ? 'OK' : 'ERROR', PHP_EOL;
+    }
+}
+
+test('null', null);
+
+test('boo:l true', true);
+test('bool: true', false);
+
+test('zero: 0', 0);
+test('small: 1', 1);
+test('small: -1', -1);
+test('medium: 1000', 1000);
+test('medium: -1000', -1000);
+test('large: 100000', 100000);
+test('large: -100000', -100000);
+
+test('double: 123.456', 123.456);
+
+test('empty: ""', "");
+test('string: "foobar"', "foobar");
+
+test('empty: array', array(), false);
+test('empty: array(1, 2, 3)', array(1, 2, 3), false);
+test('empty: array(array(1, 2, 3), arr...', array(array(1, 2, 3), array(4, 5, 6), array(7, 8, 9)), false);
+
+test('array("foo", "foo", "foo")', array("foo", "foo", "foo"), false);
+test('array("one" => 1, "two" => 2))', array("one" => 1, "two" => 2), false);
+test('array("kek" => "lol", "lol" => "kek")', array("kek" => "lol", "lol" => "kek"), false);
+test('array("" => "empty")', array("" => "empty"), false);
+
+$a = array('foo');
+test('array($a, $a)', array($a, $a), false);
+test('array(&$a, &$a)', array(&$a, &$a), false);
+
+$a = array(null);
+$b = array(&$a);
+$a[0] = &$b;
+
+test('cyclic', $a, true);
+
+$a = array(
+    'a' => array(
+        'b' => 'c',
+        'd' => 'e'
+        ),
+    'f' => array(
+        'g' => 'h'
+        )
+    );
+
+test('array', $a, false);
+
+class Obj {
+    public $a;
+    protected $b;
+    private $c;
+
+    function __construct($a, $b, $c) {
+        $this->a = $a;
+        $this->b = $b;
+        $this->c = $c;
+    }
+}
+
+test('object', new Obj(1, 2, 3), false);
+
+test('object', array(new Obj(1, 2, 3), new Obj(4, 5, 6)), false);
+
+$o = new Obj(1, 2, 3);
+
+test('object', array(&$o, &$o), false);
+--EXPECTF--
+NULL
+OK
+bool(true)
+OK
+bool(false)
+OK
+int(0)
+OK
+int(1)
+OK
+int(-1)
+OK
+int(1000)
+OK
+int(-1000)
+OK
+int(100000)
+OK
+int(-100000)
+OK
+float(123.456)
+OK
+string(0) ""
+OK
+string(6) "foobar"
+OK
+array(0) {
+}
+OK
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+OK
+array(3) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(4)
+    [1]=>
+    int(5)
+    [2]=>
+    int(6)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(7)
+    [1]=>
+    int(8)
+    [2]=>
+    int(9)
+  }
+}
+OK
+array(3) {
+  [0]=>
+  string(3) "foo"
+  [1]=>
+  string(3) "foo"
+  [2]=>
+  string(3) "foo"
+}
+OK
+array(2) {
+  ["one"]=>
+  int(1)
+  ["two"]=>
+  int(2)
+}
+OK
+array(2) {
+  ["kek"]=>
+  string(3) "lol"
+  ["lol"]=>
+  string(3) "kek"
+}
+OK
+array(1) {
+  [""]=>
+  string(5) "empty"
+}
+OK
+array(2) {
+  [0]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+  [1]=>
+  &array(1) {
+    [0]=>
+    string(3) "foo"
+  }
+}
+OK
+array(1) {
+  [0]=>
+  &array(1) {
+    [0]=>
+    &array(1) {
+      [0]=>
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
+    }
+  }
+}
+OK
+array(2) {
+  ["a"]=>
+  array(2) {
+    ["b"]=>
+    string(1) "c"
+    ["d"]=>
+    string(1) "e"
+  }
+  ["f"]=>
+  array(1) {
+    ["g"]=>
+    string(1) "h"
+  }
+}
+OK
+object(Obj)#%d (3) {
+  ["a"]=>
+  int(1)
+  [%r"?b"?:protected"?%r]=>
+  int(2)
+  [%r"?c"?:("Obj":)?private"?%r]=>
+  int(3)
+}
+OK
+array(2) {
+  [0]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  object(Obj)#%d (3) {
+    ["a"]=>
+    int(4)
+    [%r"?b"?:protected"?%r]=>
+    int(5)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(6)
+  }
+}
+OK
+array(2) {
+  [0]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+  [1]=>
+  &object(Obj)#%d (3) {
+    ["a"]=>
+    int(1)
+    [%r"?b"?:protected"?%r]=>
+    int(2)
+    [%r"?c"?:("Obj":)?private"?%r]=>
+    int(3)
+  }
+}
+OK
diff --git a/php/tests/066.phpt b/php/tests/066.phpt
new file mode 100644
index 0000000..d2e430d
--- /dev/null
+++ b/php/tests/066.phpt
@@ -0,0 +1,70 @@
+--TEST--
+Extra bytes buffered streaming unserialization (single)
+--SKIPIF--
+--FILE--
+feed($str);
+
+            while (true) {
+                if ($unpacker->execute()) {
+                    $unserialized = $unpacker->data();
+                    var_dump($unserialized);
+                    $unpacker->reset();
+                } else {
+                    break;
+                }
+            }
+            $i += $len;
+        }
+    }
+}
+
+test('array(1, 2, 3)', array('9301020392'));
+test('array(1, 2, 3), array(3, 9), 4', array('9301020392', '030904'));
+--EXPECTF--
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(2) {
+    [0]=>
+    int(3)
+    [1]=>
+    int(9)
+  }
+}
+int(4)
diff --git a/php/tests/067.phpt b/php/tests/067.phpt
new file mode 100644
index 0000000..96c1442
--- /dev/null
+++ b/php/tests/067.phpt
@@ -0,0 +1,74 @@
+--TEST--
+Extra bytes unbuffered streaming unserialization (single)
+--SKIPIF--
+--FILE--
+execute($str, $offset)) {
+                    $unserialized = $unpacker->data();
+                    var_dump($unserialized);
+
+                    $unpacker->reset();
+                    $str = substr($str, $offset);
+                    $offset = 0;
+                } else {
+                    break;
+                }
+            }
+            $i += $len;
+        }
+    }
+}
+
+test('array(1, 2, 3)', array('9301020392'));
+test('array(1, 2, 3), array(3, 9), 4', array('9301020392', '030904'));
+--EXPECTF--
+array(3) {
+  [0]=>
+  int(1)
+  [1]=>
+  int(2)
+  [2]=>
+  int(3)
+}
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(1)
+    [1]=>
+    int(2)
+    [2]=>
+    int(3)
+  }
+  [1]=>
+  array(2) {
+    [0]=>
+    int(3)
+    [1]=>
+    int(9)
+  }
+}
+int(4)
diff --git a/php/tests/070.phpt b/php/tests/070.phpt
index 893023b..affcc72 100644
--- a/php/tests/070.phpt
+++ b/php/tests/070.phpt
@@ -2,8 +2,8 @@
 Check for alias functions
 --SKIPIF--
 = 0) {
-    echo "skip tests in PHP 5.3.2 and lower";
+if (version_compare(PHP_VERSION, '5.3.2') <= 0) {
+    echo "skip tests in PHP 5.3.3 or newer";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      &array(1) {
-        [0]=>
-        &array(1) {
-          [0]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
diff --git a/php/tests/070b.phpt b/php/tests/070b.phpt
index 109ddc1..0aaa4e7 100644
--- a/php/tests/070b.phpt
+++ b/php/tests/070b.phpt
@@ -2,8 +2,8 @@
 Check for alias functions
 --SKIPIF--
 = 0) {
+    echo "skip tests in PHP 5.3.2 or older";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      *RECURSION*
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }
diff --git a/php/tests/071.phpt b/php/tests/071.phpt
index 431303b..20041d4 100644
--- a/php/tests/071.phpt
+++ b/php/tests/071.phpt
@@ -2,8 +2,8 @@
 Check for class methods
 --SKIPIF--
 = 0) {
-    echo "skip tests in PHP 5.3.2 and lower";
+if (version_compare(PHP_VERSION, '5.3.2') <= 0) {
+    echo "skip tests in PHP 5.3.3 or newer";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      &array(1) {
-        [0]=>
-        &array(1) {
-          [0]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
diff --git a/php/tests/071b.phpt b/php/tests/071b.phpt
index 770d06e..f663073 100644
--- a/php/tests/071b.phpt
+++ b/php/tests/071b.phpt
@@ -2,8 +2,8 @@
 Check for class methods
 --SKIPIF--
 = 0) {
+    echo "skip tests in PHP 5.3.2 or older";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      *RECURSION*
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }
diff --git a/php/tests/072.phpt b/php/tests/072.phpt
index ea79a56..0f89e0a 100644
--- a/php/tests/072.phpt
+++ b/php/tests/072.phpt
@@ -2,8 +2,8 @@
 Check for class methods unpacker
 --SKIPIF--
 = 0) {
-    echo "skip tests in PHP 5.3.2 and lower";
+if (version_compare(PHP_VERSION, '5.3.2') <= 0) {
+    echo "skip tests in PHP 5.3.3 or newer";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      &array(1) {
-        [0]=>
-        &array(1) {
-          [0]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
diff --git a/php/tests/072b.phpt b/php/tests/072b.phpt
index c7b0c3c..05fe41e 100644
--- a/php/tests/072b.phpt
+++ b/php/tests/072b.phpt
@@ -2,8 +2,8 @@
 Check for class methods unpacker
 --SKIPIF--
 = 0) {
+    echo "skip tests in PHP 5.3.2 or older";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      *RECURSION*
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }
diff --git a/php/tests/073.phpt b/php/tests/073.phpt
index d3a7637..f6a91a6 100644
--- a/php/tests/073.phpt
+++ b/php/tests/073.phpt
@@ -2,8 +2,8 @@
 Check for class unpacker
 --SKIPIF--
 = 0) {
-    echo "skip tests in PHP 5.3.2 and lower";
+if (version_compare(PHP_VERSION, '5.3.2') <= 0) {
+    echo "skip tests in PHP 5.3.3 or newer";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      &array(1) {
-        [0]=>
-        &array(1) {
-          [0]=>
-          *RECURSION*
-        }
-      }
+      *RECURSION*
     }
   }
 }
diff --git a/php/tests/073b.phpt b/php/tests/073b.phpt
index 131a534..065a2aa 100644
--- a/php/tests/073b.phpt
+++ b/php/tests/073b.phpt
@@ -2,8 +2,8 @@
 Check for class unpacker
 --SKIPIF--
 = 0) {
+    echo "skip tests in PHP 5.3.2 or older";
 }
 --FILE--
 
     &array(1) {
       [0]=>
-      *RECURSION*
+      &array(1) {
+        [0]=>
+        &array(1) {
+          [0]=>
+          *RECURSION*
+        }
+      }
     }
   }
 }

From 09bad2938e259df58a91ea4aadf3a78c058effca Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 2 Nov 2010 14:02:10 +0900
Subject: [PATCH 0934/1648] python: Add msgpack/_msgpack.c to .gitignore

---
 python/.gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/.gitignore b/python/.gitignore
index 8531de3..35b00cc 100644
--- a/python/.gitignore
+++ b/python/.gitignore
@@ -4,3 +4,4 @@ dist/*
 *.pyc
 *.pyo
 msgpack/__version__.py
+msgpack/_msgpack.c

From 8d3ca93be9639f4a1066b39a8ee37f626dcb8c3c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 2 Nov 2010 14:02:10 +0900
Subject: [PATCH 0935/1648] python: Add msgpack/_msgpack.c to .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 8531de3..35b00cc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@ dist/*
 *.pyc
 *.pyo
 msgpack/__version__.py
+msgpack/_msgpack.c

From b1df5d3ad77187d6a2bdf67552268511808ab06b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 2 Nov 2010 14:09:50 +0900
Subject: [PATCH 0936/1648] python: Add test for unpacking buffer object.

---
 python/test/test_buffer.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 python/test/test_buffer.py

diff --git a/python/test/test_buffer.py b/python/test/test_buffer.py
new file mode 100644
index 0000000..436b554
--- /dev/null
+++ b/python/test/test_buffer.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import packb, unpackb
+
+def test_unpack_buffer():
+    from array import array
+    buf = array('b')
+    buf.fromstring(packb(['foo', 'bar']))
+    obj = unpackb(buf)
+    assert_equal(['foo', 'bar'], obj)
+
+if __name__ == '__main__':
+    main()

From 3903979a84c4d450acc55faceb3f816ff7a32398 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 2 Nov 2010 14:09:50 +0900
Subject: [PATCH 0937/1648] python: Add test for unpacking buffer object.

---
 test/test_buffer.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 test/test_buffer.py

diff --git a/test/test_buffer.py b/test/test_buffer.py
new file mode 100644
index 0000000..436b554
--- /dev/null
+++ b/test/test_buffer.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import packb, unpackb
+
+def test_unpack_buffer():
+    from array import array
+    buf = array('b')
+    buf.fromstring(packb(['foo', 'bar']))
+    obj = unpackb(buf)
+    assert_equal(['foo', 'bar'], obj)
+
+if __name__ == '__main__':
+    main()

From 4688252bd41082a0dd077b150b53834c0346cdb1 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Nov 2010 03:11:00 +0900
Subject: [PATCH 0938/1648] python: Support old buffer protocol when unpack.
 (experimental)

---
 python/msgpack/_msgpack.pyx   | 156 ++++++++++++----------------------
 python/test/test_buffer.py    |   6 +-
 python/test/test_sequnpack.py |  16 ++--
 3 files changed, 65 insertions(+), 113 deletions(-)

diff --git a/python/msgpack/_msgpack.pyx b/python/msgpack/_msgpack.pyx
index e9d6c7b..5394055 100644
--- a/python/msgpack/_msgpack.pyx
+++ b/python/msgpack/_msgpack.pyx
@@ -1,37 +1,14 @@
 # coding: utf-8
 
+from cpython cimport *
 cdef extern from "Python.h":
     ctypedef char* const_char_ptr "const char*"
+    ctypedef char* const_void_ptr "const void*"
     ctypedef struct PyObject
+    cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
 
-    cdef object PyBytes_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
-    cdef PyObject* Py_True
-    cdef PyObject* Py_False
-    cdef object PyUnicode_AsUTF8String(object)
-
-    cdef long long PyLong_AsLongLong(object o)
-    cdef unsigned long long PyLong_AsUnsignedLongLong(object o)
-
-    cdef bint PyBool_Check(object o)
-    cdef bint PyDict_Check(object o)
-    cdef bint PySequence_Check(object o)
-    cdef bint PyLong_Check(object o)
-    cdef bint PyInt_Check(object o)
-    cdef bint PyFloat_Check(object o)
-    cdef bint PyBytes_Check(object o)
-    cdef bint PyUnicode_Check(object o)
-    cdef bint PyCallable_Check(object o)
-    cdef void Py_INCREF(object o)
-    cdef void Py_DECREF(object o)
-
-cdef extern from "stdlib.h":
-    void* malloc(size_t)
-    void* realloc(void*, size_t)
-    void free(void*)
-
-cdef extern from "string.h":
-    void* memcpy(char* dst, char* src, size_t size)
-    void* memmove(char* dst, char* src, size_t size)
+from libc.stdlib cimport *
+from libc.string cimport *
 
 cdef extern from "pack.h":
     struct msgpack_packer:
@@ -104,10 +81,10 @@ cdef class Packer(object):
                 ret = msgpack_pack_false(&self.pk)
         elif PyLong_Check(o):
             if o > 0:
-                ullval = PyLong_AsUnsignedLongLong(o)
+                ullval = o
                 ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
             else:
-                llval = PyLong_AsLongLong(o)
+                llval = o
                 ret = msgpack_pack_long_long(&self.pk, llval)
         elif PyInt_Check(o):
             longval = o
@@ -160,7 +137,7 @@ cdef class Packer(object):
 
 def pack(object o, object stream, default=None):
     """pack an object `o` and write it to stream)."""
-    packer = Packer(default)
+    packer = Packer(default=default)
     stream.write(packer.pack(o))
 
 def packb(object o, default=None):
@@ -184,17 +161,21 @@ cdef extern from "unpack.h":
         PyObject* key
 
     int template_execute(template_context* ctx, const_char_ptr data,
-            size_t len, size_t* off)
+                         size_t len, size_t* off)
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
 
-def unpackb(bytes packed_bytes, object object_hook=None, object list_hook=None):
+def unpackb(object packed, object object_hook=None, object list_hook=None):
     """Unpack packed_bytes to object. Returns an unpacked object."""
-    cdef const_char_ptr p = packed_bytes
     cdef template_context ctx
     cdef size_t off = 0
     cdef int ret
+
+    cdef char* buf
+    cdef Py_ssize_t buf_len
+    PyObject_AsReadBuffer(packed, &buf, &buf_len)
+
     template_init(&ctx)
     ctx.user.use_list = 0
     ctx.user.object_hook = ctx.user.list_hook = NULL
@@ -206,7 +187,7 @@ def unpackb(bytes packed_bytes, object object_hook=None, object list_hook=None):
         if not PyCallable_Check(list_hook):
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
-    ret = template_execute(&ctx, p, len(packed_bytes), &off)
+    ret = template_execute(&ctx, buf, buf_len, &off)
     if ret == 1:
         return template_data(&ctx)
     else:
@@ -216,8 +197,8 @@ unpacks = unpackb
 
 def unpack(object stream, object object_hook=None, object list_hook=None):
     """unpack an object from stream."""
-    packed = stream.read()
-    return unpackb(packed, object_hook=object_hook, list_hook=list_hook)
+    return unpackb(stream.read(),
+                   object_hook=object_hook, list_hook=list_hook)
 
 cdef class UnpackIterator(object):
     cdef object unpacker
@@ -232,21 +213,12 @@ cdef class UnpackIterator(object):
         return self
 
 cdef class Unpacker(object):
-    """Unpacker(file_like=None, read_size=1024*1024)
+    """Unpacker(read_size=1024*1024)
 
     Streaming unpacker.
-    file_like must have read(n) method.
     read_size is used like file_like.read(read_size)
 
-    If file_like is None, you can ``feed()`` bytes. ``feed()`` is
-    useful for unpacking from non-blocking stream.
-
-    exsample 1:
-        unpacker = Unpacker(afile)
-        for o in unpacker:
-            do_something(o)
-
-    example 2:
+    example:
         unpacker = Unpacker()
         while 1:
             buf = astream.read()
@@ -254,13 +226,11 @@ cdef class Unpacker(object):
             for o in unpacker:
                 do_something(o)
     """
-
     cdef template_context ctx
     cdef char* buf
     cdef size_t buf_size, buf_head, buf_tail
     cdef object file_like
     cdef int read_size
-    cdef object waiting_bytes
     cdef bint use_list
     cdef object object_hook
 
@@ -268,8 +238,7 @@ cdef class Unpacker(object):
         self.buf = NULL
 
     def __dealloc__(self):
-        if self.buf:
-            free(self.buf);
+        free(self.buf);
 
     def __init__(self, file_like=None, int read_size=0, bint use_list=0,
                  object object_hook=None, object list_hook=None):
@@ -278,7 +247,6 @@ cdef class Unpacker(object):
         self.use_list = use_list
         self.file_like = file_like
         self.read_size = read_size
-        self.waiting_bytes = []
         self.buf = malloc(read_size)
         self.buf_size = read_size
         self.buf_head = 0
@@ -295,65 +263,49 @@ cdef class Unpacker(object):
                 raise TypeError("object_hook must be a callable.")
             self.ctx.user.list_hook = list_hook
 
-    def feed(self, bytes next_bytes):
-        self.waiting_bytes.append(next_bytes)
+    def feed(self, object next_bytes):
+        cdef char* buf
+        cdef Py_ssize_t buf_len
+        PyObject_AsReadBuffer(next_bytes, &buf, &buf_len)
+        self.append_buffer(buf, buf_len)
 
-    cdef append_buffer(self):
-        cdef char* buf = self.buf
-        cdef Py_ssize_t tail = self.buf_tail
-        cdef Py_ssize_t l
-        cdef bytes b
+    cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
+        cdef:
+            char* buf = self.buf
+            size_t head = self.buf_head
+            size_t tail = self.buf_tail
+            size_t buf_size = self.buf_size
+            size_t new_size
 
-        for b in self.waiting_bytes:
-            l = len(b)
-            memcpy(buf + tail, (b), l)
-            tail += l
-        self.buf_tail = tail
-        del self.waiting_bytes[:]
+        if tail + _buf_len > buf_size:
+            if ((tail - head) + _buf_len)*2 < buf_size:
+                # move to front.
+                memmove(buf, buf + head, tail - head)
+                tail -= head
+                head = 0
+            else:
+                # expand buffer.
+                new_size = tail + _buf_len
+                if new_size < buf_size*2:
+                    new_size = buf_size*2
+                buf = realloc(buf, new_size)
+                buf_size = new_size
 
-    # prepare self.buf
+        memcpy(buf + tail, (_buf), _buf_len)
+        self.buf_head = head
+        self.buf_size = buf_size
+        self.buf_tail = tail + _buf_len
+
+    # prepare self.buf from file_like
     cdef fill_buffer(self):
-        cdef Py_ssize_t add_size
-
         if self.file_like is not None:
             next_bytes = self.file_like.read(self.read_size)
             if next_bytes:
-                self.waiting_bytes.append(next_bytes)
+                self.append_buffer(PyBytes_AsString(next_bytes),
+                                   PyBytes_Size(next_bytes))
             else:
                 self.file_like = None
 
-        if not self.waiting_bytes:
-            return
-
-        add_size = 0
-        for b in self.waiting_bytes:
-            add_size += len(b)
-
-        cdef char* buf = self.buf
-        cdef size_t head = self.buf_head
-        cdef size_t tail = self.buf_tail
-        cdef size_t size = self.buf_size
-
-        if self.buf_tail + add_size <= self.buf_size:
-            # do nothing.
-            pass
-        if self.buf_tail - self.buf_head + add_size < self.buf_size:
-            # move to front.
-            memmove(buf, buf + head, tail - head)
-            tail -= head
-            head = 0
-        else:
-            # expand buffer
-            size = tail + add_size
-            buf = realloc(buf, size)
-
-        self.buf = buf
-        self.buf_head = head
-        self.buf_tail = tail
-        self.buf_size = size
-
-        self.append_buffer()
-
     cpdef unpack(self):
         """unpack one object"""
         cdef int ret
diff --git a/python/test/test_buffer.py b/python/test/test_buffer.py
index 436b554..ce7a72d 100644
--- a/python/test/test_buffer.py
+++ b/python/test/test_buffer.py
@@ -7,10 +7,10 @@ from msgpack import packb, unpackb
 
 def test_unpack_buffer():
     from array import array
-    buf = array('b')
-    buf.fromstring(packb(['foo', 'bar']))
+    buf = array('c')
+    buf.fromstring(packb(('foo', 'bar')))
     obj = unpackb(buf)
-    assert_equal(['foo', 'bar'], obj)
+    assert_equal(('foo', 'bar'), obj)
 
 if __name__ == '__main__':
     main()
diff --git a/python/test/test_sequnpack.py b/python/test/test_sequnpack.py
index 789ccd2..df6e308 100644
--- a/python/test/test_sequnpack.py
+++ b/python/test/test_sequnpack.py
@@ -1,22 +1,22 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from __future__ import unicode_literals, print_function
+from __future__ import unicode_literals
 
 from msgpack import Unpacker
 
 def test_foobar():
     unpacker = Unpacker(read_size=3)
     unpacker.feed(b'foobar')
-    assert unpacker.unpack() == ord('f')
-    assert unpacker.unpack() == ord('o')
-    assert unpacker.unpack() == ord('o')
-    assert unpacker.unpack() == ord('b')
-    assert unpacker.unpack() == ord('a')
-    assert unpacker.unpack() == ord('r')
+    assert unpacker.unpack() == ord(b'f')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'b')
+    assert unpacker.unpack() == ord(b'a')
+    assert unpacker.unpack() == ord(b'r')
     try:
         o = unpacker.unpack()
-        print("Oops!", o)
+        print "Oops!", o
         assert 0
     except StopIteration:
         assert 1

From a09c85ff9c846384cf681fa955ce931010995055 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Nov 2010 03:11:00 +0900
Subject: [PATCH 0939/1648] python: Support old buffer protocol when unpack.
 (experimental)

---
 msgpack/_msgpack.pyx   | 156 ++++++++++++++---------------------------
 test/test_buffer.py    |   6 +-
 test/test_sequnpack.py |  16 ++---
 3 files changed, 65 insertions(+), 113 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index e9d6c7b..5394055 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -1,37 +1,14 @@
 # coding: utf-8
 
+from cpython cimport *
 cdef extern from "Python.h":
     ctypedef char* const_char_ptr "const char*"
+    ctypedef char* const_void_ptr "const void*"
     ctypedef struct PyObject
+    cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
 
-    cdef object PyBytes_FromStringAndSize(const_char_ptr b, Py_ssize_t len)
-    cdef PyObject* Py_True
-    cdef PyObject* Py_False
-    cdef object PyUnicode_AsUTF8String(object)
-
-    cdef long long PyLong_AsLongLong(object o)
-    cdef unsigned long long PyLong_AsUnsignedLongLong(object o)
-
-    cdef bint PyBool_Check(object o)
-    cdef bint PyDict_Check(object o)
-    cdef bint PySequence_Check(object o)
-    cdef bint PyLong_Check(object o)
-    cdef bint PyInt_Check(object o)
-    cdef bint PyFloat_Check(object o)
-    cdef bint PyBytes_Check(object o)
-    cdef bint PyUnicode_Check(object o)
-    cdef bint PyCallable_Check(object o)
-    cdef void Py_INCREF(object o)
-    cdef void Py_DECREF(object o)
-
-cdef extern from "stdlib.h":
-    void* malloc(size_t)
-    void* realloc(void*, size_t)
-    void free(void*)
-
-cdef extern from "string.h":
-    void* memcpy(char* dst, char* src, size_t size)
-    void* memmove(char* dst, char* src, size_t size)
+from libc.stdlib cimport *
+from libc.string cimport *
 
 cdef extern from "pack.h":
     struct msgpack_packer:
@@ -104,10 +81,10 @@ cdef class Packer(object):
                 ret = msgpack_pack_false(&self.pk)
         elif PyLong_Check(o):
             if o > 0:
-                ullval = PyLong_AsUnsignedLongLong(o)
+                ullval = o
                 ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
             else:
-                llval = PyLong_AsLongLong(o)
+                llval = o
                 ret = msgpack_pack_long_long(&self.pk, llval)
         elif PyInt_Check(o):
             longval = o
@@ -160,7 +137,7 @@ cdef class Packer(object):
 
 def pack(object o, object stream, default=None):
     """pack an object `o` and write it to stream)."""
-    packer = Packer(default)
+    packer = Packer(default=default)
     stream.write(packer.pack(o))
 
 def packb(object o, default=None):
@@ -184,17 +161,21 @@ cdef extern from "unpack.h":
         PyObject* key
 
     int template_execute(template_context* ctx, const_char_ptr data,
-            size_t len, size_t* off)
+                         size_t len, size_t* off)
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
 
-def unpackb(bytes packed_bytes, object object_hook=None, object list_hook=None):
+def unpackb(object packed, object object_hook=None, object list_hook=None):
     """Unpack packed_bytes to object. Returns an unpacked object."""
-    cdef const_char_ptr p = packed_bytes
     cdef template_context ctx
     cdef size_t off = 0
     cdef int ret
+
+    cdef char* buf
+    cdef Py_ssize_t buf_len
+    PyObject_AsReadBuffer(packed, &buf, &buf_len)
+
     template_init(&ctx)
     ctx.user.use_list = 0
     ctx.user.object_hook = ctx.user.list_hook = NULL
@@ -206,7 +187,7 @@ def unpackb(bytes packed_bytes, object object_hook=None, object list_hook=None):
         if not PyCallable_Check(list_hook):
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
-    ret = template_execute(&ctx, p, len(packed_bytes), &off)
+    ret = template_execute(&ctx, buf, buf_len, &off)
     if ret == 1:
         return template_data(&ctx)
     else:
@@ -216,8 +197,8 @@ unpacks = unpackb
 
 def unpack(object stream, object object_hook=None, object list_hook=None):
     """unpack an object from stream."""
-    packed = stream.read()
-    return unpackb(packed, object_hook=object_hook, list_hook=list_hook)
+    return unpackb(stream.read(),
+                   object_hook=object_hook, list_hook=list_hook)
 
 cdef class UnpackIterator(object):
     cdef object unpacker
@@ -232,21 +213,12 @@ cdef class UnpackIterator(object):
         return self
 
 cdef class Unpacker(object):
-    """Unpacker(file_like=None, read_size=1024*1024)
+    """Unpacker(read_size=1024*1024)
 
     Streaming unpacker.
-    file_like must have read(n) method.
     read_size is used like file_like.read(read_size)
 
-    If file_like is None, you can ``feed()`` bytes. ``feed()`` is
-    useful for unpacking from non-blocking stream.
-
-    exsample 1:
-        unpacker = Unpacker(afile)
-        for o in unpacker:
-            do_something(o)
-
-    example 2:
+    example:
         unpacker = Unpacker()
         while 1:
             buf = astream.read()
@@ -254,13 +226,11 @@ cdef class Unpacker(object):
             for o in unpacker:
                 do_something(o)
     """
-
     cdef template_context ctx
     cdef char* buf
     cdef size_t buf_size, buf_head, buf_tail
     cdef object file_like
     cdef int read_size
-    cdef object waiting_bytes
     cdef bint use_list
     cdef object object_hook
 
@@ -268,8 +238,7 @@ cdef class Unpacker(object):
         self.buf = NULL
 
     def __dealloc__(self):
-        if self.buf:
-            free(self.buf);
+        free(self.buf);
 
     def __init__(self, file_like=None, int read_size=0, bint use_list=0,
                  object object_hook=None, object list_hook=None):
@@ -278,7 +247,6 @@ cdef class Unpacker(object):
         self.use_list = use_list
         self.file_like = file_like
         self.read_size = read_size
-        self.waiting_bytes = []
         self.buf = malloc(read_size)
         self.buf_size = read_size
         self.buf_head = 0
@@ -295,65 +263,49 @@ cdef class Unpacker(object):
                 raise TypeError("object_hook must be a callable.")
             self.ctx.user.list_hook = list_hook
 
-    def feed(self, bytes next_bytes):
-        self.waiting_bytes.append(next_bytes)
+    def feed(self, object next_bytes):
+        cdef char* buf
+        cdef Py_ssize_t buf_len
+        PyObject_AsReadBuffer(next_bytes, &buf, &buf_len)
+        self.append_buffer(buf, buf_len)
 
-    cdef append_buffer(self):
-        cdef char* buf = self.buf
-        cdef Py_ssize_t tail = self.buf_tail
-        cdef Py_ssize_t l
-        cdef bytes b
+    cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
+        cdef:
+            char* buf = self.buf
+            size_t head = self.buf_head
+            size_t tail = self.buf_tail
+            size_t buf_size = self.buf_size
+            size_t new_size
 
-        for b in self.waiting_bytes:
-            l = len(b)
-            memcpy(buf + tail, (b), l)
-            tail += l
-        self.buf_tail = tail
-        del self.waiting_bytes[:]
+        if tail + _buf_len > buf_size:
+            if ((tail - head) + _buf_len)*2 < buf_size:
+                # move to front.
+                memmove(buf, buf + head, tail - head)
+                tail -= head
+                head = 0
+            else:
+                # expand buffer.
+                new_size = tail + _buf_len
+                if new_size < buf_size*2:
+                    new_size = buf_size*2
+                buf = realloc(buf, new_size)
+                buf_size = new_size
 
-    # prepare self.buf
+        memcpy(buf + tail, (_buf), _buf_len)
+        self.buf_head = head
+        self.buf_size = buf_size
+        self.buf_tail = tail + _buf_len
+
+    # prepare self.buf from file_like
     cdef fill_buffer(self):
-        cdef Py_ssize_t add_size
-
         if self.file_like is not None:
             next_bytes = self.file_like.read(self.read_size)
             if next_bytes:
-                self.waiting_bytes.append(next_bytes)
+                self.append_buffer(PyBytes_AsString(next_bytes),
+                                   PyBytes_Size(next_bytes))
             else:
                 self.file_like = None
 
-        if not self.waiting_bytes:
-            return
-
-        add_size = 0
-        for b in self.waiting_bytes:
-            add_size += len(b)
-
-        cdef char* buf = self.buf
-        cdef size_t head = self.buf_head
-        cdef size_t tail = self.buf_tail
-        cdef size_t size = self.buf_size
-
-        if self.buf_tail + add_size <= self.buf_size:
-            # do nothing.
-            pass
-        if self.buf_tail - self.buf_head + add_size < self.buf_size:
-            # move to front.
-            memmove(buf, buf + head, tail - head)
-            tail -= head
-            head = 0
-        else:
-            # expand buffer
-            size = tail + add_size
-            buf = realloc(buf, size)
-
-        self.buf = buf
-        self.buf_head = head
-        self.buf_tail = tail
-        self.buf_size = size
-
-        self.append_buffer()
-
     cpdef unpack(self):
         """unpack one object"""
         cdef int ret
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 436b554..ce7a72d 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -7,10 +7,10 @@ from msgpack import packb, unpackb
 
 def test_unpack_buffer():
     from array import array
-    buf = array('b')
-    buf.fromstring(packb(['foo', 'bar']))
+    buf = array('c')
+    buf.fromstring(packb(('foo', 'bar')))
     obj = unpackb(buf)
-    assert_equal(['foo', 'bar'], obj)
+    assert_equal(('foo', 'bar'), obj)
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 789ccd2..df6e308 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,22 +1,22 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from __future__ import unicode_literals, print_function
+from __future__ import unicode_literals
 
 from msgpack import Unpacker
 
 def test_foobar():
     unpacker = Unpacker(read_size=3)
     unpacker.feed(b'foobar')
-    assert unpacker.unpack() == ord('f')
-    assert unpacker.unpack() == ord('o')
-    assert unpacker.unpack() == ord('o')
-    assert unpacker.unpack() == ord('b')
-    assert unpacker.unpack() == ord('a')
-    assert unpacker.unpack() == ord('r')
+    assert unpacker.unpack() == ord(b'f')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'b')
+    assert unpacker.unpack() == ord(b'a')
+    assert unpacker.unpack() == ord(b'r')
     try:
         o = unpacker.unpack()
-        print("Oops!", o)
+        print "Oops!", o
         assert 0
     except StopIteration:
         assert 1

From d1b6e65dd8927698ced5a3111baa4e81834850bc Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Nov 2010 03:15:12 +0900
Subject: [PATCH 0940/1648] python: Port some tests from 2 to 3.

---
 python/test3/test_buffer.py | 16 ++++++++++++++++
 python/test3/test_obj.py    | 13 +++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 python/test3/test_buffer.py

diff --git a/python/test3/test_buffer.py b/python/test3/test_buffer.py
new file mode 100644
index 0000000..01310a0
--- /dev/null
+++ b/python/test3/test_buffer.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import packb, unpackb
+
+def test_unpack_buffer():
+    from array import array
+    buf = array('b')
+    buf.fromstring(packb(('foo', 'bar')))
+    obj = unpackb(buf)
+    assert_equal((b'foo', b'bar'), obj)
+
+if __name__ == '__main__':
+    main()
diff --git a/python/test3/test_obj.py b/python/test3/test_obj.py
index 972d1ca..236988d 100644
--- a/python/test3/test_obj.py
+++ b/python/test3/test_obj.py
@@ -26,6 +26,19 @@ def test_decode_hook():
     unpacked = unpacks(packed, object_hook=_decode_complex)
     eq_(unpacked[1], 1+2j)
 
+@raises(TypeError)
+def test_bad_hook():
+    packed = packs([3, 1+2j], default=lambda o: o)
+    unpacked = unpacks(packed)
+
+def _arr_to_str(arr):
+    return ''.join(str(c) for c in arr)
+
+def test_array_hook():
+    packed = packs([1,2,3])
+    unpacked = unpacks(packed, list_hook=_arr_to_str)
+    eq_(unpacked, '123')
+
 if __name__ == '__main__':
     #main()
     test_decode_hook()

From 2c9fddf085fee3fe16f86f979e42ea87cfaaa084 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Nov 2010 03:15:12 +0900
Subject: [PATCH 0941/1648] python: Port some tests from 2 to 3.

---
 test3/test_buffer.py | 16 ++++++++++++++++
 test3/test_obj.py    | 13 +++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 test3/test_buffer.py

diff --git a/test3/test_buffer.py b/test3/test_buffer.py
new file mode 100644
index 0000000..01310a0
--- /dev/null
+++ b/test3/test_buffer.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import packb, unpackb
+
+def test_unpack_buffer():
+    from array import array
+    buf = array('b')
+    buf.fromstring(packb(('foo', 'bar')))
+    obj = unpackb(buf)
+    assert_equal((b'foo', b'bar'), obj)
+
+if __name__ == '__main__':
+    main()
diff --git a/test3/test_obj.py b/test3/test_obj.py
index 972d1ca..236988d 100644
--- a/test3/test_obj.py
+++ b/test3/test_obj.py
@@ -26,6 +26,19 @@ def test_decode_hook():
     unpacked = unpacks(packed, object_hook=_decode_complex)
     eq_(unpacked[1], 1+2j)
 
+@raises(TypeError)
+def test_bad_hook():
+    packed = packs([3, 1+2j], default=lambda o: o)
+    unpacked = unpacks(packed)
+
+def _arr_to_str(arr):
+    return ''.join(str(c) for c in arr)
+
+def test_array_hook():
+    packed = packs([1,2,3])
+    unpacked = unpacks(packed, list_hook=_arr_to_str)
+    eq_(unpacked, '123')
+
 if __name__ == '__main__':
     #main()
     test_decode_hook()

From 3124576e1e177831632b84792b6297e7a1078ea0 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Nov 2010 03:26:33 +0900
Subject: [PATCH 0942/1648] python: Add ChangeLog.rst

---
 ChangeLog.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 ChangeLog.rst

diff --git a/ChangeLog.rst b/ChangeLog.rst
new file mode 100644
index 0000000..022f955
--- /dev/null
+++ b/ChangeLog.rst
@@ -0,0 +1,16 @@
+0.2.0
+======
+:release date: NOT RELEASED YET
+
+New feature
+------------
+* Add *object_hook* and *list_hook* option to unpacker. It allows you to
+  hook unpacing mapping type and array type.
+
+* Add *default* option to packer. It allows you to pack unsupported types.
+
+* unpacker accepts (old) buffer types.
+
+Bugs fixed
+----------
+* Compilation error on win32.

From e4f515166aeb5ea35094a354e5859dd39e5f9b7d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Nov 2010 03:26:33 +0900
Subject: [PATCH 0943/1648] python: Add ChangeLog.rst

---
 python/ChangeLog.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 python/ChangeLog.rst

diff --git a/python/ChangeLog.rst b/python/ChangeLog.rst
new file mode 100644
index 0000000..022f955
--- /dev/null
+++ b/python/ChangeLog.rst
@@ -0,0 +1,16 @@
+0.2.0
+======
+:release date: NOT RELEASED YET
+
+New feature
+------------
+* Add *object_hook* and *list_hook* option to unpacker. It allows you to
+  hook unpacing mapping type and array type.
+
+* Add *default* option to packer. It allows you to pack unsupported types.
+
+* unpacker accepts (old) buffer types.
+
+Bugs fixed
+----------
+* Compilation error on win32.

From 948f4d00c6e50e90fc99434905e8a1ce3f708411 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Nov 2010 03:35:52 +0900
Subject: [PATCH 0944/1648] python: Add ws2_32 library if platform is win32.

---
 setup.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index ac7ece5..aedd990 100755
--- a/setup.py
+++ b/setup.py
@@ -3,6 +3,7 @@
 version = (0, 1, 6, 'final')
 
 import os
+import sys
 from glob import glob
 from distutils.core import setup, Extension
 from distutils.command.sdist import sdist
@@ -42,10 +43,13 @@ else:
 
     Sdist = sdist
 
+libraries = ['ws2_32'] if sys.platform == 'win32' else []
+
 msgpack_mod = Extension('msgpack._msgpack',
                         sources=sources,
+                        libraries=libraries,
                         )
-del sources
+del sources, libraries
 
 
 desc = 'MessagePack (de)serializer.'
@@ -69,6 +73,7 @@ setup(name='msgpack-python',
       url='http://msgpack.sourceforge.net/',
       download_url='http://pypi.python.org/pypi/msgpack/',
       classifiers=[
+          'Programming Language :: Python :: 2',
           'Programming Language :: Python :: 3',
           'Development Status :: 4 - Beta',
           'Intended Audience :: Developers',

From bb69aa18f2036f8f2b908e219de2dc5287a19c69 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Nov 2010 03:35:52 +0900
Subject: [PATCH 0945/1648] python: Add ws2_32 library if platform is win32.

---
 python/setup.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index ac7ece5..aedd990 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -3,6 +3,7 @@
 version = (0, 1, 6, 'final')
 
 import os
+import sys
 from glob import glob
 from distutils.core import setup, Extension
 from distutils.command.sdist import sdist
@@ -42,10 +43,13 @@ else:
 
     Sdist = sdist
 
+libraries = ['ws2_32'] if sys.platform == 'win32' else []
+
 msgpack_mod = Extension('msgpack._msgpack',
                         sources=sources,
+                        libraries=libraries,
                         )
-del sources
+del sources, libraries
 
 
 desc = 'MessagePack (de)serializer.'
@@ -69,6 +73,7 @@ setup(name='msgpack-python',
       url='http://msgpack.sourceforge.net/',
       download_url='http://pypi.python.org/pypi/msgpack/',
       classifiers=[
+          'Programming Language :: Python :: 2',
           'Programming Language :: Python :: 3',
           'Development Status :: 4 - Beta',
           'Intended Audience :: Developers',

From 9d04822f35fde2eb46136b57bf73a23a6d3954f6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Nov 2010 03:43:49 +0900
Subject: [PATCH 0946/1648] python: 0.1.7

---
 ChangeLog.rst | 4 ++--
 setup.py      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 022f955..77e1707 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,6 @@
-0.2.0
+0.1.7
 ======
-:release date: NOT RELEASED YET
+:release date: 2010-11-02
 
 New feature
 ------------
diff --git a/setup.py b/setup.py
index aedd990..d21e6b5 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 6, 'final')
+version = (0, 1, 7, 'final')
 
 import os
 import sys

From d7fc215c1676da5d36f940a8a3fddca29ef61d78 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Nov 2010 03:43:49 +0900
Subject: [PATCH 0947/1648] python: 0.1.7

---
 python/ChangeLog.rst | 4 ++--
 python/setup.py      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/ChangeLog.rst b/python/ChangeLog.rst
index 022f955..77e1707 100644
--- a/python/ChangeLog.rst
+++ b/python/ChangeLog.rst
@@ -1,6 +1,6 @@
-0.2.0
+0.1.7
 ======
-:release date: NOT RELEASED YET
+:release date: 2010-11-02
 
 New feature
 ------------
diff --git a/python/setup.py b/python/setup.py
index aedd990..d21e6b5 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 6, 'final')
+version = (0, 1, 7, 'final')
 
 import os
 import sys

From 67b80f8b8a252e42119b3e9dce151a3354a58872 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 9 Jan 2011 23:06:56 +0900
Subject: [PATCH 0948/1648] python: Update ChangeLog.

---
 ChangeLog.rst | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 77e1707..19cc6b6 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,23 @@
+0.1.8
+======
+:release date: NOT YET RELEASED
+
+New feature
+------------
+* Support ``loads`` and ``dumps`` aliases for API compatibility with
+  simplejson and pickle.
+
+* Add *object_hook* and *list_hook* option to unpacker. It allows you to
+  hook unpacing mapping type and array type.
+
+* Add *default* option to packer. It allows you to pack unsupported types.
+
+* unpacker accepts (old) buffer types.
+
+Bugs fixed
+----------
+
+
 0.1.7
 ======
 :release date: 2010-11-02

From 9c61ce2c6cb8f87d6d928ef728a882bf141ccaf2 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 9 Jan 2011 23:13:35 +0900
Subject: [PATCH 0949/1648] python: Don't use ``from __future__ import
 unicode_literals``. Python 2.5 or older doesn't support it.

---
 test/test_sequnpack.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index df6e308..c92658c 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,13 +1,11 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from __future__ import unicode_literals
-
 from msgpack import Unpacker
 
 def test_foobar():
     unpacker = Unpacker(read_size=3)
-    unpacker.feed(b'foobar')
+    unpacker.feed('foobar')
     assert unpacker.unpack() == ord(b'f')
     assert unpacker.unpack() == ord(b'o')
     assert unpacker.unpack() == ord(b'o')

From 569729c3c218cdbb87b486524ab1e71c1f8bba61 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 9 Jan 2011 23:17:20 +0900
Subject: [PATCH 0950/1648] python: Make aliases for API compatibility to
 pickle. ``dumps`` is alias of ``packb`` and ``loads`` is alias of
 ``unpacks``.

---
 msgpack/_msgpack.pyx | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 5394055..3316a22 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -73,7 +73,6 @@ cdef class Packer(object):
 
         if o is None:
             ret = msgpack_pack_nil(&self.pk)
-            #elif PyBool_Check(o):
         elif isinstance(o, bool):
             if o:
                 ret = msgpack_pack_true(&self.pk)
@@ -145,7 +144,7 @@ def packb(object o, default=None):
     packer = Packer(default=default)
     return packer.pack(o)
 
-packs = packb
+dumps = packs = packb
 
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
@@ -193,7 +192,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None):
     else:
         return None
 
-unpacks = unpackb
+loads = unpacks = unpackb
 
 def unpack(object stream, object object_hook=None, object list_hook=None):
     """unpack an object from stream."""

From 5dbf2f5ef73c574d6d2d51b32abfa63b3b124ae1 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 9 Jan 2011 23:29:18 +0900
Subject: [PATCH 0951/1648] python: Add test for issue29.

---
 test/test_seq.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 test/test_seq.py

diff --git a/test/test_seq.py b/test/test_seq.py
new file mode 100644
index 0000000..993a59e
--- /dev/null
+++ b/test/test_seq.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+
+import StringIO
+import msgpack
+
+binarydata = [chr(i) for i in xrange(256)]
+binarydata = "".join(binarydata)
+
+def gen_binary_data(idx):
+    data = binarydata[:idx % 300]
+    return data
+
+def test_exceeding_unpacker_read_size():
+    dumpf = StringIO.StringIO()
+
+    packer = msgpack.Packer()
+
+    NUMBER_OF_STRINGS = 6
+    read_size = 16
+                                # 5 ok for read_size=16, while 6 glibc detected *** python: double free or corruption (fasttop):
+                                # 20 ok for read_size=256, while 25 segfaults / glibc detected *** python: double free or corruption (!prev)
+                                # 40 ok for read_size=1024, while 50 introduces errors
+                                # 7000 ok for read_size=1024*1024, while 8000 leads to  glibc detected *** python: double free or corruption (!prev):
+
+    for idx in xrange(NUMBER_OF_STRINGS):
+        data = gen_binary_data(idx)
+        dumpf.write(packer.pack(data))
+
+    f = StringIO.StringIO(dumpf.getvalue())
+    dumpf.close()
+
+    unpacker = msgpack.Unpacker(f, read_size=read_size)
+
+    read_count = 0
+    for idx, o in enumerate(unpacker):
+        assert_equal(type(o), str)
+        assert_equal(o, gen_binary_data(idx))
+        read_count += 1
+
+    assert_equal(read_count, NUMBER_OF_STRINGS)
+
+
+if __name__ == '__main__':
+    # main()
+    test_exceeding_unpacker_read_size()

From ed3f3899c1e22fc5dc61e88af3feee9029df6ec4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 9 Jan 2011 23:40:09 +0900
Subject: [PATCH 0952/1648] python: Fix segv on unpacking from stream.

---
 msgpack/_msgpack.pyx | 39 ++++++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 15 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 3316a22..5afa961 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -229,7 +229,8 @@ cdef class Unpacker(object):
     cdef char* buf
     cdef size_t buf_size, buf_head, buf_tail
     cdef object file_like
-    cdef int read_size
+    cdef object file_like_read
+    cdef Py_ssize_t read_size
     cdef bint use_list
     cdef object object_hook
 
@@ -239,12 +240,16 @@ cdef class Unpacker(object):
     def __dealloc__(self):
         free(self.buf);
 
-    def __init__(self, file_like=None, int read_size=0, bint use_list=0,
+    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
                  object object_hook=None, object list_hook=None):
         if read_size == 0:
             read_size = 1024*1024
         self.use_list = use_list
         self.file_like = file_like
+        if file_like:
+            self.file_like_read = file_like.read
+            if not PyCallable_Check(self.file_like_read):
+                raise ValueError("`file_like.read` must be a callable.")
         self.read_size = read_size
         self.buf = malloc(read_size)
         self.buf_size = read_size
@@ -265,6 +270,9 @@ cdef class Unpacker(object):
     def feed(self, object next_bytes):
         cdef char* buf
         cdef Py_ssize_t buf_len
+        if self.file_like is not None:
+            raise AssertionError(
+                    "unpacker.feed() is not be able to use with`file_like`.")
         PyObject_AsReadBuffer(next_bytes, &buf, &buf_len)
         self.append_buffer(buf, buf_len)
 
@@ -298,7 +306,7 @@ cdef class Unpacker(object):
     # prepare self.buf from file_like
     cdef fill_buffer(self):
         if self.file_like is not None:
-            next_bytes = self.file_like.read(self.read_size)
+            next_bytes = self.file_like_read(self.read_size)
             if next_bytes:
                 self.append_buffer(PyBytes_AsString(next_bytes),
                                    PyBytes_Size(next_bytes))
@@ -308,18 +316,19 @@ cdef class Unpacker(object):
     cpdef unpack(self):
         """unpack one object"""
         cdef int ret
-        self.fill_buffer()
-        ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
-        if ret == 1:
-            o = template_data(&self.ctx)
-            template_init(&self.ctx)
-            return o
-        elif ret == 0:
-            if self.file_like is not None:
-                return self.unpack()
-            raise StopIteration, "No more unpack data."
-        else:
-            raise ValueError, "Unpack failed."
+        while 1:
+            ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+            if ret == 1:
+                o = template_data(&self.ctx)
+                template_init(&self.ctx)
+                return o
+            elif ret == 0:
+                if self.file_like is not None:
+                    self.fill_buffer()
+                    continue
+                raise StopIteration("No more unpack data.")
+            else:
+                raise ValueError("Unpack failed: error = %d" % (ret,))
 
     def __iter__(self):
         return UnpackIterator(self)

From 74e7ebea4f5f93dda9b266eae8019635b6142eef Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 9 Jan 2011 23:54:06 +0900
Subject: [PATCH 0953/1648] python: Fix another segv.

---
 msgpack/_msgpack.pyx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 5afa961..0dd926f 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -299,6 +299,7 @@ cdef class Unpacker(object):
                 buf_size = new_size
 
         memcpy(buf + tail, (_buf), _buf_len)
+        self.buf = buf
         self.buf_head = head
         self.buf_size = buf_size
         self.buf_tail = tail + _buf_len

From 013d1da68f7ddc27f9a748276f3edb57b1173e02 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 9 Jan 2011 23:58:26 +0900
Subject: [PATCH 0954/1648] python: msgpack-python-0.1.8

---
 ChangeLog.rst | 3 ++-
 setup.py      | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 19cc6b6..4c70ad5 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,6 @@
 0.1.8
 ======
-:release date: NOT YET RELEASED
+:release date: 2011-01-10
 
 New feature
 ------------
@@ -16,6 +16,7 @@ New feature
 
 Bugs fixed
 ----------
+* Fix segv around ``Unpacker.feed`` or ``Unpacker(file)``.
 
 
 0.1.7
diff --git a/setup.py b/setup.py
index d21e6b5..216d255 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 7, 'final')
+version = (0, 1, 8, 'final')
 
 import os
 import sys
@@ -20,6 +20,8 @@ except ImportError:
 f = open('msgpack/__version__.py', 'w')
 f.write("version = %r\n" % (version,))
 f.close()
+del f
+
 version_str = '.'.join(str(x) for x in version[:3])
 if len(version) > 3 and version[3] != 'final':
     version_str += version[3]

From 8e38a074bbc5d3546c0eb049b838aa3afa3c5f3a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Jan 2011 00:10:29 +0900
Subject: [PATCH 0955/1648] python: Change URL in setup.py from sf.net to
 msgpack.org

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 216d255..ddace35 100755
--- a/setup.py
+++ b/setup.py
@@ -57,7 +57,7 @@ del sources, libraries
 desc = 'MessagePack (de)serializer.'
 long_desc = """MessagePack (de)serializer for Python.
 
-What's MessagePack? (from http://msgpack.sourceforge.net/)
+What's MessagePack? (from http://msgpack.org/)
 
  MessagePack is a binary-based efficient data interchange format that is
  focused on high performance. It is like JSON, but very fast and small.
@@ -72,7 +72,7 @@ setup(name='msgpack-python',
       packages=['msgpack'],
       description=desc,
       long_description=long_desc,
-      url='http://msgpack.sourceforge.net/',
+      url='http://msgpack.org/',
       download_url='http://pypi.python.org/pypi/msgpack/',
       classifiers=[
           'Programming Language :: Python :: 2',

From 56aaed36416ebe8f7df670d21b04219e67e6e2c9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Jan 2011 05:05:14 +0900
Subject: [PATCH 0956/1648] python: Fix typo in docstring. (thanks to Mateusz.)

---
 msgpack/_msgpack.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 0dd926f..c7eec81 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -264,7 +264,7 @@ cdef class Unpacker(object):
             self.ctx.user.object_hook = object_hook
         if list_hook is not None:
             if not PyCallable_Check(list_hook):
-                raise TypeError("object_hook must be a callable.")
+                raise TypeError("list_hook must be a callable.")
             self.ctx.user.list_hook = list_hook
 
     def feed(self, object next_bytes):

From 47b0c273bb70d34b253c9e908a3f2d55e99b8cef Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Jan 2011 05:07:07 +0900
Subject: [PATCH 0957/1648] python: Check if (m|re)alloc's return value is
 NULL. (Thanks to Mateusz)

---
 msgpack/_msgpack.pyx | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index c7eec81..e48f8b2 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -239,6 +239,7 @@ cdef class Unpacker(object):
 
     def __dealloc__(self):
         free(self.buf);
+        self.buf = NULL;
 
     def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
                  object object_hook=None, object list_hook=None):
@@ -252,6 +253,8 @@ cdef class Unpacker(object):
                 raise ValueError("`file_like.read` must be a callable.")
         self.read_size = read_size
         self.buf = malloc(read_size)
+        if self.buf == NULL:
+            raise MemoryError("Unable to allocate internal buffer.")
         self.buf_size = read_size
         self.buf_head = 0
         self.buf_tail = 0
@@ -295,7 +298,9 @@ cdef class Unpacker(object):
                 new_size = tail + _buf_len
                 if new_size < buf_size*2:
                     new_size = buf_size*2
-                buf = realloc(buf, new_size)
+                buf = realloc(buf, new_size) 
+                if buf == NULL:
+                    raise MemoryError("Unable to enlarge internal buffer.")   # self.buf still holds old buffer and will be freed during obj destruction
                 buf_size = new_size
 
         memcpy(buf + tail, (_buf), _buf_len)

From b453385d92e549204e7e1639f286589a38143061 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Jan 2011 20:47:23 +0900
Subject: [PATCH 0958/1648] python: Add memory error check.

---
 msgpack/_msgpack.pyx | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index e48f8b2..9f817dd 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -46,6 +46,8 @@ cdef class Packer(object):
     def __cinit__(self):
         cdef int buf_size = 1024*1024
         self.pk.buf =  malloc(buf_size);
+        if self.pk.buf == NULL:
+            raise MemoryError("Unable to allocate internal buffer.")
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
@@ -300,7 +302,9 @@ cdef class Unpacker(object):
                     new_size = buf_size*2
                 buf = realloc(buf, new_size) 
                 if buf == NULL:
-                    raise MemoryError("Unable to enlarge internal buffer.")   # self.buf still holds old buffer and will be freed during obj destruction
+                    # self.buf still holds old buffer and will be freed during
+                    # obj destruction
+                    raise MemoryError("Unable to enlarge internal buffer.")
                 buf_size = new_size
 
         memcpy(buf + tail, (_buf), _buf_len)

From 77a97b9c16bfabe1f8816c348d65e1d8277780d6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 28 Jan 2011 18:59:05 +0900
Subject: [PATCH 0959/1648] Add use_list option to unpack and unpackb

---
 msgpack/_msgpack.pyx | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 9f817dd..9564171 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -167,7 +167,7 @@ cdef extern from "unpack.h":
     object template_data(template_context* ctx)
 
 
-def unpackb(object packed, object object_hook=None, object list_hook=None):
+def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0):
     """Unpack packed_bytes to object. Returns an unpacked object."""
     cdef template_context ctx
     cdef size_t off = 0
@@ -178,7 +178,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None):
     PyObject_AsReadBuffer(packed, &buf, &buf_len)
 
     template_init(&ctx)
-    ctx.user.use_list = 0
+    ctx.user.use_list = use_list
     ctx.user.object_hook = ctx.user.list_hook = NULL
     if object_hook is not None:
         if not PyCallable_Check(object_hook):
@@ -196,9 +196,9 @@ def unpackb(object packed, object object_hook=None, object list_hook=None):
 
 loads = unpacks = unpackb
 
-def unpack(object stream, object object_hook=None, object list_hook=None):
+def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0):
     """unpack an object from stream."""
-    return unpackb(stream.read(),
+    return unpackb(stream.read(), use_list=use_list,
                    object_hook=object_hook, list_hook=list_hook)
 
 cdef class UnpackIterator(object):

From d0de96cacd3bd5912994c7b671a5fbf6cb303d31 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 29 Jan 2011 07:27:10 +0900
Subject: [PATCH 0960/1648] python: refactoring.

---
 msgpack/_msgpack.pyx | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 9564171..4df0345 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -41,7 +41,7 @@ cdef class Packer(object):
         astream.write(packer.pack(b))
     """
     cdef msgpack_packer pk
-    cdef object default
+    cdef object _default
 
     def __cinit__(self):
         cdef int buf_size = 1024*1024
@@ -55,13 +55,12 @@ cdef class Packer(object):
         if default is not None:
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")
-        self.default = default
+        self._default = default
 
     def __dealloc__(self):
         free(self.pk.buf);
 
-    cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT,
-                   default=None) except -1:
+    cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1:
         cdef long long llval
         cdef unsigned long long ullval
         cdef long longval
@@ -109,18 +108,18 @@ cdef class Packer(object):
             ret = msgpack_pack_map(&self.pk, len(d))
             if ret == 0:
                 for k,v in d.items():
-                    ret = self._pack(k, nest_limit-1, default)
+                    ret = self._pack(k, nest_limit-1)
                     if ret != 0: break
-                    ret = self._pack(v, nest_limit-1, default)
+                    ret = self._pack(v, nest_limit-1)
                     if ret != 0: break
         elif PySequence_Check(o):
             ret = msgpack_pack_array(&self.pk, len(o))
             if ret == 0:
                 for v in o:
-                    ret = self._pack(v, nest_limit-1, default)
+                    ret = self._pack(v, nest_limit-1)
                     if ret != 0: break
-        elif default is not None:
-            o = self.default(o)
+        elif self._default is not None:
+            o = self._default(o)
             ret = self._pack(o, nest_limit)
         else:
             raise TypeError("can't serialize %r" % (o,))
@@ -128,7 +127,7 @@ cdef class Packer(object):
 
     def pack(self, object obj):
         cdef int ret
-        ret = self._pack(obj, DEFAULT_RECURSE_LIMIT, self.default)
+        ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
         if ret:
             raise TypeError
         buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)

From 9e8261cbd94ca146bdf2fa72c73a19468f96ca74 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 29 Jan 2011 07:31:06 +0900
Subject: [PATCH 0961/1648] python: 0.1.9

---
 ChangeLog.rst | 14 ++++++++++++++
 setup.py      |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 4c70ad5..75e86b2 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,17 @@
+0.1.9
+======
+:release date: 2011-01-29
+
+New feature
+-----------
+* ``use_list`` option is added to unpack(b) like Unpacker.
+  (Use keyword argument because order of parameters are different)
+
+Bugs fixed
+----------
+* Fix typo.
+* Add MemoryError check.
+
 0.1.8
 ======
 :release date: 2011-01-10
diff --git a/setup.py b/setup.py
index ddace35..16987a4 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 8, 'final')
+version = (0, 1, 9, 'final')
 
 import os
 import sys

From 3aaf5f5a7afb4d51e6384f87a3f70a4235b9afb5 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 29 Jan 2011 23:22:41 +0900
Subject: [PATCH 0962/1648] python: Fix segmentation fault when `default`
 returns it's argument.

---
 msgpack/_msgpack.pyx | 4 ++--
 test/test_obj.py     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 4df0345..9a6c232 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -118,9 +118,9 @@ cdef class Packer(object):
                 for v in o:
                     ret = self._pack(v, nest_limit-1)
                     if ret != 0: break
-        elif self._default is not None:
+        elif self._default:
             o = self._default(o)
-            ret = self._pack(o, nest_limit)
+            ret = self._pack(o, nest_limit-1)
         else:
             raise TypeError("can't serialize %r" % (o,))
         return ret
diff --git a/test/test_obj.py b/test/test_obj.py
index bc85736..0eebe7b 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -26,7 +26,7 @@ def test_decode_hook():
     unpacked = unpacks(packed, object_hook=_decode_complex)
     eq_(unpacked[1], 1+2j)
 
-@raises(TypeError)
+@raises(ValueError)
 def test_bad_hook():
     packed = packs([3, 1+2j], default=lambda o: o)
     unpacked = unpacks(packed)

From 60d3ce3a180789ef5f57c22dd579c383ea7eed91 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 29 Jan 2011 23:23:56 +0900
Subject: [PATCH 0963/1648] python: Disable gc while deserializing.

---
 msgpack/_msgpack.pyx | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 9a6c232..e367444 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -9,6 +9,9 @@ cdef extern from "Python.h":
 
 from libc.stdlib cimport *
 from libc.string cimport *
+import gc
+_gc_disable = gc.disable
+_gc_enable = gc.enable
 
 cdef extern from "pack.h":
     struct msgpack_packer:
@@ -187,7 +190,9 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint
         if not PyCallable_Check(list_hook):
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
+    _gc_disable()
     ret = template_execute(&ctx, buf, buf_len, &off)
+    _gc_enable()
     if ret == 1:
         return template_data(&ctx)
     else:
@@ -326,7 +331,9 @@ cdef class Unpacker(object):
         """unpack one object"""
         cdef int ret
         while 1:
+            _gc_disable()
             ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+            _gc_enable()
             if ret == 1:
                 o = template_data(&self.ctx)
                 template_init(&self.ctx)

From af7113bb31877f337eef3d43048c0a9f1cb74258 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 30 Jan 2011 10:45:39 +0900
Subject: [PATCH 0964/1648] python: Remove UnpackIterator. Unpacker is iterator
 of itself.

---
 msgpack/_msgpack.pyx | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index e367444..cdcd0c8 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -205,18 +205,6 @@ def unpack(object stream, object object_hook=None, object list_hook=None, bint u
     return unpackb(stream.read(), use_list=use_list,
                    object_hook=object_hook, list_hook=list_hook)
 
-cdef class UnpackIterator(object):
-    cdef object unpacker
-
-    def __init__(self, unpacker):
-        self.unpacker = unpacker
-
-    def __next__(self):
-        return self.unpacker.unpack()
-
-    def __iter__(self):
-        return self
-
 cdef class Unpacker(object):
     """Unpacker(read_size=1024*1024)
 
@@ -347,7 +335,10 @@ cdef class Unpacker(object):
                 raise ValueError("Unpack failed: error = %d" % (ret,))
 
     def __iter__(self):
-        return UnpackIterator(self)
+        return self
+
+    def __next__(self):
+        return self.unpack()
 
     # for debug.
     #def _buf(self):

From 752e3d1b783fc1c12a28e05a93aa73ac7c6b751c Mon Sep 17 00:00:00 2001
From: tailhook 
Date: Fri, 15 Apr 2011 17:36:17 +0300
Subject: [PATCH 0965/1648] Implemented encoding for strings

* Packer by default uses `utf-8` encoding by default
* Unpacker uses `None` by default, so no decoding is done
* Both pack and unpack has `encoding` and `unicode_errors` arguments,
  if `encoding` is `None` no encoding/decoding is done, otherwise
  it is python codec. `unicode_errors` is supplied as `errors`
  parameter to codec
---
 msgpack/_msgpack.pyx | 72 ++++++++++++++++++++++++++++++++++----------
 msgpack/unpack.h     |  8 ++++-
 test/test_pack.py    | 57 ++++++++++++++++++++++++++++++++---
 test3/test_obj.py    |  2 +-
 test3/test_pack.py   | 55 +++++++++++++++++++++++++++++++--
 5 files changed, 169 insertions(+), 25 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index cdcd0c8..443cbd7 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -36,7 +36,7 @@ cdef int DEFAULT_RECURSE_LIMIT=511
 
 cdef class Packer(object):
     """MessagePack Packer
-    
+
     usage:
 
         packer = Packer()
@@ -45,6 +45,10 @@ cdef class Packer(object):
     """
     cdef msgpack_packer pk
     cdef object _default
+    cdef object _bencoding
+    cdef object _berrors
+    cdef char *encoding
+    cdef char *unicode_errors
 
     def __cinit__(self):
         cdef int buf_size = 1024*1024
@@ -54,11 +58,25 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
-    def __init__(self, default=None):
+    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict'):
         if default is not None:
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")
         self._default = default
+        if encoding is None:
+            self.encoding = NULL
+            self.unicode_errors = NULL
+        else:
+            if isinstance(encoding, unicode):
+                self._bencoding = encoding.encode('ascii')
+            else:
+                self._bencoding = encoding
+            self.encoding = PyBytes_AsString(self._bencoding)
+            if isinstance(unicode_errors, unicode):
+                self._berrors = unicode_errors.encode('ascii')
+            else:
+                self._berrors = unicode_errors
+            self.unicode_errors = PyBytes_AsString(self._berrors)
 
     def __dealloc__(self):
         free(self.pk.buf);
@@ -68,7 +86,7 @@ cdef class Packer(object):
         cdef unsigned long long ullval
         cdef long longval
         cdef double fval
-        cdef char* rawval 
+        cdef char* rawval
         cdef int ret
         cdef dict d
 
@@ -101,7 +119,9 @@ cdef class Packer(object):
             if ret == 0:
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
         elif PyUnicode_Check(o):
-            o = PyUnicode_AsUTF8String(o)
+            if not self.encoding:
+                raise TypeError("Can't encode utf-8 no encoding is specified")
+            o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
             rawval = o
             ret = msgpack_pack_raw(&self.pk, len(o))
             if ret == 0:
@@ -138,14 +158,14 @@ cdef class Packer(object):
         return buf
 
 
-def pack(object o, object stream, default=None):
+def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'):
     """pack an object `o` and write it to stream)."""
-    packer = Packer(default=default)
+    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
     stream.write(packer.pack(o))
 
-def packb(object o, default=None):
+def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'):
     """pack o and return packed bytes."""
-    packer = Packer(default=default)
+    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
     return packer.pack(o)
 
 dumps = packs = packb
@@ -155,6 +175,8 @@ cdef extern from "unpack.h":
         int use_list
         PyObject* object_hook
         PyObject* list_hook
+        char *encoding
+        char *unicode_errors
 
     ctypedef struct template_context:
         msgpack_user user
@@ -164,12 +186,12 @@ cdef extern from "unpack.h":
         PyObject* key
 
     int template_execute(template_context* ctx, const_char_ptr data,
-                         size_t len, size_t* off)
+                         size_t len, size_t* off) except -1
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
 
-def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0):
+def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"):
     """Unpack packed_bytes to object. Returns an unpacked object."""
     cdef template_context ctx
     cdef size_t off = 0
@@ -179,9 +201,25 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint
     cdef Py_ssize_t buf_len
     PyObject_AsReadBuffer(packed, &buf, &buf_len)
 
+    if encoding is None:
+        enc = NULL
+    else:
+        if isinstance(encoding, unicode):
+            bencoding = encoding.encode('ascii')
+        else:
+            bencoding = encoding
+        if isinstance(unicode_errors, unicode):
+            berrors = unicode_errors.encode('ascii')
+        else:
+            berrors = unicode_errors
+        enc = PyBytes_AsString(bencoding)
+        err = PyBytes_AsString(berrors)
+
     template_init(&ctx)
     ctx.user.use_list = use_list
     ctx.user.object_hook = ctx.user.list_hook = NULL
+    ctx.user.encoding = enc
+    ctx.user.unicode_errors = err
     if object_hook is not None:
         if not PyCallable_Check(object_hook):
             raise TypeError("object_hook must be a callable.")
@@ -191,8 +229,10 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
     _gc_disable()
-    ret = template_execute(&ctx, buf, buf_len, &off)
-    _gc_enable()
+    try:
+        ret = template_execute(&ctx, buf, buf_len, &off)
+    finally:
+        _gc_enable()
     if ret == 1:
         return template_data(&ctx)
     else:
@@ -200,10 +240,10 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint
 
 loads = unpacks = unpackb
 
-def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0):
+def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"):
     """unpack an object from stream."""
     return unpackb(stream.read(), use_list=use_list,
-                   object_hook=object_hook, list_hook=list_hook)
+                   object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors)
 
 cdef class Unpacker(object):
     """Unpacker(read_size=1024*1024)
@@ -236,7 +276,7 @@ cdef class Unpacker(object):
         self.buf = NULL;
 
     def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
-                 object object_hook=None, object list_hook=None):
+                 object object_hook=None, object list_hook=None, encoding=None, unicode_errors=None):
         if read_size == 0:
             read_size = 1024*1024
         self.use_list = use_list
@@ -292,7 +332,7 @@ cdef class Unpacker(object):
                 new_size = tail + _buf_len
                 if new_size < buf_size*2:
                     new_size = buf_size*2
-                buf = realloc(buf, new_size) 
+                buf = realloc(buf, new_size)
                 if buf == NULL:
                     # self.buf still holds old buffer and will be freed during
                     # obj destruction
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 453ec2b..0586ca8 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -23,6 +23,8 @@ typedef struct unpack_user {
     int use_list;
     PyObject *object_hook;
     PyObject *list_hook;
+    const char *encoding;
+    const char *unicode_errors;
 } unpack_user;
 
 
@@ -197,7 +199,11 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec
 static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
 {
     PyObject *py;
-    py = PyBytes_FromStringAndSize(p, l);
+    if(u->encoding) {
+        py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors);
+    } else {
+        py = PyBytes_FromStringAndSize(p, l);
+    }
     if (!py)
         return -1;
     *o = py;
diff --git a/test/test_pack.py b/test/test_pack.py
index 5dec068..2aef588 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -15,14 +15,63 @@ def testPack():
             0, 1, 127, 128, 255, 256, 65535, 65536,
             -1, -32, -33, -128, -129, -32768, -32769,
             1.0,
-        "", "a", "a"*31, "a"*32,
+        b"", b"a", b"a"*31, b"a"*32,
         None, True, False,
-        (), ((),), ((), None,), 
-        {None: 0}, 
-        (1<<23), 
+        (), ((),), ((), None,),
+        {None: 0},
+        (1<<23),
         ]
     for td in test_data:
         check(td)
 
+def testPackUnicode():
+    test_data = [
+        u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚",
+        ]
+    for td in test_data:
+        re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8')
+        assert_equal(re, td)
+
+def testPackUTF32():
+    test_data = [
+        u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚",
+        ]
+    for td in test_data:
+        print(packs(td, encoding='utf-32'))
+        re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32')
+        assert_equal(re, td)
+
+def testPackBytes():
+    test_data = [
+        b"", b"abcd", (b"defgh",),
+        ]
+    for td in test_data:
+        check(td)
+
+def testIgnoreUnicodeErrors():
+    re = unpacks(packs(b'abc\xeddef'),
+        encoding='utf-8', unicode_errors='ignore')
+    assert_equal(re, "abcdef")
+
+@raises(UnicodeDecodeError)
+def testStrictUnicodeUnpack():
+    unpacks(packs(b'abc\xeddef'), encoding='utf-8')
+
+@raises(UnicodeEncodeError)
+def testStrictUnicodePack():
+    packs(u"abc\xeddef", encoding='ascii', unicode_errors='strict')
+
+def testIgnoreErrorsPack():
+    re = unpacks(packs(u"abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
+    assert_equal(re, u"abcdef")
+
+@raises(TypeError)
+def testNoEncoding():
+    packs(u"abc", encoding=None)
+
+def testDecodeBinary():
+    re = unpacks(packs(u"abc"), encoding=None)
+    assert_equal(re, b"abc")
+
 if __name__ == '__main__':
     main()
diff --git a/test3/test_obj.py b/test3/test_obj.py
index 236988d..b54021f 100644
--- a/test3/test_obj.py
+++ b/test3/test_obj.py
@@ -26,7 +26,7 @@ def test_decode_hook():
     unpacked = unpacks(packed, object_hook=_decode_complex)
     eq_(unpacked[1], 1+2j)
 
-@raises(TypeError)
+@raises(ValueError)
 def test_bad_hook():
     packed = packs([3, 1+2j], default=lambda o: o)
     unpacked = unpacks(packed)
diff --git a/test3/test_pack.py b/test3/test_pack.py
index c861704..e53f7e6 100644
--- a/test3/test_pack.py
+++ b/test3/test_pack.py
@@ -17,12 +17,61 @@ def testPack():
             1.0,
         b"", b"a", b"a"*31, b"a"*32,
         None, True, False,
-        (), ((),), ((), None,), 
-        {None: 0}, 
-        (1<<23), 
+        (), ((),), ((), None,),
+        {None: 0},
+        (1<<23),
         ]
     for td in test_data:
         check(td)
 
+def testPackUnicode():
+    test_data = [
+        "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚",
+        ]
+    for td in test_data:
+        re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8')
+        assert_equal(re, td)
+
+def testPackUTF32():
+    test_data = [
+        "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚",
+        ]
+    for td in test_data:
+        print(packs(td, encoding='utf-32'))
+        re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32')
+        assert_equal(re, td)
+
+def testPackBytes():
+    test_data = [
+        b"", b"abcd", (b"defgh",),
+        ]
+    for td in test_data:
+        check(td)
+
+def testIgnoreUnicodeErrors():
+    re = unpacks(packs(b'abc\xeddef'),
+        encoding='utf-8', unicode_errors='ignore')
+    assert_equal(re, "abcdef")
+
+@raises(UnicodeDecodeError)
+def testStrictUnicodeUnpack():
+    unpacks(packs(b'abc\xeddef'), encoding='utf-8')
+
+@raises(UnicodeEncodeError)
+def testStrictUnicodePack():
+    packs("abc\xeddef", encoding='ascii', unicode_errors='strict')
+
+def testIgnoreErrorsPack():
+    re = unpacks(packs("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
+    assert_equal(re, "abcdef")
+
+@raises(TypeError)
+def testNoEncoding():
+    packs("abc", encoding=None)
+
+def testDecodeBinary():
+    re = unpacks(packs("abc"), encoding=None)
+    assert_equal(re, b"abc")
+
 if __name__ == '__main__':
     main()

From 36b0c2de411701699faadf6d7f355b95044e581c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 31 May 2011 14:10:46 +0900
Subject: [PATCH 0966/1648] (python) Change error message for unicode is passed
 but no encoding is specified.

---
 msgpack/_msgpack.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 443cbd7..14bc9d7 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -120,7 +120,7 @@ cdef class Packer(object):
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
         elif PyUnicode_Check(o):
             if not self.encoding:
-                raise TypeError("Can't encode utf-8 no encoding is specified")
+                raise TypeError("Can't pack unicode object: No encoding is specified")
             o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
             rawval = o
             ret = msgpack_pack_raw(&self.pk, len(o))

From 3ffc75928bfe5b5cf232de996a01436cb4064f65 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 31 May 2011 15:40:11 +0900
Subject: [PATCH 0967/1648] Revert "(python) Change error message for unicode
 is passed but no encoding is"

This reverts commit bd73742552cf16592662a7ec5ba3608888081131.
---
 ChangeLog.rst        | 10 ++++++++++
 msgpack/_msgpack.pyx |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 75e86b2..a0aae25 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,13 @@
+0.1.10
+======
+:release date: NOT RELEASED YET
+
+New feature
+-----------
+* Add ``encoding`` and ``unicode_erros`` option to packer and unpacker.
+  When this option is specified, (un)packs unicode object instead of bytes.
+  This enables using msgpack as a replacement of json.
+
 0.1.9
 ======
 :release date: 2011-01-29
diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 14bc9d7..443cbd7 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -120,7 +120,7 @@ cdef class Packer(object):
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
         elif PyUnicode_Check(o):
             if not self.encoding:
-                raise TypeError("Can't pack unicode object: No encoding is specified")
+                raise TypeError("Can't encode utf-8 no encoding is specified")
             o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
             rawval = o
             ret = msgpack_pack_raw(&self.pk, len(o))

From ff594d71dd85661e6307a9cca6ed184227e1dd35 Mon Sep 17 00:00:00 2001
From: inada-n 
Date: Wed, 1 Jun 2011 18:30:43 +0900
Subject: [PATCH 0968/1648] (python) make test pass with Python 2.5

---
 test/test_pack.py      | 33 +++++++++++++++++++--------------
 test/test_sequnpack.py | 20 ++++++++++----------
 2 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/test/test_pack.py b/test/test_pack.py
index 2aef588..2b5f1ad 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -3,6 +3,7 @@
 
 from nose import main
 from nose.tools import *
+from nose.plugins.skip import SkipTest
 
 from msgpack import packs, unpacks
 
@@ -15,7 +16,7 @@ def testPack():
             0, 1, 127, 128, 255, 256, 65535, 65536,
             -1, -32, -33, -128, -129, -32768, -32769,
             1.0,
-        b"", b"a", b"a"*31, b"a"*32,
+        "", "a", "a"*31, "a"*32,
         None, True, False,
         (), ((),), ((), None,),
         {None: 0},
@@ -33,36 +34,40 @@ def testPackUnicode():
         assert_equal(re, td)
 
 def testPackUTF32():
-    test_data = [
-        u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚",
-        ]
-    for td in test_data:
-        print(packs(td, encoding='utf-32'))
-        re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32')
-        assert_equal(re, td)
+    try:
+        test_data = [
+            u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚",
+            ]
+        for td in test_data:
+            re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32')
+            assert_equal(re, td)
+    except LookupError:
+        raise SkipTest
 
 def testPackBytes():
     test_data = [
-        b"", b"abcd", (b"defgh",),
+        "", "abcd", ("defgh",),
         ]
     for td in test_data:
         check(td)
 
 def testIgnoreUnicodeErrors():
-    re = unpacks(packs(b'abc\xeddef'),
-        encoding='utf-8', unicode_errors='ignore')
+    re = unpacks(packs('abc\xeddef'),
+        encoding='ascii', unicode_errors='ignore')
     assert_equal(re, "abcdef")
 
 @raises(UnicodeDecodeError)
 def testStrictUnicodeUnpack():
-    unpacks(packs(b'abc\xeddef'), encoding='utf-8')
+    unpacks(packs('abc\xeddef'), encoding='utf-8')
 
 @raises(UnicodeEncodeError)
 def testStrictUnicodePack():
     packs(u"abc\xeddef", encoding='ascii', unicode_errors='strict')
 
 def testIgnoreErrorsPack():
-    re = unpacks(packs(u"abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
+    re = unpacks(
+            packs(u"abcФФФdef", encoding='ascii', unicode_errors='ignore'),
+            encoding='utf-8')
     assert_equal(re, u"abcdef")
 
 @raises(TypeError)
@@ -71,7 +76,7 @@ def testNoEncoding():
 
 def testDecodeBinary():
     re = unpacks(packs(u"abc"), encoding=None)
-    assert_equal(re, b"abc")
+    assert_equal(re, "abc")
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index c92658c..d61be23 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -6,12 +6,12 @@ from msgpack import Unpacker
 def test_foobar():
     unpacker = Unpacker(read_size=3)
     unpacker.feed('foobar')
-    assert unpacker.unpack() == ord(b'f')
-    assert unpacker.unpack() == ord(b'o')
-    assert unpacker.unpack() == ord(b'o')
-    assert unpacker.unpack() == ord(b'b')
-    assert unpacker.unpack() == ord(b'a')
-    assert unpacker.unpack() == ord(b'r')
+    assert unpacker.unpack() == ord('f')
+    assert unpacker.unpack() == ord('o')
+    assert unpacker.unpack() == ord('o')
+    assert unpacker.unpack() == ord('b')
+    assert unpacker.unpack() == ord('a')
+    assert unpacker.unpack() == ord('r')
     try:
         o = unpacker.unpack()
         print "Oops!", o
@@ -20,14 +20,14 @@ def test_foobar():
         assert 1
     else:
         assert 0
-    unpacker.feed(b'foo')
-    unpacker.feed(b'bar')
+    unpacker.feed('foo')
+    unpacker.feed('bar')
 
     k = 0
-    for o, e in zip(unpacker, b'foobarbaz'):
+    for o, e in zip(unpacker, 'foobarbaz'):
         assert o == ord(e)
         k += 1
-    assert k == len(b'foobar')
+    assert k == len('foobar')
 
 if __name__ == '__main__':
     test_foobar()

From cd75ba495af3c153f707c127b0ce6484df6fb1a0 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 24 Jun 2011 00:04:43 +0900
Subject: [PATCH 0969/1648] (python) Fix typo in ChangeLog

---
 ChangeLog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index a0aae25..10ddf4f 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -4,7 +4,7 @@
 
 New feature
 -----------
-* Add ``encoding`` and ``unicode_erros`` option to packer and unpacker.
+* Add ``encoding`` and ``unicode_errors`` option to packer and unpacker.
   When this option is specified, (un)packs unicode object instead of bytes.
   This enables using msgpack as a replacement of json.
 

From 6bc62bd1424b6419e75b99f38199fdfc66d4ec87 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 21 Aug 2011 19:09:36 +0900
Subject: [PATCH 0970/1648] rename README to README.rst

---
 README => README.rst | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename README => README.rst (100%)

diff --git a/README b/README.rst
similarity index 100%
rename from README
rename to README.rst

From 4a1ce19addc6fc5faf26101b16124401113889d1 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 21 Aug 2011 19:29:33 +0900
Subject: [PATCH 0971/1648] Fix install command for Windows.

---
 README.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index a7e8c53..994492e 100644
--- a/README.rst
+++ b/README.rst
@@ -28,7 +28,8 @@ Windows
 MessagePack requires gcc currently. So you need to prepare
 MinGW GCC.
 
-    $ python setup.py install -c mingw32
+    $ python setup.py build -c mingw32
+    $ python setup.py install
 
 TEST
 ----

From 8c3c8a250b5f6f129e5e077a224ec6916cc87437 Mon Sep 17 00:00:00 2001
From: tailhook 
Date: Mon, 22 Aug 2011 01:52:45 +0900
Subject: [PATCH 0972/1648] Fixed `encoding` argument for unpacker in Python

---
 msgpack/_msgpack.pyx | 22 +++++++++++++++++++++-
 test/test_pack.py    |  6 ++++++
 test3/test_pack.py   |  9 +++++++--
 3 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 443cbd7..5a83ea0 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -203,6 +203,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint
 
     if encoding is None:
         enc = NULL
+        err = NULL
     else:
         if isinstance(encoding, unicode):
             bencoding = encoding.encode('ascii')
@@ -267,6 +268,10 @@ cdef class Unpacker(object):
     cdef Py_ssize_t read_size
     cdef bint use_list
     cdef object object_hook
+    cdef object _bencoding
+    cdef object _berrors
+    cdef char *encoding
+    cdef char *unicode_errors
 
     def __cinit__(self):
         self.buf = NULL
@@ -276,7 +281,8 @@ cdef class Unpacker(object):
         self.buf = NULL;
 
     def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
-                 object object_hook=None, object list_hook=None, encoding=None, unicode_errors=None):
+                 object object_hook=None, object list_hook=None,
+                 encoding=None, unicode_errors='strict'):
         if read_size == 0:
             read_size = 1024*1024
         self.use_list = use_list
@@ -303,6 +309,20 @@ cdef class Unpacker(object):
             if not PyCallable_Check(list_hook):
                 raise TypeError("list_hook must be a callable.")
             self.ctx.user.list_hook = list_hook
+        if encoding is None:
+            self.ctx.user.encoding = NULL
+            self.ctx.user.unicode_errors = NULL
+        else:
+            if isinstance(encoding, unicode):
+                self._bencoding = encoding.encode('ascii')
+            else:
+                self._bencoding = encoding
+            self.ctx.user.encoding = PyBytes_AsString(self._bencoding)
+            if isinstance(unicode_errors, unicode):
+                self._berrors = unicode_errors.encode('ascii')
+            else:
+                self._berrors = unicode_errors
+            self.ctx.user.unicode_errors = PyBytes_AsString(self._berrors)
 
     def feed(self, object next_bytes):
         cdef char* buf
diff --git a/test/test_pack.py b/test/test_pack.py
index 2b5f1ad..357cb3c 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -7,6 +7,8 @@ from nose.plugins.skip import SkipTest
 
 from msgpack import packs, unpacks
 
+from StringIO import StringIO
+
 def check(data):
     re = unpacks(packs(data))
     assert_equal(re, data)
@@ -32,6 +34,10 @@ def testPackUnicode():
     for td in test_data:
         re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8')
         assert_equal(re, td)
+        packer = Packer(encoding='utf-8')
+        data = packer.pack(td)
+        re = Unpacker(StringIO(data), encoding='utf-8').unpack()
+        assert_equal(re, td)
 
 def testPackUTF32():
     try:
diff --git a/test3/test_pack.py b/test3/test_pack.py
index e53f7e6..5ff04e7 100644
--- a/test3/test_pack.py
+++ b/test3/test_pack.py
@@ -4,7 +4,9 @@
 from nose import main
 from nose.tools import *
 
-from msgpack import packs, unpacks
+from msgpack import packs, unpacks, Unpacker, Packer
+
+from io import BytesIO
 
 def check(data):
     re = unpacks(packs(data))
@@ -31,13 +33,16 @@ def testPackUnicode():
     for td in test_data:
         re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8')
         assert_equal(re, td)
+        packer = Packer(encoding='utf-8')
+        data = packer.pack(td)
+        re = Unpacker(BytesIO(data), encoding='utf-8').unpack()
+        assert_equal(re, td)
 
 def testPackUTF32():
     test_data = [
         "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚",
         ]
     for td in test_data:
-        print(packs(td, encoding='utf-32'))
         re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32')
         assert_equal(re, td)
 

From 60762747b994138dd79c655c8b9da20704f50c0c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 22 Aug 2011 01:57:23 +0900
Subject: [PATCH 0973/1648] Fix error in tests.

---
 test/test_pack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_pack.py b/test/test_pack.py
index 357cb3c..2e1ac22 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -5,7 +5,7 @@ from nose import main
 from nose.tools import *
 from nose.plugins.skip import SkipTest
 
-from msgpack import packs, unpacks
+from msgpack import packs, unpacks, Packer, Unpacker
 
 from StringIO import StringIO
 

From 1a81f6364dcb4356d26b8e3ae48966a9c2a66832 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 22 Aug 2011 02:09:51 +0900
Subject: [PATCH 0974/1648] 0.1.10

---
 COPYING       | 2 +-
 ChangeLog.rst | 5 +++--
 setup.py      | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/COPYING b/COPYING
index 3cb1f1e..f067af3 100644
--- a/COPYING
+++ b/COPYING
@@ -1,4 +1,4 @@
-Copyright (C) 2008-2010 KLab Inc.
+Copyright (C) 2008-2011 INADA Naoki 
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
diff --git a/ChangeLog.rst b/ChangeLog.rst
index 10ddf4f..9cae3c8 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,12 +1,13 @@
 0.1.10
 ======
-:release date: NOT RELEASED YET
+:release date: 2011-08-22
 
 New feature
 -----------
 * Add ``encoding`` and ``unicode_errors`` option to packer and unpacker.
   When this option is specified, (un)packs unicode object instead of bytes.
-  This enables using msgpack as a replacement of json.
+  This enables using msgpack as a replacement of json. (tailhook)
+
 
 0.1.9
 ======
diff --git a/setup.py b/setup.py
index 16987a4..0eeb84d 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 9, 'final')
+version = (0, 1, 10)
 
 import os
 import sys

From a9483d860b0c7188cc409f2e94b0c022658d4dba Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 22 Aug 2011 03:37:14 +0900
Subject: [PATCH 0975/1648] Start 0.1.11a1.dev1

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 0eeb84d..b1a1221 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 10)
+version = (0, 1, 11, 'a1.dev1')
 
 import os
 import sys

From 8361fd8da65bb1dd939edec390a6ced969413129 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 22 Aug 2011 03:38:16 +0900
Subject: [PATCH 0976/1648] Remove extra alias.

---
 msgpack/_msgpack.pyx | 2 --
 1 file changed, 2 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 5a83ea0..ae9b69f 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -168,7 +168,6 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'):
     packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
     return packer.pack(o)
 
-dumps = packs = packb
 
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
@@ -239,7 +238,6 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint
     else:
         return None
 
-loads = unpacks = unpackb
 
 def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"):
     """unpack an object from stream."""

From 5ed2288fed84131b43e0ddeba34868b9d3a6a834 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 22 Aug 2011 03:41:24 +0900
Subject: [PATCH 0977/1648] Add signature to docstring.

---
 msgpack/_msgpack.pyx | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index ae9b69f..b004755 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -159,12 +159,18 @@ cdef class Packer(object):
 
 
 def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'):
-    """pack an object `o` and write it to stream)."""
+    """\
+    pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict')
+
+    pack an object `o` and write it to stream)."""
     packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
     stream.write(packer.pack(o))
 
 def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'):
-    """pack o and return packed bytes."""
+    """\
+    packb(object o, default=None, encoding='utf-8', unicode_errors='strict')
+
+    pack o and return packed bytes."""
     packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
     return packer.pack(o)
 
@@ -191,7 +197,11 @@ cdef extern from "unpack.h":
 
 
 def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"):
-    """Unpack packed_bytes to object. Returns an unpacked object."""
+    """\
+    unpackb(object packed, object object_hook=None, object list_hook=None,
+            bint use_list=0, encoding=None, unicode_errors="strict")
+
+    Unpack packed_bytes to object. Returns an unpacked object."""
     cdef template_context ctx
     cdef size_t off = 0
     cdef int ret
@@ -240,7 +250,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"):
-    """unpack an object from stream."""
+    """\
+    unpack(object stream, object object_hook=None, object list_hook=None,
+           bint use_list=0, encoding=None, unicode_errors="strict")
+    
+    unpack an object from stream."""
     return unpackb(stream.read(), use_list=use_list,
                    object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors)
 

From 938a1249739f1f049ef03d5a0a32336ba24f82e3 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 26 Aug 2011 04:45:05 +0900
Subject: [PATCH 0978/1648] Use cython's embedsignature directive and enhance
 some docstrings.

---
 msgpack/_msgpack.pyx | 49 +++++++++++++++++++++++++-------------------
 1 file changed, 28 insertions(+), 21 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index b004755..9c2938c 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -1,4 +1,5 @@
 # coding: utf-8
+#cython: embedsignature=True
 
 from cpython cimport *
 cdef extern from "Python.h":
@@ -159,17 +160,13 @@ cdef class Packer(object):
 
 
 def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'):
-    """\
-    pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict')
-
+    """
     pack an object `o` and write it to stream)."""
     packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
     stream.write(packer.pack(o))
 
 def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'):
-    """\
-    packb(object o, default=None, encoding='utf-8', unicode_errors='strict')
-
+    """
     pack o and return packed bytes."""
     packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
     return packer.pack(o)
@@ -197,10 +194,7 @@ cdef extern from "unpack.h":
 
 
 def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"):
-    """\
-    unpackb(object packed, object object_hook=None, object list_hook=None,
-            bint use_list=0, encoding=None, unicode_errors="strict")
-
+    """
     Unpack packed_bytes to object. Returns an unpacked object."""
     cdef template_context ctx
     cdef size_t off = 0
@@ -250,21 +244,36 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"):
-    """\
-    unpack(object stream, object object_hook=None, object list_hook=None,
-           bint use_list=0, encoding=None, unicode_errors="strict")
-    
-    unpack an object from stream."""
+    """
+    unpack an object from stream.
+    """
     return unpackb(stream.read(), use_list=use_list,
                    object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors)
 
 cdef class Unpacker(object):
-    """Unpacker(read_size=1024*1024)
-
+    """
     Streaming unpacker.
     read_size is used like file_like.read(read_size)
 
-    example:
+    `file_like` is a file-like object having `.read(n)` method.
+    When `Unpacker` initialized with `file_like`, unpacker reads serialized data
+    from it and `.feed()` method is not usable.
+
+    `read_size` is used as `file_like.read(read_size)`. (default: 1M)
+
+    If `use_list` is true, msgpack list is deserialized to Python list.
+    Otherwise, it is deserialized to Python tuple. (default: False)
+
+    `object_hook` is same to simplejson. If it is not None, it should be callable
+    and Unpacker calls it when deserializing key-value.
+
+    `encoding` is encoding used for decoding msgpack bytes. If it is None (default),
+    msgpack bytes is deserialized to Python bytes.
+
+    `unicode_errors` is used for decoding bytes.
+
+    example::
+
         unpacker = Unpacker()
         while 1:
             buf = astream.read()
@@ -292,11 +301,9 @@ cdef class Unpacker(object):
         free(self.buf);
         self.buf = NULL;
 
-    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
+    def __init__(self, file_like=None, Py_ssize_t read_size=1024*1024, bint use_list=0,
                  object object_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict'):
-        if read_size == 0:
-            read_size = 1024*1024
         self.use_list = use_list
         self.file_like = file_like
         if file_like:

From 2c53c1797975b06929e2fbb4e4b462aef502eacf Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 2 Sep 2011 16:16:08 +0900
Subject: [PATCH 0979/1648] Add test/ and test3/ to MANIFEST.in

---
 MANIFEST.in | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index cbc9ada..bf312d5 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,3 +1,5 @@
 include setup.py
 include COPYING
 recursive-include msgpack *.h *.c *.pyx
+recursive-include test *.py
+recursive-include test3 *.py

From 8312e986bfe5fea3c4d88259b06fa73d47130de4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 2 Sep 2011 16:20:01 +0900
Subject: [PATCH 0980/1648] Update changelog

---
 ChangeLog.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 9cae3c8..9b407cd 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,13 @@
+0.1.11
+=======
+:release date: NOT RELEASED YET
+
+Bugs fixed
+-------------
+
+* Include test code for Python3 to sdist. (Johan Bergström)
+
+
 0.1.10
 ======
 :release date: 2011-08-22

From d72bd0c5d4c09b78ce1912d0101709852573abc2 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 2 Sep 2011 16:22:21 +0900
Subject: [PATCH 0981/1648] Remove unnecessary semicolon.

---
 msgpack/_msgpack.pyx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 9c2938c..7a81746 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -298,8 +298,8 @@ cdef class Unpacker(object):
         self.buf = NULL
 
     def __dealloc__(self):
-        free(self.buf);
-        self.buf = NULL;
+        free(self.buf)
+        self.buf = NULL
 
     def __init__(self, file_like=None, Py_ssize_t read_size=1024*1024, bint use_list=0,
                  object object_hook=None, object list_hook=None,

From 1f829173cd62d175721fa71da481becbece8826c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 25 Dec 2011 12:48:36 +0900
Subject: [PATCH 0982/1648] add *.so to .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 35b00cc..1c29d6e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,5 +3,6 @@ build/*
 dist/*
 *.pyc
 *.pyo
+*.so
 msgpack/__version__.py
 msgpack/_msgpack.c

From b553754edf9597a11122956c567fd713e0537a3e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 25 Dec 2011 12:54:19 +0900
Subject: [PATCH 0983/1648] Fix compile error on MSVC. (davidgaleano)

---
 msgpack/pack.h            | 4 ++++
 msgpack/pack_template.h   | 4 ++--
 msgpack/unpack_template.h | 5 +++--
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/msgpack/pack.h b/msgpack/pack.h
index 2ae95d1..d36b436 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -25,6 +25,10 @@
 extern "C" {
 #endif
 
+#ifdef _MSC_VER
+#define inline __inline
+#endif
+
 typedef struct msgpack_packer {
     char *buf;
     size_t length;
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index de148bf..4b50895 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -555,8 +555,8 @@ if(sizeof(unsigned long long) == 2) {
 msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d)
 {
 	union { char buf[4]; uint32_t num; } f;
-	*((float*)&f.buf) = d;  // FIXME
 	unsigned char buf[5];
+	*((float*)&f.buf) = d;  // FIXME
 	buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num);
 	msgpack_pack_append_buffer(x, buf, 5);
 }
@@ -564,8 +564,8 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d)
 msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d)
 {
 	union { char buf[8]; uint64_t num; } f;
-	*((double*)&f.buf) = d;  // FIXME
 	unsigned char buf[9];
+	*((double*)&f.buf) = d;  // FIXME
 	buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num);
 	msgpack_pack_append_buffer(x, buf, 9);
 }
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 7a2288f..42fe3a1 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -82,8 +82,6 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context
 
 msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
 {
-	assert(len >= *off);
-
 	const unsigned char* p = (unsigned char*)data + *off;
 	const unsigned char* const pe = (unsigned char*)data + len;
 	const void* n = NULL;
@@ -91,6 +89,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	unsigned int trail = ctx->trail;
 	unsigned int cs = ctx->cs;
 	unsigned int top = ctx->top;
+
 	msgpack_unpack_struct(_stack)* stack = ctx->stack;
 	msgpack_unpack_user* user = &ctx->user;
 
@@ -99,6 +98,8 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 
 	int ret;
 
+	assert(len >= *off);
+
 #define push_simple_value(func) \
 	if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \
 	goto _push

From d2a23cc22a5998fb95a7d4c1bf1f6fbd17d27f3b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 26 Dec 2011 19:04:33 +0900
Subject: [PATCH 0984/1648] Release 0.1.11

---
 ChangeLog.rst | 3 ++-
 setup.py      | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 9b407cd..758c83e 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,11 +1,12 @@
 0.1.11
 =======
-:release date: NOT RELEASED YET
+:release date: 2011-12-26
 
 Bugs fixed
 -------------
 
 * Include test code for Python3 to sdist. (Johan Bergström)
+* Fix compilation error on MSVC. (davidgaleano)
 
 
 0.1.10
diff --git a/setup.py b/setup.py
index b1a1221..faeb9f1 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 11, 'a1.dev1')
+version = (0, 1, 11)
 
 import os
 import sys

From ac713705ebdd131acb92b52116e6ec180731add2 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 27 Dec 2011 21:34:40 +0900
Subject: [PATCH 0985/1648] 0.1.12: re-enable packs()/unpacks() aliases.

---
 ChangeLog.rst       | 10 ++++++++++
 msgpack/__init__.py |  9 +++++++++
 setup.py            |  2 +-
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 758c83e..89a97d7 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,13 @@
+0.1.12
+=======
+:release date: 2011-12-27
+
+Bugs fixed
+-------------
+
+* Re-enable packs/unpacks removed at 0.1.11. It will be removed when 0.2 is released.
+
+
 0.1.11
 =======
 :release date: 2011-12-26
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index cdf045f..6b9735e 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -9,3 +9,12 @@ loads = unpackb
 dump = pack
 dumps = packb
 
+def packs(*args, **kw):
+    from warnings import warn
+    warn("msgpack.packs() is deprecated. Use packb() instead.", DeprecationWarning)
+    return packb(*args, **kw)
+
+def unpacks(*args, **kw):
+    from warnings import warn
+    warn("msgpack.unpacks() is deprecated. Use unpackb() instead.", DeprecationWarning)
+    return unpackb(*args, **kw)
diff --git a/setup.py b/setup.py
index faeb9f1..ddacbe2 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 11)
+version = (0, 1, 12)
 
 import os
 import sys

From d685614138e588dc82b84cddae254fc120279762 Mon Sep 17 00:00:00 2001
From: Steffen Siering 
Date: Wed, 18 Jan 2012 19:02:11 +0100
Subject: [PATCH 0986/1648] adopt setup.py to work with python installation
 older 2.7

---
 setup.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index ddacbe2..f34e341 100755
--- a/setup.py
+++ b/setup.py
@@ -45,7 +45,9 @@ else:
 
     Sdist = sdist
 
-libraries = ['ws2_32'] if sys.platform == 'win32' else []
+libraries = []
+if sys.platform == 'win32':
+    libraries.append('ws2_32')
 
 msgpack_mod = Extension('msgpack._msgpack',
                         sources=sources,

From b76416977554c7b01442efe0d1f058a43372e2f5 Mon Sep 17 00:00:00 2001
From: David Wolever 
Date: Fri, 10 Feb 2012 15:08:49 -0500
Subject: [PATCH 0987/1648] Correcting 'utf-8' to 'unicode'.

---
 msgpack/_msgpack.pyx | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 7a81746..52464fb 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -121,7 +121,7 @@ cdef class Packer(object):
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
         elif PyUnicode_Check(o):
             if not self.encoding:
-                raise TypeError("Can't encode utf-8 no encoding is specified")
+                raise TypeError("Can't encode unicode string: no encoding is specified")
             o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
             rawval = o
             ret = msgpack_pack_raw(&self.pk, len(o))
@@ -425,3 +425,4 @@ cdef class Unpacker(object):
 
     #def _off(self):
     #    return self.buf_head
+

From 5b878b6038ac7d8d180f359f728531e666717f94 Mon Sep 17 00:00:00 2001
From: Steeve Morin 
Date: Thu, 16 Feb 2012 17:15:04 +0100
Subject: [PATCH 0988/1648] Be greedier when checking for tuples or lists.

---
 msgpack/_msgpack.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 52464fb..7f369a8 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -136,7 +136,7 @@ cdef class Packer(object):
                     if ret != 0: break
                     ret = self._pack(v, nest_limit-1)
                     if ret != 0: break
-        elif PySequence_Check(o):
+        elif PyTuple_Check(o) or PyList_Check(o):
             ret = msgpack_pack_array(&self.pk, len(o))
             if ret == 0:
                 for v in o:

From 31b7fda17b464176c34908da366bed5e4545823d Mon Sep 17 00:00:00 2001
From: Steeve Morin 
Date: Tue, 28 Feb 2012 15:36:58 +0100
Subject: [PATCH 0989/1648] Fix massive memory leak with object_hook and
 list_hook when unpacking.

---
 msgpack/unpack.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 0586ca8..6b443f2 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -161,8 +161,10 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj
 {
     if (u->list_hook) {
         PyObject *arglist = Py_BuildValue("(O)", *c);
-        *c = PyEval_CallObject(u->list_hook, arglist);
+        msgpack_unpack_object *new_c = PyEval_CallObject(u->list_hook, arglist);
         Py_DECREF(arglist);
+        Py_DECREF(*c);
+        *c = new_c;
     }
     return 0;
 }
@@ -190,8 +192,10 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec
 {
     if (u->object_hook) {
         PyObject *arglist = Py_BuildValue("(O)", *c);
-        *c = PyEval_CallObject(u->object_hook, arglist);
+        msgpack_unpack_object *new_c = PyEval_CallObject(u->object_hook, arglist);
         Py_DECREF(arglist);
+        Py_DECREF(*c);
+        *c = new_c;
     }
     return 0;
 }

From a5bc6b73856ca9c59f140427a26a6ffd5103b86f Mon Sep 17 00:00:00 2001
From: Steeve Morin 
Date: Tue, 28 Feb 2012 15:41:44 +0100
Subject: [PATCH 0990/1648] Better prototypes.

---
 msgpack/unpack.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 6b443f2..2659a97 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -161,7 +161,7 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj
 {
     if (u->list_hook) {
         PyObject *arglist = Py_BuildValue("(O)", *c);
-        msgpack_unpack_object *new_c = PyEval_CallObject(u->list_hook, arglist);
+        PyObject *new_c = PyEval_CallObject(u->list_hook, arglist);
         Py_DECREF(arglist);
         Py_DECREF(*c);
         *c = new_c;
@@ -192,7 +192,7 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec
 {
     if (u->object_hook) {
         PyObject *arglist = Py_BuildValue("(O)", *c);
-        msgpack_unpack_object *new_c = PyEval_CallObject(u->object_hook, arglist);
+        PyObject *new_c = PyEval_CallObject(u->object_hook, arglist);
         Py_DECREF(arglist);
         Py_DECREF(*c);
         *c = new_c;

From 64bdf6bcd633b4404cb5413f37a1c7ae428f048c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 8 Mar 2012 16:59:08 +0900
Subject: [PATCH 0991/1648] small optimization.

---
 msgpack/_msgpack.pyx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 7f369a8..612dd0f 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -131,7 +131,7 @@ cdef class Packer(object):
             d = o
             ret = msgpack_pack_map(&self.pk, len(d))
             if ret == 0:
-                for k,v in d.items():
+                for k,v in d.iteritems():
                     ret = self._pack(k, nest_limit-1)
                     if ret != 0: break
                     ret = self._pack(v, nest_limit-1)
@@ -149,7 +149,7 @@ cdef class Packer(object):
             raise TypeError("can't serialize %r" % (o,))
         return ret
 
-    def pack(self, object obj):
+    cpdef pack(self, object obj):
         cdef int ret
         ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
         if ret:

From 7a4af28fa1599f49cc0a27c66beecd5e4474dcbe Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 21 Apr 2012 14:25:07 +0900
Subject: [PATCH 0992/1648] release 0.1.13

---
 ChangeLog.rst | 21 +++++++++++++++++++++
 setup.py      |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 89a97d7..e0a4fd1 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,24 @@
+0.1.13
+=======
+:release date: 2012-04-21
+
+New
+----
+* Don't accept subtype of list and tuple as msgpack list. (Steeve Morin)
+  It allows customize how it serialized with ``default`` argument.
+
+Bugs fixed
+-----------
+* Fix wrong error message. (David Wolever)
+* Fix memory leak while unpacking when ``object_hook`` or ``list_hook`` is used.
+  (Steeve Morin)
+
+Other changes
+-------------
+* setup.py works on Python 2.5 (Steffen Siering)
+* Optimization for serializing dict.
+
+
 0.1.12
 =======
 :release date: 2011-12-27
diff --git a/setup.py b/setup.py
index f34e341..b4b1771 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 12)
+version = (0, 1, 13)
 
 import os
 import sys

From 07506667c98f51bf46b47116f9afed8fa5f381e0 Mon Sep 17 00:00:00 2001
From: Steeve Morin 
Date: Sat, 16 Jun 2012 13:56:46 +0300
Subject: [PATCH 0993/1648] Make sure objects inherited from Dict are properly
 casted (or else Cython will complain and crash).

---
 msgpack/_msgpack.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 612dd0f..990f585 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -128,7 +128,7 @@ cdef class Packer(object):
             if ret == 0:
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
         elif PyDict_Check(o):
-            d = o
+            d = o
             ret = msgpack_pack_map(&self.pk, len(d))
             if ret == 0:
                 for k,v in d.iteritems():

From 40d4b8946b5e25045eb1047f8d0d3d4c5ee71282 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 19 Jun 2012 13:39:23 +0900
Subject: [PATCH 0994/1648] Start using tox

---
 tox.ini | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 tox.ini

diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..c0a5629
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,9 @@
+[tox]
+envlist = py26,py27,py32
+[testenv]
+deps=
+    nose
+commands=nosetests -w test
+
+[testenv:py32]
+commands=nosetests -w test3

From f1dd03fe80de1c2b267d79bd3d74dec2592a2339 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 19 Jun 2012 13:39:32 +0900
Subject: [PATCH 0995/1648] Add test for subtype.

---
 test/test_subtype.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 test/test_subtype.py

diff --git a/test/test_subtype.py b/test/test_subtype.py
new file mode 100644
index 0000000..1dfd7da
--- /dev/null
+++ b/test/test_subtype.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from nose import main
+from nose.tools import *
+from msgpack import packb, unpackb
+from collections import namedtuple
+
+class MyList(list):
+    pass
+
+class MyDict(dict):
+    pass
+
+class MyTuple(tuple):
+    pass
+
+MyNamedTuple = namedtuple('MyNamedTuple', 'x y')
+
+def test_types():
+    assert_equal(packb(dict()), packb(MyDict()))
+    assert_equal(packb(list()), packb(MyList()))
+    assert_equal(packb(MyNamedTuple(1,2)), packb((1,2)))
+
+
+if __name__ == '__main__':
+    main()

From 76f34667a02f461043a70d776ec05fc1f90bd9e9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 19 Jun 2012 13:40:00 +0900
Subject: [PATCH 0996/1648] Add ".tox" to .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 1c29d6e..9452edd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 MANIFEST
 build/*
 dist/*
+.tox
 *.pyc
 *.pyo
 *.so

From 0b38e86534130f625cbea2f9446e8e52ef5f5a06 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 19 Jun 2012 13:55:14 +0900
Subject: [PATCH 0997/1648] unify tests for py2 and py3

---
 test/test_buffer.py     |   4 +-
 test/test_case.py       |  40 +++++++--------
 test/test_format.py     |  50 +++++++++----------
 test/test_obj.py        |  15 +++---
 test/test_pack.py       |  34 ++++++-------
 test/test_sequnpack.py  |  28 ++++++-----
 test3/test_buffer.py    |  16 ------
 test3/test_case.py      | 105 ----------------------------------------
 test3/test_except.py    |  14 ------
 test3/test_format.py    |  75 ----------------------------
 test3/test_obj.py       |  44 -----------------
 test3/test_pack.py      |  82 -------------------------------
 test3/test_sequnpack.py |  36 --------------
 tox.ini                 |   4 +-
 14 files changed, 85 insertions(+), 462 deletions(-)
 delete mode 100644 test3/test_buffer.py
 delete mode 100644 test3/test_case.py
 delete mode 100644 test3/test_except.py
 delete mode 100644 test3/test_format.py
 delete mode 100644 test3/test_obj.py
 delete mode 100644 test3/test_pack.py
 delete mode 100644 test3/test_sequnpack.py

diff --git a/test/test_buffer.py b/test/test_buffer.py
index ce7a72d..01310a0 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -7,10 +7,10 @@ from msgpack import packb, unpackb
 
 def test_unpack_buffer():
     from array import array
-    buf = array('c')
+    buf = array('b')
     buf.fromstring(packb(('foo', 'bar')))
     obj = unpackb(buf)
-    assert_equal(('foo', 'bar'), obj)
+    assert_equal((b'foo', b'bar'), obj)
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_case.py b/test/test_case.py
index 1cbc494..2f42316 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -40,7 +40,7 @@ def test_9():
 
 
 def check_raw(overhead, num):
-    check(num + overhead, " " * num)
+    check(num + overhead, b" " * num)
 
 def test_fixraw():
     check_raw(1, 0)
@@ -75,31 +75,31 @@ def match(obj, buf):
 
 def test_match():
     cases = [
-        (None, '\xc0'),
-        (False, '\xc2'),
-        (True, '\xc3'),
-        (0, '\x00'),
-        (127, '\x7f'),
-        (128, '\xcc\x80'),
-        (256, '\xcd\x01\x00'),
-        (-1, '\xff'),
-        (-33, '\xd0\xdf'),
-        (-129, '\xd1\xff\x7f'),
-        ({1:1}, '\x81\x01\x01'),
-        (1.0, "\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"),
-        ((), '\x90'),
-        (tuple(range(15)),"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"),
-        (tuple(range(16)),"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"),
-        ({}, '\x80'),
-        (dict([(x,x) for x in range(15)]), '\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'),
-        (dict([(x,x) for x in range(16)]), '\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'),
+        (None, b'\xc0'),
+        (False, b'\xc2'),
+        (True, b'\xc3'),
+        (0, b'\x00'),
+        (127, b'\x7f'),
+        (128, b'\xcc\x80'),
+        (256, b'\xcd\x01\x00'),
+        (-1, b'\xff'),
+        (-33, b'\xd0\xdf'),
+        (-129, b'\xd1\xff\x7f'),
+        ({1:1}, b'\x81\x01\x01'),
+        (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"),
+        ((), b'\x90'),
+        (tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"),
+        (tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"),
+        ({}, b'\x80'),
+        (dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'),
+        (dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'),
         ]
 
     for v, p in cases:
         match(v, p)
 
 def test_unicode():
-    assert_equal('foobar', unpacks(packs(u'foobar')))
+    assert_equal(b'foobar', unpacks(packs('foobar')))
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_format.py b/test/test_format.py
index 562ef54..022e680 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -9,65 +9,65 @@ def check(src, should):
     assert_equal(unpacks(src), should)
 
 def testSimpleValue():
-    check("\x93\xc0\xc2\xc3", 
+    check(b"\x93\xc0\xc2\xc3", 
             (None, False, True,))
 
 def testFixnum():
-    check("\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff",
+    check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff",
           ((0,64,127,), (-32,-16,-1,),)
           )
 
 def testFixArray():
-    check("\x92\x90\x91\x91\xc0",
+    check(b"\x92\x90\x91\x91\xc0",
           ((),((None,),),),
           )
 
 def testFixRaw():
-    check("\x94\xa0\xa1a\xa2bc\xa3def",
-          ("", "a", "bc", "def",),
+    check(b"\x94\xa0\xa1a\xa2bc\xa3def",
+          (b"", b"a", b"bc", b"def",),
           )
 
 def testFixMap():
     check(
-          "\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
+          b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
           {False: {None: None}, True:{None:{}}},
           )
 
 def testUnsignedInt():
     check(
-          "\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
-          "\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
-          "\xce\xff\xff\xff\xff",
+          b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
+          b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
+          b"\xce\xff\xff\xff\xff",
           (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,),
           )
 
 def testSignedInt():
-    check("\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
-          "\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
-          "\xd2\xff\xff\xff\xff",
+    check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
+          b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
+          b"\xd2\xff\xff\xff\xff",
           (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,))
 
 def testRaw():
-    check("\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
-        "\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
-        ("", "a", "ab", "", "a", "ab"))
+    check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
+        b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
+        (b"", b"a", b"ab", b"", b"a", b"ab"))
 
 def testArray():
-    check("\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
-        "\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02"
-        "\xc2\xc3",
+    check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
+        b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02"
+        b"\xc2\xc3",
         ((), (None,), (False,True), (), (None,), (False,True))
         )
 
 def testMap():
     check(
-        "\x96"
-            "\xde\x00\x00"
-            "\xde\x00\x01\xc0\xc2"
-            "\xde\x00\x02\xc0\xc2\xc3\xc2"
-            "\xdf\x00\x00\x00\x00"
-            "\xdf\x00\x00\x00\x01\xc0\xc2"
-            "\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
+        b"\x96"
+            b"\xde\x00\x00"
+            b"\xde\x00\x01\xc0\xc2"
+            b"\xde\x00\x02\xc0\xc2\xc3\xc2"
+            b"\xdf\x00\x00\x00\x00"
+            b"\xdf\x00\x00\x00\x01\xc0\xc2"
+            b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
         ({}, {None: False}, {True: False, None: False}, {},
             {None: False}, {True: False, None: False}))
 
diff --git a/test/test_obj.py b/test/test_obj.py
index 0eebe7b..6357cfc 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -7,22 +7,22 @@ from nose.tools import *
 from msgpack import packs, unpacks
 
 def _decode_complex(obj):
-    if '__complex__' in obj:
-        return complex(obj['real'], obj['imag'])
+    if b'__complex__' in obj:
+        return complex(obj[b'real'], obj[b'imag'])
     return obj
 
 def _encode_complex(obj):
     if isinstance(obj, complex):
-        return {'__complex__': True, 'real': 1, 'imag': 2}
+        return {b'__complex__': True, b'real': 1, b'imag': 2}
     return obj
 
 def test_encode_hook():
     packed = packs([3, 1+2j], default=_encode_complex)
     unpacked = unpacks(packed)
-    eq_(unpacked[1], {'__complex__': True, 'real': 1, 'imag': 2})
+    eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2})
 
 def test_decode_hook():
-    packed = packs([3, {'__complex__': True, 'real': 1, 'imag': 2}])
+    packed = packs([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
     unpacked = unpacks(packed, object_hook=_decode_complex)
     eq_(unpacked[1], 1+2j)
 
@@ -40,7 +40,4 @@ def test_array_hook():
     eq_(unpacked, '123')
 
 if __name__ == '__main__':
-    test_decode_hook()
-    test_encode_hook()
-    test_bad_hook()
-    test_array_hook()
+    main()
diff --git a/test/test_pack.py b/test/test_pack.py
index 2e1ac22..480af28 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -5,9 +5,9 @@ from nose import main
 from nose.tools import *
 from nose.plugins.skip import SkipTest
 
-from msgpack import packs, unpacks, Packer, Unpacker
+from msgpack import packs, unpacks, Unpacker, Packer
 
-from StringIO import StringIO
+from io import BytesIO
 
 def check(data):
     re = unpacks(packs(data))
@@ -18,7 +18,7 @@ def testPack():
             0, 1, 127, 128, 255, 256, 65535, 65536,
             -1, -32, -33, -128, -129, -32768, -32769,
             1.0,
-        "", "a", "a"*31, "a"*32,
+        b"", b"a", b"a"*31, b"a"*32,
         None, True, False,
         (), ((),), ((), None,),
         {None: 0},
@@ -29,20 +29,20 @@ def testPack():
 
 def testPackUnicode():
     test_data = [
-        u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚",
+        "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚",
         ]
     for td in test_data:
         re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8')
         assert_equal(re, td)
         packer = Packer(encoding='utf-8')
         data = packer.pack(td)
-        re = Unpacker(StringIO(data), encoding='utf-8').unpack()
+        re = Unpacker(BytesIO(data), encoding='utf-8').unpack()
         assert_equal(re, td)
 
 def testPackUTF32():
     try:
         test_data = [
-            u"", u"abcd", (u"defgh",), u"РуÑÑкий текÑÑ‚",
+            "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚",
             ]
         for td in test_data:
             re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32')
@@ -52,37 +52,35 @@ def testPackUTF32():
 
 def testPackBytes():
     test_data = [
-        "", "abcd", ("defgh",),
+        b"", b"abcd", (b"defgh",),
         ]
     for td in test_data:
         check(td)
 
 def testIgnoreUnicodeErrors():
-    re = unpacks(packs('abc\xeddef'),
-        encoding='ascii', unicode_errors='ignore')
+    re = unpacks(packs(b'abc\xeddef'),
+        encoding='utf-8', unicode_errors='ignore')
     assert_equal(re, "abcdef")
 
 @raises(UnicodeDecodeError)
 def testStrictUnicodeUnpack():
-    unpacks(packs('abc\xeddef'), encoding='utf-8')
+    unpacks(packs(b'abc\xeddef'), encoding='utf-8')
 
 @raises(UnicodeEncodeError)
 def testStrictUnicodePack():
-    packs(u"abc\xeddef", encoding='ascii', unicode_errors='strict')
+    packs("abc\xeddef", encoding='ascii', unicode_errors='strict')
 
 def testIgnoreErrorsPack():
-    re = unpacks(
-            packs(u"abcФФФdef", encoding='ascii', unicode_errors='ignore'),
-            encoding='utf-8')
-    assert_equal(re, u"abcdef")
+    re = unpacks(packs("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
+    assert_equal(re, "abcdef")
 
 @raises(TypeError)
 def testNoEncoding():
-    packs(u"abc", encoding=None)
+    packs("abc", encoding=None)
 
 def testDecodeBinary():
-    re = unpacks(packs(u"abc"), encoding=None)
-    assert_equal(re, "abc")
+    re = unpacks(packs("abc"), encoding=None)
+    assert_equal(re, b"abc")
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index d61be23..5fd377c 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,33 +1,35 @@
 #!/usr/bin/env python
 # coding: utf-8
 
+
+
 from msgpack import Unpacker
 
 def test_foobar():
     unpacker = Unpacker(read_size=3)
-    unpacker.feed('foobar')
-    assert unpacker.unpack() == ord('f')
-    assert unpacker.unpack() == ord('o')
-    assert unpacker.unpack() == ord('o')
-    assert unpacker.unpack() == ord('b')
-    assert unpacker.unpack() == ord('a')
-    assert unpacker.unpack() == ord('r')
+    unpacker.feed(b'foobar')
+    assert unpacker.unpack() == ord(b'f')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b'b')
+    assert unpacker.unpack() == ord(b'a')
+    assert unpacker.unpack() == ord(b'r')
     try:
         o = unpacker.unpack()
-        print "Oops!", o
+        print(("Oops!", o))
         assert 0
     except StopIteration:
         assert 1
     else:
         assert 0
-    unpacker.feed('foo')
-    unpacker.feed('bar')
+    unpacker.feed(b'foo')
+    unpacker.feed(b'bar')
 
     k = 0
-    for o, e in zip(unpacker, 'foobarbaz'):
-        assert o == ord(e)
+    for o, e in zip(unpacker, b'foobarbaz'):
+        assert o == e
         k += 1
-    assert k == len('foobar')
+    assert k == len(b'foobar')
 
 if __name__ == '__main__':
     test_foobar()
diff --git a/test3/test_buffer.py b/test3/test_buffer.py
deleted file mode 100644
index 01310a0..0000000
--- a/test3/test_buffer.py
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-from nose import main
-from nose.tools import *
-from msgpack import packb, unpackb
-
-def test_unpack_buffer():
-    from array import array
-    buf = array('b')
-    buf.fromstring(packb(('foo', 'bar')))
-    obj = unpackb(buf)
-    assert_equal((b'foo', b'bar'), obj)
-
-if __name__ == '__main__':
-    main()
diff --git a/test3/test_case.py b/test3/test_case.py
deleted file mode 100644
index 2f42316..0000000
--- a/test3/test_case.py
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-from nose import main
-from nose.tools import *
-from msgpack import packs, unpacks
-
-
-def check(length, obj):
-    v = packs(obj)
-    assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v)))
-    assert_equal(unpacks(v), obj)
-
-def test_1():
-    for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
-              -((1<<5)-1), -(1<<5)]:
-        check(1, o)
-
-def test_2():
-    for o in [1 << 7, (1 << 8) - 1,
-              -((1<<5)+1), -(1<<7)
-             ]:
-        check(2, o)
-
-def test_3():
-    for o in [1 << 8, (1 << 16) - 1,
-              -((1<<7)+1), -(1<<15)]:
-        check(3, o)
-
-def test_5():
-    for o in [1 << 16, (1 << 32) - 1,
-              -((1<<15)+1), -(1<<31)]:
-        check(5, o)
-
-def test_9():
-    for o in [1 << 32, (1 << 64) - 1,
-              -((1<<31)+1), -(1<<63),
-              1.0, 0.1, -0.1, -1.0]:
-        check(9, o)
-
-
-def check_raw(overhead, num):
-    check(num + overhead, b" " * num)
-
-def test_fixraw():
-    check_raw(1, 0)
-    check_raw(1, (1<<5) - 1)
-
-def test_raw16():
-    check_raw(3, 1<<5)
-    check_raw(3, (1<<16) - 1)
-
-def test_raw32():
-    check_raw(5, 1<<16)
-
-
-def check_array(overhead, num):
-    check(num + overhead, (None,) * num)
-
-def test_fixarray():
-    check_array(1, 0)
-    check_array(1, (1 << 4) - 1)
-
-def test_array16():
-    check_array(3, 1 << 4)
-    check_array(3, (1<<16)-1)
-
-def test_array32():
-    check_array(5, (1<<16))
-
-
-def match(obj, buf):
-    assert_equal(packs(obj), buf)
-    assert_equal(unpacks(buf), obj)
-
-def test_match():
-    cases = [
-        (None, b'\xc0'),
-        (False, b'\xc2'),
-        (True, b'\xc3'),
-        (0, b'\x00'),
-        (127, b'\x7f'),
-        (128, b'\xcc\x80'),
-        (256, b'\xcd\x01\x00'),
-        (-1, b'\xff'),
-        (-33, b'\xd0\xdf'),
-        (-129, b'\xd1\xff\x7f'),
-        ({1:1}, b'\x81\x01\x01'),
-        (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"),
-        ((), b'\x90'),
-        (tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"),
-        (tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"),
-        ({}, b'\x80'),
-        (dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'),
-        (dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'),
-        ]
-
-    for v, p in cases:
-        match(v, p)
-
-def test_unicode():
-    assert_equal(b'foobar', unpacks(packs('foobar')))
-
-if __name__ == '__main__':
-    main()
diff --git a/test3/test_except.py b/test3/test_except.py
deleted file mode 100644
index 574728f..0000000
--- a/test3/test_except.py
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-from nose.tools import *
-from msgpack import packs, unpacks
-
-import datetime
-
-def test_raise_on_find_unsupported_value():
-    assert_raises(TypeError, packs, datetime.datetime.now())
-
-if __name__ == '__main__':
-    from nose import main
-    main()
diff --git a/test3/test_format.py b/test3/test_format.py
deleted file mode 100644
index 022e680..0000000
--- a/test3/test_format.py
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-from nose import main
-from nose.tools import *
-from msgpack import unpacks
-
-def check(src, should):
-    assert_equal(unpacks(src), should)
-
-def testSimpleValue():
-    check(b"\x93\xc0\xc2\xc3", 
-            (None, False, True,))
-
-def testFixnum():
-    check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff",
-          ((0,64,127,), (-32,-16,-1,),)
-          )
-
-def testFixArray():
-    check(b"\x92\x90\x91\x91\xc0",
-          ((),((None,),),),
-          )
-
-def testFixRaw():
-    check(b"\x94\xa0\xa1a\xa2bc\xa3def",
-          (b"", b"a", b"bc", b"def",),
-          )
-
-def testFixMap():
-    check(
-          b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
-          {False: {None: None}, True:{None:{}}},
-          )
-
-def testUnsignedInt():
-    check(
-          b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
-          b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
-          b"\xce\xff\xff\xff\xff",
-          (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,),
-          )
-
-def testSignedInt():
-    check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
-          b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
-          b"\xd2\xff\xff\xff\xff",
-          (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,))
-
-def testRaw():
-    check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
-        b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
-        (b"", b"a", b"ab", b"", b"a", b"ab"))
-
-def testArray():
-    check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
-        b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02"
-        b"\xc2\xc3",
-        ((), (None,), (False,True), (), (None,), (False,True))
-        )
-
-def testMap():
-    check(
-        b"\x96"
-            b"\xde\x00\x00"
-            b"\xde\x00\x01\xc0\xc2"
-            b"\xde\x00\x02\xc0\xc2\xc3\xc2"
-            b"\xdf\x00\x00\x00\x00"
-            b"\xdf\x00\x00\x00\x01\xc0\xc2"
-            b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
-        ({}, {None: False}, {True: False, None: False}, {},
-            {None: False}, {True: False, None: False}))
-
-if __name__ == '__main__':
-    main()
diff --git a/test3/test_obj.py b/test3/test_obj.py
deleted file mode 100644
index b54021f..0000000
--- a/test3/test_obj.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-from nose import main
-from nose.tools import *
-
-from msgpack import packs, unpacks
-
-def _decode_complex(obj):
-    if b'__complex__' in obj:
-        return complex(obj[b'real'], obj[b'imag'])
-    return obj
-
-def _encode_complex(obj):
-    if isinstance(obj, complex):
-        return {b'__complex__': True, b'real': 1, b'imag': 2}
-    return obj
-
-def test_encode_hook():
-    packed = packs([3, 1+2j], default=_encode_complex)
-    unpacked = unpacks(packed)
-    eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2})
-
-def test_decode_hook():
-    packed = packs([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
-    unpacked = unpacks(packed, object_hook=_decode_complex)
-    eq_(unpacked[1], 1+2j)
-
-@raises(ValueError)
-def test_bad_hook():
-    packed = packs([3, 1+2j], default=lambda o: o)
-    unpacked = unpacks(packed)
-
-def _arr_to_str(arr):
-    return ''.join(str(c) for c in arr)
-
-def test_array_hook():
-    packed = packs([1,2,3])
-    unpacked = unpacks(packed, list_hook=_arr_to_str)
-    eq_(unpacked, '123')
-
-if __name__ == '__main__':
-    #main()
-    test_decode_hook()
diff --git a/test3/test_pack.py b/test3/test_pack.py
deleted file mode 100644
index 5ff04e7..0000000
--- a/test3/test_pack.py
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-from nose import main
-from nose.tools import *
-
-from msgpack import packs, unpacks, Unpacker, Packer
-
-from io import BytesIO
-
-def check(data):
-    re = unpacks(packs(data))
-    assert_equal(re, data)
-
-def testPack():
-    test_data = [
-            0, 1, 127, 128, 255, 256, 65535, 65536,
-            -1, -32, -33, -128, -129, -32768, -32769,
-            1.0,
-        b"", b"a", b"a"*31, b"a"*32,
-        None, True, False,
-        (), ((),), ((), None,),
-        {None: 0},
-        (1<<23),
-        ]
-    for td in test_data:
-        check(td)
-
-def testPackUnicode():
-    test_data = [
-        "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚",
-        ]
-    for td in test_data:
-        re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8')
-        assert_equal(re, td)
-        packer = Packer(encoding='utf-8')
-        data = packer.pack(td)
-        re = Unpacker(BytesIO(data), encoding='utf-8').unpack()
-        assert_equal(re, td)
-
-def testPackUTF32():
-    test_data = [
-        "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚",
-        ]
-    for td in test_data:
-        re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32')
-        assert_equal(re, td)
-
-def testPackBytes():
-    test_data = [
-        b"", b"abcd", (b"defgh",),
-        ]
-    for td in test_data:
-        check(td)
-
-def testIgnoreUnicodeErrors():
-    re = unpacks(packs(b'abc\xeddef'),
-        encoding='utf-8', unicode_errors='ignore')
-    assert_equal(re, "abcdef")
-
-@raises(UnicodeDecodeError)
-def testStrictUnicodeUnpack():
-    unpacks(packs(b'abc\xeddef'), encoding='utf-8')
-
-@raises(UnicodeEncodeError)
-def testStrictUnicodePack():
-    packs("abc\xeddef", encoding='ascii', unicode_errors='strict')
-
-def testIgnoreErrorsPack():
-    re = unpacks(packs("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
-    assert_equal(re, "abcdef")
-
-@raises(TypeError)
-def testNoEncoding():
-    packs("abc", encoding=None)
-
-def testDecodeBinary():
-    re = unpacks(packs("abc"), encoding=None)
-    assert_equal(re, b"abc")
-
-if __name__ == '__main__':
-    main()
diff --git a/test3/test_sequnpack.py b/test3/test_sequnpack.py
deleted file mode 100644
index 5fd377c..0000000
--- a/test3/test_sequnpack.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-
-
-from msgpack import Unpacker
-
-def test_foobar():
-    unpacker = Unpacker(read_size=3)
-    unpacker.feed(b'foobar')
-    assert unpacker.unpack() == ord(b'f')
-    assert unpacker.unpack() == ord(b'o')
-    assert unpacker.unpack() == ord(b'o')
-    assert unpacker.unpack() == ord(b'b')
-    assert unpacker.unpack() == ord(b'a')
-    assert unpacker.unpack() == ord(b'r')
-    try:
-        o = unpacker.unpack()
-        print(("Oops!", o))
-        assert 0
-    except StopIteration:
-        assert 1
-    else:
-        assert 0
-    unpacker.feed(b'foo')
-    unpacker.feed(b'bar')
-
-    k = 0
-    for o, e in zip(unpacker, b'foobarbaz'):
-        assert o == e
-        k += 1
-    assert k == len(b'foobar')
-
-if __name__ == '__main__':
-    test_foobar()
-
diff --git a/tox.ini b/tox.ini
index c0a5629..276e16a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,7 +3,5 @@ envlist = py26,py27,py32
 [testenv]
 deps=
     nose
-commands=nosetests -w test
 
-[testenv:py32]
-commands=nosetests -w test3
+commands=nosetests -w test

From 636f4529aa8426f4ee3e25e0b4931bb37384df1e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 19 Jun 2012 14:20:56 +0900
Subject: [PATCH 0998/1648] Fix tests to pass.

---
 test/test_pack.py      | 16 ++++++++++------
 test/test_seq.py       | 19 ++++++++++---------
 test/test_sequnpack.py | 14 +++++---------
 tox.ini                |  1 +
 4 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/test/test_pack.py b/test/test_pack.py
index 480af28..88b6dac 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # coding: utf-8
 
+import six
 from nose import main
 from nose.tools import *
 from nose.plugins.skip import SkipTest
@@ -29,7 +30,7 @@ def testPack():
 
 def testPackUnicode():
     test_data = [
-        "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚",
+        six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"),
         ]
     for td in test_data:
         re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8')
@@ -42,7 +43,10 @@ def testPackUnicode():
 def testPackUTF32():
     try:
         test_data = [
-            "", "abcd", ("defgh",), "РуÑÑкий текÑÑ‚",
+            six.u(""),
+            six.u("abcd"),
+            (six.u("defgh"),),
+            six.u("РуÑÑкий текÑÑ‚"),
             ]
         for td in test_data:
             re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32')
@@ -68,15 +72,15 @@ def testStrictUnicodeUnpack():
 
 @raises(UnicodeEncodeError)
 def testStrictUnicodePack():
-    packs("abc\xeddef", encoding='ascii', unicode_errors='strict')
+    packs(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict')
 
 def testIgnoreErrorsPack():
-    re = unpacks(packs("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
-    assert_equal(re, "abcdef")
+    re = unpacks(packs(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
+    assert_equal(re, six.u("abcdef"))
 
 @raises(TypeError)
 def testNoEncoding():
-    packs("abc", encoding=None)
+    packs(six.u("abc"), encoding=None)
 
 def testDecodeBinary():
     re = unpacks(packs("abc"), encoding=None)
diff --git a/test/test_seq.py b/test/test_seq.py
index 993a59e..d0f9ccc 100644
--- a/test/test_seq.py
+++ b/test/test_seq.py
@@ -1,21 +1,22 @@
 #!/usr/bin/env python
 # coding: utf-8
 
+import six
 from nose import main
 from nose.tools import *
 
-import StringIO
+import io
 import msgpack
 
-binarydata = [chr(i) for i in xrange(256)]
-binarydata = "".join(binarydata)
+binarydata = [chr(i) for i in range(256)]
+binarydata = six.b("".join(binarydata))
 
 def gen_binary_data(idx):
     data = binarydata[:idx % 300]
     return data
 
 def test_exceeding_unpacker_read_size():
-    dumpf = StringIO.StringIO()
+    dumpf = io.BytesIO()
 
     packer = msgpack.Packer()
 
@@ -26,18 +27,18 @@ def test_exceeding_unpacker_read_size():
                                 # 40 ok for read_size=1024, while 50 introduces errors
                                 # 7000 ok for read_size=1024*1024, while 8000 leads to  glibc detected *** python: double free or corruption (!prev):
 
-    for idx in xrange(NUMBER_OF_STRINGS):
+    for idx in range(NUMBER_OF_STRINGS):
         data = gen_binary_data(idx)
         dumpf.write(packer.pack(data))
 
-    f = StringIO.StringIO(dumpf.getvalue())
+    f = io.BytesIO(dumpf.getvalue())
     dumpf.close()
 
     unpacker = msgpack.Unpacker(f, read_size=read_size)
 
     read_count = 0
     for idx, o in enumerate(unpacker):
-        assert_equal(type(o), str)
+        assert_equal(type(o), bytes)
         assert_equal(o, gen_binary_data(idx))
         read_count += 1
 
@@ -45,5 +46,5 @@ def test_exceeding_unpacker_read_size():
 
 
 if __name__ == '__main__':
-    # main()
-    test_exceeding_unpacker_read_size()
+    main()
+    #test_exceeding_unpacker_read_size()
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 5fd377c..774fe1b 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-
-
 from msgpack import Unpacker
 
 def test_foobar():
@@ -16,18 +14,16 @@ def test_foobar():
     assert unpacker.unpack() == ord(b'r')
     try:
         o = unpacker.unpack()
-        print(("Oops!", o))
-        assert 0
+        assert 0, "should raise exception"
     except StopIteration:
-        assert 1
-    else:
-        assert 0
+        assert 1, "ok"
+
     unpacker.feed(b'foo')
     unpacker.feed(b'bar')
 
     k = 0
-    for o, e in zip(unpacker, b'foobarbaz'):
-        assert o == e
+    for o, e in zip(unpacker, 'foobarbaz'):
+        assert o == ord(e)
         k += 1
     assert k == len(b'foobar')
 
diff --git a/tox.ini b/tox.ini
index 276e16a..5e80dd0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,5 +3,6 @@ envlist = py26,py27,py32
 [testenv]
 deps=
     nose
+    six
 
 commands=nosetests -w test

From 5b0353eac610a23742d80f2f243884364e14f67d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 19 Jun 2012 14:31:32 +0900
Subject: [PATCH 0999/1648] Start 0.1.14dev

---
 ChangeLog.rst | 13 +++++++++++++
 setup.py      |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index e0a4fd1..3a9277c 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,16 @@
+0.1.14
+=======
+:release date: NOT RELEASED YET
+
+Changes
+-------
+* Drop supporting Python 2.5 and unify tests for Py2 and Py3.
+
+Bugs fixed
+----------
+* #8 Packing subclass of dict raises TypeError. (Thanks to Steeve Morin.)
+
+
 0.1.13
 =======
 :release date: 2012-04-21
diff --git a/setup.py b/setup.py
index b4b1771..115d3b8 100755
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 13)
+version = (0, 1, 14, 'dev')
 
 import os
 import sys

From 812c8bcff42a5b77f9ddfce70289bf754cca108c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Jun 2012 13:21:10 +0900
Subject: [PATCH 1000/1648] Update msgpack version.

---
 msgpack/pack_define.h     |   5 +-
 msgpack/pack_template.h   | 247 +++++++++++++++++++++++++-------------
 msgpack/sysdep.h          | 137 ++++++++++++++++++---
 msgpack/unpack_define.h   |   9 +-
 msgpack/unpack_template.h | 127 ++++++++++++--------
 5 files changed, 370 insertions(+), 155 deletions(-)

diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h
index f72391b..4845d52 100644
--- a/msgpack/pack_define.h
+++ b/msgpack/pack_define.h
@@ -1,7 +1,7 @@
 /*
  * MessagePack unpacking routine template
  *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
  *
  *    Licensed under the Apache License, Version 2.0 (the "License");
  *    you may not use this file except in compliance with the License.
@@ -18,8 +18,9 @@
 #ifndef MSGPACK_PACK_DEFINE_H__
 #define MSGPACK_PACK_DEFINE_H__
 
-#include "sysdep.h"
+#include "msgpack/sysdep.h"
 #include 
+#include 
 
 #endif /* msgpack/pack_define.h */
 
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 4b50895..65c959d 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -1,7 +1,7 @@
 /*
  * MessagePack packing routine template
  *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
  *
  *    Licensed under the Apache License, Version 2.0 (the "License");
  *    you may not use this file except in compliance with the License.
@@ -16,12 +16,12 @@
  *    limitations under the License.
  */
 
-#ifdef __LITTLE_ENDIAN__
+#if defined(__LITTLE_ENDIAN__)
 #define TAKE8_8(d)  ((uint8_t*)&d)[0]
 #define TAKE8_16(d) ((uint8_t*)&d)[0]
 #define TAKE8_32(d) ((uint8_t*)&d)[0]
 #define TAKE8_64(d) ((uint8_t*)&d)[0]
-#elif __BIG_ENDIAN__
+#elif defined(__BIG_ENDIAN__)
 #define TAKE8_8(d)  ((uint8_t*)&d)[0]
 #define TAKE8_16(d) ((uint8_t*)&d)[1]
 #define TAKE8_32(d) ((uint8_t*)&d)[3]
@@ -69,7 +69,7 @@ do { \
 	} else { \
 		/* unsigned 16 */ \
 		unsigned char buf[3]; \
-		buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
+		buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 		msgpack_pack_append_buffer(x, buf, 3); \
 	} \
 } while(0)
@@ -89,12 +89,12 @@ do { \
 		if(d < (1<<16)) { \
 			/* unsigned 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* unsigned 32 */ \
 			unsigned char buf[5]; \
-			buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
+			buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} \
 	} \
@@ -103,7 +103,7 @@ do { \
 #define msgpack_pack_real_uint64(x, d) \
 do { \
 	if(d < (1ULL<<8)) { \
-		if(d < (1<<7)) { \
+		if(d < (1ULL<<7)) { \
 			/* fixnum */ \
 			msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
 		} else { \
@@ -113,19 +113,19 @@ do { \
 		} \
 	} else { \
 		if(d < (1ULL<<16)) { \
-			/* signed 16 */ \
+			/* unsigned 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else if(d < (1ULL<<32)) { \
-			/* signed 32 */ \
+			/* unsigned 32 */ \
 			unsigned char buf[5]; \
-			buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
+			buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} else { \
-			/* signed 64 */ \
+			/* unsigned 64 */ \
 			unsigned char buf[9]; \
-			buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \
+			buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
 			msgpack_pack_append_buffer(x, buf, 9); \
 		} \
 	} \
@@ -149,7 +149,7 @@ do { \
 		if(d < -(1<<7)) { \
 			/* signed 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
+			buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* signed 8 */ \
@@ -167,7 +167,7 @@ do { \
 		} else { \
 			/* unsigned 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} \
 	} \
@@ -179,12 +179,12 @@ do { \
 		if(d < -(1<<15)) { \
 			/* signed 32 */ \
 			unsigned char buf[5]; \
-			buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
+			buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} else if(d < -(1<<7)) { \
 			/* signed 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
+			buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* signed 8 */ \
@@ -202,12 +202,12 @@ do { \
 		} else if(d < (1<<16)) { \
 			/* unsigned 16 */ \
 			unsigned char buf[3]; \
-			buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
+			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 			msgpack_pack_append_buffer(x, buf, 3); \
 		} else { \
 			/* unsigned 32 */ \
 			unsigned char buf[5]; \
-			buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
+			buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
 			msgpack_pack_append_buffer(x, buf, 5); \
 		} \
 	} \
@@ -220,19 +220,19 @@ do { \
 			if(d < -(1LL<<31)) { \
 				/* signed 64 */ \
 				unsigned char buf[9]; \
-				buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d); \
+				buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \
 				msgpack_pack_append_buffer(x, buf, 9); \
 			} else { \
 				/* signed 32 */ \
 				unsigned char buf[5]; \
-				buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
+				buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
 				msgpack_pack_append_buffer(x, buf, 5); \
 			} \
 		} else { \
 			if(d < -(1<<7)) { \
 				/* signed 16 */ \
 				unsigned char buf[3]; \
-				buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
+				buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
 				msgpack_pack_append_buffer(x, buf, 3); \
 			} else { \
 				/* signed 8 */ \
@@ -252,19 +252,19 @@ do { \
 			} else { \
 				/* unsigned 16 */ \
 				unsigned char buf[3]; \
-				buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d); \
+				buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
 				msgpack_pack_append_buffer(x, buf, 3); \
 			} \
 		} else { \
 			if(d < (1LL<<32)) { \
 				/* unsigned 32 */ \
 				unsigned char buf[5]; \
-				buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d); \
+				buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
 				msgpack_pack_append_buffer(x, buf, 5); \
 			} else { \
 				/* unsigned 64 */ \
 				unsigned char buf[9]; \
-				buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d); \
+				buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
 				msgpack_pack_append_buffer(x, buf, 9); \
 			} \
 		} \
@@ -272,63 +272,63 @@ do { \
 } while(0)
 
 
-#ifdef msgpack_pack_inline_func_fastint
+#ifdef msgpack_pack_inline_func_fixint
 
-msgpack_pack_inline_func_fastint(_uint8)(msgpack_pack_user x, uint8_t d)
+msgpack_pack_inline_func_fixint(_uint8)(msgpack_pack_user x, uint8_t d)
 {
 	unsigned char buf[2] = {0xcc, TAKE8_8(d)};
 	msgpack_pack_append_buffer(x, buf, 2);
 }
 
-msgpack_pack_inline_func_fastint(_uint16)(msgpack_pack_user x, uint16_t d)
+msgpack_pack_inline_func_fixint(_uint16)(msgpack_pack_user x, uint16_t d)
 {
 	unsigned char buf[3];
-	buf[0] = 0xcd; *(uint16_t*)&buf[1] = _msgpack_be16(d);
+	buf[0] = 0xcd; _msgpack_store16(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 3);
 }
 
-msgpack_pack_inline_func_fastint(_uint32)(msgpack_pack_user x, uint32_t d)
+msgpack_pack_inline_func_fixint(_uint32)(msgpack_pack_user x, uint32_t d)
 {
 	unsigned char buf[5];
-	buf[0] = 0xce; *(uint32_t*)&buf[1] = _msgpack_be32(d);
+	buf[0] = 0xce; _msgpack_store32(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 5);
 }
 
-msgpack_pack_inline_func_fastint(_uint64)(msgpack_pack_user x, uint64_t d)
+msgpack_pack_inline_func_fixint(_uint64)(msgpack_pack_user x, uint64_t d)
 {
 	unsigned char buf[9];
-	buf[0] = 0xcf; *(uint64_t*)&buf[1] = _msgpack_be64(d);
+	buf[0] = 0xcf; _msgpack_store64(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 9);
 }
 
-msgpack_pack_inline_func_fastint(_int8)(msgpack_pack_user x, int8_t d)
+msgpack_pack_inline_func_fixint(_int8)(msgpack_pack_user x, int8_t d)
 {
 	unsigned char buf[2] = {0xd0, TAKE8_8(d)};
 	msgpack_pack_append_buffer(x, buf, 2);
 }
 
-msgpack_pack_inline_func_fastint(_int16)(msgpack_pack_user x, int16_t d)
+msgpack_pack_inline_func_fixint(_int16)(msgpack_pack_user x, int16_t d)
 {
 	unsigned char buf[3];
-	buf[0] = 0xd1; *(uint16_t*)&buf[1] = _msgpack_be16(d);
+	buf[0] = 0xd1; _msgpack_store16(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 3);
 }
 
-msgpack_pack_inline_func_fastint(_int32)(msgpack_pack_user x, int32_t d)
+msgpack_pack_inline_func_fixint(_int32)(msgpack_pack_user x, int32_t d)
 {
 	unsigned char buf[5];
-	buf[0] = 0xd2; *(uint32_t*)&buf[1] = _msgpack_be32(d);
+	buf[0] = 0xd2; _msgpack_store32(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 5);
 }
 
-msgpack_pack_inline_func_fastint(_int64)(msgpack_pack_user x, int64_t d)
+msgpack_pack_inline_func_fixint(_int64)(msgpack_pack_user x, int64_t d)
 {
 	unsigned char buf[9];
-	buf[0] = 0xd3; *(uint64_t*)&buf[1] = _msgpack_be64(d);
+	buf[0] = 0xd3; _msgpack_store64(&buf[1], d);
 	msgpack_pack_append_buffer(x, buf, 9);
 }
 
-#undef msgpack_pack_inline_func_fastint
+#undef msgpack_pack_inline_func_fixint
 #endif
 
 
@@ -377,14 +377,24 @@ msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d)
 
 msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d)
 {
-#if defined(SIZEOF_SHORT) || defined(SHRT_MAX)
-#if SIZEOF_SHORT == 2 || SHRT_MAX == 0x7fff
+#if defined(SIZEOF_SHORT)
+#if SIZEOF_SHORT == 2
 	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_SHORT == 4 || SHRT_MAX == 0x7fffffff
+#elif SIZEOF_SHORT == 4
 	msgpack_pack_real_int32(x, d);
 #else
 	msgpack_pack_real_int64(x, d);
 #endif
+
+#elif defined(SHRT_MAX)
+#if SHRT_MAX == 0x7fff
+	msgpack_pack_real_int16(x, d);
+#elif SHRT_MAX == 0x7fffffff
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
 #else
 if(sizeof(short) == 2) {
 	msgpack_pack_real_int16(x, d);
@@ -398,14 +408,24 @@ if(sizeof(short) == 2) {
 
 msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d)
 {
-#if defined(SIZEOF_INT) || defined(INT_MAX)
-#if SIZEOF_INT == 2 || INT_MAX == 0x7fff
+#if defined(SIZEOF_INT)
+#if SIZEOF_INT == 2
 	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_INT == 4 || INT_MAX == 0x7fffffff
+#elif SIZEOF_INT == 4
 	msgpack_pack_real_int32(x, d);
 #else
 	msgpack_pack_real_int64(x, d);
 #endif
+
+#elif defined(INT_MAX)
+#if INT_MAX == 0x7fff
+	msgpack_pack_real_int16(x, d);
+#elif INT_MAX == 0x7fffffff
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
 #else
 if(sizeof(int) == 2) {
 	msgpack_pack_real_int16(x, d);
@@ -419,14 +439,24 @@ if(sizeof(int) == 2) {
 
 msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d)
 {
-#if defined(SIZEOF_LONG) || defined(LONG_MAX)
-#if SIZEOF_LONG == 2 || LONG_MAX == 0x7fffL
+#if defined(SIZEOF_LONG)
+#if SIZEOF_LONG == 2
 	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_LONG == 4 || LONG_MAX == 0x7fffffffL
+#elif SIZEOF_LONG == 4
 	msgpack_pack_real_int32(x, d);
 #else
 	msgpack_pack_real_int64(x, d);
 #endif
+
+#elif defined(LONG_MAX)
+#if LONG_MAX == 0x7fffL
+	msgpack_pack_real_int16(x, d);
+#elif LONG_MAX == 0x7fffffffL
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
 #else
 if(sizeof(long) == 2) {
 	msgpack_pack_real_int16(x, d);
@@ -440,14 +470,24 @@ if(sizeof(long) == 2) {
 
 msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d)
 {
-#if defined(SIZEOF_LONG_LONG) || defined(LLONG_MAX)
-#if SIZEOF_LONG_LONG == 2 || LLONG_MAX == 0x7fffL
+#if defined(SIZEOF_LONG_LONG)
+#if SIZEOF_LONG_LONG == 2
 	msgpack_pack_real_int16(x, d);
-#elif SIZEOF_LONG_LONG == 4 || LLONG_MAX == 0x7fffffffL
+#elif SIZEOF_LONG_LONG == 4
 	msgpack_pack_real_int32(x, d);
 #else
 	msgpack_pack_real_int64(x, d);
 #endif
+
+#elif defined(LLONG_MAX)
+#if LLONG_MAX == 0x7fffL
+	msgpack_pack_real_int16(x, d);
+#elif LLONG_MAX == 0x7fffffffL
+	msgpack_pack_real_int32(x, d);
+#else
+	msgpack_pack_real_int64(x, d);
+#endif
+
 #else
 if(sizeof(long long) == 2) {
 	msgpack_pack_real_int16(x, d);
@@ -461,14 +501,24 @@ if(sizeof(long long) == 2) {
 
 msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d)
 {
-#if defined(SIZEOF_SHORT) || defined(USHRT_MAX)
-#if SIZEOF_SHORT == 2 || USHRT_MAX == 0xffffU
+#if defined(SIZEOF_SHORT)
+#if SIZEOF_SHORT == 2
 	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_SHORT == 4 || USHRT_MAX == 0xffffffffU
+#elif SIZEOF_SHORT == 4
 	msgpack_pack_real_uint32(x, d);
 #else
 	msgpack_pack_real_uint64(x, d);
 #endif
+
+#elif defined(USHRT_MAX)
+#if USHRT_MAX == 0xffffU
+	msgpack_pack_real_uint16(x, d);
+#elif USHRT_MAX == 0xffffffffU
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
 #else
 if(sizeof(unsigned short) == 2) {
 	msgpack_pack_real_uint16(x, d);
@@ -482,14 +532,24 @@ if(sizeof(unsigned short) == 2) {
 
 msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d)
 {
-#if defined(SIZEOF_INT) || defined(UINT_MAX)
-#if SIZEOF_INT == 2 || UINT_MAX == 0xffffU
+#if defined(SIZEOF_INT)
+#if SIZEOF_INT == 2
 	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_INT == 4 || UINT_MAX == 0xffffffffU
+#elif SIZEOF_INT == 4
 	msgpack_pack_real_uint32(x, d);
 #else
 	msgpack_pack_real_uint64(x, d);
 #endif
+
+#elif defined(UINT_MAX)
+#if UINT_MAX == 0xffffU
+	msgpack_pack_real_uint16(x, d);
+#elif UINT_MAX == 0xffffffffU
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
 #else
 if(sizeof(unsigned int) == 2) {
 	msgpack_pack_real_uint16(x, d);
@@ -503,18 +563,28 @@ if(sizeof(unsigned int) == 2) {
 
 msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d)
 {
-#if defined(SIZEOF_LONG) || defined(ULONG_MAX)
-#if SIZEOF_LONG == 2 || ULONG_MAX == 0xffffUL
+#if defined(SIZEOF_LONG)
+#if SIZEOF_LONG == 2
 	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_LONG == 4 || ULONG_MAX == 0xffffffffUL
+#elif SIZEOF_LONG == 4
 	msgpack_pack_real_uint32(x, d);
 #else
 	msgpack_pack_real_uint64(x, d);
 #endif
-#else
-if(sizeof(unsigned int) == 2) {
+
+#elif defined(ULONG_MAX)
+#if ULONG_MAX == 0xffffUL
 	msgpack_pack_real_uint16(x, d);
-} else if(sizeof(unsigned int) == 4) {
+#elif ULONG_MAX == 0xffffffffUL
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
+#else
+if(sizeof(unsigned long) == 2) {
+	msgpack_pack_real_uint16(x, d);
+} else if(sizeof(unsigned long) == 4) {
 	msgpack_pack_real_uint32(x, d);
 } else {
 	msgpack_pack_real_uint64(x, d);
@@ -524,14 +594,24 @@ if(sizeof(unsigned int) == 2) {
 
 msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d)
 {
-#if defined(SIZEOF_LONG_LONG) || defined(ULLONG_MAX)
-#if SIZEOF_LONG_LONG == 2 || ULLONG_MAX == 0xffffUL
+#if defined(SIZEOF_LONG_LONG)
+#if SIZEOF_LONG_LONG == 2
 	msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_LONG_LONG == 4 || ULLONG_MAX == 0xffffffffUL
+#elif SIZEOF_LONG_LONG == 4
 	msgpack_pack_real_uint32(x, d);
 #else
 	msgpack_pack_real_uint64(x, d);
 #endif
+
+#elif defined(ULLONG_MAX)
+#if ULLONG_MAX == 0xffffUL
+	msgpack_pack_real_uint16(x, d);
+#elif ULLONG_MAX == 0xffffffffUL
+	msgpack_pack_real_uint32(x, d);
+#else
+	msgpack_pack_real_uint64(x, d);
+#endif
+
 #else
 if(sizeof(unsigned long long) == 2) {
 	msgpack_pack_real_uint16(x, d);
@@ -554,19 +634,24 @@ if(sizeof(unsigned long long) == 2) {
 
 msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d)
 {
-	union { char buf[4]; uint32_t num; } f;
+	union { float f; uint32_t i; } mem;
+	mem.f = d;
 	unsigned char buf[5];
-	*((float*)&f.buf) = d;  // FIXME
-	buf[0] = 0xca; *(uint32_t*)&buf[1] = _msgpack_be32(f.num);
+	buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i);
 	msgpack_pack_append_buffer(x, buf, 5);
 }
 
 msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d)
 {
-	union { char buf[8]; uint64_t num; } f;
+	union { double f; uint64_t i; } mem;
+	mem.f = d;
 	unsigned char buf[9];
-	*((double*)&f.buf) = d;  // FIXME
-	buf[0] = 0xcb; *(uint64_t*)&buf[1] = _msgpack_be64(f.num);
+	buf[0] = 0xcb;
+#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
+    // https://github.com/msgpack/msgpack-perl/pull/1
+    mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
+#endif
+    _msgpack_store64(&buf[1], mem.i);
 	msgpack_pack_append_buffer(x, buf, 9);
 }
 
@@ -610,11 +695,11 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n)
 		msgpack_pack_append_buffer(x, &d, 1);
 	} else if(n < 65536) {
 		unsigned char buf[3];
-		buf[0] = 0xdc; *(uint16_t*)&buf[1] = _msgpack_be16(n);
+		buf[0] = 0xdc; _msgpack_store16(&buf[1], (uint16_t)n);
 		msgpack_pack_append_buffer(x, buf, 3);
 	} else {
 		unsigned char buf[5];
-		buf[0] = 0xdd; *(uint32_t*)&buf[1] = _msgpack_be32(n);
+		buf[0] = 0xdd; _msgpack_store32(&buf[1], (uint32_t)n);
 		msgpack_pack_append_buffer(x, buf, 5);
 	}
 }
@@ -631,11 +716,11 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n)
 		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
 	} else if(n < 65536) {
 		unsigned char buf[3];
-		buf[0] = 0xde; *(uint16_t*)&buf[1] = _msgpack_be16(n);
+		buf[0] = 0xde; _msgpack_store16(&buf[1], (uint16_t)n);
 		msgpack_pack_append_buffer(x, buf, 3);
 	} else {
 		unsigned char buf[5];
-		buf[0] = 0xdf; *(uint32_t*)&buf[1] = _msgpack_be32(n);
+		buf[0] = 0xdf; _msgpack_store32(&buf[1], (uint32_t)n);
 		msgpack_pack_append_buffer(x, buf, 5);
 	}
 }
@@ -648,15 +733,15 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n)
 msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l)
 {
 	if(l < 32) {
-		unsigned char d = 0xa0 | l;
+		unsigned char d = 0xa0 | (uint8_t)l;
 		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
 	} else if(l < 65536) {
 		unsigned char buf[3];
-		buf[0] = 0xda; *(uint16_t*)&buf[1] = _msgpack_be16(l);
+		buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l);
 		msgpack_pack_append_buffer(x, buf, 3);
 	} else {
 		unsigned char buf[5];
-		buf[0] = 0xdb; *(uint32_t*)&buf[1] = _msgpack_be32(l);
+		buf[0] = 0xdb; _msgpack_store32(&buf[1], (uint32_t)l);
 		msgpack_pack_append_buffer(x, buf, 5);
 	}
 }
diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h
index 106158e..4fedbd8 100644
--- a/msgpack/sysdep.h
+++ b/msgpack/sysdep.h
@@ -1,7 +1,7 @@
 /*
  * MessagePack system dependencies
  *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
  *
  *    Licensed under the Apache License, Version 2.0 (the "License");
  *    you may not use this file except in compliance with the License.
@@ -18,8 +18,9 @@
 #ifndef MSGPACK_SYSDEP_H__
 #define MSGPACK_SYSDEP_H__
 
-
-#ifdef _MSC_VER
+#include 
+#include 
+#if defined(_MSC_VER) && _MSC_VER < 1600
 typedef __int8 int8_t;
 typedef unsigned __int8 uint8_t;
 typedef __int16 int16_t;
@@ -28,26 +29,38 @@ typedef __int32 int32_t;
 typedef unsigned __int32 uint32_t;
 typedef __int64 int64_t;
 typedef unsigned __int64 uint64_t;
+#elif defined(_MSC_VER)  // && _MSC_VER >= 1600
+#include 
 #else
-#include 
 #include 
 #include 
 #endif
 
-
 #ifdef _WIN32
+#define _msgpack_atomic_counter_header 
 typedef long _msgpack_atomic_counter_t;
 #define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr)
 #define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr)
+#elif defined(__GNUC__) && ((__GNUC__*10 + __GNUC_MINOR__) < 41)
+#define _msgpack_atomic_counter_header "gcc_atomic.h"
 #else
 typedef unsigned int _msgpack_atomic_counter_t;
 #define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1)
 #define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1)
 #endif
 
-
 #ifdef _WIN32
-#include 
+
+#ifdef __cplusplus
+/* numeric_limits::min,max */
+#ifdef max
+#undef max
+#endif
+#ifdef min
+#undef min
+#endif
+#endif
+
 #else
 #include   /* __BYTE_ORDER */
 #endif
@@ -57,15 +70,45 @@ typedef unsigned int _msgpack_atomic_counter_t;
 #define __LITTLE_ENDIAN__
 #elif __BYTE_ORDER == __BIG_ENDIAN
 #define __BIG_ENDIAN__
+#elif _WIN32
+#define __LITTLE_ENDIAN__
 #endif
 #endif
 
+
 #ifdef __LITTLE_ENDIAN__
 
-#define _msgpack_be16(x) ntohs(x)
-#define _msgpack_be32(x) ntohl(x)
+#ifdef _WIN32
+#  if defined(ntohs)
+#    define _msgpack_be16(x) ntohs(x)
+#  elif defined(_byteswap_ushort) || (defined(_MSC_VER) && _MSC_VER >= 1400)
+#    define _msgpack_be16(x) ((uint16_t)_byteswap_ushort((unsigned short)x))
+#  else
+#    define _msgpack_be16(x) ( \
+        ((((uint16_t)x) <<  8) ) | \
+        ((((uint16_t)x) >>  8) ) )
+#  endif
+#else
+#  define _msgpack_be16(x) ntohs(x)
+#endif
 
-#if defined(_byteswap_uint64)
+#ifdef _WIN32
+#  if defined(ntohl)
+#    define _msgpack_be32(x) ntohl(x)
+#  elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400)
+#    define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x))
+#  else
+#    define _msgpack_be32(x) \
+        ( ((((uint32_t)x) << 24)               ) | \
+          ((((uint32_t)x) <<  8) & 0x00ff0000U ) | \
+          ((((uint32_t)x) >>  8) & 0x0000ff00U ) | \
+          ((((uint32_t)x) >> 24)               ) )
+#  endif
+#else
+#  define _msgpack_be32(x) ntohl(x)
+#endif
+
+#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400)
 #  define _msgpack_be64(x) (_byteswap_uint64(x))
 #elif defined(bswap_64)
 #  define _msgpack_be64(x) bswap_64(x)
@@ -73,22 +116,80 @@ typedef unsigned int _msgpack_atomic_counter_t;
 #  define _msgpack_be64(x) __DARWIN_OSSwapInt64(x)
 #else
 #define _msgpack_be64(x) \
-	( ((((uint64_t)x) << 56) & 0xff00000000000000ULL ) | \
-	  ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \
-	  ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \
-	  ((((uint64_t)x) <<  8) & 0x000000ff00000000ULL ) | \
-	  ((((uint64_t)x) >>  8) & 0x00000000ff000000ULL ) | \
-	  ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \
-	  ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \
-	  ((((uint64_t)x) >> 56) & 0x00000000000000ffULL ) )
+    ( ((((uint64_t)x) << 56)                         ) | \
+      ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \
+      ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \
+      ((((uint64_t)x) <<  8) & 0x000000ff00000000ULL ) | \
+      ((((uint64_t)x) >>  8) & 0x00000000ff000000ULL ) | \
+      ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \
+      ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \
+      ((((uint64_t)x) >> 56)                         ) )
 #endif
 
+#define _msgpack_load16(cast, from) ((cast)( \
+        (((uint16_t)((uint8_t*)(from))[0]) << 8) | \
+        (((uint16_t)((uint8_t*)(from))[1])     ) ))
+
+#define _msgpack_load32(cast, from) ((cast)( \
+        (((uint32_t)((uint8_t*)(from))[0]) << 24) | \
+        (((uint32_t)((uint8_t*)(from))[1]) << 16) | \
+        (((uint32_t)((uint8_t*)(from))[2]) <<  8) | \
+        (((uint32_t)((uint8_t*)(from))[3])      ) ))
+
+#define _msgpack_load64(cast, from) ((cast)( \
+        (((uint64_t)((uint8_t*)(from))[0]) << 56) | \
+        (((uint64_t)((uint8_t*)(from))[1]) << 48) | \
+        (((uint64_t)((uint8_t*)(from))[2]) << 40) | \
+        (((uint64_t)((uint8_t*)(from))[3]) << 32) | \
+        (((uint64_t)((uint8_t*)(from))[4]) << 24) | \
+        (((uint64_t)((uint8_t*)(from))[5]) << 16) | \
+        (((uint64_t)((uint8_t*)(from))[6]) << 8)  | \
+        (((uint64_t)((uint8_t*)(from))[7])     )  ))
+
 #else
+
 #define _msgpack_be16(x) (x)
 #define _msgpack_be32(x) (x)
 #define _msgpack_be64(x) (x)
+
+#define _msgpack_load16(cast, from) ((cast)( \
+        (((uint16_t)((uint8_t*)from)[0]) << 8) | \
+        (((uint16_t)((uint8_t*)from)[1])     ) ))
+
+#define _msgpack_load32(cast, from) ((cast)( \
+        (((uint32_t)((uint8_t*)from)[0]) << 24) | \
+        (((uint32_t)((uint8_t*)from)[1]) << 16) | \
+        (((uint32_t)((uint8_t*)from)[2]) <<  8) | \
+        (((uint32_t)((uint8_t*)from)[3])      ) ))
+
+#define _msgpack_load64(cast, from) ((cast)( \
+        (((uint64_t)((uint8_t*)from)[0]) << 56) | \
+        (((uint64_t)((uint8_t*)from)[1]) << 48) | \
+        (((uint64_t)((uint8_t*)from)[2]) << 40) | \
+        (((uint64_t)((uint8_t*)from)[3]) << 32) | \
+        (((uint64_t)((uint8_t*)from)[4]) << 24) | \
+        (((uint64_t)((uint8_t*)from)[5]) << 16) | \
+        (((uint64_t)((uint8_t*)from)[6]) << 8)  | \
+        (((uint64_t)((uint8_t*)from)[7])     )  ))
 #endif
 
 
+#define _msgpack_store16(to, num) \
+    do { uint16_t val = _msgpack_be16(num); memcpy(to, &val, 2); } while(0)
+#define _msgpack_store32(to, num) \
+    do { uint32_t val = _msgpack_be32(num); memcpy(to, &val, 4); } while(0)
+#define _msgpack_store64(to, num) \
+    do { uint64_t val = _msgpack_be64(num); memcpy(to, &val, 8); } while(0)
+
+/*
+#define _msgpack_load16(cast, from) \
+    ({ cast val; memcpy(&val, (char*)from, 2); _msgpack_be16(val); })
+#define _msgpack_load32(cast, from) \
+    ({ cast val; memcpy(&val, (char*)from, 4); _msgpack_be32(val); })
+#define _msgpack_load64(cast, from) \
+    ({ cast val; memcpy(&val, (char*)from, 8); _msgpack_be64(val); })
+*/
+
+
 #endif /* msgpack/sysdep.h */
 
diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h
index 63d90a8..959d351 100644
--- a/msgpack/unpack_define.h
+++ b/msgpack/unpack_define.h
@@ -1,7 +1,7 @@
 /*
  * MessagePack unpacking routine template
  *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
  *
  *    Licensed under the Apache License, Version 2.0 (the "License");
  *    you may not use this file except in compliance with the License.
@@ -18,7 +18,8 @@
 #ifndef MSGPACK_UNPACK_DEFINE_H__
 #define MSGPACK_UNPACK_DEFINE_H__
 
-#include "sysdep.h"
+#include "msgpack/sysdep.h"
+#include 
 #include 
 #include 
 #include 
@@ -28,8 +29,8 @@ extern "C" {
 #endif
 
 
-#ifndef MSGPACK_MAX_STACK_SIZE
-#define MSGPACK_MAX_STACK_SIZE 16
+#ifndef MSGPACK_EMBED_STACK_SIZE
+#define MSGPACK_EMBED_STACK_SIZE 32
 #endif
 
 
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 42fe3a1..711b163 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -1,7 +1,7 @@
 /*
  * MessagePack unpacking routine template
  *
- * Copyright (C) 2008-2009 FURUHASHI Sadayuki
+ * Copyright (C) 2008-2010 FURUHASHI Sadayuki
  *
  *    Licensed under the Apache License, Version 2.0 (the "License");
  *    you may not use this file except in compliance with the License.
@@ -50,11 +50,7 @@ msgpack_unpack_struct_decl(_stack) {
 	msgpack_unpack_object obj;
 	size_t count;
 	unsigned int ct;
-
-    union {
-        size_t curr;
-        msgpack_unpack_object map_key;
-    };
+	msgpack_unpack_object map_key;
 };
 
 msgpack_unpack_struct_decl(_context) {
@@ -62,7 +58,12 @@ msgpack_unpack_struct_decl(_context) {
 	unsigned int cs;
 	unsigned int trail;
 	unsigned int top;
-	msgpack_unpack_struct(_stack) stack[MSGPACK_MAX_STACK_SIZE];
+	/*
+	msgpack_unpack_struct(_stack)* stack;
+	unsigned int stack_size;
+	msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE];
+	*/
+	msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE];
 };
 
 
@@ -71,9 +72,22 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx)
 	ctx->cs = CS_HEADER;
 	ctx->trail = 0;
 	ctx->top = 0;
+	/*
+	ctx->stack = ctx->embed_stack;
+	ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
+	*/
 	ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user);
 }
 
+/*
+msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx)
+{
+	if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
+		free(ctx->stack);
+	}
+}
+*/
+
 msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx)
 {
 	return (ctx)->stack[0].obj;
@@ -82,6 +96,8 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context
 
 msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
 {
+	assert(len >= *off);
+
 	const unsigned char* p = (unsigned char*)data + *off;
 	const unsigned char* const pe = (unsigned char*)data + len;
 	const void* n = NULL;
@@ -89,8 +105,10 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	unsigned int trail = ctx->trail;
 	unsigned int cs = ctx->cs;
 	unsigned int top = ctx->top;
-
 	msgpack_unpack_struct(_stack)* stack = ctx->stack;
+	/*
+	unsigned int stack_size = ctx->stack_size;
+	*/
 	msgpack_unpack_user* user = &ctx->user;
 
 	msgpack_unpack_object obj;
@@ -98,8 +116,6 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 
 	int ret;
 
-	assert(len >= *off);
-
 #define push_simple_value(func) \
 	if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \
 	goto _push
@@ -122,25 +138,38 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	goto _fixed_trail_again
 
 #define start_container(func, count_, ct_) \
+	if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
 	if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
 	if((count_) == 0) { obj = stack[top].obj; goto _push; } \
-	if(top >= MSGPACK_MAX_STACK_SIZE) { goto _failed; } \
 	stack[top].ct = ct_; \
-    stack[top].curr = 0; \
 	stack[top].count = count_; \
+	++top; \
 	/*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
 	/*printf("stack push %d\n", top);*/ \
-	++top; \
+	/* FIXME \
+	if(top >= stack_size) { \
+		if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
+			size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \
+			size_t nsize = csize * 2; \
+			msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \
+			if(tmp == NULL) { goto _failed; } \
+			memcpy(tmp, ctx->stack, csize); \
+			ctx->stack = stack = tmp; \
+			ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
+		} else { \
+			size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \
+			msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \
+			if(tmp == NULL) { goto _failed; } \
+			ctx->stack = stack = tmp; \
+			ctx->stack_size = stack_size = stack_size * 2; \
+		} \
+	} \
+	*/ \
 	goto _header_again
 
 #define NEXT_CS(p) \
 	((unsigned int)*p & 0x1f)
 
-#define PTR_CAST_8(ptr)   (*(uint8_t*)ptr)
-#define PTR_CAST_16(ptr)  _msgpack_be16(*(uint16_t*)ptr)
-#define PTR_CAST_32(ptr)  _msgpack_be32(*(uint32_t*)ptr)
-#define PTR_CAST_64(ptr)  _msgpack_be64(*(uint64_t*)ptr)
-
 #ifdef USE_CASE_RANGE
 #define SWITCH_RANGE_BEGIN     switch(*p) {
 #define SWITCH_RANGE(FROM, TO) case FROM ... TO:
@@ -228,70 +257,74 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 			//case CS_
 			//case CS_
 			case CS_FLOAT: {
-					union { uint32_t num; char buf[4]; } f;
-					f.num = PTR_CAST_32(n);  // FIXME
-					push_fixed_value(_float, *((float*)f.buf)); }
+					union { uint32_t i; float f; } mem;
+					mem.i = _msgpack_load32(uint32_t,n);
+					push_fixed_value(_float, mem.f); }
 			case CS_DOUBLE: {
-					union { uint64_t num; char buf[8]; } f;
-					f.num = PTR_CAST_64(n);  // FIXME
-					push_fixed_value(_double, *((double*)f.buf)); }
+					union { uint64_t i; double f; } mem;
+					mem.i = _msgpack_load64(uint64_t,n);
+#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
+                    // https://github.com/msgpack/msgpack-perl/pull/1
+                    mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
+#endif
+					push_fixed_value(_double, mem.f); }
 			case CS_UINT_8:
-				push_fixed_value(_uint8, (uint8_t)PTR_CAST_8(n));
+				push_fixed_value(_uint8, *(uint8_t*)n);
 			case CS_UINT_16:
-				push_fixed_value(_uint16, (uint16_t)PTR_CAST_16(n));
+				push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
 			case CS_UINT_32:
-				push_fixed_value(_uint32, (uint32_t)PTR_CAST_32(n));
+				push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
 			case CS_UINT_64:
-				push_fixed_value(_uint64, (uint64_t)PTR_CAST_64(n));
+				push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
 
 			case CS_INT_8:
-				push_fixed_value(_int8, (int8_t)PTR_CAST_8(n));
+				push_fixed_value(_int8, *(int8_t*)n);
 			case CS_INT_16:
-				push_fixed_value(_int16, (int16_t)PTR_CAST_16(n));
+				push_fixed_value(_int16, _msgpack_load16(int16_t,n));
 			case CS_INT_32:
-				push_fixed_value(_int32, (int32_t)PTR_CAST_32(n));
+				push_fixed_value(_int32, _msgpack_load32(int32_t,n));
 			case CS_INT_64:
-				push_fixed_value(_int64, (int64_t)PTR_CAST_64(n));
+				push_fixed_value(_int64, _msgpack_load64(int64_t,n));
 
 			//case CS_
 			//case CS_
 			//case CS_BIG_INT_16:
-			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint16_t)PTR_CAST_16(n), _big_int_zero);
+			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero);
 			//case CS_BIG_INT_32:
-			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, (uint32_t)PTR_CAST_32(n), _big_int_zero);
+			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero);
 			//case ACS_BIG_INT_VALUE:
 			//_big_int_zero:
 			//	// FIXME
 			//	push_variable_value(_big_int, data, n, trail);
 
 			//case CS_BIG_FLOAT_16:
-			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint16_t)PTR_CAST_16(n), _big_float_zero);
+			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero);
 			//case CS_BIG_FLOAT_32:
-			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, (uint32_t)PTR_CAST_32(n), _big_float_zero);
+			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero);
 			//case ACS_BIG_FLOAT_VALUE:
 			//_big_float_zero:
 			//	// FIXME
 			//	push_variable_value(_big_float, data, n, trail);
 
 			case CS_RAW_16:
-				again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint16_t)PTR_CAST_16(n), _raw_zero);
+				again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
 			case CS_RAW_32:
-				again_fixed_trail_if_zero(ACS_RAW_VALUE, (uint32_t)PTR_CAST_32(n), _raw_zero);
+				again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
 			case ACS_RAW_VALUE:
 			_raw_zero:
 				push_variable_value(_raw, data, n, trail);
 
 			case CS_ARRAY_16:
-				start_container(_array, (uint16_t)PTR_CAST_16(n), CT_ARRAY_ITEM);
+				start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
 			case CS_ARRAY_32:
 				/* FIXME security guard */
-				start_container(_array, (uint32_t)PTR_CAST_32(n), CT_ARRAY_ITEM);
+				start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
 
 			case CS_MAP_16:
-				start_container(_map, (uint16_t)PTR_CAST_16(n), CT_MAP_KEY);
+				start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
 			case CS_MAP_32:
 				/* FIXME security guard */
-				start_container(_map, (uint32_t)PTR_CAST_32(n), CT_MAP_KEY);
+				start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
 
 			default:
 				goto _failed;
@@ -303,9 +336,8 @@ _push:
 	c = &stack[top-1];
 	switch(c->ct) {
 	case CT_ARRAY_ITEM:
-		if(msgpack_unpack_callback(_array_item)(user, c->curr, &c->obj, obj) < 0) { goto _failed; }
-		if(++c->curr == c->count) {
-			msgpack_unpack_callback(_array_end)(user, &c->obj);
+		if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; }
+		if(--c->count == 0) {
 			obj = c->obj;
 			--top;
 			/*printf("stack pop %d\n", top);*/
@@ -319,7 +351,6 @@ _push:
 	case CT_MAP_VALUE:
 		if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
 		if(--c->count == 0) {
-			msgpack_unpack_callback(_map_end)(user, &c->obj);
 			obj = c->obj;
 			--top;
 			/*printf("stack pop %d\n", top);*/
@@ -379,8 +410,4 @@ _end:
 #undef start_container
 
 #undef NEXT_CS
-#undef PTR_CAST_8
-#undef PTR_CAST_16
-#undef PTR_CAST_32
-#undef PTR_CAST_64
 

From 188da01777d83e4024c3c7799892a492e4ec25e8 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Jun 2012 15:19:59 +0900
Subject: [PATCH 1001/1648] Fix new version of msgpack.

---
 msgpack/unpack.h          |  2 +-
 msgpack/unpack_template.h | 17 ++++++++++++-----
 setup.py                  |  1 +
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 2659a97..a106f9c 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -16,7 +16,7 @@
  *    limitations under the License.
  */
 
-#define MSGPACK_MAX_STACK_SIZE  (1024)
+#define MSGPACK_EMBED_STACK_SIZE  (1024)
 #include "unpack_define.h"
 
 typedef struct unpack_user {
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 711b163..b844a24 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -48,6 +48,7 @@
 
 msgpack_unpack_struct_decl(_stack) {
 	msgpack_unpack_object obj;
+	size_t size;
 	size_t count;
 	unsigned int ct;
 	msgpack_unpack_object map_key;
@@ -140,9 +141,12 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 #define start_container(func, count_, ct_) \
 	if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
 	if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
-	if((count_) == 0) { obj = stack[top].obj; goto _push; } \
+	if((count_) == 0) { obj = stack[top].obj; \
+		msgpack_unpack_callback(func##_end)(user, &obj); \
+		goto _push; } \
 	stack[top].ct = ct_; \
-	stack[top].count = count_; \
+	stack[top].size  = count_; \
+	stack[top].count = 0; \
 	++top; \
 	/*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
 	/*printf("stack push %d\n", top);*/ \
@@ -336,9 +340,10 @@ _push:
 	c = &stack[top-1];
 	switch(c->ct) {
 	case CT_ARRAY_ITEM:
-		if(msgpack_unpack_callback(_array_item)(user, &c->obj, obj) < 0) { goto _failed; }
-		if(--c->count == 0) {
+		if(msgpack_unpack_callback(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
+		if(++c->count == c->size) {
 			obj = c->obj;
+			msgpack_unpack_callback(_array_end)(user, &obj);
 			--top;
 			/*printf("stack pop %d\n", top);*/
 			goto _push;
@@ -350,8 +355,9 @@ _push:
 		goto _header_again;
 	case CT_MAP_VALUE:
 		if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
-		if(--c->count == 0) {
+		if(++c->count == c->size) {
 			obj = c->obj;
+			msgpack_unpack_callback(_map_end)(user, &obj);
 			--top;
 			/*printf("stack pop %d\n", top);*/
 			goto _push;
@@ -411,3 +417,4 @@ _end:
 
 #undef NEXT_CS
 
+/* vim: set ts=4 sw=4 noexpandtab  */
diff --git a/setup.py b/setup.py
index 115d3b8..7e78b4c 100755
--- a/setup.py
+++ b/setup.py
@@ -52,6 +52,7 @@ if sys.platform == 'win32':
 msgpack_mod = Extension('msgpack._msgpack',
                         sources=sources,
                         libraries=libraries,
+                        include_dirs=['.'],
                         )
 del sources, libraries
 

From 58eb7d0ce857b0ffb5e0ee4addf5ad1b3b47afa9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Jun 2012 15:34:51 +0900
Subject: [PATCH 1002/1648] Update changelog

---
 ChangeLog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 3a9277c..2fc9c18 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -5,6 +5,7 @@
 Changes
 -------
 * Drop supporting Python 2.5 and unify tests for Py2 and Py3.
+* Use new version of msgpack-c. It packs correctly on big endian platforms.
 
 Bugs fixed
 ----------

From ebe4c1f4bc31bc1f47831d72819cac5a2a584604 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Jun 2012 17:27:29 +0900
Subject: [PATCH 1003/1648] manage to compile on windows

Use C++ compiler to build.
---
 msgpack/_msgpack.pyx | 4 ++--
 msgpack/pack.h       | 2 +-
 setup.py             | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)
 mode change 100755 => 100644 setup.py

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 990f585..526c003 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -222,8 +222,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint
     template_init(&ctx)
     ctx.user.use_list = use_list
     ctx.user.object_hook = ctx.user.list_hook = NULL
-    ctx.user.encoding = enc
-    ctx.user.unicode_errors = err
+    ctx.user.encoding = enc
+    ctx.user.unicode_errors = err
     if object_hook is not None:
         if not PyCallable_Check(object_hook):
             raise TypeError("object_hook must be a callable.")
diff --git a/msgpack/pack.h b/msgpack/pack.h
index d36b436..4c0373e 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -77,7 +77,7 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_
 
     if (len + l > bs) {
         bs = (len + l) * 2;
-        buf = realloc(buf, bs);
+        buf = (char*)realloc(buf, bs);
         if (!buf) return -1;
     }
     memcpy(buf + len, data, l);
diff --git a/setup.py b/setup.py
old mode 100755
new mode 100644
index 7e78b4c..149b282
--- a/setup.py
+++ b/setup.py
@@ -53,6 +53,7 @@ msgpack_mod = Extension('msgpack._msgpack',
                         sources=sources,
                         libraries=libraries,
                         include_dirs=['.'],
+                        language='c++',
                         )
 del sources, libraries
 

From 06ed24a529eba164b23089a7653427f62dfb32e2 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Jun 2012 17:37:22 +0900
Subject: [PATCH 1004/1648] Fix setup scripts.

Support _msgpack.cpp
---
 MANIFEST.in | 2 +-
 setup.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index bf312d5..4e85759 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,5 @@
 include setup.py
 include COPYING
-recursive-include msgpack *.h *.c *.pyx
+recursive-include msgpack *.h *.c *.pyx *.cpp
 recursive-include test *.py
 recursive-include test3 *.py
diff --git a/setup.py b/setup.py
index 149b282..3f8f9f8 100644
--- a/setup.py
+++ b/setup.py
@@ -37,7 +37,7 @@ if have_cython:
                                         cython_compiler.default_options)
             sdist.__init__(self, *args, **kwargs)
 else:
-    sources = ['msgpack/_msgpack.c']
+    sources = ['msgpack/_msgpack.cpp']
 
     for f in sources:
         if not os.path.exists(f):

From 288e82029353e3c3256341c0e32d16ca32d33700 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 27 Jun 2012 18:05:35 +0900
Subject: [PATCH 1005/1648] prepare 0.2

---
 MANIFEST.in |  2 +-
 README.rst  | 78 +++++++++++++++++++++++++++++++++++++++++++----------
 setup.py    | 13 ++++-----
 3 files changed, 70 insertions(+), 23 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 4e85759..e1912ca 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,5 @@
 include setup.py
 include COPYING
+include README.rst
 recursive-include msgpack *.h *.c *.pyx *.cpp
 recursive-include test *.py
-recursive-include test3 *.py
diff --git a/README.rst b/README.rst
index 994492e..f392b67 100644
--- a/README.rst
+++ b/README.rst
@@ -3,33 +3,83 @@ MessagePack Python Binding
 ===========================
 
 :author: INADA Naoki
-:version: 0.1.0
-:date: 2009-07-12
+:version: 0.2.0
+:date: 2012-06-27
 
 HOW TO USE
 -----------
-You can read document in docstring after `import msgpack`
+
+one-shot pack & unpack
+^^^^^^^^^^^^^^^^^^^^^^
+
+Use ``packb`` for packing and ``unpackb`` for unpacking.
+msgpack provides ``dumps`` and ``loads`` as alias for compatibility with
+``json`` and ``pickle``.
+
+``pack`` and ``dump`` packs to file-like object.
+``unpack`` and ``load`` unpacks from file-like object.
+
+   >>> import msgpack
+   >>> msgpack.packb([1, 2, 3])
+   '\x93\x01\x02\x03'
+   >>> msgpack.unpackb(_)
+   (1, 2, 3)
+
+
+``unpack`` unpacks msgpack's array to Python's tuple.
+To unpack it to list, Use ``use_list`` option.
+
+   >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=True)
+   [1, 2, 3]
+
+Read docstring for other options.
+
+
+streaming unpacking
+^^^^^^^^^^^^^^^^^^^
+
+``Unpacker`` is "streaming unpacker". It unpacks multiple objects from one
+stream.
+
+::
+
+   import msgpack
+   from io import BytesIO
+
+   buf = BytesIO()
+   for i in range(100):
+      buf.write(msgpack.packb(range(i)))
+
+   buf.seek(0)
+
+   unpacker = msgpack.Unpacker()
+   while True:
+       data = buf.read(4)
+       if not data:
+           break
+       unpacker.seed(buf.read(16))
+       for unpacked in unpacker:
+           print unpacked
 
 
 INSTALL
 ---------
-Cython_ is required to build msgpack.
+You can use ``pip`` or ``easy_install`` to install msgpack::
 
-.. _Cython: http://www.cython.org/
+   $ easy_install msgpack-python
+     or
+   $ pip install msgpack-python
 
-posix
-''''''
-You can install msgpack in common way.
-
-    $ python setup.py install
 
 Windows
 ''''''''
-MessagePack requires gcc currently. So you need to prepare
-MinGW GCC.
+msgpack provides some binary distribution for Windows.
+You can install msgpack without compiler with them.
+
+When you can't use binary distribution, you need to install Visual Studio
+or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support
+amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.)
 
-    $ python setup.py build -c mingw32
-    $ python setup.py install
 
 TEST
 ----
diff --git a/setup.py b/setup.py
index 3f8f9f8..5802c3c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 1, 14, 'dev')
+version = (0, 2, 0, 'dev')
 
 import os
 import sys
@@ -59,13 +59,10 @@ del sources, libraries
 
 
 desc = 'MessagePack (de)serializer.'
-long_desc = """MessagePack (de)serializer for Python.
-
-What's MessagePack? (from http://msgpack.org/)
-
- MessagePack is a binary-based efficient data interchange format that is
- focused on high performance. It is like JSON, but very fast and small.
-"""
+f = open('README.rst')
+long_desc = f.read()
+f.close()
+del f
 
 setup(name='msgpack-python',
       author='INADA Naoki',

From dd5b1e265a24a6ddc6ffe681d9d632a8240ce2ad Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 27 Jun 2012 18:07:02 +0900
Subject: [PATCH 1006/1648] remove deprecated api.

---
 ChangeLog.rst       | 5 +++--
 msgpack/__init__.py | 9 ---------
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 2fc9c18..e255e19 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,11 +1,12 @@
-0.1.14
+0.2.0
 =======
-:release date: NOT RELEASED YET
+:release date: 2012-06-27
 
 Changes
 -------
 * Drop supporting Python 2.5 and unify tests for Py2 and Py3.
 * Use new version of msgpack-c. It packs correctly on big endian platforms.
+* Remove deprecated packs and unpacks API.
 
 Bugs fixed
 ----------
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 6b9735e..cdf045f 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -9,12 +9,3 @@ loads = unpackb
 dump = pack
 dumps = packb
 
-def packs(*args, **kw):
-    from warnings import warn
-    warn("msgpack.packs() is deprecated. Use packb() instead.", DeprecationWarning)
-    return packb(*args, **kw)
-
-def unpacks(*args, **kw):
-    from warnings import warn
-    warn("msgpack.unpacks() is deprecated. Use unpackb() instead.", DeprecationWarning)
-    return unpackb(*args, **kw)

From 52a7c02a50309fba0652279fdb4065821cf4d69c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 27 Jun 2012 18:15:44 +0900
Subject: [PATCH 1007/1648] update .gitignore

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index 9452edd..7918394 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,5 +5,8 @@ dist/*
 *.pyc
 *.pyo
 *.so
+*~
 msgpack/__version__.py
 msgpack/_msgpack.c
+msgpack/_msgpack.cpp
+*.egg-info

From 002a941b432d62e4a24f17309c8c09cbf394cb56 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 27 Jun 2012 18:16:59 +0900
Subject: [PATCH 1008/1648] Use setuptools to build egg package.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 5802c3c..70d9b06 100644
--- a/setup.py
+++ b/setup.py
@@ -5,8 +5,8 @@ version = (0, 2, 0, 'dev')
 import os
 import sys
 from glob import glob
-from distutils.core import setup, Extension
 from distutils.command.sdist import sdist
+from setuptools import setup, Extension
 
 try:
     from Cython.Distutils import build_ext

From 8514871c9b5b3063ed8042bbd8043522696cb5b1 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 27 Jun 2012 18:23:47 +0900
Subject: [PATCH 1009/1648] release 0.2.0

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 70d9b06..43f0d3b 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 2, 0, 'dev')
+version = (0, 2, 0)
 
 import os
 import sys

From 4bff55db9fe73e23b6da63a8b34c3679754549de Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 27 Jun 2012 18:25:56 +0900
Subject: [PATCH 1010/1648] Fix rst syntax.

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index f392b67..1da6ece 100644
--- a/README.rst
+++ b/README.rst
@@ -72,7 +72,7 @@ You can use ``pip`` or ``easy_install`` to install msgpack::
 
 
 Windows
-''''''''
+^^^^^^^
 msgpack provides some binary distribution for Windows.
 You can install msgpack without compiler with them.
 

From 2122b46b8489f3cf414f5ebb137f76ecc9fd9598 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 4 Jul 2012 14:58:36 +0900
Subject: [PATCH 1011/1648] Fix using deprecated api in tests.

---
 test/test_case.py   | 12 ++++++------
 test/test_except.py |  4 ++--
 test/test_format.py |  4 ++--
 test/test_obj.py    | 18 +++++++++---------
 test/test_pack.py   | 20 ++++++++++----------
 5 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/test/test_case.py b/test/test_case.py
index 2f42316..b88714d 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -3,13 +3,13 @@
 
 from nose import main
 from nose.tools import *
-from msgpack import packs, unpacks
+from msgpack import packb, unpackb
 
 
 def check(length, obj):
-    v = packs(obj)
+    v = packb(obj)
     assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v)))
-    assert_equal(unpacks(v), obj)
+    assert_equal(unpackb(v), obj)
 
 def test_1():
     for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
@@ -70,8 +70,8 @@ def test_array32():
 
 
 def match(obj, buf):
-    assert_equal(packs(obj), buf)
-    assert_equal(unpacks(buf), obj)
+    assert_equal(packb(obj), buf)
+    assert_equal(unpackb(buf), obj)
 
 def test_match():
     cases = [
@@ -99,7 +99,7 @@ def test_match():
         match(v, p)
 
 def test_unicode():
-    assert_equal(b'foobar', unpacks(packs('foobar')))
+    assert_equal(b'foobar', unpackb(packb('foobar')))
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_except.py b/test/test_except.py
index 574728f..ad02cb6 100644
--- a/test/test_except.py
+++ b/test/test_except.py
@@ -2,12 +2,12 @@
 # coding: utf-8
 
 from nose.tools import *
-from msgpack import packs, unpacks
+from msgpack import packb, unpackb
 
 import datetime
 
 def test_raise_on_find_unsupported_value():
-    assert_raises(TypeError, packs, datetime.datetime.now())
+    assert_raises(TypeError, packb, datetime.datetime.now())
 
 if __name__ == '__main__':
     from nose import main
diff --git a/test/test_format.py b/test/test_format.py
index 022e680..c03b3e2 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -3,10 +3,10 @@
 
 from nose import main
 from nose.tools import *
-from msgpack import unpacks
+from msgpack import unpackb
 
 def check(src, should):
-    assert_equal(unpacks(src), should)
+    assert_equal(unpackb(src), should)
 
 def testSimpleValue():
     check(b"\x93\xc0\xc2\xc3", 
diff --git a/test/test_obj.py b/test/test_obj.py
index 6357cfc..d155b73 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -4,7 +4,7 @@
 from nose import main
 from nose.tools import *
 
-from msgpack import packs, unpacks
+from msgpack import packb, unpackb
 
 def _decode_complex(obj):
     if b'__complex__' in obj:
@@ -17,26 +17,26 @@ def _encode_complex(obj):
     return obj
 
 def test_encode_hook():
-    packed = packs([3, 1+2j], default=_encode_complex)
-    unpacked = unpacks(packed)
+    packed = packb([3, 1+2j], default=_encode_complex)
+    unpacked = unpackb(packed)
     eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2})
 
 def test_decode_hook():
-    packed = packs([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
-    unpacked = unpacks(packed, object_hook=_decode_complex)
+    packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
+    unpacked = unpackb(packed, object_hook=_decode_complex)
     eq_(unpacked[1], 1+2j)
 
 @raises(ValueError)
 def test_bad_hook():
-    packed = packs([3, 1+2j], default=lambda o: o)
-    unpacked = unpacks(packed)
+    packed = packb([3, 1+2j], default=lambda o: o)
+    unpacked = unpackb(packed)
 
 def _arr_to_str(arr):
     return ''.join(str(c) for c in arr)
 
 def test_array_hook():
-    packed = packs([1,2,3])
-    unpacked = unpacks(packed, list_hook=_arr_to_str)
+    packed = packb([1,2,3])
+    unpacked = unpackb(packed, list_hook=_arr_to_str)
     eq_(unpacked, '123')
 
 if __name__ == '__main__':
diff --git a/test/test_pack.py b/test/test_pack.py
index 88b6dac..898cdb9 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -6,12 +6,12 @@ from nose import main
 from nose.tools import *
 from nose.plugins.skip import SkipTest
 
-from msgpack import packs, unpacks, Unpacker, Packer
+from msgpack import packb, unpackb, Unpacker, Packer
 
 from io import BytesIO
 
 def check(data):
-    re = unpacks(packs(data))
+    re = unpackb(packb(data))
     assert_equal(re, data)
 
 def testPack():
@@ -33,7 +33,7 @@ def testPackUnicode():
         six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"),
         ]
     for td in test_data:
-        re = unpacks(packs(td, encoding='utf-8'), encoding='utf-8')
+        re = unpackb(packb(td, encoding='utf-8'), encoding='utf-8')
         assert_equal(re, td)
         packer = Packer(encoding='utf-8')
         data = packer.pack(td)
@@ -49,7 +49,7 @@ def testPackUTF32():
             six.u("РуÑÑкий текÑÑ‚"),
             ]
         for td in test_data:
-            re = unpacks(packs(td, encoding='utf-32'), encoding='utf-32')
+            re = unpackb(packb(td, encoding='utf-32'), encoding='utf-32')
             assert_equal(re, td)
     except LookupError:
         raise SkipTest
@@ -62,28 +62,28 @@ def testPackBytes():
         check(td)
 
 def testIgnoreUnicodeErrors():
-    re = unpacks(packs(b'abc\xeddef'),
+    re = unpackb(packb(b'abc\xeddef'),
         encoding='utf-8', unicode_errors='ignore')
     assert_equal(re, "abcdef")
 
 @raises(UnicodeDecodeError)
 def testStrictUnicodeUnpack():
-    unpacks(packs(b'abc\xeddef'), encoding='utf-8')
+    unpackb(packb(b'abc\xeddef'), encoding='utf-8')
 
 @raises(UnicodeEncodeError)
 def testStrictUnicodePack():
-    packs(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict')
+    packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict')
 
 def testIgnoreErrorsPack():
-    re = unpacks(packs(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
+    re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
     assert_equal(re, six.u("abcdef"))
 
 @raises(TypeError)
 def testNoEncoding():
-    packs(six.u("abc"), encoding=None)
+    packb(six.u("abc"), encoding=None)
 
 def testDecodeBinary():
-    re = unpacks(packs("abc"), encoding=None)
+    re = unpackb(packb("abc"), encoding=None)
     assert_equal(re, b"abc")
 
 if __name__ == '__main__':

From e133c7fd27a515feba7691d6ad37cc9d58dc5b42 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Jul 2012 21:15:11 +0900
Subject: [PATCH 1012/1648] Start 0.2.1

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 43f0d3b..b46fa91 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 2, 0)
+version = (0, 2, 1, 'dev1')
 
 import os
 import sys

From 7b1167044b17572126fc69b89eb6fa9c0a5afb91 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Jul 2012 21:28:16 +0900
Subject: [PATCH 1013/1648] Add max_buffer_size to Unpacker.

---
 ChangeLog.rst        | 16 ++++++++++
 msgpack/_msgpack.pyx | 71 +++++++++++++++++++++++++++++++++++---------
 2 files changed, 73 insertions(+), 14 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index e255e19..e86e3bd 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,19 @@
+0.2.1
+=======
+:release date: NOT RELEASED YET
+
+Changes
+-------
+* Add ``max_buffer_size`` parameter to Unpacker. It limits internal buffer size
+  and allows unpack data from untrusted source safely.
+
+* Unpacker's buffer reallocation algorithm is less greedy now. It cause perforamce
+  derease in rare case but memory efficient and don't allocate than ``max_buffer_size``.
+
+Bugs fixed
+----------
+
+
 0.2.0
 =======
 :release date: 2012-06-27
diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 526c003..96abb42 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -10,6 +10,9 @@ cdef extern from "Python.h":
 
 from libc.stdlib cimport *
 from libc.string cimport *
+from libc.limits cimport *
+
+
 import gc
 _gc_disable = gc.disable
 _gc_enable = gc.enable
@@ -35,6 +38,11 @@ cdef extern from "pack.h":
 
 cdef int DEFAULT_RECURSE_LIMIT=511
 
+
+class BufferFull(Exception):
+    pass
+
+
 cdef class Packer(object):
     """MessagePack Packer
 
@@ -193,7 +201,9 @@ cdef extern from "unpack.h":
     object template_data(template_context* ctx)
 
 
-def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"):
+def unpackb(object packed, object object_hook=None, object list_hook=None,
+            bint use_list=0, encoding=None, unicode_errors="strict",
+            ):
     """
     Unpack packed_bytes to object. Returns an unpacked object."""
     cdef template_context ctx
@@ -243,12 +253,16 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint
         return None
 
 
-def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=0, encoding=None, unicode_errors="strict"):
+def unpack(object stream, object object_hook=None, object list_hook=None,
+           bint use_list=0, encoding=None, unicode_errors="strict",
+           ):
     """
     unpack an object from stream.
     """
     return unpackb(stream.read(), use_list=use_list,
-                   object_hook=object_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors)
+                   object_hook=object_hook, list_hook=list_hook,
+                   encoding=encoding, unicode_errors=unicode_errors,
+                   )
 
 cdef class Unpacker(object):
     """
@@ -259,7 +273,7 @@ cdef class Unpacker(object):
     When `Unpacker` initialized with `file_like`, unpacker reads serialized data
     from it and `.feed()` method is not usable.
 
-    `read_size` is used as `file_like.read(read_size)`. (default: 1M)
+    `read_size` is used as `file_like.read(read_size)`. (default: 1024**2)
 
     If `use_list` is true, msgpack list is deserialized to Python list.
     Otherwise, it is deserialized to Python tuple. (default: False)
@@ -272,11 +286,24 @@ cdef class Unpacker(object):
 
     `unicode_errors` is used for decoding bytes.
 
-    example::
+    `max_buffer_size` limits size of data waiting unpacked. 0 means unlimited
+    (default).
+    Raises `BufferFull` exception when it is insufficient.
+    You shoud set this parameter when unpacking data from untrasted source.
+
+    example of streaming deserialize from file-like object::
+
+        unpacker = Unpacker(file_like)
+        for o in unpacker:
+            do_something(o)
+
+    example of streaming deserialize from socket::
 
         unpacker = Unpacker()
         while 1:
-            buf = astream.read()
+            buf = sock.recv(1024**2)
+            if not buf:
+                break
             unpacker.feed(buf)
             for o in unpacker:
                 do_something(o)
@@ -293,6 +320,7 @@ cdef class Unpacker(object):
     cdef object _berrors
     cdef char *encoding
     cdef char *unicode_errors
+    cdef size_t max_buffer_size
 
     def __cinit__(self):
         self.buf = NULL
@@ -303,7 +331,7 @@ cdef class Unpacker(object):
 
     def __init__(self, file_like=None, Py_ssize_t read_size=1024*1024, bint use_list=0,
                  object object_hook=None, object list_hook=None,
-                 encoding=None, unicode_errors='strict'):
+                 encoding=None, unicode_errors='strict', int max_buffer_size=0):
         self.use_list = use_list
         self.file_like = file_like
         if file_like:
@@ -314,6 +342,10 @@ cdef class Unpacker(object):
         self.buf = malloc(read_size)
         if self.buf == NULL:
             raise MemoryError("Unable to allocate internal buffer.")
+        if max_buffer_size:
+            self.max_buffer_size = max_buffer_size
+        else:
+            self.max_buffer_size = INT_MAX
         self.buf_size = read_size
         self.buf_head = 0
         self.buf_tail = 0
@@ -355,28 +387,36 @@ cdef class Unpacker(object):
     cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
         cdef:
             char* buf = self.buf
+            char* new_buf
             size_t head = self.buf_head
             size_t tail = self.buf_tail
             size_t buf_size = self.buf_size
             size_t new_size
 
         if tail + _buf_len > buf_size:
-            if ((tail - head) + _buf_len)*2 < buf_size:
+            if ((tail - head) + _buf_len) <= buf_size:
                 # move to front.
                 memmove(buf, buf + head, tail - head)
                 tail -= head
                 head = 0
             else:
                 # expand buffer.
-                new_size = tail + _buf_len
-                if new_size < buf_size*2:
-                    new_size = buf_size*2
-                buf = realloc(buf, new_size)
-                if buf == NULL:
+                new_size = (tail-head) + _buf_len
+                if new_size > self.max_buffer_size:
+                    raise BufferFull
+                new_size = min(new_size*2, self.max_buffer_size)
+                new_buf = malloc(new_size)
+                if new_buf == NULL:
                     # self.buf still holds old buffer and will be freed during
                     # obj destruction
                     raise MemoryError("Unable to enlarge internal buffer.")
+                memcpy(new_buf, buf + head, tail - head)
+                free(buf)
+
+                buf = new_buf
                 buf_size = new_size
+                tail -= head
+                head = 0
 
         memcpy(buf + tail, (_buf), _buf_len)
         self.buf = buf
@@ -387,7 +427,10 @@ cdef class Unpacker(object):
     # prepare self.buf from file_like
     cdef fill_buffer(self):
         if self.file_like is not None:
-            next_bytes = self.file_like_read(self.read_size)
+            next_bytes = self.file_like_read(
+                    max(self.read_size,
+                        self.max_buffer_size - (self.buf_tail - self.buf_head)
+                        ))
             if next_bytes:
                 self.append_buffer(PyBytes_AsString(next_bytes),
                                    PyBytes_Size(next_bytes))

From e5462ff72f16517e5e9eb434efad084d874e3967 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 20 Jul 2012 02:02:37 +0900
Subject: [PATCH 1014/1648] Add test for max_buffer_size.

---
 test/test_sequnpack.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 774fe1b..b1b80b2 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,7 +1,8 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from msgpack import Unpacker
+from msgpack import Unpacker, BufferFull
+import nose
 
 def test_foobar():
     unpacker = Unpacker(read_size=3)
@@ -27,6 +28,19 @@ def test_foobar():
         k += 1
     assert k == len(b'foobar')
 
-if __name__ == '__main__':
-    test_foobar()
 
+def test_maxbuffersize():
+    nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3)
+    unpacker = Unpacker(read_size=3, max_buffer_size=3)
+    unpacker.feed(b'fo')
+    nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob')
+    unpacker.feed(b'o')
+    assert ord('f') == next(unpacker)
+    unpacker.feed(b'b')
+    assert ord('o') == next(unpacker)
+    assert ord('o') == next(unpacker)
+    assert ord('b') == next(unpacker)
+
+
+if __name__ == '__main__':
+    nose.main()

From 53ca2bb648d423462b970493b8ed63cdaf70afe4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 20 Jul 2012 02:02:54 +0900
Subject: [PATCH 1015/1648] raise ValueError when read_size > max_buffer_size.

---
 msgpack/_msgpack.pyx | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 96abb42..00d6d90 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -338,14 +338,15 @@ cdef class Unpacker(object):
             self.file_like_read = file_like.read
             if not PyCallable_Check(self.file_like_read):
                 raise ValueError("`file_like.read` must be a callable.")
+        if not max_buffer_size:
+            max_buffer_size = INT_MAX
+        self.max_buffer_size = max_buffer_size
+        if read_size > max_buffer_size:
+            raise ValueError("read_size should be less or equal to max_buffer_size")
         self.read_size = read_size
         self.buf = malloc(read_size)
         if self.buf == NULL:
             raise MemoryError("Unable to allocate internal buffer.")
-        if max_buffer_size:
-            self.max_buffer_size = max_buffer_size
-        else:
-            self.max_buffer_size = INT_MAX
         self.buf_size = read_size
         self.buf_head = 0
         self.buf_tail = 0

From 59c8b51e5b3df8322bc4d2f639bcafd6a25eecaf Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 20 Jul 2012 02:05:43 +0900
Subject: [PATCH 1016/1648] Default value of read_size is min(1024**2,
 max_buffer_size)

---
 msgpack/_msgpack.pyx | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 00d6d90..ebf1592 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -267,13 +267,13 @@ def unpack(object stream, object object_hook=None, object list_hook=None,
 cdef class Unpacker(object):
     """
     Streaming unpacker.
-    read_size is used like file_like.read(read_size)
 
     `file_like` is a file-like object having `.read(n)` method.
     When `Unpacker` initialized with `file_like`, unpacker reads serialized data
     from it and `.feed()` method is not usable.
 
-    `read_size` is used as `file_like.read(read_size)`. (default: 1024**2)
+    `read_size` is used as `file_like.read(read_size)`.
+    (default: min(1024**2, max_buffer_size))
 
     If `use_list` is true, msgpack list is deserialized to Python list.
     Otherwise, it is deserialized to Python tuple. (default: False)
@@ -329,7 +329,7 @@ cdef class Unpacker(object):
         free(self.buf)
         self.buf = NULL
 
-    def __init__(self, file_like=None, Py_ssize_t read_size=1024*1024, bint use_list=0,
+    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
                  object object_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict', int max_buffer_size=0):
         self.use_list = use_list
@@ -340,6 +340,8 @@ cdef class Unpacker(object):
                 raise ValueError("`file_like.read` must be a callable.")
         if not max_buffer_size:
             max_buffer_size = INT_MAX
+        if not read_size:
+            read_size = min(max_buffer_size, 1024**2)
         self.max_buffer_size = max_buffer_size
         if read_size > max_buffer_size:
             raise ValueError("read_size should be less or equal to max_buffer_size")

From bf4124f592b4ba54c9ec09fbc8a4f898bc229967 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 25 Jul 2012 09:19:10 +0900
Subject: [PATCH 1017/1648] Fix setup.py sdist doesn't generates c++ source.

---
 setup.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index b46fa91..2e6ec84 100644
--- a/setup.py
+++ b/setup.py
@@ -32,9 +32,10 @@ if have_cython:
 
     class Sdist(sdist):
         def __init__(self, *args, **kwargs):
+            cy_opt = cython_compiler.default_options.copy()
+            cy_opt['cplus'] = True
             for src in glob('msgpack/*.pyx'):
-                cython_compiler.compile(glob('msgpack/*.pyx'),
-                                        cython_compiler.default_options)
+                cython_compiler.compile(glob('msgpack/*.pyx'), cy_opt)
             sdist.__init__(self, *args, **kwargs)
 else:
     sources = ['msgpack/_msgpack.cpp']

From 7c03f322fa73901620b2cd188186003dd9e02841 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 19 Aug 2012 01:05:41 +0900
Subject: [PATCH 1018/1648] Fix on SPARC Solaris.

Use C++ only on Windows.
Define ENDIAN macros from `sys.byteorder`.
---
 setup.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/setup.py b/setup.py
index 2e6ec84..7d8a5d4 100644
--- a/setup.py
+++ b/setup.py
@@ -33,12 +33,12 @@ if have_cython:
     class Sdist(sdist):
         def __init__(self, *args, **kwargs):
             cy_opt = cython_compiler.default_options.copy()
-            cy_opt['cplus'] = True
+            #cy_opt['cplus'] = True
             for src in glob('msgpack/*.pyx'):
                 cython_compiler.compile(glob('msgpack/*.pyx'), cy_opt)
             sdist.__init__(self, *args, **kwargs)
 else:
-    sources = ['msgpack/_msgpack.cpp']
+    sources = ['msgpack/_msgpack.c']
 
     for f in sources:
         if not os.path.exists(f):
@@ -47,16 +47,24 @@ else:
     Sdist = sdist
 
 libraries = []
+language = 'c'
 if sys.platform == 'win32':
     libraries.append('ws2_32')
+    language = 'c++'
+
+if sys.byteorder == 'big':
+    macros = [('__BIG_ENDIAN__', '1')]
+else:
+    macros = [('__LITTLE_ENDIAN__', '1')]
 
 msgpack_mod = Extension('msgpack._msgpack',
                         sources=sources,
                         libraries=libraries,
                         include_dirs=['.'],
-                        language='c++',
+                        language=language,
+                        define_macros=macros,
                         )
-del sources, libraries
+del sources, libraries, language, macros
 
 
 desc = 'MessagePack (de)serializer.'
@@ -79,7 +87,6 @@ setup(name='msgpack-python',
       classifiers=[
           'Programming Language :: Python :: 2',
           'Programming Language :: Python :: 3',
-          'Development Status :: 4 - Beta',
           'Intended Audience :: Developers',
           'License :: OSI Approved :: Apache Software License',
           ]

From 670bb3ca154d5f86fe72fe34a73d0ca8b7bd315a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 19 Aug 2012 02:53:16 +0900
Subject: [PATCH 1019/1648] Use C++ compiler on win32.

---
 setup.py | 69 +++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 48 insertions(+), 21 deletions(-)

diff --git a/setup.py b/setup.py
index 7d8a5d4..074700b 100644
--- a/setup.py
+++ b/setup.py
@@ -4,23 +4,62 @@ version = (0, 2, 1, 'dev1')
 
 import os
 import sys
+import shutil
 from glob import glob
 from distutils.command.sdist import sdist
 from setuptools import setup, Extension
 
+from distutils.command.build_ext import build_ext
+
 try:
-    from Cython.Distutils import build_ext
     import Cython.Compiler.Main as cython_compiler
     have_cython = True
 except ImportError:
-    from distutils.command.build_ext import build_ext
     have_cython = False
 
+
+def cythonize(src):
+    sys.stderr.write("cythonize: %r\n" % (src,))
+    cython_compiler.compile([src])
+
+def ensure_source(src):
+    pyx = os.path.splitext(src)[0] + '.pyx'
+
+    if not os.path.exists(src):
+        if not have_cython:
+            raise Exception("""\
+Cython is required for building extension from checkout.
+Install Cython >= 0.16 or install msgpack from PyPI.
+""")
+        cythonize(src)
+    elif (os.path.exists(pyx) and
+          os.stat(src).st_mtime < os.stat(pyx).st_mtime and
+          have_cython):
+        cythonize(src)
+
+    # Use C++ compiler on win32.
+    # MSVC9 doesn't provide stdint.h when using C Compiler.
+    if sys.platform == 'win32':
+        cpp = src + 'pp'
+        shutil.copy(src, cpp)
+        return cpp
+    else:
+        return src
+
+
+class BuildExt(build_ext):
+    def build_extension(self, ext):
+        ext.sources = map(ensure_source, ext.sources)
+        return build_ext.build_extension(self, ext)
+
+
 # make msgpack/__verison__.py
 f = open('msgpack/__version__.py', 'w')
-f.write("version = %r\n" % (version,))
-f.close()
-del f
+try:
+    f.write("version = %r\n" % (version,))
+finally:
+    f.close()
+    del f
 
 version_str = '.'.join(str(x) for x in version[:3])
 if len(version) > 3 and version[3] != 'final':
@@ -28,29 +67,18 @@ if len(version) > 3 and version[3] != 'final':
 
 # take care of extension modules.
 if have_cython:
-    sources = ['msgpack/_msgpack.pyx']
-
     class Sdist(sdist):
         def __init__(self, *args, **kwargs):
-            cy_opt = cython_compiler.default_options.copy()
-            #cy_opt['cplus'] = True
             for src in glob('msgpack/*.pyx'):
-                cython_compiler.compile(glob('msgpack/*.pyx'), cy_opt)
+                cythonize(src)
             sdist.__init__(self, *args, **kwargs)
 else:
-    sources = ['msgpack/_msgpack.c']
-
-    for f in sources:
-        if not os.path.exists(f):
-            raise ImportError("Building msgpack from VCS needs Cython. Install Cython or use sdist package.")
-
     Sdist = sdist
 
+sources = ['msgpack/_msgpack.c']
 libraries = []
-language = 'c'
 if sys.platform == 'win32':
     libraries.append('ws2_32')
-    language = 'c++'
 
 if sys.byteorder == 'big':
     macros = [('__BIG_ENDIAN__', '1')]
@@ -61,10 +89,9 @@ msgpack_mod = Extension('msgpack._msgpack',
                         sources=sources,
                         libraries=libraries,
                         include_dirs=['.'],
-                        language=language,
                         define_macros=macros,
                         )
-del sources, libraries, language, macros
+del sources, libraries, macros
 
 
 desc = 'MessagePack (de)serializer.'
@@ -77,7 +104,7 @@ setup(name='msgpack-python',
       author='INADA Naoki',
       author_email='songofacandy@gmail.com',
       version=version_str,
-      cmdclass={'build_ext': build_ext, 'sdist': Sdist},
+      cmdclass={'build_ext': BuildExt, 'sdist': Sdist},
       ext_modules=[msgpack_mod],
       packages=['msgpack'],
       description=desc,

From 29b4b785d0152c964819bc613a0321214df78c47 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 19 Aug 2012 03:04:19 +0900
Subject: [PATCH 1020/1648] Fix build_ext doesn't work on Python 3.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 074700b..b03647a 100644
--- a/setup.py
+++ b/setup.py
@@ -49,7 +49,7 @@ Install Cython >= 0.16 or install msgpack from PyPI.
 
 class BuildExt(build_ext):
     def build_extension(self, ext):
-        ext.sources = map(ensure_source, ext.sources)
+        ext.sources = list(map(ensure_source, ext.sources))
         return build_ext.build_extension(self, ext)
 
 

From 814c42c2917ca62637ee37344c042174dfb10b4b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 19 Aug 2012 04:17:56 +0900
Subject: [PATCH 1021/1648] Change the way to manage version number.

---
 msgpack/__init__.py | 2 +-
 msgpack/_version.py | 1 +
 setup.py            | 8 +-------
 3 files changed, 3 insertions(+), 8 deletions(-)
 create mode 100644 msgpack/_version.py

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index cdf045f..98b1ab7 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,5 +1,5 @@
 # coding: utf-8
-from msgpack.__version__ import *
+from msgpack._version import version
 from msgpack._msgpack import *
 
 # alias for compatibility to simplejson/marshal/pickle.
diff --git a/msgpack/_version.py b/msgpack/_version.py
new file mode 100644
index 0000000..5355679
--- /dev/null
+++ b/msgpack/_version.py
@@ -0,0 +1 @@
+version = (0, 2, 1, 'dev1')
diff --git a/setup.py b/setup.py
index b03647a..6148dbd 100644
--- a/setup.py
+++ b/setup.py
@@ -53,13 +53,7 @@ class BuildExt(build_ext):
         return build_ext.build_extension(self, ext)
 
 
-# make msgpack/__verison__.py
-f = open('msgpack/__version__.py', 'w')
-try:
-    f.write("version = %r\n" % (version,))
-finally:
-    f.close()
-    del f
+exec(open('msgpack/_version.py').read())
 
 version_str = '.'.join(str(x) for x in version[:3])
 if len(version) > 3 and version[3] != 'final':

From f74ce3caaa5d1aa3432dca1558a3d13a15d3fae3 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 20 Aug 2012 00:11:38 +0900
Subject: [PATCH 1022/1648] 0.2.1

---
 ChangeLog.rst       | 5 ++++-
 msgpack/_version.py | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index e86e3bd..c328f35 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,6 @@
 0.2.1
 =======
-:release date: NOT RELEASED YET
+:release date: 2012-08-20
 
 Changes
 -------
@@ -12,6 +12,9 @@ Changes
 
 Bugs fixed
 ----------
+* Fix msgpack didn't work on SPARC Solaris. It was because choosing wrong byteorder
+  on compilation time. Use ``sys.byteorder`` to get correct byte order.
+  Very thanks to Chris Casey for giving test environment to me.
 
 
 0.2.0
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 5355679..68ae707 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 2, 1, 'dev1')
+version = (0, 2, 1)

From e63a9437538919c669317b695d842807803301fe Mon Sep 17 00:00:00 2001
From: TobiasSimon 
Date: Mon, 20 Aug 2012 21:56:55 +0200
Subject: [PATCH 1023/1648] added float serialization support

---
 msgpack/_msgpack.pyx | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index ebf1592..62af8d3 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -30,6 +30,7 @@ cdef extern from "pack.h":
     int msgpack_pack_long(msgpack_packer* pk, long d)
     int msgpack_pack_long_long(msgpack_packer* pk, long long d)
     int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d)
+    int msgpack_pack_float(msgpack_packer* pk, float d)
     int msgpack_pack_double(msgpack_packer* pk, double d)
     int msgpack_pack_array(msgpack_packer* pk, size_t l)
     int msgpack_pack_map(msgpack_packer* pk, size_t l)
@@ -58,6 +59,7 @@ cdef class Packer(object):
     cdef object _berrors
     cdef char *encoding
     cdef char *unicode_errors
+    cdef bool use_float
 
     def __cinit__(self):
         cdef int buf_size = 1024*1024
@@ -67,7 +69,8 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
-    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict'):
+    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_float=False):
+        self.use_float = use_float
         if default is not None:
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")
@@ -94,7 +97,8 @@ cdef class Packer(object):
         cdef long long llval
         cdef unsigned long long ullval
         cdef long longval
-        cdef double fval
+        cdef float fval
+        cdef double dval
         cdef char* rawval
         cdef int ret
         cdef dict d
@@ -120,8 +124,12 @@ cdef class Packer(object):
             longval = o
             ret = msgpack_pack_long(&self.pk, longval)
         elif PyFloat_Check(o):
-            fval = o
-            ret = msgpack_pack_double(&self.pk, fval)
+            if self.use_float:
+               fval = o
+               ret = msgpack_pack_float(&self.pk, fval)
+            else:
+               dval = o
+               ret = msgpack_pack_double(&self.pk, dval)
         elif PyBytes_Check(o):
             rawval = o
             ret = msgpack_pack_raw(&self.pk, len(o))

From 6aa4aead31e01e22f5297fadc7c3e4b0e90ba8ae Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 21 Aug 2012 14:56:32 +0900
Subject: [PATCH 1024/1648] Fix build from pyx doesn't work.

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 6148dbd..2a20e16 100644
--- a/setup.py
+++ b/setup.py
@@ -31,11 +31,11 @@ def ensure_source(src):
 Cython is required for building extension from checkout.
 Install Cython >= 0.16 or install msgpack from PyPI.
 """)
-        cythonize(src)
+        cythonize(pyx)
     elif (os.path.exists(pyx) and
           os.stat(src).st_mtime < os.stat(pyx).st_mtime and
           have_cython):
-        cythonize(src)
+        cythonize(pyx)
 
     # Use C++ compiler on win32.
     # MSVC9 doesn't provide stdint.h when using C Compiler.

From 235b928be7b43f1a5f1753ff08e685cbcf55bba4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 24 Aug 2012 09:53:18 +0900
Subject: [PATCH 1025/1648] Stop disable/enable gc.

json and pickle modules don't stop gc. It's a very dirty hack.
---
 msgpack/_msgpack.pyx | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index ebf1592..7ff0cff 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -13,10 +13,6 @@ from libc.string cimport *
 from libc.limits cimport *
 
 
-import gc
-_gc_disable = gc.disable
-_gc_enable = gc.enable
-
 cdef extern from "pack.h":
     struct msgpack_packer:
         char* buf
@@ -242,11 +238,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         if not PyCallable_Check(list_hook):
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
-    _gc_disable()
-    try:
-        ret = template_execute(&ctx, buf, buf_len, &off)
-    finally:
-        _gc_enable()
+    ret = template_execute(&ctx, buf, buf_len, &off)
     if ret == 1:
         return template_data(&ctx)
     else:
@@ -444,9 +436,7 @@ cdef class Unpacker(object):
         """unpack one object"""
         cdef int ret
         while 1:
-            _gc_disable()
             ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
-            _gc_enable()
             if ret == 1:
                 o = template_data(&self.ctx)
                 template_init(&self.ctx)

From 56ec7ee1b1bdca7dd528cb6337f6b391581aefd0 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 24 Aug 2012 10:05:38 +0900
Subject: [PATCH 1026/1648] Update changelog

---
 ChangeLog.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index c328f35..67eac68 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,13 @@
+0.2.2
+=======
+:release date: NOT RELEASED YET
+
+Bugs fixed
+-----------
+* ``unpack()`` didn't restores gc state when it called with gc disabled.
+  ``unpack()`` doesn't control gc now instead of restoring gc state collectly.
+  User can control gc state when gc cause performance issue.
+
 0.2.1
 =======
 :release date: 2012-08-20

From 0297b36bda332fe21ab7d4c4c549cc68ccd344bc Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 21 Sep 2012 13:58:56 +0900
Subject: [PATCH 1027/1648] Fix reading more than read_size.

---
 msgpack/_msgpack.pyx | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 7ff0cff..15bf5a7 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -278,8 +278,8 @@ cdef class Unpacker(object):
 
     `unicode_errors` is used for decoding bytes.
 
-    `max_buffer_size` limits size of data waiting unpacked. 0 means unlimited
-    (default).
+    `max_buffer_size` limits size of data waiting unpacked.
+    0 means system's INT_MAX (default).
     Raises `BufferFull` exception when it is insufficient.
     You shoud set this parameter when unpacking data from untrasted source.
 
@@ -332,11 +332,11 @@ cdef class Unpacker(object):
                 raise ValueError("`file_like.read` must be a callable.")
         if not max_buffer_size:
             max_buffer_size = INT_MAX
+        if read_size > max_buffer_size:
+            raise ValueError("read_size should be less or equal to max_buffer_size")
         if not read_size:
             read_size = min(max_buffer_size, 1024**2)
         self.max_buffer_size = max_buffer_size
-        if read_size > max_buffer_size:
-            raise ValueError("read_size should be less or equal to max_buffer_size")
         self.read_size = read_size
         self.buf = malloc(read_size)
         if self.buf == NULL:
@@ -419,18 +419,15 @@ cdef class Unpacker(object):
         self.buf_size = buf_size
         self.buf_tail = tail + _buf_len
 
-    # prepare self.buf from file_like
-    cdef fill_buffer(self):
-        if self.file_like is not None:
-            next_bytes = self.file_like_read(
-                    max(self.read_size,
-                        self.max_buffer_size - (self.buf_tail - self.buf_head)
-                        ))
-            if next_bytes:
-                self.append_buffer(PyBytes_AsString(next_bytes),
-                                   PyBytes_Size(next_bytes))
-            else:
-                self.file_like = None
+    cdef read_from_file(self):
+        next_bytes = self.file_like_read(
+                min(self.read_size,
+                    self.max_buffer_size - (self.buf_tail - self.buf_head)
+                    ))
+        if next_bytes:
+            self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes))
+        else:
+            self.file_like = None
 
     cpdef unpack(self):
         """unpack one object"""
@@ -443,7 +440,7 @@ cdef class Unpacker(object):
                 return o
             elif ret == 0:
                 if self.file_like is not None:
-                    self.fill_buffer()
+                    self.read_from_file()
                     continue
                 raise StopIteration("No more unpack data.")
             else:

From 397d772e110ad7000e0952d41c74d8efd322f01f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 21 Sep 2012 14:08:34 +0900
Subject: [PATCH 1028/1648] Rename use_float to use_single_float.

---
 msgpack/_msgpack.pyx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index aef1228..af005cb 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -65,8 +65,8 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
-    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_float=False):
-        self.use_float = use_float
+    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False):
+        self.use_float = use_single_float
         if default is not None:
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")

From 51335bbee4502ac3af81363a10ef6718439377d1 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 21 Sep 2012 14:15:30 +0900
Subject: [PATCH 1029/1648] packb supports use_single_float option.

---
 ChangeLog.rst        | 9 ++++++++-
 msgpack/_msgpack.pyx | 5 +++--
 test/test_pack.py    | 5 +++++
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 67eac68..4fd5cbc 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,11 @@
 0.2.2
 =======
-:release date: NOT RELEASED YET
+:release date: 2012-09-21
+
+Changes
+-------
+* Add ``use_single_float`` option to ``Packer``. When it is true, packs float
+  object in single precision format.
 
 Bugs fixed
 -----------
@@ -8,6 +13,8 @@ Bugs fixed
   ``unpack()`` doesn't control gc now instead of restoring gc state collectly.
   User can control gc state when gc cause performance issue.
 
+* ``Unpacker``'s ``read_size`` option didn't used.
+
 0.2.1
 =======
 :release date: 2012-08-20
diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index af005cb..c9f5e31 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -177,10 +177,11 @@ def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors
     packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
     stream.write(packer.pack(o))
 
-def packb(object o, default=None, encoding='utf-8', unicode_errors='strict'):
+def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False):
     """
     pack o and return packed bytes."""
-    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
+    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
+                    use_single_float=use_single_float)
     return packer.pack(o)
 
 
diff --git a/test/test_pack.py b/test/test_pack.py
index 898cdb9..85d11a0 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -2,6 +2,7 @@
 # coding: utf-8
 
 import six
+import struct
 from nose import main
 from nose.tools import *
 from nose.plugins.skip import SkipTest
@@ -86,5 +87,9 @@ def testDecodeBinary():
     re = unpackb(packb("abc"), encoding=None)
     assert_equal(re, b"abc")
 
+def testPackFloat():
+    assert_equal(packb(1.0, use_single_float=True),  b'\xca' + struct.pack('>f', 1.0))
+    assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0))
+
 if __name__ == '__main__':
     main()

From be405ec5cfe98a42283055baa901ec31de2632ba Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 21 Sep 2012 14:16:40 +0900
Subject: [PATCH 1030/1648] 0.2.2

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 2a20e16..ae72221 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 2, 1, 'dev1')
+version = (0, 2, 2)
 
 import os
 import sys

From 5b66edaa156c43793b6f68013a738f545885b8d6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 21 Sep 2012 14:17:34 +0900
Subject: [PATCH 1031/1648] 0.2.2 (again)

---
 msgpack/_version.py | 2 +-
 setup.py            | 2 --
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index 68ae707..f343b7a 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 2, 1)
+version = (0, 2, 2)
diff --git a/setup.py b/setup.py
index ae72221..86b0b34 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 # coding: utf-8
-version = (0, 2, 2)
-
 import os
 import sys
 import shutil

From ffec10dff3839ae182cff3d9fff67ab3fe6165be Mon Sep 17 00:00:00 2001
From: jnothman 
Date: Fri, 21 Sep 2012 16:03:41 +1000
Subject: [PATCH 1032/1648] Expose packed stream with Unpacker.read_bytes()

At present, Unpacker buffers reading from the stream, meaning the stream can no longer be read directly. Unpacker.read_bytes(n) provides access to the underlying data, allowing content of known size to be read without unpacking.
---
 msgpack/_msgpack.pyx | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index c9f5e31..d7ea4b4 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -455,6 +455,18 @@ cdef class Unpacker(object):
             else:
                 raise ValueError("Unpack failed: error = %d" % (ret,))
 
+    def read_bytes(self, Py_ssize_t nbytes):
+        """read a specified number of raw bytes from the stream"""
+        cdef size_t nread
+        ret = ''
+        while len(ret) < nbytes and self.file_like is not None:
+            if self.buf_head == self.buf_tail:
+                self.fill_buffer()
+            nread = min(self.buf_tail - self.buf_head, nbytes - len(ret))
+            ret += PyBytes_FromStringAndSize(self.buf + self.buf_head, nread)
+            self.buf_head += nread
+        return ret
+
     def __iter__(self):
         return self
 

From 28058fb53d21097947d190bcc47e3609a6794e7a Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Sat, 22 Sep 2012 22:57:00 +1000
Subject: [PATCH 1033/1648] A first implementation of Unpacker.skip()

---
 msgpack/_msgpack.pyx      | 25 +++++++++++++++++--------
 msgpack/unpack_template.h | 24 ++++++++++++++----------
 2 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 80d34ab..c1e3e75 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -200,7 +200,7 @@ cdef extern from "unpack.h":
         PyObject* key
 
     int template_execute(template_context* ctx, const_char_ptr data,
-                         size_t len, size_t* off) except -1
+                         size_t len, size_t* off, bool construct) except -1
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
@@ -246,7 +246,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         if not PyCallable_Check(list_hook):
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
-    ret = template_execute(&ctx, buf, buf_len, &off)
+    ret = template_execute(&ctx, buf, buf_len, &off, True)
     if ret == 1:
         return template_data(&ctx)
     else:
@@ -440,15 +440,12 @@ cdef class Unpacker(object):
             else:
                 self.file_like = None
 
-    cpdef unpack(self):
-        """unpack one object"""
+    cpdef _unpack(self, bool construct):
         cdef int ret
         while 1:
-            ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+            ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head, construct)
             if ret == 1:
-                o = template_data(&self.ctx)
-                template_init(&self.ctx)
-                return o
+                return
             elif ret == 0:
                 if self.file_like is not None:
                     self.fill_buffer()
@@ -457,6 +454,18 @@ cdef class Unpacker(object):
             else:
                 raise ValueError("Unpack failed: error = %d" % (ret,))
 
+    cpdef unpack(self):
+        """unpack one object"""
+        self._unpack(True)
+        o = template_data(&self.ctx)
+        template_init(&self.ctx)
+
+
+    cpdef skip(self):
+        """read and ignore one object, returning None"""
+        self._unpack(False)
+        template_init(&self.ctx)
+
     def __iter__(self):
         return self
 
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index b844a24..10e41e1 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -95,7 +95,7 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context
 }
 
 
-msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
+msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, bool construct)
 {
 	assert(len >= *off);
 
@@ -117,14 +117,17 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 
 	int ret;
 
+#define construct_cb(name) \
+    construct && msgpack_unpack_callback(name)
+
 #define push_simple_value(func) \
-	if(msgpack_unpack_callback(func)(user, &obj) < 0) { goto _failed; } \
+	if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \
 	goto _push
 #define push_fixed_value(func, arg) \
-	if(msgpack_unpack_callback(func)(user, arg, &obj) < 0) { goto _failed; } \
+	if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \
 	goto _push
 #define push_variable_value(func, base, pos, len) \
-	if(msgpack_unpack_callback(func)(user, \
+	if(construct_cb(func)(user, \
 		(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
 	goto _push
 
@@ -140,9 +143,9 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 
 #define start_container(func, count_, ct_) \
 	if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
-	if(msgpack_unpack_callback(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
+	if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
 	if((count_) == 0) { obj = stack[top].obj; \
-		msgpack_unpack_callback(func##_end)(user, &obj); \
+		construct_cb(func##_end)(user, &obj); \
 		goto _push; } \
 	stack[top].ct = ct_; \
 	stack[top].size  = count_; \
@@ -340,10 +343,10 @@ _push:
 	c = &stack[top-1];
 	switch(c->ct) {
 	case CT_ARRAY_ITEM:
-		if(msgpack_unpack_callback(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
+		if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
 		if(++c->count == c->size) {
 			obj = c->obj;
-			msgpack_unpack_callback(_array_end)(user, &obj);
+			construct_cb(_array_end)(user, &obj);
 			--top;
 			/*printf("stack pop %d\n", top);*/
 			goto _push;
@@ -354,10 +357,10 @@ _push:
 		c->ct = CT_MAP_VALUE;
 		goto _header_again;
 	case CT_MAP_VALUE:
-		if(msgpack_unpack_callback(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
+		if(construct_cb(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
 		if(++c->count == c->size) {
 			obj = c->obj;
-			msgpack_unpack_callback(_map_end)(user, &obj);
+			construct_cb(_map_end)(user, &obj);
 			--top;
 			/*printf("stack pop %d\n", top);*/
 			goto _push;
@@ -399,6 +402,7 @@ _end:
 	*off = p - (const unsigned char*)data;
 
 	return ret;
+#undef construct_cb
 }
 
 

From 4d643894a1ab02b0836245b8a456200cac5ae314 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 23 Sep 2012 02:13:32 +0900
Subject: [PATCH 1034/1648] Support packing subclass of dict.

---
 ChangeLog.rst        | 11 +++++++++++
 msgpack/_msgpack.pyx | 16 +++++++++++++---
 test/test_pack.py    | 26 ++++++++++++++++++++++++++
 3 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 4fd5cbc..46b83ee 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,14 @@
+0.2.3
+=======
+:release date: in development
+
+Changes
+-------
+
+Bugs fixed
+-----------
+* Can't pack subclass of dict.
+
 0.2.2
 =======
 :release date: 2012-09-21
diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index c9f5e31..976871e 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -139,11 +139,19 @@ cdef class Packer(object):
             ret = msgpack_pack_raw(&self.pk, len(o))
             if ret == 0:
                 ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
-        elif PyDict_Check(o):
+        elif PyDict_CheckExact(o):
             d = o
             ret = msgpack_pack_map(&self.pk, len(d))
             if ret == 0:
-                for k,v in d.iteritems():
+                for k, v in d.iteritems():
+                    ret = self._pack(k, nest_limit-1)
+                    if ret != 0: break
+                    ret = self._pack(v, nest_limit-1)
+                    if ret != 0: break
+        elif PyDict_Check(o):
+            ret = msgpack_pack_map(&self.pk, len(o))
+            if ret == 0:
+                for k, v in o.items():
                     ret = self._pack(k, nest_limit-1)
                     if ret != 0: break
                     ret = self._pack(v, nest_limit-1)
@@ -332,7 +340,9 @@ cdef class Unpacker(object):
 
     def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
                  object object_hook=None, object list_hook=None,
-                 encoding=None, unicode_errors='strict', int max_buffer_size=0):
+                 encoding=None, unicode_errors='strict', int max_buffer_size=0,
+                 object object_pairs_hook=None,
+                 ):
         self.use_list = use_list
         self.file_like = file_like
         if file_like:
diff --git a/test/test_pack.py b/test/test_pack.py
index 85d11a0..b216c46 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -91,5 +91,31 @@ def testPackFloat():
     assert_equal(packb(1.0, use_single_float=True),  b'\xca' + struct.pack('>f', 1.0))
     assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0))
 
+
+class odict(dict):
+    '''Reimplement OrderedDict to run test on Python 2.6'''
+    def __init__(self, seq):
+        self._seq = seq
+        dict.__init__(self, seq)
+
+    def items(self):
+        return self._seq[:]
+
+    def iteritems(self):
+        return iter(self._seq)
+
+    def keys(self):
+        return [x[0] for x in self._seq]
+
+def test_odict():
+    seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
+    od = odict(seq)
+    assert_equal(unpackb(packb(od)), dict(seq))
+    # After object_pairs_hook is implemented.
+    #def pair_hook(seq):
+    #    return seq
+    #assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq)
+
+
 if __name__ == '__main__':
     main()

From 8b2959bc0ab086a3dbe47176b3c241dd1a1ecf6c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 23 Sep 2012 03:39:14 +0900
Subject: [PATCH 1035/1648] pack and packb raises ValueError when extra data
 passed.

---
 msgpack/_msgpack.pyx | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 976871e..12ee2ea 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -257,7 +257,10 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         ctx.user.list_hook = list_hook
     ret = template_execute(&ctx, buf, buf_len, &off)
     if ret == 1:
-        return template_data(&ctx)
+        obj = template_data(&ctx)
+        if off < buf_len:
+            raise ValueError("Extra data.")
+        return obj
     else:
         return None
 
@@ -461,7 +464,7 @@ cdef class Unpacker(object):
                 if self.file_like is not None:
                     self.read_from_file()
                     continue
-                raise StopIteration("No more unpack data.")
+                raise StopIteration("No more data to unpack.")
             else:
                 raise ValueError("Unpack failed: error = %d" % (ret,))
 

From 36b88b407718cb7b96f7ae12cffbb0c8a27e493b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 23 Sep 2012 03:44:41 +0900
Subject: [PATCH 1036/1648] Add Roadmap.

---
 ROADMAP.md | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 ROADMAP.md

diff --git a/ROADMAP.md b/ROADMAP.md
new file mode 100644
index 0000000..5245cc0
--- /dev/null
+++ b/ROADMAP.md
@@ -0,0 +1,7 @@
+0.2 series
+==========
+Improve compatibility to simplejson.
+
+0.3 series
+==========
+Add features msgpack-ruby has.

From e8842efdedb1917a28147aa8ad1bf6f7b729a751 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 23 Sep 2012 08:57:32 +0900
Subject: [PATCH 1037/1648] Add py33 to tox.

---
 tox.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 5e80dd0..214c4c4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,6 @@
 [tox]
-envlist = py26,py27,py32
+envlist = py26,py27,py32,py33
+
 [testenv]
 deps=
     nose

From eaf9891b4255f3b1ca5cf2ea5b631091523b913d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 23 Sep 2012 10:00:18 +0900
Subject: [PATCH 1038/1648] clean some cython code.

---
 msgpack/_msgpack.pyx      | 28 +++++++++++++++-------------
 msgpack/unpack_template.h |  2 +-
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 9061d42..c8ee7bb 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -209,7 +209,7 @@ cdef extern from "unpack.h":
         PyObject* key
 
     int template_execute(template_context* ctx, const_char_ptr data,
-                         size_t len, size_t* off, bool construct) except -1
+                         size_t len, size_t* off, bint construct) except -1
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
@@ -255,7 +255,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         if not PyCallable_Check(list_hook):
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
-    ret = template_execute(&ctx, buf, buf_len, &off, True)
+    ret = template_execute(&ctx, buf, buf_len, &off, 1)
     if ret == 1:
         obj = template_data(&ctx)
         if off < buf_len:
@@ -451,12 +451,18 @@ cdef class Unpacker(object):
         else:
             self.file_like = None
 
-    cpdef _unpack(self, bool construct):
+    cdef _unpack(self, bint construct):
         cdef int ret
+        cdef object obj
         while 1:
             ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head, construct)
             if ret == 1:
-                return
+                if construct:
+                    obj = template_data(&self.ctx)
+                else:
+                    obj = None
+                template_init(&self.ctx)
+                return obj
             elif ret == 0:
                 if self.file_like is not None:
                     self.read_from_file()
@@ -465,23 +471,19 @@ cdef class Unpacker(object):
             else:
                 raise ValueError("Unpack failed: error = %d" % (ret,))
 
-    cpdef unpack(self):
+    def unpack(self):
         """unpack one object"""
-        self._unpack(True)
-        o = template_data(&self.ctx)
-        template_init(&self.ctx)
+        return self._unpack(1)
 
-
-    cpdef skip(self):
+    def skip(self):
         """read and ignore one object, returning None"""
-        self._unpack(False)
-        template_init(&self.ctx)
+        return self._unpack(0)
 
     def __iter__(self):
         return self
 
     def __next__(self):
-        return self.unpack()
+        return self._unpack(1)
 
     # for debug.
     #def _buf(self):
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 10e41e1..5495a51 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -95,7 +95,7 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context
 }
 
 
-msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, bool construct)
+msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, int construct)
 {
 	assert(len >= *off);
 

From 7d142d2bef0805a528f1cd84e173579209298380 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 23 Sep 2012 10:02:11 +0900
Subject: [PATCH 1039/1648] Add changelog

---
 ChangeLog.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 46b83ee..fe5b820 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,12 @@
+0.3.0
+=====
+:release date: in development
+
+Changes
+-------
+* Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman)
+
+
 0.2.3
 =======
 :release date: in development

From 48d693c1b9613fd976a3bf668f692ec22ad4a520 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 23 Sep 2012 10:09:51 +0900
Subject: [PATCH 1040/1648] Add test for `.skip()`

---
 msgpack/_msgpack.pyx   |  2 +-
 test/test_sequnpack.py | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index c8ee7bb..8d37aaa 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -451,7 +451,7 @@ cdef class Unpacker(object):
         else:
             self.file_like = None
 
-    cdef _unpack(self, bint construct):
+    cdef object _unpack(self, bint construct):
         cdef int ret
         cdef object obj
         while 1:
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index b1b80b2..aa47d3c 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -28,6 +28,20 @@ def test_foobar():
         k += 1
     assert k == len(b'foobar')
 
+def test_foobar_skip():
+    unpacker = Unpacker(read_size=3)
+    unpacker.feed(b'foobar')
+    assert unpacker.unpack() == ord(b'f')
+    unpacker.skip()
+    assert unpacker.unpack() == ord(b'o')
+    unpacker.skip()
+    assert unpacker.unpack() == ord(b'a')
+    unpacker.skip()
+    try:
+        o = unpacker.unpack()
+        assert 0, "should raise exception"
+    except StopIteration:
+        assert 1, "ok"
 
 def test_maxbuffersize():
     nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3)

From c3da8458681fc479233910d4c92dc84374e5efed Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 23 Sep 2012 11:16:59 +0900
Subject: [PATCH 1041/1648] Add docstring about raising ValueError when there
 are extra bytes.

---
 msgpack/_msgpack.pyx | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 12ee2ea..0886580 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -217,8 +217,10 @@ cdef extern from "unpack.h":
 def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=0, encoding=None, unicode_errors="strict",
             ):
+    """Unpack packed_bytes to object. Returns an unpacked object.
+
+    Raises `ValueError` when `packed` contains extra bytes.
     """
-    Unpack packed_bytes to object. Returns an unpacked object."""
     cdef template_context ctx
     cdef size_t off = 0
     cdef int ret
@@ -268,14 +270,16 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 def unpack(object stream, object object_hook=None, object list_hook=None,
            bint use_list=0, encoding=None, unicode_errors="strict",
            ):
-    """
-    unpack an object from stream.
+    """Unpack an object from `stream`.
+
+    Raises `ValueError` when `stream` has extra bytes.
     """
     return unpackb(stream.read(), use_list=use_list,
                    object_hook=object_hook, list_hook=list_hook,
                    encoding=encoding, unicode_errors=unicode_errors,
                    )
 
+
 cdef class Unpacker(object):
     """
     Streaming unpacker.

From b06ed8eb75563111ef88a119f9f7a45e67f61736 Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Sun, 23 Sep 2012 18:11:49 +1000
Subject: [PATCH 1042/1648] Factor context initialisation from unpackb and
 Unpacker

---
 msgpack/_msgpack.pyx | 82 +++++++++++++++-----------------------------
 1 file changed, 28 insertions(+), 54 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index e0a1043..823ed62 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -213,6 +213,32 @@ cdef extern from "unpack.h":
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
+cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook, bint use_list, encoding, unicode_errors):
+    template_init(ctx)
+    ctx.user.use_list = use_list
+    ctx.user.object_hook = ctx.user.list_hook = NULL
+    if object_hook is not None:
+        if not PyCallable_Check(object_hook):
+            raise TypeError("object_hook must be a callable.")
+        ctx.user.object_hook = object_hook
+    if list_hook is not None:
+        if not PyCallable_Check(list_hook):
+            raise TypeError("list_hook must be a callable.")
+        ctx.user.list_hook = list_hook
+    if encoding is None:
+        ctx.user.encoding = NULL
+        ctx.user.unicode_errors = NULL
+    else:
+        if isinstance(encoding, unicode):
+            _bencoding = encoding.encode('ascii')
+        else:
+            _bencoding = encoding
+        ctx.user.encoding = PyBytes_AsString(_bencoding)
+        if isinstance(unicode_errors, unicode):
+            _berrors = unicode_errors.encode('ascii')
+        else:
+            _berrors = unicode_errors
+        ctx.user.unicode_errors = PyBytes_AsString(_berrors)
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=0, encoding=None, unicode_errors="strict",
@@ -229,34 +255,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef Py_ssize_t buf_len
     PyObject_AsReadBuffer(packed, &buf, &buf_len)
 
-    if encoding is None:
-        enc = NULL
-        err = NULL
-    else:
-        if isinstance(encoding, unicode):
-            bencoding = encoding.encode('ascii')
-        else:
-            bencoding = encoding
-        if isinstance(unicode_errors, unicode):
-            berrors = unicode_errors.encode('ascii')
-        else:
-            berrors = unicode_errors
-        enc = PyBytes_AsString(bencoding)
-        err = PyBytes_AsString(berrors)
-
-    template_init(&ctx)
-    ctx.user.use_list = use_list
-    ctx.user.object_hook = ctx.user.list_hook = NULL
-    ctx.user.encoding = enc
-    ctx.user.unicode_errors = err
-    if object_hook is not None:
-        if not PyCallable_Check(object_hook):
-            raise TypeError("object_hook must be a callable.")
-        ctx.user.object_hook = object_hook
-    if list_hook is not None:
-        if not PyCallable_Check(list_hook):
-            raise TypeError("list_hook must be a callable.")
-        ctx.user.list_hook = list_hook
+    init_ctx(&ctx, object_hook, list_hook, use_list, encoding, unicode_errors)
     ret = template_execute(&ctx, buf, buf_len, &off, 1)
     if ret == 1:
         obj = template_data(&ctx)
@@ -348,7 +347,6 @@ cdef class Unpacker(object):
     def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
                  object object_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict', int max_buffer_size=0,
-                 object object_pairs_hook=None,
                  ):
         self.use_list = use_list
         self.file_like = file_like
@@ -370,31 +368,7 @@ cdef class Unpacker(object):
         self.buf_size = read_size
         self.buf_head = 0
         self.buf_tail = 0
-        template_init(&self.ctx)
-        self.ctx.user.use_list = use_list
-        self.ctx.user.object_hook = self.ctx.user.list_hook = NULL
-        if object_hook is not None:
-            if not PyCallable_Check(object_hook):
-                raise TypeError("object_hook must be a callable.")
-            self.ctx.user.object_hook = object_hook
-        if list_hook is not None:
-            if not PyCallable_Check(list_hook):
-                raise TypeError("list_hook must be a callable.")
-            self.ctx.user.list_hook = list_hook
-        if encoding is None:
-            self.ctx.user.encoding = NULL
-            self.ctx.user.unicode_errors = NULL
-        else:
-            if isinstance(encoding, unicode):
-                self._bencoding = encoding.encode('ascii')
-            else:
-                self._bencoding = encoding
-            self.ctx.user.encoding = PyBytes_AsString(self._bencoding)
-            if isinstance(unicode_errors, unicode):
-                self._berrors = unicode_errors.encode('ascii')
-            else:
-                self._berrors = unicode_errors
-            self.ctx.user.unicode_errors = PyBytes_AsString(self._berrors)
+        init_ctx(&self.ctx, object_hook, list_hook, use_list, encoding, unicode_errors)
 
     def feed(self, object next_bytes):
         cdef char* buf

From 77942514db0c5a80e9f3f9bcb1e1939ecc8705e6 Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Sun, 23 Sep 2012 19:37:28 +1000
Subject: [PATCH 1043/1648] Implement object_pairs_hook

---
 msgpack/_msgpack.pyx      | 34 ++++++++++++++++++++++++++--------
 msgpack/unpack.h          | 30 +++++++++++++++++++++---------
 msgpack/unpack_template.h |  2 +-
 test/test_obj.py          | 10 ++++++++++
 test/test_pack.py         |  7 +++----
 5 files changed, 61 insertions(+), 22 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 823ed62..b6d8e8b 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -197,6 +197,7 @@ cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
         int use_list
         PyObject* object_hook
+        bint has_pairs_hook # call object_hook with k-v pairs
         PyObject* list_hook
         char *encoding
         char *unicode_errors
@@ -213,18 +214,32 @@ cdef extern from "unpack.h":
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
-cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook, bint use_list, encoding, unicode_errors):
+cdef inline init_ctx(template_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, encoding, unicode_errors):
     template_init(ctx)
     ctx.user.use_list = use_list
     ctx.user.object_hook = ctx.user.list_hook = NULL
+
+    if object_hook is not None and object_pairs_hook is not None:
+        raise ValueError("object_pairs_hook and object_hook are mutually exclusive.")
+
     if object_hook is not None:
         if not PyCallable_Check(object_hook):
             raise TypeError("object_hook must be a callable.")
         ctx.user.object_hook = object_hook
+
+    if object_pairs_hook is None:
+        ctx.user.has_pairs_hook = False
+    else:
+        if not PyCallable_Check(object_pairs_hook):
+            raise TypeError("object_pairs_hook must be a callable.")
+        ctx.user.object_hook = object_pairs_hook
+        ctx.user.has_pairs_hook = True
+
     if list_hook is not None:
         if not PyCallable_Check(list_hook):
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
+
     if encoding is None:
         ctx.user.encoding = NULL
         ctx.user.unicode_errors = NULL
@@ -240,7 +255,7 @@ cdef inline init_ctx(template_context *ctx, object object_hook, object list_hook
             _berrors = unicode_errors
         ctx.user.unicode_errors = PyBytes_AsString(_berrors)
 
-def unpackb(object packed, object object_hook=None, object list_hook=None,
+def unpackb(object packed, object object_hook=None, object object_pairs_hook=None, object list_hook=None,
             bint use_list=0, encoding=None, unicode_errors="strict",
             ):
     """Unpack packed_bytes to object. Returns an unpacked object.
@@ -255,7 +270,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef Py_ssize_t buf_len
     PyObject_AsReadBuffer(packed, &buf, &buf_len)
 
-    init_ctx(&ctx, object_hook, list_hook, use_list, encoding, unicode_errors)
+    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors)
     ret = template_execute(&ctx, buf, buf_len, &off, 1)
     if ret == 1:
         obj = template_data(&ctx)
@@ -266,7 +281,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         return None
 
 
-def unpack(object stream, object object_hook=None, object list_hook=None,
+def unpack(object stream, object object_hook=None, object object_pairs_hook=None, object list_hook=None,
            bint use_list=0, encoding=None, unicode_errors="strict",
            ):
     """Unpack an object from `stream`.
@@ -274,7 +289,7 @@ def unpack(object stream, object object_hook=None, object list_hook=None,
     Raises `ValueError` when `stream` has extra bytes.
     """
     return unpackb(stream.read(), use_list=use_list,
-                   object_hook=object_hook, list_hook=list_hook,
+                   object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
                    encoding=encoding, unicode_errors=unicode_errors,
                    )
 
@@ -294,7 +309,10 @@ cdef class Unpacker(object):
     Otherwise, it is deserialized to Python tuple. (default: False)
 
     `object_hook` is same to simplejson. If it is not None, it should be callable
-    and Unpacker calls it when deserializing key-value.
+    and Unpacker calls it with a dict argument after deserializing a map.
+
+    `object_pairs_hook` is same to simplejson. If it is not None, it should be callable
+    and Unpacker calls it with a list of key-value pairs after deserializing a map.
 
     `encoding` is encoding used for decoding msgpack bytes. If it is None (default),
     msgpack bytes is deserialized to Python bytes.
@@ -345,7 +363,7 @@ cdef class Unpacker(object):
         self.buf = NULL
 
     def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
-                 object object_hook=None, object list_hook=None,
+                 object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict', int max_buffer_size=0,
                  ):
         self.use_list = use_list
@@ -368,7 +386,7 @@ cdef class Unpacker(object):
         self.buf_size = read_size
         self.buf_head = 0
         self.buf_tail = 0
-        init_ctx(&self.ctx, object_hook, list_hook, use_list, encoding, unicode_errors)
+        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors)
 
     def feed(self, object next_bytes):
         cdef char* buf
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index a106f9c..7064a1b 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -22,6 +22,7 @@
 typedef struct unpack_user {
     int use_list;
     PyObject *object_hook;
+    bool has_pairs_hook;
     PyObject *list_hook;
     const char *encoding;
     const char *unicode_errors;
@@ -160,9 +161,7 @@ static inline int template_callback_array_item(unpack_user* u, unsigned int curr
 static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->list_hook) {
-        PyObject *arglist = Py_BuildValue("(O)", *c);
-        PyObject *new_c = PyEval_CallObject(u->list_hook, arglist);
-        Py_DECREF(arglist);
+        PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c);
         Py_DECREF(*c);
         *c = new_c;
     }
@@ -171,16 +170,31 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj
 
 static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
 {
-    PyObject *p = PyDict_New();
+    PyObject *p;
+    if (u->has_pairs_hook) {
+        p = PyList_New(n); // Or use tuple?
+    }
+    else {
+        p = PyDict_New();
+    }
     if (!p)
         return -1;
     *o = p;
     return 0;
 }
 
-static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
+static inline int template_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
 {
-    if (PyDict_SetItem(*c, k, v) == 0) {
+    if (u->has_pairs_hook) {
+        msgpack_unpack_object item = PyTuple_Pack(2, k, v);
+        if (!item)
+            return -1;
+        Py_DECREF(k);
+        Py_DECREF(v);
+        PyList_SET_ITEM(*c, current, item);
+        return 0;
+    }
+    else if (PyDict_SetItem(*c, k, v) == 0) {
         Py_DECREF(k);
         Py_DECREF(v);
         return 0;
@@ -191,9 +205,7 @@ static inline int template_callback_map_item(unpack_user* u, msgpack_unpack_obje
 static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->object_hook) {
-        PyObject *arglist = Py_BuildValue("(O)", *c);
-        PyObject *new_c = PyEval_CallObject(u->object_hook, arglist);
-        Py_DECREF(arglist);
+        PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c);
         Py_DECREF(*c);
         *c = new_c;
     }
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 5495a51..6080a51 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -357,7 +357,7 @@ _push:
 		c->ct = CT_MAP_VALUE;
 		goto _header_again;
 	case CT_MAP_VALUE:
-		if(construct_cb(_map_item)(user, &c->obj, c->map_key, obj) < 0) { goto _failed; }
+		if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
 		if(++c->count == c->size) {
 			obj = c->obj;
 			construct_cb(_map_end)(user, &obj);
diff --git a/test/test_obj.py b/test/test_obj.py
index d155b73..e0d89fc 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -26,6 +26,16 @@ def test_decode_hook():
     unpacked = unpackb(packed, object_hook=_decode_complex)
     eq_(unpacked[1], 1+2j)
 
+def test_decode_pairs_hook():
+    packed = packb([3, {1: 2, 3: 4}])
+    prod_sum = 1 * 2 + 3 * 4
+    unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l))
+    eq_(unpacked[1], prod_sum)
+
+@raises(ValueError)
+def test_only_one_obj_hook():
+    unpackb('', object_hook=lambda x: x, object_pairs_hook=lambda x: x)
+
 @raises(ValueError)
 def test_bad_hook():
     packed = packb([3, 1+2j], default=lambda o: o)
diff --git a/test/test_pack.py b/test/test_pack.py
index b216c46..2c99873 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -111,10 +111,9 @@ def test_odict():
     seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
     od = odict(seq)
     assert_equal(unpackb(packb(od)), dict(seq))
-    # After object_pairs_hook is implemented.
-    #def pair_hook(seq):
-    #    return seq
-    #assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq)
+    def pair_hook(seq):
+        return seq
+    assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq)
 
 
 if __name__ == '__main__':

From e7c51d9089e9270ce197c00a6af1c60e45f36e97 Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Sun, 23 Sep 2012 11:13:44 +1000
Subject: [PATCH 1044/1648] Cleaner read_bytes and a test case

No longer reads via buffer for unbuffered bytes
---
 msgpack/_msgpack.pyx   | 12 +++++-------
 test/test_sequnpack.py | 16 ++++++++++++++++
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index d7ea4b4..30fb9fc 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -458,13 +458,11 @@ cdef class Unpacker(object):
     def read_bytes(self, Py_ssize_t nbytes):
         """read a specified number of raw bytes from the stream"""
         cdef size_t nread
-        ret = ''
-        while len(ret) < nbytes and self.file_like is not None:
-            if self.buf_head == self.buf_tail:
-                self.fill_buffer()
-            nread = min(self.buf_tail - self.buf_head, nbytes - len(ret))
-            ret += PyBytes_FromStringAndSize(self.buf + self.buf_head, nread)
-            self.buf_head += nread
+        nread = min(self.buf_tail - self.buf_head, nbytes)
+        ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread)
+        self.buf_head += nread
+        if len(ret) < nbytes and self.file_like is not None:
+            ret += self.file_like.read(nbytes - len(ret))
         return ret
 
     def __iter__(self):
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index b1b80b2..c763f40 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # coding: utf-8
 
+import six
 from msgpack import Unpacker, BufferFull
 import nose
 
@@ -42,5 +43,20 @@ def test_maxbuffersize():
     assert ord('b') == next(unpacker)
 
 
+def test_readbytes():
+    unpacker = Unpacker(read_size=3)
+    unpacker.feed(b'foobar')
+    assert unpacker.unpack() == ord(b'f')
+    assert unpacker.read_bytes(3) == b'oob'
+    assert unpacker.unpack() == ord(b'a')
+    assert unpacker.unpack() == ord(b'r')
+
+    # Test buffer refill
+    unpacker = Unpacker(six.BytesIO(b'foobar'), read_size=3)
+    assert unpacker.unpack() == ord(b'f')
+    assert unpacker.read_bytes(3) == b'oob'
+    assert unpacker.unpack() == ord(b'a')
+    assert unpacker.unpack() == ord(b'r')
+
 if __name__ == '__main__':
     nose.main()

From 60df5eadaf507594b73e5e5a887da1fc52cb3f32 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 24 Sep 2012 02:12:55 +0900
Subject: [PATCH 1045/1648] Warn when use_list is not specified.

---
 msgpack/_msgpack.pyx   | 30 ++++++++++++++++++++++--------
 setup.py               |  2 +-
 test/test_buffer.py    |  4 ++--
 test/test_format.py    |  4 ++--
 test/test_pack.py      | 12 ++++++------
 test/test_seq.py       |  2 +-
 test/test_sequnpack.py |  6 +++---
 7 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index e0a1043..d0c4541 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -1,12 +1,16 @@
 # coding: utf-8
 #cython: embedsignature=True
 
+import warnings
+
 from cpython cimport *
 cdef extern from "Python.h":
     ctypedef char* const_char_ptr "const char*"
     ctypedef char* const_void_ptr "const void*"
     ctypedef struct PyObject
     cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
+    char* __FILE__
+    int __LINE__
 
 from libc.stdlib cimport *
 from libc.string cimport *
@@ -195,7 +199,7 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use
 
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
-        int use_list
+        bint use_list
         PyObject* object_hook
         PyObject* list_hook
         char *encoding
@@ -215,7 +219,7 @@ cdef extern from "unpack.h":
 
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
-            bint use_list=0, encoding=None, unicode_errors="strict",
+            use_list=None, encoding=None, unicode_errors="strict",
             ):
     """Unpack packed_bytes to object. Returns an unpacked object.
 
@@ -227,6 +231,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
     cdef char* buf
     cdef Py_ssize_t buf_len
+
     PyObject_AsReadBuffer(packed, &buf, &buf_len)
 
     if encoding is None:
@@ -245,7 +250,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         err = PyBytes_AsString(berrors)
 
     template_init(&ctx)
-    ctx.user.use_list = use_list
+    if use_list is None:
+        warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1)
+        ctx.user.use_list = 0
+    else:
+        ctx.user.use_list = use_list
     ctx.user.object_hook = ctx.user.list_hook = NULL
     ctx.user.encoding = enc
     ctx.user.unicode_errors = err
@@ -268,12 +277,15 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None,
-           bint use_list=0, encoding=None, unicode_errors="strict",
+           use_list=None, encoding=None, unicode_errors="strict",
            ):
     """Unpack an object from `stream`.
 
     Raises `ValueError` when `stream` has extra bytes.
     """
+    if use_list is None:
+        warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1)
+        use_list = 0
     return unpackb(stream.read(), use_list=use_list,
                    object_hook=object_hook, list_hook=list_hook,
                    encoding=encoding, unicode_errors=unicode_errors,
@@ -292,7 +304,7 @@ cdef class Unpacker(object):
     (default: min(1024**2, max_buffer_size))
 
     If `use_list` is true, msgpack list is deserialized to Python list.
-    Otherwise, it is deserialized to Python tuple. (default: False)
+    Otherwise, it is deserialized to Python tuple.
 
     `object_hook` is same to simplejson. If it is not None, it should be callable
     and Unpacker calls it when deserializing key-value.
@@ -330,7 +342,6 @@ cdef class Unpacker(object):
     cdef object file_like
     cdef object file_like_read
     cdef Py_ssize_t read_size
-    cdef bint use_list
     cdef object object_hook
     cdef object _bencoding
     cdef object _berrors
@@ -345,12 +356,15 @@ cdef class Unpacker(object):
         free(self.buf)
         self.buf = NULL
 
-    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
+    def __init__(self, file_like=None, Py_ssize_t read_size=0, use_list=None,
                  object object_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict', int max_buffer_size=0,
                  object object_pairs_hook=None,
                  ):
-        self.use_list = use_list
+        if use_list is None:
+            warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1)
+            use_list = 0
+
         self.file_like = file_like
         if file_like:
             self.file_like_read = file_like.read
diff --git a/setup.py b/setup.py
index 86b0b34..9f0ce5d 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,7 @@ except ImportError:
 
 def cythonize(src):
     sys.stderr.write("cythonize: %r\n" % (src,))
-    cython_compiler.compile([src])
+    cython_compiler.compile([src], emit_linenums=True)
 
 def ensure_source(src):
     pyx = os.path.splitext(src)[0] + '.pyx'
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 01310a0..785fb60 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -9,8 +9,8 @@ def test_unpack_buffer():
     from array import array
     buf = array('b')
     buf.fromstring(packb(('foo', 'bar')))
-    obj = unpackb(buf)
-    assert_equal((b'foo', b'bar'), obj)
+    obj = unpackb(buf, use_list=1)
+    assert_equal([b'foo', b'bar'], obj)
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_format.py b/test/test_format.py
index c03b3e2..ac08709 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -5,8 +5,8 @@ from nose import main
 from nose.tools import *
 from msgpack import unpackb
 
-def check(src, should):
-    assert_equal(unpackb(src), should)
+def check(src, should, use_list=0):
+    assert_equal(unpackb(src, use_list=use_list), should)
 
 def testSimpleValue():
     check(b"\x93\xc0\xc2\xc3", 
diff --git a/test/test_pack.py b/test/test_pack.py
index b216c46..dc77dfe 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -11,8 +11,8 @@ from msgpack import packb, unpackb, Unpacker, Packer
 
 from io import BytesIO
 
-def check(data):
-    re = unpackb(packb(data))
+def check(data, use_list=False):
+    re = unpackb(packb(data), use_list=use_list)
     assert_equal(re, data)
 
 def testPack():
@@ -34,7 +34,7 @@ def testPackUnicode():
         six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"),
         ]
     for td in test_data:
-        re = unpackb(packb(td, encoding='utf-8'), encoding='utf-8')
+        re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8')
         assert_equal(re, td)
         packer = Packer(encoding='utf-8')
         data = packer.pack(td)
@@ -46,11 +46,11 @@ def testPackUTF32():
         test_data = [
             six.u(""),
             six.u("abcd"),
-            (six.u("defgh"),),
+            [six.u("defgh")],
             six.u("РуÑÑкий текÑÑ‚"),
             ]
         for td in test_data:
-            re = unpackb(packb(td, encoding='utf-32'), encoding='utf-32')
+            re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
             assert_equal(re, td)
     except LookupError:
         raise SkipTest
@@ -110,7 +110,7 @@ class odict(dict):
 def test_odict():
     seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
     od = odict(seq)
-    assert_equal(unpackb(packb(od)), dict(seq))
+    assert_equal(unpackb(packb(od), use_list=1), dict(seq))
     # After object_pairs_hook is implemented.
     #def pair_hook(seq):
     #    return seq
diff --git a/test/test_seq.py b/test/test_seq.py
index d0f9ccc..72e935a 100644
--- a/test/test_seq.py
+++ b/test/test_seq.py
@@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size():
     f = io.BytesIO(dumpf.getvalue())
     dumpf.close()
 
-    unpacker = msgpack.Unpacker(f, read_size=read_size)
+    unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1)
 
     read_count = 0
     for idx, o in enumerate(unpacker):
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index aa47d3c..dac36a8 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -5,7 +5,7 @@ from msgpack import Unpacker, BufferFull
 import nose
 
 def test_foobar():
-    unpacker = Unpacker(read_size=3)
+    unpacker = Unpacker(read_size=3, use_list=1)
     unpacker.feed(b'foobar')
     assert unpacker.unpack() == ord(b'f')
     assert unpacker.unpack() == ord(b'o')
@@ -29,7 +29,7 @@ def test_foobar():
     assert k == len(b'foobar')
 
 def test_foobar_skip():
-    unpacker = Unpacker(read_size=3)
+    unpacker = Unpacker(read_size=3, use_list=1)
     unpacker.feed(b'foobar')
     assert unpacker.unpack() == ord(b'f')
     unpacker.skip()
@@ -45,7 +45,7 @@ def test_foobar_skip():
 
 def test_maxbuffersize():
     nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3)
-    unpacker = Unpacker(read_size=3, max_buffer_size=3)
+    unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
     unpacker.feed(b'fo')
     nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob')
     unpacker.feed(b'o')

From c2a2d417f1e3a95992b41c3b24e6470077c99c6f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 24 Sep 2012 02:20:53 +0900
Subject: [PATCH 1046/1648] Fix warnings in tests.

---
 test/test_case.py |  6 +++---
 test/test_obj.py  |  8 ++++----
 test/test_pack.py | 15 +++++++--------
 3 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/test/test_case.py b/test/test_case.py
index b88714d..9cbf9bd 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -9,7 +9,7 @@ from msgpack import packb, unpackb
 def check(length, obj):
     v = packb(obj)
     assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v)))
-    assert_equal(unpackb(v), obj)
+    assert_equal(unpackb(v, use_list=0), obj)
 
 def test_1():
     for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
@@ -71,7 +71,7 @@ def test_array32():
 
 def match(obj, buf):
     assert_equal(packb(obj), buf)
-    assert_equal(unpackb(buf), obj)
+    assert_equal(unpackb(buf, use_list=0), obj)
 
 def test_match():
     cases = [
@@ -99,7 +99,7 @@ def test_match():
         match(v, p)
 
 def test_unicode():
-    assert_equal(b'foobar', unpackb(packb('foobar')))
+    assert_equal(b'foobar', unpackb(packb('foobar'), use_list=1))
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_obj.py b/test/test_obj.py
index d155b73..d809093 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -18,25 +18,25 @@ def _encode_complex(obj):
 
 def test_encode_hook():
     packed = packb([3, 1+2j], default=_encode_complex)
-    unpacked = unpackb(packed)
+    unpacked = unpackb(packed, use_list=1)
     eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2})
 
 def test_decode_hook():
     packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
-    unpacked = unpackb(packed, object_hook=_decode_complex)
+    unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1)
     eq_(unpacked[1], 1+2j)
 
 @raises(ValueError)
 def test_bad_hook():
     packed = packb([3, 1+2j], default=lambda o: o)
-    unpacked = unpackb(packed)
+    unpacked = unpackb(packed, use_list=1)
 
 def _arr_to_str(arr):
     return ''.join(str(c) for c in arr)
 
 def test_array_hook():
     packed = packb([1,2,3])
-    unpacked = unpackb(packed, list_hook=_arr_to_str)
+    unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1)
     eq_(unpacked, '123')
 
 if __name__ == '__main__':
diff --git a/test/test_pack.py b/test/test_pack.py
index dc77dfe..9bd2b32 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -31,14 +31,14 @@ def testPack():
 
 def testPackUnicode():
     test_data = [
-        six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"),
+        six.u(""), six.u("abcd"), [six.u("defgh")], six.u("РуÑÑкий текÑÑ‚"),
         ]
     for td in test_data:
-        re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8')
+        re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8')
         assert_equal(re, td)
         packer = Packer(encoding='utf-8')
         data = packer.pack(td)
-        re = Unpacker(BytesIO(data), encoding='utf-8').unpack()
+        re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack()
         assert_equal(re, td)
 
 def testPackUTF32():
@@ -63,20 +63,19 @@ def testPackBytes():
         check(td)
 
 def testIgnoreUnicodeErrors():
-    re = unpackb(packb(b'abc\xeddef'),
-        encoding='utf-8', unicode_errors='ignore')
+    re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
     assert_equal(re, "abcdef")
 
 @raises(UnicodeDecodeError)
 def testStrictUnicodeUnpack():
-    unpackb(packb(b'abc\xeddef'), encoding='utf-8')
+    unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1)
 
 @raises(UnicodeEncodeError)
 def testStrictUnicodePack():
     packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict')
 
 def testIgnoreErrorsPack():
-    re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
+    re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1)
     assert_equal(re, six.u("abcdef"))
 
 @raises(TypeError)
@@ -84,7 +83,7 @@ def testNoEncoding():
     packb(six.u("abc"), encoding=None)
 
 def testDecodeBinary():
-    re = unpackb(packb("abc"), encoding=None)
+    re = unpackb(packb("abc"), encoding=None, use_list=1)
     assert_equal(re, b"abc")
 
 def testPackFloat():

From d503788e9537498ff2ed0da1f836dc4de6074981 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 24 Sep 2012 02:12:55 +0900
Subject: [PATCH 1047/1648] Warn when use_list is not specified.

Conflicts:
	test/test_sequnpack.py
---
 msgpack/_msgpack.pyx   | 30 ++++++++++++++++++++++--------
 setup.py               |  2 +-
 test/test_buffer.py    |  4 ++--
 test/test_format.py    |  4 ++--
 test/test_pack.py      | 12 ++++++------
 test/test_seq.py       |  2 +-
 test/test_sequnpack.py |  5 ++---
 7 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 0886580..44a50ae 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -1,12 +1,16 @@
 # coding: utf-8
 #cython: embedsignature=True
 
+import warnings
+
 from cpython cimport *
 cdef extern from "Python.h":
     ctypedef char* const_char_ptr "const char*"
     ctypedef char* const_void_ptr "const void*"
     ctypedef struct PyObject
     cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
+    char* __FILE__
+    int __LINE__
 
 from libc.stdlib cimport *
 from libc.string cimport *
@@ -195,7 +199,7 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use
 
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
-        int use_list
+        bint use_list
         PyObject* object_hook
         PyObject* list_hook
         char *encoding
@@ -215,7 +219,7 @@ cdef extern from "unpack.h":
 
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
-            bint use_list=0, encoding=None, unicode_errors="strict",
+            use_list=None, encoding=None, unicode_errors="strict",
             ):
     """Unpack packed_bytes to object. Returns an unpacked object.
 
@@ -227,6 +231,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
     cdef char* buf
     cdef Py_ssize_t buf_len
+
     PyObject_AsReadBuffer(packed, &buf, &buf_len)
 
     if encoding is None:
@@ -245,7 +250,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         err = PyBytes_AsString(berrors)
 
     template_init(&ctx)
-    ctx.user.use_list = use_list
+    if use_list is None:
+        warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1)
+        ctx.user.use_list = 0
+    else:
+        ctx.user.use_list = use_list
     ctx.user.object_hook = ctx.user.list_hook = NULL
     ctx.user.encoding = enc
     ctx.user.unicode_errors = err
@@ -268,12 +277,15 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None,
-           bint use_list=0, encoding=None, unicode_errors="strict",
+           use_list=None, encoding=None, unicode_errors="strict",
            ):
     """Unpack an object from `stream`.
 
     Raises `ValueError` when `stream` has extra bytes.
     """
+    if use_list is None:
+        warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1)
+        use_list = 0
     return unpackb(stream.read(), use_list=use_list,
                    object_hook=object_hook, list_hook=list_hook,
                    encoding=encoding, unicode_errors=unicode_errors,
@@ -292,7 +304,7 @@ cdef class Unpacker(object):
     (default: min(1024**2, max_buffer_size))
 
     If `use_list` is true, msgpack list is deserialized to Python list.
-    Otherwise, it is deserialized to Python tuple. (default: False)
+    Otherwise, it is deserialized to Python tuple.
 
     `object_hook` is same to simplejson. If it is not None, it should be callable
     and Unpacker calls it when deserializing key-value.
@@ -330,7 +342,6 @@ cdef class Unpacker(object):
     cdef object file_like
     cdef object file_like_read
     cdef Py_ssize_t read_size
-    cdef bint use_list
     cdef object object_hook
     cdef object _bencoding
     cdef object _berrors
@@ -345,12 +356,15 @@ cdef class Unpacker(object):
         free(self.buf)
         self.buf = NULL
 
-    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=0,
+    def __init__(self, file_like=None, Py_ssize_t read_size=0, use_list=None,
                  object object_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict', int max_buffer_size=0,
                  object object_pairs_hook=None,
                  ):
-        self.use_list = use_list
+        if use_list is None:
+            warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1)
+            use_list = 0
+
         self.file_like = file_like
         if file_like:
             self.file_like_read = file_like.read
diff --git a/setup.py b/setup.py
index 86b0b34..9f0ce5d 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,7 @@ except ImportError:
 
 def cythonize(src):
     sys.stderr.write("cythonize: %r\n" % (src,))
-    cython_compiler.compile([src])
+    cython_compiler.compile([src], emit_linenums=True)
 
 def ensure_source(src):
     pyx = os.path.splitext(src)[0] + '.pyx'
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 01310a0..785fb60 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -9,8 +9,8 @@ def test_unpack_buffer():
     from array import array
     buf = array('b')
     buf.fromstring(packb(('foo', 'bar')))
-    obj = unpackb(buf)
-    assert_equal((b'foo', b'bar'), obj)
+    obj = unpackb(buf, use_list=1)
+    assert_equal([b'foo', b'bar'], obj)
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_format.py b/test/test_format.py
index c03b3e2..ac08709 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -5,8 +5,8 @@ from nose import main
 from nose.tools import *
 from msgpack import unpackb
 
-def check(src, should):
-    assert_equal(unpackb(src), should)
+def check(src, should, use_list=0):
+    assert_equal(unpackb(src, use_list=use_list), should)
 
 def testSimpleValue():
     check(b"\x93\xc0\xc2\xc3", 
diff --git a/test/test_pack.py b/test/test_pack.py
index b216c46..dc77dfe 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -11,8 +11,8 @@ from msgpack import packb, unpackb, Unpacker, Packer
 
 from io import BytesIO
 
-def check(data):
-    re = unpackb(packb(data))
+def check(data, use_list=False):
+    re = unpackb(packb(data), use_list=use_list)
     assert_equal(re, data)
 
 def testPack():
@@ -34,7 +34,7 @@ def testPackUnicode():
         six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"),
         ]
     for td in test_data:
-        re = unpackb(packb(td, encoding='utf-8'), encoding='utf-8')
+        re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8')
         assert_equal(re, td)
         packer = Packer(encoding='utf-8')
         data = packer.pack(td)
@@ -46,11 +46,11 @@ def testPackUTF32():
         test_data = [
             six.u(""),
             six.u("abcd"),
-            (six.u("defgh"),),
+            [six.u("defgh")],
             six.u("РуÑÑкий текÑÑ‚"),
             ]
         for td in test_data:
-            re = unpackb(packb(td, encoding='utf-32'), encoding='utf-32')
+            re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
             assert_equal(re, td)
     except LookupError:
         raise SkipTest
@@ -110,7 +110,7 @@ class odict(dict):
 def test_odict():
     seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
     od = odict(seq)
-    assert_equal(unpackb(packb(od)), dict(seq))
+    assert_equal(unpackb(packb(od), use_list=1), dict(seq))
     # After object_pairs_hook is implemented.
     #def pair_hook(seq):
     #    return seq
diff --git a/test/test_seq.py b/test/test_seq.py
index d0f9ccc..72e935a 100644
--- a/test/test_seq.py
+++ b/test/test_seq.py
@@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size():
     f = io.BytesIO(dumpf.getvalue())
     dumpf.close()
 
-    unpacker = msgpack.Unpacker(f, read_size=read_size)
+    unpacker = msgpack.Unpacker(f, read_size=read_size, use_list=1)
 
     read_count = 0
     for idx, o in enumerate(unpacker):
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index b1b80b2..21fc3be 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -5,7 +5,7 @@ from msgpack import Unpacker, BufferFull
 import nose
 
 def test_foobar():
-    unpacker = Unpacker(read_size=3)
+    unpacker = Unpacker(read_size=3, use_list=1)
     unpacker.feed(b'foobar')
     assert unpacker.unpack() == ord(b'f')
     assert unpacker.unpack() == ord(b'o')
@@ -28,10 +28,9 @@ def test_foobar():
         k += 1
     assert k == len(b'foobar')
 
-
 def test_maxbuffersize():
     nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3)
-    unpacker = Unpacker(read_size=3, max_buffer_size=3)
+    unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
     unpacker.feed(b'fo')
     nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob')
     unpacker.feed(b'o')

From c280e589884f2b4afd064cffd08b0f353db93036 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 24 Sep 2012 02:20:53 +0900
Subject: [PATCH 1048/1648] Fix warnings in tests.

---
 test/test_case.py |  6 +++---
 test/test_obj.py  |  8 ++++----
 test/test_pack.py | 15 +++++++--------
 3 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/test/test_case.py b/test/test_case.py
index b88714d..9cbf9bd 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -9,7 +9,7 @@ from msgpack import packb, unpackb
 def check(length, obj):
     v = packb(obj)
     assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v)))
-    assert_equal(unpackb(v), obj)
+    assert_equal(unpackb(v, use_list=0), obj)
 
 def test_1():
     for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
@@ -71,7 +71,7 @@ def test_array32():
 
 def match(obj, buf):
     assert_equal(packb(obj), buf)
-    assert_equal(unpackb(buf), obj)
+    assert_equal(unpackb(buf, use_list=0), obj)
 
 def test_match():
     cases = [
@@ -99,7 +99,7 @@ def test_match():
         match(v, p)
 
 def test_unicode():
-    assert_equal(b'foobar', unpackb(packb('foobar')))
+    assert_equal(b'foobar', unpackb(packb('foobar'), use_list=1))
 
 if __name__ == '__main__':
     main()
diff --git a/test/test_obj.py b/test/test_obj.py
index d155b73..d809093 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -18,25 +18,25 @@ def _encode_complex(obj):
 
 def test_encode_hook():
     packed = packb([3, 1+2j], default=_encode_complex)
-    unpacked = unpackb(packed)
+    unpacked = unpackb(packed, use_list=1)
     eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2})
 
 def test_decode_hook():
     packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
-    unpacked = unpackb(packed, object_hook=_decode_complex)
+    unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1)
     eq_(unpacked[1], 1+2j)
 
 @raises(ValueError)
 def test_bad_hook():
     packed = packb([3, 1+2j], default=lambda o: o)
-    unpacked = unpackb(packed)
+    unpacked = unpackb(packed, use_list=1)
 
 def _arr_to_str(arr):
     return ''.join(str(c) for c in arr)
 
 def test_array_hook():
     packed = packb([1,2,3])
-    unpacked = unpackb(packed, list_hook=_arr_to_str)
+    unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1)
     eq_(unpacked, '123')
 
 if __name__ == '__main__':
diff --git a/test/test_pack.py b/test/test_pack.py
index dc77dfe..9bd2b32 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -31,14 +31,14 @@ def testPack():
 
 def testPackUnicode():
     test_data = [
-        six.u(""), six.u("abcd"), (six.u("defgh"),), six.u("РуÑÑкий текÑÑ‚"),
+        six.u(""), six.u("abcd"), [six.u("defgh")], six.u("РуÑÑкий текÑÑ‚"),
         ]
     for td in test_data:
-        re = unpackb(packb(td, encoding='utf-8'), use_list=0, encoding='utf-8')
+        re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8')
         assert_equal(re, td)
         packer = Packer(encoding='utf-8')
         data = packer.pack(td)
-        re = Unpacker(BytesIO(data), encoding='utf-8').unpack()
+        re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack()
         assert_equal(re, td)
 
 def testPackUTF32():
@@ -63,20 +63,19 @@ def testPackBytes():
         check(td)
 
 def testIgnoreUnicodeErrors():
-    re = unpackb(packb(b'abc\xeddef'),
-        encoding='utf-8', unicode_errors='ignore')
+    re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
     assert_equal(re, "abcdef")
 
 @raises(UnicodeDecodeError)
 def testStrictUnicodeUnpack():
-    unpackb(packb(b'abc\xeddef'), encoding='utf-8')
+    unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1)
 
 @raises(UnicodeEncodeError)
 def testStrictUnicodePack():
     packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict')
 
 def testIgnoreErrorsPack():
-    re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8')
+    re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1)
     assert_equal(re, six.u("abcdef"))
 
 @raises(TypeError)
@@ -84,7 +83,7 @@ def testNoEncoding():
     packb(six.u("abc"), encoding=None)
 
 def testDecodeBinary():
-    re = unpackb(packb("abc"), encoding=None)
+    re = unpackb(packb("abc"), encoding=None, use_list=1)
     assert_equal(re, b"abc")
 
 def testPackFloat():

From 15a46eb143d979c7b55f30d12d8bfe6a846f2a7e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 24 Sep 2012 02:42:38 +0900
Subject: [PATCH 1049/1648] use_list=1 is default

---
 msgpack/_msgpack.pyx | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index d0c4541..e932ba9 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -219,7 +219,7 @@ cdef extern from "unpack.h":
 
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
-            use_list=None, encoding=None, unicode_errors="strict",
+            bint use_list=1, encoding=None, unicode_errors="strict",
             ):
     """Unpack packed_bytes to object. Returns an unpacked object.
 
@@ -250,11 +250,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         err = PyBytes_AsString(berrors)
 
     template_init(&ctx)
-    if use_list is None:
-        warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1)
-        ctx.user.use_list = 0
-    else:
-        ctx.user.use_list = use_list
+    ctx.user.use_list = use_list
     ctx.user.object_hook = ctx.user.list_hook = NULL
     ctx.user.encoding = enc
     ctx.user.unicode_errors = err
@@ -277,15 +273,12 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None,
-           use_list=None, encoding=None, unicode_errors="strict",
+           bint use_list=1, encoding=None, unicode_errors="strict",
            ):
     """Unpack an object from `stream`.
 
     Raises `ValueError` when `stream` has extra bytes.
     """
-    if use_list is None:
-        warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1)
-        use_list = 0
     return unpackb(stream.read(), use_list=use_list,
                    object_hook=object_hook, list_hook=list_hook,
                    encoding=encoding, unicode_errors=unicode_errors,
@@ -356,15 +349,11 @@ cdef class Unpacker(object):
         free(self.buf)
         self.buf = NULL
 
-    def __init__(self, file_like=None, Py_ssize_t read_size=0, use_list=None,
+    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
                  object object_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict', int max_buffer_size=0,
                  object object_pairs_hook=None,
                  ):
-        if use_list is None:
-            warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1)
-            use_list = 0
-
         self.file_like = file_like
         if file_like:
             self.file_like_read = file_like.read

From ac403ef68da911ac4735407ada88db02015bd520 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 24 Sep 2012 02:45:37 +0900
Subject: [PATCH 1050/1648] Start 0.2.3dev

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index f343b7a..9bffd02 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 2, 2)
+version = (0, 2, 3, 'dev1')

From 927d29131dc8d2a9f606cf7c881606d47ace557b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 24 Sep 2012 02:55:50 +0900
Subject: [PATCH 1051/1648] Write about warning in changelog.

---
 ChangeLog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 46b83ee..f49b577 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -4,6 +4,7 @@
 
 Changes
 -------
+* Warn when use_list is not specified. It's default value will be changed in 0.3.
 
 Bugs fixed
 -----------

From 477d3b152f5d36a48a8083b3720def2dd1f5d1a7 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 24 Sep 2012 03:08:13 +0900
Subject: [PATCH 1052/1648] Fix warnings.

---
 test/test_obj.py  | 2 +-
 test/test_pack.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_obj.py b/test/test_obj.py
index 12a149f..881e627 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -29,7 +29,7 @@ def test_decode_hook():
 def test_decode_pairs_hook():
     packed = packb([3, {1: 2, 3: 4}])
     prod_sum = 1 * 2 + 3 * 4
-    unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l))
+    unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1)
     eq_(unpacked[1], prod_sum)
 
 @raises(ValueError)
diff --git a/test/test_pack.py b/test/test_pack.py
index 9009d35..6af87fd 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -112,7 +112,7 @@ def test_odict():
     assert_equal(unpackb(packb(od), use_list=1), dict(seq))
     def pair_hook(seq):
         return seq
-    assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook), seq)
+    assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1), seq)
 
 
 if __name__ == '__main__':

From d56e2b2c8aa1005fbac3b584cd003ba0cdece2e2 Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Tue, 25 Sep 2012 00:30:15 +1000
Subject: [PATCH 1053/1648] Use C++ function templating for skip()/construct()

---
 msgpack/_msgpack.pyx      | 23 +++++++++++------------
 msgpack/unpack.h          |  1 +
 msgpack/unpack_template.h | 21 +++++++++++++--------
 setup.py                  |  4 ++--
 4 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index e0a1043..0fc3739 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -208,8 +208,10 @@ cdef extern from "unpack.h":
         unsigned int ct
         PyObject* key
 
-    int template_execute(template_context* ctx, const_char_ptr data,
-                         size_t len, size_t* off, bint construct) except -1
+    ctypedef int (*execute_fn)(template_context* ctx, const_char_ptr data,
+                               size_t len, size_t* off) except -1
+    execute_fn template_construct
+    execute_fn template_skip
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
@@ -257,7 +259,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         if not PyCallable_Check(list_hook):
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
-    ret = template_execute(&ctx, buf, buf_len, &off, 1)
+    ret = template_construct(&ctx, buf, buf_len, &off)
     if ret == 1:
         obj = template_data(&ctx)
         if off < buf_len:
@@ -455,16 +457,13 @@ cdef class Unpacker(object):
         else:
             self.file_like = None
 
-    cdef object _unpack(self, bint construct):
+    cdef object _unpack(self, execute_fn execute):
         cdef int ret
         cdef object obj
         while 1:
-            ret = template_execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head, construct)
+            ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
             if ret == 1:
-                if construct:
-                    obj = template_data(&self.ctx)
-                else:
-                    obj = None
+                obj = template_data(&self.ctx)
                 template_init(&self.ctx)
                 return obj
             elif ret == 0:
@@ -477,17 +476,17 @@ cdef class Unpacker(object):
 
     def unpack(self):
         """unpack one object"""
-        return self._unpack(1)
+        return self._unpack(template_construct)
 
     def skip(self):
         """read and ignore one object, returning None"""
-        return self._unpack(0)
+        return self._unpack(template_skip)
 
     def __iter__(self):
         return self
 
     def __next__(self):
-        return self._unpack(1)
+        return self._unpack(template_construct)
 
     # for debug.
     #def _buf(self):
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index a106f9c..3c9d4be 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -41,6 +41,7 @@ typedef struct unpack_user {
 
 #define msgpack_unpack_user unpack_user
 
+typedef int (*execute_fn)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off);
 
 struct template_context;
 typedef struct template_context template_context;
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 5495a51..e0cf42e 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -95,7 +95,8 @@ msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context
 }
 
 
-msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off, int construct)
+template 
+msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
 {
 	assert(len >= *off);
 
@@ -380,6 +381,8 @@ _header_again:
 
 
 _finish:
+	if (!construct)
+		msgpack_unpack_callback(_nil)(user, &obj);
 	stack[0].obj = obj;
 	++p;
 	ret = 1;
@@ -405,13 +408,6 @@ _end:
 #undef construct_cb
 }
 
-
-#undef msgpack_unpack_func
-#undef msgpack_unpack_callback
-#undef msgpack_unpack_struct
-#undef msgpack_unpack_object
-#undef msgpack_unpack_user
-
 #undef push_simple_value
 #undef push_fixed_value
 #undef push_variable_value
@@ -419,6 +415,15 @@ _end:
 #undef again_fixed_trail_if_zero
 #undef start_container
 
+static const execute_fn template_construct = &template_execute;
+static const execute_fn template_skip = &template_execute;
+
+#undef msgpack_unpack_func
+#undef msgpack_unpack_callback
+#undef msgpack_unpack_struct
+#undef msgpack_unpack_object
+#undef msgpack_unpack_user
+
 #undef NEXT_CS
 
 /* vim: set ts=4 sw=4 noexpandtab  */
diff --git a/setup.py b/setup.py
index 86b0b34..708fa13 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,7 @@ except ImportError:
 
 def cythonize(src):
     sys.stderr.write("cythonize: %r\n" % (src,))
-    cython_compiler.compile([src])
+    cython_compiler.compile([src], cplus=True)
 
 def ensure_source(src):
     pyx = os.path.splitext(src)[0] + '.pyx'
@@ -67,7 +67,7 @@ if have_cython:
 else:
     Sdist = sdist
 
-sources = ['msgpack/_msgpack.c']
+sources = ['msgpack/_msgpack.cpp']
 libraries = []
 if sys.platform == 'win32':
     libraries.append('ws2_32')

From 0431a766f4e069d74627441aa3facbc7e64e4511 Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Tue, 25 Sep 2012 01:18:33 +1000
Subject: [PATCH 1054/1648] read_array/map_header functionality

---
 msgpack/_msgpack.pyx      | 10 ++++++
 msgpack/unpack_template.h | 63 +++++++++++++++++++++++++++++++++++++
 test/test_read_size.py    | 66 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 139 insertions(+)
 create mode 100644 test/test_read_size.py

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 0fc3739..7131d1f 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -212,6 +212,8 @@ cdef extern from "unpack.h":
                                size_t len, size_t* off) except -1
     execute_fn template_construct
     execute_fn template_skip
+    execute_fn read_array_header
+    execute_fn read_map_header
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
@@ -482,6 +484,14 @@ cdef class Unpacker(object):
         """read and ignore one object, returning None"""
         return self._unpack(template_skip)
 
+    def read_array_header(self):
+        """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents."""
+        return self._unpack(read_array_header)
+
+    def read_map_header(self):
+        """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs."""
+        return self._unpack(read_map_header)
+
     def __iter__(self):
         return self
 
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index e0cf42e..69ef6e2 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -408,6 +408,10 @@ _end:
 #undef construct_cb
 }
 
+#undef SWITCH_RANGE_BEGIN
+#undef SWITCH_RANGE
+#undef SWITCH_RANGE_DEFAULT
+#undef SWITCH_RANGE_END
 #undef push_simple_value
 #undef push_fixed_value
 #undef push_variable_value
@@ -415,8 +419,67 @@ _end:
 #undef again_fixed_trail_if_zero
 #undef start_container
 
+template 
+msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
+{
+	assert(len >= *off);
+	uint32_t size;
+	const unsigned char *const p = (unsigned char*)data + *off;
+
+#define inc_offset(inc) \
+	if (len - *off < inc) \
+		return 0; \
+	*off += inc;
+
+	switch (*p) {
+	case var_offset:
+		inc_offset(3);
+		size = _msgpack_load16(uint16_t, p + 1);
+		break;
+	case var_offset + 1:
+		inc_offset(5);
+		size = _msgpack_load32(uint32_t, p + 1);
+		break;
+#ifdef USE_CASE_RANGE
+	case fixed_offset + 0x0 ... fixed_offset + 0xf:
+#else
+	case fixed_offset + 0x0:
+	case fixed_offset + 0x1:
+	case fixed_offset + 0x2:
+	case fixed_offset + 0x3:
+	case fixed_offset + 0x4:
+	case fixed_offset + 0x5:
+	case fixed_offset + 0x6:
+	case fixed_offset + 0x7:
+	case fixed_offset + 0x8:
+	case fixed_offset + 0x9:
+	case fixed_offset + 0xa:
+	case fixed_offset + 0xb:
+	case fixed_offset + 0xc:
+	case fixed_offset + 0xd:
+	case fixed_offset + 0xe:
+	case fixed_offset + 0xf:
+#endif
+		++*off;
+		size = ((unsigned int)*p) & 0x0f;
+		break;
+	default:
+		PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
+		return -1;
+    }
+	msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj);
+	return 1;
+}
+
+#undef SWITCH_RANGE_BEGIN
+#undef SWITCH_RANGE
+#undef SWITCH_RANGE_DEFAULT
+#undef SWITCH_RANGE_END
+
 static const execute_fn template_construct = &template_execute;
 static const execute_fn template_skip = &template_execute;
+static const execute_fn read_array_header = &template_container_header<0x90, 0xdc>;
+static const execute_fn read_map_header = &template_container_header<0x80, 0xde>;
 
 #undef msgpack_unpack_func
 #undef msgpack_unpack_callback
diff --git a/test/test_read_size.py b/test/test_read_size.py
new file mode 100644
index 0000000..714f963
--- /dev/null
+++ b/test/test_read_size.py
@@ -0,0 +1,66 @@
+"""Test Unpacker's read_array_header and read_map_header methods"""
+from msgpack import packb, Unpacker
+UnexpectedTypeException = ValueError
+
+def test_read_array_header():
+    unpacker = Unpacker()
+    unpacker.feed(packb(['a', 'b', 'c']))
+    assert unpacker.read_array_header() == 3
+    assert unpacker.unpack() == 'a'
+    assert unpacker.unpack() == 'b'
+    assert unpacker.unpack() == 'c'
+    try:
+        unpacker.unpack()
+        assert 0, 'should raise exception'
+    except StopIteration:
+        assert 1, 'okay'
+
+
+def test_read_map_header():
+    unpacker = Unpacker()
+    unpacker.feed(packb({'a': 'A'}))
+    assert unpacker.read_map_header() == 1
+    assert unpacker.unpack() == 'a'
+    assert unpacker.unpack() == 'A'
+    try:
+        unpacker.unpack()
+        assert 0, 'should raise exception'
+    except StopIteration:
+        assert 1, 'okay'
+
+def test_incorrect_type_array():
+    unpacker = Unpacker()
+    unpacker.feed(packb(1))
+    try:
+        unpacker.read_array_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'
+
+def test_incorrect_type_map():
+    unpacker = Unpacker()
+    unpacker.feed(packb(1))
+    try:
+        unpacker.read_map_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'
+
+def test_correct_type_nested_array():
+    unpacker = Unpacker()
+    unpacker.feed(packb({'a': ['b', 'c', 'd']}))
+    try:
+        unpacker.read_array_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'
+
+def test_incorrect_type_nested_map():
+    unpacker = Unpacker()
+    unpacker.feed(packb([{'a': 'b'}]))
+    try:
+        unpacker.read_map_header()
+        assert 0, 'should raise exception'
+    except UnexpectedTypeException:
+        assert 1, 'okay'
+

From 9d9c3eecb846c6a927a31aae394dea39fa75aef4 Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Sun, 23 Sep 2012 17:26:16 +1000
Subject: [PATCH 1055/1648] Packer.pack_array/map_header to correspond to read
 functions

---
 msgpack/_msgpack.pyx | 11 +++++++++++
 test/test_pack.py    | 29 +++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 7131d1f..18a75ca 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -178,6 +178,17 @@ cdef class Packer(object):
         self.pk.length = 0
         return buf
 
+    cpdef pack_array_header(self, size_t size):
+        msgpack_pack_array(&self.pk, size)
+        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+        self.pk.length = 0
+        return buf
+
+    cpdef pack_map_header(self, size_t size):
+        msgpack_pack_map(&self.pk, size)
+        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+        self.pk.length = 0
+        return buf
 
 def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'):
     """
diff --git a/test/test_pack.py b/test/test_pack.py
index b216c46..937141d 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -91,6 +91,35 @@ def testPackFloat():
     assert_equal(packb(1.0, use_single_float=True),  b'\xca' + struct.pack('>f', 1.0))
     assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0))
 
+def testArraySize(sizes=[0, 5, 50, 1000]):
+    bio = six.BytesIO()
+    packer = Packer()
+    for size in sizes:
+        bio.write(packer.pack_array_header(size))
+        for i in range(size):
+            bio.write(packer.pack(i))
+
+    bio.seek(0)
+    unpacker = Unpacker(bio)
+    for size in sizes:
+        assert unpacker.unpack() == tuple(range(size))
+
+def testMapSize(sizes=[0, 5, 50, 1000]):
+    bio = six.BytesIO()
+    packer = Packer()
+    for size in sizes:
+        bio.write(packer.pack_map_header(size))
+        for i in range(size):
+            bio.write(packer.pack(i)) # key
+            bio.write(packer.pack(i * 2)) # value
+
+    bio.seek(0)
+    unpacker = Unpacker(bio)
+    for size in sizes:
+        assert unpacker.unpack() == {i: i * 2 for i in range(size)}
+
+
+
 
 class odict(dict):
     '''Reimplement OrderedDict to run test on Python 2.6'''

From d5f99959cc2ec393c13fc9e44714351272bac7fc Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 1 Oct 2012 01:34:58 +0900
Subject: [PATCH 1056/1648] Fix some test failure.

---
 test/test_pack.py      |  6 +++---
 test/test_read_size.py | 10 +++++-----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/test_pack.py b/test/test_pack.py
index ff1eeef..21c2bd7 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -99,9 +99,9 @@ def testArraySize(sizes=[0, 5, 50, 1000]):
             bio.write(packer.pack(i))
 
     bio.seek(0)
-    unpacker = Unpacker(bio)
+    unpacker = Unpacker(bio, use_list=1)
     for size in sizes:
-        assert unpacker.unpack() == tuple(range(size))
+        assert unpacker.unpack() == list(range(size))
 
 def testMapSize(sizes=[0, 5, 50, 1000]):
     bio = six.BytesIO()
@@ -115,7 +115,7 @@ def testMapSize(sizes=[0, 5, 50, 1000]):
     bio.seek(0)
     unpacker = Unpacker(bio)
     for size in sizes:
-        assert unpacker.unpack() == {i: i * 2 for i in range(size)}
+        assert unpacker.unpack() == dict((i, i * 2) for i in range(size))
 
 
 
diff --git a/test/test_read_size.py b/test/test_read_size.py
index 714f963..e130805 100644
--- a/test/test_read_size.py
+++ b/test/test_read_size.py
@@ -6,9 +6,9 @@ def test_read_array_header():
     unpacker = Unpacker()
     unpacker.feed(packb(['a', 'b', 'c']))
     assert unpacker.read_array_header() == 3
-    assert unpacker.unpack() == 'a'
-    assert unpacker.unpack() == 'b'
-    assert unpacker.unpack() == 'c'
+    assert unpacker.unpack() == b'a'
+    assert unpacker.unpack() == b'b'
+    assert unpacker.unpack() == b'c'
     try:
         unpacker.unpack()
         assert 0, 'should raise exception'
@@ -20,8 +20,8 @@ def test_read_map_header():
     unpacker = Unpacker()
     unpacker.feed(packb({'a': 'A'}))
     assert unpacker.read_map_header() == 1
-    assert unpacker.unpack() == 'a'
-    assert unpacker.unpack() == 'A'
+    assert unpacker.unpack() == B'a'
+    assert unpacker.unpack() == B'A'
     try:
         unpacker.unpack()
         assert 0, 'should raise exception'

From 87f292cbf929c0cf8c8de867a5e7dd285cf1550a Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Thu, 4 Oct 2012 11:26:29 +1000
Subject: [PATCH 1057/1648] Allow packed data to be captured while executing
 skip(), etc.

---
 msgpack/_msgpack.pyx    | 37 +++++++++++++++++++++++++------------
 test/test_unpack_raw.py | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 12 deletions(-)
 create mode 100644 test/test_unpack_raw.py

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 7fcfcb0..71f9810 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -467,11 +467,16 @@ cdef class Unpacker(object):
         else:
             self.file_like = None
 
-    cdef object _unpack(self, execute_fn execute):
+    cdef object _unpack(self, execute_fn execute, object write_bytes):
         cdef int ret
         cdef object obj
+        cdef size_t prev_head
         while 1:
+            prev_head = self.buf_head
             ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+            if write_bytes is not None:
+                write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
+
             if ret == 1:
                 obj = template_data(&self.ctx)
                 template_init(&self.ctx)
@@ -484,27 +489,35 @@ cdef class Unpacker(object):
             else:
                 raise ValueError("Unpack failed: error = %d" % (ret,))
 
-    def unpack(self):
-        """unpack one object"""
-        return self._unpack(template_construct)
+    def unpack(self, object write_bytes=None):
+        """
+        unpack one object
 
-    def skip(self):
-        """read and ignore one object, returning None"""
-        return self._unpack(template_skip)
+        If write_bytes is not None, it will be called with parts of the raw message as it is unpacked.
+        """
+        return self._unpack(template_construct, write_bytes)
 
-    def read_array_header(self):
+    def skip(self, object write_bytes=None):
+        """
+        read and ignore one object, returning None
+
+        If write_bytes is not None, it will be called with parts of the raw message as it is unpacked.
+        """
+        return self._unpack(template_skip, write_bytes)
+
+    def read_array_header(self, object write_bytes=None):
         """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents."""
-        return self._unpack(read_array_header)
+        return self._unpack(read_array_header, write_bytes)
 
-    def read_map_header(self):
+    def read_map_header(self, object write_bytes=None):
         """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs."""
-        return self._unpack(read_map_header)
+        return self._unpack(read_map_header, write_bytes)
 
     def __iter__(self):
         return self
 
     def __next__(self):
-        return self._unpack(template_construct)
+        return self._unpack(template_construct, None)
 
     # for debug.
     #def _buf(self):
diff --git a/test/test_unpack_raw.py b/test/test_unpack_raw.py
new file mode 100644
index 0000000..ed2e04b
--- /dev/null
+++ b/test/test_unpack_raw.py
@@ -0,0 +1,32 @@
+"""Tests for cases where the user seeks to obtain packed msgpack objects"""
+
+from nose import main
+from nose.tools import *
+import six
+from msgpack import Unpacker, packb
+
+def test_write_bytes():
+    unpacker = Unpacker()
+    unpacker.feed(b'abc')
+    f = six.BytesIO()
+    assert_equal(unpacker.unpack(f.write), ord('a'))
+    assert_equal(f.getvalue(), b'a')
+    f.truncate(0)
+    assert_is_none(unpacker.skip(f.write))
+    assert_equal(f.getvalue(), b'b')
+    f.truncate(0)
+    assert_is_none(unpacker.skip())
+    assert_equal(f.getvalue(), b'')
+
+def test_write_bytes_multi_buffer():
+    long_val = (5) * 100
+    expected = packb(long_val)
+    unpacker = Unpacker(six.BytesIO(expected), read_size=3, max_buffer_size=3)
+
+    f = six.BytesIO()
+    unpacked = unpacker.unpack(f.write)
+    assert_equal(unpacked, long_val)
+    assert_equal(f.getvalue(), expected)
+
+if __name__ == '__main__':
+    main()

From 89ce16df39f67ac77785a63b4111c353f0a606a3 Mon Sep 17 00:00:00 2001
From: Alexei Romanoff 
Date: Fri, 12 Oct 2012 12:32:32 +0300
Subject: [PATCH 1058/1648] A segfault fixed in the issue
 https://github.com/msgpack/msgpack-python/issues/28

---
 msgpack/unpack.h          | 3 +++
 msgpack/unpack_template.h | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 5ec7dbc..61e5d91 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -207,6 +207,9 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec
 {
     if (u->object_hook) {
         PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c);
+        if (!new_c)
+            return -1;
+
         Py_DECREF(*c);
         *c = new_c;
     }
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 7d07601..9d13062 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -361,7 +361,7 @@ _push:
 		if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
 		if(++c->count == c->size) {
 			obj = c->obj;
-			construct_cb(_map_end)(user, &obj);
+			if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; }
 			--top;
 			/*printf("stack pop %d\n", top);*/
 			goto _push;

From 4ea952f39dc5ff1231f780252d8d4efb16b2be3b Mon Sep 17 00:00:00 2001
From: Alexei Romanoff 
Date: Fri, 12 Oct 2012 12:34:18 +0300
Subject: [PATCH 1059/1648] Added unit-test for issue
 https://github.com/msgpack/msgpack-python/issues/28

---
 test/test_obj.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/test_obj.py b/test/test_obj.py
index 881e627..15e192c 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -49,5 +49,18 @@ def test_array_hook():
     unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1)
     eq_(unpacked, '123')
 
+
+class DecodeError(Exception):
+    pass
+
+def bad_complex_decoder(o):
+    raise DecodeError("Ooops!")
+
+
+@raises(DecodeError)
+def test_an_exception_in_objecthook1():
+    packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}})
+    unpackb(packed, object_hook=bad_complex_decoder)
+
 if __name__ == '__main__':
     main()

From cf89f18be7614d6d55bb9eb7e9bf0e10d42a8508 Mon Sep 17 00:00:00 2001
From: Alexei Romanoff 
Date: Fri, 12 Oct 2012 13:19:53 +0300
Subject: [PATCH 1060/1648] segfault fixed when data is unpacked using
 `list_hook`,

this bug is a twin to #28.
Unit-test is also attached.
---
 msgpack/unpack.h          | 2 ++
 msgpack/unpack_template.h | 2 +-
 test/test_obj.py          | 8 ++++++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 61e5d91..3dc88e5 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -163,6 +163,8 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj
 {
     if (u->list_hook) {
         PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c);
+        if (!new_c)
+            return -1;
         Py_DECREF(*c);
         *c = new_c;
     }
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 9d13062..8a57f0d 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -347,7 +347,7 @@ _push:
 		if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
 		if(++c->count == c->size) {
 			obj = c->obj;
-			construct_cb(_array_end)(user, &obj);
+			if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; }
 			--top;
 			/*printf("stack pop %d\n", top);*/
 			goto _push;
diff --git a/test/test_obj.py b/test/test_obj.py
index 15e192c..1d9024b 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -62,5 +62,13 @@ def test_an_exception_in_objecthook1():
     packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}})
     unpackb(packed, object_hook=bad_complex_decoder)
 
+
+@raises(DecodeError)
+def test_an_exception_in_objecthook2():
+    packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]})
+    unpackb(packed, list_hook=bad_complex_decoder, use_list=1)
+
+
+
 if __name__ == '__main__':
     main()

From 541d22d00434863ce1e3607950c199edbd2b850a Mon Sep 17 00:00:00 2001
From: Alexei Romanoff 
Date: Fri, 12 Oct 2012 13:32:29 +0300
Subject: [PATCH 1061/1648] Added example of using default/object_hook into
 README

---
 README.rst | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/README.rst b/README.rst
index 1da6ece..ee896ce 100644
--- a/README.rst
+++ b/README.rst
@@ -61,6 +61,36 @@ stream.
        for unpacked in unpacker:
            print unpacked
 
+packing/unpacking of custom data type
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Also possible to pack/unpack user's data types. Here is an example for
+``datetime.datetime``.
+
+::
+    import datetime
+
+    import msgpack
+
+    useful_dict = {
+        "id": 1,
+        "created": datetime.datetime.now(),
+    }
+
+    def decode_datetime(obj):
+        if b'__datetime__' in obj:
+            obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f")
+        return obj
+
+    def encode_datetime(obj):
+        if isinstance(obj, datetime.datetime):
+            return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")}
+        return obj
+
+
+    packed_dict = msgpack.packb(useful_dict, default=encode_datetime)
+    this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime)
+
 
 INSTALL
 ---------

From fa1c4745ec4c8cf8bad5f35be8ddb7fd9e28532e Mon Sep 17 00:00:00 2001
From: Alexei Romanoff 
Date: Fri, 12 Oct 2012 15:25:14 +0300
Subject: [PATCH 1062/1648] README formatting has been improved

---
 README.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.rst b/README.rst
index ee896ce..834c86c 100644
--- a/README.rst
+++ b/README.rst
@@ -68,6 +68,7 @@ Also possible to pack/unpack user's data types. Here is an example for
 ``datetime.datetime``.
 
 ::
+
     import datetime
 
     import msgpack

From 62e8f400244a449ab4ca72991ae8f06610298dc6 Mon Sep 17 00:00:00 2001
From: Spiros Eliopoulos 
Date: Thu, 1 Nov 2012 00:41:06 -0400
Subject: [PATCH 1063/1648] Fix typo in README

seed -> feed
---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 834c86c..bcfd98c 100644
--- a/README.rst
+++ b/README.rst
@@ -57,7 +57,7 @@ stream.
        data = buf.read(4)
        if not data:
            break
-       unpacker.seed(buf.read(16))
+       unpacker.feed(buf.read(16))
        for unpacked in unpacker:
            print unpacked
 

From 30233a5a995e23b72319598d11ebba084497a18b Mon Sep 17 00:00:00 2001
From: Spiros Eliopoulos 
Date: Thu, 1 Nov 2012 00:55:33 -0400
Subject: [PATCH 1064/1648] Fix Unpacker example in README

The example did not properly deserialize, since it was dropping bytes
from the input stream.
---
 README.rst | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/README.rst b/README.rst
index bcfd98c..c15d46b 100644
--- a/README.rst
+++ b/README.rst
@@ -52,14 +52,15 @@ stream.
 
    buf.seek(0)
 
-   unpacker = msgpack.Unpacker()
-   while True:
-       data = buf.read(4)
+    unpacker = msgpack.Unpacker()
+    while True:
+       data = buf.read(16)
        if not data:
            break
-       unpacker.feed(buf.read(16))
-       for unpacked in unpacker:
-           print unpacked
+       unpacker.feed(data)
+
+    for unpacked in unpacker:
+       print unpacked
 
 packing/unpacking of custom data type
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

From d025d908821f60423fd4ad20f04500f3a289783a Mon Sep 17 00:00:00 2001
From: Spiros Eliopoulos 
Date: Thu, 1 Nov 2012 11:28:11 -0400
Subject: [PATCH 1065/1648] Put Unpacker read loop back into buffer read loop

So it works for streaming as intended.
---
 README.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.rst b/README.rst
index c15d46b..dcd3eb1 100644
--- a/README.rst
+++ b/README.rst
@@ -52,15 +52,15 @@ stream.
 
    buf.seek(0)
 
-    unpacker = msgpack.Unpacker()
-    while True:
+   unpacker = msgpack.Unpacker()
+   while True:
        data = buf.read(16)
        if not data:
            break
        unpacker.feed(data)
 
-    for unpacked in unpacker:
-       print unpacked
+       for unpacked in unpacker:
+           print unpacked
 
 packing/unpacking of custom data type
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

From 2f0078d3955ad151452b92d570661d1b5779d7b6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 6 Nov 2012 02:10:36 +0900
Subject: [PATCH 1066/1648] Add travis config.

---
 .travis.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..593d61e
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,10 @@
+language: python
+python:
+    - 2.5
+    - 2.6
+    - 2.7
+    - 3.2
+    - 3.3
+
+install: "pip install cython --use-mirrors"
+script: nosetests

From 3ec2e6e729ee71d0284140d3f494529788cbcfd5 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 6 Nov 2012 02:32:59 +0900
Subject: [PATCH 1067/1648] Add six to .travis.yml

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 593d61e..7286637 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,5 +6,5 @@ python:
     - 3.2
     - 3.3
 
-install: "pip install cython --use-mirrors"
+install: "pip install cython six --use-mirrors"
 script: nosetests

From dbf50c9f789b831e0e8ec582a9947708175b5d5e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 6 Nov 2012 09:35:06 +0900
Subject: [PATCH 1068/1648] Add travis status.

---
 README.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.rst b/README.rst
index dcd3eb1..8cebc01 100644
--- a/README.rst
+++ b/README.rst
@@ -6,6 +6,9 @@ MessagePack Python Binding
 :version: 0.2.0
 :date: 2012-06-27
 
+.. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
+   :target: https://travis-ci.org/#!/msgpack/msgpack-python
+
 HOW TO USE
 -----------
 

From c75ef976d7230ceee884e24127736b6771351d80 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 6 Nov 2012 09:37:55 +0900
Subject: [PATCH 1069/1648] Fix build error on first time.

---
 setup.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 5be92bf..b9af8c3 100644
--- a/setup.py
+++ b/setup.py
@@ -34,8 +34,7 @@ Install Cython >= 0.16 or install msgpack from PyPI.
           os.stat(src).st_mtime < os.stat(pyx).st_mtime and
           have_cython):
         cythonize(pyx)
-    else:
-        return src
+    return src
 
 
 class BuildExt(build_ext):

From eb61b4de9e052e4c126b21348f83695bcb30534a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 6 Nov 2012 09:46:02 +0900
Subject: [PATCH 1070/1648] (travis) Install msgpack-python itself.

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 7286637..4701750 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,5 +6,5 @@ python:
     - 3.2
     - 3.3
 
-install: "pip install cython six --use-mirrors"
+install: "pip install cython six . --use-mirrors"
 script: nosetests

From e404c9845f5eab127e4d0c66b7ddf7cd9733c22b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 6 Nov 2012 10:39:10 +0900
Subject: [PATCH 1071/1648] (travis) Install itself.

---
 .travis.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 4701750..6bf4336 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,5 +6,8 @@ python:
     - 3.2
     - 3.3
 
-install: "pip install cython six . --use-mirrors"
+install:
+    - "pip install cython six --use-mirrors"
+    - "python setup.py install"
+
 script: nosetests

From 0ef52869e39b597a6937f46fe00147227034ce02 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 7 Nov 2012 02:00:08 +0900
Subject: [PATCH 1072/1648] Fix unpack error on Python 3.2.

ctx.user.encoding and ctx.user.unicode_errors may refer to deallocated string.
---
 msgpack/_msgpack.pyx | 58 ++++++++++++++++++++++++++------------------
 1 file changed, 35 insertions(+), 23 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 7fcfcb0..f665306 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -1,8 +1,6 @@
 # coding: utf-8
 #cython: embedsignature=True
 
-import warnings
-
 from cpython cimport *
 cdef extern from "Python.h":
     ctypedef char* const_char_ptr "const char*"
@@ -233,7 +231,9 @@ cdef extern from "unpack.h":
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
-cdef inline init_ctx(template_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, encoding, unicode_errors):
+cdef inline init_ctx(template_context *ctx,
+                     object object_hook, object object_pairs_hook, object list_hook,
+                     bint use_list, char* encoding, char* unicode_errors):
     template_init(ctx)
     ctx.user.use_list = use_list
     ctx.user.object_hook = ctx.user.list_hook = NULL
@@ -259,20 +259,8 @@ cdef inline init_ctx(template_context *ctx, object object_hook, object object_pa
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
 
-    if encoding is None:
-        ctx.user.encoding = NULL
-        ctx.user.unicode_errors = NULL
-    else:
-        if isinstance(encoding, unicode):
-            _bencoding = encoding.encode('ascii')
-        else:
-            _bencoding = encoding
-        ctx.user.encoding = PyBytes_AsString(_bencoding)
-        if isinstance(unicode_errors, unicode):
-            _berrors = unicode_errors.encode('ascii')
-        else:
-            _berrors = unicode_errors
-        ctx.user.unicode_errors = PyBytes_AsString(_berrors)
+    ctx.user.encoding = encoding
+    ctx.user.unicode_errors = unicode_errors
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=1, encoding=None, unicode_errors="strict",
@@ -288,10 +276,22 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
     cdef char* buf
     cdef Py_ssize_t buf_len
+    cdef char* cenc = NULL
+    cdef char* cerr = NULL
 
     PyObject_AsReadBuffer(packed, &buf, &buf_len)
 
-    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors)
+    if encoding is not None:
+        if isinstance(encoding, unicode):
+            encoding = encoding.encode('ascii')
+        cenc = PyBytes_AsString(encoding)
+
+    if unicode_errors is not None:
+        if isinstance(unicode_errors, unicode):
+            unicode_errors = unicode_errors.encode('ascii')
+        cerr = PyBytes_AsString(unicode_errors)
+
+    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
     ret = template_construct(&ctx, buf, buf_len, &off)
     if ret == 1:
         obj = template_data(&ctx)
@@ -370,10 +370,7 @@ cdef class Unpacker(object):
     cdef object file_like_read
     cdef Py_ssize_t read_size
     cdef object object_hook
-    cdef object _bencoding
-    cdef object _berrors
-    cdef char *encoding
-    cdef char *unicode_errors
+    cdef object encoding, unicode_errors
     cdef size_t max_buffer_size
 
     def __cinit__(self):
@@ -387,6 +384,8 @@ cdef class Unpacker(object):
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict', int max_buffer_size=0,
                  ):
+        cdef char *cenc, *cerr
+
         self.file_like = file_like
         if file_like:
             self.file_like_read = file_like.read
@@ -406,7 +405,20 @@ cdef class Unpacker(object):
         self.buf_size = read_size
         self.buf_head = 0
         self.buf_tail = 0
-        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors)
+
+        if encoding is not None:
+            if isinstance(encoding, unicode):
+                encoding = encoding.encode('ascii')
+            self.encoding = encoding
+            cenc = PyBytes_AsString(encoding)
+
+        if unicode_errors is not None:
+            if isinstance(unicode_errors, unicode):
+                unicode_errors = unicode_errors.encode('ascii')
+            self.unicode_errors = unicode_errors
+            cerr = PyBytes_AsString(unicode_errors)
+
+        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
 
     def feed(self, object next_bytes):
         cdef char* buf

From df6b969a8d40fc1b2f934801a1088eda24b73ab2 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 7 Nov 2012 02:04:42 +0900
Subject: [PATCH 1073/1648] (travis) Fix test script

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 6bf4336..bbd5ff1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,4 +10,4 @@ install:
     - "pip install cython six --use-mirrors"
     - "python setup.py install"
 
-script: nosetests
+script: "nosetests -w test"

From ec655b9f2ccd4a0fb6e9fc9c9f1d8b88ace69e72 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 7 Nov 2012 02:23:57 +0900
Subject: [PATCH 1074/1648] Fix segmentation fault.

---
 msgpack/_msgpack.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index f665306..9e79910 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -384,7 +384,7 @@ cdef class Unpacker(object):
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict', int max_buffer_size=0,
                  ):
-        cdef char *cenc, *cerr
+        cdef char *cenc=NULL, *cerr=NULL
 
         self.file_like = file_like
         if file_like:

From 67d8cc6c4f0e657674175fadda702b7bf95937c3 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 7 Nov 2012 02:24:26 +0900
Subject: [PATCH 1075/1648] (travis) Python 2.5 can't pass tests.

Python 2.5 doesn't have b'' literal.
---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index bbd5ff1..9247c78 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,5 @@
 language: python
 python:
-    - 2.5
     - 2.6
     - 2.7
     - 3.2

From b14caa419c9ac35779b3212d367c24f36f40e484 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 7 Nov 2012 08:15:46 +0900
Subject: [PATCH 1076/1648] Use system cython.

---
 .travis.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 9247c78..2f3171c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,10 @@ python:
     - 3.3
 
 install:
-    - "pip install cython six --use-mirrors"
-    - "python setup.py install"
+    - sudo apt-get update -qq
+    - sudo apt-get install -q cython
+    - cython --cplus msgpack/_msgpack.pyx
+    - pip install six --use-mirrors
+    - python setup.py install
 
 script: "nosetests -w test"

From 1c389135b88d3fc8be4151228fff1fb5764f119b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 7 Nov 2012 08:28:30 +0900
Subject: [PATCH 1077/1648] (travis) Travis doesn't support Python 3.3 yet.

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 2f3171c..11835cb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,7 +3,6 @@ python:
     - 2.6
     - 2.7
     - 3.2
-    - 3.3
 
 install:
     - sudo apt-get update -qq

From 6b782232317b3e00f8896fb36567f8ad8c3c3345 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 6 Dec 2012 19:15:04 +0900
Subject: [PATCH 1078/1648] Fix test failuar.

---
 test/test_unpack_raw.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/test/test_unpack_raw.py b/test/test_unpack_raw.py
index ed2e04b..15d9c93 100644
--- a/test/test_unpack_raw.py
+++ b/test/test_unpack_raw.py
@@ -5,19 +5,21 @@ from nose.tools import *
 import six
 from msgpack import Unpacker, packb
 
+
 def test_write_bytes():
     unpacker = Unpacker()
     unpacker.feed(b'abc')
     f = six.BytesIO()
     assert_equal(unpacker.unpack(f.write), ord('a'))
     assert_equal(f.getvalue(), b'a')
-    f.truncate(0)
-    assert_is_none(unpacker.skip(f.write))
+    f = six.BytesIO()
+    assert unpacker.skip(f.write) is None
     assert_equal(f.getvalue(), b'b')
-    f.truncate(0)
-    assert_is_none(unpacker.skip())
+    f = six.BytesIO()
+    assert unpacker.skip() is None
     assert_equal(f.getvalue(), b'')
 
+
 def test_write_bytes_multi_buffer():
     long_val = (5) * 100
     expected = packb(long_val)

From caecc0098eeb631bb0aeca1fdba397ddd96ca329 Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Thu, 6 Dec 2012 23:01:12 +1100
Subject: [PATCH 1079/1648] Change Unpacker example to read from stream

---
 README.rst | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/README.rst b/README.rst
index 8cebc01..8154098 100644
--- a/README.rst
+++ b/README.rst
@@ -55,15 +55,9 @@ stream.
 
    buf.seek(0)
 
-   unpacker = msgpack.Unpacker()
-   while True:
-       data = buf.read(16)
-       if not data:
-           break
-       unpacker.feed(data)
-
-       for unpacked in unpacker:
-           print unpacked
+   unpacker = msgpack.Unpacker(buf)
+   for unpacked in unpacker:
+       print unpacked
 
 packing/unpacking of custom data type
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

From c567ad1c5220a9da145d41b4e61f3a411d32ce57 Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Thu, 6 Dec 2012 23:10:25 +1100
Subject: [PATCH 1080/1648] Describe object_pairs_hook in README

---
 README.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.rst b/README.rst
index 8154098..f3779f1 100644
--- a/README.rst
+++ b/README.rst
@@ -90,6 +90,9 @@ Also possible to pack/unpack user's data types. Here is an example for
     packed_dict = msgpack.packb(useful_dict, default=encode_datetime)
     this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime)
 
+``Unpacker``'s ``object_hook`` callback receives a dict; the
+``object_pairs_hook`` callback may instead be used to receive a list of
+key-value pairs.
 
 INSTALL
 ---------

From 53b67f14496d71b3aa896d3311367401dc3c00a3 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 6 Dec 2012 21:17:44 +0900
Subject: [PATCH 1081/1648] Add changelog

---
 ChangeLog.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 6a4d27b..43ee723 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -5,7 +5,8 @@
 Changes
 -------
 * Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman)
-
+* Add capturing feature. You can pass the writable object to
+  ``Unpacker.unpack()`` as a second parameter.
 
 0.2.3
 =======

From de3724c1de8b0320e6cab3736404ab9857b7a951 Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Thu, 6 Dec 2012 23:34:18 +1100
Subject: [PATCH 1082/1648] README documentation of advanced Unpacker features

---
 README.rst | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/README.rst b/README.rst
index f3779f1..3f5e823 100644
--- a/README.rst
+++ b/README.rst
@@ -94,6 +94,36 @@ Also possible to pack/unpack user's data types. Here is an example for
 ``object_pairs_hook`` callback may instead be used to receive a list of
 key-value pairs.
 
+
+advanced unpacking control
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As an alternative to iteration, ``Unpacker`` objects provide ``unpack``,
+``skip``, ``read_array_header`` and ``read_map_header`` methods. The former two
+read an entire message from the stream, respectively deserialising and returning
+the result, or ignoring it. The latter two methods return the number of elements
+in the upcoming container, so that each element in an array, or key-value pair
+in a map, can be unpacked or skipped individually.
+
+Each of these methods may optionally write the packed data it reads to a
+callback function:
+
+::
+
+    from io import BytesIO
+
+    def distribute(unpacker, get_worker):
+        nelems = unpacker.read_map_header()
+        for i in range(nelems):
+            # Select a worker for the given key
+            key = unpacker.unpack()
+            worker = get_worker(key)
+
+            # Send the value as a packed message to worker
+            bytestream = BytesIO()
+            unpacker.skip(bytestream.write)
+            worker.send(bytestream.getvalue())
+
 INSTALL
 ---------
 You can use ``pip`` or ``easy_install`` to install msgpack::

From 6d4115f64bb180f34161b8a517c9c2165af81deb Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Thu, 6 Dec 2012 23:36:16 +1100
Subject: [PATCH 1083/1648] Minor grammar fixes

---
 README.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index 3f5e823..9a437eb 100644
--- a/README.rst
+++ b/README.rst
@@ -35,13 +35,13 @@ To unpack it to list, Use ``use_list`` option.
    >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=True)
    [1, 2, 3]
 
-Read docstring for other options.
+Read the docstring for other options.
 
 
 streaming unpacking
 ^^^^^^^^^^^^^^^^^^^
 
-``Unpacker`` is "streaming unpacker". It unpacks multiple objects from one
+``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one
 stream.
 
 ::
@@ -59,10 +59,11 @@ stream.
    for unpacked in unpacker:
        print unpacked
 
+
 packing/unpacking of custom data type
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Also possible to pack/unpack user's data types. Here is an example for
+It is also possible to pack/unpack custom data types. Here is an example for
 ``datetime.datetime``.
 
 ::

From 659d0961a319bbaabf994a972b06022c99e4465c Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Thu, 6 Dec 2012 23:36:46 +1100
Subject: [PATCH 1084/1648] Brief mention of Unpacker.feed in README

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 9a437eb..1f5b1a2 100644
--- a/README.rst
+++ b/README.rst
@@ -42,7 +42,7 @@ streaming unpacking
 ^^^^^^^^^^^^^^^^^^^
 
 ``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one
-stream.
+stream (or from bytes provided through its ``feed`` method).
 
 ::
 

From fc41ed606d06becae4b53c8614b74768ae755cba Mon Sep 17 00:00:00 2001
From: Joel Nothman 
Date: Thu, 6 Dec 2012 23:44:27 +1100
Subject: [PATCH 1085/1648] Mention CPython and MessagePack in README

---
 README.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/README.rst b/README.rst
index 1f5b1a2..5264ce9 100644
--- a/README.rst
+++ b/README.rst
@@ -9,6 +9,13 @@ MessagePack Python Binding
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
    :target: https://travis-ci.org/#!/msgpack/msgpack-python
 
+WHAT IT IS
+----------
+
+`MessagePack `_ is a fast, compact binary serialization format, suitable for
+similar data to JSON. This package provides CPython bindings for reading and
+writing MessagePack data.
+
 HOW TO USE
 -----------
 

From 15f309c0b197abe86e94fb3840ff8979b35c7e0c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 6 Dec 2012 22:26:39 +0900
Subject: [PATCH 1086/1648] Add note about use_list.

---
 README.rst | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/README.rst b/README.rst
index 5264ce9..30a264d 100644
--- a/README.rst
+++ b/README.rst
@@ -42,6 +42,9 @@ To unpack it to list, Use ``use_list`` option.
    >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=True)
    [1, 2, 3]
 
+The default behavior will be changed in the future. (probably 0.4)
+You should always pass the ``use_list`` keyword argument.
+
 Read the docstring for other options.
 
 
@@ -151,6 +154,27 @@ or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support
 amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.)
 
 
+PERFORMANCE NOTE
+-----------------
+
+GC
+^^
+
+CPython's GC starts when growing allocated object.
+This means unpacking may cause useless GC.
+You can use ``gc.disable()`` when unpacking large message.
+
+use_list
+^^^^^^^^^
+List is the default sequence type of Python.
+But tuple is lighter than list.
+You can use ``use_list=False`` while unpacking when performance is important.
+
+Python's dict can't use list as key and MessagePack allows array for key of mapping.
+``use_list=False`` allows unpacking such message.
+Another way to unpacking such object is using ``object_pairs_hook``.
+
+
 TEST
 ----
 MessagePack uses `nosetest` for testing.

From edd2e52373246c7d55ee59bf3bd46cda60e3be6d Mon Sep 17 00:00:00 2001
From: jnothman 
Date: Fri, 7 Dec 2012 00:53:17 +1100
Subject: [PATCH 1087/1648] Fix README re default use_list=True

---
 README.rst | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/README.rst b/README.rst
index 30a264d..cdfb9e1 100644
--- a/README.rst
+++ b/README.rst
@@ -29,21 +29,20 @@ msgpack provides ``dumps`` and ``loads`` as alias for compatibility with
 ``pack`` and ``dump`` packs to file-like object.
 ``unpack`` and ``load`` unpacks from file-like object.
 
+::
+
    >>> import msgpack
    >>> msgpack.packb([1, 2, 3])
    '\x93\x01\x02\x03'
    >>> msgpack.unpackb(_)
-   (1, 2, 3)
-
-
-``unpack`` unpacks msgpack's array to Python's tuple.
-To unpack it to list, Use ``use_list`` option.
-
-   >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=True)
    [1, 2, 3]
 
-The default behavior will be changed in the future. (probably 0.4)
-You should always pass the ``use_list`` keyword argument.
+``unpack`` unpacks msgpack's array to Python's list, but can unpack to tuple::
+
+   >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False)
+   (1, 2, 3)
+
+You should always pass the ``use_list`` keyword argument. See performance issues relating to use_list_ below.
 
 Read the docstring for other options.
 

From 56ef0d07debe19af355f284d54ee5f1dcf542592 Mon Sep 17 00:00:00 2001
From: jnothman 
Date: Fri, 7 Dec 2012 11:12:19 +1100
Subject: [PATCH 1088/1648] Warn about StopIteration in README

---
 README.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.rst b/README.rst
index 30a264d..61cc56c 100644
--- a/README.rst
+++ b/README.rst
@@ -116,6 +116,9 @@ the result, or ignoring it. The latter two methods return the number of elements
 in the upcoming container, so that each element in an array, or key-value pair
 in a map, can be unpacked or skipped individually.
 
+Warning: these methods raise ``StopIteration`` when called at the end of the
+stream. Unless caught, this may silently break an iteration.
+
 Each of these methods may optionally write the packed data it reads to a
 callback function:
 

From dd5c76b9552e371503535ce10a0314151e62fa28 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 7 Dec 2012 11:35:16 +0900
Subject: [PATCH 1089/1648] Add NOTE for changing default value of use_list.

---
 ChangeLog.rst        |  6 ++++++
 README.rst           | 11 +++++++++--
 msgpack/_msgpack.pyx |  2 +-
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 43ee723..279cb81 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -2,6 +2,12 @@
 =====
 :release date: in development
 
+Inconpatible Changes
+--------------------
+
+* Default value of ``use_list`` is ``True`` for now. (It was ``False`` for 0.2.x)
+  You should pass it explicitly for compatibility to 0.2.x.
+
 Changes
 -------
 * Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman)
diff --git a/README.rst b/README.rst
index 58f6cb1..aa447f3 100644
--- a/README.rst
+++ b/README.rst
@@ -3,8 +3,8 @@ MessagePack Python Binding
 ===========================
 
 :author: INADA Naoki
-:version: 0.2.0
-:date: 2012-06-27
+:version: 0.3.0
+:date: 2012-12-07
 
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
    :target: https://travis-ci.org/#!/msgpack/msgpack-python
@@ -16,6 +16,13 @@ WHAT IT IS
 similar data to JSON. This package provides CPython bindings for reading and
 writing MessagePack data.
 
+NOTE for msgpack 0.2.x users
+----------------------------
+
+The default value of ``use_list`` keyword argument is ``True`` from 0.3.x.
+You should pass the argument explicitly for backward compatibility.
+
+
 HOW TO USE
 -----------
 
diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 38940f0..5c202fc 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -327,7 +327,7 @@ cdef class Unpacker(object):
     `read_size` is used as `file_like.read(read_size)`.
     (default: min(1024**2, max_buffer_size))
 
-    If `use_list` is true, msgpack list is deserialized to Python list.
+    If `use_list` is true (default), msgpack list is deserialized to Python list.
     Otherwise, it is deserialized to Python tuple.
 
     `object_hook` is same to simplejson. If it is not None, it should be callable

From 219d47503ca982996b03f4396fdd2929c9905356 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Dec 2012 00:31:19 +0900
Subject: [PATCH 1090/1648] Split exceptions.

---
 msgpack/_msgpack.pyx   | 24 ++++++++++++++++--------
 msgpack/exceptions.py  | 23 +++++++++++++++++++++++
 test/test_read_size.py |  6 +++---
 test/test_sequnpack.py |  5 +++--
 4 files changed, 45 insertions(+), 13 deletions(-)
 create mode 100644 msgpack/exceptions.py

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 5c202fc..da70f0d 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -38,8 +38,13 @@ cdef extern from "pack.h":
 cdef int DEFAULT_RECURSE_LIMIT=511
 
 
-class BufferFull(Exception):
-    pass
+from msgpack.exceptions import (
+        UnpackException,
+        BufferFull,
+        OutOfData,
+        UnpackValueError,
+        ExtraData,
+        )
 
 
 cdef class Packer(object):
@@ -102,7 +107,7 @@ cdef class Packer(object):
         cdef dict d
 
         if nest_limit < 0:
-            raise ValueError("Too deep.")
+            raise UnpackValueError("recursion limit exceeded.")
 
         if o is None:
             ret = msgpack_pack_nil(&self.pk)
@@ -296,7 +301,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     if ret == 1:
         obj = template_data(&ctx)
         if off < buf_len:
-            raise ValueError("Extra data.")
+            raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
         return obj
     else:
         return None
@@ -425,7 +430,7 @@ cdef class Unpacker(object):
         cdef Py_ssize_t buf_len
         if self.file_like is not None:
             raise AssertionError(
-                    "unpacker.feed() is not be able to use with`file_like`.")
+                    "unpacker.feed() is not be able to use with `file_like`.")
         PyObject_AsReadBuffer(next_bytes, &buf, &buf_len)
         self.append_buffer(buf, buf_len)
 
@@ -479,7 +484,7 @@ cdef class Unpacker(object):
         else:
             self.file_like = None
 
-    cdef object _unpack(self, execute_fn execute, object write_bytes):
+    cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0):
         cdef int ret
         cdef object obj
         cdef size_t prev_head
@@ -497,7 +502,10 @@ cdef class Unpacker(object):
                 if self.file_like is not None:
                     self.read_from_file()
                     continue
-                raise StopIteration("No more data to unpack.")
+                if iter:
+                    raise StopIteration("No more data to unpack.")
+                else:
+                    raise OutOfData("No more data to unpack.")
             else:
                 raise ValueError("Unpack failed: error = %d" % (ret,))
 
@@ -539,7 +547,7 @@ cdef class Unpacker(object):
         return self
 
     def __next__(self):
-        return self._unpack(template_construct, None)
+        return self._unpack(template_construct, None, 1)
 
     # for debug.
     #def _buf(self):
diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py
new file mode 100644
index 0000000..0a75430
--- /dev/null
+++ b/msgpack/exceptions.py
@@ -0,0 +1,23 @@
+class UnpackException(Exception):
+    pass
+
+
+class BufferFull(UnpackException):
+    pass
+
+
+class OutOfData(UnpackException):
+    pass
+
+
+class UnpackValueError(UnpackException, ValueError):
+    pass
+
+
+class ExtraData(ValueError):
+    def __init__(self, unpacked, extra):
+        self.unpacked = unpacked
+        self.extra = extra
+
+    def __str__(self):
+        return "unpack(b) recieved extra data."
diff --git a/test/test_read_size.py b/test/test_read_size.py
index e130805..4e6c2b9 100644
--- a/test/test_read_size.py
+++ b/test/test_read_size.py
@@ -1,5 +1,5 @@
 """Test Unpacker's read_array_header and read_map_header methods"""
-from msgpack import packb, Unpacker
+from msgpack import packb, Unpacker, OutOfData
 UnexpectedTypeException = ValueError
 
 def test_read_array_header():
@@ -12,7 +12,7 @@ def test_read_array_header():
     try:
         unpacker.unpack()
         assert 0, 'should raise exception'
-    except StopIteration:
+    except OutOfData:
         assert 1, 'okay'
 
 
@@ -25,7 +25,7 @@ def test_read_map_header():
     try:
         unpacker.unpack()
         assert 0, 'should raise exception'
-    except StopIteration:
+    except OutOfData:
         assert 1, 'okay'
 
 def test_incorrect_type_array():
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 769f3ff..f767726 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -3,6 +3,7 @@
 
 import six
 from msgpack import Unpacker, BufferFull
+from msgpack.exceptions import OutOfData
 import nose
 
 def test_foobar():
@@ -17,7 +18,7 @@ def test_foobar():
     try:
         o = unpacker.unpack()
         assert 0, "should raise exception"
-    except StopIteration:
+    except OutOfData:
         assert 1, "ok"
 
     unpacker.feed(b'foo')
@@ -41,7 +42,7 @@ def test_foobar_skip():
     try:
         o = unpacker.unpack()
         assert 0, "should raise exception"
-    except StopIteration:
+    except OutOfData:
         assert 1, "ok"
 
 def test_maxbuffersize():

From c7161e9403ce15c4bbea99cb482ea37a74311b45 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Dec 2012 00:35:26 +0900
Subject: [PATCH 1091/1648] Add note for StopIteration => OutOfData.

---
 README.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.rst b/README.rst
index aa447f3..270b8ff 100644
--- a/README.rst
+++ b/README.rst
@@ -19,9 +19,15 @@ writing MessagePack data.
 NOTE for msgpack 0.2.x users
 ----------------------------
 
+The msgpack 0.3 have some incompatible changes.
+
 The default value of ``use_list`` keyword argument is ``True`` from 0.3.x.
 You should pass the argument explicitly for backward compatibility.
 
+`Unpacker.unpack()` and some unpack methods now raises `OutOfData`
+instead of `StopIteration`.
+`StopIteration` is used for iterator protocol only.
+
 
 HOW TO USE
 -----------

From 4480227e066e66fd771dc1702adf4929dede453d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Dec 2012 00:39:04 +0900
Subject: [PATCH 1092/1648] Improve docstrings.

---
 msgpack/_msgpack.pyx | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index da70f0d..2c81cb4 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -426,6 +426,7 @@ cdef class Unpacker(object):
         init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
 
     def feed(self, object next_bytes):
+        """Append `next_bytes` to internal buffer."""
         cdef char* buf
         cdef Py_ssize_t buf_len
         if self.file_like is not None:
@@ -523,7 +524,11 @@ cdef class Unpacker(object):
         """
         unpack one object
 
-        If write_bytes is not None, it will be called with parts of the raw message as it is unpacked.
+        If write_bytes is not None, it will be called with parts of the raw
+        message as it is unpacked.
+
+        When there are not enough bytes for unpacking, `unpack()` raises
+        `OutOfData` Exception.
         """
         return self._unpack(template_construct, write_bytes)
 
@@ -531,16 +536,30 @@ cdef class Unpacker(object):
         """
         read and ignore one object, returning None
 
-        If write_bytes is not None, it will be called with parts of the raw message as it is unpacked.
+        If write_bytes is not None, it will be called with parts of the raw
+        message as it is unpacked.
+
+        When there are not enough bytes for unpacking, `unpack()` raises
+        `OutOfData` Exception.
         """
         return self._unpack(template_skip, write_bytes)
 
     def read_array_header(self, object write_bytes=None):
-        """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents."""
+        """assuming the next object is an array, return its size n, such that
+        the next n unpack() calls will iterate over its contents.
+
+        When there are not enough bytes for unpacking, `unpack()` raises
+        `OutOfData` Exception.
+        """
         return self._unpack(read_array_header, write_bytes)
 
     def read_map_header(self, object write_bytes=None):
-        """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs."""
+        """assuming the next object is a map, return its size n, such that the
+        next n * 2 unpack() calls will iterate over its key-value pairs.
+
+        When there are not enough bytes for unpacking, `unpack()` raises
+        `OutOfData` Exception.
+        """
         return self._unpack(read_map_header, write_bytes)
 
     def __iter__(self):

From ed40c671dac175e437fe67689053b5fe8a269ad3 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Dec 2012 01:42:38 +0900
Subject: [PATCH 1093/1648] `pack` raise MemoryError when realloc is failed.

---
 msgpack/_msgpack.pyx | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 2c81cb4..b2b5222 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -179,7 +179,9 @@ cdef class Packer(object):
     cpdef pack(self, object obj):
         cdef int ret
         ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
-        if ret:
+        if ret == -1:
+            raise MemoryError
+        elif ret:  # should not happen.
             raise TypeError
         buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
         self.pk.length = 0

From 30025c7ea0a53e17a907ad35ffa56b7b7efd99d1 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Dec 2012 20:06:00 +0900
Subject: [PATCH 1094/1648] Improve docstring.

---
 msgpack/_msgpack.pyx | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index b2b5222..74a4cc4 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -529,8 +529,7 @@ cdef class Unpacker(object):
         If write_bytes is not None, it will be called with parts of the raw
         message as it is unpacked.
 
-        When there are not enough bytes for unpacking, `unpack()` raises
-        `OutOfData` Exception.
+        Raises `OutOfData` when there are no more bytes to unpack.
         """
         return self._unpack(template_construct, write_bytes)
 
@@ -541,8 +540,7 @@ cdef class Unpacker(object):
         If write_bytes is not None, it will be called with parts of the raw
         message as it is unpacked.
 
-        When there are not enough bytes for unpacking, `unpack()` raises
-        `OutOfData` Exception.
+        Raises `OutOfData` when there are no more bytes to unpack.
         """
         return self._unpack(template_skip, write_bytes)
 
@@ -550,8 +548,7 @@ cdef class Unpacker(object):
         """assuming the next object is an array, return its size n, such that
         the next n unpack() calls will iterate over its contents.
 
-        When there are not enough bytes for unpacking, `unpack()` raises
-        `OutOfData` Exception.
+        Raises `OutOfData` when there are no more bytes to unpack.
         """
         return self._unpack(read_array_header, write_bytes)
 
@@ -559,8 +556,7 @@ cdef class Unpacker(object):
         """assuming the next object is a map, return its size n, such that the
         next n * 2 unpack() calls will iterate over its key-value pairs.
 
-        When there are not enough bytes for unpacking, `unpack()` raises
-        `OutOfData` Exception.
+        Raises `OutOfData` when there are no more bytes to unpack.
         """
         return self._unpack(read_map_header, write_bytes)
 

From 1c0fe10a2fd5a32cf3f558683695220f3718a4b8 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Dec 2012 20:12:38 +0900
Subject: [PATCH 1095/1648] Remove unused UnpackException.

---
 msgpack/_msgpack.pyx | 1 -
 1 file changed, 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 74a4cc4..52d63cc 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -39,7 +39,6 @@ cdef int DEFAULT_RECURSE_LIMIT=511
 
 
 from msgpack.exceptions import (
-        UnpackException,
         BufferFull,
         OutOfData,
         UnpackValueError,

From 0c7ab7c3441e96b33fa7bca8fcaa10d3087aba2a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Dec 2012 20:17:18 +0900
Subject: [PATCH 1096/1648] refactoring: remove pack_define.h

---
 msgpack/pack.h        |  3 ++-
 msgpack/pack_define.h | 26 --------------------------
 2 files changed, 2 insertions(+), 27 deletions(-)
 delete mode 100644 msgpack/pack_define.h

diff --git a/msgpack/pack.h b/msgpack/pack.h
index 4c0373e..bb939d9 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -19,7 +19,8 @@
 #include 
 #include 
 #include "sysdep.h"
-#include "pack_define.h"
+#include 
+#include 
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/msgpack/pack_define.h b/msgpack/pack_define.h
deleted file mode 100644
index 4845d52..0000000
--- a/msgpack/pack_define.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * MessagePack unpacking routine template
- *
- * Copyright (C) 2008-2010 FURUHASHI Sadayuki
- *
- *    Licensed under the Apache License, Version 2.0 (the "License");
- *    you may not use this file except in compliance with the License.
- *    You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- */
-#ifndef MSGPACK_PACK_DEFINE_H__
-#define MSGPACK_PACK_DEFINE_H__
-
-#include "msgpack/sysdep.h"
-#include 
-#include 
-
-#endif /* msgpack/pack_define.h */
-

From 537a2ab3f262d65f5a85741ddbb4ac6604ed7e0b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Dec 2012 21:26:41 +0900
Subject: [PATCH 1097/1648] Add `Packer.pack_pairs`.

---
 msgpack/_msgpack.pyx | 39 +++++++++++++++++++++++++++++++++++----
 test/test_pack.py    | 11 +++++++++--
 2 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 52d63cc..2d97f21 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -186,18 +186,49 @@ cdef class Packer(object):
         self.pk.length = 0
         return buf
 
-    cpdef pack_array_header(self, size_t size):
-        msgpack_pack_array(&self.pk, size)
+    def pack_array_header(self, size_t size):
+        cdef int ret = msgpack_pack_array(&self.pk, size)
+        if ret == -1:
+            raise MemoryError
+        elif ret:  # should not happen
+            raise TypeError
         buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
         self.pk.length = 0
         return buf
 
-    cpdef pack_map_header(self, size_t size):
-        msgpack_pack_map(&self.pk, size)
+    def pack_map_header(self, size_t size):
+        cdef int ret = msgpack_pack_map(&self.pk, size)
+        if ret == -1:
+            raise MemoryError
+        elif ret:  # should not happen
+            raise TypeError
         buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
         self.pk.length = 0
         return buf
 
+    def pack_map_pairs(self, object pairs):
+        """
+        Pack *pairs* as msgpack map type.
+
+        *pairs* should sequence of pair.
+        (`len(pairs)` and `for k, v in *pairs*:` should be supported.)
+        """
+        cdef int ret = msgpack_pack_map(&self.pk, len(pairs))
+        if ret == 0:
+            for k, v in pairs:
+                ret = self._pack(k)
+                if ret != 0: break
+                ret = self._pack(v)
+                if ret != 0: break
+        if ret == -1:
+            raise MemoryError
+        elif ret:  # should not happen
+            raise TypeError
+        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+        self.pk.length = 0
+        return buf
+
+
 def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'):
     """
     pack an object `o` and write it to stream)."""
diff --git a/test/test_pack.py b/test/test_pack.py
index 21c2bd7..b918f8e 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -118,8 +118,6 @@ def testMapSize(sizes=[0, 5, 50, 1000]):
         assert unpacker.unpack() == dict((i, i * 2) for i in range(size))
 
 
-
-
 class odict(dict):
     '''Reimplement OrderedDict to run test on Python 2.6'''
     def __init__(self, seq):
@@ -144,5 +142,14 @@ def test_odict():
     assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1), seq)
 
 
+def test_pairlist():
+    pairlist = [(b'a', 1), (2, b'b'), (b'foo', b'bar')]
+    packer = Packer()
+    packed = packer.pack_map_pairs(pairlist)
+    unpacked = unpackb(packed, object_pairs_hook=list)
+    assert pairlist == unpacked
+
+
+
 if __name__ == '__main__':
     main()

From 4adc6f194d9986eee4cfa4999d30862e4251d4cf Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Dec 2012 21:47:18 +0900
Subject: [PATCH 1098/1648] Add `autoreset` option to Packer.

---
 msgpack/_msgpack.pyx | 48 +++++++++++++++++++++++++++++++++-----------
 test/test_pack.py    | 15 ++++++++++++++
 2 files changed, 51 insertions(+), 12 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 2d97f21..d58255e 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -54,6 +54,16 @@ cdef class Packer(object):
         packer = Packer()
         astream.write(packer.pack(a))
         astream.write(packer.pack(b))
+
+    Packer's constructor has some keyword arguments:
+
+    * *defaut* - Convert user type to builtin type that Packer supports.
+      See also simplejson's document.
+    * *encoding* - Convert unicode to bytes with this encoding. (default: 'utf-8')
+    * *unicode_erros* - Error handler for encoding unicode. (default: 'strict')
+    * *use_single_float* - Use single precision float type for float. (default: False)
+    * *autoreset* - Reset buffer after each pack and return it's content as `bytes`. (default: True).
+      If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
     """
     cdef msgpack_packer pk
     cdef object _default
@@ -62,6 +72,7 @@ cdef class Packer(object):
     cdef char *encoding
     cdef char *unicode_errors
     cdef bool use_float
+    cdef bint autoreset
 
     def __cinit__(self):
         cdef int buf_size = 1024*1024
@@ -71,8 +82,9 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
-    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False):
+    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1):
         self.use_float = use_single_float
+        self.autoreset = autoreset
         if default is not None:
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")
@@ -182,9 +194,10 @@ cdef class Packer(object):
             raise MemoryError
         elif ret:  # should not happen.
             raise TypeError
-        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
-        self.pk.length = 0
-        return buf
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
 
     def pack_array_header(self, size_t size):
         cdef int ret = msgpack_pack_array(&self.pk, size)
@@ -192,9 +205,10 @@ cdef class Packer(object):
             raise MemoryError
         elif ret:  # should not happen
             raise TypeError
-        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
-        self.pk.length = 0
-        return buf
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
 
     def pack_map_header(self, size_t size):
         cdef int ret = msgpack_pack_map(&self.pk, size)
@@ -202,9 +216,10 @@ cdef class Packer(object):
             raise MemoryError
         elif ret:  # should not happen
             raise TypeError
-        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
-        self.pk.length = 0
-        return buf
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
 
     def pack_map_pairs(self, object pairs):
         """
@@ -224,9 +239,18 @@ cdef class Packer(object):
             raise MemoryError
         elif ret:  # should not happen
             raise TypeError
-        buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+        if self.autoreset:
+            buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+            self.pk.length = 0
+            return buf
+
+    def reset(self):
+        """Clear internal buffer."""
         self.pk.length = 0
-        return buf
+
+    def bytes(self):
+        """Return buffer content."""
+        return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
 
 
 def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'):
diff --git a/test/test_pack.py b/test/test_pack.py
index b918f8e..b934dd2 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -103,6 +103,21 @@ def testArraySize(sizes=[0, 5, 50, 1000]):
     for size in sizes:
         assert unpacker.unpack() == list(range(size))
 
+def test_manualreset(sizes=[0, 5, 50, 1000]):
+    packer = Packer(autoreset=False)
+    for size in sizes:
+        packer.pack_array_header(size)
+        for i in range(size):
+            packer.pack(i)
+
+    bio = six.BytesIO(packer.bytes())
+    unpacker = Unpacker(bio, use_list=1)
+    for size in sizes:
+        assert unpacker.unpack() == list(range(size))
+
+    packer.reset()
+    assert packer.bytes() == b''
+
 def testMapSize(sizes=[0, 5, 50, 1000]):
     bio = six.BytesIO()
     packer = Packer()

From 171145e56290a2254d6b648d438dde33cd631fc4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 10 Dec 2012 21:54:17 +0900
Subject: [PATCH 1099/1648] Update changelog.

---
 ChangeLog.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 279cb81..a66a276 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -13,6 +13,12 @@ Changes
 * Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman)
 * Add capturing feature. You can pass the writable object to
   ``Unpacker.unpack()`` as a second parameter.
+* Add ``Packer.pack_array_header`` and ``Packer.pack_map_header``.
+  These methods only pack header of each type.
+* Add ``autoreset`` option to ``Packer`` (default: True).
+  Packer doesn't return packed bytes and clear internal buffer.
+* Add ``Packer.pack_map_pairs``. It packs sequence of pair to map type.
+
 
 0.2.3
 =======

From 3478406537fc36badb2501d2110a2bceb48a45b6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 02:46:13 +0900
Subject: [PATCH 1100/1648] Fix tests.

---
 msgpack/_msgpack.pyx   | 62 ++++++++++++++++++++++++------------------
 setup.py               |  2 +-
 test/test_obj.py       |  2 +-
 test/test_sequnpack.py |  2 +-
 4 files changed, 39 insertions(+), 29 deletions(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index dca6237..5feba11 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -1,21 +1,17 @@
 # coding: utf-8
 #cython: embedsignature=True
 
-import warnings
-
 from cpython cimport *
 cdef extern from "Python.h":
     ctypedef char* const_char_ptr "const char*"
     ctypedef char* const_void_ptr "const void*"
     ctypedef struct PyObject
     cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
-    char* __FILE__
-    int __LINE__
 
 from libc.stdlib cimport *
 from libc.string cimport *
 from libc.limits cimport *
-
+import warnings
 
 cdef extern from "pack.h":
     struct msgpack_packer:
@@ -218,7 +214,9 @@ cdef extern from "unpack.h":
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
-cdef inline init_ctx(template_context *ctx, object object_hook, object object_pairs_hook, object list_hook, bint use_list, encoding, unicode_errors):
+cdef inline init_ctx(template_context *ctx,
+                     object object_hook, object object_pairs_hook, object list_hook,
+                     bint use_list, char* encoding, char* unicode_errors):
     template_init(ctx)
     ctx.user.use_list = use_list
     ctx.user.object_hook = ctx.user.list_hook = NULL
@@ -244,20 +242,8 @@ cdef inline init_ctx(template_context *ctx, object object_hook, object object_pa
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
 
-    if encoding is None:
-        ctx.user.encoding = NULL
-        ctx.user.unicode_errors = NULL
-    else:
-        if isinstance(encoding, unicode):
-            _bencoding = encoding.encode('ascii')
-        else:
-            _bencoding = encoding
-        ctx.user.encoding = PyBytes_AsString(_bencoding)
-        if isinstance(unicode_errors, unicode):
-            _berrors = unicode_errors.encode('ascii')
-        else:
-            _berrors = unicode_errors
-        ctx.user.unicode_errors = PyBytes_AsString(_berrors)
+    ctx.user.encoding = encoding
+    ctx.user.unicode_errors = unicode_errors
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
             use_list=None, encoding=None, unicode_errors="strict",
@@ -273,13 +259,26 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
     cdef char* buf
     cdef Py_ssize_t buf_len
+    cdef char* cenc = NULL
+    cdef char* cerr = NULL
 
     PyObject_AsReadBuffer(packed, &buf, &buf_len)
 
     if use_list is None:
         warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1)
         use_list = 0
-    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors)
+
+    if encoding is not None:
+        if isinstance(encoding, unicode):
+            encoding = encoding.encode('ascii')
+        cenc = PyBytes_AsString(encoding)
+
+    if unicode_errors is not None:
+        if isinstance(unicode_errors, unicode):
+            unicode_errors = unicode_errors.encode('ascii')
+        cerr = PyBytes_AsString(unicode_errors)
+
+    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
     ret = template_execute(&ctx, buf, buf_len, &off, 1)
     if ret == 1:
         obj = template_data(&ctx)
@@ -361,10 +360,7 @@ cdef class Unpacker(object):
     cdef object file_like_read
     cdef Py_ssize_t read_size
     cdef object object_hook
-    cdef object _bencoding
-    cdef object _berrors
-    cdef char *encoding
-    cdef char *unicode_errors
+    cdef object encoding, unicode_errors
     cdef size_t max_buffer_size
 
     def __cinit__(self):
@@ -378,6 +374,7 @@ cdef class Unpacker(object):
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict', int max_buffer_size=0,
                  ):
+        cdef char *cenc=NULL, *cerr=NULL
         if use_list is None:
             warnings.warn("Set use_list explicitly.", category=DeprecationWarning, stacklevel=1)
             use_list = 0
@@ -401,7 +398,20 @@ cdef class Unpacker(object):
         self.buf_size = read_size
         self.buf_head = 0
         self.buf_tail = 0
-        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, encoding, unicode_errors)
+
+        if encoding is not None:
+            if isinstance(encoding, unicode):
+                encoding = encoding.encode('ascii')
+            self.encoding = encoding
+            cenc = PyBytes_AsString(encoding)
+
+        if unicode_errors is not None:
+            if isinstance(unicode_errors, unicode):
+                unicode_errors = unicode_errors.encode('ascii')
+            self.unicode_errors = unicode_errors
+            cerr = PyBytes_AsString(unicode_errors)
+
+        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
 
     def feed(self, object next_bytes):
         cdef char* buf
diff --git a/setup.py b/setup.py
index 9f0ce5d..86b0b34 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,7 @@ except ImportError:
 
 def cythonize(src):
     sys.stderr.write("cythonize: %r\n" % (src,))
-    cython_compiler.compile([src], emit_linenums=True)
+    cython_compiler.compile([src])
 
 def ensure_source(src):
     pyx = os.path.splitext(src)[0] + '.pyx'
diff --git a/test/test_obj.py b/test/test_obj.py
index 881e627..c38d6dc 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -34,7 +34,7 @@ def test_decode_pairs_hook():
 
 @raises(ValueError)
 def test_only_one_obj_hook():
-    unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x)
+    unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x, use_list=1)
 
 @raises(ValueError)
 def test_bad_hook():
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index dac36a8..eac0828 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -44,7 +44,7 @@ def test_foobar_skip():
         assert 1, "ok"
 
 def test_maxbuffersize():
-    nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3)
+    nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3, use_list=1)
     unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
     unpacker.feed(b'fo')
     nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob')

From 6ae363ea27e409c63e951eb32d7c16df0309bb52 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 02:52:24 +0900
Subject: [PATCH 1101/1648] Update changelog for 0.2.3

---
 ChangeLog.rst | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 6a4d27b..833c97c 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,12 +1,3 @@
-0.3.0
-=====
-:release date: in development
-
-Changes
--------
-* Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman)
-
-
 0.2.3
 =======
 :release date: in development
@@ -14,6 +5,7 @@ Changes
 Changes
 -------
 * Warn when use_list is not specified. It's default value will be changed in 0.3.
+* Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman)
 
 Bugs fixed
 -----------

From 4a20700e2000d4baa952e8db48801672e53bab99 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 02:56:20 +0900
Subject: [PATCH 1102/1648] prepare 0.2.3

---
 README.rst          | 4 ++--
 msgpack/_version.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index 1da6ece..5c0829a 100644
--- a/README.rst
+++ b/README.rst
@@ -3,8 +3,8 @@ MessagePack Python Binding
 ===========================
 
 :author: INADA Naoki
-:version: 0.2.0
-:date: 2012-06-27
+:version: 0.2.3
+:date: 2012-12-11
 
 HOW TO USE
 -----------
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 9bffd02..1bbdf2a 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 2, 3, 'dev1')
+version = (0, 2, 3)

From cb7dff331911f836f9bfac2e5f0450760e362e89 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 02:59:45 +0900
Subject: [PATCH 1103/1648] Add .travis.yml

---
 .travis.yml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..11835cb
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,14 @@
+language: python
+python:
+    - 2.6
+    - 2.7
+    - 3.2
+
+install:
+    - sudo apt-get update -qq
+    - sudo apt-get install -q cython
+    - cython --cplus msgpack/_msgpack.pyx
+    - pip install six --use-mirrors
+    - python setup.py install
+
+script: "nosetests -w test"

From e9f9e9e83ff5b69e7aab5384d9cb507363dd88e3 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 09:26:15 +0900
Subject: [PATCH 1104/1648] Don't use c++ on travis.

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 11835cb..e8c2cec 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,7 +7,7 @@ python:
 install:
     - sudo apt-get update -qq
     - sudo apt-get install -q cython
-    - cython --cplus msgpack/_msgpack.pyx
+    - cython msgpack/_msgpack.pyx
     - pip install six --use-mirrors
     - python setup.py install
 

From 280d56eb9b61f68ff87e3e217eeec0b5fd645c38 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 22:05:00 +0900
Subject: [PATCH 1105/1648] rename _msgpack.pyx => _packer.pyx

---
 msgpack/__init__.py                   | 3 ++-
 msgpack/{_msgpack.pyx => _packer.pyx} | 0
 setup.py                              | 4 ++--
 3 files changed, 4 insertions(+), 3 deletions(-)
 rename msgpack/{_msgpack.pyx => _packer.pyx} (100%)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 98b1ab7..1aacee1 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 from msgpack._version import version
-from msgpack._msgpack import *
+from msgpack.exceptions import *
+from msgpack._packer import pack, packb, Packer, unpack, unpackb, Unpacker
 
 # alias for compatibility to simplejson/marshal/pickle.
 load = unpack
diff --git a/msgpack/_msgpack.pyx b/msgpack/_packer.pyx
similarity index 100%
rename from msgpack/_msgpack.pyx
rename to msgpack/_packer.pyx
diff --git a/setup.py b/setup.py
index b9af8c3..5a342e0 100644
--- a/setup.py
+++ b/setup.py
@@ -59,7 +59,7 @@ if have_cython:
 else:
     Sdist = sdist
 
-sources = ['msgpack/_msgpack.cpp']
+sources = ['msgpack/_packer.cpp']
 libraries = []
 if sys.platform == 'win32':
     libraries.append('ws2_32')
@@ -69,7 +69,7 @@ if sys.byteorder == 'big':
 else:
     macros = [('__LITTLE_ENDIAN__', '1')]
 
-msgpack_mod = Extension('msgpack._msgpack',
+msgpack_mod = Extension('msgpack._packer',
                         sources=sources,
                         libraries=libraries,
                         include_dirs=['.'],

From eec02b87294a84199d3eb527b785d6091307d868 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 22:05:48 +0900
Subject: [PATCH 1106/1648] remove unused import

---
 setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.py b/setup.py
index 5a342e0..c73b16e 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,6 @@
 # coding: utf-8
 import os
 import sys
-import shutil
 from glob import glob
 from distutils.command.sdist import sdist
 from setuptools import setup, Extension

From b79e5ba4e5e16eb9907e2e0081593ce6aa14eb66 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 22:15:21 +0900
Subject: [PATCH 1107/1648] Split _msgpack.pyx

---
 msgpack/__init__.py   |   3 +-
 msgpack/_packer.pyx   | 374 +----------------------------------------
 msgpack/_unpacker.pyx | 382 ++++++++++++++++++++++++++++++++++++++++++
 msgpack/exceptions.py |   6 +
 setup.py              |  24 ++-
 5 files changed, 407 insertions(+), 382 deletions(-)
 create mode 100644 msgpack/_unpacker.pyx

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 1aacee1..0bfe6fe 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,7 +1,8 @@
 # coding: utf-8
 from msgpack._version import version
 from msgpack.exceptions import *
-from msgpack._packer import pack, packb, Packer, unpack, unpackb, Unpacker
+from msgpack._packer import pack, packb, Packer
+from msgpack._unpacker import unpack, unpackb, Unpacker
 
 # alias for compatibility to simplejson/marshal/pickle.
 load = unpack
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index d58255e..a5bc570 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -7,13 +7,12 @@ cdef extern from "Python.h":
     ctypedef char* const_void_ptr "const void*"
     ctypedef struct PyObject
     cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
-    char* __FILE__
-    int __LINE__
 
 from libc.stdlib cimport *
 from libc.string cimport *
 from libc.limits cimport *
 
+from msgpack.exceptions import PackValueError
 
 cdef extern from "pack.h":
     struct msgpack_packer:
@@ -38,13 +37,6 @@ cdef extern from "pack.h":
 cdef int DEFAULT_RECURSE_LIMIT=511
 
 
-from msgpack.exceptions import (
-        BufferFull,
-        OutOfData,
-        UnpackValueError,
-        ExtraData,
-        )
-
 
 cdef class Packer(object):
     """MessagePack Packer
@@ -118,7 +110,7 @@ cdef class Packer(object):
         cdef dict d
 
         if nest_limit < 0:
-            raise UnpackValueError("recursion limit exceeded.")
+            raise PackValueError("recursion limit exceeded.")
 
         if o is None:
             ret = msgpack_pack_nil(&self.pk)
@@ -265,365 +257,3 @@ def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use
     packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
                     use_single_float=use_single_float)
     return packer.pack(o)
-
-
-cdef extern from "unpack.h":
-    ctypedef struct msgpack_user:
-        bint use_list
-        PyObject* object_hook
-        bint has_pairs_hook # call object_hook with k-v pairs
-        PyObject* list_hook
-        char *encoding
-        char *unicode_errors
-
-    ctypedef struct template_context:
-        msgpack_user user
-        PyObject* obj
-        size_t count
-        unsigned int ct
-        PyObject* key
-
-    ctypedef int (*execute_fn)(template_context* ctx, const_char_ptr data,
-                               size_t len, size_t* off) except -1
-    execute_fn template_construct
-    execute_fn template_skip
-    execute_fn read_array_header
-    execute_fn read_map_header
-    void template_init(template_context* ctx)
-    object template_data(template_context* ctx)
-
-cdef inline init_ctx(template_context *ctx,
-                     object object_hook, object object_pairs_hook, object list_hook,
-                     bint use_list, char* encoding, char* unicode_errors):
-    template_init(ctx)
-    ctx.user.use_list = use_list
-    ctx.user.object_hook = ctx.user.list_hook = NULL
-
-    if object_hook is not None and object_pairs_hook is not None:
-        raise ValueError("object_pairs_hook and object_hook are mutually exclusive.")
-
-    if object_hook is not None:
-        if not PyCallable_Check(object_hook):
-            raise TypeError("object_hook must be a callable.")
-        ctx.user.object_hook = object_hook
-
-    if object_pairs_hook is None:
-        ctx.user.has_pairs_hook = False
-    else:
-        if not PyCallable_Check(object_pairs_hook):
-            raise TypeError("object_pairs_hook must be a callable.")
-        ctx.user.object_hook = object_pairs_hook
-        ctx.user.has_pairs_hook = True
-
-    if list_hook is not None:
-        if not PyCallable_Check(list_hook):
-            raise TypeError("list_hook must be a callable.")
-        ctx.user.list_hook = list_hook
-
-    ctx.user.encoding = encoding
-    ctx.user.unicode_errors = unicode_errors
-
-def unpackb(object packed, object object_hook=None, object list_hook=None,
-            bint use_list=1, encoding=None, unicode_errors="strict",
-            object_pairs_hook=None,
-            ):
-    """Unpack packed_bytes to object. Returns an unpacked object.
-
-    Raises `ValueError` when `packed` contains extra bytes.
-    """
-    cdef template_context ctx
-    cdef size_t off = 0
-    cdef int ret
-
-    cdef char* buf
-    cdef Py_ssize_t buf_len
-    cdef char* cenc = NULL
-    cdef char* cerr = NULL
-
-    PyObject_AsReadBuffer(packed, &buf, &buf_len)
-
-    if encoding is not None:
-        if isinstance(encoding, unicode):
-            encoding = encoding.encode('ascii')
-        cenc = PyBytes_AsString(encoding)
-
-    if unicode_errors is not None:
-        if isinstance(unicode_errors, unicode):
-            unicode_errors = unicode_errors.encode('ascii')
-        cerr = PyBytes_AsString(unicode_errors)
-
-    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
-    ret = template_construct(&ctx, buf, buf_len, &off)
-    if ret == 1:
-        obj = template_data(&ctx)
-        if off < buf_len:
-            raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
-        return obj
-    else:
-        return None
-
-
-def unpack(object stream, object object_hook=None, object list_hook=None,
-           bint use_list=1, encoding=None, unicode_errors="strict",
-           object_pairs_hook=None,
-           ):
-    """Unpack an object from `stream`.
-
-    Raises `ValueError` when `stream` has extra bytes.
-    """
-    return unpackb(stream.read(), use_list=use_list,
-                   object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
-                   encoding=encoding, unicode_errors=unicode_errors,
-                   )
-
-
-cdef class Unpacker(object):
-    """
-    Streaming unpacker.
-
-    `file_like` is a file-like object having `.read(n)` method.
-    When `Unpacker` initialized with `file_like`, unpacker reads serialized data
-    from it and `.feed()` method is not usable.
-
-    `read_size` is used as `file_like.read(read_size)`.
-    (default: min(1024**2, max_buffer_size))
-
-    If `use_list` is true (default), msgpack list is deserialized to Python list.
-    Otherwise, it is deserialized to Python tuple.
-
-    `object_hook` is same to simplejson. If it is not None, it should be callable
-    and Unpacker calls it with a dict argument after deserializing a map.
-
-    `object_pairs_hook` is same to simplejson. If it is not None, it should be callable
-    and Unpacker calls it with a list of key-value pairs after deserializing a map.
-
-    `encoding` is encoding used for decoding msgpack bytes. If it is None (default),
-    msgpack bytes is deserialized to Python bytes.
-
-    `unicode_errors` is used for decoding bytes.
-
-    `max_buffer_size` limits size of data waiting unpacked.
-    0 means system's INT_MAX (default).
-    Raises `BufferFull` exception when it is insufficient.
-    You shoud set this parameter when unpacking data from untrasted source.
-
-    example of streaming deserialize from file-like object::
-
-        unpacker = Unpacker(file_like)
-        for o in unpacker:
-            do_something(o)
-
-    example of streaming deserialize from socket::
-
-        unpacker = Unpacker()
-        while 1:
-            buf = sock.recv(1024**2)
-            if not buf:
-                break
-            unpacker.feed(buf)
-            for o in unpacker:
-                do_something(o)
-    """
-    cdef template_context ctx
-    cdef char* buf
-    cdef size_t buf_size, buf_head, buf_tail
-    cdef object file_like
-    cdef object file_like_read
-    cdef Py_ssize_t read_size
-    cdef object object_hook
-    cdef object encoding, unicode_errors
-    cdef size_t max_buffer_size
-
-    def __cinit__(self):
-        self.buf = NULL
-
-    def __dealloc__(self):
-        free(self.buf)
-        self.buf = NULL
-
-    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
-                 object object_hook=None, object object_pairs_hook=None, object list_hook=None,
-                 encoding=None, unicode_errors='strict', int max_buffer_size=0,
-                 ):
-        cdef char *cenc=NULL, *cerr=NULL
-
-        self.file_like = file_like
-        if file_like:
-            self.file_like_read = file_like.read
-            if not PyCallable_Check(self.file_like_read):
-                raise ValueError("`file_like.read` must be a callable.")
-        if not max_buffer_size:
-            max_buffer_size = INT_MAX
-        if read_size > max_buffer_size:
-            raise ValueError("read_size should be less or equal to max_buffer_size")
-        if not read_size:
-            read_size = min(max_buffer_size, 1024**2)
-        self.max_buffer_size = max_buffer_size
-        self.read_size = read_size
-        self.buf = malloc(read_size)
-        if self.buf == NULL:
-            raise MemoryError("Unable to allocate internal buffer.")
-        self.buf_size = read_size
-        self.buf_head = 0
-        self.buf_tail = 0
-
-        if encoding is not None:
-            if isinstance(encoding, unicode):
-                encoding = encoding.encode('ascii')
-            self.encoding = encoding
-            cenc = PyBytes_AsString(encoding)
-
-        if unicode_errors is not None:
-            if isinstance(unicode_errors, unicode):
-                unicode_errors = unicode_errors.encode('ascii')
-            self.unicode_errors = unicode_errors
-            cerr = PyBytes_AsString(unicode_errors)
-
-        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
-
-    def feed(self, object next_bytes):
-        """Append `next_bytes` to internal buffer."""
-        cdef char* buf
-        cdef Py_ssize_t buf_len
-        if self.file_like is not None:
-            raise AssertionError(
-                    "unpacker.feed() is not be able to use with `file_like`.")
-        PyObject_AsReadBuffer(next_bytes, &buf, &buf_len)
-        self.append_buffer(buf, buf_len)
-
-    cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
-        cdef:
-            char* buf = self.buf
-            char* new_buf
-            size_t head = self.buf_head
-            size_t tail = self.buf_tail
-            size_t buf_size = self.buf_size
-            size_t new_size
-
-        if tail + _buf_len > buf_size:
-            if ((tail - head) + _buf_len) <= buf_size:
-                # move to front.
-                memmove(buf, buf + head, tail - head)
-                tail -= head
-                head = 0
-            else:
-                # expand buffer.
-                new_size = (tail-head) + _buf_len
-                if new_size > self.max_buffer_size:
-                    raise BufferFull
-                new_size = min(new_size*2, self.max_buffer_size)
-                new_buf = malloc(new_size)
-                if new_buf == NULL:
-                    # self.buf still holds old buffer and will be freed during
-                    # obj destruction
-                    raise MemoryError("Unable to enlarge internal buffer.")
-                memcpy(new_buf, buf + head, tail - head)
-                free(buf)
-
-                buf = new_buf
-                buf_size = new_size
-                tail -= head
-                head = 0
-
-        memcpy(buf + tail, (_buf), _buf_len)
-        self.buf = buf
-        self.buf_head = head
-        self.buf_size = buf_size
-        self.buf_tail = tail + _buf_len
-
-    cdef read_from_file(self):
-        next_bytes = self.file_like_read(
-                min(self.read_size,
-                    self.max_buffer_size - (self.buf_tail - self.buf_head)
-                    ))
-        if next_bytes:
-            self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes))
-        else:
-            self.file_like = None
-
-    cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0):
-        cdef int ret
-        cdef object obj
-        cdef size_t prev_head
-        while 1:
-            prev_head = self.buf_head
-            ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
-            if write_bytes is not None:
-                write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
-
-            if ret == 1:
-                obj = template_data(&self.ctx)
-                template_init(&self.ctx)
-                return obj
-            elif ret == 0:
-                if self.file_like is not None:
-                    self.read_from_file()
-                    continue
-                if iter:
-                    raise StopIteration("No more data to unpack.")
-                else:
-                    raise OutOfData("No more data to unpack.")
-            else:
-                raise ValueError("Unpack failed: error = %d" % (ret,))
-
-    def read_bytes(self, Py_ssize_t nbytes):
-        """read a specified number of raw bytes from the stream"""
-        cdef size_t nread
-        nread = min(self.buf_tail - self.buf_head, nbytes)
-        ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread)
-        self.buf_head += nread
-        if len(ret) < nbytes and self.file_like is not None:
-            ret += self.file_like.read(nbytes - len(ret))
-        return ret
-
-    def unpack(self, object write_bytes=None):
-        """
-        unpack one object
-
-        If write_bytes is not None, it will be called with parts of the raw
-        message as it is unpacked.
-
-        Raises `OutOfData` when there are no more bytes to unpack.
-        """
-        return self._unpack(template_construct, write_bytes)
-
-    def skip(self, object write_bytes=None):
-        """
-        read and ignore one object, returning None
-
-        If write_bytes is not None, it will be called with parts of the raw
-        message as it is unpacked.
-
-        Raises `OutOfData` when there are no more bytes to unpack.
-        """
-        return self._unpack(template_skip, write_bytes)
-
-    def read_array_header(self, object write_bytes=None):
-        """assuming the next object is an array, return its size n, such that
-        the next n unpack() calls will iterate over its contents.
-
-        Raises `OutOfData` when there are no more bytes to unpack.
-        """
-        return self._unpack(read_array_header, write_bytes)
-
-    def read_map_header(self, object write_bytes=None):
-        """assuming the next object is a map, return its size n, such that the
-        next n * 2 unpack() calls will iterate over its key-value pairs.
-
-        Raises `OutOfData` when there are no more bytes to unpack.
-        """
-        return self._unpack(read_map_header, write_bytes)
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        return self._unpack(template_construct, None, 1)
-
-    # for debug.
-    #def _buf(self):
-    #    return PyString_FromStringAndSize(self.buf, self.buf_tail)
-
-    #def _off(self):
-    #    return self.buf_head
-
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
new file mode 100644
index 0000000..4a27d9e
--- /dev/null
+++ b/msgpack/_unpacker.pyx
@@ -0,0 +1,382 @@
+# coding: utf-8
+#cython: embedsignature=True
+
+from cpython cimport *
+cdef extern from "Python.h":
+    ctypedef char* const_char_ptr "const char*"
+    ctypedef char* const_void_ptr "const void*"
+    ctypedef struct PyObject
+    cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
+
+from libc.stdlib cimport *
+from libc.string cimport *
+from libc.limits cimport *
+
+from msgpack.exceptions import (
+        BufferFull,
+        OutOfData,
+        UnpackValueError,
+        ExtraData,
+        )
+
+
+
+cdef extern from "unpack.h":
+    ctypedef struct msgpack_user:
+        bint use_list
+        PyObject* object_hook
+        bint has_pairs_hook # call object_hook with k-v pairs
+        PyObject* list_hook
+        char *encoding
+        char *unicode_errors
+
+    ctypedef struct template_context:
+        msgpack_user user
+        PyObject* obj
+        size_t count
+        unsigned int ct
+        PyObject* key
+
+    ctypedef int (*execute_fn)(template_context* ctx, const_char_ptr data,
+                               size_t len, size_t* off) except -1
+    execute_fn template_construct
+    execute_fn template_skip
+    execute_fn read_array_header
+    execute_fn read_map_header
+    void template_init(template_context* ctx)
+    object template_data(template_context* ctx)
+
+cdef inline init_ctx(template_context *ctx,
+                     object object_hook, object object_pairs_hook, object list_hook,
+                     bint use_list, char* encoding, char* unicode_errors):
+    template_init(ctx)
+    ctx.user.use_list = use_list
+    ctx.user.object_hook = ctx.user.list_hook = NULL
+
+    if object_hook is not None and object_pairs_hook is not None:
+        raise ValueError("object_pairs_hook and object_hook are mutually exclusive.")
+
+    if object_hook is not None:
+        if not PyCallable_Check(object_hook):
+            raise TypeError("object_hook must be a callable.")
+        ctx.user.object_hook = object_hook
+
+    if object_pairs_hook is None:
+        ctx.user.has_pairs_hook = False
+    else:
+        if not PyCallable_Check(object_pairs_hook):
+            raise TypeError("object_pairs_hook must be a callable.")
+        ctx.user.object_hook = object_pairs_hook
+        ctx.user.has_pairs_hook = True
+
+    if list_hook is not None:
+        if not PyCallable_Check(list_hook):
+            raise TypeError("list_hook must be a callable.")
+        ctx.user.list_hook = list_hook
+
+    ctx.user.encoding = encoding
+    ctx.user.unicode_errors = unicode_errors
+
+def unpackb(object packed, object object_hook=None, object list_hook=None,
+            bint use_list=1, encoding=None, unicode_errors="strict",
+            object_pairs_hook=None,
+            ):
+    """Unpack packed_bytes to object. Returns an unpacked object.
+
+    Raises `ValueError` when `packed` contains extra bytes.
+    """
+    cdef template_context ctx
+    cdef size_t off = 0
+    cdef int ret
+
+    cdef char* buf
+    cdef Py_ssize_t buf_len
+    cdef char* cenc = NULL
+    cdef char* cerr = NULL
+
+    PyObject_AsReadBuffer(packed, &buf, &buf_len)
+
+    if encoding is not None:
+        if isinstance(encoding, unicode):
+            encoding = encoding.encode('ascii')
+        cenc = PyBytes_AsString(encoding)
+
+    if unicode_errors is not None:
+        if isinstance(unicode_errors, unicode):
+            unicode_errors = unicode_errors.encode('ascii')
+        cerr = PyBytes_AsString(unicode_errors)
+
+    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
+    ret = template_construct(&ctx, buf, buf_len, &off)
+    if ret == 1:
+        obj = template_data(&ctx)
+        if off < buf_len:
+            raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
+        return obj
+    else:
+        raise UnpackValueError
+
+
+def unpack(object stream, object object_hook=None, object list_hook=None,
+           bint use_list=1, encoding=None, unicode_errors="strict",
+           object_pairs_hook=None,
+           ):
+    """Unpack an object from `stream`.
+
+    Raises `ValueError` when `stream` has extra bytes.
+    """
+    return unpackb(stream.read(), use_list=use_list,
+                   object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
+                   encoding=encoding, unicode_errors=unicode_errors,
+                   )
+
+
+cdef class Unpacker(object):
+    """
+    Streaming unpacker.
+
+    `file_like` is a file-like object having `.read(n)` method.
+    When `Unpacker` initialized with `file_like`, unpacker reads serialized data
+    from it and `.feed()` method is not usable.
+
+    `read_size` is used as `file_like.read(read_size)`.
+    (default: min(1024**2, max_buffer_size))
+
+    If `use_list` is true (default), msgpack list is deserialized to Python list.
+    Otherwise, it is deserialized to Python tuple.
+
+    `object_hook` is same to simplejson. If it is not None, it should be callable
+    and Unpacker calls it with a dict argument after deserializing a map.
+
+    `object_pairs_hook` is same to simplejson. If it is not None, it should be callable
+    and Unpacker calls it with a list of key-value pairs after deserializing a map.
+
+    `encoding` is encoding used for decoding msgpack bytes. If it is None (default),
+    msgpack bytes is deserialized to Python bytes.
+
+    `unicode_errors` is used for decoding bytes.
+
+    `max_buffer_size` limits size of data waiting unpacked.
+    0 means system's INT_MAX (default).
+    Raises `BufferFull` exception when it is insufficient.
+    You shoud set this parameter when unpacking data from untrasted source.
+
+    example of streaming deserialize from file-like object::
+
+        unpacker = Unpacker(file_like)
+        for o in unpacker:
+            do_something(o)
+
+    example of streaming deserialize from socket::
+
+        unpacker = Unpacker()
+        while 1:
+            buf = sock.recv(1024**2)
+            if not buf:
+                break
+            unpacker.feed(buf)
+            for o in unpacker:
+                do_something(o)
+    """
+    cdef template_context ctx
+    cdef char* buf
+    cdef size_t buf_size, buf_head, buf_tail
+    cdef object file_like
+    cdef object file_like_read
+    cdef Py_ssize_t read_size
+    cdef object object_hook
+    cdef object encoding, unicode_errors
+    cdef size_t max_buffer_size
+
+    def __cinit__(self):
+        self.buf = NULL
+
+    def __dealloc__(self):
+        free(self.buf)
+        self.buf = NULL
+
+    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
+                 object object_hook=None, object object_pairs_hook=None, object list_hook=None,
+                 encoding=None, unicode_errors='strict', int max_buffer_size=0,
+                 ):
+        cdef char *cenc=NULL, *cerr=NULL
+
+        self.file_like = file_like
+        if file_like:
+            self.file_like_read = file_like.read
+            if not PyCallable_Check(self.file_like_read):
+                raise ValueError("`file_like.read` must be a callable.")
+        if not max_buffer_size:
+            max_buffer_size = INT_MAX
+        if read_size > max_buffer_size:
+            raise ValueError("read_size should be less or equal to max_buffer_size")
+        if not read_size:
+            read_size = min(max_buffer_size, 1024**2)
+        self.max_buffer_size = max_buffer_size
+        self.read_size = read_size
+        self.buf = malloc(read_size)
+        if self.buf == NULL:
+            raise MemoryError("Unable to allocate internal buffer.")
+        self.buf_size = read_size
+        self.buf_head = 0
+        self.buf_tail = 0
+
+        if encoding is not None:
+            if isinstance(encoding, unicode):
+                encoding = encoding.encode('ascii')
+            self.encoding = encoding
+            cenc = PyBytes_AsString(encoding)
+
+        if unicode_errors is not None:
+            if isinstance(unicode_errors, unicode):
+                unicode_errors = unicode_errors.encode('ascii')
+            self.unicode_errors = unicode_errors
+            cerr = PyBytes_AsString(unicode_errors)
+
+        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
+
+    def feed(self, object next_bytes):
+        """Append `next_bytes` to internal buffer."""
+        cdef char* buf
+        cdef Py_ssize_t buf_len
+        if self.file_like is not None:
+            raise AssertionError(
+                    "unpacker.feed() is not be able to use with `file_like`.")
+        PyObject_AsReadBuffer(next_bytes, &buf, &buf_len)
+        self.append_buffer(buf, buf_len)
+
+    cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
+        cdef:
+            char* buf = self.buf
+            char* new_buf
+            size_t head = self.buf_head
+            size_t tail = self.buf_tail
+            size_t buf_size = self.buf_size
+            size_t new_size
+
+        if tail + _buf_len > buf_size:
+            if ((tail - head) + _buf_len) <= buf_size:
+                # move to front.
+                memmove(buf, buf + head, tail - head)
+                tail -= head
+                head = 0
+            else:
+                # expand buffer.
+                new_size = (tail-head) + _buf_len
+                if new_size > self.max_buffer_size:
+                    raise BufferFull
+                new_size = min(new_size*2, self.max_buffer_size)
+                new_buf = malloc(new_size)
+                if new_buf == NULL:
+                    # self.buf still holds old buffer and will be freed during
+                    # obj destruction
+                    raise MemoryError("Unable to enlarge internal buffer.")
+                memcpy(new_buf, buf + head, tail - head)
+                free(buf)
+
+                buf = new_buf
+                buf_size = new_size
+                tail -= head
+                head = 0
+
+        memcpy(buf + tail, (_buf), _buf_len)
+        self.buf = buf
+        self.buf_head = head
+        self.buf_size = buf_size
+        self.buf_tail = tail + _buf_len
+
+    cdef read_from_file(self):
+        next_bytes = self.file_like_read(
+                min(self.read_size,
+                    self.max_buffer_size - (self.buf_tail - self.buf_head)
+                    ))
+        if next_bytes:
+            self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes))
+        else:
+            self.file_like = None
+
+    cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0):
+        cdef int ret
+        cdef object obj
+        cdef size_t prev_head
+        while 1:
+            prev_head = self.buf_head
+            ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+            if write_bytes is not None:
+                write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
+
+            if ret == 1:
+                obj = template_data(&self.ctx)
+                template_init(&self.ctx)
+                return obj
+            elif ret == 0:
+                if self.file_like is not None:
+                    self.read_from_file()
+                    continue
+                if iter:
+                    raise StopIteration("No more data to unpack.")
+                else:
+                    raise OutOfData("No more data to unpack.")
+            else:
+                raise ValueError("Unpack failed: error = %d" % (ret,))
+
+    def read_bytes(self, Py_ssize_t nbytes):
+        """read a specified number of raw bytes from the stream"""
+        cdef size_t nread
+        nread = min(self.buf_tail - self.buf_head, nbytes)
+        ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread)
+        self.buf_head += nread
+        if len(ret) < nbytes and self.file_like is not None:
+            ret += self.file_like.read(nbytes - len(ret))
+        return ret
+
+    def unpack(self, object write_bytes=None):
+        """
+        unpack one object
+
+        If write_bytes is not None, it will be called with parts of the raw
+        message as it is unpacked.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(template_construct, write_bytes)
+
+    def skip(self, object write_bytes=None):
+        """
+        read and ignore one object, returning None
+
+        If write_bytes is not None, it will be called with parts of the raw
+        message as it is unpacked.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(template_skip, write_bytes)
+
+    def read_array_header(self, object write_bytes=None):
+        """assuming the next object is an array, return its size n, such that
+        the next n unpack() calls will iterate over its contents.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(read_array_header, write_bytes)
+
+    def read_map_header(self, object write_bytes=None):
+        """assuming the next object is a map, return its size n, such that the
+        next n * 2 unpack() calls will iterate over its key-value pairs.
+
+        Raises `OutOfData` when there are no more bytes to unpack.
+        """
+        return self._unpack(read_map_header, write_bytes)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return self._unpack(template_construct, None, 1)
+
+    # for debug.
+    #def _buf(self):
+    #    return PyString_FromStringAndSize(self.buf, self.buf_tail)
+
+    #def _off(self):
+    #    return self.buf_head
diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py
index 0a75430..2565541 100644
--- a/msgpack/exceptions.py
+++ b/msgpack/exceptions.py
@@ -21,3 +21,9 @@ class ExtraData(ValueError):
 
     def __str__(self):
         return "unpack(b) recieved extra data."
+
+class PackException(Exception):
+    pass
+
+class PackValueError(PackException, ValueError):
+    pass
diff --git a/setup.py b/setup.py
index c73b16e..ac3eeb5 100644
--- a/setup.py
+++ b/setup.py
@@ -58,7 +58,6 @@ if have_cython:
 else:
     Sdist = sdist
 
-sources = ['msgpack/_packer.cpp']
 libraries = []
 if sys.platform == 'win32':
     libraries.append('ws2_32')
@@ -68,13 +67,20 @@ if sys.byteorder == 'big':
 else:
     macros = [('__LITTLE_ENDIAN__', '1')]
 
-msgpack_mod = Extension('msgpack._packer',
-                        sources=sources,
-                        libraries=libraries,
-                        include_dirs=['.'],
-                        define_macros=macros,
-                        )
-del sources, libraries, macros
+ext_modules = []
+ext_modules.append(Extension('msgpack._packer',
+                             sources=['msgpack/_packer.cpp'],
+                             libraries=libraries,
+                             include_dirs=['.'],
+                             define_macros=macros,
+                             ))
+ext_modules.append(Extension('msgpack._unpacker',
+                             sources=['msgpack/_unpacker.cpp'],
+                             libraries=libraries,
+                             include_dirs=['.'],
+                             define_macros=macros,
+                             ))
+del libraries, macros
 
 
 desc = 'MessagePack (de)serializer.'
@@ -88,7 +94,7 @@ setup(name='msgpack-python',
       author_email='songofacandy@gmail.com',
       version=version_str,
       cmdclass={'build_ext': BuildExt, 'sdist': Sdist},
-      ext_modules=[msgpack_mod],
+      ext_modules=ext_modules,
       packages=['msgpack'],
       description=desc,
       long_description=long_desc,

From 1c5b865db36a9cd2db5eb8de7625573a78085c56 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 22:15:53 +0900
Subject: [PATCH 1108/1648] Update .gitignore

---
 .gitignore | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 7918394..1bd68b4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,5 @@ dist/*
 *.so
 *~
 msgpack/__version__.py
-msgpack/_msgpack.c
-msgpack/_msgpack.cpp
+msgpack/*.cpp
 *.egg-info

From 78c345555b4ffd4cdba1b587e747ba59d794838d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 22:26:23 +0900
Subject: [PATCH 1109/1648] Update .travis.yml

---
 .travis.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 11835cb..16812bc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,7 +7,8 @@ python:
 install:
     - sudo apt-get update -qq
     - sudo apt-get install -q cython
-    - cython --cplus msgpack/_msgpack.pyx
+    - cython --cplus msgpack/_packer.pyx
+    - cython --cplus msgpack/_unpacker.pyx
     - pip install six --use-mirrors
     - python setup.py install
 

From 2ad02bb11a9e1a54bee11040b04a6c7138779e44 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 22:30:38 +0900
Subject: [PATCH 1110/1648] Add Python 3.3 to .travis.yml

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 16812bc..dd4a10a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,6 +3,7 @@ python:
     - 2.6
     - 2.7
     - 3.2
+    - 3.3
 
 install:
     - sudo apt-get update -qq

From 8b27482f5f86c91c2a0d1fc62b4d72b0accb2756 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 22:36:09 +0900
Subject: [PATCH 1111/1648] Use tox on Travis.

---
 .travis.yml | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index dd4a10a..f6d8a48 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,16 +1,12 @@
 language: python
 python:
-    - 2.6
     - 2.7
-    - 3.2
-    - 3.3
 
 install:
     - sudo apt-get update -qq
     - sudo apt-get install -q cython
+    - sudo apt-get install -q python26-dev python27-dev python32-dev python33-dev
     - cython --cplus msgpack/_packer.pyx
     - cython --cplus msgpack/_unpacker.pyx
-    - pip install six --use-mirrors
-    - python setup.py install
 
-script: "nosetests -w test"
+script: "tox"

From ef054cef51db34438aeb8df530dbf8c3ef819b64 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 22:40:07 +0900
Subject: [PATCH 1112/1648] fix package name

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index f6d8a48..e636061 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,7 @@ python:
 install:
     - sudo apt-get update -qq
     - sudo apt-get install -q cython
-    - sudo apt-get install -q python26-dev python27-dev python32-dev python33-dev
+    - sudo apt-get install -q python2.6-dev python2.7-dev python3.2-dev python3.3-dev
     - cython --cplus msgpack/_packer.pyx
     - cython --cplus msgpack/_unpacker.pyx
 

From 1dd9280bffe2c26f0efd6e38dc212e35ed23dc0d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 22:42:08 +0900
Subject: [PATCH 1113/1648] Install tox

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index e636061..ec2fa3b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,6 +3,7 @@ python:
     - 2.7
 
 install:
+    - pip install --use-mirrors tox
     - sudo apt-get update -qq
     - sudo apt-get install -q cython
     - sudo apt-get install -q python2.6-dev python2.7-dev python3.2-dev python3.3-dev

From 7bebb665fbca11f9bbb83545b5d60639d26c6338 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 22:46:28 +0900
Subject: [PATCH 1114/1648] Set PIP_USE_MIRRORS=true for faster venv creation.

---
 .travis.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index ec2fa3b..efa5f8b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,9 @@ language: python
 python:
     - 2.7
 
+env:
+    - PIP_USE_MIRRORS=true
+
 install:
     - pip install --use-mirrors tox
     - sudo apt-get update -qq

From 34611a8ccdc61ba2a8bbb5612c32ed8054315b3f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 11 Dec 2012 22:49:20 +0900
Subject: [PATCH 1115/1648] Use newer cython.

---
 .travis.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index efa5f8b..2e6fc56 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,10 +6,9 @@ env:
     - PIP_USE_MIRRORS=true
 
 install:
-    - pip install --use-mirrors tox
     - sudo apt-get update -qq
-    - sudo apt-get install -q cython
-    - sudo apt-get install -q python2.6-dev python2.7-dev python3.2-dev python3.3-dev
+    - sudo apt-get install -q python3.3-dev
+    - pip install --use-mirrors tox cython
     - cython --cplus msgpack/_packer.pyx
     - cython --cplus msgpack/_unpacker.pyx
 

From 0fa8c102d7f8ed618284e1b039fa867cfd9baa2f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 22 Dec 2012 12:12:32 +0900
Subject: [PATCH 1116/1648] Add test reproducing SEGV

---
 test/test_except.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/test/test_except.py b/test/test_except.py
index ad02cb6..e142dd6 100644
--- a/test/test_except.py
+++ b/test/test_except.py
@@ -6,9 +6,28 @@ from msgpack import packb, unpackb
 
 import datetime
 
+class DummyException(Exception):
+    pass
+
+
 def test_raise_on_find_unsupported_value():
     assert_raises(TypeError, packb, datetime.datetime.now())
 
+
+def test_raise_from_object_hook():
+    def hook(obj):
+        raise DummyException
+    assert_raises(DummyException, unpackb, packb({}), object_hook=hook)
+    assert_raises(DummyException, unpackb, packb({'fizz': 'buzz'}),
+                  object_hook=hook)
+    assert_raises(DummyException, unpackb, packb({'fizz': 'buzz'}),
+                  object_pairs_hook=hook)
+    assert_raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}),
+                  object_hook=hook)
+    assert_raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}),
+                  object_pairs_hook=hook)
+
+
 if __name__ == '__main__':
     from nose import main
     main()

From 79e44f86c9ec554f56b9807f909e21887c36d166 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 22 Dec 2012 12:14:05 +0900
Subject: [PATCH 1117/1648] Add NULL check.

---
 msgpack/unpack.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 7064a1b..0851ec2 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -206,6 +206,9 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec
 {
     if (u->object_hook) {
         PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c);
+        if (new_c == NULL) {
+            return -1;
+        }
         Py_DECREF(*c);
         *c = new_c;
     }

From ce2c5b22efcbbbc5598c7a631b3558d757b8b9dc Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 22 Dec 2012 12:42:36 +0900
Subject: [PATCH 1118/1648] Check return value of _end functions.

---
 msgpack/unpack_template.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 6080a51..9450943 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -145,7 +145,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
 	if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
 	if((count_) == 0) { obj = stack[top].obj; \
-		construct_cb(func##_end)(user, &obj); \
+		if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \
 		goto _push; } \
 	stack[top].ct = ct_; \
 	stack[top].size  = count_; \
@@ -346,7 +346,7 @@ _push:
 		if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
 		if(++c->count == c->size) {
 			obj = c->obj;
-			construct_cb(_array_end)(user, &obj);
+			if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; }
 			--top;
 			/*printf("stack pop %d\n", top);*/
 			goto _push;
@@ -360,7 +360,7 @@ _push:
 		if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
 		if(++c->count == c->size) {
 			obj = c->obj;
-			construct_cb(_map_end)(user, &obj);
+			if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; }
 			--top;
 			/*printf("stack pop %d\n", top);*/
 			goto _push;

From d796d696d193277504b76e4ba3144154bdc1163b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 22 Dec 2012 13:09:35 +0900
Subject: [PATCH 1119/1648] revert unwanted changes.

---
 test/test_obj.py       | 2 +-
 test/test_sequnpack.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_obj.py b/test/test_obj.py
index bfc9ab2..1d9024b 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -34,7 +34,7 @@ def test_decode_pairs_hook():
 
 @raises(ValueError)
 def test_only_one_obj_hook():
-    unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x, use_list=1)
+    unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x)
 
 @raises(ValueError)
 def test_bad_hook():
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index ca4c8db..f767726 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -46,7 +46,7 @@ def test_foobar_skip():
         assert 1, "ok"
 
 def test_maxbuffersize():
-    nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3, use_list=1)
+    nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3)
     unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
     unpacker.feed(b'fo')
     nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob')

From 431fe8f9e09ad8615c6dbcf8e299a8e8f2dc5ba2 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 22 Dec 2012 17:04:53 +0900
Subject: [PATCH 1120/1648] Update changelog

---
 ChangeLog.rst | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 833c97c..69adcaa 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,14 @@
+0.2.4
+=======
+
+Bugs fixed
+----------
+
+* Fix SEGV when object_hook or object_pairs_hook raise Exception. (#39)
+
 0.2.3
 =======
-:release date: in development
+:release date: 2012-12-11
 
 Changes
 -------

From 9dc299bd8da75e61f512e6ba50bd06face71f8a0 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 22 Dec 2012 17:11:30 +0900
Subject: [PATCH 1121/1648] 0.2.4

---
 ChangeLog.rst       | 1 +
 msgpack/_version.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 69adcaa..f684846 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,5 +1,6 @@
 0.2.4
 =======
+:release date: 2012-12-22
 
 Bugs fixed
 ----------
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 1bbdf2a..0f5f95a 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 2, 3)
+version = (0, 2, 4)

From 7b11a42825a631ba85608a3c0943919073d9899e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 22 Dec 2012 17:13:45 +0900
Subject: [PATCH 1122/1648] Update ChangeLog.

---
 ChangeLog.rst | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index a66a276..55e296f 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -20,9 +20,19 @@ Changes
 * Add ``Packer.pack_map_pairs``. It packs sequence of pair to map type.
 
 
+
+0.2.4
+=======
+:release date: 2012-12-22
+
+Bugs fixed
+----------
+
+* Fix SEGV when object_hook or object_pairs_hook raise Exception. (#39)
+
 0.2.3
 =======
-:release date: in development
+:release date: 2012-12-11
 
 Changes
 -------

From 72416e403c6d70b6fbc4ea8e05e1fe7f02a6561a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 29 Dec 2012 01:39:59 +0900
Subject: [PATCH 1123/1648] Fix unpacker doesn't raise exception for invalid
 input.

---
 msgpack/_msgpack.pyx | 4 +++-
 test/test_except.py  | 6 ++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
index 5feba11..78b2aa7 100644
--- a/msgpack/_msgpack.pyx
+++ b/msgpack/_msgpack.pyx
@@ -210,7 +210,7 @@ cdef extern from "unpack.h":
         PyObject* key
 
     int template_execute(template_context* ctx, const_char_ptr data,
-                         size_t len, size_t* off, bint construct) except -1
+                         size_t len, size_t* off, bint construct) except? -1
     void template_init(template_context* ctx)
     object template_data(template_context* ctx)
 
@@ -285,6 +285,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         if off < buf_len:
             raise ValueError("Extra data.")
         return obj
+    elif ret < 0:
+        raise ValueError("Unpack failed: error = %d" % (ret,))
     else:
         return None
 
diff --git a/test/test_except.py b/test/test_except.py
index e142dd6..35287df 100644
--- a/test/test_except.py
+++ b/test/test_except.py
@@ -6,6 +6,7 @@ from msgpack import packb, unpackb
 
 import datetime
 
+
 class DummyException(Exception):
     pass
 
@@ -28,6 +29,11 @@ def test_raise_from_object_hook():
                   object_pairs_hook=hook)
 
 
+@raises(ValueError)
+def test_invalidvalue():
+    unpackb(b'\xd9\x97#DL_')
+
+
 if __name__ == '__main__':
     from nose import main
     main()

From 593c832ab00372b4a44dd47de94e4c2546fc1193 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 29 Dec 2012 11:24:25 +0900
Subject: [PATCH 1124/1648] Use py.test instead of nosetests.

---
 test/test_buffer.py     |  7 ++----
 test/test_case.py       | 15 ++++++-------
 test/test_except.py     | 28 +++++++++---------------
 test/test_format.py     |  7 +-----
 test/test_obj.py        | 39 ++++++++++++++--------------------
 test/test_pack.py       | 47 ++++++++++++++++++-----------------------
 test/test_seq.py        | 14 +++---------
 test/test_sequnpack.py  | 25 +++++++++-------------
 test/test_subtype.py    | 12 +++--------
 test/test_unpack_raw.py | 17 ++++++---------
 10 files changed, 77 insertions(+), 134 deletions(-)

diff --git a/test/test_buffer.py b/test/test_buffer.py
index 785fb60..04cc02d 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -1,16 +1,13 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from nose import main
-from nose.tools import *
 from msgpack import packb, unpackb
 
+
 def test_unpack_buffer():
     from array import array
     buf = array('b')
     buf.fromstring(packb(('foo', 'bar')))
     obj = unpackb(buf, use_list=1)
-    assert_equal([b'foo', b'bar'], obj)
+    assert [b'foo', b'bar'] == obj
 
-if __name__ == '__main__':
-    main()
diff --git a/test/test_case.py b/test/test_case.py
index 9cbf9bd..5a4bb6c 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -1,15 +1,14 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from nose import main
-from nose.tools import *
 from msgpack import packb, unpackb
 
 
 def check(length, obj):
     v = packb(obj)
-    assert_equal(len(v), length, "%r length should be %r but get %r" % (obj, length, len(v)))
-    assert_equal(unpackb(v, use_list=0), obj)
+    assert len(v) == length, \
+        "%r length should be %r but get %r" % (obj, length, len(v))
+    assert unpackb(v, use_list=0) == obj
 
 def test_1():
     for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
@@ -70,8 +69,8 @@ def test_array32():
 
 
 def match(obj, buf):
-    assert_equal(packb(obj), buf)
-    assert_equal(unpackb(buf, use_list=0), obj)
+    assert packb(obj) == buf
+    assert unpackb(buf, use_list=0) == obj
 
 def test_match():
     cases = [
@@ -99,7 +98,5 @@ def test_match():
         match(v, p)
 
 def test_unicode():
-    assert_equal(b'foobar', unpackb(packb('foobar'), use_list=1))
+    assert unpackb(packb('foobar'), use_list=1) == b'foobar'
 
-if __name__ == '__main__':
-    main()
diff --git a/test/test_except.py b/test/test_except.py
index 35287df..361d4ea 100644
--- a/test/test_except.py
+++ b/test/test_except.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from nose.tools import *
+from pytest import raises
 from msgpack import packb, unpackb
 
 import datetime
@@ -12,28 +12,20 @@ class DummyException(Exception):
 
 
 def test_raise_on_find_unsupported_value():
-    assert_raises(TypeError, packb, datetime.datetime.now())
+    with raises(TypeError):
+        packb(datetime.datetime.now())
 
 
 def test_raise_from_object_hook():
     def hook(obj):
         raise DummyException
-    assert_raises(DummyException, unpackb, packb({}), object_hook=hook)
-    assert_raises(DummyException, unpackb, packb({'fizz': 'buzz'}),
-                  object_hook=hook)
-    assert_raises(DummyException, unpackb, packb({'fizz': 'buzz'}),
-                  object_pairs_hook=hook)
-    assert_raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}),
-                  object_hook=hook)
-    assert_raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}),
-                  object_pairs_hook=hook)
+    raises(DummyException, unpackb, packb({}), object_hook=hook)
+    raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_hook=hook)
+    raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_pairs_hook=hook)
+    raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_hook=hook)
+    raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_pairs_hook=hook)
 
 
-@raises(ValueError)
 def test_invalidvalue():
-    unpackb(b'\xd9\x97#DL_')
-
-
-if __name__ == '__main__':
-    from nose import main
-    main()
+    with raises(ValueError):
+        unpackb(b'\xd9\x97#DL_')
diff --git a/test/test_format.py b/test/test_format.py
index ac08709..5fec0c3 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -1,12 +1,10 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from nose import main
-from nose.tools import *
 from msgpack import unpackb
 
 def check(src, should, use_list=0):
-    assert_equal(unpackb(src, use_list=use_list), should)
+    assert unpackb(src, use_list=use_list) == should
 
 def testSimpleValue():
     check(b"\x93\xc0\xc2\xc3", 
@@ -70,6 +68,3 @@ def testMap():
             b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
         ({}, {None: False}, {True: False, None: False}, {},
             {None: False}, {True: False, None: False}))
-
-if __name__ == '__main__':
-    main()
diff --git a/test/test_obj.py b/test/test_obj.py
index 1d9024b..fbf610c 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from nose import main
-from nose.tools import *
-
+from pytest import raises
 from msgpack import packb, unpackb
 
 def _decode_complex(obj):
@@ -19,27 +17,27 @@ def _encode_complex(obj):
 def test_encode_hook():
     packed = packb([3, 1+2j], default=_encode_complex)
     unpacked = unpackb(packed, use_list=1)
-    eq_(unpacked[1], {b'__complex__': True, b'real': 1, b'imag': 2})
+    assert unpacked[1] == {b'__complex__': True, b'real': 1, b'imag': 2}
 
 def test_decode_hook():
     packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
     unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1)
-    eq_(unpacked[1], 1+2j)
+    assert unpacked[1] == 1+2j
 
 def test_decode_pairs_hook():
     packed = packb([3, {1: 2, 3: 4}])
     prod_sum = 1 * 2 + 3 * 4
     unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1)
-    eq_(unpacked[1], prod_sum)
+    assert unpacked[1] == prod_sum
 
-@raises(ValueError)
 def test_only_one_obj_hook():
-    unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x)
+    with raises(ValueError):
+        unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x)
 
-@raises(ValueError)
 def test_bad_hook():
-    packed = packb([3, 1+2j], default=lambda o: o)
-    unpacked = unpackb(packed, use_list=1)
+    with raises(ValueError):
+        packed = packb([3, 1+2j], default=lambda o: o)
+        unpacked = unpackb(packed, use_list=1)
 
 def _arr_to_str(arr):
     return ''.join(str(c) for c in arr)
@@ -47,7 +45,7 @@ def _arr_to_str(arr):
 def test_array_hook():
     packed = packb([1,2,3])
     unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1)
-    eq_(unpacked, '123')
+    assert unpacked == '123'
 
 
 class DecodeError(Exception):
@@ -57,18 +55,13 @@ def bad_complex_decoder(o):
     raise DecodeError("Ooops!")
 
 
-@raises(DecodeError)
 def test_an_exception_in_objecthook1():
-    packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}})
-    unpackb(packed, object_hook=bad_complex_decoder)
+    with raises(DecodeError):
+        packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}})
+        unpackb(packed, object_hook=bad_complex_decoder)
 
 
-@raises(DecodeError)
 def test_an_exception_in_objecthook2():
-    packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]})
-    unpackb(packed, list_hook=bad_complex_decoder, use_list=1)
-
-
-
-if __name__ == '__main__':
-    main()
+    with raises(DecodeError):
+        packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]})
+        unpackb(packed, list_hook=bad_complex_decoder, use_list=1)
diff --git a/test/test_pack.py b/test/test_pack.py
index b934dd2..8f4117c 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -3,9 +3,7 @@
 
 import six
 import struct
-from nose import main
-from nose.tools import *
-from nose.plugins.skip import SkipTest
+from pytest import raises, xfail
 
 from msgpack import packb, unpackb, Unpacker, Packer
 
@@ -13,7 +11,7 @@ from io import BytesIO
 
 def check(data, use_list=False):
     re = unpackb(packb(data), use_list=use_list)
-    assert_equal(re, data)
+    assert re == data
 
 def testPack():
     test_data = [
@@ -35,11 +33,11 @@ def testPackUnicode():
         ]
     for td in test_data:
         re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8')
-        assert_equal(re, td)
+        assert re == td
         packer = Packer(encoding='utf-8')
         data = packer.pack(td)
         re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack()
-        assert_equal(re, td)
+        assert re == td
 
 def testPackUTF32():
     try:
@@ -51,9 +49,9 @@ def testPackUTF32():
             ]
         for td in test_data:
             re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
-            assert_equal(re, td)
-    except LookupError:
-        raise SkipTest
+            assert re == td
+    except LookupError as e:
+        xfail(e)
 
 def testPackBytes():
     test_data = [
@@ -64,31 +62,31 @@ def testPackBytes():
 
 def testIgnoreUnicodeErrors():
     re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
-    assert_equal(re, "abcdef")
+    assert re == "abcdef"
 
-@raises(UnicodeDecodeError)
 def testStrictUnicodeUnpack():
-    unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1)
+    with raises(UnicodeDecodeError):
+        unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1)
 
-@raises(UnicodeEncodeError)
 def testStrictUnicodePack():
-    packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict')
+    with raises(UnicodeEncodeError):
+        packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict')
 
 def testIgnoreErrorsPack():
     re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1)
-    assert_equal(re, six.u("abcdef"))
+    assert re == six.u("abcdef")
 
-@raises(TypeError)
 def testNoEncoding():
-    packb(six.u("abc"), encoding=None)
+    with raises(TypeError):
+        packb(six.u("abc"), encoding=None)
 
 def testDecodeBinary():
     re = unpackb(packb("abc"), encoding=None, use_list=1)
-    assert_equal(re, b"abc")
+    assert re == b"abc"
 
 def testPackFloat():
-    assert_equal(packb(1.0, use_single_float=True),  b'\xca' + struct.pack('>f', 1.0))
-    assert_equal(packb(1.0, use_single_float=False), b'\xcb' + struct.pack('>d', 1.0))
+    assert packb(1.0, use_single_float=True)  == b'\xca' + struct.pack('>f', 1.0)
+    assert packb(1.0, use_single_float=False) == b'\xcb' + struct.pack('>d', 1.0)
 
 def testArraySize(sizes=[0, 5, 50, 1000]):
     bio = six.BytesIO()
@@ -151,10 +149,10 @@ class odict(dict):
 def test_odict():
     seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
     od = odict(seq)
-    assert_equal(unpackb(packb(od), use_list=1), dict(seq))
+    assert unpackb(packb(od), use_list=1) == dict(seq)
     def pair_hook(seq):
         return seq
-    assert_equal(unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1), seq)
+    assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
 
 
 def test_pairlist():
@@ -163,8 +161,3 @@ def test_pairlist():
     packed = packer.pack_map_pairs(pairlist)
     unpacked = unpackb(packed, object_pairs_hook=list)
     assert pairlist == unpacked
-
-
-
-if __name__ == '__main__':
-    main()
diff --git a/test/test_seq.py b/test/test_seq.py
index 72e935a..af719b0 100644
--- a/test/test_seq.py
+++ b/test/test_seq.py
@@ -2,9 +2,6 @@
 # coding: utf-8
 
 import six
-from nose import main
-from nose.tools import *
-
 import io
 import msgpack
 
@@ -38,13 +35,8 @@ def test_exceeding_unpacker_read_size():
 
     read_count = 0
     for idx, o in enumerate(unpacker):
-        assert_equal(type(o), bytes)
-        assert_equal(o, gen_binary_data(idx))
+        assert type(o) == bytes
+        assert o == gen_binary_data(idx)
         read_count += 1
 
-    assert_equal(read_count, NUMBER_OF_STRINGS)
-
-
-if __name__ == '__main__':
-    main()
-    #test_exceeding_unpacker_read_size()
+    assert read_count == NUMBER_OF_STRINGS
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index f767726..fc1f712 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -4,7 +4,8 @@
 import six
 from msgpack import Unpacker, BufferFull
 from msgpack.exceptions import OutOfData
-import nose
+from pytest import raises
+
 
 def test_foobar():
     unpacker = Unpacker(read_size=3, use_list=1)
@@ -15,11 +16,8 @@ def test_foobar():
     assert unpacker.unpack() == ord(b'b')
     assert unpacker.unpack() == ord(b'a')
     assert unpacker.unpack() == ord(b'r')
-    try:
-        o = unpacker.unpack()
-        assert 0, "should raise exception"
-    except OutOfData:
-        assert 1, "ok"
+    with raises(OutOfData):
+        unpacker.unpack()
 
     unpacker.feed(b'foo')
     unpacker.feed(b'bar')
@@ -39,17 +37,16 @@ def test_foobar_skip():
     unpacker.skip()
     assert unpacker.unpack() == ord(b'a')
     unpacker.skip()
-    try:
-        o = unpacker.unpack()
-        assert 0, "should raise exception"
-    except OutOfData:
-        assert 1, "ok"
+    with raises(OutOfData):
+        unpacker.unpack()
 
 def test_maxbuffersize():
-    nose.tools.assert_raises(ValueError, Unpacker, read_size=5, max_buffer_size=3)
+    with raises(ValueError):
+        Unpacker(read_size=5, max_buffer_size=3)
     unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
     unpacker.feed(b'fo')
-    nose.tools.assert_raises(BufferFull, unpacker.feed, b'ob')
+    with raises(BufferFull):
+        unpacker.feed(b'ob')
     unpacker.feed(b'o')
     assert ord('f') == next(unpacker)
     unpacker.feed(b'b')
@@ -73,5 +70,3 @@ def test_readbytes():
     assert unpacker.unpack() == ord(b'a')
     assert unpacker.unpack() == ord(b'r')
 
-if __name__ == '__main__':
-    nose.main()
diff --git a/test/test_subtype.py b/test/test_subtype.py
index 1dfd7da..6807508 100644
--- a/test/test_subtype.py
+++ b/test/test_subtype.py
@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from nose import main
-from nose.tools import *
 from msgpack import packb, unpackb
 from collections import namedtuple
 
@@ -18,10 +16,6 @@ class MyTuple(tuple):
 MyNamedTuple = namedtuple('MyNamedTuple', 'x y')
 
 def test_types():
-    assert_equal(packb(dict()), packb(MyDict()))
-    assert_equal(packb(list()), packb(MyList()))
-    assert_equal(packb(MyNamedTuple(1,2)), packb((1,2)))
-
-
-if __name__ == '__main__':
-    main()
+    assert packb(MyDict()) == packb(dict())
+    assert packb(MyList()) == packb(list())
+    assert packb(MyNamedTuple(1, 2)) == packb((1, 2))
diff --git a/test/test_unpack_raw.py b/test/test_unpack_raw.py
index 15d9c93..9f3784c 100644
--- a/test/test_unpack_raw.py
+++ b/test/test_unpack_raw.py
@@ -1,7 +1,5 @@
 """Tests for cases where the user seeks to obtain packed msgpack objects"""
 
-from nose import main
-from nose.tools import *
 import six
 from msgpack import Unpacker, packb
 
@@ -10,14 +8,14 @@ def test_write_bytes():
     unpacker = Unpacker()
     unpacker.feed(b'abc')
     f = six.BytesIO()
-    assert_equal(unpacker.unpack(f.write), ord('a'))
-    assert_equal(f.getvalue(), b'a')
+    assert unpacker.unpack(f.write) == ord('a')
+    assert f.getvalue() == b'a'
     f = six.BytesIO()
     assert unpacker.skip(f.write) is None
-    assert_equal(f.getvalue(), b'b')
+    assert f.getvalue() == b'b'
     f = six.BytesIO()
     assert unpacker.skip() is None
-    assert_equal(f.getvalue(), b'')
+    assert f.getvalue() == b''
 
 
 def test_write_bytes_multi_buffer():
@@ -27,8 +25,5 @@ def test_write_bytes_multi_buffer():
 
     f = six.BytesIO()
     unpacked = unpacker.unpack(f.write)
-    assert_equal(unpacked, long_val)
-    assert_equal(f.getvalue(), expected)
-
-if __name__ == '__main__':
-    main()
+    assert unpacked == long_val
+    assert f.getvalue() == expected

From 97a9f3f05ccff6ba548172a7c07cdfe80ae2a12b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 29 Dec 2012 11:27:28 +0900
Subject: [PATCH 1125/1648] Switching to py.test

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 214c4c4..1951352 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,7 +3,7 @@ envlist = py26,py27,py32,py33
 
 [testenv]
 deps=
-    nose
+    pytest
     six
 
-commands=nosetests -w test
+commands=py.test test

From 5f55e4c6dbc3ec723bea5b9fead2e36224e70b81 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 29 Dec 2012 11:28:28 +0900
Subject: [PATCH 1126/1648] Switchng to py.test

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 245c09c..875a3f6 100644
--- a/Makefile
+++ b/Makefile
@@ -9,4 +9,4 @@ python3:
 	python3 setup.py build sdist
 
 test:
-	nosetests test
+	py.test test

From 6a28b28c6314158ee099c95d452406278cdb93b0 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Mon, 28 Jan 2013 12:27:24 +0100
Subject: [PATCH 1127/1648] Add pure Python fallback module

Signed-off-by: Bas Westerbaan 
---
 msgpack/__init__.py |   8 +-
 msgpack/fallback.py | 476 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 482 insertions(+), 2 deletions(-)
 create mode 100644 msgpack/fallback.py

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 0bfe6fe..49a32d9 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,8 +1,12 @@
 # coding: utf-8
 from msgpack._version import version
 from msgpack.exceptions import *
-from msgpack._packer import pack, packb, Packer
-from msgpack._unpacker import unpack, unpackb, Unpacker
+
+try:
+    from msgpack._packer import pack, packb, Packer
+    from msgpack._unpacker import unpack, unpackb, Unpacker
+except ImportError:
+    from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker
 
 # alias for compatibility to simplejson/marshal/pickle.
 load = unpack
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
new file mode 100644
index 0000000..2ca3a8f
--- /dev/null
+++ b/msgpack/fallback.py
@@ -0,0 +1,476 @@
+# Fallback pure Python implementation of msgpack
+
+import sys
+import array
+import struct
+
+try:
+    import cStringIO as StringIO
+except ImportError:
+    import StringIO
+
+from msgpack.exceptions import (
+        BufferFull,
+        OutOfData,
+        UnpackValueError,
+        PackValueError,
+        ExtraData)
+
+EX_SKIP                 = 0 
+EX_CONSTRUCT            = 1
+EX_READ_ARRAY_HEADER    = 2
+EX_READ_MAP_HEADER      = 3
+
+TYPE_IMMEDIATE          = 0
+TYPE_ARRAY              = 1
+TYPE_MAP                = 2
+TYPE_RAW                = 3
+
+DEFAULT_RECURSE_LIMIT=511
+
+def pack(o, stream, default=None, encoding='utf-8', unicode_errors='strict'):
+    """ Pack object `o` and write it to `stream` """
+    packer = Packer(default=default, encoding=encoding,
+                        unicode_errors=unicode_errors)
+    stream.write(packer.pack(o))
+
+def packb(o, default=None, encoding='utf-8', unicode_errors='struct',
+                use_single_float=False):
+    """ Pack object `o` and return packed bytes """
+    packer = Packer(default=default,
+                    encoding=encoding,
+                    unicode_errors=unicode_errors,
+                    use_single_float=use_single_float)
+    return packer.pack(o)
+
+def unpack(stream, object_hook=None, list_hook=None, use_list=True,
+                encoding=None, unicode_errors='strict',
+                object_pairs_hook=None):
+    """ Unpack an object from `stream`.
+
+    Raises `ExtraData` when `stream` has extra bytes. """
+    unpacker = Unpacker(stream, object_hook=object_hook, list_hook=list_hook,
+            use_list=use_list, encoding=encoding, unicode_errors=unicode_errors,
+            object_pairs_hook=object_pairs_hook)
+    ret = unpacker._fb_unpack()
+    unpacker._fb_check_for_extradata()
+    return ret
+
+def unpackb(packed, object_hook=None, list_hook=None, use_list=True,
+                encoding=None, unicode_errors='strict',
+                object_pairs_hook=None):
+    """ Unpack an object from `packed`.
+
+    Raises `ExtraData` when `packed` contains extra bytes. """
+    unpacker = Unpacker(None, object_hook=object_hook, list_hook=list_hook,
+            use_list=use_list, encoding=encoding, unicode_errors=unicode_errors,
+            object_pairs_hook=object_pairs_hook)
+    unpacker.feed(packed)
+    ret = unpacker._fb_unpack()
+    unpacker._fb_check_for_extradata()
+    return ret
+
+class Unpacker(object):
+    """
+    Streaming unpacker.
+
+    `file_like` is a file-like object having a `.read(n)` method.
+    When `Unpacker` is initialized with a `file_like`, `.feed()` is not
+    usable.
+
+    `read_size` is used for `file_like.read(read_size)`.
+
+    If `use_list` is True (default), msgpack lists are deserialized to Python
+    lists.  Otherwise they are deserialized to tuples.
+
+    `object_hook` is the same as in simplejson.  If it is not None, it should
+    be callable and Unpacker calls it with a dict argument after deserializing
+    a map.
+
+    `object_pairs_hook` is the same as in simplejson.  If it is not None, it
+    should be callable and Unpacker calls it with a list of key-value pairs
+    after deserializing a map.
+
+    `encoding` is the encoding used for decoding msgpack bytes.  If it is
+    None (default), msgpack bytes are deserialized to Python bytes.
+
+    `unicode_errors` is used for decoding bytes.
+
+    `max_buffer_size` limits the buffer size.  0 means INT_MAX (default).
+
+    Raises `BufferFull` exception when it is unsufficient.
+
+    You should set this parameter when unpacking data from an untrustred source.
+
+    example of streaming deserialization from file-like object::
+
+        unpacker = Unpacker(file_like)
+        for o in unpacker:
+            do_something(o)
+
+    example of streaming deserialization from socket::
+
+        unpacker = Unapcker()
+        while 1:
+            buf = sock.recv(1024*2)
+            if not buf:
+                break
+            unpacker.feed(buf)
+            for o in unpacker:
+                do_something(o)
+    """
+
+    def __init__(self, file_like=None, read_size=0, use_list=True,
+            object_hook=None, object_pairs_hook=None, list_hook=None,
+            encoding=None, unicode_errors='strict', max_buffer_size=0):
+        if file_like is None:
+            self._fb_feeding = True
+        else:
+            if not callable(file_like.read):
+                raise ValueError("`file_like.read` must be callable")
+            self.file_like = file_like
+            self._fb_feeding = False
+        self._fb_buffers = []
+        self._fb_buf_o = 0
+        self._fb_buf_i = 0
+        self._fb_buf_n = 0
+        self.max_buffer_size = (sys.maxint if max_buffer_size == 0
+                                        else max_buffer_size)
+        self.read_size = (read_size if read_size != 0
+                            else min(self.max_buffer_size, 2048))
+        if read_size > self.max_buffer_size:
+            raise ValueError("read_size must be smaller than max_buffer_size")
+        self.encoding = encoding
+        self.unicode_errors = unicode_errors
+        self.use_list = use_list
+        self.list_hook = list_hook
+        self.object_hook = object_hook
+        self.object_pairs_hook = object_pairs_hook
+
+        if list_hook is not None and not callable(list_hook):
+            raise ValueError('`list_hook` is not callable')
+        if object_hook is not None and not callable(object_hook):
+            raise ValueError('`object_hook` is not callable')
+        if object_pairs_hook is not None and not callable(object_pairs_hook):
+            raise ValueError('`object_pairs_hook` is not callable')
+        if object_hook is not None and object_pairs_hook is not None:
+            raise ValueError("object_pairs_hook and object_hook are mutually "+
+                                "exclusive")
+
+    def feed(self, next_bytes):
+        if isinstance(next_bytes, array.array):
+            next_bytes = next_bytes.tostring()
+        assert self._fb_feeding
+        if self._fb_buf_n + len(next_bytes) > self.max_buffer_size:
+            raise BufferFull
+        self._fb_buf_n += len(next_bytes)
+        self._fb_buffers.append(next_bytes)
+
+    def _fb_consume(self):
+        self._fb_buffers = self._fb_buffers[self._fb_buf_i:]
+        if self._fb_buffers:
+            self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:]
+        self._fb_buf_o = 0
+        self._fb_buf_i = 0
+        self._fb_buf_n = sum(map(len, self._fb_buffers))
+
+    def _fb_check_for_extradata(self):
+        if self._fb_buf_i != len(self._fb_buffers):
+            raise ExtraData
+        if self._fb_feeding:
+            return
+        if not self.file_like:
+            return
+        if not self.file_like.read(1):
+            raise ExtraData
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        try:
+            ret = self._fb_unpack(None, None)
+            self._fb_consume()
+            return ret
+        except OutOfData:
+            raise StopIteration
+
+    def read_bytes(self, n):
+        return self._fb_read(n)
+
+    def _fb_read(self, n, write_bytes=None):
+        ret = ''
+        while len(ret) != n:
+            if self._fb_buf_i == len(self._fb_buffers):
+                if self._fb_feeding:
+                    break
+                tmp = self.file_like.read(self.read_size)
+                if not tmp:
+                    break
+                self._fb_buffers.append(tmp)
+                continue
+            sliced = n - len(ret)
+            ret += self._fb_buffers[self._fb_buf_i][
+                        self._fb_buf_o:self._fb_buf_o + sliced]
+            self._fb_buf_o += sliced
+            if self._fb_buf_o >= len(self._fb_buffers[self._fb_buf_i]):
+                self._fb_buf_o = 0
+                self._fb_buf_i += 1
+        if len(ret) != n:
+            raise OutOfData
+        if write_bytes is not None:
+            write_bytes(ret)
+        return ret
+
+    def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
+        typ = TYPE_IMMEDIATE
+        b = ord(self._fb_read(1, write_bytes))
+        if   b & 0b10000000 == 0:
+            obj = b
+        elif b & 0b11100000 == 0b11100000:
+            obj = struct.unpack("b", chr(b))[0]
+        elif b & 0b11100000 == 0b10100000:
+            n = b & 0b00011111
+            obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0]
+            typ = TYPE_RAW
+        elif b & 0b11110000 == 0b10010000:
+            n = b & 0b00001111
+            typ = TYPE_ARRAY
+        elif b & 0b11110000 == 0b10000000:
+            n = b & 0b00001111
+            typ = TYPE_MAP
+        elif b == 0xc0:
+            obj = None
+        elif b == 0xc2:
+            obj = False
+        elif b == 0xc3:
+            obj = True
+        elif b == 0xca:
+            obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0]
+        elif b == 0xcb:
+            obj = struct.unpack(">d", self._fb_read(8, write_bytes))[0]
+        elif b == 0xcc:
+            obj = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+        elif b == 0xcd:
+            obj = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+        elif b == 0xce:
+            obj = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+        elif b == 0xcf:
+            obj = struct.unpack(">Q", self._fb_read(8, write_bytes))[0]
+        elif b == 0xd0:
+            obj = struct.unpack("b", self._fb_read(1, write_bytes))[0]
+        elif b == 0xd1:
+            obj = struct.unpack(">h", self._fb_read(2, write_bytes))[0]
+        elif b == 0xd2:
+            obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0]
+        elif b == 0xd3:
+            obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0]
+        elif b == 0xda:
+            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0]
+            typ = TYPE_RAW
+        elif b == 0xdb:
+            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0]
+            typ = TYPE_RAW
+        elif b == 0xdc:
+            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            typ = TYPE_ARRAY
+        elif b == 0xdd:
+            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            typ = TYPE_ARRAY
+        elif b == 0xde:
+            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            typ = TYPE_MAP
+        elif b == 0xdf:
+            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            typ = TYPE_MAP
+        else:
+            raise UnpackValueError("Unknown header: 0x%x" % b)
+        if execute == EX_READ_ARRAY_HEADER:
+            if typ != TYPE_ARRAY:
+                raise UnpackValueError("Expected array")
+            return n
+        if execute == EX_READ_MAP_HEADER:
+            if typ != TYPE_MAP:
+                raise UnpackValueError("Expected map")
+            return n
+        # TODO should we eliminate the recursion?
+        if typ == TYPE_ARRAY:
+            if execute == EX_SKIP:
+                for i in xrange(n):
+                    # TODO check whether we need to call `list_hook`
+                    self._fb_unpack(EX_SKIP, write_bytes)
+                return
+            ret = []
+            for i in xrange(n):
+                ret.append(self._fb_unpack(EX_CONSTRUCT, write_bytes))
+            if self.list_hook is not None:
+                ret = self.list_hook(ret)
+            # TODO is the interaction between `list_hook` and `use_list` ok?
+            return ret if self.use_list else tuple(ret)
+        if typ == TYPE_MAP:
+            if execute == EX_SKIP:
+                for i in xrange(n):
+                    # TODO check whether we need to call hooks
+                    self._fb_unpack(EX_SKIP, write_bytes)
+                    self._fb_unpack(EX_SKIP, write_bytes)
+                return
+            ret = []
+            for i in xrange(n):
+                ret.append((self._fb_unpack(EX_CONSTRUCT, write_bytes),
+                            self._fb_unpack(EX_CONSTRUCT, write_bytes)))
+            if self.object_pairs_hook is not None:
+                ret = self.object_pairs_hook(ret)
+            else:
+                ret = dict(ret)
+            if self.object_hook is not None:
+                ret = self.object_hook(ret)
+            return ret
+        if execute == EX_SKIP:
+            return
+        if typ == TYPE_RAW:
+            if self.encoding is not None:
+                obj = obj.decode(self.encoding, self.unicode_errors)
+            return obj
+        assert typ == TYPE_IMMEDIATE
+        return obj
+
+    def skip(self, write_bytes=None):
+        self._fb_unpack(EX_SKIP, write_bytes)
+        self._fb_consume()
+
+    def unpack(self, write_bytes=None):
+        ret = self._fb_unpack(EX_CONSTRUCT, write_bytes)
+        self._fb_consume()
+        return ret
+
+    def read_array_header(self, write_bytes=None):
+        ret = self._fb_unpack(EX_READ_ARRAY_HEADER, write_bytes)
+        self._fb_consume()
+        return ret
+
+    def read_map_header(self, write_bytes=None):
+        ret = self._fb_unpack(EX_READ_MAP_HEADER, write_bytes)
+        self._fb_consume()
+        return ret
+
+class Packer(object):
+    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
+                    use_single_float=False, autoreset=True):
+        self.use_float = use_single_float
+        self.autoreset = autoreset
+        self.encoding = encoding
+        self.unicode_errors = unicode_errors
+        self.buffer = StringIO.StringIO()
+        if default is not None:
+            if not callable(default):
+                raise TypeError("default must be callable")
+        self._default = default
+    def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT):
+        if nest_limit < 0:
+            raise PackValueError("recursion limit exceeded")
+        if obj is None:
+            return self.buffer.write(chr(0xc0))
+        if isinstance(obj, bool):
+            if obj:
+                return self.buffer.write(chr(0xc3))
+            return self.buffer.write(chr(0xc2))
+        if isinstance(obj, int) or isinstance(obj, long):
+            if 0 <= obj < 0x80:
+                return self.buffer.write(struct.pack("B", obj))
+            if -0x20 <= obj < 0:
+                return self.buffer.write(struct.pack("b", obj))
+            if 0x80 <= obj <= 0xff:
+                return self.buffer.write(struct.pack("BB", 0xcc, obj))
+            if -0x80 <= obj < 0:
+                return self.buffer.write(struct.pack(">Bb", 0xd0, obj))
+            if 0xff < obj <= 0xffff:
+                return self.buffer.write(struct.pack(">BH", 0xcd, obj))
+            if -0x8000 <= obj < -0x80:
+                return self.buffer.write(struct.pack(">Bh", 0xd1, obj))
+            if 0xffff < obj <= 0xffffffff:
+                return self.buffer.write(struct.pack(">BI", 0xce, obj))
+            if -0x80000000 <= obj < -0x8000:
+                return self.buffer.write(struct.pack(">Bi", 0xd2, obj))
+            if 0xffffffff < obj <= 0xffffffffffffffff:
+                return self.buffer.write(struct.pack(">BQ", 0xcf, obj))
+            if -0x8000000000000000 <= obj < -0x80000000:
+                return self.buffer.write(struct.pack(">Bq", 0xd3, obj))
+            raise PackValueError("Integer value out of range")
+        if isinstance(obj, str) or isinstance(obj, unicode):
+            if isinstance(obj, unicode):
+                obj = obj.encode(self.encoding, self.unicode_errors)
+            n = len(obj)
+            if n <= 0x1f:
+                return self.buffer.write(chr(0xa0 + n) + obj)
+            if n <= 0xffff:
+                return self.buffer.write(struct.pack(">BH%ds" % n,0xda, n, obj))
+            if n <= 0xffffffff:
+                return self.buffer.write(struct.pack(">BI%ds" % n,0xdb, n, obj))
+            raise PackValueError("String is too large")
+        if isinstance(obj, float):
+            if self.use_float:
+                return self.buffer.write(struct.pack(">Bf", 0xca, obj))
+            return self.buffer.write(struct.pack(">Bd", 0xcb, obj))
+        if isinstance(obj, list) or isinstance(obj, tuple):
+            n = len(obj)
+            self._fb_pack_array_header(n)
+            for i in xrange(n):
+                self._pack(obj[i], nest_limit - 1)
+            return
+        if isinstance(obj, dict):
+            return self._fb_pack_map_pairs(len(obj), obj.iteritems(),
+                                            nest_limit - 1)
+        if self._default is not None:
+            return self._pack(self._default(obj), nest_limit - 1)
+        raise TypeError("Cannot serialize %r" % obj)
+    def pack(self, obj):
+        self._pack(obj)
+        ret = self.buffer.getvalue()
+        if self.autoreset:
+            self.buffer = StringIO.StringIO()
+        return ret
+    def pack_map_pairs(self, pairs):
+        self._fb_pack_map_pairs(len(pairs), pairs)
+        ret = self.buffer.getvalue()
+        if self.autoreset:
+            self.buffer = StringIO.StringIO()
+        return ret
+    def pack_array_header(self, n):
+        self._fb_pack_array_header(n)
+        ret = self.buffer.getvalue()
+        if self.autoreset:
+            self.buffer = StringIO.StringIO()
+        return ret
+    def pack_map_header(self, n):
+        self._fb_pack_map_header(n)
+        ret = self.buffer.getvalue()
+        if self.autoreset:
+            self.buffer = StringIO.StringIO()
+        return ret
+    def _fb_pack_array_header(self, n):
+        if n <= 0x0f:
+            return self.buffer.write(chr(0x90 + n))
+        if n <= 0xffff:
+            return self.buffer.write(struct.pack(">BH", 0xdc, n))
+        if n <= 0xffffffff:
+            return self.buffer.write(struct.pack(">BI", 0xdd, n))
+        raise PackValueError("Array is too large")
+    def _fb_pack_map_header(self, n):
+        if n <= 0x0f:
+            return self.buffer.write(chr(0x80 + n))
+        if n <= 0xffff:
+            return self.buffer.write(struct.pack(">BH", 0xde, n))
+        if n <= 0xffffffff:
+            return self.buffer.write(struct.pack(">BI", 0xdf, n))
+        raise PackValueError("Dict is too large")
+    def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
+        self._fb_pack_map_header(n)
+        for (k, v) in pairs:
+            self._pack(k, nest_limit - 1)
+            self._pack(v, nest_limit - 1)
+    def bytes(self):
+        return self.buffer.getvalue()
+    def reset(self):
+        self.buffer = StringIO.StringIO()

From 2627b6ae9f0606b388a3a3ec0110d1fdb33d082e Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Mon, 28 Jan 2013 12:27:46 +0100
Subject: [PATCH 1128/1648] setup: automatically fallback to pure Python module

Signed-off-by: Bas Westerbaan 
---
 setup.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/setup.py b/setup.py
index ac3eeb5..3d7fc04 100644
--- a/setup.py
+++ b/setup.py
@@ -8,6 +8,9 @@ from setuptools import setup, Extension
 
 from distutils.command.build_ext import build_ext
 
+class NoCython(Exception):
+    pass
+
 try:
     import Cython.Compiler.Main as cython_compiler
     have_cython = True
@@ -24,10 +27,7 @@ def ensure_source(src):
 
     if not os.path.exists(src):
         if not have_cython:
-            raise Exception("""\
-Cython is required for building extension from checkout.
-Install Cython >= 0.16 or install msgpack from PyPI.
-""")
+            raise NoCython
         cythonize(pyx)
     elif (os.path.exists(pyx) and
           os.stat(src).st_mtime < os.stat(pyx).st_mtime and
@@ -38,7 +38,14 @@ Install Cython >= 0.16 or install msgpack from PyPI.
 
 class BuildExt(build_ext):
     def build_extension(self, ext):
-        ext.sources = list(map(ensure_source, ext.sources))
+        try:
+            ext.sources = list(map(ensure_source, ext.sources))
+        except NoCython:
+            print "WARNING"
+            print "Cython is required for building extension from checkout."
+            print "Install Cython >= 0.16 or install msgpack from PyPI."
+            print "Falling back to pure Python implementation."
+            return
         return build_ext.build_extension(self, ext)
 
 

From 69ba3c9bf9737df1ade7832986b0fa6f1659b04d Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Mon, 28 Jan 2013 13:30:32 +0100
Subject: [PATCH 1129/1648] fallback: use __pypy__.builders.StringBuilder when
 available

This increases performance *a lot* on PyPy.

Signed-off-by: Bas Westerbaan 
---
 msgpack/fallback.py | 46 +++++++++++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 10 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 2ca3a8f..220e0fe 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -4,10 +4,28 @@ import sys
 import array
 import struct
 
-try:
-    import cStringIO as StringIO
-except ImportError:
-    import StringIO
+if hasattr(sys, 'pypy_version_info'):
+    # cStringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
+    # StringBuilder is fastest.
+    from __pypy__.builders import StringBuilder
+    USING_STRINGBUILDER = True
+    class StringIO(object):
+        def __init__(self, s=''):
+            if s:
+                self.builder = StringBuilder(len(s))
+                self.builder.append(s)
+            else:
+                self.builder = StringBuilder()
+        def write(self, s):
+            self.builder.append(s)
+        def getvalue(self):
+            return self.builder.build()
+else:
+    USING_STRINGBUILDER = False
+    try:
+        from cStringIO import StringIO
+    except ImportError:
+        from StringIO import StringIO
 
 from msgpack.exceptions import (
         BufferFull,
@@ -362,7 +380,7 @@ class Packer(object):
         self.autoreset = autoreset
         self.encoding = encoding
         self.unicode_errors = unicode_errors
-        self.buffer = StringIO.StringIO()
+        self.buffer = StringIO()
         if default is not None:
             if not callable(default):
                 raise TypeError("default must be callable")
@@ -429,25 +447,33 @@ class Packer(object):
         self._pack(obj)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = StringIO.StringIO()
+            self.buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self.buffer = StringIO(ret)
         return ret
     def pack_map_pairs(self, pairs):
         self._fb_pack_map_pairs(len(pairs), pairs)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = StringIO.StringIO()
+            self.buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self.buffer = StringIO(ret)
         return ret
     def pack_array_header(self, n):
         self._fb_pack_array_header(n)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = StringIO.StringIO()
+            self.buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self.buffer = StringIO(ret)
         return ret
     def pack_map_header(self, n):
         self._fb_pack_map_header(n)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = StringIO.StringIO()
+            self.buffer = StringIO()
+        elif USING_STRINGBUILDER:
+            self.buffer = StringIO(ret)
         return ret
     def _fb_pack_array_header(self, n):
         if n <= 0x0f:
@@ -473,4 +499,4 @@ class Packer(object):
     def bytes(self):
         return self.buffer.getvalue()
     def reset(self):
-        self.buffer = StringIO.StringIO()
+        self.buffer = StringIO()

From 6fa0f46a122c4f6be35415a3e65dcdc542fd3acd Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Mon, 28 Jan 2013 14:32:01 +0100
Subject: [PATCH 1130/1648] setup: remove Python 2 only syntax

---
 setup.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/setup.py b/setup.py
index 3d7fc04..d4808d6 100644
--- a/setup.py
+++ b/setup.py
@@ -41,10 +41,10 @@ class BuildExt(build_ext):
         try:
             ext.sources = list(map(ensure_source, ext.sources))
         except NoCython:
-            print "WARNING"
-            print "Cython is required for building extension from checkout."
-            print "Install Cython >= 0.16 or install msgpack from PyPI."
-            print "Falling back to pure Python implementation."
+            print("WARNING")
+            print("Cython is required for building extension from checkout.")
+            print("Install Cython >= 0.16 or install msgpack from PyPI.")
+            print("Falling back to pure Python implementation.")
             return
         return build_ext.build_extension(self, ext)
 

From b940802032adc16b074b1e29bed72825580c5c9f Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Mon, 28 Jan 2013 22:29:23 +0100
Subject: [PATCH 1131/1648] fallback: two fixes for raising ExtraData

---
 msgpack/fallback.py | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 220e0fe..392f7eb 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -71,7 +71,8 @@ def unpack(stream, object_hook=None, list_hook=None, use_list=True,
             use_list=use_list, encoding=encoding, unicode_errors=unicode_errors,
             object_pairs_hook=object_pairs_hook)
     ret = unpacker._fb_unpack()
-    unpacker._fb_check_for_extradata()
+    if unpacker._fb_got_extradata():
+        raise ExtraData(ret, unpacker._fb_get_extradata())
     return ret
 
 def unpackb(packed, object_hook=None, list_hook=None, use_list=True,
@@ -85,7 +86,8 @@ def unpackb(packed, object_hook=None, list_hook=None, use_list=True,
             object_pairs_hook=object_pairs_hook)
     unpacker.feed(packed)
     ret = unpacker._fb_unpack()
-    unpacker._fb_check_for_extradata()
+    if unpacker._fb_got_extradata():
+        raise ExtraData(ret, unpacker._fb_get_extradata())
     return ret
 
 class Unpacker(object):
@@ -192,15 +194,16 @@ class Unpacker(object):
         self._fb_buf_i = 0
         self._fb_buf_n = sum(map(len, self._fb_buffers))
 
-    def _fb_check_for_extradata(self):
+    def _fb_got_extradata(self):
         if self._fb_buf_i != len(self._fb_buffers):
-            raise ExtraData
+            return True
         if self._fb_feeding:
-            return
+            return False
         if not self.file_like:
-            return
-        if not self.file_like.read(1):
-            raise ExtraData
+            return False
+        if self.file_like.read(1):
+            return True
+        return False
 
     def __iter__(self):
         return self
@@ -216,6 +219,12 @@ class Unpacker(object):
     def read_bytes(self, n):
         return self._fb_read(n)
 
+    def _fb_get_extradata(self):
+        bufs = self._fb_buffers[self._fb_buf_i:]
+        if bufs:
+            bufs[0] = bufs[0][self._fb_buf_o:]
+        return ''.join(bufs)
+
     def _fb_read(self, n, write_bytes=None):
         ret = ''
         while len(ret) != n:

From af9c9ca2c9cae01cf603da90530b9ba396007e5b Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Tue, 29 Jan 2013 02:01:34 +0100
Subject: [PATCH 1132/1648] fallback: performance: write(a+b) -> write(a);
 write(b)

---
 msgpack/fallback.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 392f7eb..d6fd533 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -430,7 +430,8 @@ class Packer(object):
                 obj = obj.encode(self.encoding, self.unicode_errors)
             n = len(obj)
             if n <= 0x1f:
-                return self.buffer.write(chr(0xa0 + n) + obj)
+                self.buffer.write(chr(0xa0 + n))
+                return self.buffer.write(obj)
             if n <= 0xffff:
                 return self.buffer.write(struct.pack(">BH%ds" % n,0xda, n, obj))
             if n <= 0xffffffff:

From 94925acb124998aaf9438e7c8152fbfbb7d5d8a8 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Tue, 29 Jan 2013 02:15:29 +0100
Subject: [PATCH 1133/1648] fallback: do not use dynamic format strings for
 struct.(un)pack

Increases performance on PyPy.
---
 msgpack/fallback.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index d6fd533..fe2b1ac 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -258,7 +258,7 @@ class Unpacker(object):
             obj = struct.unpack("b", chr(b))[0]
         elif b & 0b11100000 == 0b10100000:
             n = b & 0b00011111
-            obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
             typ = TYPE_RAW
         elif b & 0b11110000 == 0b10010000:
             n = b & 0b00001111
@@ -294,11 +294,11 @@ class Unpacker(object):
             obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0]
         elif b == 0xda:
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
-            obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
             typ = TYPE_RAW
         elif b == 0xdb:
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
-            obj = struct.unpack("%ds" % n, self._fb_read(n, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
             typ = TYPE_RAW
         elif b == 0xdc:
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
@@ -433,9 +433,11 @@ class Packer(object):
                 self.buffer.write(chr(0xa0 + n))
                 return self.buffer.write(obj)
             if n <= 0xffff:
-                return self.buffer.write(struct.pack(">BH%ds" % n,0xda, n, obj))
+                self.buffer.write(struct.pack(">BH", 0xda, n))
+                return self.buffer.write(obj)
             if n <= 0xffffffff:
-                return self.buffer.write(struct.pack(">BI%ds" % n,0xdb, n, obj))
+                self.buffer.write(struct.pack(">BI", 0xdb, n))
+                return self.buffer.write(obj)
             raise PackValueError("String is too large")
         if isinstance(obj, float):
             if self.use_float:

From fb81f80d14613bd2ac3e63029a47bb0512c25dd5 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Tue, 29 Jan 2013 02:47:41 +0100
Subject: [PATCH 1134/1648] fallback: bugfix in next()

---
 msgpack/fallback.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index fe2b1ac..a866ff1 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -210,7 +210,7 @@ class Unpacker(object):
 
     def next(self):
         try:
-            ret = self._fb_unpack(None, None)
+            ret = self._fb_unpack(EX_CONSTRUCT, None)
             self._fb_consume()
             return ret
         except OutOfData:

From d2f549a47094b2a29cc94bc50029ebdc85508861 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Tue, 29 Jan 2013 02:58:26 +0100
Subject: [PATCH 1135/1648] fallback: add actual rollback and add a testcase
 for it

Signed-off-by: Bas Westerbaan 
---
 msgpack/fallback.py    | 21 +++++++++++++--------
 test/test_sequnpack.py | 15 +++++++++++++++
 2 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index a866ff1..2f83a20 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -208,17 +208,13 @@ class Unpacker(object):
     def __iter__(self):
         return self
 
-    def next(self):
-        try:
-            ret = self._fb_unpack(EX_CONSTRUCT, None)
-            self._fb_consume()
-            return ret
-        except OutOfData:
-            raise StopIteration
-
     def read_bytes(self, n):
         return self._fb_read(n)
 
+    def _fb_rollback(self):
+        self._fb_buf_i = 0
+        self._fb_buf_o = 0
+
     def _fb_get_extradata(self):
         bufs = self._fb_buffers[self._fb_buf_i:]
         if bufs:
@@ -244,6 +240,7 @@ class Unpacker(object):
                 self._fb_buf_o = 0
                 self._fb_buf_i += 1
         if len(ret) != n:
+            self._fb_rollback()
             raise OutOfData
         if write_bytes is not None:
             write_bytes(ret)
@@ -363,6 +360,14 @@ class Unpacker(object):
         assert typ == TYPE_IMMEDIATE
         return obj
 
+    def next(self):
+        try:
+            ret = self._fb_unpack(EX_CONSTRUCT, None)
+            self._fb_consume()
+            return ret
+        except OutOfData:
+            raise StopIteration
+
     def skip(self, write_bytes=None):
         self._fb_unpack(EX_SKIP, write_bytes)
         self._fb_consume()
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index fc1f712..1da383c 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -7,6 +7,21 @@ from msgpack.exceptions import OutOfData
 from pytest import raises
 
 
+def test_partialdata():
+    unpacker = Unpacker()
+    unpacker.feed(b'\xa5')
+    with raises(StopIteration): next(iter(unpacker))
+    unpacker.feed(b'h')
+    with raises(StopIteration): next(iter(unpacker))
+    unpacker.feed(b'a')
+    with raises(StopIteration): next(iter(unpacker))
+    unpacker.feed(b'l')
+    with raises(StopIteration): next(iter(unpacker))
+    unpacker.feed(b'l')
+    with raises(StopIteration): next(iter(unpacker))
+    unpacker.feed(b'o')
+    assert next(iter(unpacker)) == 'hallo'
+
 def test_foobar():
     unpacker = Unpacker(read_size=3, use_list=1)
     unpacker.feed(b'foobar')

From b9e9199eea3a1654dca3e6faf6530d438788ad77 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Tue, 29 Jan 2013 03:03:13 +0100
Subject: [PATCH 1136/1648] fallback: python3 bugfix for new testcase of
 d2f549a4

Signed-off-by: Bas Westerbaan 
---
 test/test_sequnpack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 1da383c..9db14ca 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -20,7 +20,7 @@ def test_partialdata():
     unpacker.feed(b'l')
     with raises(StopIteration): next(iter(unpacker))
     unpacker.feed(b'o')
-    assert next(iter(unpacker)) == 'hallo'
+    assert next(iter(unpacker)) == b'hallo'
 
 def test_foobar():
     unpacker = Unpacker(read_size=3, use_list=1)

From 770fed6b7f1ce1f6e9c20881897a417680c8b79a Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Tue, 29 Jan 2013 03:39:46 +0100
Subject: [PATCH 1137/1648] fallback: Use mmap objects instead of strings to
 unpack

Signed-off-by: Bas Westerbaan 
---
 msgpack/fallback.py | 80 +++++++++++++++++++++++++++++----------------
 1 file changed, 51 insertions(+), 29 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 2f83a20..c0fcc18 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -1,15 +1,27 @@
 # Fallback pure Python implementation of msgpack
 
+#
+# Easy imports
+#
 import sys
 import array
 import struct
 
+#
+# Tricky imports
+#
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+# We will use wStringIO for buffering the writes for packing.
+# Normally, we will use cStringIO.StringIO.
+# On PyPy we will use PyPy's own StringBuilder.
 if hasattr(sys, 'pypy_version_info'):
-    # cStringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
-    # StringBuilder is fastest.
     from __pypy__.builders import StringBuilder
     USING_STRINGBUILDER = True
-    class StringIO(object):
+    class wStringIO(object):
         def __init__(self, s=''):
             if s:
                 self.builder = StringBuilder(len(s))
@@ -22,10 +34,18 @@ if hasattr(sys, 'pypy_version_info'):
             return self.builder.build()
 else:
     USING_STRINGBUILDER = False
-    try:
-        from cStringIO import StringIO
-    except ImportError:
-        from StringIO import StringIO
+    wStringIO = StringIO
+
+# We will use rStringIO for unpacking.
+# Normally, this is a mmap.  A normal StringIO is not a drop-in replacement ---
+# it misses the __len__ operation.
+# TODO add fallback for when mmap is unavailable
+import mmap
+def rStringIO(s):
+    m = mmap.mmap(-1, len(s))
+    m.write(s)
+    m.seek(0)
+    return m
 
 from msgpack.exceptions import (
         BufferFull,
@@ -184,13 +204,13 @@ class Unpacker(object):
         if self._fb_buf_n + len(next_bytes) > self.max_buffer_size:
             raise BufferFull
         self._fb_buf_n += len(next_bytes)
-        self._fb_buffers.append(next_bytes)
+        self._fb_buffers.append(rStringIO(next_bytes))
 
     def _fb_consume(self):
         self._fb_buffers = self._fb_buffers[self._fb_buf_i:]
         if self._fb_buffers:
-            self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:]
-        self._fb_buf_o = 0
+            self._fb_buffers[0] = rStringIO(self._fb_buffers[0][
+                                            self._fb_buffers[0].tell():])
         self._fb_buf_i = 0
         self._fb_buf_n = sum(map(len, self._fb_buffers))
 
@@ -212,16 +232,20 @@ class Unpacker(object):
         return self._fb_read(n)
 
     def _fb_rollback(self):
+        for buf in self._fb_buffers:
+            buf.seek(0)
         self._fb_buf_i = 0
-        self._fb_buf_o = 0
 
     def _fb_get_extradata(self):
         bufs = self._fb_buffers[self._fb_buf_i:]
         if bufs:
-            bufs[0] = bufs[0][self._fb_buf_o:]
-        return ''.join(bufs)
+            bufs[0] = rStringIO(bufs[0][bufs[0].tell():])
+        return ''.join([buf[:] for buf in bufs])
 
     def _fb_read(self, n, write_bytes=None):
+        if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers)
+                and self._fb_buffers[0].tell() + n < len(self._fb_buffers[0])):
+            return self._fb_buffers[0].read(n)
         ret = ''
         while len(ret) != n:
             if self._fb_buf_i == len(self._fb_buffers):
@@ -230,14 +254,12 @@ class Unpacker(object):
                 tmp = self.file_like.read(self.read_size)
                 if not tmp:
                     break
-                self._fb_buffers.append(tmp)
+                self._fb_buffers.append(rStringIO(tmp))
                 continue
             sliced = n - len(ret)
-            ret += self._fb_buffers[self._fb_buf_i][
-                        self._fb_buf_o:self._fb_buf_o + sliced]
-            self._fb_buf_o += sliced
-            if self._fb_buf_o >= len(self._fb_buffers[self._fb_buf_i]):
-                self._fb_buf_o = 0
+            ret += self._fb_buffers[self._fb_buf_i].read(sliced)
+            if (self._fb_buffers[self._fb_buf_i].tell()
+                    ==  len(self._fb_buffers[self._fb_buf_i])):
                 self._fb_buf_i += 1
         if len(ret) != n:
             self._fb_rollback()
@@ -394,7 +416,7 @@ class Packer(object):
         self.autoreset = autoreset
         self.encoding = encoding
         self.unicode_errors = unicode_errors
-        self.buffer = StringIO()
+        self.buffer = wStringIO()
         if default is not None:
             if not callable(default):
                 raise TypeError("default must be callable")
@@ -464,33 +486,33 @@ class Packer(object):
         self._pack(obj)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = StringIO()
+            self.buffer = wStringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = StringIO(ret)
+            self.buffer = wStringIO(ret)
         return ret
     def pack_map_pairs(self, pairs):
         self._fb_pack_map_pairs(len(pairs), pairs)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = StringIO()
+            self.buffer = wStringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = StringIO(ret)
+            self.buffer = wStringIO(ret)
         return ret
     def pack_array_header(self, n):
         self._fb_pack_array_header(n)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = StringIO()
+            self.buffer = wStringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = StringIO(ret)
+            self.buffer = wStringIO(ret)
         return ret
     def pack_map_header(self, n):
         self._fb_pack_map_header(n)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = StringIO()
+            self.buffer = wStringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = StringIO(ret)
+            self.buffer = wStringIO(ret)
         return ret
     def _fb_pack_array_header(self, n):
         if n <= 0x0f:
@@ -516,4 +538,4 @@ class Packer(object):
     def bytes(self):
         return self.buffer.getvalue()
     def reset(self):
-        self.buffer = StringIO()
+        self.buffer = wStringIO()

From d91a0d3d68bef38be667ba8e931e3f53cf90be6f Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Tue, 29 Jan 2013 03:45:17 +0100
Subject: [PATCH 1138/1648] Revert "fallback: Use mmap objects instead of
 strings to unpack"

See next commit.

This reverts commit 770fed6b7f1ce1f6e9c20881897a417680c8b79a.
---
 msgpack/fallback.py | 80 ++++++++++++++++-----------------------------
 1 file changed, 29 insertions(+), 51 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index c0fcc18..2f83a20 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -1,27 +1,15 @@
 # Fallback pure Python implementation of msgpack
 
-#
-# Easy imports
-#
 import sys
 import array
 import struct
 
-#
-# Tricky imports
-#
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
-
-# We will use wStringIO for buffering the writes for packing.
-# Normally, we will use cStringIO.StringIO.
-# On PyPy we will use PyPy's own StringBuilder.
 if hasattr(sys, 'pypy_version_info'):
+    # cStringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
+    # StringBuilder is fastest.
     from __pypy__.builders import StringBuilder
     USING_STRINGBUILDER = True
-    class wStringIO(object):
+    class StringIO(object):
         def __init__(self, s=''):
             if s:
                 self.builder = StringBuilder(len(s))
@@ -34,18 +22,10 @@ if hasattr(sys, 'pypy_version_info'):
             return self.builder.build()
 else:
     USING_STRINGBUILDER = False
-    wStringIO = StringIO
-
-# We will use rStringIO for unpacking.
-# Normally, this is a mmap.  A normal StringIO is not a drop-in replacement ---
-# it misses the __len__ operation.
-# TODO add fallback for when mmap is unavailable
-import mmap
-def rStringIO(s):
-    m = mmap.mmap(-1, len(s))
-    m.write(s)
-    m.seek(0)
-    return m
+    try:
+        from cStringIO import StringIO
+    except ImportError:
+        from StringIO import StringIO
 
 from msgpack.exceptions import (
         BufferFull,
@@ -204,13 +184,13 @@ class Unpacker(object):
         if self._fb_buf_n + len(next_bytes) > self.max_buffer_size:
             raise BufferFull
         self._fb_buf_n += len(next_bytes)
-        self._fb_buffers.append(rStringIO(next_bytes))
+        self._fb_buffers.append(next_bytes)
 
     def _fb_consume(self):
         self._fb_buffers = self._fb_buffers[self._fb_buf_i:]
         if self._fb_buffers:
-            self._fb_buffers[0] = rStringIO(self._fb_buffers[0][
-                                            self._fb_buffers[0].tell():])
+            self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:]
+        self._fb_buf_o = 0
         self._fb_buf_i = 0
         self._fb_buf_n = sum(map(len, self._fb_buffers))
 
@@ -232,20 +212,16 @@ class Unpacker(object):
         return self._fb_read(n)
 
     def _fb_rollback(self):
-        for buf in self._fb_buffers:
-            buf.seek(0)
         self._fb_buf_i = 0
+        self._fb_buf_o = 0
 
     def _fb_get_extradata(self):
         bufs = self._fb_buffers[self._fb_buf_i:]
         if bufs:
-            bufs[0] = rStringIO(bufs[0][bufs[0].tell():])
-        return ''.join([buf[:] for buf in bufs])
+            bufs[0] = bufs[0][self._fb_buf_o:]
+        return ''.join(bufs)
 
     def _fb_read(self, n, write_bytes=None):
-        if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers)
-                and self._fb_buffers[0].tell() + n < len(self._fb_buffers[0])):
-            return self._fb_buffers[0].read(n)
         ret = ''
         while len(ret) != n:
             if self._fb_buf_i == len(self._fb_buffers):
@@ -254,12 +230,14 @@ class Unpacker(object):
                 tmp = self.file_like.read(self.read_size)
                 if not tmp:
                     break
-                self._fb_buffers.append(rStringIO(tmp))
+                self._fb_buffers.append(tmp)
                 continue
             sliced = n - len(ret)
-            ret += self._fb_buffers[self._fb_buf_i].read(sliced)
-            if (self._fb_buffers[self._fb_buf_i].tell()
-                    ==  len(self._fb_buffers[self._fb_buf_i])):
+            ret += self._fb_buffers[self._fb_buf_i][
+                        self._fb_buf_o:self._fb_buf_o + sliced]
+            self._fb_buf_o += sliced
+            if self._fb_buf_o >= len(self._fb_buffers[self._fb_buf_i]):
+                self._fb_buf_o = 0
                 self._fb_buf_i += 1
         if len(ret) != n:
             self._fb_rollback()
@@ -416,7 +394,7 @@ class Packer(object):
         self.autoreset = autoreset
         self.encoding = encoding
         self.unicode_errors = unicode_errors
-        self.buffer = wStringIO()
+        self.buffer = StringIO()
         if default is not None:
             if not callable(default):
                 raise TypeError("default must be callable")
@@ -486,33 +464,33 @@ class Packer(object):
         self._pack(obj)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = wStringIO()
+            self.buffer = StringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = wStringIO(ret)
+            self.buffer = StringIO(ret)
         return ret
     def pack_map_pairs(self, pairs):
         self._fb_pack_map_pairs(len(pairs), pairs)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = wStringIO()
+            self.buffer = StringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = wStringIO(ret)
+            self.buffer = StringIO(ret)
         return ret
     def pack_array_header(self, n):
         self._fb_pack_array_header(n)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = wStringIO()
+            self.buffer = StringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = wStringIO(ret)
+            self.buffer = StringIO(ret)
         return ret
     def pack_map_header(self, n):
         self._fb_pack_map_header(n)
         ret = self.buffer.getvalue()
         if self.autoreset:
-            self.buffer = wStringIO()
+            self.buffer = StringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = wStringIO(ret)
+            self.buffer = StringIO(ret)
         return ret
     def _fb_pack_array_header(self, n):
         if n <= 0x0f:
@@ -538,4 +516,4 @@ class Packer(object):
     def bytes(self):
         return self.buffer.getvalue()
     def reset(self):
-        self.buffer = wStringIO()
+        self.buffer = StringIO()

From 4cde7f080c16fd396d758930faf9ebf15b138047 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Tue, 29 Jan 2013 03:46:07 +0100
Subject: [PATCH 1139/1648] fallback: _fb_read: add fast-path

---
 msgpack/fallback.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 2f83a20..bd50b47 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -222,6 +222,11 @@ class Unpacker(object):
         return ''.join(bufs)
 
     def _fb_read(self, n, write_bytes=None):
+        if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers)
+                and self._fb_buf_o + n < len(self._fb_buffers[self._fb_buf_i])):
+            self._fb_buf_o += n
+            return self._fb_buffers[self._fb_buf_i][
+                    self._fb_buf_o-n:self._fb_buf_o]
         ret = ''
         while len(ret) != n:
             if self._fb_buf_i == len(self._fb_buffers):

From 328369e52e8ef41a8b4ec09e48870f8f3edb190c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 29 Jan 2013 14:33:37 +0900
Subject: [PATCH 1140/1648] pep8 friendly.

---
 msgpack/fallback.py | 46 ++++++++++++++++++++++++++++-----------------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index bd50b47..3d733f4 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -1,4 +1,4 @@
-# Fallback pure Python implementation of msgpack
+"""Fallback pure Python implementation of msgpack"""
 
 import sys
 import array
@@ -49,11 +49,11 @@ DEFAULT_RECURSE_LIMIT=511
 def pack(o, stream, default=None, encoding='utf-8', unicode_errors='strict'):
     """ Pack object `o` and write it to `stream` """
     packer = Packer(default=default, encoding=encoding,
-                        unicode_errors=unicode_errors)
+                    unicode_errors=unicode_errors)
     stream.write(packer.pack(o))
 
 def packb(o, default=None, encoding='utf-8', unicode_errors='struct',
-                use_single_float=False):
+          use_single_float=False):
     """ Pack object `o` and return packed bytes """
     packer = Packer(default=default,
                     encoding=encoding,
@@ -62,28 +62,30 @@ def packb(o, default=None, encoding='utf-8', unicode_errors='struct',
     return packer.pack(o)
 
 def unpack(stream, object_hook=None, list_hook=None, use_list=True,
-                encoding=None, unicode_errors='strict',
-                object_pairs_hook=None):
+           encoding=None, unicode_errors='strict',
+           object_pairs_hook=None):
     """ Unpack an object from `stream`.
 
     Raises `ExtraData` when `stream` has extra bytes. """
     unpacker = Unpacker(stream, object_hook=object_hook, list_hook=list_hook,
-            use_list=use_list, encoding=encoding, unicode_errors=unicode_errors,
-            object_pairs_hook=object_pairs_hook)
+                        use_list=use_list,
+                        encoding=encoding, unicode_errors=unicode_errors,
+                        object_pairs_hook=object_pairs_hook)
     ret = unpacker._fb_unpack()
     if unpacker._fb_got_extradata():
         raise ExtraData(ret, unpacker._fb_get_extradata())
     return ret
 
 def unpackb(packed, object_hook=None, list_hook=None, use_list=True,
-                encoding=None, unicode_errors='strict',
-                object_pairs_hook=None):
+            encoding=None, unicode_errors='strict',
+            object_pairs_hook=None):
     """ Unpack an object from `packed`.
 
     Raises `ExtraData` when `packed` contains extra bytes. """
     unpacker = Unpacker(None, object_hook=object_hook, list_hook=list_hook,
-            use_list=use_list, encoding=encoding, unicode_errors=unicode_errors,
-            object_pairs_hook=object_pairs_hook)
+                        use_list=use_list,
+                        encoding=encoding, unicode_errors=unicode_errors,
+                        object_pairs_hook=object_pairs_hook)
     unpacker.feed(packed)
     ret = unpacker._fb_unpack()
     if unpacker._fb_got_extradata():
@@ -141,8 +143,8 @@ class Unpacker(object):
     """
 
     def __init__(self, file_like=None, read_size=0, use_list=True,
-            object_hook=None, object_pairs_hook=None, list_hook=None,
-            encoding=None, unicode_errors='strict', max_buffer_size=0):
+                 object_hook=None, object_pairs_hook=None, list_hook=None,
+                 encoding=None, unicode_errors='strict', max_buffer_size=0):
         if file_like is None:
             self._fb_feeding = True
         else:
@@ -174,8 +176,8 @@ class Unpacker(object):
         if object_pairs_hook is not None and not callable(object_pairs_hook):
             raise ValueError('`object_pairs_hook` is not callable')
         if object_hook is not None and object_pairs_hook is not None:
-            raise ValueError("object_pairs_hook and object_hook are mutually "+
-                                "exclusive")
+            raise ValueError("object_pairs_hook and object_hook are mutually "
+                             "exclusive")
 
     def feed(self, next_bytes):
         if isinstance(next_bytes, array.array):
@@ -394,7 +396,7 @@ class Unpacker(object):
 
 class Packer(object):
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
-                    use_single_float=False, autoreset=True):
+                 use_single_float=False, autoreset=True):
         self.use_float = use_single_float
         self.autoreset = autoreset
         self.encoding = encoding
@@ -404,6 +406,7 @@ class Packer(object):
             if not callable(default):
                 raise TypeError("default must be callable")
         self._default = default
+
     def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT):
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded")
@@ -461,10 +464,11 @@ class Packer(object):
             return
         if isinstance(obj, dict):
             return self._fb_pack_map_pairs(len(obj), obj.iteritems(),
-                                            nest_limit - 1)
+                                           nest_limit - 1)
         if self._default is not None:
             return self._pack(self._default(obj), nest_limit - 1)
         raise TypeError("Cannot serialize %r" % obj)
+
     def pack(self, obj):
         self._pack(obj)
         ret = self.buffer.getvalue()
@@ -473,6 +477,7 @@ class Packer(object):
         elif USING_STRINGBUILDER:
             self.buffer = StringIO(ret)
         return ret
+
     def pack_map_pairs(self, pairs):
         self._fb_pack_map_pairs(len(pairs), pairs)
         ret = self.buffer.getvalue()
@@ -481,6 +486,7 @@ class Packer(object):
         elif USING_STRINGBUILDER:
             self.buffer = StringIO(ret)
         return ret
+
     def pack_array_header(self, n):
         self._fb_pack_array_header(n)
         ret = self.buffer.getvalue()
@@ -489,6 +495,7 @@ class Packer(object):
         elif USING_STRINGBUILDER:
             self.buffer = StringIO(ret)
         return ret
+
     def pack_map_header(self, n):
         self._fb_pack_map_header(n)
         ret = self.buffer.getvalue()
@@ -497,6 +504,7 @@ class Packer(object):
         elif USING_STRINGBUILDER:
             self.buffer = StringIO(ret)
         return ret
+
     def _fb_pack_array_header(self, n):
         if n <= 0x0f:
             return self.buffer.write(chr(0x90 + n))
@@ -505,6 +513,7 @@ class Packer(object):
         if n <= 0xffffffff:
             return self.buffer.write(struct.pack(">BI", 0xdd, n))
         raise PackValueError("Array is too large")
+
     def _fb_pack_map_header(self, n):
         if n <= 0x0f:
             return self.buffer.write(chr(0x80 + n))
@@ -513,12 +522,15 @@ class Packer(object):
         if n <= 0xffffffff:
             return self.buffer.write(struct.pack(">BI", 0xdf, n))
         raise PackValueError("Dict is too large")
+
     def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
         self._fb_pack_map_header(n)
         for (k, v) in pairs:
             self._pack(k, nest_limit - 1)
             self._pack(v, nest_limit - 1)
+
     def bytes(self):
         return self.buffer.getvalue()
+
     def reset(self):
         self.buffer = StringIO()

From cbabeebc95e9e42c0356e089b742588a4de75d56 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 29 Jan 2013 14:47:16 +0900
Subject: [PATCH 1141/1648] Use MSGPACK_PUREPYTHON envvar to test fallback
 module

---
 .travis.yml         |  2 +-
 msgpack/__init__.py | 12 ++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2e6fc56..e536fdc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,4 +12,4 @@ install:
     - cython --cplus msgpack/_packer.pyx
     - cython --cplus msgpack/_unpacker.pyx
 
-script: "tox"
+script: "tox && MSGPACK_PUREPYTHON=x tox"
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 49a32d9..77f6b81 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -2,11 +2,15 @@
 from msgpack._version import version
 from msgpack.exceptions import *
 
-try:
-    from msgpack._packer import pack, packb, Packer
-    from msgpack._unpacker import unpack, unpackb, Unpacker
-except ImportError:
+import os
+if os.environ.get('MSGPACK_PUREPYTHON'):
     from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker
+else:
+    try:
+        from msgpack._packer import pack, packb, Packer
+        from msgpack._unpacker import unpack, unpackb, Unpacker
+    except ImportError:
+        from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker
 
 # alias for compatibility to simplejson/marshal/pickle.
 load = unpack

From 8d6a387dff10dd2150aa86cd96e2bece26546268 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 29 Jan 2013 15:10:22 +0900
Subject: [PATCH 1142/1648] fallback: Support Python 3.

---
 msgpack/fallback.py | 54 ++++++++++++++++++++++++++++-----------------
 1 file changed, 34 insertions(+), 20 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 3d733f4..ac6dbf9 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -4,13 +4,28 @@ import sys
 import array
 import struct
 
+if sys.version_info[0] == 3:
+    PY3 = True
+    int_types = int
+    Unicode = str
+    xrange = range
+    def dict_iteritems(d):
+        return d.items()
+else:
+    PY3 = False
+    int_types = (int, long)
+    Unicode = unicode
+    def dict_iteritems(d):
+        return d.iteritems()
+
+
 if hasattr(sys, 'pypy_version_info'):
     # cStringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
     # StringBuilder is fastest.
     from __pypy__.builders import StringBuilder
     USING_STRINGBUILDER = True
     class StringIO(object):
-        def __init__(self, s=''):
+        def __init__(self, s=b''):
             if s:
                 self.builder = StringBuilder(len(s))
                 self.builder.append(s)
@@ -22,10 +37,7 @@ if hasattr(sys, 'pypy_version_info'):
             return self.builder.build()
 else:
     USING_STRINGBUILDER = False
-    try:
-        from cStringIO import StringIO
-    except ImportError:
-        from StringIO import StringIO
+    from io import BytesIO as StringIO
 
 from msgpack.exceptions import (
         BufferFull,
@@ -156,7 +168,7 @@ class Unpacker(object):
         self._fb_buf_o = 0
         self._fb_buf_i = 0
         self._fb_buf_n = 0
-        self.max_buffer_size = (sys.maxint if max_buffer_size == 0
+        self.max_buffer_size = (2**31-1 if max_buffer_size == 0
                                         else max_buffer_size)
         self.read_size = (read_size if read_size != 0
                             else min(self.max_buffer_size, 2048))
@@ -221,7 +233,7 @@ class Unpacker(object):
         bufs = self._fb_buffers[self._fb_buf_i:]
         if bufs:
             bufs[0] = bufs[0][self._fb_buf_o:]
-        return ''.join(bufs)
+        return b''.join(bufs)
 
     def _fb_read(self, n, write_bytes=None):
         if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers)
@@ -229,7 +241,7 @@ class Unpacker(object):
             self._fb_buf_o += n
             return self._fb_buffers[self._fb_buf_i][
                     self._fb_buf_o-n:self._fb_buf_o]
-        ret = ''
+        ret = b''
         while len(ret) != n:
             if self._fb_buf_i == len(self._fb_buffers):
                 if self._fb_feeding:
@@ -255,11 +267,12 @@ class Unpacker(object):
 
     def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
         typ = TYPE_IMMEDIATE
-        b = ord(self._fb_read(1, write_bytes))
+        c = self._fb_read(1, write_bytes)
+        b = ord(c)
         if   b & 0b10000000 == 0:
             obj = b
         elif b & 0b11100000 == 0b11100000:
-            obj = struct.unpack("b", chr(b))[0]
+            obj = struct.unpack("b", c)[0]
         elif b & 0b11100000 == 0b10100000:
             n = b & 0b00011111
             obj = self._fb_read(n, write_bytes)
@@ -374,6 +387,7 @@ class Unpacker(object):
             return ret
         except OutOfData:
             raise StopIteration
+    __next__ = next
 
     def skip(self, write_bytes=None):
         self._fb_unpack(EX_SKIP, write_bytes)
@@ -411,12 +425,12 @@ class Packer(object):
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded")
         if obj is None:
-            return self.buffer.write(chr(0xc0))
+            return self.buffer.write(b"\xc0")
         if isinstance(obj, bool):
             if obj:
-                return self.buffer.write(chr(0xc3))
-            return self.buffer.write(chr(0xc2))
-        if isinstance(obj, int) or isinstance(obj, long):
+                return self.buffer.write(b"\xc3")
+            return self.buffer.write(b"\xc2")
+        if isinstance(obj, int_types):
             if 0 <= obj < 0x80:
                 return self.buffer.write(struct.pack("B", obj))
             if -0x20 <= obj < 0:
@@ -438,12 +452,12 @@ class Packer(object):
             if -0x8000000000000000 <= obj < -0x80000000:
                 return self.buffer.write(struct.pack(">Bq", 0xd3, obj))
             raise PackValueError("Integer value out of range")
-        if isinstance(obj, str) or isinstance(obj, unicode):
-            if isinstance(obj, unicode):
+        if isinstance(obj, (Unicode, bytes)):
+            if isinstance(obj, Unicode):
                 obj = obj.encode(self.encoding, self.unicode_errors)
             n = len(obj)
             if n <= 0x1f:
-                self.buffer.write(chr(0xa0 + n))
+                self.buffer.write(struct.pack('B', 0xa0 + n))
                 return self.buffer.write(obj)
             if n <= 0xffff:
                 self.buffer.write(struct.pack(">BH", 0xda, n))
@@ -463,7 +477,7 @@ class Packer(object):
                 self._pack(obj[i], nest_limit - 1)
             return
         if isinstance(obj, dict):
-            return self._fb_pack_map_pairs(len(obj), obj.iteritems(),
+            return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj),
                                            nest_limit - 1)
         if self._default is not None:
             return self._pack(self._default(obj), nest_limit - 1)
@@ -507,7 +521,7 @@ class Packer(object):
 
     def _fb_pack_array_header(self, n):
         if n <= 0x0f:
-            return self.buffer.write(chr(0x90 + n))
+            return self.buffer.write(struct.pack('B', 0x90 + n))
         if n <= 0xffff:
             return self.buffer.write(struct.pack(">BH", 0xdc, n))
         if n <= 0xffffffff:
@@ -516,7 +530,7 @@ class Packer(object):
 
     def _fb_pack_map_header(self, n):
         if n <= 0x0f:
-            return self.buffer.write(chr(0x80 + n))
+            return self.buffer.write(struct.pack('B', 0x80 + n))
         if n <= 0xffff:
             return self.buffer.write(struct.pack(">BH", 0xde, n))
         if n <= 0xffffffff:

From 266eaf813d4e958dce5a2a8c4a84babf331369f0 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 29 Jan 2013 15:13:20 +0900
Subject: [PATCH 1143/1648] changelog: describe purepython fallback.

---
 ChangeLog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 55e296f..fdbb3ea 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -10,6 +10,7 @@ Inconpatible Changes
 
 Changes
 -------
+* Pure Python fallback module is added. (thanks to bwesterb)
 * Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman)
 * Add capturing feature. You can pass the writable object to
   ``Unpacker.unpack()`` as a second parameter.

From 95dfec808a700c99144e454fbf9d98be420f2c51 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 3 Feb 2013 00:02:37 +0900
Subject: [PATCH 1144/1648] Add simple benchmark.

---
 benchmark/benchmark.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 benchmark/benchmark.py

diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py
new file mode 100644
index 0000000..1439289
--- /dev/null
+++ b/benchmark/benchmark.py
@@ -0,0 +1,32 @@
+from msgpack import fallback
+try:
+    from msgpack import _unpacker, _packer
+    has_ext = True
+except ImportError:
+    has_ext = False
+import timeit
+
+
+def profile(name, func):
+    times = timeit.repeat(func, number=1000, repeat=4)
+    times = ', '.join(["%8f" % t for t in times])
+    print("%-30s %40s" % (name, times))
+
+
+def simple(name, data):
+    if has_ext:
+        profile("packing %s (ext)" % name, lambda: _packer.packb(data))
+    profile('packing %s (fallback)' % name, lambda: fallback.packb(data))
+
+    data = fallback.packb(data)
+    if has_ext:
+        profile('unpacking %s (ext)' % name, lambda: _unpacker.unpackb(data))
+    profile('unpacking %s (fallback)' % name, lambda: fallback.unpackb(data))
+
+def main():
+    simple("integers", [7]*10000)
+    simple("bytes", [b'x'*n for n in range(100)]*10)
+    simple("lists", [[]]*10000)
+    simple("dicts", [{}]*10000)
+
+main()

From 0536d1bd0cb5c781b41e8cf7ede66448396dc993 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 3 Feb 2013 00:11:26 +0900
Subject: [PATCH 1145/1648] Don't compile extension module when running on pypy

---
 setup.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/setup.py b/setup.py
index d4808d6..888de09 100644
--- a/setup.py
+++ b/setup.py
@@ -75,18 +75,19 @@ else:
     macros = [('__LITTLE_ENDIAN__', '1')]
 
 ext_modules = []
-ext_modules.append(Extension('msgpack._packer',
-                             sources=['msgpack/_packer.cpp'],
-                             libraries=libraries,
-                             include_dirs=['.'],
-                             define_macros=macros,
-                             ))
-ext_modules.append(Extension('msgpack._unpacker',
-                             sources=['msgpack/_unpacker.cpp'],
-                             libraries=libraries,
-                             include_dirs=['.'],
-                             define_macros=macros,
-                             ))
+if not hasattr(sys, 'pypy_version_info'):
+    ext_modules.append(Extension('msgpack._packer',
+                                 sources=['msgpack/_packer.cpp'],
+                                 libraries=libraries,
+                                 include_dirs=['.'],
+                                 define_macros=macros,
+                                 ))
+    ext_modules.append(Extension('msgpack._unpacker',
+                                 sources=['msgpack/_unpacker.cpp'],
+                                 libraries=libraries,
+                                 include_dirs=['.'],
+                                 define_macros=macros,
+                                 ))
 del libraries, macros
 
 

From 22920baae6957e2259a82d0595c2b97fc58fcd02 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 3 Feb 2013 00:20:00 +0900
Subject: [PATCH 1146/1648] Fix minor bugs and tuning unpacking dict.

---
 msgpack/fallback.py | 25 ++++++++++++++++---------
 test/test_pack.py   |  2 +-
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index ac6dbf9..c10c6ac 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -360,16 +360,19 @@ class Unpacker(object):
                     self._fb_unpack(EX_SKIP, write_bytes)
                     self._fb_unpack(EX_SKIP, write_bytes)
                 return
-            ret = []
-            for i in xrange(n):
-                ret.append((self._fb_unpack(EX_CONSTRUCT, write_bytes),
-                            self._fb_unpack(EX_CONSTRUCT, write_bytes)))
             if self.object_pairs_hook is not None:
-                ret = self.object_pairs_hook(ret)
+                ret = self.object_pairs_hook(
+                        (self._fb_unpack(EX_CONSTRUCT, write_bytes),
+                         self._fb_unpack(EX_CONSTRUCT, write_bytes))
+                            for _ in xrange(n)
+                        )
             else:
-                ret = dict(ret)
-            if self.object_hook is not None:
-                ret = self.object_hook(ret)
+                ret = {}
+                for _ in xrange(n):
+                    key = self._fb_unpack(EX_CONSTRUCT, write_bytes)
+                    ret[key] = self._fb_unpack(EX_CONSTRUCT, write_bytes)
+                if self.object_hook is not None:
+                    ret = self.object_hook(ret)
             return ret
         if execute == EX_SKIP:
             return
@@ -421,7 +424,7 @@ class Packer(object):
                 raise TypeError("default must be callable")
         self._default = default
 
-    def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT):
+    def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance):
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded")
         if obj is None:
@@ -454,6 +457,10 @@ class Packer(object):
             raise PackValueError("Integer value out of range")
         if isinstance(obj, (Unicode, bytes)):
             if isinstance(obj, Unicode):
+                if self.encoding is None:
+                    raise TypeError(
+                            "Can't encode unicode string: "
+                            "no encoding is specified")
                 obj = obj.encode(self.encoding, self.unicode_errors)
             n = len(obj)
             if n <= 0x1f:
diff --git a/test/test_pack.py b/test/test_pack.py
index 8f4117c..3225f41 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -151,7 +151,7 @@ def test_odict():
     od = odict(seq)
     assert unpackb(packb(od), use_list=1) == dict(seq)
     def pair_hook(seq):
-        return seq
+        return list(seq)
     assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
 
 

From 1951b197b547c3f12b755790717d799272fbeb34 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 3 Feb 2013 00:52:05 +0900
Subject: [PATCH 1147/1648] Skip compile error for extension modules.

---
 setup.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 888de09..1055a61 100644
--- a/setup.py
+++ b/setup.py
@@ -46,7 +46,12 @@ class BuildExt(build_ext):
             print("Install Cython >= 0.16 or install msgpack from PyPI.")
             print("Falling back to pure Python implementation.")
             return
-        return build_ext.build_extension(self, ext)
+        try:
+            return build_ext.build_extension(self, ext)
+        except Exception as e:
+            print("WARNING: Failed to compile extensiom modules.")
+            print("msgpack uses fallback pure python implementation.")
+            print(e)
 
 
 exec(open('msgpack/_version.py').read())

From a865f8f7e94ea5b484771c1be3fe57ff3a63aa2a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 4 Feb 2013 15:14:30 +0900
Subject: [PATCH 1148/1648] Use _private names for non public data members.
 (fix #44)

---
 msgpack/fallback.py | 151 ++++++++++++++++++++++----------------------
 1 file changed, 76 insertions(+), 75 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index c10c6ac..a834229 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -168,18 +168,18 @@ class Unpacker(object):
         self._fb_buf_o = 0
         self._fb_buf_i = 0
         self._fb_buf_n = 0
-        self.max_buffer_size = (2**31-1 if max_buffer_size == 0
-                                        else max_buffer_size)
-        self.read_size = (read_size if read_size != 0
-                            else min(self.max_buffer_size, 2048))
-        if read_size > self.max_buffer_size:
+        self._max_buffer_size = (2**31-1 if max_buffer_size == 0
+                                         else max_buffer_size)
+        self._read_size = (read_size if read_size != 0
+                            else min(self._max_buffer_size, 2048))
+        if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")
-        self.encoding = encoding
-        self.unicode_errors = unicode_errors
-        self.use_list = use_list
-        self.list_hook = list_hook
-        self.object_hook = object_hook
-        self.object_pairs_hook = object_pairs_hook
+        self._encoding = encoding
+        self._unicode_errors = unicode_errors
+        self._use_list = use_list
+        self._list_hook = list_hook
+        self._object_hook = object_hook
+        self._object_pairs_hook = object_pairs_hook
 
         if list_hook is not None and not callable(list_hook):
             raise ValueError('`list_hook` is not callable')
@@ -195,7 +195,7 @@ class Unpacker(object):
         if isinstance(next_bytes, array.array):
             next_bytes = next_bytes.tostring()
         assert self._fb_feeding
-        if self._fb_buf_n + len(next_bytes) > self.max_buffer_size:
+        if self._fb_buf_n + len(next_bytes) > self._max_buffer_size:
             raise BufferFull
         self._fb_buf_n += len(next_bytes)
         self._fb_buffers.append(next_bytes)
@@ -246,7 +246,7 @@ class Unpacker(object):
             if self._fb_buf_i == len(self._fb_buffers):
                 if self._fb_feeding:
                     break
-                tmp = self.file_like.read(self.read_size)
+                tmp = self.file_like.read(self._read_size)
                 if not tmp:
                     break
                 self._fb_buffers.append(tmp)
@@ -349,10 +349,10 @@ class Unpacker(object):
             ret = []
             for i in xrange(n):
                 ret.append(self._fb_unpack(EX_CONSTRUCT, write_bytes))
-            if self.list_hook is not None:
-                ret = self.list_hook(ret)
+            if self._list_hook is not None:
+                ret = self._list_hook(ret)
             # TODO is the interaction between `list_hook` and `use_list` ok?
-            return ret if self.use_list else tuple(ret)
+            return ret if self._use_list else tuple(ret)
         if typ == TYPE_MAP:
             if execute == EX_SKIP:
                 for i in xrange(n):
@@ -360,8 +360,8 @@ class Unpacker(object):
                     self._fb_unpack(EX_SKIP, write_bytes)
                     self._fb_unpack(EX_SKIP, write_bytes)
                 return
-            if self.object_pairs_hook is not None:
-                ret = self.object_pairs_hook(
+            if self._object_pairs_hook is not None:
+                ret = self._object_pairs_hook(
                         (self._fb_unpack(EX_CONSTRUCT, write_bytes),
                          self._fb_unpack(EX_CONSTRUCT, write_bytes))
                             for _ in xrange(n)
@@ -371,14 +371,14 @@ class Unpacker(object):
                 for _ in xrange(n):
                     key = self._fb_unpack(EX_CONSTRUCT, write_bytes)
                     ret[key] = self._fb_unpack(EX_CONSTRUCT, write_bytes)
-                if self.object_hook is not None:
-                    ret = self.object_hook(ret)
+                if self._object_hook is not None:
+                    ret = self._object_hook(ret)
             return ret
         if execute == EX_SKIP:
             return
         if typ == TYPE_RAW:
-            if self.encoding is not None:
-                obj = obj.decode(self.encoding, self.unicode_errors)
+            if self._encoding is not None:
+                obj = obj.decode(self._encoding, self._unicode_errors)
             return obj
         assert typ == TYPE_IMMEDIATE
         return obj
@@ -411,14 +411,15 @@ class Unpacker(object):
         self._fb_consume()
         return ret
 
+
 class Packer(object):
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
                  use_single_float=False, autoreset=True):
-        self.use_float = use_single_float
-        self.autoreset = autoreset
-        self.encoding = encoding
-        self.unicode_errors = unicode_errors
-        self.buffer = StringIO()
+        self._use_float = use_single_float
+        self._autoreset = autoreset
+        self._encoding = encoding
+        self._unicode_errors = unicode_errors
+        self._buffer = StringIO()
         if default is not None:
             if not callable(default):
                 raise TypeError("default must be callable")
@@ -428,55 +429,55 @@ class Packer(object):
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded")
         if obj is None:
-            return self.buffer.write(b"\xc0")
+            return self._buffer.write(b"\xc0")
         if isinstance(obj, bool):
             if obj:
-                return self.buffer.write(b"\xc3")
-            return self.buffer.write(b"\xc2")
+                return self._buffer.write(b"\xc3")
+            return self._buffer.write(b"\xc2")
         if isinstance(obj, int_types):
             if 0 <= obj < 0x80:
-                return self.buffer.write(struct.pack("B", obj))
+                return self._buffer.write(struct.pack("B", obj))
             if -0x20 <= obj < 0:
-                return self.buffer.write(struct.pack("b", obj))
+                return self._buffer.write(struct.pack("b", obj))
             if 0x80 <= obj <= 0xff:
-                return self.buffer.write(struct.pack("BB", 0xcc, obj))
+                return self._buffer.write(struct.pack("BB", 0xcc, obj))
             if -0x80 <= obj < 0:
-                return self.buffer.write(struct.pack(">Bb", 0xd0, obj))
+                return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
             if 0xff < obj <= 0xffff:
-                return self.buffer.write(struct.pack(">BH", 0xcd, obj))
+                return self._buffer.write(struct.pack(">BH", 0xcd, obj))
             if -0x8000 <= obj < -0x80:
-                return self.buffer.write(struct.pack(">Bh", 0xd1, obj))
+                return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
             if 0xffff < obj <= 0xffffffff:
-                return self.buffer.write(struct.pack(">BI", 0xce, obj))
+                return self._buffer.write(struct.pack(">BI", 0xce, obj))
             if -0x80000000 <= obj < -0x8000:
-                return self.buffer.write(struct.pack(">Bi", 0xd2, obj))
+                return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
             if 0xffffffff < obj <= 0xffffffffffffffff:
-                return self.buffer.write(struct.pack(">BQ", 0xcf, obj))
+                return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
             if -0x8000000000000000 <= obj < -0x80000000:
-                return self.buffer.write(struct.pack(">Bq", 0xd3, obj))
+                return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
             raise PackValueError("Integer value out of range")
         if isinstance(obj, (Unicode, bytes)):
             if isinstance(obj, Unicode):
-                if self.encoding is None:
+                if self._encoding is None:
                     raise TypeError(
                             "Can't encode unicode string: "
                             "no encoding is specified")
-                obj = obj.encode(self.encoding, self.unicode_errors)
+                obj = obj.encode(self._encoding, self._unicode_errors)
             n = len(obj)
             if n <= 0x1f:
-                self.buffer.write(struct.pack('B', 0xa0 + n))
-                return self.buffer.write(obj)
+                self._buffer.write(struct.pack('B', 0xa0 + n))
+                return self._buffer.write(obj)
             if n <= 0xffff:
-                self.buffer.write(struct.pack(">BH", 0xda, n))
-                return self.buffer.write(obj)
+                self._buffer.write(struct.pack(">BH", 0xda, n))
+                return self._buffer.write(obj)
             if n <= 0xffffffff:
-                self.buffer.write(struct.pack(">BI", 0xdb, n))
-                return self.buffer.write(obj)
+                self._buffer.write(struct.pack(">BI", 0xdb, n))
+                return self._buffer.write(obj)
             raise PackValueError("String is too large")
         if isinstance(obj, float):
-            if self.use_float:
-                return self.buffer.write(struct.pack(">Bf", 0xca, obj))
-            return self.buffer.write(struct.pack(">Bd", 0xcb, obj))
+            if self._use_float:
+                return self._buffer.write(struct.pack(">Bf", 0xca, obj))
+            return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
         if isinstance(obj, list) or isinstance(obj, tuple):
             n = len(obj)
             self._fb_pack_array_header(n)
@@ -492,56 +493,56 @@ class Packer(object):
 
     def pack(self, obj):
         self._pack(obj)
-        ret = self.buffer.getvalue()
-        if self.autoreset:
-            self.buffer = StringIO()
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = StringIO(ret)
+            self._buffer = StringIO(ret)
         return ret
 
     def pack_map_pairs(self, pairs):
         self._fb_pack_map_pairs(len(pairs), pairs)
-        ret = self.buffer.getvalue()
-        if self.autoreset:
-            self.buffer = StringIO()
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = StringIO(ret)
+            self._buffer = StringIO(ret)
         return ret
 
     def pack_array_header(self, n):
         self._fb_pack_array_header(n)
-        ret = self.buffer.getvalue()
-        if self.autoreset:
-            self.buffer = StringIO()
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = StringIO(ret)
+            self._buffer = StringIO(ret)
         return ret
 
     def pack_map_header(self, n):
         self._fb_pack_map_header(n)
-        ret = self.buffer.getvalue()
-        if self.autoreset:
-            self.buffer = StringIO()
+        ret = self._buffer.getvalue()
+        if self._autoreset:
+            self._buffer = StringIO()
         elif USING_STRINGBUILDER:
-            self.buffer = StringIO(ret)
+            self._buffer = StringIO(ret)
         return ret
 
     def _fb_pack_array_header(self, n):
         if n <= 0x0f:
-            return self.buffer.write(struct.pack('B', 0x90 + n))
+            return self._buffer.write(struct.pack('B', 0x90 + n))
         if n <= 0xffff:
-            return self.buffer.write(struct.pack(">BH", 0xdc, n))
+            return self._buffer.write(struct.pack(">BH", 0xdc, n))
         if n <= 0xffffffff:
-            return self.buffer.write(struct.pack(">BI", 0xdd, n))
+            return self._buffer.write(struct.pack(">BI", 0xdd, n))
         raise PackValueError("Array is too large")
 
     def _fb_pack_map_header(self, n):
         if n <= 0x0f:
-            return self.buffer.write(struct.pack('B', 0x80 + n))
+            return self._buffer.write(struct.pack('B', 0x80 + n))
         if n <= 0xffff:
-            return self.buffer.write(struct.pack(">BH", 0xde, n))
+            return self._buffer.write(struct.pack(">BH", 0xde, n))
         if n <= 0xffffffff:
-            return self.buffer.write(struct.pack(">BI", 0xdf, n))
+            return self._buffer.write(struct.pack(">BI", 0xdf, n))
         raise PackValueError("Dict is too large")
 
     def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
@@ -551,7 +552,7 @@ class Packer(object):
             self._pack(v, nest_limit - 1)
 
     def bytes(self):
-        return self.buffer.getvalue()
+        return self._buffer.getvalue()
 
     def reset(self):
-        self.buffer = StringIO()
+        self._buffer = StringIO()

From df6449f17366b9d608895766cc1a151167460ae0 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 16 Feb 2013 09:28:29 +0900
Subject: [PATCH 1149/1648] 0.3.0

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index e750485..5999ede 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 3, 0, 'dev1')
+version = (0, 3, 0)

From 626ae51017cdbf62a2c7b00b338747d1b2070513 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 16 Feb 2013 14:03:39 +0900
Subject: [PATCH 1150/1648] README: s/nosetest/pytest/

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 270b8ff..f2e5962 100644
--- a/README.rst
+++ b/README.rst
@@ -192,10 +192,10 @@ Another way to unpacking such object is using ``object_pairs_hook``.
 
 TEST
 ----
-MessagePack uses `nosetest` for testing.
+MessagePack uses `pytest` for testing.
 Run test with following command:
 
-    $ nosetests test
+    $ py.test
 
 
 ..

From 3f12846d40e035573993cc9c5195bf3607ee98bb Mon Sep 17 00:00:00 2001
From: Alex Gaynor 
Date: Sat, 16 Feb 2013 12:08:14 -0800
Subject: [PATCH 1151/1648] On PyPy, preallocate lists

When deserealizing arrays, preallocate the resulting list at the correct size.
---
 msgpack/fallback.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index a834229..0016bd1 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -22,6 +22,7 @@ else:
 if hasattr(sys, 'pypy_version_info'):
     # cStringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
     # StringBuilder is fastest.
+    from __pypy__ import newlist_hint
     from __pypy__.builders import StringBuilder
     USING_STRINGBUILDER = True
     class StringIO(object):
@@ -38,6 +39,7 @@ if hasattr(sys, 'pypy_version_info'):
 else:
     USING_STRINGBUILDER = False
     from io import BytesIO as StringIO
+    newlist_hint = lambda size: []
 
 from msgpack.exceptions import (
         BufferFull,
@@ -346,7 +348,7 @@ class Unpacker(object):
                     # TODO check whether we need to call `list_hook`
                     self._fb_unpack(EX_SKIP, write_bytes)
                 return
-            ret = []
+            ret = newlist_hint(n)
             for i in xrange(n):
                 ret.append(self._fb_unpack(EX_CONSTRUCT, write_bytes))
             if self._list_hook is not None:

From 952403319478ac4a95c4d5d9c85908ba6dda5672 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 21 Feb 2013 14:01:12 +0900
Subject: [PATCH 1152/1648] skip reserved byte.

---
 msgpack/fallback.py       | 14 ++++++-
 msgpack/unpack_template.h | 85 ++++++++++++++++++++-------------------
 2 files changed, 56 insertions(+), 43 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 0016bd1..311f16b 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -57,6 +57,7 @@ TYPE_IMMEDIATE          = 0
 TYPE_ARRAY              = 1
 TYPE_MAP                = 2
 TYPE_RAW                = 3
+TYPE_RESERVED           = 4
 
 DEFAULT_RECURSE_LIMIT=511
 
@@ -267,8 +268,10 @@ class Unpacker(object):
             write_bytes(ret)
         return ret
 
-    def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
+    def _read_header(self, execute=EX_CONSTRUCT, write_bytes=None):
         typ = TYPE_IMMEDIATE
+        n = 0
+        obj = None
         c = self._fb_read(1, write_bytes)
         b = ord(c)
         if   b & 0b10000000 == 0:
@@ -332,7 +335,14 @@ class Unpacker(object):
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             typ = TYPE_MAP
         else:
-            raise UnpackValueError("Unknown header: 0x%x" % b)
+            typ = TYPE_RESERVED
+        return typ, n, obj
+
+    def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
+        typ = TYPE_RESERVED
+        while typ == TYPE_RESERVED:
+            typ, n, obj = self._read_header(self, execute, write_bytes)
+
         if execute == EX_READ_ARRAY_HEADER:
             if typ != TYPE_ARRAY:
                 raise UnpackValueError("Expected array")
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 83b6918..8fd179d 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -422,53 +422,56 @@ _end:
 template 
 msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
 {
-	assert(len >= *off);
-	uint32_t size;
-	const unsigned char *const p = (unsigned char*)data + *off;
+	for (;;) {
+		if (len < *off) {
+			return 0;
+		}
+		uint32_t size;
+		const unsigned char *const p = (unsigned char*)data + *off;
 
 #define inc_offset(inc) \
-	if (len - *off < inc) \
-		return 0; \
-	*off += inc;
+		if (len - *off < inc) \
+			return 0; \
+		*off += inc;
 
-	switch (*p) {
-	case var_offset:
-		inc_offset(3);
-		size = _msgpack_load16(uint16_t, p + 1);
-		break;
-	case var_offset + 1:
-		inc_offset(5);
-		size = _msgpack_load32(uint32_t, p + 1);
-		break;
+		switch (*p) {
+		case var_offset:
+			inc_offset(3);
+			size = _msgpack_load16(uint16_t, p + 1);
+			break;
+		case var_offset + 1:
+			inc_offset(5);
+			size = _msgpack_load32(uint32_t, p + 1);
+			break;
 #ifdef USE_CASE_RANGE
-	case fixed_offset + 0x0 ... fixed_offset + 0xf:
+		case fixed_offset + 0x0 ... fixed_offset + 0xf:
 #else
-	case fixed_offset + 0x0:
-	case fixed_offset + 0x1:
-	case fixed_offset + 0x2:
-	case fixed_offset + 0x3:
-	case fixed_offset + 0x4:
-	case fixed_offset + 0x5:
-	case fixed_offset + 0x6:
-	case fixed_offset + 0x7:
-	case fixed_offset + 0x8:
-	case fixed_offset + 0x9:
-	case fixed_offset + 0xa:
-	case fixed_offset + 0xb:
-	case fixed_offset + 0xc:
-	case fixed_offset + 0xd:
-	case fixed_offset + 0xe:
-	case fixed_offset + 0xf:
+		case fixed_offset + 0x0:
+		case fixed_offset + 0x1:
+		case fixed_offset + 0x2:
+		case fixed_offset + 0x3:
+		case fixed_offset + 0x4:
+		case fixed_offset + 0x5:
+		case fixed_offset + 0x6:
+		case fixed_offset + 0x7:
+		case fixed_offset + 0x8:
+		case fixed_offset + 0x9:
+		case fixed_offset + 0xa:
+		case fixed_offset + 0xb:
+		case fixed_offset + 0xc:
+		case fixed_offset + 0xd:
+		case fixed_offset + 0xe:
+		case fixed_offset + 0xf:
 #endif
-		++*off;
-		size = ((unsigned int)*p) & 0x0f;
-		break;
-	default:
-		PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
-		return -1;
-    }
-	msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj);
-	return 1;
+			++*off;
+			size = ((unsigned int)*p) & 0x0f;
+			break;
+		default:
+			continue;
+		}
+		msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj);
+		return 1;
+	}
 }
 
 #undef SWITCH_RANGE_BEGIN

From ff3342aeedb9d21897b0b7275dd68d781f149995 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 21 Feb 2013 16:02:33 +0900
Subject: [PATCH 1153/1648] Skip reserved byte

---
 msgpack/unpack_template.h | 298 +++++++++++++++++++++-----------------
 test/test_reserved.py     |   8 +
 2 files changed, 171 insertions(+), 135 deletions(-)
 create mode 100644 test/test_reserved.py

diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 8fd179d..6f86695 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -60,10 +60,10 @@ msgpack_unpack_struct_decl(_context) {
 	unsigned int trail;
 	unsigned int top;
 	/*
-	msgpack_unpack_struct(_stack)* stack;
-	unsigned int stack_size;
-	msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE];
-	*/
+	   msgpack_unpack_struct(_stack)* stack;
+	   unsigned int stack_size;
+	   msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE];
+	   */
 	msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE];
 };
 
@@ -74,20 +74,20 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx)
 	ctx->trail = 0;
 	ctx->top = 0;
 	/*
-	ctx->stack = ctx->embed_stack;
-	ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
-	*/
+	   ctx->stack = ctx->embed_stack;
+	   ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
+	   */
 	ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user);
 }
 
 /*
-msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx)
-{
-	if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
-		free(ctx->stack);
-	}
-}
-*/
+   msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx)
+   {
+   if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
+   free(ctx->stack);
+   }
+   }
+   */
 
 msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx)
 {
@@ -109,8 +109,8 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	unsigned int top = ctx->top;
 	msgpack_unpack_struct(_stack)* stack = ctx->stack;
 	/*
-	unsigned int stack_size = ctx->stack_size;
-	*/
+	   unsigned int stack_size = ctx->stack_size;
+	   */
 	msgpack_unpack_user* user = &ctx->user;
 
 	msgpack_unpack_object obj;
@@ -119,7 +119,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	int ret;
 
 #define construct_cb(name) \
-    construct && msgpack_unpack_callback(name)
+	construct && msgpack_unpack_callback(name)
 
 #define push_simple_value(func) \
 	if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \
@@ -129,7 +129,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	goto _push
 #define push_variable_value(func, base, pos, len) \
 	if(construct_cb(func)(user, \
-		(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
+				(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
 	goto _push
 
 #define again_fixed_trail(_cs, trail_len) \
@@ -155,24 +155,24 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	/*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
 	/*printf("stack push %d\n", top);*/ \
 	/* FIXME \
-	if(top >= stack_size) { \
-		if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
-			size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \
-			size_t nsize = csize * 2; \
-			msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \
-			if(tmp == NULL) { goto _failed; } \
-			memcpy(tmp, ctx->stack, csize); \
-			ctx->stack = stack = tmp; \
-			ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
-		} else { \
-			size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \
-			msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \
-			if(tmp == NULL) { goto _failed; } \
-			ctx->stack = stack = tmp; \
-			ctx->stack_size = stack_size = stack_size * 2; \
-		} \
-	} \
-	*/ \
+	   if(top >= stack_size) { \
+	   if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
+	   size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \
+	   size_t nsize = csize * 2; \
+	   msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \
+	   if(tmp == NULL) { goto _failed; } \
+	   memcpy(tmp, ctx->stack, csize); \
+	   ctx->stack = stack = tmp; \
+	   ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
+	   } else { \
+	   size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \
+	   msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \
+	   if(tmp == NULL) { goto _failed; } \
+	   ctx->stack = stack = tmp; \
+	   ctx->stack_size = stack_size = stack_size * 2; \
+	   } \
+	   } \
+	   */ \
 	goto _header_again
 
 #define NEXT_CS(p) \
@@ -195,7 +195,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 		switch(cs) {
 		case CS_HEADER:
 			SWITCH_RANGE_BEGIN
-			SWITCH_RANGE(0x00, 0x7f)  // Positive Fixnum
+				SWITCH_RANGE(0x00, 0x7f)  // Positive Fixnum
 				push_fixed_value(_uint8, *(uint8_t*)p);
 			SWITCH_RANGE(0xe0, 0xff)  // Negative Fixnum
 				push_fixed_value(_int8, *(int8_t*)p);
@@ -203,18 +203,24 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				switch(*p) {
 				case 0xc0:  // nil
 					push_simple_value(_nil);
-				//case 0xc1:  // string
-				//	again_terminal_trail(NEXT_CS(p), p+1);
+					// reserved.
+				case 0xc1:  // string
+					fprintf(stderr, "skipping %x\n", (int)*p);
+					goto _header_again;
+					//	again_terminal_trail(NEXT_CS(p), p+1);
 				case 0xc2:  // false
 					push_simple_value(_false);
 				case 0xc3:  // true
 					push_simple_value(_true);
-				//case 0xc4:
-				//case 0xc5:
-				//case 0xc6:
-				//case 0xc7:
-				//case 0xc8:
-				//case 0xc9:
+					// reserved.
+				case 0xc4:
+				case 0xc5:
+				case 0xc6:
+				case 0xc7:
+				case 0xc8:
+				case 0xc9:
+					fprintf(stderr, "skipping %x\n", (int)*p);
+					goto _header_again;
 				case 0xca:  // float
 				case 0xcb:  // double
 				case 0xcc:  // unsigned int  8
@@ -226,12 +232,15 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				case 0xd2:  // signed int 32
 				case 0xd3:  // signed int 64
 					again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
-				//case 0xd4:
-				//case 0xd5:
-				//case 0xd6:  // big integer 16
-				//case 0xd7:  // big integer 32
-				//case 0xd8:  // big float 16
-				//case 0xd9:  // big float 32
+					// reserved.
+				case 0xd4:
+				case 0xd5:
+				case 0xd6:  // big integer 16
+				case 0xd7:  // big integer 32
+				case 0xd8:  // big float 16
+				case 0xd9:  // big float 32
+					fprintf(stderr, "skipping %x\n", (int)*p);
+					goto _header_again;
 				case 0xda:  // raw 16
 				case 0xdb:  // raw 32
 				case 0xdc:  // array 16
@@ -240,6 +249,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				case 0xdf:  // map 32
 					again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
 				default:
+					fprintf(stderr, "failed %x\n", (int)*p);
 					goto _failed;
 				}
 			SWITCH_RANGE(0xa0, 0xbf)  // FixRaw
@@ -250,128 +260,129 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
 
 			SWITCH_RANGE_DEFAULT
-				goto _failed;
+				fprintf(stderr, "failed2 %x\n", (int)*p);
+			goto _failed;
 			SWITCH_RANGE_END
-			// end CS_HEADER
+				// end CS_HEADER
 
 
-		_fixed_trail_again:
-			++p;
+				_fixed_trail_again:
+				++p;
 
 		default:
 			if((size_t)(pe - p) < trail) { goto _out; }
 			n = p;  p += trail - 1;
 			switch(cs) {
-			//case CS_
-			//case CS_
+				//case CS_
+				//case CS_
 			case CS_FLOAT: {
-					union { uint32_t i; float f; } mem;
-					mem.i = _msgpack_load32(uint32_t,n);
-					push_fixed_value(_float, mem.f); }
+							   union { uint32_t i; float f; } mem;
+							   mem.i = _msgpack_load32(uint32_t,n);
+							   push_fixed_value(_float, mem.f); }
 			case CS_DOUBLE: {
-					union { uint64_t i; double f; } mem;
-					mem.i = _msgpack_load64(uint64_t,n);
+								union { uint64_t i; double f; } mem;
+								mem.i = _msgpack_load64(uint64_t,n);
 #if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
-                    // https://github.com/msgpack/msgpack-perl/pull/1
-                    mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
+								// https://github.com/msgpack/msgpack-perl/pull/1
+								mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
 #endif
-					push_fixed_value(_double, mem.f); }
+								push_fixed_value(_double, mem.f); }
 			case CS_UINT_8:
-				push_fixed_value(_uint8, *(uint8_t*)n);
+							push_fixed_value(_uint8, *(uint8_t*)n);
 			case CS_UINT_16:
-				push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
+							push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
 			case CS_UINT_32:
-				push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
+							push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
 			case CS_UINT_64:
-				push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
+							push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
 
 			case CS_INT_8:
-				push_fixed_value(_int8, *(int8_t*)n);
+							push_fixed_value(_int8, *(int8_t*)n);
 			case CS_INT_16:
-				push_fixed_value(_int16, _msgpack_load16(int16_t,n));
+							push_fixed_value(_int16, _msgpack_load16(int16_t,n));
 			case CS_INT_32:
-				push_fixed_value(_int32, _msgpack_load32(int32_t,n));
+							push_fixed_value(_int32, _msgpack_load32(int32_t,n));
 			case CS_INT_64:
-				push_fixed_value(_int64, _msgpack_load64(int64_t,n));
+							push_fixed_value(_int64, _msgpack_load64(int64_t,n));
 
-			//case CS_
-			//case CS_
-			//case CS_BIG_INT_16:
-			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero);
-			//case CS_BIG_INT_32:
-			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero);
-			//case ACS_BIG_INT_VALUE:
-			//_big_int_zero:
-			//	// FIXME
-			//	push_variable_value(_big_int, data, n, trail);
+							//case CS_
+							//case CS_
+							//case CS_BIG_INT_16:
+							//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero);
+							//case CS_BIG_INT_32:
+							//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero);
+							//case ACS_BIG_INT_VALUE:
+							//_big_int_zero:
+							//	// FIXME
+							//	push_variable_value(_big_int, data, n, trail);
 
-			//case CS_BIG_FLOAT_16:
-			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero);
-			//case CS_BIG_FLOAT_32:
-			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero);
-			//case ACS_BIG_FLOAT_VALUE:
-			//_big_float_zero:
-			//	// FIXME
-			//	push_variable_value(_big_float, data, n, trail);
+							//case CS_BIG_FLOAT_16:
+							//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero);
+							//case CS_BIG_FLOAT_32:
+							//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero);
+							//case ACS_BIG_FLOAT_VALUE:
+							//_big_float_zero:
+							//	// FIXME
+							//	push_variable_value(_big_float, data, n, trail);
 
 			case CS_RAW_16:
-				again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
+							again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
 			case CS_RAW_32:
-				again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
+							again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
 			case ACS_RAW_VALUE:
-			_raw_zero:
-				push_variable_value(_raw, data, n, trail);
+_raw_zero:
+							push_variable_value(_raw, data, n, trail);
 
 			case CS_ARRAY_16:
-				start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
+							start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
 			case CS_ARRAY_32:
-				/* FIXME security guard */
-				start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
+							/* FIXME security guard */
+							start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
 
 			case CS_MAP_16:
-				start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
+							start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
 			case CS_MAP_32:
-				/* FIXME security guard */
-				start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
+							/* FIXME security guard */
+							start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
 
 			default:
-				goto _failed;
+							goto _failed;
 			}
 		}
 
 _push:
-	if(top == 0) { goto _finish; }
-	c = &stack[top-1];
-	switch(c->ct) {
-	case CT_ARRAY_ITEM:
-		if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
-		if(++c->count == c->size) {
-			obj = c->obj;
-			if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; }
-			--top;
-			/*printf("stack pop %d\n", top);*/
-			goto _push;
-		}
-		goto _header_again;
-	case CT_MAP_KEY:
-		c->map_key = obj;
-		c->ct = CT_MAP_VALUE;
-		goto _header_again;
-	case CT_MAP_VALUE:
-		if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
-		if(++c->count == c->size) {
-			obj = c->obj;
-			if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; }
-			--top;
-			/*printf("stack pop %d\n", top);*/
-			goto _push;
-		}
-		c->ct = CT_MAP_KEY;
-		goto _header_again;
+		if(top == 0) { goto _finish; }
+		c = &stack[top-1];
+		switch(c->ct) {
+		case CT_ARRAY_ITEM:
+			if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
+			if(++c->count == c->size) {
+				obj = c->obj;
+				if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; }
+				--top;
+				/*printf("stack pop %d\n", top);*/
+				goto _push;
+			}
+			goto _header_again;
+		case CT_MAP_KEY:
+			c->map_key = obj;
+			c->ct = CT_MAP_VALUE;
+			goto _header_again;
+		case CT_MAP_VALUE:
+			if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
+			if(++c->count == c->size) {
+				obj = c->obj;
+				if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; }
+				--top;
+				/*printf("stack pop %d\n", top);*/
+				goto _push;
+			}
+			c->ct = CT_MAP_KEY;
+			goto _header_again;
 
-	default:
-		goto _failed;
-	}
+		default:
+			goto _failed;
+		}
 
 _header_again:
 		cs = CS_HEADER;
@@ -431,7 +442,7 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx
 
 #define inc_offset(inc) \
 		if (len - *off < inc) \
-			return 0; \
+		return 0; \
 		*off += inc;
 
 		switch (*p) {
@@ -466,8 +477,25 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx
 			++*off;
 			size = ((unsigned int)*p) & 0x0f;
 			break;
-		default:
+			// reserved:
+		case 0xc1:
+		case 0xc4:
+		case 0xc5:
+		case 0xc6:
+		case 0xc7:
+		case 0xc8:
+		case 0xc9:
+		case 0xd4:
+		case 0xd5:
+		case 0xd6:  // big integer 16
+		case 0xd7:  // big integer 32
+		case 0xd8:  // big float 16
+		case 0xd9:  // big float 32
+			++*off;
 			continue;
+		default:
+			PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
+			return -1;
 		}
 		msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj);
 		return 1;
diff --git a/test/test_reserved.py b/test/test_reserved.py
new file mode 100644
index 0000000..56b4455
--- /dev/null
+++ b/test/test_reserved.py
@@ -0,0 +1,8 @@
+import msgpack
+
+reserved_bytes = b"\xc1\xc4\xc5\xc6\xc7\xc8\xc9\xd4\xd5\xd6\xd7\xd8\xd9"
+
+def test_skip_reserved():
+    packed_list = msgpack.packb([])
+    for b in reserved_bytes:
+        assert msgpack.unpackb(b+packed_list, use_list=1) == []

From 43dd224d52eb4711c68101b40b910d252dacc290 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 21 Feb 2013 16:04:58 +0900
Subject: [PATCH 1154/1648] Fix test for Python 3.

---
 test/test_reserved.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/test/test_reserved.py b/test/test_reserved.py
index 56b4455..5094064 100644
--- a/test/test_reserved.py
+++ b/test/test_reserved.py
@@ -1,6 +1,20 @@
 import msgpack
 
-reserved_bytes = b"\xc1\xc4\xc5\xc6\xc7\xc8\xc9\xd4\xd5\xd6\xd7\xd8\xd9"
+reserved_bytes = [
+        b"\xc1",
+        b"\xc4",
+        b"\xc5",
+        b"\xc6",
+        b"\xc7",
+        b"\xc8",
+        b"\xc9",
+        b"\xd4",
+        b"\xd5",
+        b"\xd6",
+        b"\xd7",
+        b"\xd8",
+        b"\xd9",
+        ]
 
 def test_skip_reserved():
     packed_list = msgpack.packb([])

From d766820421c9c67972f899cf6333fbb00e595162 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 21 Feb 2013 16:55:42 +0900
Subject: [PATCH 1155/1648] Fix easy bug.

---
 msgpack/fallback.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 311f16b..8dae9a7 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -341,7 +341,7 @@ class Unpacker(object):
     def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
         typ = TYPE_RESERVED
         while typ == TYPE_RESERVED:
-            typ, n, obj = self._read_header(self, execute, write_bytes)
+            typ, n, obj = self._read_header(execute, write_bytes)
 
         if execute == EX_READ_ARRAY_HEADER:
             if typ != TYPE_ARRAY:

From f0fd90a759f07bccfd36e4f9eac8c4b545fd47a4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 22 Feb 2013 17:41:52 +0900
Subject: [PATCH 1156/1648] Fix exception incompatibility.

---
 msgpack/_unpacker.pyx | 4 +---
 msgpack/fallback.py   | 5 ++++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index b83a6e4..aeda02a 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -113,10 +113,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         if off < buf_len:
             raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
         return obj
-    elif ret < 0:
-        raise ValueError("Unpack failed: error = %d" % (ret,))
     else:
-        raise UnpackValueError
+        raise UnpackValueError("Unpack failed: error = %d" % (ret,))
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None,
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 8dae9a7..f893e1a 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -102,7 +102,10 @@ def unpackb(packed, object_hook=None, list_hook=None, use_list=True,
                         encoding=encoding, unicode_errors=unicode_errors,
                         object_pairs_hook=object_pairs_hook)
     unpacker.feed(packed)
-    ret = unpacker._fb_unpack()
+    try:
+        ret = unpacker._fb_unpack()
+    except OutOfData:
+        raise UnpackValueError("Data is not enough.")
     if unpacker._fb_got_extradata():
         raise ExtraData(ret, unpacker._fb_get_extradata())
     return ret

From 5c51203d14ba2420d98f7c9f85a21b3a7b89e1b9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 23 Feb 2013 18:00:54 +0900
Subject: [PATCH 1157/1648] Revert "Fix test for Python 3."

This reverts commit 43dd224d52eb4711c68101b40b910d252dacc290.
---
 test/test_reserved.py | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/test/test_reserved.py b/test/test_reserved.py
index 5094064..56b4455 100644
--- a/test/test_reserved.py
+++ b/test/test_reserved.py
@@ -1,20 +1,6 @@
 import msgpack
 
-reserved_bytes = [
-        b"\xc1",
-        b"\xc4",
-        b"\xc5",
-        b"\xc6",
-        b"\xc7",
-        b"\xc8",
-        b"\xc9",
-        b"\xd4",
-        b"\xd5",
-        b"\xd6",
-        b"\xd7",
-        b"\xd8",
-        b"\xd9",
-        ]
+reserved_bytes = b"\xc1\xc4\xc5\xc6\xc7\xc8\xc9\xd4\xd5\xd6\xd7\xd8\xd9"
 
 def test_skip_reserved():
     packed_list = msgpack.packb([])

From a6859791a2048f5b2aae1ece03a603af63d69633 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 23 Feb 2013 18:01:43 +0900
Subject: [PATCH 1158/1648] Revert "Skip reserved byte"

This reverts commit ff3342aeedb9d21897b0b7275dd68d781f149995.
---
 msgpack/unpack_template.h | 298 +++++++++++++++++---------------------
 test/test_reserved.py     |   8 -
 2 files changed, 135 insertions(+), 171 deletions(-)
 delete mode 100644 test/test_reserved.py

diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 6f86695..8fd179d 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -60,10 +60,10 @@ msgpack_unpack_struct_decl(_context) {
 	unsigned int trail;
 	unsigned int top;
 	/*
-	   msgpack_unpack_struct(_stack)* stack;
-	   unsigned int stack_size;
-	   msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE];
-	   */
+	msgpack_unpack_struct(_stack)* stack;
+	unsigned int stack_size;
+	msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE];
+	*/
 	msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE];
 };
 
@@ -74,20 +74,20 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx)
 	ctx->trail = 0;
 	ctx->top = 0;
 	/*
-	   ctx->stack = ctx->embed_stack;
-	   ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
-	   */
+	ctx->stack = ctx->embed_stack;
+	ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
+	*/
 	ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user);
 }
 
 /*
-   msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx)
-   {
-   if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
-   free(ctx->stack);
-   }
-   }
-   */
+msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx)
+{
+	if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
+		free(ctx->stack);
+	}
+}
+*/
 
 msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx)
 {
@@ -109,8 +109,8 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	unsigned int top = ctx->top;
 	msgpack_unpack_struct(_stack)* stack = ctx->stack;
 	/*
-	   unsigned int stack_size = ctx->stack_size;
-	   */
+	unsigned int stack_size = ctx->stack_size;
+	*/
 	msgpack_unpack_user* user = &ctx->user;
 
 	msgpack_unpack_object obj;
@@ -119,7 +119,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	int ret;
 
 #define construct_cb(name) \
-	construct && msgpack_unpack_callback(name)
+    construct && msgpack_unpack_callback(name)
 
 #define push_simple_value(func) \
 	if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \
@@ -129,7 +129,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	goto _push
 #define push_variable_value(func, base, pos, len) \
 	if(construct_cb(func)(user, \
-				(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
+		(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
 	goto _push
 
 #define again_fixed_trail(_cs, trail_len) \
@@ -155,24 +155,24 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	/*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
 	/*printf("stack push %d\n", top);*/ \
 	/* FIXME \
-	   if(top >= stack_size) { \
-	   if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
-	   size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \
-	   size_t nsize = csize * 2; \
-	   msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \
-	   if(tmp == NULL) { goto _failed; } \
-	   memcpy(tmp, ctx->stack, csize); \
-	   ctx->stack = stack = tmp; \
-	   ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
-	   } else { \
-	   size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \
-	   msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \
-	   if(tmp == NULL) { goto _failed; } \
-	   ctx->stack = stack = tmp; \
-	   ctx->stack_size = stack_size = stack_size * 2; \
-	   } \
-	   } \
-	   */ \
+	if(top >= stack_size) { \
+		if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
+			size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \
+			size_t nsize = csize * 2; \
+			msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \
+			if(tmp == NULL) { goto _failed; } \
+			memcpy(tmp, ctx->stack, csize); \
+			ctx->stack = stack = tmp; \
+			ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
+		} else { \
+			size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \
+			msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \
+			if(tmp == NULL) { goto _failed; } \
+			ctx->stack = stack = tmp; \
+			ctx->stack_size = stack_size = stack_size * 2; \
+		} \
+	} \
+	*/ \
 	goto _header_again
 
 #define NEXT_CS(p) \
@@ -195,7 +195,7 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 		switch(cs) {
 		case CS_HEADER:
 			SWITCH_RANGE_BEGIN
-				SWITCH_RANGE(0x00, 0x7f)  // Positive Fixnum
+			SWITCH_RANGE(0x00, 0x7f)  // Positive Fixnum
 				push_fixed_value(_uint8, *(uint8_t*)p);
 			SWITCH_RANGE(0xe0, 0xff)  // Negative Fixnum
 				push_fixed_value(_int8, *(int8_t*)p);
@@ -203,24 +203,18 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				switch(*p) {
 				case 0xc0:  // nil
 					push_simple_value(_nil);
-					// reserved.
-				case 0xc1:  // string
-					fprintf(stderr, "skipping %x\n", (int)*p);
-					goto _header_again;
-					//	again_terminal_trail(NEXT_CS(p), p+1);
+				//case 0xc1:  // string
+				//	again_terminal_trail(NEXT_CS(p), p+1);
 				case 0xc2:  // false
 					push_simple_value(_false);
 				case 0xc3:  // true
 					push_simple_value(_true);
-					// reserved.
-				case 0xc4:
-				case 0xc5:
-				case 0xc6:
-				case 0xc7:
-				case 0xc8:
-				case 0xc9:
-					fprintf(stderr, "skipping %x\n", (int)*p);
-					goto _header_again;
+				//case 0xc4:
+				//case 0xc5:
+				//case 0xc6:
+				//case 0xc7:
+				//case 0xc8:
+				//case 0xc9:
 				case 0xca:  // float
 				case 0xcb:  // double
 				case 0xcc:  // unsigned int  8
@@ -232,15 +226,12 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				case 0xd2:  // signed int 32
 				case 0xd3:  // signed int 64
 					again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
-					// reserved.
-				case 0xd4:
-				case 0xd5:
-				case 0xd6:  // big integer 16
-				case 0xd7:  // big integer 32
-				case 0xd8:  // big float 16
-				case 0xd9:  // big float 32
-					fprintf(stderr, "skipping %x\n", (int)*p);
-					goto _header_again;
+				//case 0xd4:
+				//case 0xd5:
+				//case 0xd6:  // big integer 16
+				//case 0xd7:  // big integer 32
+				//case 0xd8:  // big float 16
+				//case 0xd9:  // big float 32
 				case 0xda:  // raw 16
 				case 0xdb:  // raw 32
 				case 0xdc:  // array 16
@@ -249,7 +240,6 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				case 0xdf:  // map 32
 					again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
 				default:
-					fprintf(stderr, "failed %x\n", (int)*p);
 					goto _failed;
 				}
 			SWITCH_RANGE(0xa0, 0xbf)  // FixRaw
@@ -260,129 +250,128 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 				start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
 
 			SWITCH_RANGE_DEFAULT
-				fprintf(stderr, "failed2 %x\n", (int)*p);
-			goto _failed;
+				goto _failed;
 			SWITCH_RANGE_END
-				// end CS_HEADER
+			// end CS_HEADER
 
 
-				_fixed_trail_again:
-				++p;
+		_fixed_trail_again:
+			++p;
 
 		default:
 			if((size_t)(pe - p) < trail) { goto _out; }
 			n = p;  p += trail - 1;
 			switch(cs) {
-				//case CS_
-				//case CS_
+			//case CS_
+			//case CS_
 			case CS_FLOAT: {
-							   union { uint32_t i; float f; } mem;
-							   mem.i = _msgpack_load32(uint32_t,n);
-							   push_fixed_value(_float, mem.f); }
+					union { uint32_t i; float f; } mem;
+					mem.i = _msgpack_load32(uint32_t,n);
+					push_fixed_value(_float, mem.f); }
 			case CS_DOUBLE: {
-								union { uint64_t i; double f; } mem;
-								mem.i = _msgpack_load64(uint64_t,n);
+					union { uint64_t i; double f; } mem;
+					mem.i = _msgpack_load64(uint64_t,n);
 #if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
-								// https://github.com/msgpack/msgpack-perl/pull/1
-								mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
+                    // https://github.com/msgpack/msgpack-perl/pull/1
+                    mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
 #endif
-								push_fixed_value(_double, mem.f); }
+					push_fixed_value(_double, mem.f); }
 			case CS_UINT_8:
-							push_fixed_value(_uint8, *(uint8_t*)n);
+				push_fixed_value(_uint8, *(uint8_t*)n);
 			case CS_UINT_16:
-							push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
+				push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
 			case CS_UINT_32:
-							push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
+				push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
 			case CS_UINT_64:
-							push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
+				push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
 
 			case CS_INT_8:
-							push_fixed_value(_int8, *(int8_t*)n);
+				push_fixed_value(_int8, *(int8_t*)n);
 			case CS_INT_16:
-							push_fixed_value(_int16, _msgpack_load16(int16_t,n));
+				push_fixed_value(_int16, _msgpack_load16(int16_t,n));
 			case CS_INT_32:
-							push_fixed_value(_int32, _msgpack_load32(int32_t,n));
+				push_fixed_value(_int32, _msgpack_load32(int32_t,n));
 			case CS_INT_64:
-							push_fixed_value(_int64, _msgpack_load64(int64_t,n));
+				push_fixed_value(_int64, _msgpack_load64(int64_t,n));
 
-							//case CS_
-							//case CS_
-							//case CS_BIG_INT_16:
-							//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero);
-							//case CS_BIG_INT_32:
-							//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero);
-							//case ACS_BIG_INT_VALUE:
-							//_big_int_zero:
-							//	// FIXME
-							//	push_variable_value(_big_int, data, n, trail);
+			//case CS_
+			//case CS_
+			//case CS_BIG_INT_16:
+			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero);
+			//case CS_BIG_INT_32:
+			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero);
+			//case ACS_BIG_INT_VALUE:
+			//_big_int_zero:
+			//	// FIXME
+			//	push_variable_value(_big_int, data, n, trail);
 
-							//case CS_BIG_FLOAT_16:
-							//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero);
-							//case CS_BIG_FLOAT_32:
-							//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero);
-							//case ACS_BIG_FLOAT_VALUE:
-							//_big_float_zero:
-							//	// FIXME
-							//	push_variable_value(_big_float, data, n, trail);
+			//case CS_BIG_FLOAT_16:
+			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero);
+			//case CS_BIG_FLOAT_32:
+			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero);
+			//case ACS_BIG_FLOAT_VALUE:
+			//_big_float_zero:
+			//	// FIXME
+			//	push_variable_value(_big_float, data, n, trail);
 
 			case CS_RAW_16:
-							again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
+				again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
 			case CS_RAW_32:
-							again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
+				again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
 			case ACS_RAW_VALUE:
-_raw_zero:
-							push_variable_value(_raw, data, n, trail);
+			_raw_zero:
+				push_variable_value(_raw, data, n, trail);
 
 			case CS_ARRAY_16:
-							start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
+				start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
 			case CS_ARRAY_32:
-							/* FIXME security guard */
-							start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
+				/* FIXME security guard */
+				start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
 
 			case CS_MAP_16:
-							start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
+				start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
 			case CS_MAP_32:
-							/* FIXME security guard */
-							start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
+				/* FIXME security guard */
+				start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
 
 			default:
-							goto _failed;
+				goto _failed;
 			}
 		}
 
 _push:
-		if(top == 0) { goto _finish; }
-		c = &stack[top-1];
-		switch(c->ct) {
-		case CT_ARRAY_ITEM:
-			if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
-			if(++c->count == c->size) {
-				obj = c->obj;
-				if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; }
-				--top;
-				/*printf("stack pop %d\n", top);*/
-				goto _push;
-			}
-			goto _header_again;
-		case CT_MAP_KEY:
-			c->map_key = obj;
-			c->ct = CT_MAP_VALUE;
-			goto _header_again;
-		case CT_MAP_VALUE:
-			if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
-			if(++c->count == c->size) {
-				obj = c->obj;
-				if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; }
-				--top;
-				/*printf("stack pop %d\n", top);*/
-				goto _push;
-			}
-			c->ct = CT_MAP_KEY;
-			goto _header_again;
-
-		default:
-			goto _failed;
+	if(top == 0) { goto _finish; }
+	c = &stack[top-1];
+	switch(c->ct) {
+	case CT_ARRAY_ITEM:
+		if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
+		if(++c->count == c->size) {
+			obj = c->obj;
+			if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; }
+			--top;
+			/*printf("stack pop %d\n", top);*/
+			goto _push;
 		}
+		goto _header_again;
+	case CT_MAP_KEY:
+		c->map_key = obj;
+		c->ct = CT_MAP_VALUE;
+		goto _header_again;
+	case CT_MAP_VALUE:
+		if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
+		if(++c->count == c->size) {
+			obj = c->obj;
+			if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; }
+			--top;
+			/*printf("stack pop %d\n", top);*/
+			goto _push;
+		}
+		c->ct = CT_MAP_KEY;
+		goto _header_again;
+
+	default:
+		goto _failed;
+	}
 
 _header_again:
 		cs = CS_HEADER;
@@ -442,7 +431,7 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx
 
 #define inc_offset(inc) \
 		if (len - *off < inc) \
-		return 0; \
+			return 0; \
 		*off += inc;
 
 		switch (*p) {
@@ -477,25 +466,8 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx
 			++*off;
 			size = ((unsigned int)*p) & 0x0f;
 			break;
-			// reserved:
-		case 0xc1:
-		case 0xc4:
-		case 0xc5:
-		case 0xc6:
-		case 0xc7:
-		case 0xc8:
-		case 0xc9:
-		case 0xd4:
-		case 0xd5:
-		case 0xd6:  // big integer 16
-		case 0xd7:  // big integer 32
-		case 0xd8:  // big float 16
-		case 0xd9:  // big float 32
-			++*off;
-			continue;
 		default:
-			PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
-			return -1;
+			continue;
 		}
 		msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj);
 		return 1;
diff --git a/test/test_reserved.py b/test/test_reserved.py
deleted file mode 100644
index 56b4455..0000000
--- a/test/test_reserved.py
+++ /dev/null
@@ -1,8 +0,0 @@
-import msgpack
-
-reserved_bytes = b"\xc1\xc4\xc5\xc6\xc7\xc8\xc9\xd4\xd5\xd6\xd7\xd8\xd9"
-
-def test_skip_reserved():
-    packed_list = msgpack.packb([])
-    for b in reserved_bytes:
-        assert msgpack.unpackb(b+packed_list, use_list=1) == []

From 38a9ad98c9e8e8b73efb9726554737a60ec215c9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 23 Feb 2013 18:11:46 +0900
Subject: [PATCH 1159/1648] Revert skipping reserved byte.

---
 msgpack/fallback.py       |  9 ++---
 msgpack/unpack_template.h | 85 +++++++++++++++++++--------------------
 2 files changed, 44 insertions(+), 50 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index f893e1a..e9dddd6 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -48,7 +48,7 @@ from msgpack.exceptions import (
         PackValueError,
         ExtraData)
 
-EX_SKIP                 = 0 
+EX_SKIP                 = 0
 EX_CONSTRUCT            = 1
 EX_READ_ARRAY_HEADER    = 2
 EX_READ_MAP_HEADER      = 3
@@ -57,7 +57,6 @@ TYPE_IMMEDIATE          = 0
 TYPE_ARRAY              = 1
 TYPE_MAP                = 2
 TYPE_RAW                = 3
-TYPE_RESERVED           = 4
 
 DEFAULT_RECURSE_LIMIT=511
 
@@ -338,13 +337,11 @@ class Unpacker(object):
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             typ = TYPE_MAP
         else:
-            typ = TYPE_RESERVED
+            raise UnpackValueError("Unknown header: 0x%x" % b)
         return typ, n, obj
 
     def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
-        typ = TYPE_RESERVED
-        while typ == TYPE_RESERVED:
-            typ, n, obj = self._read_header(execute, write_bytes)
+        typ, n, obj = self._read_header(execute, write_bytes)
 
         if execute == EX_READ_ARRAY_HEADER:
             if typ != TYPE_ARRAY:
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 8fd179d..83b6918 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -422,56 +422,53 @@ _end:
 template 
 msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
 {
-	for (;;) {
-		if (len < *off) {
-			return 0;
-		}
-		uint32_t size;
-		const unsigned char *const p = (unsigned char*)data + *off;
+	assert(len >= *off);
+	uint32_t size;
+	const unsigned char *const p = (unsigned char*)data + *off;
 
 #define inc_offset(inc) \
-		if (len - *off < inc) \
-			return 0; \
-		*off += inc;
+	if (len - *off < inc) \
+		return 0; \
+	*off += inc;
 
-		switch (*p) {
-		case var_offset:
-			inc_offset(3);
-			size = _msgpack_load16(uint16_t, p + 1);
-			break;
-		case var_offset + 1:
-			inc_offset(5);
-			size = _msgpack_load32(uint32_t, p + 1);
-			break;
+	switch (*p) {
+	case var_offset:
+		inc_offset(3);
+		size = _msgpack_load16(uint16_t, p + 1);
+		break;
+	case var_offset + 1:
+		inc_offset(5);
+		size = _msgpack_load32(uint32_t, p + 1);
+		break;
 #ifdef USE_CASE_RANGE
-		case fixed_offset + 0x0 ... fixed_offset + 0xf:
+	case fixed_offset + 0x0 ... fixed_offset + 0xf:
 #else
-		case fixed_offset + 0x0:
-		case fixed_offset + 0x1:
-		case fixed_offset + 0x2:
-		case fixed_offset + 0x3:
-		case fixed_offset + 0x4:
-		case fixed_offset + 0x5:
-		case fixed_offset + 0x6:
-		case fixed_offset + 0x7:
-		case fixed_offset + 0x8:
-		case fixed_offset + 0x9:
-		case fixed_offset + 0xa:
-		case fixed_offset + 0xb:
-		case fixed_offset + 0xc:
-		case fixed_offset + 0xd:
-		case fixed_offset + 0xe:
-		case fixed_offset + 0xf:
+	case fixed_offset + 0x0:
+	case fixed_offset + 0x1:
+	case fixed_offset + 0x2:
+	case fixed_offset + 0x3:
+	case fixed_offset + 0x4:
+	case fixed_offset + 0x5:
+	case fixed_offset + 0x6:
+	case fixed_offset + 0x7:
+	case fixed_offset + 0x8:
+	case fixed_offset + 0x9:
+	case fixed_offset + 0xa:
+	case fixed_offset + 0xb:
+	case fixed_offset + 0xc:
+	case fixed_offset + 0xd:
+	case fixed_offset + 0xe:
+	case fixed_offset + 0xf:
 #endif
-			++*off;
-			size = ((unsigned int)*p) & 0x0f;
-			break;
-		default:
-			continue;
-		}
-		msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj);
-		return 1;
-	}
+		++*off;
+		size = ((unsigned int)*p) & 0x0f;
+		break;
+	default:
+		PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
+		return -1;
+    }
+	msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj);
+	return 1;
 }
 
 #undef SWITCH_RANGE_BEGIN

From f4cb6fb87749491bce5e3e6b2d7bfafdf7764435 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 24 Feb 2013 18:06:36 +0900
Subject: [PATCH 1160/1648] Add apidoc

---
 docs/Makefile  | 156 +++++++++++++++++++++++++++
 docs/api.rst   |  38 +++++++
 docs/conf.py   | 285 +++++++++++++++++++++++++++++++++++++++++++++++++
 docs/index.rst |  11 ++
 4 files changed, 490 insertions(+)
 create mode 100644 docs/Makefile
 create mode 100644 docs/api.rst
 create mode 100644 docs/conf.py
 create mode 100644 docs/index.rst

diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..10d4d4b
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,156 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make ' where  is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	-rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/msgpack.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/msgpack.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/msgpack"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/msgpack"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+serve: html
+	cd _build/html && python3.3 -m http.server
diff --git a/docs/api.rst b/docs/api.rst
new file mode 100644
index 0000000..a26e7c7
--- /dev/null
+++ b/docs/api.rst
@@ -0,0 +1,38 @@
+API reference
+===============
+
+.. module:: msgpack
+
+.. autofunction:: pack
+
+:func:`dump` is alias for :func:`pack`
+
+.. autofunction:: packb
+
+:func:`dumps` is alias for :func:`packb`
+
+.. autofunction:: unpack
+
+:func:`load` is alias for :func:`unpack`
+
+.. autofunction:: unpackb
+
+:func:`loads` is alias for :func:`unpackb`
+
+.. autoclass:: Packer
+    :members:
+
+.. autoclass:: Unpacker
+    :members:
+
+exceptions
+-----------
+
+These exceptions are accessible via `msgpack` package.
+(For example, `msgpack.OutOfData` is shortcut for `msgpack.exceptions.OutOfData`)
+
+.. automodule:: msgpack.exceptions
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 0000000..0821ad4
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,285 @@
+# -*- coding: utf-8 -*-
+#
+# msgpack documentation build configuration file, created by
+# sphinx-quickstart on Sun Feb 24 14:20:50 2013.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'msgpack'
+copyright = u'2013, Author'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = ''
+# The full version, including alpha/beta/rc tags.
+release = ''
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinxdoc'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# " v documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a  tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'msgpackdoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+  ('index', 'msgpack.tex', u'msgpack Documentation',
+   u'Author', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'msgpack', u'msgpack Documentation',
+     [u'Author'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'msgpack', u'msgpack Documentation',
+   u'Author', 'msgpack', 'One line description of project.',
+   'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+
+# -- Options for Epub output ---------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = u'msgpack'
+epub_author = u'Author'
+epub_publisher = u'Author'
+epub_copyright = u'2013, Author'
+
+# The language of the text. It defaults to the language option
+# or en if the language is not set.
+#epub_language = ''
+
+# The scheme of the identifier. Typical schemes are ISBN or URL.
+#epub_scheme = ''
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#epub_identifier = ''
+
+# A unique identification for the text.
+#epub_uid = ''
+
+# A tuple containing the cover image and cover page html template filenames.
+#epub_cover = ()
+
+# HTML files that should be inserted before the pages created by sphinx.
+# The format is a list of tuples containing the path and title.
+#epub_pre_files = []
+
+# HTML files shat should be inserted after the pages created by sphinx.
+# The format is a list of tuples containing the path and title.
+#epub_post_files = []
+
+# A list of files that should not be packed into the epub file.
+#epub_exclude_files = []
+
+# The depth of the table of contents in toc.ncx.
+#epub_tocdepth = 3
+
+# Allow duplicate toc entries.
+#epub_tocdup = True
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 0000000..5dcff6f
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,11 @@
+msgpack document
+==================
+
+`MessagePack `_ is a efficient format for inter
+language data exchange.
+
+.. toctree::
+   :maxdepth: 1
+
+   ../README
+   api

From 7991530cecf061a50792bcb423153c6a150bc9e6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 24 Feb 2013 18:06:50 +0900
Subject: [PATCH 1161/1648] Update README

---
 README.rst | 77 ++++++++++++++++++++++++++++--------------------------
 1 file changed, 40 insertions(+), 37 deletions(-)

diff --git a/README.rst b/README.rst
index f2e5962..da2f023 100644
--- a/README.rst
+++ b/README.rst
@@ -1,6 +1,6 @@
-===========================
-MessagePack Python Binding
-===========================
+=======================
+MessagePack for Python
+=======================
 
 :author: INADA Naoki
 :version: 0.3.0
@@ -9,19 +9,42 @@ MessagePack Python Binding
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
    :target: https://travis-ci.org/#!/msgpack/msgpack-python
 
-WHAT IT IS
-----------
+What's this
+------------
 
 `MessagePack `_ is a fast, compact binary serialization format, suitable for
 similar data to JSON. This package provides CPython bindings for reading and
 writing MessagePack data.
 
-NOTE for msgpack 0.2.x users
+Install
+---------
+You can use ``pip`` or ``easy_install`` to install msgpack::
+
+   $ easy_install msgpack-python
+     or
+   $ pip install msgpack-python
+
+PyPy
+^^^^^
+
+msgpack-python provides pure python implementation.
+PyPy can use this.
+
+Windows
+^^^^^^^
+
+When you can't use binary distribution, you need to install Visual Studio
+or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support
+amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.)
+
+Without extension, using pure python implementation on CPython runs slowly.
+
+Note for msgpack 0.2.x users
 ----------------------------
 
 The msgpack 0.3 have some incompatible changes.
 
-The default value of ``use_list`` keyword argument is ``True`` from 0.3.x.
+The default value of ``use_list`` keyword argument is ``True`` from 0.3.
 You should pass the argument explicitly for backward compatibility.
 
 `Unpacker.unpack()` and some unpack methods now raises `OutOfData`
@@ -29,10 +52,10 @@ instead of `StopIteration`.
 `StopIteration` is used for iterator protocol only.
 
 
-HOW TO USE
+How to use
 -----------
 
-one-shot pack & unpack
+One-shot pack & unpack
 ^^^^^^^^^^^^^^^^^^^^^^
 
 Use ``packb`` for packing and ``unpackb`` for unpacking.
@@ -60,7 +83,7 @@ You should always pass the ``use_list`` keyword argument. See performance issues
 Read the docstring for other options.
 
 
-streaming unpacking
+Streaming unpacking
 ^^^^^^^^^^^^^^^^^^^
 
 ``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one
@@ -82,7 +105,7 @@ stream (or from bytes provided through its ``feed`` method).
        print unpacked
 
 
-packing/unpacking of custom data type
+Packing/unpacking of custom data type
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 It is also possible to pack/unpack custom data types. Here is an example for
@@ -118,7 +141,7 @@ It is also possible to pack/unpack custom data types. Here is an example for
 key-value pairs.
 
 
-advanced unpacking control
+Advanced unpacking control
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 As an alternative to iteration, ``Unpacker`` objects provide ``unpack``,
@@ -150,27 +173,8 @@ callback function:
             unpacker.skip(bytestream.write)
             worker.send(bytestream.getvalue())
 
-INSTALL
----------
-You can use ``pip`` or ``easy_install`` to install msgpack::
-
-   $ easy_install msgpack-python
-     or
-   $ pip install msgpack-python
-
-
-Windows
-^^^^^^^
-msgpack provides some binary distribution for Windows.
-You can install msgpack without compiler with them.
-
-When you can't use binary distribution, you need to install Visual Studio
-or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support
-amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.)
-
-
-PERFORMANCE NOTE
------------------
+Note about performance
+------------------------
 
 GC
 ^^
@@ -179,8 +183,8 @@ CPython's GC starts when growing allocated object.
 This means unpacking may cause useless GC.
 You can use ``gc.disable()`` when unpacking large message.
 
-use_list
-^^^^^^^^^
+`use_list` option
+^^^^^^^^^^^^^^^^^^
 List is the default sequence type of Python.
 But tuple is lighter than list.
 You can use ``use_list=False`` while unpacking when performance is important.
@@ -190,13 +194,12 @@ Python's dict can't use list as key and MessagePack allows array for key of mapp
 Another way to unpacking such object is using ``object_pairs_hook``.
 
 
-TEST
+Test
 ----
 MessagePack uses `pytest` for testing.
 Run test with following command:
 
     $ py.test
 
-
 ..
     vim: filetype=rst

From 230537cf287e5b6985f1fd06ebb01a45776065e6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 25 Feb 2013 14:29:49 +0900
Subject: [PATCH 1162/1648] Update docs.

---
 docs/api.rst   | 2 +-
 docs/conf.py   | 6 +++---
 docs/index.rst | 1 -
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/docs/api.rst b/docs/api.rst
index a26e7c7..7efc04a 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -1,5 +1,5 @@
 API reference
-===============
+=============
 
 .. module:: msgpack
 
diff --git a/docs/conf.py b/docs/conf.py
index 0821ad4..fba09b7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -41,16 +41,15 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'msgpack'
-copyright = u'2013, Author'
+copyright = u'2013, INADA Naoki'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = ''
 # The full version, including alpha/beta/rc tags.
-release = ''
+version = release = '0.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -61,6 +60,7 @@ release = ''
 #today = ''
 # Else, today_fmt is used as the format for a strftime call.
 #today_fmt = '%B %d, %Y'
+today_fmt = "%Y-%m-%d"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
diff --git a/docs/index.rst b/docs/index.rst
index 5dcff6f..72d4499 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -7,5 +7,4 @@ language data exchange.
 .. toctree::
    :maxdepth: 1
 
-   ../README
    api

From 1e38bfa1235896f2df5d34211166276b48afa4b9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 25 Feb 2013 18:23:42 +0900
Subject: [PATCH 1163/1648] fallback: refactor

---
 msgpack/fallback.py | 58 ++++++++++++++++++++++-----------------------
 1 file changed, 28 insertions(+), 30 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index e9dddd6..28ce036 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -60,46 +60,44 @@ TYPE_RAW                = 3
 
 DEFAULT_RECURSE_LIMIT=511
 
-def pack(o, stream, default=None, encoding='utf-8', unicode_errors='strict'):
-    """ Pack object `o` and write it to `stream` """
-    packer = Packer(default=default, encoding=encoding,
-                    unicode_errors=unicode_errors)
+def pack(o, stream, **kwargs):
+    """
+    Pack object `o` and write it to `stream`
+
+    See :class:`Packer` for options.
+    """
+    packer = Packer(**kwargs)
     stream.write(packer.pack(o))
 
-def packb(o, default=None, encoding='utf-8', unicode_errors='struct',
-          use_single_float=False):
-    """ Pack object `o` and return packed bytes """
-    packer = Packer(default=default,
-                    encoding=encoding,
-                    unicode_errors=unicode_errors,
-                    use_single_float=use_single_float)
-    return packer.pack(o)
+def packb(o, **kwargs):
+    """
+    Pack object `o` and return packed bytes
 
-def unpack(stream, object_hook=None, list_hook=None, use_list=True,
-           encoding=None, unicode_errors='strict',
-           object_pairs_hook=None):
-    """ Unpack an object from `stream`.
+    See :class:`Packer` for options.
+    """
+    return Packer(**kwargs).pack(o)
 
-    Raises `ExtraData` when `stream` has extra bytes. """
-    unpacker = Unpacker(stream, object_hook=object_hook, list_hook=list_hook,
-                        use_list=use_list,
-                        encoding=encoding, unicode_errors=unicode_errors,
-                        object_pairs_hook=object_pairs_hook)
+def unpack(stream, **kwargs):
+    """
+    Unpack an object from `stream`.
+
+    Raises `ExtraData` when `packed` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    unpacker = Unpacker(stream, **kwargs)
     ret = unpacker._fb_unpack()
     if unpacker._fb_got_extradata():
         raise ExtraData(ret, unpacker._fb_get_extradata())
     return ret
 
-def unpackb(packed, object_hook=None, list_hook=None, use_list=True,
-            encoding=None, unicode_errors='strict',
-            object_pairs_hook=None):
-    """ Unpack an object from `packed`.
+def unpackb(packed, **kwargs):
+    """
+    Unpack an object from `packed`.
 
-    Raises `ExtraData` when `packed` contains extra bytes. """
-    unpacker = Unpacker(None, object_hook=object_hook, list_hook=list_hook,
-                        use_list=use_list,
-                        encoding=encoding, unicode_errors=unicode_errors,
-                        object_pairs_hook=object_pairs_hook)
+    Raises `ExtraData` when `packed` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    unpacker = Unpacker(None, **kwargs)
     unpacker.feed(packed)
     try:
         ret = unpacker._fb_unpack()

From 3ce005cf377eaeb5a517ce82698e4ae4381e67b9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Feb 2013 09:20:44 +0900
Subject: [PATCH 1164/1648] better packer docstring

---
 Makefile            | 12 ++++++++----
 docs/Makefile       |  2 +-
 docs/api.rst        |  5 +++++
 msgpack/_packer.pyx | 42 ++++++++++++++++++++++++++++--------------
 msgpack/fallback.py | 24 ++++++++++++++++++++++++
 5 files changed, 66 insertions(+), 19 deletions(-)

diff --git a/Makefile b/Makefile
index 875a3f6..da17b81 100644
--- a/Makefile
+++ b/Makefile
@@ -1,12 +1,16 @@
 .PHONY: test all python3
 
-all:
+all: cython
 	python setup.py build_ext -i -f
-	python setup.py build sdist
 
-python3:
+doc-serve: all
+	cd docs && make serve
+
+cython:
+	cython msgpack/*.pyx
+
+python3: cython
 	python3 setup.py build_ext -i -f
-	python3 setup.py build sdist
 
 test:
 	py.test test
diff --git a/docs/Makefile b/docs/Makefile
index 10d4d4b..427a980 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -10,7 +10,7 @@ BUILDDIR      = _build
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+ALLSPHINXOPTS   = -E -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 
diff --git a/docs/api.rst b/docs/api.rst
index 7efc04a..ccc7952 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -36,3 +36,8 @@ These exceptions are accessible via `msgpack` package.
     :undoc-members:
     :show-inheritance:
 
+.. automodule:: msgpack.fallback
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index a5bc570..562c92c 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -39,9 +39,10 @@ cdef int DEFAULT_RECURSE_LIMIT=511
 
 
 cdef class Packer(object):
-    """MessagePack Packer
+    """
+    MessagePack Packer
 
-    usage:
+    usage::
 
         packer = Packer()
         astream.write(packer.pack(a))
@@ -49,13 +50,18 @@ cdef class Packer(object):
 
     Packer's constructor has some keyword arguments:
 
-    * *defaut* - Convert user type to builtin type that Packer supports.
-      See also simplejson's document.
-    * *encoding* - Convert unicode to bytes with this encoding. (default: 'utf-8')
-    * *unicode_erros* - Error handler for encoding unicode. (default: 'strict')
-    * *use_single_float* - Use single precision float type for float. (default: False)
-    * *autoreset* - Reset buffer after each pack and return it's content as `bytes`. (default: True).
-      If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+    :param callable default:
+        Convert user type to builtin type that Packer supports.
+        See also simplejson's document.
+    :param str encoding:
+        Convert unicode to bytes with this encoding. (default: 'utf-8')
+    :param str unicode_erros:
+        Error handler for encoding unicode. (default: 'strict')
+    :param bool use_single_float:
+        Use single precision float type for float. (default: False)
+    :param bool autoreset:
+        Reset buffer after each pack and return it's content as `bytes`. (default: True).
+        If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
     """
     cdef msgpack_packer pk
     cdef object _default
@@ -75,6 +81,8 @@ cdef class Packer(object):
         self.pk.length = 0
 
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1):
+        """
+        """
         self.use_float = use_single_float
         self.autoreset = autoreset
         if default is not None:
@@ -218,7 +226,7 @@ cdef class Packer(object):
         Pack *pairs* as msgpack map type.
 
         *pairs* should sequence of pair.
-        (`len(pairs)` and `for k, v in *pairs*:` should be supported.)
+        (`len(pairs)` and `for k, v in pairs:` should be supported.)
         """
         cdef int ret = msgpack_pack_map(&self.pk, len(pairs))
         if ret == 0:
@@ -245,15 +253,21 @@ cdef class Packer(object):
         return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
 
 
-def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'):
+def pack(object o, object stream, default=None, str encoding='utf-8', str unicode_errors='strict'):
+    """
+    pack an object `o` and write it to stream)
+
+    See :class:`Packer` for options.
     """
-    pack an object `o` and write it to stream)."""
     packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
     stream.write(packer.pack(o))
 
-def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False):
+def packb(object o, default=None, encoding='utf-8', str unicode_errors='strict', bint use_single_float=False):
+    """
+    pack o and return packed bytes
+
+    See :class:`Packer` for options.
     """
-    pack o and return packed bytes."""
     packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
                     use_single_float=use_single_float)
     return packer.pack(o)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 28ce036..f9a2f5e 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -423,6 +423,30 @@ class Unpacker(object):
 
 
 class Packer(object):
+    """
+    MessagePack Packer
+
+    usage:
+
+        packer = Packer()
+        astream.write(packer.pack(a))
+        astream.write(packer.pack(b))
+
+    Packer's constructor has some keyword arguments:
+
+    :param callable default:
+        Convert user type to builtin type that Packer supports.
+        See also simplejson's document.
+    :param str encoding:
+            Convert unicode to bytes with this encoding. (default: 'utf-8')
+    :param str unicode_erros:
+        Error handler for encoding unicode. (default: 'strict')
+    :param bool use_single_float:
+        Use single precision float type for float. (default: False)
+    :param bool autoreset:
+        Reset buffer after each pack and return it's content as `bytes`. (default: True).
+        If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+    """
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
                  use_single_float=False, autoreset=True):
         self._use_float = use_single_float

From 8e13598a36cbf56992454a04e1c4e92e4cfb15b8 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Feb 2013 09:49:25 +0900
Subject: [PATCH 1165/1648] docs: better unpacker docstring.

---
 Makefile              |  3 ++
 docs/Makefile         |  3 ++
 msgpack/_unpacker.pyx | 64 +++++++++++++++++++++++++++----------------
 3 files changed, 46 insertions(+), 24 deletions(-)

diff --git a/Makefile b/Makefile
index da17b81..3fe278e 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,9 @@ all: cython
 doc-serve: all
 	cd docs && make serve
 
+doc:
+	cd docs && make zip
+
 cython:
 	cython msgpack/*.pyx
 
diff --git a/docs/Makefile b/docs/Makefile
index 427a980..0869604 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -154,3 +154,6 @@ doctest:
 
 serve: html
 	cd _build/html && python3.3 -m http.server
+
+zip: html
+	cd _build/html && zip -r ../../../msgpack-doc.zip .
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index aeda02a..bd838f2 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -81,9 +81,12 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=1, encoding=None, unicode_errors="strict",
             object_pairs_hook=None,
             ):
-    """Unpack packed_bytes to object. Returns an unpacked object.
+    """
+    Unpack packed_bytes to object. Returns an unpacked object.
 
     Raises `ValueError` when `packed` contains extra bytes.
+
+    See :class:`Unpacker` for options.
     """
     cdef template_context ctx
     cdef size_t off = 0
@@ -121,9 +124,12 @@ def unpack(object stream, object object_hook=None, object list_hook=None,
            bint use_list=1, encoding=None, unicode_errors="strict",
            object_pairs_hook=None,
            ):
-    """Unpack an object from `stream`.
+    """
+    Unpack an object from `stream`.
 
     Raises `ValueError` when `stream` has extra bytes.
+
+    See :class:`Unpacker` for options.
     """
     return unpackb(stream.read(), use_list=use_list,
                    object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
@@ -135,48 +141,58 @@ cdef class Unpacker(object):
     """
     Streaming unpacker.
 
-    `file_like` is a file-like object having `.read(n)` method.
-    When `Unpacker` initialized with `file_like`, unpacker reads serialized data
-    from it and `.feed()` method is not usable.
+    arguments:
 
-    `read_size` is used as `file_like.read(read_size)`.
-    (default: min(1024**2, max_buffer_size))
+    :param file_like:
+        File-like object having `.read(n)` method.
+        If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
 
-    If `use_list` is true (default), msgpack list is deserialized to Python list.
-    Otherwise, it is deserialized to Python tuple.
+    :param int read_size:
+        Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`)
 
-    `object_hook` is same to simplejson. If it is not None, it should be callable
-    and Unpacker calls it with a dict argument after deserializing a map.
+    :param bool use_list:
+        If true, unpack msgpack array to Python list.
+        Otherwise, unpack to Python tuple. (default: True)
 
-    `object_pairs_hook` is same to simplejson. If it is not None, it should be callable
-    and Unpacker calls it with a list of key-value pairs after deserializing a map.
+    :param callable object_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a dict argument after unpacking msgpack map.
+        (See also simplejson)
 
-    `encoding` is encoding used for decoding msgpack bytes. If it is None (default),
-    msgpack bytes is deserialized to Python bytes.
+    :param callable object_pairs_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
+        (See also simplejson)
 
-    `unicode_errors` is used for decoding bytes.
+    :param str encoding:
+        Encoding used for decoding msgpack raw.
+        If it is None (default), msgpack raw is deserialized to Python bytes.
 
-    `max_buffer_size` limits size of data waiting unpacked.
-    0 means system's INT_MAX (default).
-    Raises `BufferFull` exception when it is insufficient.
-    You shoud set this parameter when unpacking data from untrasted source.
+    :param str unicode_errors:
+        Used for decoding msgpack raw with *encoding*.
+        (default: `'strict'`)
+
+    :param int max_buffer_size:
+        Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
+        Raises `BufferFull` exception when it is insufficient.
+        You shoud set this parameter when unpacking data from untrasted source.
 
     example of streaming deserialize from file-like object::
 
         unpacker = Unpacker(file_like)
         for o in unpacker:
-            do_something(o)
+            process(o)
 
     example of streaming deserialize from socket::
 
         unpacker = Unpacker()
-        while 1:
+        while True:
             buf = sock.recv(1024**2)
             if not buf:
                 break
             unpacker.feed(buf)
             for o in unpacker:
-                do_something(o)
+                process(o)
     """
     cdef template_context ctx
     cdef char* buf
@@ -197,7 +213,7 @@ cdef class Unpacker(object):
 
     def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
-                 encoding=None, unicode_errors='strict', int max_buffer_size=0,
+                 str encoding=None, str unicode_errors='strict', int max_buffer_size=0,
                  ):
         cdef char *cenc=NULL, *cerr=NULL
 

From 5176e92d995f95b1b7db2554f6d44112ed04e8f8 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Feb 2013 09:55:13 +0900
Subject: [PATCH 1166/1648] Fix typeerror.

---
 msgpack/_unpacker.pyx | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index bd838f2..daeb6d7 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -239,15 +239,17 @@ cdef class Unpacker(object):
 
         if encoding is not None:
             if isinstance(encoding, unicode):
-                encoding = encoding.encode('ascii')
-            self.encoding = encoding
-            cenc = PyBytes_AsString(encoding)
+                self.encoding = encoding.encode('ascii')
+            else:
+                self.encoding = encoding
+            cenc = PyBytes_AsString(self.encoding)
 
         if unicode_errors is not None:
             if isinstance(unicode_errors, unicode):
-                unicode_errors = unicode_errors.encode('ascii')
-            self.unicode_errors = unicode_errors
-            cerr = PyBytes_AsString(unicode_errors)
+                self.unicode_errors = unicode_errors.encode('ascii')
+            else:
+                self.unicode_errors = unicode_errors
+            cerr = PyBytes_AsString(self.unicode_errors)
 
         init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
 

From d2feb1362946c87609e3753a058ddbd0958ee0bd Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 26 Feb 2013 14:20:44 +0900
Subject: [PATCH 1167/1648] doc: remove fallback from api reference.

---
 docs/api.rst | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/docs/api.rst b/docs/api.rst
index ccc7952..50a84c4 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -35,9 +35,3 @@ These exceptions are accessible via `msgpack` package.
     :members:
     :undoc-members:
     :show-inheritance:
-
-.. automodule:: msgpack.fallback
-    :members:
-    :undoc-members:
-    :show-inheritance:
-

From 58d8effc357ae06b68dec9f434c8039a6ddad0d3 Mon Sep 17 00:00:00 2001
From: jnothman 
Date: Wed, 27 Feb 2013 18:26:57 +1100
Subject: [PATCH 1168/1648] Copy note on OutOfData from pypi to ChangeLog

---
 ChangeLog.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index fdbb3ea..fe64ff8 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -7,6 +7,8 @@ Inconpatible Changes
 
 * Default value of ``use_list`` is ``True`` for now. (It was ``False`` for 0.2.x)
   You should pass it explicitly for compatibility to 0.2.x.
+* `Unpacker.unpack()` and some unpack methods now raise `OutOfData` instead of
+  `StopIteration`. `StopIteration` is used for iterator protocol only.
 
 Changes
 -------

From c91131f49f47b2af0349c1807d1c0f183d2b99b3 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 27 Feb 2013 20:37:07 +0900
Subject: [PATCH 1169/1648] remove msgpack_pack* macros

---
 msgpack/pack.h          |   8 ---
 msgpack/pack_template.h | 120 +++++++++-------------------------------
 2 files changed, 25 insertions(+), 103 deletions(-)

diff --git a/msgpack/pack.h b/msgpack/pack.h
index bb939d9..59764d9 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -90,14 +90,6 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_
     return 0;
 }
 
-#define msgpack_pack_inline_func(name) \
-	static inline int msgpack_pack ## name
-
-#define msgpack_pack_inline_func_cint(name) \
-	static inline int msgpack_pack ## name
-
-#define msgpack_pack_user msgpack_packer*
-
 #define msgpack_pack_append_buffer(user, buf, len) \
         return msgpack_pack_write(user, (const char*)buf, len)
 
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 65c959d..d6bf820 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -28,14 +28,6 @@
 #define TAKE8_64(d) ((uint8_t*)&d)[7]
 #endif
 
-#ifndef msgpack_pack_inline_func
-#error msgpack_pack_inline_func template is not defined
-#endif
-
-#ifndef msgpack_pack_user
-#error msgpack_pack_user type is not defined
-#endif
-
 #ifndef msgpack_pack_append_buffer
 #error msgpack_pack_append_buffer callback is not defined
 #endif
@@ -272,102 +264,42 @@ do { \
 } while(0)
 
 
-#ifdef msgpack_pack_inline_func_fixint
-
-msgpack_pack_inline_func_fixint(_uint8)(msgpack_pack_user x, uint8_t d)
-{
-	unsigned char buf[2] = {0xcc, TAKE8_8(d)};
-	msgpack_pack_append_buffer(x, buf, 2);
-}
-
-msgpack_pack_inline_func_fixint(_uint16)(msgpack_pack_user x, uint16_t d)
-{
-	unsigned char buf[3];
-	buf[0] = 0xcd; _msgpack_store16(&buf[1], d);
-	msgpack_pack_append_buffer(x, buf, 3);
-}
-
-msgpack_pack_inline_func_fixint(_uint32)(msgpack_pack_user x, uint32_t d)
-{
-	unsigned char buf[5];
-	buf[0] = 0xce; _msgpack_store32(&buf[1], d);
-	msgpack_pack_append_buffer(x, buf, 5);
-}
-
-msgpack_pack_inline_func_fixint(_uint64)(msgpack_pack_user x, uint64_t d)
-{
-	unsigned char buf[9];
-	buf[0] = 0xcf; _msgpack_store64(&buf[1], d);
-	msgpack_pack_append_buffer(x, buf, 9);
-}
-
-msgpack_pack_inline_func_fixint(_int8)(msgpack_pack_user x, int8_t d)
-{
-	unsigned char buf[2] = {0xd0, TAKE8_8(d)};
-	msgpack_pack_append_buffer(x, buf, 2);
-}
-
-msgpack_pack_inline_func_fixint(_int16)(msgpack_pack_user x, int16_t d)
-{
-	unsigned char buf[3];
-	buf[0] = 0xd1; _msgpack_store16(&buf[1], d);
-	msgpack_pack_append_buffer(x, buf, 3);
-}
-
-msgpack_pack_inline_func_fixint(_int32)(msgpack_pack_user x, int32_t d)
-{
-	unsigned char buf[5];
-	buf[0] = 0xd2; _msgpack_store32(&buf[1], d);
-	msgpack_pack_append_buffer(x, buf, 5);
-}
-
-msgpack_pack_inline_func_fixint(_int64)(msgpack_pack_user x, int64_t d)
-{
-	unsigned char buf[9];
-	buf[0] = 0xd3; _msgpack_store64(&buf[1], d);
-	msgpack_pack_append_buffer(x, buf, 9);
-}
-
-#undef msgpack_pack_inline_func_fixint
-#endif
-
-
-msgpack_pack_inline_func(_uint8)(msgpack_pack_user x, uint8_t d)
+static inline int msgpack_pack_uint8(msgpack_packer* x, uint8_t d)
 {
 	msgpack_pack_real_uint8(x, d);
 }
 
-msgpack_pack_inline_func(_uint16)(msgpack_pack_user x, uint16_t d)
+static inline int msgpack_pack_uint16(msgpack_packer* x, uint16_t d)
 {
 	msgpack_pack_real_uint16(x, d);
 }
 
-msgpack_pack_inline_func(_uint32)(msgpack_pack_user x, uint32_t d)
+static inline int msgpack_pack_uint32(msgpack_packer* x, uint32_t d)
 {
 	msgpack_pack_real_uint32(x, d);
 }
 
-msgpack_pack_inline_func(_uint64)(msgpack_pack_user x, uint64_t d)
+static inline int msgpack_pack_uint64(msgpack_packer* x, uint64_t d)
 {
 	msgpack_pack_real_uint64(x, d);
 }
 
-msgpack_pack_inline_func(_int8)(msgpack_pack_user x, int8_t d)
+static inline int msgpack_pack_int8(msgpack_packer* x, int8_t d)
 {
 	msgpack_pack_real_int8(x, d);
 }
 
-msgpack_pack_inline_func(_int16)(msgpack_pack_user x, int16_t d)
+static inline int msgpack_pack_int16(msgpack_packer* x, int16_t d)
 {
 	msgpack_pack_real_int16(x, d);
 }
 
-msgpack_pack_inline_func(_int32)(msgpack_pack_user x, int32_t d)
+static inline int msgpack_pack_int32(msgpack_packer* x, int32_t d)
 {
 	msgpack_pack_real_int32(x, d);
 }
 
-msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d)
+static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d)
 {
 	msgpack_pack_real_int64(x, d);
 }
@@ -375,7 +307,7 @@ msgpack_pack_inline_func(_int64)(msgpack_pack_user x, int64_t d)
 
 #ifdef msgpack_pack_inline_func_cint
 
-msgpack_pack_inline_func_cint(_short)(msgpack_pack_user x, short d)
+static inline int msgpack_pack_short(msgpack_packer* x, short d)
 {
 #if defined(SIZEOF_SHORT)
 #if SIZEOF_SHORT == 2
@@ -406,7 +338,7 @@ if(sizeof(short) == 2) {
 #endif
 }
 
-msgpack_pack_inline_func_cint(_int)(msgpack_pack_user x, int d)
+static inline int msgpack_pack_int(msgpack_packer* x, int d)
 {
 #if defined(SIZEOF_INT)
 #if SIZEOF_INT == 2
@@ -437,7 +369,7 @@ if(sizeof(int) == 2) {
 #endif
 }
 
-msgpack_pack_inline_func_cint(_long)(msgpack_pack_user x, long d)
+static inline int msgpack_pack_long(msgpack_packer* x, long d)
 {
 #if defined(SIZEOF_LONG)
 #if SIZEOF_LONG == 2
@@ -468,7 +400,7 @@ if(sizeof(long) == 2) {
 #endif
 }
 
-msgpack_pack_inline_func_cint(_long_long)(msgpack_pack_user x, long long d)
+static inline int msgpack_pack_long_long(msgpack_packer* x, long long d)
 {
 #if defined(SIZEOF_LONG_LONG)
 #if SIZEOF_LONG_LONG == 2
@@ -499,7 +431,7 @@ if(sizeof(long long) == 2) {
 #endif
 }
 
-msgpack_pack_inline_func_cint(_unsigned_short)(msgpack_pack_user x, unsigned short d)
+static inline int msgpack_pack_unsigned_short(msgpack_packer* x, unsigned short d)
 {
 #if defined(SIZEOF_SHORT)
 #if SIZEOF_SHORT == 2
@@ -530,7 +462,7 @@ if(sizeof(unsigned short) == 2) {
 #endif
 }
 
-msgpack_pack_inline_func_cint(_unsigned_int)(msgpack_pack_user x, unsigned int d)
+static inline int msgpack_pack_unsigned_int(msgpack_packer* x, unsigned int d)
 {
 #if defined(SIZEOF_INT)
 #if SIZEOF_INT == 2
@@ -561,7 +493,7 @@ if(sizeof(unsigned int) == 2) {
 #endif
 }
 
-msgpack_pack_inline_func_cint(_unsigned_long)(msgpack_pack_user x, unsigned long d)
+static inline int msgpack_pack_unsigned_long(msgpack_packer* x, unsigned long d)
 {
 #if defined(SIZEOF_LONG)
 #if SIZEOF_LONG == 2
@@ -592,7 +524,7 @@ if(sizeof(unsigned long) == 2) {
 #endif
 }
 
-msgpack_pack_inline_func_cint(_unsigned_long_long)(msgpack_pack_user x, unsigned long long d)
+static inline int msgpack_pack_unsigned_long_long(msgpack_packer* x, unsigned long long d)
 {
 #if defined(SIZEOF_LONG_LONG)
 #if SIZEOF_LONG_LONG == 2
@@ -632,7 +564,7 @@ if(sizeof(unsigned long long) == 2) {
  * Float
  */
 
-msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d)
+static inline int msgpack_pack_float(msgpack_packer* x, float d)
 {
 	union { float f; uint32_t i; } mem;
 	mem.f = d;
@@ -641,7 +573,7 @@ msgpack_pack_inline_func(_float)(msgpack_pack_user x, float d)
 	msgpack_pack_append_buffer(x, buf, 5);
 }
 
-msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d)
+static inline int msgpack_pack_double(msgpack_packer* x, double d)
 {
 	union { double f; uint64_t i; } mem;
 	mem.f = d;
@@ -660,7 +592,7 @@ msgpack_pack_inline_func(_double)(msgpack_pack_user x, double d)
  * Nil
  */
 
-msgpack_pack_inline_func(_nil)(msgpack_pack_user x)
+static inline int msgpack_pack_nil(msgpack_packer* x)
 {
 	static const unsigned char d = 0xc0;
 	msgpack_pack_append_buffer(x, &d, 1);
@@ -671,13 +603,13 @@ msgpack_pack_inline_func(_nil)(msgpack_pack_user x)
  * Boolean
  */
 
-msgpack_pack_inline_func(_true)(msgpack_pack_user x)
+static inline int msgpack_pack_true(msgpack_packer* x)
 {
 	static const unsigned char d = 0xc3;
 	msgpack_pack_append_buffer(x, &d, 1);
 }
 
-msgpack_pack_inline_func(_false)(msgpack_pack_user x)
+static inline int msgpack_pack_false(msgpack_packer* x)
 {
 	static const unsigned char d = 0xc2;
 	msgpack_pack_append_buffer(x, &d, 1);
@@ -688,7 +620,7 @@ msgpack_pack_inline_func(_false)(msgpack_pack_user x)
  * Array
  */
 
-msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n)
+static inline int msgpack_pack_array(msgpack_packer* x, unsigned int n)
 {
 	if(n < 16) {
 		unsigned char d = 0x90 | n;
@@ -709,7 +641,7 @@ msgpack_pack_inline_func(_array)(msgpack_pack_user x, unsigned int n)
  * Map
  */
 
-msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n)
+static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n)
 {
 	if(n < 16) {
 		unsigned char d = 0x80 | n;
@@ -730,7 +662,7 @@ msgpack_pack_inline_func(_map)(msgpack_pack_user x, unsigned int n)
  * Raw
  */
 
-msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l)
+static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
 {
 	if(l < 32) {
 		unsigned char d = 0xa0 | (uint8_t)l;
@@ -746,13 +678,11 @@ msgpack_pack_inline_func(_raw)(msgpack_pack_user x, size_t l)
 	}
 }
 
-msgpack_pack_inline_func(_raw_body)(msgpack_pack_user x, const void* b, size_t l)
+static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l)
 {
 	msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
 }
 
-#undef msgpack_pack_inline_func
-#undef msgpack_pack_user
 #undef msgpack_pack_append_buffer
 
 #undef TAKE8_8

From c49489cd372903fc5c26b330c91bdd3679029e17 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 27 Feb 2013 21:12:20 +0900
Subject: [PATCH 1170/1648] remove some macros.

---
 msgpack/_unpacker.pyx     | 34 +++++++--------
 msgpack/pack.h            |  2 +-
 msgpack/pack_template.h   |  6 +--
 msgpack/unpack.h          | 72 +++++++++++++------------------
 msgpack/unpack_template.h | 90 +++++++++++++--------------------------
 5 files changed, 80 insertions(+), 124 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index daeb6d7..5813dc9 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -30,26 +30,26 @@ cdef extern from "unpack.h":
         char *encoding
         char *unicode_errors
 
-    ctypedef struct template_context:
+    ctypedef struct unpack_context:
         msgpack_user user
         PyObject* obj
         size_t count
         unsigned int ct
         PyObject* key
 
-    ctypedef int (*execute_fn)(template_context* ctx, const_char_ptr data,
+    ctypedef int (*execute_fn)(unpack_context* ctx, const_char_ptr data,
                                size_t len, size_t* off) except? -1
-    execute_fn template_construct
-    execute_fn template_skip
+    execute_fn unpack_construct
+    execute_fn unpack_skip
     execute_fn read_array_header
     execute_fn read_map_header
-    void template_init(template_context* ctx)
-    object template_data(template_context* ctx)
+    void unpack_init(unpack_context* ctx)
+    object unpack_data(unpack_context* ctx)
 
-cdef inline init_ctx(template_context *ctx,
+cdef inline init_ctx(unpack_context *ctx,
                      object object_hook, object object_pairs_hook, object list_hook,
                      bint use_list, char* encoding, char* unicode_errors):
-    template_init(ctx)
+    unpack_init(ctx)
     ctx.user.use_list = use_list
     ctx.user.object_hook = ctx.user.list_hook = NULL
 
@@ -88,7 +88,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
     See :class:`Unpacker` for options.
     """
-    cdef template_context ctx
+    cdef unpack_context ctx
     cdef size_t off = 0
     cdef int ret
 
@@ -110,9 +110,9 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         cerr = PyBytes_AsString(unicode_errors)
 
     init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
-    ret = template_construct(&ctx, buf, buf_len, &off)
+    ret = unpack_construct(&ctx, buf, buf_len, &off)
     if ret == 1:
-        obj = template_data(&ctx)
+        obj = unpack_data(&ctx)
         if off < buf_len:
             raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
         return obj
@@ -194,7 +194,7 @@ cdef class Unpacker(object):
             for o in unpacker:
                 process(o)
     """
-    cdef template_context ctx
+    cdef unpack_context ctx
     cdef char* buf
     cdef size_t buf_size, buf_head, buf_tail
     cdef object file_like
@@ -324,8 +324,8 @@ cdef class Unpacker(object):
                 write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
 
             if ret == 1:
-                obj = template_data(&self.ctx)
-                template_init(&self.ctx)
+                obj = unpack_data(&self.ctx)
+                unpack_init(&self.ctx)
                 return obj
             elif ret == 0:
                 if self.file_like is not None:
@@ -357,7 +357,7 @@ cdef class Unpacker(object):
 
         Raises `OutOfData` when there are no more bytes to unpack.
         """
-        return self._unpack(template_construct, write_bytes)
+        return self._unpack(unpack_construct, write_bytes)
 
     def skip(self, object write_bytes=None):
         """
@@ -368,7 +368,7 @@ cdef class Unpacker(object):
 
         Raises `OutOfData` when there are no more bytes to unpack.
         """
-        return self._unpack(template_skip, write_bytes)
+        return self._unpack(unpack_skip, write_bytes)
 
     def read_array_header(self, object write_bytes=None):
         """assuming the next object is an array, return its size n, such that
@@ -390,7 +390,7 @@ cdef class Unpacker(object):
         return self
 
     def __next__(self):
-        return self._unpack(template_construct, None, 1)
+        return self._unpack(unpack_construct, None, 1)
 
     # for debug.
     #def _buf(self):
diff --git a/msgpack/pack.h b/msgpack/pack.h
index 59764d9..1539991 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -45,7 +45,7 @@ static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
 static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d);
 static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d);
 static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d);
-static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d);
+//static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d);
 
 static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d);
 static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d);
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index d6bf820..35deb83 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -305,7 +305,7 @@ static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d)
 }
 
 
-#ifdef msgpack_pack_inline_func_cint
+//#ifdef msgpack_pack_inline_func_cint
 
 static inline int msgpack_pack_short(msgpack_packer* x, short d)
 {
@@ -555,8 +555,8 @@ if(sizeof(unsigned long long) == 2) {
 #endif
 }
 
-#undef msgpack_pack_inline_func_cint
-#endif
+//#undef msgpack_pack_inline_func_cint
+//#endif
 
 
 
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 3dc88e5..595b8df 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -28,31 +28,17 @@ typedef struct unpack_user {
     const char *unicode_errors;
 } unpack_user;
 
+typedef PyObject* msgpack_unpack_object;
+struct unpack_context;
+typedef struct unpack_context unpack_context;
+typedef int (*execute_fn)(unpack_context *ctx, const char* data, size_t len, size_t* off);
 
-#define msgpack_unpack_struct(name) \
-	struct template ## name
-
-#define msgpack_unpack_func(ret, name) \
-	static inline ret template ## name
-
-#define msgpack_unpack_callback(name) \
-	template_callback ## name
-
-#define msgpack_unpack_object PyObject*
-
-#define msgpack_unpack_user unpack_user
-
-typedef int (*execute_fn)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off);
-
-struct template_context;
-typedef struct template_context template_context;
-
-static inline msgpack_unpack_object template_callback_root(unpack_user* u)
+static inline msgpack_unpack_object unpack_callback_root(unpack_user* u)
 {
     return NULL;
 }
 
-static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o)
+static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o)
 {
     PyObject *p = PyInt_FromLong((long)d);
     if (!p)
@@ -60,13 +46,13 @@ static inline int template_callback_uint16(unpack_user* u, uint16_t d, msgpack_u
     *o = p;
     return 0;
 }
-static inline int template_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o)
+static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpack_object* o)
 {
-    return template_callback_uint16(u, d, o);
+    return unpack_callback_uint16(u, d, o);
 }
 
 
-static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
+static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
 {
     PyObject *p;
     if (d > LONG_MAX) {
@@ -80,7 +66,7 @@ static inline int template_callback_uint32(unpack_user* u, uint32_t d, msgpack_u
     return 0;
 }
 
-static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o)
+static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o)
 {
     PyObject *p = PyLong_FromUnsignedLongLong(d);
     if (!p)
@@ -89,7 +75,7 @@ static inline int template_callback_uint64(unpack_user* u, uint64_t d, msgpack_u
     return 0;
 }
 
-static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o)
+static inline int unpack_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o)
 {
     PyObject *p = PyInt_FromLong(d);
     if (!p)
@@ -98,17 +84,17 @@ static inline int template_callback_int32(unpack_user* u, int32_t d, msgpack_unp
     return 0;
 }
 
-static inline int template_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o)
+static inline int unpack_callback_int16(unpack_user* u, int16_t d, msgpack_unpack_object* o)
 {
-    return template_callback_int32(u, d, o);
+    return unpack_callback_int32(u, d, o);
 }
 
-static inline int template_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o)
+static inline int unpack_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_object* o)
 {
-    return template_callback_int32(u, d, o);
+    return unpack_callback_int32(u, d, o);
 }
 
-static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o)
+static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o)
 {
     PyObject *p = PyLong_FromLongLong(d);
     if (!p)
@@ -117,7 +103,7 @@ static inline int template_callback_int64(unpack_user* u, int64_t d, msgpack_unp
     return 0;
 }
 
-static inline int template_callback_double(unpack_user* u, double d, msgpack_unpack_object* o)
+static inline int unpack_callback_double(unpack_user* u, double d, msgpack_unpack_object* o)
 {
     PyObject *p = PyFloat_FromDouble(d);
     if (!p)
@@ -126,21 +112,21 @@ static inline int template_callback_double(unpack_user* u, double d, msgpack_unp
     return 0;
 }
 
-static inline int template_callback_float(unpack_user* u, float d, msgpack_unpack_object* o)
+static inline int unpack_callback_float(unpack_user* u, float d, msgpack_unpack_object* o)
 {
-    return template_callback_double(u, d, o);
+    return unpack_callback_double(u, d, o);
 }
 
-static inline int template_callback_nil(unpack_user* u, msgpack_unpack_object* o)
+static inline int unpack_callback_nil(unpack_user* u, msgpack_unpack_object* o)
 { Py_INCREF(Py_None); *o = Py_None; return 0; }
 
-static inline int template_callback_true(unpack_user* u, msgpack_unpack_object* o)
+static inline int unpack_callback_true(unpack_user* u, msgpack_unpack_object* o)
 { Py_INCREF(Py_True); *o = Py_True; return 0; }
 
-static inline int template_callback_false(unpack_user* u, msgpack_unpack_object* o)
+static inline int unpack_callback_false(unpack_user* u, msgpack_unpack_object* o)
 { Py_INCREF(Py_False); *o = Py_False; return 0; }
 
-static inline int template_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+static inline int unpack_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
 {
     PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n);
 
@@ -150,7 +136,7 @@ static inline int template_callback_array(unpack_user* u, unsigned int n, msgpac
     return 0;
 }
 
-static inline int template_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o)
+static inline int unpack_callback_array_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object o)
 {
     if (u->use_list)
         PyList_SET_ITEM(*c, current, o);
@@ -159,7 +145,7 @@ static inline int template_callback_array_item(unpack_user* u, unsigned int curr
     return 0;
 }
 
-static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
+static inline int unpack_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->list_hook) {
         PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c);
@@ -171,7 +157,7 @@ static inline int template_callback_array_end(unpack_user* u, msgpack_unpack_obj
     return 0;
 }
 
-static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
+static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
 {
     PyObject *p;
     if (u->has_pairs_hook) {
@@ -186,7 +172,7 @@ static inline int template_callback_map(unpack_user* u, unsigned int n, msgpack_
     return 0;
 }
 
-static inline int template_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
+static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
 {
     if (u->has_pairs_hook) {
         msgpack_unpack_object item = PyTuple_Pack(2, k, v);
@@ -205,7 +191,7 @@ static inline int template_callback_map_item(unpack_user* u, unsigned int curren
     return -1;
 }
 
-static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
+static inline int unpack_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->object_hook) {
         PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c);
@@ -218,7 +204,7 @@ static inline int template_callback_map_end(unpack_user* u, msgpack_unpack_objec
     return 0;
 }
 
-static inline int template_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
+static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
 {
     PyObject *py;
     if(u->encoding) {
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 83b6918..153340c 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -16,59 +16,35 @@
  *    limitations under the License.
  */
 
-#ifndef msgpack_unpack_func
-#error msgpack_unpack_func template is not defined
-#endif
-
-#ifndef msgpack_unpack_callback
-#error msgpack_unpack_callback template is not defined
-#endif
-
-#ifndef msgpack_unpack_struct
-#error msgpack_unpack_struct template is not defined
-#endif
-
-#ifndef msgpack_unpack_struct_decl
-#define msgpack_unpack_struct_decl(name) msgpack_unpack_struct(name)
-#endif
-
-#ifndef msgpack_unpack_object
-#error msgpack_unpack_object type is not defined
-#endif
-
-#ifndef msgpack_unpack_user
-#error msgpack_unpack_user type is not defined
-#endif
-
 #ifndef USE_CASE_RANGE
 #if !defined(_MSC_VER)
 #define USE_CASE_RANGE
 #endif
 #endif
 
-msgpack_unpack_struct_decl(_stack) {
-	msgpack_unpack_object obj;
+typedef struct unpack_stack {
+	PyObject* obj;
 	size_t size;
 	size_t count;
 	unsigned int ct;
-	msgpack_unpack_object map_key;
-};
+	PyObject* map_key;
+} unpack_stack;
 
-msgpack_unpack_struct_decl(_context) {
-	msgpack_unpack_user user;
+struct unpack_context {
+	unpack_user user;
 	unsigned int cs;
 	unsigned int trail;
 	unsigned int top;
 	/*
-	msgpack_unpack_struct(_stack)* stack;
+	unpack_stack* stack;
 	unsigned int stack_size;
-	msgpack_unpack_struct(_stack) embed_stack[MSGPACK_EMBED_STACK_SIZE];
+	unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE];
 	*/
-	msgpack_unpack_struct(_stack) stack[MSGPACK_EMBED_STACK_SIZE];
+	unpack_stack stack[MSGPACK_EMBED_STACK_SIZE];
 };
 
 
-msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx)
+static inline void unpack_init(unpack_context* ctx)
 {
 	ctx->cs = CS_HEADER;
 	ctx->trail = 0;
@@ -77,11 +53,11 @@ msgpack_unpack_func(void, _init)(msgpack_unpack_struct(_context)* ctx)
 	ctx->stack = ctx->embed_stack;
 	ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
 	*/
-	ctx->stack[0].obj = msgpack_unpack_callback(_root)(&ctx->user);
+	ctx->stack[0].obj = unpack_callback_root(&ctx->user);
 }
 
 /*
-msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx)
+static inline void unpack_destroy(unpack_context* ctx)
 {
 	if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
 		free(ctx->stack);
@@ -89,14 +65,14 @@ msgpack_unpack_func(void, _destroy)(msgpack_unpack_struct(_context)* ctx)
 }
 */
 
-msgpack_unpack_func(msgpack_unpack_object, _data)(msgpack_unpack_struct(_context)* ctx)
+static inline PyObject* unpack_data(unpack_context* ctx)
 {
 	return (ctx)->stack[0].obj;
 }
 
 
 template 
-msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
+static inline int unpack_execute(unpack_context* ctx, const char* data, size_t len, size_t* off)
 {
 	assert(len >= *off);
 
@@ -107,19 +83,19 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	unsigned int trail = ctx->trail;
 	unsigned int cs = ctx->cs;
 	unsigned int top = ctx->top;
-	msgpack_unpack_struct(_stack)* stack = ctx->stack;
+	unpack_stack* stack = ctx->stack;
 	/*
 	unsigned int stack_size = ctx->stack_size;
 	*/
-	msgpack_unpack_user* user = &ctx->user;
+	unpack_user* user = &ctx->user;
 
-	msgpack_unpack_object obj;
-	msgpack_unpack_struct(_stack)* c = NULL;
+	PyObject* obj;
+	unpack_stack* c = NULL;
 
 	int ret;
 
 #define construct_cb(name) \
-    construct && msgpack_unpack_callback(name)
+    construct && unpack_callback ## name
 
 #define push_simple_value(func) \
 	if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \
@@ -157,16 +133,16 @@ msgpack_unpack_func(int, _execute)(msgpack_unpack_struct(_context)* ctx, const c
 	/* FIXME \
 	if(top >= stack_size) { \
 		if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
-			size_t csize = sizeof(msgpack_unpack_struct(_stack)) * MSGPACK_EMBED_STACK_SIZE; \
+			size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \
 			size_t nsize = csize * 2; \
-			msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)malloc(nsize); \
+			unpack_stack* tmp = (unpack_stack*)malloc(nsize); \
 			if(tmp == NULL) { goto _failed; } \
 			memcpy(tmp, ctx->stack, csize); \
 			ctx->stack = stack = tmp; \
 			ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
 		} else { \
-			size_t nsize = sizeof(msgpack_unpack_struct(_stack)) * ctx->stack_size * 2; \
-			msgpack_unpack_struct(_stack)* tmp = (msgpack_unpack_struct(_stack)*)realloc(ctx->stack, nsize); \
+			size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \
+			unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \
 			if(tmp == NULL) { goto _failed; } \
 			ctx->stack = stack = tmp; \
 			ctx->stack_size = stack_size = stack_size * 2; \
@@ -382,7 +358,7 @@ _header_again:
 
 _finish:
 	if (!construct)
-		msgpack_unpack_callback(_nil)(user, &obj);
+		unpack_callback_nil(user, &obj);
 	stack[0].obj = obj;
 	++p;
 	ret = 1;
@@ -420,7 +396,7 @@ _end:
 #undef start_container
 
 template 
-msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx, const char* data, size_t len, size_t* off)
+static inline int unpack_container_header(unpack_context* ctx, const char* data, size_t len, size_t* off)
 {
 	assert(len >= *off);
 	uint32_t size;
@@ -467,7 +443,7 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx
 		PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
 		return -1;
     }
-	msgpack_unpack_callback(_uint32)(&ctx->user, size, &ctx->stack[0].obj);
+	unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj);
 	return 1;
 }
 
@@ -476,16 +452,10 @@ msgpack_unpack_func(int, _container_header)(msgpack_unpack_struct(_context)* ctx
 #undef SWITCH_RANGE_DEFAULT
 #undef SWITCH_RANGE_END
 
-static const execute_fn template_construct = &template_execute;
-static const execute_fn template_skip = &template_execute;
-static const execute_fn read_array_header = &template_container_header<0x90, 0xdc>;
-static const execute_fn read_map_header = &template_container_header<0x80, 0xde>;
-
-#undef msgpack_unpack_func
-#undef msgpack_unpack_callback
-#undef msgpack_unpack_struct
-#undef msgpack_unpack_object
-#undef msgpack_unpack_user
+static const execute_fn unpack_construct = &unpack_execute;
+static const execute_fn unpack_skip = &unpack_execute;
+static const execute_fn read_array_header = &unpack_container_header<0x90, 0xdc>;
+static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>;
 
 #undef NEXT_CS
 

From c9b6e5b65d8cbc8a1ed5450eb274326e1c061406 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 27 Feb 2013 21:24:25 +0900
Subject: [PATCH 1171/1648] s/\t/    /g

---
 msgpack/pack_template.h   | 655 +++++++++++++++++++-------------------
 msgpack/sysdep.h          |   1 -
 msgpack/unpack_define.h   |  76 ++---
 msgpack/unpack_template.h | 628 ++++++++++++++++++------------------
 4 files changed, 679 insertions(+), 681 deletions(-)

diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 35deb83..9e00d7e 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -39,269 +39,269 @@
 
 #define msgpack_pack_real_uint8(x, d) \
 do { \
-	if(d < (1<<7)) { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
-	} else { \
-		/* unsigned 8 */ \
-		unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \
-		msgpack_pack_append_buffer(x, buf, 2); \
-	} \
+    if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
+    } else { \
+        /* unsigned 8 */ \
+        unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \
+        msgpack_pack_append_buffer(x, buf, 2); \
+    } \
 } while(0)
 
 #define msgpack_pack_real_uint16(x, d) \
 do { \
-	if(d < (1<<7)) { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
-	} else if(d < (1<<8)) { \
-		/* unsigned 8 */ \
-		unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
-		msgpack_pack_append_buffer(x, buf, 2); \
-	} else { \
-		/* unsigned 16 */ \
-		unsigned char buf[3]; \
-		buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
-		msgpack_pack_append_buffer(x, buf, 3); \
-	} \
+    if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
+    } else if(d < (1<<8)) { \
+        /* unsigned 8 */ \
+        unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
+        msgpack_pack_append_buffer(x, buf, 2); \
+    } else { \
+        /* unsigned 16 */ \
+        unsigned char buf[3]; \
+        buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+        msgpack_pack_append_buffer(x, buf, 3); \
+    } \
 } while(0)
 
 #define msgpack_pack_real_uint32(x, d) \
 do { \
-	if(d < (1<<8)) { \
-		if(d < (1<<7)) { \
-			/* fixnum */ \
-			msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
-		} else { \
-			/* unsigned 8 */ \
-			unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} \
-	} else { \
-		if(d < (1<<16)) { \
-			/* unsigned 16 */ \
-			unsigned char buf[3]; \
-			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} else { \
-			/* unsigned 32 */ \
-			unsigned char buf[5]; \
-			buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
-			msgpack_pack_append_buffer(x, buf, 5); \
-		} \
-	} \
+    if(d < (1<<8)) { \
+        if(d < (1<<7)) { \
+            /* fixnum */ \
+            msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
+        } else { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else { \
+        if(d < (1<<16)) { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* unsigned 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } \
+    } \
 } while(0)
 
 #define msgpack_pack_real_uint64(x, d) \
 do { \
-	if(d < (1ULL<<8)) { \
-		if(d < (1ULL<<7)) { \
-			/* fixnum */ \
-			msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
-		} else { \
-			/* unsigned 8 */ \
-			unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} \
-	} else { \
-		if(d < (1ULL<<16)) { \
-			/* unsigned 16 */ \
-			unsigned char buf[3]; \
-			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} else if(d < (1ULL<<32)) { \
-			/* unsigned 32 */ \
-			unsigned char buf[5]; \
-			buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
-			msgpack_pack_append_buffer(x, buf, 5); \
-		} else { \
-			/* unsigned 64 */ \
-			unsigned char buf[9]; \
-			buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
-			msgpack_pack_append_buffer(x, buf, 9); \
-		} \
-	} \
+    if(d < (1ULL<<8)) { \
+        if(d < (1ULL<<7)) { \
+            /* fixnum */ \
+            msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
+        } else { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else { \
+        if(d < (1ULL<<16)) { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else if(d < (1ULL<<32)) { \
+            /* unsigned 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } else { \
+            /* unsigned 64 */ \
+            unsigned char buf[9]; \
+            buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
+            msgpack_pack_append_buffer(x, buf, 9); \
+        } \
+    } \
 } while(0)
 
 #define msgpack_pack_real_int8(x, d) \
 do { \
-	if(d < -(1<<5)) { \
-		/* signed 8 */ \
-		unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \
-		msgpack_pack_append_buffer(x, buf, 2); \
-	} else { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
-	} \
+    if(d < -(1<<5)) { \
+        /* signed 8 */ \
+        unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \
+        msgpack_pack_append_buffer(x, buf, 2); \
+    } else { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
+    } \
 } while(0)
 
 #define msgpack_pack_real_int16(x, d) \
 do { \
-	if(d < -(1<<5)) { \
-		if(d < -(1<<7)) { \
-			/* signed 16 */ \
-			unsigned char buf[3]; \
-			buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} else { \
-			/* signed 8 */ \
-			unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} \
-	} else if(d < (1<<7)) { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
-	} else { \
-		if(d < (1<<8)) { \
-			/* unsigned 8 */ \
-			unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} else { \
-			/* unsigned 16 */ \
-			unsigned char buf[3]; \
-			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} \
-	} \
+    if(d < -(1<<5)) { \
+        if(d < -(1<<7)) { \
+            /* signed 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* signed 8 */ \
+            unsigned char buf[2] = {0xd0, TAKE8_16(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_16(d), 1); \
+    } else { \
+        if(d < (1<<8)) { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_16(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } else { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } \
+    } \
 } while(0)
 
 #define msgpack_pack_real_int32(x, d) \
 do { \
-	if(d < -(1<<5)) { \
-		if(d < -(1<<15)) { \
-			/* signed 32 */ \
-			unsigned char buf[5]; \
-			buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
-			msgpack_pack_append_buffer(x, buf, 5); \
-		} else if(d < -(1<<7)) { \
-			/* signed 16 */ \
-			unsigned char buf[3]; \
-			buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} else { \
-			/* signed 8 */ \
-			unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} \
-	} else if(d < (1<<7)) { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
-	} else { \
-		if(d < (1<<8)) { \
-			/* unsigned 8 */ \
-			unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
-			msgpack_pack_append_buffer(x, buf, 2); \
-		} else if(d < (1<<16)) { \
-			/* unsigned 16 */ \
-			unsigned char buf[3]; \
-			buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
-			msgpack_pack_append_buffer(x, buf, 3); \
-		} else { \
-			/* unsigned 32 */ \
-			unsigned char buf[5]; \
-			buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
-			msgpack_pack_append_buffer(x, buf, 5); \
-		} \
-	} \
+    if(d < -(1<<5)) { \
+        if(d < -(1<<15)) { \
+            /* signed 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } else if(d < -(1<<7)) { \
+            /* signed 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* signed 8 */ \
+            unsigned char buf[2] = {0xd0, TAKE8_32(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } \
+    } else if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_32(d), 1); \
+    } else { \
+        if(d < (1<<8)) { \
+            /* unsigned 8 */ \
+            unsigned char buf[2] = {0xcc, TAKE8_32(d)}; \
+            msgpack_pack_append_buffer(x, buf, 2); \
+        } else if(d < (1<<16)) { \
+            /* unsigned 16 */ \
+            unsigned char buf[3]; \
+            buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+            msgpack_pack_append_buffer(x, buf, 3); \
+        } else { \
+            /* unsigned 32 */ \
+            unsigned char buf[5]; \
+            buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+            msgpack_pack_append_buffer(x, buf, 5); \
+        } \
+    } \
 } while(0)
 
 #define msgpack_pack_real_int64(x, d) \
 do { \
-	if(d < -(1LL<<5)) { \
-		if(d < -(1LL<<15)) { \
-			if(d < -(1LL<<31)) { \
-				/* signed 64 */ \
-				unsigned char buf[9]; \
-				buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \
-				msgpack_pack_append_buffer(x, buf, 9); \
-			} else { \
-				/* signed 32 */ \
-				unsigned char buf[5]; \
-				buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
-				msgpack_pack_append_buffer(x, buf, 5); \
-			} \
-		} else { \
-			if(d < -(1<<7)) { \
-				/* signed 16 */ \
-				unsigned char buf[3]; \
-				buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
-				msgpack_pack_append_buffer(x, buf, 3); \
-			} else { \
-				/* signed 8 */ \
-				unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \
-				msgpack_pack_append_buffer(x, buf, 2); \
-			} \
-		} \
-	} else if(d < (1<<7)) { \
-		/* fixnum */ \
-		msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
-	} else { \
-		if(d < (1LL<<16)) { \
-			if(d < (1<<8)) { \
-				/* unsigned 8 */ \
-				unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
-				msgpack_pack_append_buffer(x, buf, 2); \
-			} else { \
-				/* unsigned 16 */ \
-				unsigned char buf[3]; \
-				buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
-				msgpack_pack_append_buffer(x, buf, 3); \
-			} \
-		} else { \
-			if(d < (1LL<<32)) { \
-				/* unsigned 32 */ \
-				unsigned char buf[5]; \
-				buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
-				msgpack_pack_append_buffer(x, buf, 5); \
-			} else { \
-				/* unsigned 64 */ \
-				unsigned char buf[9]; \
-				buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
-				msgpack_pack_append_buffer(x, buf, 9); \
-			} \
-		} \
-	} \
+    if(d < -(1LL<<5)) { \
+        if(d < -(1LL<<15)) { \
+            if(d < -(1LL<<31)) { \
+                /* signed 64 */ \
+                unsigned char buf[9]; \
+                buf[0] = 0xd3; _msgpack_store64(&buf[1], d); \
+                msgpack_pack_append_buffer(x, buf, 9); \
+            } else { \
+                /* signed 32 */ \
+                unsigned char buf[5]; \
+                buf[0] = 0xd2; _msgpack_store32(&buf[1], (int32_t)d); \
+                msgpack_pack_append_buffer(x, buf, 5); \
+            } \
+        } else { \
+            if(d < -(1<<7)) { \
+                /* signed 16 */ \
+                unsigned char buf[3]; \
+                buf[0] = 0xd1; _msgpack_store16(&buf[1], (int16_t)d); \
+                msgpack_pack_append_buffer(x, buf, 3); \
+            } else { \
+                /* signed 8 */ \
+                unsigned char buf[2] = {0xd0, TAKE8_64(d)}; \
+                msgpack_pack_append_buffer(x, buf, 2); \
+            } \
+        } \
+    } else if(d < (1<<7)) { \
+        /* fixnum */ \
+        msgpack_pack_append_buffer(x, &TAKE8_64(d), 1); \
+    } else { \
+        if(d < (1LL<<16)) { \
+            if(d < (1<<8)) { \
+                /* unsigned 8 */ \
+                unsigned char buf[2] = {0xcc, TAKE8_64(d)}; \
+                msgpack_pack_append_buffer(x, buf, 2); \
+            } else { \
+                /* unsigned 16 */ \
+                unsigned char buf[3]; \
+                buf[0] = 0xcd; _msgpack_store16(&buf[1], (uint16_t)d); \
+                msgpack_pack_append_buffer(x, buf, 3); \
+            } \
+        } else { \
+            if(d < (1LL<<32)) { \
+                /* unsigned 32 */ \
+                unsigned char buf[5]; \
+                buf[0] = 0xce; _msgpack_store32(&buf[1], (uint32_t)d); \
+                msgpack_pack_append_buffer(x, buf, 5); \
+            } else { \
+                /* unsigned 64 */ \
+                unsigned char buf[9]; \
+                buf[0] = 0xcf; _msgpack_store64(&buf[1], d); \
+                msgpack_pack_append_buffer(x, buf, 9); \
+            } \
+        } \
+    } \
 } while(0)
 
 
 static inline int msgpack_pack_uint8(msgpack_packer* x, uint8_t d)
 {
-	msgpack_pack_real_uint8(x, d);
+    msgpack_pack_real_uint8(x, d);
 }
 
 static inline int msgpack_pack_uint16(msgpack_packer* x, uint16_t d)
 {
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 }
 
 static inline int msgpack_pack_uint32(msgpack_packer* x, uint32_t d)
 {
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 }
 
 static inline int msgpack_pack_uint64(msgpack_packer* x, uint64_t d)
 {
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 }
 
 static inline int msgpack_pack_int8(msgpack_packer* x, int8_t d)
 {
-	msgpack_pack_real_int8(x, d);
+    msgpack_pack_real_int8(x, d);
 }
 
 static inline int msgpack_pack_int16(msgpack_packer* x, int16_t d)
 {
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 }
 
 static inline int msgpack_pack_int32(msgpack_packer* x, int32_t d)
 {
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 }
 
 static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d)
 {
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 }
 
 
@@ -311,29 +311,29 @@ static inline int msgpack_pack_short(msgpack_packer* x, short d)
 {
 #if defined(SIZEOF_SHORT)
 #if SIZEOF_SHORT == 2
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 #elif SIZEOF_SHORT == 4
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 #else
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 #endif
 
 #elif defined(SHRT_MAX)
 #if SHRT_MAX == 0x7fff
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 #elif SHRT_MAX == 0x7fffffff
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 #else
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 #endif
 
 #else
 if(sizeof(short) == 2) {
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 } else if(sizeof(short) == 4) {
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 } else {
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 }
 #endif
 }
@@ -342,29 +342,29 @@ static inline int msgpack_pack_int(msgpack_packer* x, int d)
 {
 #if defined(SIZEOF_INT)
 #if SIZEOF_INT == 2
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 #elif SIZEOF_INT == 4
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 #else
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 #endif
 
 #elif defined(INT_MAX)
 #if INT_MAX == 0x7fff
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 #elif INT_MAX == 0x7fffffff
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 #else
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 #endif
 
 #else
 if(sizeof(int) == 2) {
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 } else if(sizeof(int) == 4) {
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 } else {
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 }
 #endif
 }
@@ -373,29 +373,29 @@ static inline int msgpack_pack_long(msgpack_packer* x, long d)
 {
 #if defined(SIZEOF_LONG)
 #if SIZEOF_LONG == 2
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 #elif SIZEOF_LONG == 4
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 #else
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 #endif
 
 #elif defined(LONG_MAX)
 #if LONG_MAX == 0x7fffL
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 #elif LONG_MAX == 0x7fffffffL
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 #else
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 #endif
 
 #else
 if(sizeof(long) == 2) {
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 } else if(sizeof(long) == 4) {
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 } else {
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 }
 #endif
 }
@@ -404,29 +404,29 @@ static inline int msgpack_pack_long_long(msgpack_packer* x, long long d)
 {
 #if defined(SIZEOF_LONG_LONG)
 #if SIZEOF_LONG_LONG == 2
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 #elif SIZEOF_LONG_LONG == 4
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 #else
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 #endif
 
 #elif defined(LLONG_MAX)
 #if LLONG_MAX == 0x7fffL
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 #elif LLONG_MAX == 0x7fffffffL
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 #else
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 #endif
 
 #else
 if(sizeof(long long) == 2) {
-	msgpack_pack_real_int16(x, d);
+    msgpack_pack_real_int16(x, d);
 } else if(sizeof(long long) == 4) {
-	msgpack_pack_real_int32(x, d);
+    msgpack_pack_real_int32(x, d);
 } else {
-	msgpack_pack_real_int64(x, d);
+    msgpack_pack_real_int64(x, d);
 }
 #endif
 }
@@ -435,29 +435,29 @@ static inline int msgpack_pack_unsigned_short(msgpack_packer* x, unsigned short
 {
 #if defined(SIZEOF_SHORT)
 #if SIZEOF_SHORT == 2
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 #elif SIZEOF_SHORT == 4
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 #else
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 #endif
 
 #elif defined(USHRT_MAX)
 #if USHRT_MAX == 0xffffU
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 #elif USHRT_MAX == 0xffffffffU
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 #else
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 #endif
 
 #else
 if(sizeof(unsigned short) == 2) {
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 } else if(sizeof(unsigned short) == 4) {
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 } else {
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 }
 #endif
 }
@@ -466,29 +466,29 @@ static inline int msgpack_pack_unsigned_int(msgpack_packer* x, unsigned int d)
 {
 #if defined(SIZEOF_INT)
 #if SIZEOF_INT == 2
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 #elif SIZEOF_INT == 4
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 #else
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 #endif
 
 #elif defined(UINT_MAX)
 #if UINT_MAX == 0xffffU
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 #elif UINT_MAX == 0xffffffffU
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 #else
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 #endif
 
 #else
 if(sizeof(unsigned int) == 2) {
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 } else if(sizeof(unsigned int) == 4) {
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 } else {
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 }
 #endif
 }
@@ -497,29 +497,29 @@ static inline int msgpack_pack_unsigned_long(msgpack_packer* x, unsigned long d)
 {
 #if defined(SIZEOF_LONG)
 #if SIZEOF_LONG == 2
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 #elif SIZEOF_LONG == 4
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 #else
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 #endif
 
 #elif defined(ULONG_MAX)
 #if ULONG_MAX == 0xffffUL
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 #elif ULONG_MAX == 0xffffffffUL
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 #else
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 #endif
 
 #else
 if(sizeof(unsigned long) == 2) {
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 } else if(sizeof(unsigned long) == 4) {
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 } else {
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 }
 #endif
 }
@@ -528,29 +528,29 @@ static inline int msgpack_pack_unsigned_long_long(msgpack_packer* x, unsigned lo
 {
 #if defined(SIZEOF_LONG_LONG)
 #if SIZEOF_LONG_LONG == 2
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 #elif SIZEOF_LONG_LONG == 4
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 #else
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 #endif
 
 #elif defined(ULLONG_MAX)
 #if ULLONG_MAX == 0xffffUL
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 #elif ULLONG_MAX == 0xffffffffUL
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 #else
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 #endif
 
 #else
 if(sizeof(unsigned long long) == 2) {
-	msgpack_pack_real_uint16(x, d);
+    msgpack_pack_real_uint16(x, d);
 } else if(sizeof(unsigned long long) == 4) {
-	msgpack_pack_real_uint32(x, d);
+    msgpack_pack_real_uint32(x, d);
 } else {
-	msgpack_pack_real_uint64(x, d);
+    msgpack_pack_real_uint64(x, d);
 }
 #endif
 }
@@ -566,25 +566,25 @@ if(sizeof(unsigned long long) == 2) {
 
 static inline int msgpack_pack_float(msgpack_packer* x, float d)
 {
-	union { float f; uint32_t i; } mem;
-	mem.f = d;
-	unsigned char buf[5];
-	buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i);
-	msgpack_pack_append_buffer(x, buf, 5);
+    union { float f; uint32_t i; } mem;
+    mem.f = d;
+    unsigned char buf[5];
+    buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i);
+    msgpack_pack_append_buffer(x, buf, 5);
 }
 
 static inline int msgpack_pack_double(msgpack_packer* x, double d)
 {
-	union { double f; uint64_t i; } mem;
-	mem.f = d;
-	unsigned char buf[9];
-	buf[0] = 0xcb;
+    union { double f; uint64_t i; } mem;
+    mem.f = d;
+    unsigned char buf[9];
+    buf[0] = 0xcb;
 #if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
     // https://github.com/msgpack/msgpack-perl/pull/1
     mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
 #endif
     _msgpack_store64(&buf[1], mem.i);
-	msgpack_pack_append_buffer(x, buf, 9);
+    msgpack_pack_append_buffer(x, buf, 9);
 }
 
 
@@ -594,8 +594,8 @@ static inline int msgpack_pack_double(msgpack_packer* x, double d)
 
 static inline int msgpack_pack_nil(msgpack_packer* x)
 {
-	static const unsigned char d = 0xc0;
-	msgpack_pack_append_buffer(x, &d, 1);
+    static const unsigned char d = 0xc0;
+    msgpack_pack_append_buffer(x, &d, 1);
 }
 
 
@@ -605,14 +605,14 @@ static inline int msgpack_pack_nil(msgpack_packer* x)
 
 static inline int msgpack_pack_true(msgpack_packer* x)
 {
-	static const unsigned char d = 0xc3;
-	msgpack_pack_append_buffer(x, &d, 1);
+    static const unsigned char d = 0xc3;
+    msgpack_pack_append_buffer(x, &d, 1);
 }
 
 static inline int msgpack_pack_false(msgpack_packer* x)
 {
-	static const unsigned char d = 0xc2;
-	msgpack_pack_append_buffer(x, &d, 1);
+    static const unsigned char d = 0xc2;
+    msgpack_pack_append_buffer(x, &d, 1);
 }
 
 
@@ -622,18 +622,18 @@ static inline int msgpack_pack_false(msgpack_packer* x)
 
 static inline int msgpack_pack_array(msgpack_packer* x, unsigned int n)
 {
-	if(n < 16) {
-		unsigned char d = 0x90 | n;
-		msgpack_pack_append_buffer(x, &d, 1);
-	} else if(n < 65536) {
-		unsigned char buf[3];
-		buf[0] = 0xdc; _msgpack_store16(&buf[1], (uint16_t)n);
-		msgpack_pack_append_buffer(x, buf, 3);
-	} else {
-		unsigned char buf[5];
-		buf[0] = 0xdd; _msgpack_store32(&buf[1], (uint32_t)n);
-		msgpack_pack_append_buffer(x, buf, 5);
-	}
+    if(n < 16) {
+        unsigned char d = 0x90 | n;
+        msgpack_pack_append_buffer(x, &d, 1);
+    } else if(n < 65536) {
+        unsigned char buf[3];
+        buf[0] = 0xdc; _msgpack_store16(&buf[1], (uint16_t)n);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5];
+        buf[0] = 0xdd; _msgpack_store32(&buf[1], (uint32_t)n);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
 }
 
 
@@ -643,18 +643,18 @@ static inline int msgpack_pack_array(msgpack_packer* x, unsigned int n)
 
 static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n)
 {
-	if(n < 16) {
-		unsigned char d = 0x80 | n;
-		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
-	} else if(n < 65536) {
-		unsigned char buf[3];
-		buf[0] = 0xde; _msgpack_store16(&buf[1], (uint16_t)n);
-		msgpack_pack_append_buffer(x, buf, 3);
-	} else {
-		unsigned char buf[5];
-		buf[0] = 0xdf; _msgpack_store32(&buf[1], (uint32_t)n);
-		msgpack_pack_append_buffer(x, buf, 5);
-	}
+    if(n < 16) {
+        unsigned char d = 0x80 | n;
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
+    } else if(n < 65536) {
+        unsigned char buf[3];
+        buf[0] = 0xde; _msgpack_store16(&buf[1], (uint16_t)n);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5];
+        buf[0] = 0xdf; _msgpack_store32(&buf[1], (uint32_t)n);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
 }
 
 
@@ -664,23 +664,23 @@ static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n)
 
 static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
 {
-	if(l < 32) {
-		unsigned char d = 0xa0 | (uint8_t)l;
-		msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
-	} else if(l < 65536) {
-		unsigned char buf[3];
-		buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l);
-		msgpack_pack_append_buffer(x, buf, 3);
-	} else {
-		unsigned char buf[5];
-		buf[0] = 0xdb; _msgpack_store32(&buf[1], (uint32_t)l);
-		msgpack_pack_append_buffer(x, buf, 5);
-	}
+    if(l < 32) {
+        unsigned char d = 0xa0 | (uint8_t)l;
+        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
+    } else if(l < 65536) {
+        unsigned char buf[3];
+        buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5];
+        buf[0] = 0xdb; _msgpack_store32(&buf[1], (uint32_t)l);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
 }
 
 static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l)
 {
-	msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
+    msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
 }
 
 #undef msgpack_pack_append_buffer
@@ -698,4 +698,3 @@ static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t
 #undef msgpack_pack_real_int16
 #undef msgpack_pack_real_int32
 #undef msgpack_pack_real_int64
-
diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h
index 4fedbd8..ed9c1bc 100644
--- a/msgpack/sysdep.h
+++ b/msgpack/sysdep.h
@@ -192,4 +192,3 @@ typedef unsigned int _msgpack_atomic_counter_t;
 
 
 #endif /* msgpack/sysdep.h */
-
diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h
index 959d351..c81b990 100644
--- a/msgpack/unpack_define.h
+++ b/msgpack/unpack_define.h
@@ -35,53 +35,53 @@ extern "C" {
 
 
 typedef enum {
-	CS_HEADER            = 0x00,  // nil
+    CS_HEADER            = 0x00,  // nil
 
-	//CS_                = 0x01,
-	//CS_                = 0x02,  // false
-	//CS_                = 0x03,  // true
+    //CS_                = 0x01,
+    //CS_                = 0x02,  // false
+    //CS_                = 0x03,  // true
 
-	//CS_                = 0x04,
-	//CS_                = 0x05,
-	//CS_                = 0x06,
-	//CS_                = 0x07,
+    //CS_                = 0x04,
+    //CS_                = 0x05,
+    //CS_                = 0x06,
+    //CS_                = 0x07,
 
-	//CS_                = 0x08,
-	//CS_                = 0x09,
-	CS_FLOAT             = 0x0a,
-	CS_DOUBLE            = 0x0b,
-	CS_UINT_8            = 0x0c,
-	CS_UINT_16           = 0x0d,
-	CS_UINT_32           = 0x0e,
-	CS_UINT_64           = 0x0f,
-	CS_INT_8             = 0x10,
-	CS_INT_16            = 0x11,
-	CS_INT_32            = 0x12,
-	CS_INT_64            = 0x13,
+    //CS_                = 0x08,
+    //CS_                = 0x09,
+    CS_FLOAT             = 0x0a,
+    CS_DOUBLE            = 0x0b,
+    CS_UINT_8            = 0x0c,
+    CS_UINT_16           = 0x0d,
+    CS_UINT_32           = 0x0e,
+    CS_UINT_64           = 0x0f,
+    CS_INT_8             = 0x10,
+    CS_INT_16            = 0x11,
+    CS_INT_32            = 0x12,
+    CS_INT_64            = 0x13,
 
-	//CS_                = 0x14,
-	//CS_                = 0x15,
-	//CS_BIG_INT_16        = 0x16,
-	//CS_BIG_INT_32        = 0x17,
-	//CS_BIG_FLOAT_16      = 0x18,
-	//CS_BIG_FLOAT_32      = 0x19,
-	CS_RAW_16            = 0x1a,
-	CS_RAW_32            = 0x1b,
-	CS_ARRAY_16          = 0x1c,
-	CS_ARRAY_32          = 0x1d,
-	CS_MAP_16            = 0x1e,
-	CS_MAP_32            = 0x1f,
+    //CS_                = 0x14,
+    //CS_                = 0x15,
+    //CS_BIG_INT_16        = 0x16,
+    //CS_BIG_INT_32        = 0x17,
+    //CS_BIG_FLOAT_16      = 0x18,
+    //CS_BIG_FLOAT_32      = 0x19,
+    CS_RAW_16            = 0x1a,
+    CS_RAW_32            = 0x1b,
+    CS_ARRAY_16          = 0x1c,
+    CS_ARRAY_32          = 0x1d,
+    CS_MAP_16            = 0x1e,
+    CS_MAP_32            = 0x1f,
 
-	//ACS_BIG_INT_VALUE,
-	//ACS_BIG_FLOAT_VALUE,
-	ACS_RAW_VALUE,
+    //ACS_BIG_INT_VALUE,
+    //ACS_BIG_FLOAT_VALUE,
+    ACS_RAW_VALUE,
 } msgpack_unpack_state;
 
 
 typedef enum {
-	CT_ARRAY_ITEM,
-	CT_MAP_KEY,
-	CT_MAP_VALUE,
+    CT_ARRAY_ITEM,
+    CT_MAP_KEY,
+    CT_MAP_VALUE,
 } msgpack_container_type;
 
 
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 153340c..29ac935 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -23,136 +23,136 @@
 #endif
 
 typedef struct unpack_stack {
-	PyObject* obj;
-	size_t size;
-	size_t count;
-	unsigned int ct;
-	PyObject* map_key;
+    PyObject* obj;
+    size_t size;
+    size_t count;
+    unsigned int ct;
+    PyObject* map_key;
 } unpack_stack;
 
 struct unpack_context {
-	unpack_user user;
-	unsigned int cs;
-	unsigned int trail;
-	unsigned int top;
-	/*
-	unpack_stack* stack;
-	unsigned int stack_size;
-	unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE];
-	*/
-	unpack_stack stack[MSGPACK_EMBED_STACK_SIZE];
+    unpack_user user;
+    unsigned int cs;
+    unsigned int trail;
+    unsigned int top;
+    /*
+    unpack_stack* stack;
+    unsigned int stack_size;
+    unpack_stack embed_stack[MSGPACK_EMBED_STACK_SIZE];
+    */
+    unpack_stack stack[MSGPACK_EMBED_STACK_SIZE];
 };
 
 
 static inline void unpack_init(unpack_context* ctx)
 {
-	ctx->cs = CS_HEADER;
-	ctx->trail = 0;
-	ctx->top = 0;
-	/*
-	ctx->stack = ctx->embed_stack;
-	ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
-	*/
-	ctx->stack[0].obj = unpack_callback_root(&ctx->user);
+    ctx->cs = CS_HEADER;
+    ctx->trail = 0;
+    ctx->top = 0;
+    /*
+    ctx->stack = ctx->embed_stack;
+    ctx->stack_size = MSGPACK_EMBED_STACK_SIZE;
+    */
+    ctx->stack[0].obj = unpack_callback_root(&ctx->user);
 }
 
 /*
 static inline void unpack_destroy(unpack_context* ctx)
 {
-	if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
-		free(ctx->stack);
-	}
+    if(ctx->stack_size != MSGPACK_EMBED_STACK_SIZE) {
+        free(ctx->stack);
+    }
 }
 */
 
 static inline PyObject* unpack_data(unpack_context* ctx)
 {
-	return (ctx)->stack[0].obj;
+    return (ctx)->stack[0].obj;
 }
 
 
 template 
 static inline int unpack_execute(unpack_context* ctx, const char* data, size_t len, size_t* off)
 {
-	assert(len >= *off);
+    assert(len >= *off);
 
-	const unsigned char* p = (unsigned char*)data + *off;
-	const unsigned char* const pe = (unsigned char*)data + len;
-	const void* n = NULL;
+    const unsigned char* p = (unsigned char*)data + *off;
+    const unsigned char* const pe = (unsigned char*)data + len;
+    const void* n = NULL;
 
-	unsigned int trail = ctx->trail;
-	unsigned int cs = ctx->cs;
-	unsigned int top = ctx->top;
-	unpack_stack* stack = ctx->stack;
-	/*
-	unsigned int stack_size = ctx->stack_size;
-	*/
-	unpack_user* user = &ctx->user;
+    unsigned int trail = ctx->trail;
+    unsigned int cs = ctx->cs;
+    unsigned int top = ctx->top;
+    unpack_stack* stack = ctx->stack;
+    /*
+    unsigned int stack_size = ctx->stack_size;
+    */
+    unpack_user* user = &ctx->user;
 
-	PyObject* obj;
-	unpack_stack* c = NULL;
+    PyObject* obj;
+    unpack_stack* c = NULL;
 
-	int ret;
+    int ret;
 
 #define construct_cb(name) \
     construct && unpack_callback ## name
 
 #define push_simple_value(func) \
-	if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \
-	goto _push
+    if(construct_cb(func)(user, &obj) < 0) { goto _failed; } \
+    goto _push
 #define push_fixed_value(func, arg) \
-	if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \
-	goto _push
+    if(construct_cb(func)(user, arg, &obj) < 0) { goto _failed; } \
+    goto _push
 #define push_variable_value(func, base, pos, len) \
-	if(construct_cb(func)(user, \
-		(const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
-	goto _push
+    if(construct_cb(func)(user, \
+        (const char*)base, (const char*)pos, len, &obj) < 0) { goto _failed; } \
+    goto _push
 
 #define again_fixed_trail(_cs, trail_len) \
-	trail = trail_len; \
-	cs = _cs; \
-	goto _fixed_trail_again
+    trail = trail_len; \
+    cs = _cs; \
+    goto _fixed_trail_again
 #define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \
-	trail = trail_len; \
-	if(trail == 0) { goto ifzero; } \
-	cs = _cs; \
-	goto _fixed_trail_again
+    trail = trail_len; \
+    if(trail == 0) { goto ifzero; } \
+    cs = _cs; \
+    goto _fixed_trail_again
 
 #define start_container(func, count_, ct_) \
-	if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
-	if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
-	if((count_) == 0) { obj = stack[top].obj; \
-		if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \
-		goto _push; } \
-	stack[top].ct = ct_; \
-	stack[top].size  = count_; \
-	stack[top].count = 0; \
-	++top; \
-	/*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
-	/*printf("stack push %d\n", top);*/ \
-	/* FIXME \
-	if(top >= stack_size) { \
-		if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
-			size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \
-			size_t nsize = csize * 2; \
-			unpack_stack* tmp = (unpack_stack*)malloc(nsize); \
-			if(tmp == NULL) { goto _failed; } \
-			memcpy(tmp, ctx->stack, csize); \
-			ctx->stack = stack = tmp; \
-			ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
-		} else { \
-			size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \
-			unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \
-			if(tmp == NULL) { goto _failed; } \
-			ctx->stack = stack = tmp; \
-			ctx->stack_size = stack_size = stack_size * 2; \
-		} \
-	} \
-	*/ \
-	goto _header_again
+    if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
+    if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
+    if((count_) == 0) { obj = stack[top].obj; \
+        if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \
+        goto _push; } \
+    stack[top].ct = ct_; \
+    stack[top].size  = count_; \
+    stack[top].count = 0; \
+    ++top; \
+    /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
+    /*printf("stack push %d\n", top);*/ \
+    /* FIXME \
+    if(top >= stack_size) { \
+        if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
+            size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \
+            size_t nsize = csize * 2; \
+            unpack_stack* tmp = (unpack_stack*)malloc(nsize); \
+            if(tmp == NULL) { goto _failed; } \
+            memcpy(tmp, ctx->stack, csize); \
+            ctx->stack = stack = tmp; \
+            ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
+        } else { \
+            size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \
+            unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \
+            if(tmp == NULL) { goto _failed; } \
+            ctx->stack = stack = tmp; \
+            ctx->stack_size = stack_size = stack_size * 2; \
+        } \
+    } \
+    */ \
+    goto _header_again
 
 #define NEXT_CS(p) \
-	((unsigned int)*p & 0x1f)
+    ((unsigned int)*p & 0x1f)
 
 #ifdef USE_CASE_RANGE
 #define SWITCH_RANGE_BEGIN     switch(*p) {
@@ -166,221 +166,221 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
 #define SWITCH_RANGE_END       } }
 #endif
 
-	if(p == pe) { goto _out; }
-	do {
-		switch(cs) {
-		case CS_HEADER:
-			SWITCH_RANGE_BEGIN
-			SWITCH_RANGE(0x00, 0x7f)  // Positive Fixnum
-				push_fixed_value(_uint8, *(uint8_t*)p);
-			SWITCH_RANGE(0xe0, 0xff)  // Negative Fixnum
-				push_fixed_value(_int8, *(int8_t*)p);
-			SWITCH_RANGE(0xc0, 0xdf)  // Variable
-				switch(*p) {
-				case 0xc0:  // nil
-					push_simple_value(_nil);
-				//case 0xc1:  // string
-				//	again_terminal_trail(NEXT_CS(p), p+1);
-				case 0xc2:  // false
-					push_simple_value(_false);
-				case 0xc3:  // true
-					push_simple_value(_true);
-				//case 0xc4:
-				//case 0xc5:
-				//case 0xc6:
-				//case 0xc7:
-				//case 0xc8:
-				//case 0xc9:
-				case 0xca:  // float
-				case 0xcb:  // double
-				case 0xcc:  // unsigned int  8
-				case 0xcd:  // unsigned int 16
-				case 0xce:  // unsigned int 32
-				case 0xcf:  // unsigned int 64
-				case 0xd0:  // signed int  8
-				case 0xd1:  // signed int 16
-				case 0xd2:  // signed int 32
-				case 0xd3:  // signed int 64
-					again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
-				//case 0xd4:
-				//case 0xd5:
-				//case 0xd6:  // big integer 16
-				//case 0xd7:  // big integer 32
-				//case 0xd8:  // big float 16
-				//case 0xd9:  // big float 32
-				case 0xda:  // raw 16
-				case 0xdb:  // raw 32
-				case 0xdc:  // array 16
-				case 0xdd:  // array 32
-				case 0xde:  // map 16
-				case 0xdf:  // map 32
-					again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
-				default:
-					goto _failed;
-				}
-			SWITCH_RANGE(0xa0, 0xbf)  // FixRaw
-				again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
-			SWITCH_RANGE(0x90, 0x9f)  // FixArray
-				start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM);
-			SWITCH_RANGE(0x80, 0x8f)  // FixMap
-				start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
+    if(p == pe) { goto _out; }
+    do {
+        switch(cs) {
+        case CS_HEADER:
+            SWITCH_RANGE_BEGIN
+            SWITCH_RANGE(0x00, 0x7f)  // Positive Fixnum
+                push_fixed_value(_uint8, *(uint8_t*)p);
+            SWITCH_RANGE(0xe0, 0xff)  // Negative Fixnum
+                push_fixed_value(_int8, *(int8_t*)p);
+            SWITCH_RANGE(0xc0, 0xdf)  // Variable
+                switch(*p) {
+                case 0xc0:  // nil
+                    push_simple_value(_nil);
+                //case 0xc1:  // string
+                //    again_terminal_trail(NEXT_CS(p), p+1);
+                case 0xc2:  // false
+                    push_simple_value(_false);
+                case 0xc3:  // true
+                    push_simple_value(_true);
+                //case 0xc4:
+                //case 0xc5:
+                //case 0xc6:
+                //case 0xc7:
+                //case 0xc8:
+                //case 0xc9:
+                case 0xca:  // float
+                case 0xcb:  // double
+                case 0xcc:  // unsigned int  8
+                case 0xcd:  // unsigned int 16
+                case 0xce:  // unsigned int 32
+                case 0xcf:  // unsigned int 64
+                case 0xd0:  // signed int  8
+                case 0xd1:  // signed int 16
+                case 0xd2:  // signed int 32
+                case 0xd3:  // signed int 64
+                    again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
+                //case 0xd4:
+                //case 0xd5:
+                //case 0xd6:  // big integer 16
+                //case 0xd7:  // big integer 32
+                //case 0xd8:  // big float 16
+                //case 0xd9:  // big float 32
+                case 0xda:  // raw 16
+                case 0xdb:  // raw 32
+                case 0xdc:  // array 16
+                case 0xdd:  // array 32
+                case 0xde:  // map 16
+                case 0xdf:  // map 32
+                    again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
+                default:
+                    goto _failed;
+                }
+            SWITCH_RANGE(0xa0, 0xbf)  // FixRaw
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
+            SWITCH_RANGE(0x90, 0x9f)  // FixArray
+                start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM);
+            SWITCH_RANGE(0x80, 0x8f)  // FixMap
+                start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
 
-			SWITCH_RANGE_DEFAULT
-				goto _failed;
-			SWITCH_RANGE_END
-			// end CS_HEADER
+            SWITCH_RANGE_DEFAULT
+                goto _failed;
+            SWITCH_RANGE_END
+            // end CS_HEADER
 
 
-		_fixed_trail_again:
-			++p;
+        _fixed_trail_again:
+            ++p;
 
-		default:
-			if((size_t)(pe - p) < trail) { goto _out; }
-			n = p;  p += trail - 1;
-			switch(cs) {
-			//case CS_
-			//case CS_
-			case CS_FLOAT: {
-					union { uint32_t i; float f; } mem;
-					mem.i = _msgpack_load32(uint32_t,n);
-					push_fixed_value(_float, mem.f); }
-			case CS_DOUBLE: {
-					union { uint64_t i; double f; } mem;
-					mem.i = _msgpack_load64(uint64_t,n);
+        default:
+            if((size_t)(pe - p) < trail) { goto _out; }
+            n = p;  p += trail - 1;
+            switch(cs) {
+            //case CS_
+            //case CS_
+            case CS_FLOAT: {
+                    union { uint32_t i; float f; } mem;
+                    mem.i = _msgpack_load32(uint32_t,n);
+                    push_fixed_value(_float, mem.f); }
+            case CS_DOUBLE: {
+                    union { uint64_t i; double f; } mem;
+                    mem.i = _msgpack_load64(uint64_t,n);
 #if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
                     // https://github.com/msgpack/msgpack-perl/pull/1
                     mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
 #endif
-					push_fixed_value(_double, mem.f); }
-			case CS_UINT_8:
-				push_fixed_value(_uint8, *(uint8_t*)n);
-			case CS_UINT_16:
-				push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
-			case CS_UINT_32:
-				push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
-			case CS_UINT_64:
-				push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
+                    push_fixed_value(_double, mem.f); }
+            case CS_UINT_8:
+                push_fixed_value(_uint8, *(uint8_t*)n);
+            case CS_UINT_16:
+                push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
+            case CS_UINT_32:
+                push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
+            case CS_UINT_64:
+                push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
 
-			case CS_INT_8:
-				push_fixed_value(_int8, *(int8_t*)n);
-			case CS_INT_16:
-				push_fixed_value(_int16, _msgpack_load16(int16_t,n));
-			case CS_INT_32:
-				push_fixed_value(_int32, _msgpack_load32(int32_t,n));
-			case CS_INT_64:
-				push_fixed_value(_int64, _msgpack_load64(int64_t,n));
+            case CS_INT_8:
+                push_fixed_value(_int8, *(int8_t*)n);
+            case CS_INT_16:
+                push_fixed_value(_int16, _msgpack_load16(int16_t,n));
+            case CS_INT_32:
+                push_fixed_value(_int32, _msgpack_load32(int32_t,n));
+            case CS_INT_64:
+                push_fixed_value(_int64, _msgpack_load64(int64_t,n));
 
-			//case CS_
-			//case CS_
-			//case CS_BIG_INT_16:
-			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero);
-			//case CS_BIG_INT_32:
-			//	again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero);
-			//case ACS_BIG_INT_VALUE:
-			//_big_int_zero:
-			//	// FIXME
-			//	push_variable_value(_big_int, data, n, trail);
+            //case CS_
+            //case CS_
+            //case CS_BIG_INT_16:
+            //    again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load16(uint16_t,n), _big_int_zero);
+            //case CS_BIG_INT_32:
+            //    again_fixed_trail_if_zero(ACS_BIG_INT_VALUE, _msgpack_load32(uint32_t,n), _big_int_zero);
+            //case ACS_BIG_INT_VALUE:
+            //_big_int_zero:
+            //    // FIXME
+            //    push_variable_value(_big_int, data, n, trail);
 
-			//case CS_BIG_FLOAT_16:
-			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero);
-			//case CS_BIG_FLOAT_32:
-			//	again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero);
-			//case ACS_BIG_FLOAT_VALUE:
-			//_big_float_zero:
-			//	// FIXME
-			//	push_variable_value(_big_float, data, n, trail);
+            //case CS_BIG_FLOAT_16:
+            //    again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load16(uint16_t,n), _big_float_zero);
+            //case CS_BIG_FLOAT_32:
+            //    again_fixed_trail_if_zero(ACS_BIG_FLOAT_VALUE, _msgpack_load32(uint32_t,n), _big_float_zero);
+            //case ACS_BIG_FLOAT_VALUE:
+            //_big_float_zero:
+            //    // FIXME
+            //    push_variable_value(_big_float, data, n, trail);
 
-			case CS_RAW_16:
-				again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
-			case CS_RAW_32:
-				again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
-			case ACS_RAW_VALUE:
-			_raw_zero:
-				push_variable_value(_raw, data, n, trail);
+            case CS_RAW_16:
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
+            case CS_RAW_32:
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
+            case ACS_RAW_VALUE:
+            _raw_zero:
+                push_variable_value(_raw, data, n, trail);
 
-			case CS_ARRAY_16:
-				start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
-			case CS_ARRAY_32:
-				/* FIXME security guard */
-				start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
+            case CS_ARRAY_16:
+                start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
+            case CS_ARRAY_32:
+                /* FIXME security guard */
+                start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
 
-			case CS_MAP_16:
-				start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
-			case CS_MAP_32:
-				/* FIXME security guard */
-				start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
+            case CS_MAP_16:
+                start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
+            case CS_MAP_32:
+                /* FIXME security guard */
+                start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
 
-			default:
-				goto _failed;
-			}
-		}
+            default:
+                goto _failed;
+            }
+        }
 
 _push:
-	if(top == 0) { goto _finish; }
-	c = &stack[top-1];
-	switch(c->ct) {
-	case CT_ARRAY_ITEM:
-		if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
-		if(++c->count == c->size) {
-			obj = c->obj;
-			if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; }
-			--top;
-			/*printf("stack pop %d\n", top);*/
-			goto _push;
-		}
-		goto _header_again;
-	case CT_MAP_KEY:
-		c->map_key = obj;
-		c->ct = CT_MAP_VALUE;
-		goto _header_again;
-	case CT_MAP_VALUE:
-		if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
-		if(++c->count == c->size) {
-			obj = c->obj;
-			if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; }
-			--top;
-			/*printf("stack pop %d\n", top);*/
-			goto _push;
-		}
-		c->ct = CT_MAP_KEY;
-		goto _header_again;
+    if(top == 0) { goto _finish; }
+    c = &stack[top-1];
+    switch(c->ct) {
+    case CT_ARRAY_ITEM:
+        if(construct_cb(_array_item)(user, c->count, &c->obj, obj) < 0) { goto _failed; }
+        if(++c->count == c->size) {
+            obj = c->obj;
+            if (construct_cb(_array_end)(user, &obj) < 0) { goto _failed; }
+            --top;
+            /*printf("stack pop %d\n", top);*/
+            goto _push;
+        }
+        goto _header_again;
+    case CT_MAP_KEY:
+        c->map_key = obj;
+        c->ct = CT_MAP_VALUE;
+        goto _header_again;
+    case CT_MAP_VALUE:
+        if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; }
+        if(++c->count == c->size) {
+            obj = c->obj;
+            if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; }
+            --top;
+            /*printf("stack pop %d\n", top);*/
+            goto _push;
+        }
+        c->ct = CT_MAP_KEY;
+        goto _header_again;
 
-	default:
-		goto _failed;
-	}
+    default:
+        goto _failed;
+    }
 
 _header_again:
-		cs = CS_HEADER;
-		++p;
-	} while(p != pe);
-	goto _out;
+        cs = CS_HEADER;
+        ++p;
+    } while(p != pe);
+    goto _out;
 
 
 _finish:
-	if (!construct)
-		unpack_callback_nil(user, &obj);
-	stack[0].obj = obj;
-	++p;
-	ret = 1;
-	/*printf("-- finish --\n"); */
-	goto _end;
+    if (!construct)
+        unpack_callback_nil(user, &obj);
+    stack[0].obj = obj;
+    ++p;
+    ret = 1;
+    /*printf("-- finish --\n"); */
+    goto _end;
 
 _failed:
-	/*printf("** FAILED **\n"); */
-	ret = -1;
-	goto _end;
+    /*printf("** FAILED **\n"); */
+    ret = -1;
+    goto _end;
 
 _out:
-	ret = 0;
-	goto _end;
+    ret = 0;
+    goto _end;
 
 _end:
-	ctx->cs = cs;
-	ctx->trail = trail;
-	ctx->top = top;
-	*off = p - (const unsigned char*)data;
+    ctx->cs = cs;
+    ctx->trail = trail;
+    ctx->top = top;
+    *off = p - (const unsigned char*)data;
 
-	return ret;
+    return ret;
 #undef construct_cb
 }
 
@@ -398,53 +398,53 @@ _end:
 template 
 static inline int unpack_container_header(unpack_context* ctx, const char* data, size_t len, size_t* off)
 {
-	assert(len >= *off);
-	uint32_t size;
-	const unsigned char *const p = (unsigned char*)data + *off;
+    assert(len >= *off);
+    uint32_t size;
+    const unsigned char *const p = (unsigned char*)data + *off;
 
 #define inc_offset(inc) \
-	if (len - *off < inc) \
-		return 0; \
-	*off += inc;
+    if (len - *off < inc) \
+        return 0; \
+    *off += inc;
 
-	switch (*p) {
-	case var_offset:
-		inc_offset(3);
-		size = _msgpack_load16(uint16_t, p + 1);
-		break;
-	case var_offset + 1:
-		inc_offset(5);
-		size = _msgpack_load32(uint32_t, p + 1);
-		break;
+    switch (*p) {
+    case var_offset:
+        inc_offset(3);
+        size = _msgpack_load16(uint16_t, p + 1);
+        break;
+    case var_offset + 1:
+        inc_offset(5);
+        size = _msgpack_load32(uint32_t, p + 1);
+        break;
 #ifdef USE_CASE_RANGE
-	case fixed_offset + 0x0 ... fixed_offset + 0xf:
+    case fixed_offset + 0x0 ... fixed_offset + 0xf:
 #else
-	case fixed_offset + 0x0:
-	case fixed_offset + 0x1:
-	case fixed_offset + 0x2:
-	case fixed_offset + 0x3:
-	case fixed_offset + 0x4:
-	case fixed_offset + 0x5:
-	case fixed_offset + 0x6:
-	case fixed_offset + 0x7:
-	case fixed_offset + 0x8:
-	case fixed_offset + 0x9:
-	case fixed_offset + 0xa:
-	case fixed_offset + 0xb:
-	case fixed_offset + 0xc:
-	case fixed_offset + 0xd:
-	case fixed_offset + 0xe:
-	case fixed_offset + 0xf:
+    case fixed_offset + 0x0:
+    case fixed_offset + 0x1:
+    case fixed_offset + 0x2:
+    case fixed_offset + 0x3:
+    case fixed_offset + 0x4:
+    case fixed_offset + 0x5:
+    case fixed_offset + 0x6:
+    case fixed_offset + 0x7:
+    case fixed_offset + 0x8:
+    case fixed_offset + 0x9:
+    case fixed_offset + 0xa:
+    case fixed_offset + 0xb:
+    case fixed_offset + 0xc:
+    case fixed_offset + 0xd:
+    case fixed_offset + 0xe:
+    case fixed_offset + 0xf:
 #endif
-		++*off;
-		size = ((unsigned int)*p) & 0x0f;
-		break;
-	default:
-		PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
-		return -1;
+        ++*off;
+        size = ((unsigned int)*p) & 0x0f;
+        break;
+    default:
+        PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
+        return -1;
     }
-	unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj);
-	return 1;
+    unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj);
+    return 1;
 }
 
 #undef SWITCH_RANGE_BEGIN
@@ -459,4 +459,4 @@ static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>;
 
 #undef NEXT_CS
 
-/* vim: set ts=4 sw=4 noexpandtab  */
+/* vim: set ts=4 sw=4 sts=4 expandtab  */

From 5c90f953dae37610f478183849b77413d6ec8e9f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 13 Mar 2013 18:15:43 +0900
Subject: [PATCH 1172/1648] Add build batch for Windows

---
 build_windows.bat  | 24 ++++++++++++++++++++++++
 upload_windows.bat |  4 ++++
 2 files changed, 28 insertions(+)
 create mode 100644 build_windows.bat
 create mode 100644 upload_windows.bat

diff --git a/build_windows.bat b/build_windows.bat
new file mode 100644
index 0000000..a71c0e0
--- /dev/null
+++ b/build_windows.bat
@@ -0,0 +1,24 @@
+set MSSdk=1
+set DISTUTILS_USE_SDK=1
+
+rem Python27 x86
+rem call "C:\Program Files\Microsoft SDKs\Windows\v6.1\Bin\SetEnv.cmd" /Release /x86 /xp
+call "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars32.bat"
+c:\Python27\python setup.py build_ext -f build install
+pause
+
+rem Python27 amd64
+rem call "C:\Program Files\Microsoft SDKs\Windows\v6.1\Bin\SetEnv.cmd" /Release /x64 /xp
+call "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat"
+c:\Python27_amd64\python setup.py build_ext -f build install
+pause
+
+rem Python33 x86
+call "C:\Program Files\Microsoft SDKs\Windows\v7.1\bin\SetEnv.cmd" /Release /x86 /xp
+c:\Python33\python setup.py build_ext -f build install
+pause
+
+rem Python33 amd64
+call "C:\Program Files\Microsoft SDKs\Windows\v7.1\bin\SetEnv.cmd" /Release /x64 /xp
+c:\Python33_amd64\python setup.py build_ext -f build install
+pause
diff --git a/upload_windows.bat b/upload_windows.bat
new file mode 100644
index 0000000..5cd9a7c
--- /dev/null
+++ b/upload_windows.bat
@@ -0,0 +1,4 @@
+c:\Python27\python       setup.py bdist_egg bdist_wininst upload
+c:\Python33\python       setup.py bdist_egg bdist_wininst upload
+c:\Python27_amd64\python setup.py bdist_egg bdist_wininst upload
+c:\Python33_amd64\python setup.py bdist_egg bdist_wininst upload

From 0faa1bb558ab671b9503c5e22a97c0ec24ab5a19 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 8 Apr 2013 01:57:37 +0900
Subject: [PATCH 1173/1648] Remove unnecessary type declaration.

---
 msgpack/_packer.pyx   | 6 ------
 msgpack/_unpacker.pyx | 3 +--
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 562c92c..93323c4 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -2,12 +2,6 @@
 #cython: embedsignature=True
 
 from cpython cimport *
-cdef extern from "Python.h":
-    ctypedef char* const_char_ptr "const char*"
-    ctypedef char* const_void_ptr "const void*"
-    ctypedef struct PyObject
-    cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
-
 from libc.stdlib cimport *
 from libc.string cimport *
 from libc.limits cimport *
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 5813dc9..1ad71a0 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -3,7 +3,6 @@
 
 from cpython cimport *
 cdef extern from "Python.h":
-    ctypedef char* const_char_ptr "const char*"
     ctypedef char* const_void_ptr "const void*"
     ctypedef struct PyObject
     cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
@@ -37,7 +36,7 @@ cdef extern from "unpack.h":
         unsigned int ct
         PyObject* key
 
-    ctypedef int (*execute_fn)(unpack_context* ctx, const_char_ptr data,
+    ctypedef int (*execute_fn)(unpack_context* ctx, const char* data,
                                size_t len, size_t* off) except? -1
     execute_fn unpack_construct
     execute_fn unpack_skip

From 18215b01bb272f28d98af5c9373fac8cbb24b30e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 8 Apr 2013 10:52:11 +0900
Subject: [PATCH 1174/1648] Unpacker.feed() uses new buffer interface.

---
 msgpack/_unpacker.pyx | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 1ad71a0..2644f15 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -254,13 +254,15 @@ cdef class Unpacker(object):
 
     def feed(self, object next_bytes):
         """Append `next_bytes` to internal buffer."""
-        cdef char* buf
-        cdef Py_ssize_t buf_len
+        cdef Py_buffer pybuff
         if self.file_like is not None:
             raise AssertionError(
                     "unpacker.feed() is not be able to use with `file_like`.")
-        PyObject_AsReadBuffer(next_bytes, &buf, &buf_len)
-        self.append_buffer(buf, buf_len)
+        PyObject_GetBuffer(next_bytes, &pybuff, PyBUF_SIMPLE)
+        try:
+            self.append_buffer(pybuff.buf, pybuff.len)
+        finally:
+            PyBuffer_Release(&pybuff)
 
     cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
         cdef:

From 085db7f8dca2b4e2497bfb291238cd3ff2d06e28 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 8 Apr 2013 10:57:21 +0900
Subject: [PATCH 1175/1648] Use new buffer interface.

---
 msgpack/_unpacker.pyx | 49 ++++++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 26 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 2644f15..46be7e0 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -2,10 +2,6 @@
 #cython: embedsignature=True
 
 from cpython cimport *
-cdef extern from "Python.h":
-    ctypedef char* const_void_ptr "const void*"
-    ctypedef struct PyObject
-    cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
 
 from libc.stdlib cimport *
 from libc.string cimport *
@@ -19,8 +15,8 @@ from msgpack.exceptions import (
         )
 
 
-
 cdef extern from "unpack.h":
+    ctypedef struct PyObject
     ctypedef struct msgpack_user:
         bint use_list
         PyObject* object_hook
@@ -91,32 +87,33 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef size_t off = 0
     cdef int ret
 
-    cdef char* buf
-    cdef Py_ssize_t buf_len
+    cdef Py_buffer buff
     cdef char* cenc = NULL
     cdef char* cerr = NULL
 
-    PyObject_AsReadBuffer(packed, &buf, &buf_len)
+    PyObject_GetBuffer(packed, &buff, PyBUF_SIMPLE)
+    try:
+        if encoding is not None:
+            if isinstance(encoding, unicode):
+                encoding = encoding.encode('ascii')
+            cenc = PyBytes_AsString(encoding)
 
-    if encoding is not None:
-        if isinstance(encoding, unicode):
-            encoding = encoding.encode('ascii')
-        cenc = PyBytes_AsString(encoding)
+        if unicode_errors is not None:
+            if isinstance(unicode_errors, unicode):
+                unicode_errors = unicode_errors.encode('ascii')
+            cerr = PyBytes_AsString(unicode_errors)
 
-    if unicode_errors is not None:
-        if isinstance(unicode_errors, unicode):
-            unicode_errors = unicode_errors.encode('ascii')
-        cerr = PyBytes_AsString(unicode_errors)
-
-    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
-    ret = unpack_construct(&ctx, buf, buf_len, &off)
-    if ret == 1:
-        obj = unpack_data(&ctx)
-        if off < buf_len:
-            raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
-        return obj
-    else:
-        raise UnpackValueError("Unpack failed: error = %d" % (ret,))
+        init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
+        ret = unpack_construct(&ctx, buff.buf, buff.len, &off)
+        if ret == 1:
+            obj = unpack_data(&ctx)
+            if off < buff.len:
+                raise ExtraData(obj, PyBytes_FromStringAndSize((buff)+off, buff.len-off))
+            return obj
+        else:
+            raise UnpackValueError("Unpack failed: error = %d" % (ret,))
+    finally:
+        PyBuffer_Release(&buff)
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None,

From a692bf98520b7836f6b456c1fcc33f20bb45f952 Mon Sep 17 00:00:00 2001
From: jnothman 
Date: Wed, 10 Apr 2013 21:03:41 +1000
Subject: [PATCH 1176/1648] Remove obsolete StopIteration warning

---
 README.rst | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/README.rst b/README.rst
index da2f023..294cd63 100644
--- a/README.rst
+++ b/README.rst
@@ -151,9 +151,6 @@ the result, or ignoring it. The latter two methods return the number of elements
 in the upcoming container, so that each element in an array, or key-value pair
 in a map, can be unpacked or skipped individually.
 
-Warning: these methods raise ``StopIteration`` when called at the end of the
-stream. Unless caught, this may silently break an iteration.
-
 Each of these methods may optionally write the packed data it reads to a
 callback function:
 

From 0c3fecf91bb39c6672fbf2f56d1c6e9a25ff90f0 Mon Sep 17 00:00:00 2001
From: YAMAMOTO Takashi 
Date: Wed, 1 May 2013 12:15:12 +0900
Subject: [PATCH 1177/1648] fix a typo in a comment

---
 msgpack/fallback.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index f9a2f5e..c66592c 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -147,7 +147,7 @@ class Unpacker(object):
 
     example of streaming deserialization from socket::
 
-        unpacker = Unapcker()
+        unpacker = Unpacker()
         while 1:
             buf = sock.recv(1024*2)
             if not buf:

From 56dbf7f9beba6a01040f7a513f85a8079f7b26f3 Mon Sep 17 00:00:00 2001
From: YAMAMOTO Takashi 
Date: Tue, 7 May 2013 13:56:39 +0900
Subject: [PATCH 1178/1648] fix more comment typos

---
 msgpack/_packer.pyx | 2 +-
 msgpack/fallback.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 93323c4..6289192 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -49,7 +49,7 @@ cdef class Packer(object):
         See also simplejson's document.
     :param str encoding:
         Convert unicode to bytes with this encoding. (default: 'utf-8')
-    :param str unicode_erros:
+    :param str unicode_errors:
         Error handler for encoding unicode. (default: 'strict')
     :param bool use_single_float:
         Use single precision float type for float. (default: False)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index c66592c..8f9d646 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -439,7 +439,7 @@ class Packer(object):
         See also simplejson's document.
     :param str encoding:
             Convert unicode to bytes with this encoding. (default: 'utf-8')
-    :param str unicode_erros:
+    :param str unicode_errors:
         Error handler for encoding unicode. (default: 'strict')
     :param bool use_single_float:
         Use single precision float type for float. (default: False)

From b0c193f3e07fdabc3d489bd5e134e3e41733f204 Mon Sep 17 00:00:00 2001
From: YAMAMOTO Takashi 
Date: Thu, 16 May 2013 12:41:02 +0900
Subject: [PATCH 1179/1648] fix long/int confusions in pyx version of unpack

---
 msgpack/unpack.h | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 595b8df..fb13b4e 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -68,7 +68,12 @@ static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unp
 
 static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o)
 {
-    PyObject *p = PyLong_FromUnsignedLongLong(d);
+    PyObject *p;
+    if (d > LONG_MAX) {
+        p = PyLong_FromUnsignedLongLong((unsigned long)d);
+    } else {
+        p = PyInt_FromLong((long)d);
+    }
     if (!p)
         return -1;
     *o = p;
@@ -96,9 +101,12 @@ static inline int unpack_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_
 
 static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o)
 {
-    PyObject *p = PyLong_FromLongLong(d);
-    if (!p)
-        return -1;
+    PyObject *p;
+    if (d > LONG_MAX || d < LONG_MIN) {
+        p = PyLong_FromLongLong((unsigned long)d);
+    } else {
+        p = PyInt_FromLong((long)d);
+    }
     *o = p;
     return 0;
 }

From 63b9fa5843ad9e38e3797c061de495f49659ab08 Mon Sep 17 00:00:00 2001
From: YAMAMOTO Takashi 
Date: Thu, 16 May 2013 12:57:28 +0900
Subject: [PATCH 1180/1648] fix a compilation error

msgpack/_unpacker.pyx: In function 'PyObject* __pyx_pf_7msgpack_9_unpacker_unpac
kb(PyObject*, PyObject*, PyObject*, PyObject*, int, PyObject*, PyObject*, PyObje
ct*)':
msgpack/_unpacker.pyx:111:70: error: invalid cast from type 'Py_buffer' to type 'char*'
---
 msgpack/_unpacker.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 46be7e0..06806bd 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -108,7 +108,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         if ret == 1:
             obj = unpack_data(&ctx)
             if off < buff.len:
-                raise ExtraData(obj, PyBytes_FromStringAndSize((buff)+off, buff.len-off))
+                raise ExtraData(obj, PyBytes_FromStringAndSize(buff.buf+off, buff.len-off))
             return obj
         else:
             raise UnpackValueError("Unpack failed: error = %d" % (ret,))

From 08c56d66f6eecb99850897dd3c4987dea01e6d44 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 19 May 2013 01:13:21 +0900
Subject: [PATCH 1181/1648] Use --cplus for cythoning

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3fe278e..71be6af 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ doc:
 	cd docs && make zip
 
 cython:
-	cython msgpack/*.pyx
+	cython --cplus msgpack/*.pyx
 
 python3: cython
 	python3 setup.py build_ext -i -f

From bbe86e7a925f113337f6e24d3ff24ddc30137dd6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 19 May 2013 12:30:23 +0900
Subject: [PATCH 1182/1648] Revert "Use new buffer interface."

This reverts commit 085db7f8dca2b4e2497bfb291238cd3ff2d06e28.

Conflicts:
	msgpack/_unpacker.pyx
---
 msgpack/_unpacker.pyx | 49 +++++++++++++++++++++++--------------------
 1 file changed, 26 insertions(+), 23 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 06806bd..2644f15 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -2,6 +2,10 @@
 #cython: embedsignature=True
 
 from cpython cimport *
+cdef extern from "Python.h":
+    ctypedef char* const_void_ptr "const void*"
+    ctypedef struct PyObject
+    cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
 
 from libc.stdlib cimport *
 from libc.string cimport *
@@ -15,8 +19,8 @@ from msgpack.exceptions import (
         )
 
 
+
 cdef extern from "unpack.h":
-    ctypedef struct PyObject
     ctypedef struct msgpack_user:
         bint use_list
         PyObject* object_hook
@@ -87,33 +91,32 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef size_t off = 0
     cdef int ret
 
-    cdef Py_buffer buff
+    cdef char* buf
+    cdef Py_ssize_t buf_len
     cdef char* cenc = NULL
     cdef char* cerr = NULL
 
-    PyObject_GetBuffer(packed, &buff, PyBUF_SIMPLE)
-    try:
-        if encoding is not None:
-            if isinstance(encoding, unicode):
-                encoding = encoding.encode('ascii')
-            cenc = PyBytes_AsString(encoding)
+    PyObject_AsReadBuffer(packed, &buf, &buf_len)
 
-        if unicode_errors is not None:
-            if isinstance(unicode_errors, unicode):
-                unicode_errors = unicode_errors.encode('ascii')
-            cerr = PyBytes_AsString(unicode_errors)
+    if encoding is not None:
+        if isinstance(encoding, unicode):
+            encoding = encoding.encode('ascii')
+        cenc = PyBytes_AsString(encoding)
 
-        init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
-        ret = unpack_construct(&ctx, buff.buf, buff.len, &off)
-        if ret == 1:
-            obj = unpack_data(&ctx)
-            if off < buff.len:
-                raise ExtraData(obj, PyBytes_FromStringAndSize(buff.buf+off, buff.len-off))
-            return obj
-        else:
-            raise UnpackValueError("Unpack failed: error = %d" % (ret,))
-    finally:
-        PyBuffer_Release(&buff)
+    if unicode_errors is not None:
+        if isinstance(unicode_errors, unicode):
+            unicode_errors = unicode_errors.encode('ascii')
+        cerr = PyBytes_AsString(unicode_errors)
+
+    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
+    ret = unpack_construct(&ctx, buf, buf_len, &off)
+    if ret == 1:
+        obj = unpack_data(&ctx)
+        if off < buf_len:
+            raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
+        return obj
+    else:
+        raise UnpackValueError("Unpack failed: error = %d" % (ret,))
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None,

From 956f55ecdf3b98e8072b4073da650e9a8fa05e0b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 19 May 2013 12:32:33 +0900
Subject: [PATCH 1183/1648] Stop using const_void_ptr typedef.

New Cython supports const natively.
---
 msgpack/_unpacker.pyx | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 2644f15..9ff1085 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -3,9 +3,8 @@
 
 from cpython cimport *
 cdef extern from "Python.h":
-    ctypedef char* const_void_ptr "const void*"
     ctypedef struct PyObject
-    cdef int PyObject_AsReadBuffer(object o, const_void_ptr* buff, Py_ssize_t* buf_len) except -1
+    cdef int PyObject_AsReadBuffer(object o, const void* buff, Py_ssize_t* buf_len) except -1
 
 from libc.stdlib cimport *
 from libc.string cimport *
@@ -96,7 +95,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef char* cenc = NULL
     cdef char* cerr = NULL
 
-    PyObject_AsReadBuffer(packed, &buf, &buf_len)
+    PyObject_AsReadBuffer(packed, &buf, &buf_len)
 
     if encoding is not None:
         if isinstance(encoding, unicode):

From 3dbb2d1e7e97a61a9c36e316336552dc0e20577f Mon Sep 17 00:00:00 2001
From: YAMAMOTO Takashi 
Date: Mon, 3 Jun 2013 13:18:52 +0900
Subject: [PATCH 1184/1648] fix compilation errors

---
 msgpack/_unpacker.pyx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 9ff1085..1f4dd85 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -4,7 +4,7 @@
 from cpython cimport *
 cdef extern from "Python.h":
     ctypedef struct PyObject
-    cdef int PyObject_AsReadBuffer(object o, const void* buff, Py_ssize_t* buf_len) except -1
+    cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1
 
 from libc.stdlib cimport *
 from libc.string cimport *
@@ -95,7 +95,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef char* cenc = NULL
     cdef char* cerr = NULL
 
-    PyObject_AsReadBuffer(packed, &buf, &buf_len)
+    PyObject_AsReadBuffer(packed, &buf, &buf_len)
 
     if encoding is not None:
         if isinstance(encoding, unicode):

From d1b9ecbc8e33663663f2fb2f13ac17854dd6c8d6 Mon Sep 17 00:00:00 2001
From: YAMAMOTO Takashi 
Date: Mon, 3 Jun 2013 13:49:44 +0900
Subject: [PATCH 1185/1648] fix long vs long long bugs

these bugs were introduced by "fix long/int confusions in pyx version of
unpack" commit.
---
 msgpack/unpack.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index fb13b4e..baeed1f 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -70,7 +70,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp
 {
     PyObject *p;
     if (d > LONG_MAX) {
-        p = PyLong_FromUnsignedLongLong((unsigned long)d);
+        p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d);
     } else {
         p = PyInt_FromLong((long)d);
     }
@@ -103,7 +103,7 @@ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpac
 {
     PyObject *p;
     if (d > LONG_MAX || d < LONG_MIN) {
-        p = PyLong_FromLongLong((unsigned long)d);
+        p = PyLong_FromLongLong((unsigned PY_LONG_LONG)d);
     } else {
         p = PyInt_FromLong((long)d);
     }

From e250b89920527d69e30566746a1ef12a2f43b25c Mon Sep 17 00:00:00 2001
From: YAMAMOTO Takashi 
Date: Mon, 3 Jun 2013 13:17:03 +0900
Subject: [PATCH 1186/1648] more tests

---
 test/test_pack.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_pack.py b/test/test_pack.py
index 3225f41..9dc5ada 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -12,11 +12,12 @@ from io import BytesIO
 def check(data, use_list=False):
     re = unpackb(packb(data), use_list=use_list)
     assert re == data
+    assert type(re) == type(data)
 
 def testPack():
     test_data = [
-            0, 1, 127, 128, 255, 256, 65535, 65536,
-            -1, -32, -33, -128, -129, -32768, -32769,
+            0, 1, 127, 128, 255, 256, 65535, 65536, 4294967295, 4294967296,
+            -1, -32, -33, -128, -129, -32768, -32769, -4294967296, -4294967297,
             1.0,
         b"", b"a", b"a"*31, b"a"*32,
         None, True, False,

From 12f87147b5e2c1ffadc9fed51dca35d940038798 Mon Sep 17 00:00:00 2001
From: Lieven Govaerts 
Date: Fri, 13 Sep 2013 13:47:13 +0200
Subject: [PATCH 1187/1648] * msgpack/exceptions.py: Fix typo in error message.

---
 msgpack/exceptions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py
index 2565541..f7678f1 100644
--- a/msgpack/exceptions.py
+++ b/msgpack/exceptions.py
@@ -20,7 +20,7 @@ class ExtraData(ValueError):
         self.extra = extra
 
     def __str__(self):
-        return "unpack(b) recieved extra data."
+        return "unpack(b) received extra data."
 
 class PackException(Exception):
     pass

From d61097511a1caa0e3bc5a70c1d2d92f448bd5025 Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Tue, 15 Oct 2013 16:59:43 +0200
Subject: [PATCH 1188/1648] add support for extended types: you can now
 pack/unpack custom python objects by subclassing Packer and Unpacker

---
 msgpack/fallback.py    | 131 ++++++++++++++++++++++++++---------------
 test/test_extension.py |  24 ++++++++
 2 files changed, 108 insertions(+), 47 deletions(-)
 create mode 100644 test/test_extension.py

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 8f9d646..b7f455b 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -58,55 +58,10 @@ TYPE_ARRAY              = 1
 TYPE_MAP                = 2
 TYPE_RAW                = 3
 
+EXTENDED_TYPE          = 1000
+
 DEFAULT_RECURSE_LIMIT=511
 
-def pack(o, stream, **kwargs):
-    """
-    Pack object `o` and write it to `stream`
-
-    See :class:`Packer` for options.
-    """
-    packer = Packer(**kwargs)
-    stream.write(packer.pack(o))
-
-def packb(o, **kwargs):
-    """
-    Pack object `o` and return packed bytes
-
-    See :class:`Packer` for options.
-    """
-    return Packer(**kwargs).pack(o)
-
-def unpack(stream, **kwargs):
-    """
-    Unpack an object from `stream`.
-
-    Raises `ExtraData` when `packed` contains extra bytes.
-    See :class:`Unpacker` for options.
-    """
-    unpacker = Unpacker(stream, **kwargs)
-    ret = unpacker._fb_unpack()
-    if unpacker._fb_got_extradata():
-        raise ExtraData(ret, unpacker._fb_get_extradata())
-    return ret
-
-def unpackb(packed, **kwargs):
-    """
-    Unpack an object from `packed`.
-
-    Raises `ExtraData` when `packed` contains extra bytes.
-    See :class:`Unpacker` for options.
-    """
-    unpacker = Unpacker(None, **kwargs)
-    unpacker.feed(packed)
-    try:
-        ret = unpacker._fb_unpack()
-    except OutOfData:
-        raise UnpackValueError("Data is not enough.")
-    if unpacker._fb_got_extradata():
-        raise ExtraData(ret, unpacker._fb_get_extradata())
-    return ret
-
 class Unpacker(object):
     """
     Streaming unpacker.
@@ -334,6 +289,9 @@ class Unpacker(object):
         elif b == 0xdf:
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             typ = TYPE_MAP
+        elif b == 0xc9:
+            n, typ = struct.unpack(">Ib", self._fb_read(5, write_bytes))
+            typ += EXTENDED_TYPE
         else:
             raise UnpackValueError("Unknown header: 0x%x" % b)
         return typ, n, obj
@@ -390,6 +348,10 @@ class Unpacker(object):
             if self._encoding is not None:
                 obj = obj.decode(self._encoding, self._unicode_errors)
             return obj
+        if typ >= EXTENDED_TYPE:
+            typ -= EXTENDED_TYPE
+            data = self._fb_read(n, write_bytes)
+            return self.handle_extended_type(typ, data)
         assert typ == TYPE_IMMEDIATE
         return obj
 
@@ -411,6 +373,9 @@ class Unpacker(object):
         self._fb_consume()
         return ret
 
+    def handle_extended_type(self, typecode, data):
+        raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode)
+
     def read_array_header(self, write_bytes=None):
         ret = self._fb_unpack(EX_READ_ARRAY_HEADER, write_bytes)
         self._fb_consume()
@@ -521,10 +486,33 @@ class Packer(object):
         if isinstance(obj, dict):
             return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj),
                                            nest_limit - 1)
+        if self.pack_extended_type(obj):
+            # it means that obj was succesfully handled by
+            # handle_extended_type, so we are done
+            return
         if self._default is not None:
             return self._pack(self._default(obj), nest_limit - 1)
         raise TypeError("Cannot serialize %r" % obj)
 
+    def pack_extended_type(self, obj):
+        res = self.handle_extended_type(obj)
+        if res is None:
+            return False
+        fmt, typecode, data = res
+        # for now we support only this. We should add support for the other
+        # fixext/ext formats
+        assert fmt == "ext 32"
+        assert 0 <= typecode <= 127
+        N = len(data)
+        self._buffer.write(struct.pack('>BIB', 0xc9, N, typecode))
+        self._buffer.write(data)
+        return True
+
+    def handle_extended_type(self, obj):
+        # by default we don't support any extended type. This can be
+        # overridden by subclasses
+        return None
+
     def pack(self, obj):
         self._pack(obj)
         ret = self._buffer.getvalue()
@@ -590,3 +578,52 @@ class Packer(object):
 
     def reset(self):
         self._buffer = StringIO()
+
+
+def pack(o, stream, Packer=Packer, **kwargs):
+    """
+    Pack object `o` and write it to `stream`
+
+    See :class:`Packer` for options.
+    """
+    packer = Packer(**kwargs)
+    stream.write(packer.pack(o))
+
+def packb(o, Packer=Packer, **kwargs):
+    """
+    Pack object `o` and return packed bytes
+
+    See :class:`Packer` for options.
+    """
+    return Packer(**kwargs).pack(o)
+
+def unpack(stream, Unpacker=Unpacker, **kwargs):
+    """
+    Unpack an object from `stream`.
+
+    Raises `ExtraData` when `packed` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    unpacker = Unpacker(stream, **kwargs)
+    ret = unpacker._fb_unpack()
+    if unpacker._fb_got_extradata():
+        raise ExtraData(ret, unpacker._fb_get_extradata())
+    return ret
+
+def unpackb(packed, Unpacker=Unpacker, **kwargs):
+    """
+    Unpack an object from `packed`.
+
+    Raises `ExtraData` when `packed` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    unpacker = Unpacker(None, **kwargs)
+    unpacker.feed(packed)
+    try:
+        ret = unpacker._fb_unpack()
+    except OutOfData:
+        raise UnpackValueError("Data is not enough.")
+    if unpacker._fb_got_extradata():
+        raise ExtraData(ret, unpacker._fb_get_extradata())
+    return ret
+
diff --git a/test/test_extension.py b/test/test_extension.py
new file mode 100644
index 0000000..45e6027
--- /dev/null
+++ b/test/test_extension.py
@@ -0,0 +1,24 @@
+import array
+import msgpack
+
+def test_extension_type():
+    class MyPacker(msgpack.Packer):
+        def handle_extended_type(self, obj):
+            if isinstance(obj, array.array):
+                fmt = "ext 32"
+                typecode = 123 # application specific typecode
+                data = obj.tostring()
+                return fmt, typecode, data
+
+    class MyUnpacker(msgpack.Unpacker):
+        def handle_extended_type(self, typecode, data):
+            assert typecode == 123
+            obj = array.array('d')
+            obj.fromstring(data)
+            return obj
+
+    obj = [42, 'hello', array.array('d', [1.1, 2.2, 3.3])]
+    s = msgpack.packb(obj, MyPacker)
+    obj2 = msgpack.unpackb(s, MyUnpacker)
+    assert obj == obj2
+    

From da12e177a31445492795f4003bbe0328645325b2 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 17 Oct 2013 08:35:08 +0900
Subject: [PATCH 1189/1648] Add bin type support.

---
 msgpack/_packer.pyx       | 28 +++++++++++++++++++++-------
 msgpack/pack.h            |  3 ++-
 msgpack/pack_template.h   | 27 ++++++++++++++++++++++++++-
 msgpack/unpack.h          |  9 +++++++++
 msgpack/unpack_define.h   | 17 ++++++-----------
 msgpack/unpack_template.h | 23 +++++++++++++++++------
 6 files changed, 81 insertions(+), 26 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 6289192..b9ae505 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -26,6 +26,7 @@ cdef extern from "pack.h":
     int msgpack_pack_array(msgpack_packer* pk, size_t l)
     int msgpack_pack_map(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw(msgpack_packer* pk, size_t l)
+    int msgpack_pack_bin(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
 
 cdef int DEFAULT_RECURSE_LIMIT=511
@@ -56,6 +57,9 @@ cdef class Packer(object):
     :param bool autoreset:
         Reset buffer after each pack and return it's content as `bytes`. (default: True).
         If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+    :param bool use_bin_type:
+        Use bin type introduced in msgpack spec 2.0 for bytes.
+        It also enable str8 type for unicode.
     """
     cdef msgpack_packer pk
     cdef object _default
@@ -64,6 +68,7 @@ cdef class Packer(object):
     cdef char *encoding
     cdef char *unicode_errors
     cdef bool use_float
+    cdef bool use_bin_type
     cdef bint autoreset
 
     def __cinit__(self):
@@ -74,11 +79,13 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
-    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1):
+    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
+                 use_single_float=False, bint autoreset=1, bint use_bin_type=0):
         """
         """
         self.use_float = use_single_float
         self.autoreset = autoreset
+        self.pk.use_bin_type = use_bin_type
         if default is not None:
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")
@@ -110,6 +117,7 @@ cdef class Packer(object):
         cdef char* rawval
         cdef int ret
         cdef dict d
+        cdef size_t L
 
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded.")
@@ -140,9 +148,10 @@ cdef class Packer(object):
                ret = msgpack_pack_double(&self.pk, dval)
         elif PyBytes_Check(o):
             rawval = o
-            ret = msgpack_pack_raw(&self.pk, len(o))
+            L = len(o)
+            ret = msgpack_pack_bin(&self.pk, L)
             if ret == 0:
-                ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
+                ret = msgpack_pack_raw_body(&self.pk, rawval, L)
         elif PyUnicode_Check(o):
             if not self.encoding:
                 raise TypeError("Can't encode unicode string: no encoding is specified")
@@ -247,21 +256,26 @@ cdef class Packer(object):
         return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
 
 
-def pack(object o, object stream, default=None, str encoding='utf-8', str unicode_errors='strict'):
+def pack(object o, object stream,
+         default=None, str encoding='utf-8', str unicode_errors='strict',
+         bint use_single_float=False, bint use_bin_type=False):
     """
     pack an object `o` and write it to stream)
 
     See :class:`Packer` for options.
     """
-    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors)
+    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
+                    use_single_float=use_single_float, use_bin_type=use_bin_type)
     stream.write(packer.pack(o))
 
-def packb(object o, default=None, encoding='utf-8', str unicode_errors='strict', bint use_single_float=False):
+def packb(object o,
+          default=None, str encoding='utf-8', str unicode_errors='strict',
+          bint use_single_float=False, bint use_bin_type=False):
     """
     pack o and return packed bytes
 
     See :class:`Packer` for options.
     """
     packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
-                    use_single_float=use_single_float)
+                    use_single_float=use_single_float, use_bin_type=use_bin_type)
     return packer.pack(o)
diff --git a/msgpack/pack.h b/msgpack/pack.h
index 1539991..001a0c1 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -34,11 +34,11 @@ typedef struct msgpack_packer {
     char *buf;
     size_t length;
     size_t buf_size;
+    bool use_bin_type;
 } msgpack_packer;
 
 typedef struct Packer Packer;
 
-static inline int msgpack_pack_short(msgpack_packer* pk, short d);
 static inline int msgpack_pack_int(msgpack_packer* pk, int d);
 static inline int msgpack_pack_long(msgpack_packer* pk, long d);
 static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
@@ -68,6 +68,7 @@ static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
 static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
 
 static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
+static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l);
 static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
 
 static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 9e00d7e..ff1cbaa 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -667,7 +667,10 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
     if(l < 32) {
         unsigned char d = 0xa0 | (uint8_t)l;
         msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
-    } else if(l < 65536) {
+    } else if (x->use_bin_type && l < 256) {  // str8 is new format introduced with bin.
+        unsigned char buf[2] = {0xd9, (uint8_t)l};
+        msgpack_pack_append_buffer(x, buf, 2);
+    } else if (l < 65536) {
         unsigned char buf[3];
         buf[0] = 0xda; _msgpack_store16(&buf[1], (uint16_t)l);
         msgpack_pack_append_buffer(x, buf, 3);
@@ -678,6 +681,28 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
     }
 }
 
+/*
+ * bin
+ */
+static inline int msgpack_pack_bin(msgpack_packer *x, size_t l)
+{
+    if (!x->use_bin_type) {
+        return msgpack_pack_raw(x, l)
+    }
+    if (l < 256) {
+        unsigned char buf[2] = {0xc4, (unsigned char)l};
+        msgpack_pack_append_buffer(x, buf, 2);
+    } else if (l < 65536) {
+        unsigned char buf[3] = {0xc5};
+        _msgpack_store16(&buf[1], (uint16_t)l);
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else {
+        unsigned char buf[5] = {0xc6};
+        _msgpack_store32(&buf[1], (uint32_t)l);
+        msgpack_pack_append_buffer(x, buf, 5);
+    }
+}
+
 static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l)
 {
     msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index baeed1f..03c735e 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -226,4 +226,13 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
     return 0;
 }
 
+static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
+{
+    PyObject *py = PyBytes_FromStringAndSize(p, l);
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
 #include "unpack_template.h"
diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h
index c81b990..0b14f52 100644
--- a/msgpack/unpack_define.h
+++ b/msgpack/unpack_define.h
@@ -34,6 +34,7 @@ extern "C" {
 #endif
 
 
+// CS is first byte & 0x1f
 typedef enum {
     CS_HEADER            = 0x00,  // nil
 
@@ -41,9 +42,9 @@ typedef enum {
     //CS_                = 0x02,  // false
     //CS_                = 0x03,  // true
 
-    //CS_                = 0x04,
-    //CS_                = 0x05,
-    //CS_                = 0x06,
+    CS_BIN_8             = 0x04,
+    CS_BIN_16            = 0x05,
+    CS_BIN_32            = 0x06,
     //CS_                = 0x07,
 
     //CS_                = 0x08,
@@ -59,12 +60,7 @@ typedef enum {
     CS_INT_32            = 0x12,
     CS_INT_64            = 0x13,
 
-    //CS_                = 0x14,
-    //CS_                = 0x15,
-    //CS_BIG_INT_16        = 0x16,
-    //CS_BIG_INT_32        = 0x17,
-    //CS_BIG_FLOAT_16      = 0x18,
-    //CS_BIG_FLOAT_32      = 0x19,
+    CS_RAW_8             = 0x19,
     CS_RAW_16            = 0x1a,
     CS_RAW_32            = 0x1b,
     CS_ARRAY_16          = 0x1c,
@@ -72,9 +68,8 @@ typedef enum {
     CS_MAP_16            = 0x1e,
     CS_MAP_32            = 0x1f,
 
-    //ACS_BIG_INT_VALUE,
-    //ACS_BIG_FLOAT_VALUE,
     ACS_RAW_VALUE,
+    ACS_BIN_VALUE,
 } msgpack_unpack_state;
 
 
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 29ac935..25229ac 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -151,8 +151,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
     */ \
     goto _header_again
 
-#define NEXT_CS(p) \
-    ((unsigned int)*p & 0x1f)
+#define NEXT_CS(p)  ((unsigned int)*p & 0x1f)
 
 #ifdef USE_CASE_RANGE
 #define SWITCH_RANGE_BEGIN     switch(*p) {
@@ -185,9 +184,6 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
                     push_simple_value(_false);
                 case 0xc3:  // true
                     push_simple_value(_true);
-                //case 0xc4:
-                //case 0xc5:
-                //case 0xc6:
                 //case 0xc7:
                 //case 0xc8:
                 //case 0xc9:
@@ -202,12 +198,15 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
                 case 0xd2:  // signed int 32
                 case 0xd3:  // signed int 64
                     again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
+                case 0xc4:  // bin 8
+                case 0xc5:  // bin 16
+                case 0xc6:  // bin 32
                 //case 0xd4:
                 //case 0xd5:
                 //case 0xd6:  // big integer 16
                 //case 0xd7:  // big integer 32
                 //case 0xd8:  // big float 16
-                //case 0xd9:  // big float 32
+                case 0xd9:  // raw 8
                 case 0xda:  // raw 16
                 case 0xdb:  // raw 32
                 case 0xdc:  // array 16
@@ -290,6 +289,18 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
             //    // FIXME
             //    push_variable_value(_big_float, data, n, trail);
 
+            case CS_BIN_8:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero);
+            case CS_BIN_16:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero);
+            case CS_BIN_32:
+                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero);
+            case ACS_BIN_VALUE:
+            _bin_zero:
+                push_variable_value(_bin, data, n, trail);
+
+            case CS_RAW_8:
+                again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero);
             case CS_RAW_16:
                 again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
             case CS_RAW_32:

From 171c5381135bdfc6db154100ad5a9913955408a7 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 17 Oct 2013 08:44:25 +0900
Subject: [PATCH 1190/1648] refactoring.

---
 msgpack/__init__.py | 22 ++++++++++++++++++++--
 msgpack/_packer.pyx | 27 +--------------------------
 msgpack/fallback.py | 17 -----------------
 3 files changed, 21 insertions(+), 45 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 77f6b81..5ce531e 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -4,14 +4,32 @@ from msgpack.exceptions import *
 
 import os
 if os.environ.get('MSGPACK_PUREPYTHON'):
-    from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker
+    from msgpack.fallback import Packer, unpack, unpackb, Unpacker
 else:
     try:
-        from msgpack._packer import pack, packb, Packer
+        from msgpack._packer import Packer
         from msgpack._unpacker import unpack, unpackb, Unpacker
     except ImportError:
         from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker
 
+
+def pack(o, stream, **kwargs):
+    """
+    Pack object `o` and write it to `stream`
+
+    See :class:`Packer` for options.
+    """
+    packer = Packer(**kwargs)
+    stream.write(packer.pack(o))
+
+def packb(o, **kwargs):
+    """
+    Pack object `o` and return packed bytes
+
+    See :class:`Packer` for options.
+    """
+    return Packer(**kwargs).pack(o)
+
 # alias for compatibility to simplejson/marshal/pickle.
 load = unpack
 loads = unpackb
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index b9ae505..7082445 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -13,6 +13,7 @@ cdef extern from "pack.h":
         char* buf
         size_t length
         size_t buf_size
+        bint use_bin_type
 
     int msgpack_pack_int(msgpack_packer* pk, int d)
     int msgpack_pack_nil(msgpack_packer* pk)
@@ -68,7 +69,6 @@ cdef class Packer(object):
     cdef char *encoding
     cdef char *unicode_errors
     cdef bool use_float
-    cdef bool use_bin_type
     cdef bint autoreset
 
     def __cinit__(self):
@@ -254,28 +254,3 @@ cdef class Packer(object):
     def bytes(self):
         """Return buffer content."""
         return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
-
-
-def pack(object o, object stream,
-         default=None, str encoding='utf-8', str unicode_errors='strict',
-         bint use_single_float=False, bint use_bin_type=False):
-    """
-    pack an object `o` and write it to stream)
-
-    See :class:`Packer` for options.
-    """
-    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
-                    use_single_float=use_single_float, use_bin_type=use_bin_type)
-    stream.write(packer.pack(o))
-
-def packb(object o,
-          default=None, str encoding='utf-8', str unicode_errors='strict',
-          bint use_single_float=False, bint use_bin_type=False):
-    """
-    pack o and return packed bytes
-
-    See :class:`Packer` for options.
-    """
-    packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors,
-                    use_single_float=use_single_float, use_bin_type=use_bin_type)
-    return packer.pack(o)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 8f9d646..3bf0489 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -60,23 +60,6 @@ TYPE_RAW                = 3
 
 DEFAULT_RECURSE_LIMIT=511
 
-def pack(o, stream, **kwargs):
-    """
-    Pack object `o` and write it to `stream`
-
-    See :class:`Packer` for options.
-    """
-    packer = Packer(**kwargs)
-    stream.write(packer.pack(o))
-
-def packb(o, **kwargs):
-    """
-    Pack object `o` and return packed bytes
-
-    See :class:`Packer` for options.
-    """
-    return Packer(**kwargs).pack(o)
-
 def unpack(stream, **kwargs):
     """
     Unpack an object from `stream`.

From 84f6b100190049a5d1bceb916208eeae2a3d2591 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 17 Oct 2013 08:52:59 +0900
Subject: [PATCH 1191/1648] Add bin type support to pure Python packer.

---
 msgpack/fallback.py     | 32 ++++++++++++++++++++++++--------
 msgpack/pack_template.h |  2 +-
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 3bf0489..49a2bc5 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -429,11 +429,15 @@ class Packer(object):
     :param bool autoreset:
         Reset buffer after each pack and return it's content as `bytes`. (default: True).
         If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+    :param bool use_bin_type:
+        Use bin type introduced in msgpack spec 2.0 for bytes.
+        It also enable str8 type for unicode.
     """
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
-                 use_single_float=False, autoreset=True):
+                 use_single_float=False, autoreset=True, use_bin_type=False):
         self._use_float = use_single_float
         self._autoreset = autoreset
+        self._use_bin_type = use_bin_type
         self._encoding = encoding
         self._unicode_errors = unicode_errors
         self._buffer = StringIO()
@@ -473,6 +477,17 @@ class Packer(object):
             if -0x8000000000000000 <= obj < -0x80000000:
                 return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
             raise PackValueError("Integer value out of range")
+        if self._use_bin_type and isinstance(obj, bytes):
+            n = len(obj)
+            if n <= 0xff:
+                self._buffer.write(struct.pack('>BB', 0xc4, n))
+            elif n <= 0xffff:
+                self._buffer.write(struct.pack(">BH", 0xc5, n))
+            elif n <= 0xffffffff:
+                self._buffer.write(struct.pack(">BI", 0xc6, n))
+            else:
+                raise PackValueError("Bytes is too large")
+            return self._buffer.write(obj)
         if isinstance(obj, (Unicode, bytes)):
             if isinstance(obj, Unicode):
                 if self._encoding is None:
@@ -483,19 +498,20 @@ class Packer(object):
             n = len(obj)
             if n <= 0x1f:
                 self._buffer.write(struct.pack('B', 0xa0 + n))
-                return self._buffer.write(obj)
-            if n <= 0xffff:
+            elif self._use_bin_type and n <= 0xff:
+                self._buffer.write(struct.pack('>BB', 0xd9, n))
+            elif n <= 0xffff:
                 self._buffer.write(struct.pack(">BH", 0xda, n))
-                return self._buffer.write(obj)
-            if n <= 0xffffffff:
+            elif n <= 0xffffffff:
                 self._buffer.write(struct.pack(">BI", 0xdb, n))
-                return self._buffer.write(obj)
-            raise PackValueError("String is too large")
+            else:
+                raise PackValueError("String is too large")
+            return self._buffer.write(obj)
         if isinstance(obj, float):
             if self._use_float:
                 return self._buffer.write(struct.pack(">Bf", 0xca, obj))
             return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
-        if isinstance(obj, list) or isinstance(obj, tuple):
+        if isinstance(obj, (list, tuple)):
             n = len(obj)
             self._fb_pack_array_header(n)
             for i in xrange(n):
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index ff1cbaa..d228d7a 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -664,7 +664,7 @@ static inline int msgpack_pack_map(msgpack_packer* x, unsigned int n)
 
 static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
 {
-    if(l < 32) {
+    if (l < 32) {
         unsigned char d = 0xa0 | (uint8_t)l;
         msgpack_pack_append_buffer(x, &TAKE8_8(d), 1);
     } else if (x->use_bin_type && l < 256) {  // str8 is new format introduced with bin.

From 85eaff344b74092ccf076ba2465e9bdb1537409b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 17 Oct 2013 09:15:19 +0900
Subject: [PATCH 1192/1648] Add bin type support for fallback Unpacker.

---
 msgpack/__init__.py   |  2 +-
 msgpack/_unpacker.pyx |  1 -
 msgpack/fallback.py   | 19 +++++++++++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 5ce531e..7f0519e 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -10,7 +10,7 @@ else:
         from msgpack._packer import Packer
         from msgpack._unpacker import unpack, unpackb, Unpacker
     except ImportError:
-        from msgpack.fallback import pack, packb, Packer, unpack, unpackb, Unpacker
+        from msgpack.fallback import Packer, unpack, unpackb, Unpacker
 
 
 def pack(o, stream, **kwargs):
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 1f4dd85..7b0c8a6 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -18,7 +18,6 @@ from msgpack.exceptions import (
         )
 
 
-
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
         bint use_list
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 49a2bc5..901c6c8 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -57,6 +57,7 @@ TYPE_IMMEDIATE          = 0
 TYPE_ARRAY              = 1
 TYPE_MAP                = 2
 TYPE_RAW                = 3
+TYPE_BIN                = 4
 
 DEFAULT_RECURSE_LIMIT=511
 
@@ -297,6 +298,10 @@ class Unpacker(object):
             obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0]
         elif b == 0xd3:
             obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0]
+        elif b == 0xd9:
+            n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+            typ = TYPE_RAW
         elif b == 0xda:
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
             obj = self._fb_read(n, write_bytes)
@@ -305,6 +310,18 @@ class Unpacker(object):
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             obj = self._fb_read(n, write_bytes)
             typ = TYPE_RAW
+        elif b == 0xc4:
+            n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+            typ = TYPE_BIN
+        elif b == 0xc5:
+            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+            typ = TYPE_BIN
+        elif b == 0xc6:
+            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+            typ = TYPE_BIN
         elif b == 0xdc:
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
             typ = TYPE_ARRAY
@@ -373,6 +390,8 @@ class Unpacker(object):
             if self._encoding is not None:
                 obj = obj.decode(self._encoding, self._unicode_errors)
             return obj
+        if typ == TYPE_BIN:
+            return obj
         assert typ == TYPE_IMMEDIATE
         return obj
 

From f162bf6f79d7a50e10530e42630b063af3bb8eb1 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 17 Oct 2013 09:37:20 +0900
Subject: [PATCH 1193/1648] Add tests for str8 and bin types.

---
 test/test_newspec.py | 69 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 test/test_newspec.py

diff --git a/test/test_newspec.py b/test/test_newspec.py
new file mode 100644
index 0000000..8bc2cfe
--- /dev/null
+++ b/test/test_newspec.py
@@ -0,0 +1,69 @@
+# coding: utf-8
+
+from msgpack import packb, unpackb
+
+
+def test_str8():
+    header = b'\xd9'
+    data = b'x' * 32
+    b = packb(data.decode(), use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\x20'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+    data = b'x' * 255
+    b = packb(data.decode(), use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\xff'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+
+def test_bin8():
+    header = b'\xc4'
+    data = b''
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\x00'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+    data = b'x' * 255
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 2
+    assert b[0:2] == header + b'\xff'
+    assert b[2:] == data
+    assert unpackb(b) == data
+
+
+def test_bin16():
+    header = b'\xc5'
+    data = b'x' * 256
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 3
+    assert b[0:1] == header
+    assert b[1:3] == b'\x01\x00'
+    assert b[3:] == data
+    assert unpackb(b) == data
+
+    data = b'x' * 65535
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 3
+    assert b[0:1] == header
+    assert b[1:3] == b'\xff\xff'
+    assert b[3:] == data
+    assert unpackb(b) == data
+
+
+def test_bin32():
+    header = b'\xc6'
+    data = b'x' * 65536
+    b = packb(data, use_bin_type=True)
+    assert len(b) == len(data) + 5
+    assert b[0:1] == header
+    assert b[1:5] == b'\x00\x01\x00\x00'
+    assert b[5:] == data
+    assert unpackb(b) == data
+
+

From d9439204c77a0ab58248c0b78237f16c4496647c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 17 Oct 2013 11:29:36 +0900
Subject: [PATCH 1194/1648] Add ext type support to fallback.Unpacker.

---
 msgpack/__init__.py |  4 +++
 msgpack/fallback.py | 72 +++++++++++++++++++++++++++++++++++----------
 2 files changed, 60 insertions(+), 16 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 7f0519e..79107b6 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -2,6 +2,10 @@
 from msgpack._version import version
 from msgpack.exceptions import *
 
+from collections import namedtuple
+
+ExtType = namedtuple('ExtType', 'code data')
+
 import os
 if os.environ.get('MSGPACK_PUREPYTHON'):
     from msgpack.fallback import Packer, unpack, unpackb, Unpacker
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 901c6c8..074fcba 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -48,6 +48,9 @@ from msgpack.exceptions import (
         PackValueError,
         ExtraData)
 
+from msgpack import ExtType
+
+
 EX_SKIP                 = 0
 EX_CONSTRUCT            = 1
 EX_READ_ARRAY_HEADER    = 2
@@ -58,8 +61,9 @@ TYPE_ARRAY              = 1
 TYPE_MAP                = 2
 TYPE_RAW                = 3
 TYPE_BIN                = 4
+TYPE_EXT                = 5
 
-DEFAULT_RECURSE_LIMIT=511
+DEFAULT_RECURSE_LIMIT = 511
 
 def unpack(stream, **kwargs):
     """
@@ -112,6 +116,9 @@ class Unpacker(object):
     should be callable and Unpacker calls it with a list of key-value pairs
     after deserializing a map.
 
+    `ext_hook` is callback for ext (User defined) type. It called with two
+    arguments: (code, bytes). default: `msgpack.ExtType`
+
     `encoding` is the encoding used for decoding msgpack bytes.  If it is
     None (default), msgpack bytes are deserialized to Python bytes.
 
@@ -143,7 +150,8 @@ class Unpacker(object):
 
     def __init__(self, file_like=None, read_size=0, use_list=True,
                  object_hook=None, object_pairs_hook=None, list_hook=None,
-                 encoding=None, unicode_errors='strict', max_buffer_size=0):
+                 encoding=None, unicode_errors='strict', max_buffer_size=0,
+                 ext_hook=ExtType):
         if file_like is None:
             self._fb_feeding = True
         else:
@@ -167,6 +175,7 @@ class Unpacker(object):
         self._list_hook = list_hook
         self._object_hook = object_hook
         self._object_pairs_hook = object_pairs_hook
+        self._ext_hook = ext_hook
 
         if list_hook is not None and not callable(list_hook):
             raise ValueError('`list_hook` is not callable')
@@ -177,6 +186,8 @@ class Unpacker(object):
         if object_hook is not None and object_pairs_hook is not None:
             raise ValueError("object_pairs_hook and object_hook are mutually "
                              "exclusive")
+        if not callable(ext_hook):
+            raise ValueError("`ext_hook` is not callable")
 
     def feed(self, next_bytes):
         if isinstance(next_bytes, array.array):
@@ -278,6 +289,30 @@ class Unpacker(object):
             obj = False
         elif b == 0xc3:
             obj = True
+        elif b == 0xc4:
+            typ = TYPE_BIN
+            n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xc5:
+            typ = TYPE_BIN
+            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xc6:
+            typ = TYPE_BIN
+            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            obj = self._fb_read(n, write_bytes)
+        elif b == 0xc7:  # ext 8
+            typ = TYPE_EXT
+            L, n = struct.unpack('Bb', self._fb_read(2, write_bytes))
+            obj = self._fb_read(L, write_bytes)
+        elif b == 0xc8:  # ext 16
+            typ = TYPE_EXT
+            L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes))
+            obj = self._fb_read(L, write_bytes)
+        elif b == 0xc9:  # ext 32
+            typ = TYPE_EXT
+            L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes))
+            obj = self._fb_read(L, write_bytes)
         elif b == 0xca:
             obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0]
         elif b == 0xcb:
@@ -298,30 +333,33 @@ class Unpacker(object):
             obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0]
         elif b == 0xd3:
             obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0]
+        elif b == 0xd4:  # fixext 1
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes))
+        elif b == 0xd5:  # fixext 2
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes))
+        elif b == 0xd6:  # fixext 4
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes))
+        elif b == 0xd7:  # fixext 8
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes))
+        elif b == 0xd8:  # fixext 16
+            typ = TYPE_EXT
+            n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes))
         elif b == 0xd9:
+            typ = TYPE_RAW
             n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
             obj = self._fb_read(n, write_bytes)
-            typ = TYPE_RAW
         elif b == 0xda:
+            typ = TYPE_RAW
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
             obj = self._fb_read(n, write_bytes)
-            typ = TYPE_RAW
         elif b == 0xdb:
-            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
-            obj = self._fb_read(n, write_bytes)
             typ = TYPE_RAW
-        elif b == 0xc4:
-            n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
-            obj = self._fb_read(n, write_bytes)
-            typ = TYPE_BIN
-        elif b == 0xc5:
-            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
-            obj = self._fb_read(n, write_bytes)
-            typ = TYPE_BIN
-        elif b == 0xc6:
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             obj = self._fb_read(n, write_bytes)
-            typ = TYPE_BIN
         elif b == 0xdc:
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
             typ = TYPE_ARRAY
@@ -390,6 +428,8 @@ class Unpacker(object):
             if self._encoding is not None:
                 obj = obj.decode(self._encoding, self._unicode_errors)
             return obj
+        if typ == TYPE_EXT:
+            return self._ext_hook(n, obj)
         if typ == TYPE_BIN:
             return obj
         assert typ == TYPE_IMMEDIATE

From 5529dfe59660f3c2fc5058e6fa42b24fe764a255 Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Fri, 18 Oct 2013 14:38:52 +0200
Subject: [PATCH 1195/1648] kill some duplicate code from unpack/unpackb and
 move the logic to Unpacker.unpack_one. By doing this we no longer need to
 make the module-level pack/unpack parametric on the class, because they
 contain no logic at all

---
 msgpack/fallback.py    | 95 +++++++++++++++++++++---------------------
 test/test_extension.py |  8 ++--
 2 files changed, 52 insertions(+), 51 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index b7f455b..2c79482 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -62,6 +62,44 @@ EXTENDED_TYPE          = 1000
 
 DEFAULT_RECURSE_LIMIT=511
 
+def pack(o, stream, **kwargs):
+    """
+    Pack object `o` and write it to `stream`
+
+    See :class:`Packer` for options.
+    """
+    packer = Packer(**kwargs)
+    stream.write(packer.pack(o))
+
+def packb(o, **kwargs):
+    """
+    Pack object `o` and return packed bytes
+
+    See :class:`Packer` for options.
+    """
+    return Packer(**kwargs).pack(o)
+
+def unpack(stream, **kwargs):
+    """
+    Unpack an object from `stream`.
+
+    Raises `ExtraData` when `packed` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    unpacker = Unpacker(stream, **kwargs)
+    return unpacker.unpack_one()
+
+def unpackb(packed, **kwargs):
+    """
+    Unpack an object from `packed`.
+
+    Raises `ExtraData` when `packed` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    unpacker = Unpacker(None, **kwargs)
+    unpacker.feed(packed)
+    return unpacker.unpack_one()
+
 class Unpacker(object):
     """
     Streaming unpacker.
@@ -149,6 +187,15 @@ class Unpacker(object):
             raise ValueError("object_pairs_hook and object_hook are mutually "
                              "exclusive")
 
+    def unpack_one(self):
+        try:
+            ret = self._fb_unpack()
+        except OutOfData:
+            raise UnpackValueError("Data is not enough.")
+        if self._fb_got_extradata():
+            raise ExtraData(ret, self._fb_get_extradata())
+        return ret
+
     def feed(self, next_bytes):
         if isinstance(next_bytes, array.array):
             next_bytes = next_bytes.tostring()
@@ -579,51 +626,3 @@ class Packer(object):
     def reset(self):
         self._buffer = StringIO()
 
-
-def pack(o, stream, Packer=Packer, **kwargs):
-    """
-    Pack object `o` and write it to `stream`
-
-    See :class:`Packer` for options.
-    """
-    packer = Packer(**kwargs)
-    stream.write(packer.pack(o))
-
-def packb(o, Packer=Packer, **kwargs):
-    """
-    Pack object `o` and return packed bytes
-
-    See :class:`Packer` for options.
-    """
-    return Packer(**kwargs).pack(o)
-
-def unpack(stream, Unpacker=Unpacker, **kwargs):
-    """
-    Unpack an object from `stream`.
-
-    Raises `ExtraData` when `packed` contains extra bytes.
-    See :class:`Unpacker` for options.
-    """
-    unpacker = Unpacker(stream, **kwargs)
-    ret = unpacker._fb_unpack()
-    if unpacker._fb_got_extradata():
-        raise ExtraData(ret, unpacker._fb_get_extradata())
-    return ret
-
-def unpackb(packed, Unpacker=Unpacker, **kwargs):
-    """
-    Unpack an object from `packed`.
-
-    Raises `ExtraData` when `packed` contains extra bytes.
-    See :class:`Unpacker` for options.
-    """
-    unpacker = Unpacker(None, **kwargs)
-    unpacker.feed(packed)
-    try:
-        ret = unpacker._fb_unpack()
-    except OutOfData:
-        raise UnpackValueError("Data is not enough.")
-    if unpacker._fb_got_extradata():
-        raise ExtraData(ret, unpacker._fb_get_extradata())
-    return ret
-
diff --git a/test/test_extension.py b/test/test_extension.py
index 45e6027..0a9c14f 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -18,7 +18,9 @@ def test_extension_type():
             return obj
 
     obj = [42, 'hello', array.array('d', [1.1, 2.2, 3.3])]
-    s = msgpack.packb(obj, MyPacker)
-    obj2 = msgpack.unpackb(s, MyUnpacker)
+    packer = MyPacker()
+    unpacker = MyUnpacker(None)
+    s = packer.pack(obj)
+    unpacker.feed(s)
+    obj2 = unpacker.unpack_one()
     assert obj == obj2
-    

From 522c4bfc7993c296b78df9c9c91aac5fd40ae8e0 Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Fri, 18 Oct 2013 15:03:58 +0200
Subject: [PATCH 1196/1648] slightly change to API

---
 msgpack/fallback.py    | 26 ++++++++++----------------
 test/test_extension.py |  7 ++++---
 2 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 2c79482..101bd0f 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -398,7 +398,7 @@ class Unpacker(object):
         if typ >= EXTENDED_TYPE:
             typ -= EXTENDED_TYPE
             data = self._fb_read(n, write_bytes)
-            return self.handle_extended_type(typ, data)
+            return self.read_extended_type(typ, data)
         assert typ == TYPE_IMMEDIATE
         return obj
 
@@ -420,7 +420,7 @@ class Unpacker(object):
         self._fb_consume()
         return ret
 
-    def handle_extended_type(self, typecode, data):
+    def read_extended_type(self, typecode, data):
         raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode)
 
     def read_array_header(self, write_bytes=None):
@@ -533,19 +533,19 @@ class Packer(object):
         if isinstance(obj, dict):
             return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj),
                                            nest_limit - 1)
-        if self.pack_extended_type(obj):
-            # it means that obj was succesfully handled by
-            # handle_extended_type, so we are done
+        if self.handle_unknown_type(obj):
+            # it means that obj was succesfully packed, so we are done
             return
         if self._default is not None:
             return self._pack(self._default(obj), nest_limit - 1)
         raise TypeError("Cannot serialize %r" % obj)
 
-    def pack_extended_type(self, obj):
-        res = self.handle_extended_type(obj)
-        if res is None:
-            return False
-        fmt, typecode, data = res
+    def handle_unknown_type(self, obj):
+        # by default we don't support any extended type. This can be
+        # overridden by subclasses
+        return None
+
+    def pack_extended_type(self, fmt, typecode, data):
         # for now we support only this. We should add support for the other
         # fixext/ext formats
         assert fmt == "ext 32"
@@ -553,12 +553,6 @@ class Packer(object):
         N = len(data)
         self._buffer.write(struct.pack('>BIB', 0xc9, N, typecode))
         self._buffer.write(data)
-        return True
-
-    def handle_extended_type(self, obj):
-        # by default we don't support any extended type. This can be
-        # overridden by subclasses
-        return None
 
     def pack(self, obj):
         self._pack(obj)
diff --git a/test/test_extension.py b/test/test_extension.py
index 0a9c14f..0b26f8e 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -3,15 +3,16 @@ import msgpack
 
 def test_extension_type():
     class MyPacker(msgpack.Packer):
-        def handle_extended_type(self, obj):
+        def handle_unknown_type(self, obj):
             if isinstance(obj, array.array):
                 fmt = "ext 32"
                 typecode = 123 # application specific typecode
                 data = obj.tostring()
-                return fmt, typecode, data
+                self.pack_extended_type(fmt, typecode, data)
+                return True
 
     class MyUnpacker(msgpack.Unpacker):
-        def handle_extended_type(self, typecode, data):
+        def read_extended_type(self, typecode, data):
             assert typecode == 123
             obj = array.array('d')
             obj.fromstring(data)

From c727440ba5fe2f77d6cc03171ad7c193a3f481ee Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Fri, 18 Oct 2013 15:45:50 +0200
Subject: [PATCH 1197/1648] automatically find the best format to encode
 extended types

---
 msgpack/fallback.py    | 28 ++++++++++++++++++++++------
 test/test_extension.py | 18 ++++++++++++++++--
 2 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 101bd0f..f984dcd 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -545,13 +545,29 @@ class Packer(object):
         # overridden by subclasses
         return None
 
-    def pack_extended_type(self, fmt, typecode, data):
-        # for now we support only this. We should add support for the other
-        # fixext/ext formats
-        assert fmt == "ext 32"
+    def pack_extended_type(self, typecode, data):
         assert 0 <= typecode <= 127
-        N = len(data)
-        self._buffer.write(struct.pack('>BIB', 0xc9, N, typecode))
+        n = len(data)
+        if n == 1:
+            header = struct.pack(">BB", 0xd4, typecode) # fixext 1
+        elif n == 2:
+            header = struct.pack(">BB", 0xd5, typecode) # fixext 2
+        elif n == 4:
+            header = struct.pack(">BB", 0xd6, typecode) # fixext 4
+        elif n == 8:
+            header = struct.pack(">BB", 0xd7, typecode) # fixext 8
+        elif n == 16:
+            header = struct.pack(">BB", 0xd8, typecode) # fixext 16
+        elif n <= 2**8-1:
+            header = struct.pack(">BBB", 0xc7, n, typecode) # ext 8
+        elif n <= 2**16-1:
+            header = struct.pack(">BHB", 0xc8, n, typecode) # ext 16
+        elif n <= 2**32-1:
+            header = struct.pack(">BIB", 0xc9, n, typecode) # ext 32
+        else:
+            raise PackValueError("ext data too large")            
+        #
+        self._buffer.write(header)
         self._buffer.write(data)
 
     def pack(self, obj):
diff --git a/test/test_extension.py b/test/test_extension.py
index 0b26f8e..1908fa2 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -1,14 +1,28 @@
 import array
+import struct
 import msgpack
 
+def test_pack_extended_type():
+    def p(s):
+        packer = msgpack.Packer()
+        packer.pack_extended_type(0x42, s)
+        return packer._buffer.getvalue()
+    assert p('A')        == '\xd4\x42A'          # fixext 1
+    assert p('AB')       == '\xd5\x42AB'         # fixext 2
+    assert p('ABCD')     == '\xd6\x42ABCD'       # fixext 4
+    assert p('ABCDEFGH') == '\xd7\x42ABCDEFGH'   # fixext 8
+    assert p('A'*16)     == '\xd8\x42' + 'A'*16  # fixext 16
+    assert p('ABC')      == '\xc7\x03\x42ABC'        # ext 8
+    assert p('A'*0x0123)        == '\xc8\x01\x23\x42' + 'A'*0x0123 # ext 16
+    assert p('A'*0x00012345)    == '\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345 # ext 32
+
 def test_extension_type():
     class MyPacker(msgpack.Packer):
         def handle_unknown_type(self, obj):
             if isinstance(obj, array.array):
-                fmt = "ext 32"
                 typecode = 123 # application specific typecode
                 data = obj.tostring()
-                self.pack_extended_type(fmt, typecode, data)
+                self.pack_extended_type(typecode, data)
                 return True
 
     class MyUnpacker(msgpack.Unpacker):

From afa28fb2051cb00f03c83e020745e1eb238ff4ac Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Fri, 18 Oct 2013 15:54:12 +0200
Subject: [PATCH 1198/1648] add support to unpack all ext formats

---
 msgpack/fallback.py | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index f984dcd..c272420 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -336,7 +336,33 @@ class Unpacker(object):
         elif b == 0xdf:
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             typ = TYPE_MAP
-        elif b == 0xc9:
+        elif b == 0xd4: # fixext 1
+            typ = struct.unpack(">B", self._fb_read(1, write_bytes))[0]
+            n = 1
+            typ += EXTENDED_TYPE
+        elif b == 0xd5: # fixext 2
+            typ = struct.unpack(">B", self._fb_read(1, write_bytes))[0]
+            n = 2
+            typ += EXTENDED_TYPE
+        elif b == 0xd6: # fixext 4
+            typ = struct.unpack(">B", self._fb_read(1, write_bytes))[0]
+            n = 4
+            typ += EXTENDED_TYPE
+        elif b == 0xd7: # fixext 8
+            typ = struct.unpack(">B", self._fb_read(1, write_bytes))[0]
+            n = 8
+            typ += EXTENDED_TYPE
+        elif b == 0xd8: # fixext 16
+            typ = struct.unpack(">B", self._fb_read(1, write_bytes))[0]
+            n = 16
+            typ += EXTENDED_TYPE
+        elif b == 0xc7: # ext 8
+            n, typ = struct.unpack(">Bb", self._fb_read(2, write_bytes))
+            typ += EXTENDED_TYPE
+        elif b == 0xc8: # ext 16
+            n, typ = struct.unpack(">Hb", self._fb_read(3, write_bytes))
+            typ += EXTENDED_TYPE
+        elif b == 0xc9: # ext 32
             n, typ = struct.unpack(">Ib", self._fb_read(5, write_bytes))
             typ += EXTENDED_TYPE
         else:

From 5467515065b95496b9f5b9d842ffc73c9ccb806e Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Fri, 18 Oct 2013 17:33:54 +0200
Subject: [PATCH 1199/1648] implement Packer.pack_extended_type also in the
 cython version of the code

---
 msgpack/_packer.pyx     |  6 +++++
 msgpack/pack.h          |  2 ++
 msgpack/pack_template.h | 60 +++++++++++++++++++++++++++++++++++++++++
 test/test_extension.py  |  2 +-
 4 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 6289192..985559c 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -5,6 +5,7 @@ from cpython cimport *
 from libc.stdlib cimport *
 from libc.string cimport *
 from libc.limits cimport *
+from libc.stdint cimport int8_t
 
 from msgpack.exceptions import PackValueError
 
@@ -27,6 +28,7 @@ cdef extern from "pack.h":
     int msgpack_pack_map(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
+    int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l)
 
 cdef int DEFAULT_RECURSE_LIMIT=511
 
@@ -193,6 +195,10 @@ cdef class Packer(object):
             self.pk.length = 0
             return buf
 
+    def pack_extended_type(self, typecode, data):
+        msgpack_pack_ext(&self.pk, typecode, len(data))
+        msgpack_pack_raw_body(&self.pk, data, len(data))
+
     def pack_array_header(self, size_t size):
         cdef int ret = msgpack_pack_array(&self.pk, size)
         if ret == -1:
diff --git a/msgpack/pack.h b/msgpack/pack.h
index 1539991..08fdd82 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -70,6 +70,8 @@ static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
 static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
 static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
 
+static inline int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l);
+
 static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)
 {
     char* buf = pk->buf;
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 9e00d7e..ac9815f 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -683,6 +683,66 @@ static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t
     msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
 }
 
+/*
+ * Ext
+ */
+
+static inline int msgpack_pack_ext(msgpack_packer* x, int8_t typecode, size_t l)
+{
+    if (l == 1) {
+        unsigned char buf[2];
+        buf[0] = 0xd4;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 2) {
+        unsigned char buf[2];
+        buf[0] = 0xd5;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 4) {
+        unsigned char buf[2];
+        buf[0] = 0xd6;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 8) {
+        unsigned char buf[2];
+        buf[0] = 0xd7;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l == 16) {
+        unsigned char buf[2];
+        buf[0] = 0xd8;
+        buf[1] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 2);
+    }
+    else if(l < 256) {
+        unsigned char buf[3];
+        buf[0] = 0xc7;
+        buf[1] = l;
+        buf[2] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 3);
+    } else if(l < 65536) {
+        unsigned char buf[4];
+        buf[0] = 0xc8;
+        _msgpack_store16(&buf[1], (uint16_t)l);
+        buf[3] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 4);
+    } else {
+        unsigned char buf[6];
+        buf[0] = 0xc9;
+        _msgpack_store32(&buf[1], (uint32_t)l);
+        buf[5] = (unsigned char)typecode;
+        msgpack_pack_append_buffer(x, buf, 6);
+    }
+
+}
+
+
+
 #undef msgpack_pack_append_buffer
 
 #undef TAKE8_8
diff --git a/test/test_extension.py b/test/test_extension.py
index 1908fa2..9ec1153 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -6,7 +6,7 @@ def test_pack_extended_type():
     def p(s):
         packer = msgpack.Packer()
         packer.pack_extended_type(0x42, s)
-        return packer._buffer.getvalue()
+        return packer.bytes()
     assert p('A')        == '\xd4\x42A'          # fixext 1
     assert p('AB')       == '\xd5\x42AB'         # fixext 2
     assert p('ABCD')     == '\xd6\x42ABCD'       # fixext 4

From a7485eccb2e5fcebbd76612a658f2e18bdebe745 Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Fri, 18 Oct 2013 17:46:42 +0200
Subject: [PATCH 1200/1648] add the hook for unknown types also to the cython
 Packer

---
 msgpack/_packer.pyx | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 985559c..f2a0f76 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -176,6 +176,9 @@ cdef class Packer(object):
                 for v in o:
                     ret = self._pack(v, nest_limit-1)
                     if ret != 0: break
+        elif self.handle_unknown_type(o):
+            # it means that obj was succesfully packed, so we are done
+            return 0
         elif self._default:
             o = self._default(o)
             ret = self._pack(o, nest_limit-1)
@@ -195,6 +198,9 @@ cdef class Packer(object):
             self.pk.length = 0
             return buf
 
+    def handle_unknown_type(self, obj):
+        return None
+
     def pack_extended_type(self, typecode, data):
         msgpack_pack_ext(&self.pk, typecode, len(data))
         msgpack_pack_raw_body(&self.pk, data, len(data))

From ff858387d37d37ec4472f6b6ac7010d8f2b0744f Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Sat, 19 Oct 2013 01:49:03 +0200
Subject: [PATCH 1201/1648] implement unpack_one also for the cython version,
 and add a test for it

---
 msgpack/_unpacker.pyx  | 18 ++++++++++++++++++
 test/test_sequnpack.py | 15 ++++++++++++++-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 1f4dd85..e05b9ed 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -359,6 +359,24 @@ cdef class Unpacker(object):
         """
         return self._unpack(unpack_construct, write_bytes)
 
+    def unpack_one(self, object write_bytes=None):
+        """
+        unpack one object
+
+        If write_bytes is not None, it will be called with parts of the raw
+        message as it is unpacked.
+
+        Raises `UnpackValueError` if there are no more bytes to unpack.
+        Raises ``ExtraData`` if there are still bytes left after the unpacking.
+        """
+        try:
+            result = self.unpack()
+        except OutOfData:
+            raise UnpackValueError("Data is not enough")
+        if self.buf_head < self.buf_tail:
+            raise ExtraData(result, self.buf[self.buf_head:])
+        return result
+
     def skip(self, object write_bytes=None):
         """
         read and ignore one object, returning None
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 9db14ca..abc447a 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,9 +1,10 @@
 #!/usr/bin/env python
 # coding: utf-8
 
+import py
 import six
 from msgpack import Unpacker, BufferFull
-from msgpack.exceptions import OutOfData
+from msgpack.exceptions import OutOfData, ExtraData, UnpackValueError
 from pytest import raises
 
 
@@ -85,3 +86,15 @@ def test_readbytes():
     assert unpacker.unpack() == ord(b'a')
     assert unpacker.unpack() == ord(b'r')
 
+def test_unpack_one():
+    unpacker = Unpacker()
+    unpacker.feed('\xda\x00\x03abc')
+    assert unpacker.unpack_one() == 'abc'
+    #
+    unpacker = Unpacker()
+    unpacker.feed('\xda\x00\x03abcd')
+    py.test.raises(ExtraData, "unpacker.unpack_one()")
+    #
+    unpacker = Unpacker()
+    unpacker.feed('\xda\x00\x03ab')
+    py.test.raises(UnpackValueError, "unpacker.unpack_one()")

From 985d4c1496d8c9186079ebc4e42aee319e67c385 Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Sat, 19 Oct 2013 11:34:28 +0200
Subject: [PATCH 1202/1648] add a test for unpacking extended types

---
 test/test_extension.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/test/test_extension.py b/test/test_extension.py
index 9ec1153..96944a3 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -16,6 +16,28 @@ def test_pack_extended_type():
     assert p('A'*0x0123)        == '\xc8\x01\x23\x42' + 'A'*0x0123 # ext 16
     assert p('A'*0x00012345)    == '\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345 # ext 32
 
+def test_unpack_extended_type():
+    class MyUnpacker(msgpack.Unpacker):
+        def read_extended_type(self, typecode, data):
+            return (typecode, data)
+
+    def u(s):
+        unpacker = MyUnpacker()
+        unpacker.feed(s)
+        return unpacker.unpack_one()
+
+    assert u('\xd4\x42A')         == (0x42, 'A')        # fixext 1
+    assert u('\xd5\x42AB')        == (0x42, 'AB')       # fixext 2
+    assert u('\xd6\x42ABCD')      == (0x42, 'ABCD')     # fixext 4
+    assert u('\xd7\x42ABCDEFGH')  == (0x42, 'ABCDEFGH') # fixext 8
+    assert u('\xd8\x42' + 'A'*16) == (0x42, 'A'*16)     # fixext 16
+    assert u('\xc7\x03\x42ABC')   == (0x42, 'ABC')      # ext 8
+    assert (u('\xc8\x01\x23\x42' + 'A'*0x0123) ==
+            (0x42, 'A'*0x0123))                         # ext 16
+    assert (u('\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345) ==
+            (0x42, 'A'*0x00012345))                     # ext 32
+
+
 def test_extension_type():
     class MyPacker(msgpack.Packer):
         def handle_unknown_type(self, obj):

From 56dd1650a42a454027ba335b494100a9f211758e Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Sat, 19 Oct 2013 17:27:16 +0200
Subject: [PATCH 1203/1648] implement unpacking for all the fixtext formats

---
 msgpack/_unpacker.pyx     | 21 +++++++++++++++++++--
 msgpack/unpack.h          | 18 ++++++++++++++++++
 msgpack/unpack_define.h   | 14 ++++++++------
 msgpack/unpack_template.h | 22 +++++++++++++++-------
 setup.py                  |  1 +
 test/test_extension.py    |  1 +
 6 files changed, 62 insertions(+), 15 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index e05b9ed..6500ef7 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -25,6 +25,7 @@ cdef extern from "unpack.h":
         PyObject* object_hook
         bint has_pairs_hook # call object_hook with k-v pairs
         PyObject* list_hook
+        PyObject* ext_type_hook
         char *encoding
         char *unicode_errors
 
@@ -46,6 +47,7 @@ cdef extern from "unpack.h":
 
 cdef inline init_ctx(unpack_context *ctx,
                      object object_hook, object object_pairs_hook, object list_hook,
+                     object ext_type_hook,
                      bint use_list, char* encoding, char* unicode_errors):
     unpack_init(ctx)
     ctx.user.use_list = use_list
@@ -72,9 +74,17 @@ cdef inline init_ctx(unpack_context *ctx,
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
 
+    if ext_type_hook is not None:
+        if not PyCallable_Check(ext_type_hook):
+            raise TypeError("ext_type_hook must be a callable.")
+        ctx.user.ext_type_hook = ext_type_hook
+
     ctx.user.encoding = encoding
     ctx.user.unicode_errors = unicode_errors
 
+def default_read_extended_type(typecode, data):
+    raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode)
+
 def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=1, encoding=None, unicode_errors="strict",
             object_pairs_hook=None,
@@ -107,7 +117,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
             unicode_errors = unicode_errors.encode('ascii')
         cerr = PyBytes_AsString(unicode_errors)
 
-    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
+    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, default_read_extended_type,
+              use_list, cenc, cerr)
     ret = unpack_construct(&ctx, buf, buf_len, &off)
     if ret == 1:
         obj = unpack_data(&ctx)
@@ -249,7 +260,10 @@ cdef class Unpacker(object):
                 self.unicode_errors = unicode_errors
             cerr = PyBytes_AsString(self.unicode_errors)
 
-        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr)
+        ext_type_hook = self.read_extended_type
+        Py_INCREF(ext_type_hook)
+        init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
+                  ext_type_hook, use_list, cenc, cerr)
 
     def feed(self, object next_bytes):
         """Append `next_bytes` to internal buffer."""
@@ -404,6 +418,9 @@ cdef class Unpacker(object):
         """
         return self._unpack(read_map_header, write_bytes)
 
+    def read_extended_type(self, typecode, data):
+        return default_read_extended_type(typecode, data)
+
     def __iter__(self):
         return self
 
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index baeed1f..97ebd3f 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -24,6 +24,7 @@ typedef struct unpack_user {
     PyObject *object_hook;
     bool has_pairs_hook;
     PyObject *list_hook;
+    PyObject *ext_type_hook;
     const char *encoding;
     const char *unicode_errors;
 } unpack_user;
@@ -226,4 +227,21 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
     return 0;
 }
 
+static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
+                                      unsigned int lenght, msgpack_unpack_object* o)
+{
+    PyObject *py;
+    int8_t typecode = (int8_t)*pos++;
+    if (!u->ext_type_hook) {
+        PyErr_SetString(PyExc_AssertionError, "u->ext_type_hook cannot be NULL");
+        return -1;
+    }
+    // lenght also includes the typecode, so the actual data is lenght-1
+    py = PyEval_CallFunction(u->ext_type_hook, "(is#)", typecode, pos, lenght-1);
+    if (!py)
+        return -1;
+    *o = py;
+    return 0;
+}
+
 #include "unpack_template.h"
diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h
index c81b990..986fa91 100644
--- a/msgpack/unpack_define.h
+++ b/msgpack/unpack_define.h
@@ -59,12 +59,12 @@ typedef enum {
     CS_INT_32            = 0x12,
     CS_INT_64            = 0x13,
 
-    //CS_                = 0x14,
-    //CS_                = 0x15,
-    //CS_BIG_INT_16        = 0x16,
-    //CS_BIG_INT_32        = 0x17,
-    //CS_BIG_FLOAT_16      = 0x18,
-    //CS_BIG_FLOAT_32      = 0x19,
+    CS_FIXEXT1           = 0x14,
+    CS_FIXEXT2           = 0x15,
+    CS_FIXEXT4           = 0x16,
+    CS_FIXEXT8           = 0x17,
+    CS_FIXEXT16          = 0x18,
+
     CS_RAW_16            = 0x1a,
     CS_RAW_32            = 0x1b,
     CS_ARRAY_16          = 0x1c,
@@ -75,6 +75,8 @@ typedef enum {
     //ACS_BIG_INT_VALUE,
     //ACS_BIG_FLOAT_VALUE,
     ACS_RAW_VALUE,
+    ACS_EXT_VALUE,
+
 } msgpack_unpack_state;
 
 
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 29ac935..b051075 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -202,12 +202,16 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
                 case 0xd2:  // signed int 32
                 case 0xd3:  // signed int 64
                     again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
-                //case 0xd4:
-                //case 0xd5:
-                //case 0xd6:  // big integer 16
-                //case 0xd7:  // big integer 32
-                //case 0xd8:  // big float 16
-                //case 0xd9:  // big float 32
+                case 0xd4:  // fixext 1
+                case 0xd5:  // fixext 2
+                case 0xd6:  // fixext 4
+                case 0xd7:  // fixext 8
+                    again_fixed_trail_if_zero(ACS_EXT_VALUE, 
+                                              (1 << (((unsigned int)*p) & 0x03))+1,
+                                              _ext_zero);
+                case 0xd8:  // fixext 16
+                    again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero);
+                //case 0xd9:
                 case 0xda:  // raw 16
                 case 0xdb:  // raw 32
                 case 0xdc:  // array 16
@@ -298,6 +302,10 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
             _raw_zero:
                 push_variable_value(_raw, data, n, trail);
 
+            case ACS_EXT_VALUE:
+            _ext_zero:
+                push_variable_value(_ext, data, n, trail);
+
             case CS_ARRAY_16:
                 start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
             case CS_ARRAY_32:
@@ -309,7 +317,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
             case CS_MAP_32:
                 /* FIXME security guard */
                 start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
-
+            
             default:
                 goto _failed;
             }
diff --git a/setup.py b/setup.py
index 1055a61..83ae79f 100644
--- a/setup.py
+++ b/setup.py
@@ -92,6 +92,7 @@ if not hasattr(sys, 'pypy_version_info'):
                                  libraries=libraries,
                                  include_dirs=['.'],
                                  define_macros=macros,
+                                 extra_compile_args=['-O0'],
                                  ))
 del libraries, macros
 
diff --git a/test/test_extension.py b/test/test_extension.py
index 96944a3..94117e1 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -1,3 +1,4 @@
+import py
 import array
 import struct
 import msgpack

From c9b97f078854dda02dc9404ee9d88ca5e16fb493 Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Sat, 19 Oct 2013 18:04:30 +0200
Subject: [PATCH 1204/1648] implement unpacking of ext 8,16,32

---
 msgpack/unpack_define.h   | 17 ++++++++---------
 msgpack/unpack_template.h | 21 ++++++++++++++++-----
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h
index 986fa91..17c287e 100644
--- a/msgpack/unpack_define.h
+++ b/msgpack/unpack_define.h
@@ -44,10 +44,9 @@ typedef enum {
     //CS_                = 0x04,
     //CS_                = 0x05,
     //CS_                = 0x06,
-    //CS_                = 0x07,
-
-    //CS_                = 0x08,
-    //CS_                = 0x09,
+    CS_EXT_8             = 0x07,
+    CS_EXT_16            = 0x08,
+    CS_EXT_32            = 0x09,
     CS_FLOAT             = 0x0a,
     CS_DOUBLE            = 0x0b,
     CS_UINT_8            = 0x0c,
@@ -59,11 +58,11 @@ typedef enum {
     CS_INT_32            = 0x12,
     CS_INT_64            = 0x13,
 
-    CS_FIXEXT1           = 0x14,
-    CS_FIXEXT2           = 0x15,
-    CS_FIXEXT4           = 0x16,
-    CS_FIXEXT8           = 0x17,
-    CS_FIXEXT16          = 0x18,
+    //CS_FIXEXT1           = 0x14,
+    //CS_FIXEXT2           = 0x15,
+    //CS_FIXEXT4           = 0x16,
+    //CS_FIXEXT8           = 0x17,
+    //CS_FIXEXT16          = 0x18,
 
     CS_RAW_16            = 0x1a,
     CS_RAW_32            = 0x1b,
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index b051075..0c6af0e 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -188,9 +188,12 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
                 //case 0xc4:
                 //case 0xc5:
                 //case 0xc6:
-                //case 0xc7:
-                //case 0xc8:
-                //case 0xc9:
+                case 0xc7:  // ext 8
+                    again_fixed_trail(NEXT_CS(p), 1);
+                case 0xc8:  // ext 16
+                    again_fixed_trail(NEXT_CS(p), 2);
+                case 0xc9:  // ext 32
+                    again_fixed_trail(NEXT_CS(p), 4);
                 case 0xca:  // float
                 case 0xcb:  // double
                 case 0xcc:  // unsigned int  8
@@ -242,8 +245,16 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
             if((size_t)(pe - p) < trail) { goto _out; }
             n = p;  p += trail - 1;
             switch(cs) {
-            //case CS_
-            //case CS_
+            case CS_EXT_8:
+                again_fixed_trail_if_zero(ACS_EXT_VALUE, *(uint8_t*)n+1, _ext_zero);
+            case CS_EXT_16:
+                again_fixed_trail_if_zero(ACS_EXT_VALUE,
+                                          _msgpack_load16(uint16_t,n)+1,
+                                          _ext_zero);
+            case CS_EXT_32:
+                again_fixed_trail_if_zero(ACS_EXT_VALUE,
+                                          _msgpack_load32(uint32_t,n)+1,
+                                          _ext_zero);
             case CS_FLOAT: {
                     union { uint32_t i; float f; } mem;
                     mem.i = _msgpack_load32(uint32_t,n);

From 6386481024ec045d9ef991a2c975902276812508 Mon Sep 17 00:00:00 2001
From: Antonio Cuni 
Date: Sat, 19 Oct 2013 18:43:16 +0200
Subject: [PATCH 1205/1648] add a note in the README

---
 README.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.rst b/README.rst
index 294cd63..600d7f7 100644
--- a/README.rst
+++ b/README.rst
@@ -140,6 +140,14 @@ It is also possible to pack/unpack custom data types. Here is an example for
 ``object_pairs_hook`` callback may instead be used to receive a list of
 key-value pairs.
 
+Extended types
+^^^^^^^^^^^^^^^
+
+It is also possible to pack/unpack custom data types using the msgpack feature
+of "extended types". For example, msgpack-pypy uses it to provide very fast serialization of int/float lists on top of PyPy (experimental for now):
+
+https://bitbucket.org/antocuni/msgpack-pypy/src/default/msgpack_pypy.py
+
 
 Advanced unpacking control
 ^^^^^^^^^^^^^^^^^^^^^^^^^^

From 7123341ca89a9a3afee8521cc16a1a419ea8871e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 20 Oct 2013 14:34:36 +0900
Subject: [PATCH 1206/1648] code refactoring.

---
 msgpack/fallback.py | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 074fcba..dfaaa54 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -163,12 +163,10 @@ class Unpacker(object):
         self._fb_buf_o = 0
         self._fb_buf_i = 0
         self._fb_buf_n = 0
-        self._max_buffer_size = (2**31-1 if max_buffer_size == 0
-                                         else max_buffer_size)
-        self._read_size = (read_size if read_size != 0
-                            else min(self._max_buffer_size, 2048))
+        self._max_buffer_size = max_buffer_size or 2**31-1
         if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")
+        self._read_size = read_size or min(self._max_buffer_size, 2048)
         self._encoding = encoding
         self._unicode_errors = unicode_errors
         self._use_list = use_list
@@ -234,26 +232,26 @@ class Unpacker(object):
         return b''.join(bufs)
 
     def _fb_read(self, n, write_bytes=None):
-        if (write_bytes is None and self._fb_buf_i < len(self._fb_buffers)
-                and self._fb_buf_o + n < len(self._fb_buffers[self._fb_buf_i])):
+        buffs = self._fb_buffers
+        if (write_bytes is None and self._fb_buf_i < len(buffs) and
+                self._fb_buf_o + n < len(buffs[self._fb_buf_i])):
             self._fb_buf_o += n
-            return self._fb_buffers[self._fb_buf_i][
-                    self._fb_buf_o-n:self._fb_buf_o]
+            return buffs[self._fb_buf_i][self._fb_buf_o - n:self._fb_buf_o]
+
         ret = b''
         while len(ret) != n:
-            if self._fb_buf_i == len(self._fb_buffers):
+            if self._fb_buf_i == len(buffs):
                 if self._fb_feeding:
                     break
                 tmp = self.file_like.read(self._read_size)
                 if not tmp:
                     break
-                self._fb_buffers.append(tmp)
+                buffs.append(tmp)
                 continue
             sliced = n - len(ret)
-            ret += self._fb_buffers[self._fb_buf_i][
-                        self._fb_buf_o:self._fb_buf_o + sliced]
+            ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced]
             self._fb_buf_o += sliced
-            if self._fb_buf_o >= len(self._fb_buffers[self._fb_buf_i]):
+            if self._fb_buf_o >= len(buffs[self._fb_buf_i]):
                 self._fb_buf_o = 0
                 self._fb_buf_i += 1
         if len(ret) != n:
@@ -410,10 +408,9 @@ class Unpacker(object):
                 return
             if self._object_pairs_hook is not None:
                 ret = self._object_pairs_hook(
-                        (self._fb_unpack(EX_CONSTRUCT, write_bytes),
-                         self._fb_unpack(EX_CONSTRUCT, write_bytes))
-                            for _ in xrange(n)
-                        )
+                    (self._fb_unpack(EX_CONSTRUCT, write_bytes),
+                     self._fb_unpack(EX_CONSTRUCT, write_bytes))
+                    for _ in xrange(n))
             else:
                 ret = {}
                 for _ in xrange(n):

From aa68c9b8330b130d600b22ec47d5c3841499b536 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 20 Oct 2013 15:40:20 +0900
Subject: [PATCH 1207/1648] fallback: Support pack_ext_type.

---
 msgpack/_packer.pyx    |  2 +-
 msgpack/_unpacker.pyx  |  2 --
 msgpack/fallback.py    | 46 +++++++++++++++++++++++++++++++++++-------
 test/test_extension.py | 26 ++++++++++++------------
 4 files changed, 53 insertions(+), 23 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index df670ed..f033263 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -210,7 +210,7 @@ cdef class Packer(object):
     def handle_unknown_type(self, obj):
         return None
 
-    def pack_extended_type(self, typecode, data):
+    def pack_ext_type(self, typecode, data):
         msgpack_pack_ext(&self.pk, typecode, len(data))
         msgpack_pack_raw_body(&self.pk, data, len(data))
 
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index cf30670..b0e66db 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -32,8 +32,6 @@ cdef extern from "unpack.h":
         msgpack_user user
         PyObject* obj
         size_t count
-        unsigned int ct
-        PyObject* key
 
     ctypedef int (*execute_fn)(unpack_context* ctx, const char* data,
                                size_t len, size_t* off) except? -1
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index dfaaa54..0b29700 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -42,11 +42,11 @@ else:
     newlist_hint = lambda size: []
 
 from msgpack.exceptions import (
-        BufferFull,
-        OutOfData,
-        UnpackValueError,
-        PackValueError,
-        ExtraData)
+    BufferFull,
+    OutOfData,
+    UnpackValueError,
+    PackValueError,
+    ExtraData)
 
 from msgpack import ExtType
 
@@ -65,6 +65,7 @@ TYPE_EXT                = 5
 
 DEFAULT_RECURSE_LIMIT = 511
 
+
 def unpack(stream, **kwargs):
     """
     Unpack an object from `stream`.
@@ -78,6 +79,7 @@ def unpack(stream, **kwargs):
         raise ExtraData(ret, unpacker._fb_get_extradata())
     return ret
 
+
 def unpackb(packed, **kwargs):
     """
     Unpack an object from `packed`.
@@ -95,6 +97,7 @@ def unpackb(packed, **kwargs):
         raise ExtraData(ret, unpacker._fb_get_extradata())
     return ret
 
+
 class Unpacker(object):
     """
     Streaming unpacker.
@@ -548,8 +551,8 @@ class Packer(object):
             if isinstance(obj, Unicode):
                 if self._encoding is None:
                     raise TypeError(
-                            "Can't encode unicode string: "
-                            "no encoding is specified")
+                        "Can't encode unicode string: "
+                        "no encoding is specified")
                 obj = obj.encode(self._encoding, self._unicode_errors)
             n = len(obj)
             if n <= 0x1f:
@@ -616,6 +619,35 @@ class Packer(object):
             self._buffer = StringIO(ret)
         return ret
 
+    def pack_ext_type(self, typecode, data):
+        if not isinstance(typecode, int):
+            raise TypeError("typecode must have int type.")
+        if not 0 <= typecode <= 127:
+            raise ValueError("typecode should be 0-127")
+        if not isinstance(data, bytes):
+            raise TypeError("data must have bytes type")
+        L = len(data)
+        if L > 0xffffffff:
+            raise ValueError("Too large data")
+        if L == 1:
+            self._buffer.write(b'\xd4')
+        elif L == 2:
+            self._buffer.write(b'\xd5')
+        elif L == 4:
+            self._buffer.write(b'\xd6')
+        elif L == 8:
+            self._buffer.write(b'\xd7')
+        elif L == 16:
+            self._buffer.write(b'\xd8')
+        elif L <= 0xff:
+            self._buffer.write(b'\xc7' + struct.pack('B', L))
+        elif L <= 0xffff:
+            self._buffer.write(b'\xc8' + struct.pack('>H', L))
+        else:
+            self._buffer.write(b'\xc9' + struct.pack('>I', L))
+        self._buffer.write(struct.pack('B', typecode))
+        self._buffer.write(data)
+
     def _fb_pack_array_header(self, n):
         if n <= 0x0f:
             return self._buffer.write(struct.pack('B', 0x90 + n))
diff --git a/test/test_extension.py b/test/test_extension.py
index 94117e1..f2fa363 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -1,21 +1,21 @@
-import py
 import array
-import struct
 import msgpack
 
-def test_pack_extended_type():
+
+def test_pack_ext_type():
     def p(s):
         packer = msgpack.Packer()
-        packer.pack_extended_type(0x42, s)
+        packer.pack_ext_type(0x42, s)
         return packer.bytes()
-    assert p('A')        == '\xd4\x42A'          # fixext 1
-    assert p('AB')       == '\xd5\x42AB'         # fixext 2
-    assert p('ABCD')     == '\xd6\x42ABCD'       # fixext 4
-    assert p('ABCDEFGH') == '\xd7\x42ABCDEFGH'   # fixext 8
-    assert p('A'*16)     == '\xd8\x42' + 'A'*16  # fixext 16
-    assert p('ABC')      == '\xc7\x03\x42ABC'        # ext 8
-    assert p('A'*0x0123)        == '\xc8\x01\x23\x42' + 'A'*0x0123 # ext 16
-    assert p('A'*0x00012345)    == '\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345 # ext 32
+    assert p(b'A')        == b'\xd4\x42A'          # fixext 1
+    assert p(b'AB')       == b'\xd5\x42AB'         # fixext 2
+    assert p(b'ABCD')     == b'\xd6\x42ABCD'       # fixext 4
+    assert p(b'ABCDEFGH') == b'\xd7\x42ABCDEFGH'   # fixext 8
+    assert p(b'A'*16)     == b'\xd8\x42' + 'A'*16  # fixext 16
+    assert p(b'ABC')      == b'\xc7\x03\x42ABC'        # ext 8
+    assert p(b'A'*0x0123)     == b'\xc8\x01\x23\x42' + b'A'*0x0123 # ext 16
+    assert p(b'A'*0x00012345) == b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345 # ext 32
+
 
 def test_unpack_extended_type():
     class MyUnpacker(msgpack.Unpacker):
@@ -45,7 +45,7 @@ def test_extension_type():
             if isinstance(obj, array.array):
                 typecode = 123 # application specific typecode
                 data = obj.tostring()
-                self.pack_extended_type(typecode, data)
+                self.pack_ext_type(typecode, data)
                 return True
 
     class MyUnpacker(msgpack.Unpacker):

From 96bcd76f49afd00f5b7def1ff7cfd002a7fa477d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 20 Oct 2013 20:28:32 +0900
Subject: [PATCH 1208/1648] Packing ExtType and some cleanup

---
 msgpack/__init__.py     |   2 +-
 msgpack/_packer.pyx     | 150 +++++++++++++++++++++-------------------
 msgpack/_unpacker.pyx   |  49 ++++---------
 msgpack/pack_template.h |   3 +-
 msgpack/unpack.h        |  10 +--
 test/test_extension.py  |  71 +++++++++----------
 test/test_sequnpack.py  |  13 ----
 7 files changed, 130 insertions(+), 168 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 79107b6..a7b47b1 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -26,6 +26,7 @@ def pack(o, stream, **kwargs):
     packer = Packer(**kwargs)
     stream.write(packer.pack(o))
 
+
 def packb(o, **kwargs):
     """
     Pack object `o` and return packed bytes
@@ -40,4 +41,3 @@ loads = unpackb
 
 dump = pack
 dumps = packb
-
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index f033263..f63667c 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -8,6 +8,8 @@ from libc.limits cimport *
 from libc.stdint cimport int8_t
 
 from msgpack.exceptions import PackValueError
+from msgpack import ExtType
+
 
 cdef extern from "pack.h":
     struct msgpack_packer:
@@ -120,80 +122,87 @@ cdef class Packer(object):
         cdef int ret
         cdef dict d
         cdef size_t L
+        cdef int default_used = 0
 
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded.")
 
-        if o is None:
-            ret = msgpack_pack_nil(&self.pk)
-        elif isinstance(o, bool):
-            if o:
-                ret = msgpack_pack_true(&self.pk)
-            else:
-                ret = msgpack_pack_false(&self.pk)
-        elif PyLong_Check(o):
-            if o > 0:
-                ullval = o
-                ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
-            else:
-                llval = o
-                ret = msgpack_pack_long_long(&self.pk, llval)
-        elif PyInt_Check(o):
-            longval = o
-            ret = msgpack_pack_long(&self.pk, longval)
-        elif PyFloat_Check(o):
-            if self.use_float:
-               fval = o
-               ret = msgpack_pack_float(&self.pk, fval)
-            else:
-               dval = o
-               ret = msgpack_pack_double(&self.pk, dval)
-        elif PyBytes_Check(o):
-            rawval = o
-            L = len(o)
-            ret = msgpack_pack_bin(&self.pk, L)
-            if ret == 0:
+        while True:
+            if o is None:
+                ret = msgpack_pack_nil(&self.pk)
+            elif isinstance(o, bool):
+                if o:
+                    ret = msgpack_pack_true(&self.pk)
+                else:
+                    ret = msgpack_pack_false(&self.pk)
+            elif PyLong_Check(o):
+                if o > 0:
+                    ullval = o
+                    ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
+                else:
+                    llval = o
+                    ret = msgpack_pack_long_long(&self.pk, llval)
+            elif PyInt_Check(o):
+                longval = o
+                ret = msgpack_pack_long(&self.pk, longval)
+            elif PyFloat_Check(o):
+                if self.use_float:
+                   fval = o
+                   ret = msgpack_pack_float(&self.pk, fval)
+                else:
+                   dval = o
+                   ret = msgpack_pack_double(&self.pk, dval)
+            elif PyBytes_Check(o):
+                rawval = o
+                L = len(o)
+                ret = msgpack_pack_bin(&self.pk, L)
+                if ret == 0:
+                    ret = msgpack_pack_raw_body(&self.pk, rawval, L)
+            elif PyUnicode_Check(o):
+                if not self.encoding:
+                    raise TypeError("Can't encode unicode string: no encoding is specified")
+                o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
+                rawval = o
+                ret = msgpack_pack_raw(&self.pk, len(o))
+                if ret == 0:
+                    ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
+            elif PyDict_CheckExact(o):
+                d = o
+                ret = msgpack_pack_map(&self.pk, len(d))
+                if ret == 0:
+                    for k, v in d.iteritems():
+                        ret = self._pack(k, nest_limit-1)
+                        if ret != 0: break
+                        ret = self._pack(v, nest_limit-1)
+                        if ret != 0: break
+            elif PyDict_Check(o):
+                ret = msgpack_pack_map(&self.pk, len(o))
+                if ret == 0:
+                    for k, v in o.items():
+                        ret = self._pack(k, nest_limit-1)
+                        if ret != 0: break
+                        ret = self._pack(v, nest_limit-1)
+                        if ret != 0: break
+            elif isinstance(o, ExtType):
+                # This should be before Tuple because ExtType is namedtuple.
+                longval = o[0]
+                rawval = o[1]
+                L = len(o[1])
+                ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
-        elif PyUnicode_Check(o):
-            if not self.encoding:
-                raise TypeError("Can't encode unicode string: no encoding is specified")
-            o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
-            rawval = o
-            ret = msgpack_pack_raw(&self.pk, len(o))
-            if ret == 0:
-                ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
-        elif PyDict_CheckExact(o):
-            d = o
-            ret = msgpack_pack_map(&self.pk, len(d))
-            if ret == 0:
-                for k, v in d.iteritems():
-                    ret = self._pack(k, nest_limit-1)
-                    if ret != 0: break
-                    ret = self._pack(v, nest_limit-1)
-                    if ret != 0: break
-        elif PyDict_Check(o):
-            ret = msgpack_pack_map(&self.pk, len(o))
-            if ret == 0:
-                for k, v in o.items():
-                    ret = self._pack(k, nest_limit-1)
-                    if ret != 0: break
-                    ret = self._pack(v, nest_limit-1)
-                    if ret != 0: break
-        elif PyTuple_Check(o) or PyList_Check(o):
-            ret = msgpack_pack_array(&self.pk, len(o))
-            if ret == 0:
-                for v in o:
-                    ret = self._pack(v, nest_limit-1)
-                    if ret != 0: break
-        elif self.handle_unknown_type(o):
-            # it means that obj was succesfully packed, so we are done
-            return 0
-        elif self._default:
-            o = self._default(o)
-            ret = self._pack(o, nest_limit-1)
-        else:
-            raise TypeError("can't serialize %r" % (o,))
-        return ret
+            elif PyTuple_Check(o) or PyList_Check(o):
+                ret = msgpack_pack_array(&self.pk, len(o))
+                if ret == 0:
+                    for v in o:
+                        ret = self._pack(v, nest_limit-1)
+                        if ret != 0: break
+            elif not default_used and self._default:
+                o = self._default(o)
+                default_used = 1
+                continue
+            else:
+                raise TypeError("can't serialize %r" % (o,))
+            return ret
 
     cpdef pack(self, object obj):
         cdef int ret
@@ -207,9 +216,6 @@ cdef class Packer(object):
             self.pk.length = 0
             return buf
 
-    def handle_unknown_type(self, obj):
-        return None
-
     def pack_ext_type(self, typecode, data):
         msgpack_pack_ext(&self.pk, typecode, len(data))
         msgpack_pack_raw_body(&self.pk, data, len(data))
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index b0e66db..d5aa46e 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -16,6 +16,7 @@ from msgpack.exceptions import (
         UnpackValueError,
         ExtraData,
         )
+from msgpack import ExtType
 
 
 cdef extern from "unpack.h":
@@ -24,7 +25,7 @@ cdef extern from "unpack.h":
         PyObject* object_hook
         bint has_pairs_hook # call object_hook with k-v pairs
         PyObject* list_hook
-        PyObject* ext_type_hook
+        PyObject* ext_hook
         char *encoding
         char *unicode_errors
 
@@ -43,8 +44,8 @@ cdef extern from "unpack.h":
     object unpack_data(unpack_context* ctx)
 
 cdef inline init_ctx(unpack_context *ctx,
-                     object object_hook, object object_pairs_hook, object list_hook,
-                     object ext_type_hook,
+                     object object_hook, object object_pairs_hook,
+                     object list_hook, object ext_hook,
                      bint use_list, char* encoding, char* unicode_errors):
     unpack_init(ctx)
     ctx.user.use_list = use_list
@@ -71,10 +72,10 @@ cdef inline init_ctx(unpack_context *ctx,
             raise TypeError("list_hook must be a callable.")
         ctx.user.list_hook = list_hook
 
-    if ext_type_hook is not None:
-        if not PyCallable_Check(ext_type_hook):
-            raise TypeError("ext_type_hook must be a callable.")
-        ctx.user.ext_type_hook = ext_type_hook
+    if ext_hook is not None:
+        if not PyCallable_Check(ext_hook):
+            raise TypeError("ext_hook must be a callable.")
+        ctx.user.ext_hook = ext_hook
 
     ctx.user.encoding = encoding
     ctx.user.unicode_errors = unicode_errors
@@ -84,8 +85,7 @@ def default_read_extended_type(typecode, data):
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=1, encoding=None, unicode_errors="strict",
-            object_pairs_hook=None,
-            ):
+            object_pairs_hook=None, ext_hook=ExtType):
     """
     Unpack packed_bytes to object. Returns an unpacked object.
 
@@ -114,8 +114,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
             unicode_errors = unicode_errors.encode('ascii')
         cerr = PyBytes_AsString(unicode_errors)
 
-    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, default_read_extended_type,
-              use_list, cenc, cerr)
+    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
+             use_list, cenc, cerr)
     ret = unpack_construct(&ctx, buf, buf_len, &off)
     if ret == 1:
         obj = unpack_data(&ctx)
@@ -220,7 +220,7 @@ cdef class Unpacker(object):
     def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  str encoding=None, str unicode_errors='strict', int max_buffer_size=0,
-                 ):
+                 object ext_hook=ExtType):
         cdef char *cenc=NULL, *cerr=NULL
 
         self.file_like = file_like
@@ -257,10 +257,8 @@ cdef class Unpacker(object):
                 self.unicode_errors = unicode_errors
             cerr = PyBytes_AsString(self.unicode_errors)
 
-        ext_type_hook = self.read_extended_type
-        Py_INCREF(ext_type_hook)
         init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
-                  ext_type_hook, use_list, cenc, cerr)
+                 ext_hook, use_list, cenc, cerr)
 
     def feed(self, object next_bytes):
         """Append `next_bytes` to internal buffer."""
@@ -370,24 +368,6 @@ cdef class Unpacker(object):
         """
         return self._unpack(unpack_construct, write_bytes)
 
-    def unpack_one(self, object write_bytes=None):
-        """
-        unpack one object
-
-        If write_bytes is not None, it will be called with parts of the raw
-        message as it is unpacked.
-
-        Raises `UnpackValueError` if there are no more bytes to unpack.
-        Raises ``ExtraData`` if there are still bytes left after the unpacking.
-        """
-        try:
-            result = self.unpack()
-        except OutOfData:
-            raise UnpackValueError("Data is not enough")
-        if self.buf_head < self.buf_tail:
-            raise ExtraData(result, self.buf[self.buf_head:])
-        return result
-
     def skip(self, object write_bytes=None):
         """
         read and ignore one object, returning None
@@ -415,9 +395,6 @@ cdef class Unpacker(object):
         """
         return self._unpack(read_map_header, write_bytes)
 
-    def read_extended_type(self, typecode, data):
-        return default_read_extended_type(typecode, data)
-
     def __iter__(self):
         return self
 
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 0fe9514..8b91619 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -687,7 +687,7 @@ static inline int msgpack_pack_raw(msgpack_packer* x, size_t l)
 static inline int msgpack_pack_bin(msgpack_packer *x, size_t l)
 {
     if (!x->use_bin_type) {
-        return msgpack_pack_raw(x, l)
+        return msgpack_pack_raw(x, l);
     }
     if (l < 256) {
         unsigned char buf[2] = {0xc4, (unsigned char)l};
@@ -711,7 +711,6 @@ static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t
 /*
  * Ext
  */
-
 static inline int msgpack_pack_ext(msgpack_packer* x, int8_t typecode, size_t l)
 {
     if (l == 1) {
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 327a524..3c09747 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -24,7 +24,7 @@ typedef struct unpack_user {
     PyObject *object_hook;
     bool has_pairs_hook;
     PyObject *list_hook;
-    PyObject *ext_type_hook;
+    PyObject *ext_hook;
     const char *encoding;
     const char *unicode_errors;
 } unpack_user;
@@ -241,12 +241,12 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch
 {
     PyObject *py;
     int8_t typecode = (int8_t)*pos++;
-    if (!u->ext_type_hook) {
-        PyErr_SetString(PyExc_AssertionError, "u->ext_type_hook cannot be NULL");
+    if (!u->ext_hook) {
+        PyErr_SetString(PyExc_AssertionError, "u->ext_hook cannot be NULL");
         return -1;
     }
-    // lenght also includes the typecode, so the actual data is lenght-1
-    py = PyEval_CallFunction(u->ext_type_hook, "(is#)", typecode, pos, lenght-1);
+    // length also includes the typecode, so the actual data is lenght-1
+    py = PyEval_CallFunction(u->ext_hook, "(is#)", typecode, pos, lenght-1);
     if (!py)
         return -1;
     *o = py;
diff --git a/test/test_extension.py b/test/test_extension.py
index f2fa363..2f85ce3 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -1,5 +1,7 @@
+from __future__ import print_function
 import array
 import msgpack
+from msgpack import ExtType
 
 
 def test_pack_ext_type():
@@ -11,54 +13,45 @@ def test_pack_ext_type():
     assert p(b'AB')       == b'\xd5\x42AB'         # fixext 2
     assert p(b'ABCD')     == b'\xd6\x42ABCD'       # fixext 4
     assert p(b'ABCDEFGH') == b'\xd7\x42ABCDEFGH'   # fixext 8
-    assert p(b'A'*16)     == b'\xd8\x42' + 'A'*16  # fixext 16
+    assert p(b'A'*16)     == b'\xd8\x42' + b'A'*16 # fixext 16
     assert p(b'ABC')      == b'\xc7\x03\x42ABC'        # ext 8
     assert p(b'A'*0x0123)     == b'\xc8\x01\x23\x42' + b'A'*0x0123 # ext 16
     assert p(b'A'*0x00012345) == b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345 # ext 32
 
 
-def test_unpack_extended_type():
-    class MyUnpacker(msgpack.Unpacker):
-        def read_extended_type(self, typecode, data):
-            return (typecode, data)
+def test_unpack_ext_type():
+    def check(b, expected):
+        assert msgpack.unpackb(b) == expected
 
-    def u(s):
-        unpacker = MyUnpacker()
-        unpacker.feed(s)
-        return unpacker.unpack_one()
-
-    assert u('\xd4\x42A')         == (0x42, 'A')        # fixext 1
-    assert u('\xd5\x42AB')        == (0x42, 'AB')       # fixext 2
-    assert u('\xd6\x42ABCD')      == (0x42, 'ABCD')     # fixext 4
-    assert u('\xd7\x42ABCDEFGH')  == (0x42, 'ABCDEFGH') # fixext 8
-    assert u('\xd8\x42' + 'A'*16) == (0x42, 'A'*16)     # fixext 16
-    assert u('\xc7\x03\x42ABC')   == (0x42, 'ABC')      # ext 8
-    assert (u('\xc8\x01\x23\x42' + 'A'*0x0123) ==
-            (0x42, 'A'*0x0123))                         # ext 16
-    assert (u('\xc9\x00\x01\x23\x45\x42' + 'A'*0x00012345) ==
-            (0x42, 'A'*0x00012345))                     # ext 32
+    check(b'\xd4\x42A',         ExtType(0x42, b'A'))        # fixext 1
+    check(b'\xd5\x42AB',        ExtType(0x42, b'AB'))       # fixext 2
+    check(b'\xd6\x42ABCD',      ExtType(0x42, b'ABCD'))     # fixext 4
+    check(b'\xd7\x42ABCDEFGH',  ExtType(0x42, b'ABCDEFGH')) # fixext 8
+    check(b'\xd8\x42' + b'A'*16, ExtType(0x42, b'A'*16))    # fixext 16
+    check(b'\xc7\x03\x42ABC',   ExtType(0x42, b'ABC'))      # ext 8
+    check(b'\xc8\x01\x23\x42' + b'A'*0x0123,
+          ExtType(0x42, b'A'*0x0123))                        # ext 16
+    check(b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345,
+          ExtType(0x42, b'A'*0x00012345))                   # ext 32
 
 
 def test_extension_type():
-    class MyPacker(msgpack.Packer):
-        def handle_unknown_type(self, obj):
-            if isinstance(obj, array.array):
-                typecode = 123 # application specific typecode
-                data = obj.tostring()
-                self.pack_ext_type(typecode, data)
-                return True
+    def default(obj):
+        print('default called', obj)
+        if isinstance(obj, array.array):
+            typecode = 123 # application specific typecode
+            data = obj.tostring()
+            return ExtType(typecode, data)
+        raise TypeError("Unknwon type object %r" % (obj,))
 
-    class MyUnpacker(msgpack.Unpacker):
-        def read_extended_type(self, typecode, data):
-            assert typecode == 123
-            obj = array.array('d')
-            obj.fromstring(data)
-            return obj
+    def ext_hook(code, data):
+        print('ext_hook called', code, data)
+        assert code == 123
+        obj = array.array('d')
+        obj.fromstring(data)
+        return obj
 
-    obj = [42, 'hello', array.array('d', [1.1, 2.2, 3.3])]
-    packer = MyPacker()
-    unpacker = MyUnpacker(None)
-    s = packer.pack(obj)
-    unpacker.feed(s)
-    obj2 = unpacker.unpack_one()
+    obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])]
+    s = msgpack.packb(obj, default=default)
+    obj2 = msgpack.unpackb(s, ext_hook=ext_hook)
     assert obj == obj2
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index abc447a..af66b78 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -85,16 +85,3 @@ def test_readbytes():
     assert unpacker.read_bytes(3) == b'oob'
     assert unpacker.unpack() == ord(b'a')
     assert unpacker.unpack() == ord(b'r')
-
-def test_unpack_one():
-    unpacker = Unpacker()
-    unpacker.feed('\xda\x00\x03abc')
-    assert unpacker.unpack_one() == 'abc'
-    #
-    unpacker = Unpacker()
-    unpacker.feed('\xda\x00\x03abcd')
-    py.test.raises(ExtraData, "unpacker.unpack_one()")
-    #
-    unpacker = Unpacker()
-    unpacker.feed('\xda\x00\x03ab')
-    py.test.raises(UnpackValueError, "unpacker.unpack_one()")

From 822cce823cfea8e9f7625598a125897718b4ab58 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 20 Oct 2013 22:59:27 +0900
Subject: [PATCH 1209/1648] Support unpacking new types.

---
 msgpack/unpack.h          | 10 +++++++---
 msgpack/unpack_define.h   |  1 -
 msgpack/unpack_template.h | 15 +++++++++------
 test/test_obj.py          |  2 +-
 4 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 3c09747..c733b24 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -157,7 +157,7 @@ static inline int unpack_callback_array_item(unpack_user* u, unsigned int curren
 static inline int unpack_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->list_hook) {
-        PyObject *new_c = PyEval_CallFunction(u->list_hook, "(O)", *c);
+        PyObject *new_c = PyObject_CallFunction(u->list_hook, "(O)", *c);
         if (!new_c)
             return -1;
         Py_DECREF(*c);
@@ -203,7 +203,7 @@ static inline int unpack_callback_map_item(unpack_user* u, unsigned int current,
 static inline int unpack_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->object_hook) {
-        PyObject *new_c = PyEval_CallFunction(u->object_hook, "(O)", *c);
+        PyObject *new_c = PyObject_CallFunction(u->object_hook, "(O)", *c);
         if (!new_c)
             return -1;
 
@@ -246,7 +246,11 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch
         return -1;
     }
     // length also includes the typecode, so the actual data is lenght-1
-    py = PyEval_CallFunction(u->ext_hook, "(is#)", typecode, pos, lenght-1);
+#if PY_MAJOR_VERSION == 2
+    py = PyObject_CallFunction(u->ext_hook, "(is#)", typecode, pos, lenght-1);
+#else
+    py = PyObject_CallFunction(u->ext_hook, "(iy#)", typecode, pos, lenght-1);
+#endif
     if (!py)
         return -1;
     *o = py;
diff --git a/msgpack/unpack_define.h b/msgpack/unpack_define.h
index 2ee92b5..0dd708d 100644
--- a/msgpack/unpack_define.h
+++ b/msgpack/unpack_define.h
@@ -93,4 +93,3 @@ typedef enum {
 #endif
 
 #endif /* msgpack/unpack_define.h */
-
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 1a709ec..7646896 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -178,15 +178,17 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
                 switch(*p) {
                 case 0xc0:  // nil
                     push_simple_value(_nil);
-                //case 0xc1:  // string
-                //    again_terminal_trail(NEXT_CS(p), p+1);
+                //case 0xc1:  // never used
                 case 0xc2:  // false
                     push_simple_value(_false);
                 case 0xc3:  // true
                     push_simple_value(_true);
-                //case 0xc4:
-                //case 0xc5:
-                //case 0xc6:
+                case 0xc4:  // bin 8
+                    again_fixed_trail(NEXT_CS(p), 1);
+                case 0xc5:  // bin 16
+                    again_fixed_trail(NEXT_CS(p), 2);
+                case 0xc6:  // bin 32
+                    again_fixed_trail(NEXT_CS(p), 4);
                 case 0xc7:  // ext 8
                     again_fixed_trail(NEXT_CS(p), 1);
                 case 0xc8:  // ext 16
@@ -213,7 +215,8 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
                                               _ext_zero);
                 case 0xd8:  // fixext 16
                     again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero);
-                //case 0xd9:
+                case 0xd9:  // str 8
+                    again_fixed_trail(NEXT_CS(p), 1);
                 case 0xda:  // raw 16
                 case 0xdb:  // raw 32
                 case 0xdc:  // array 16
diff --git a/test/test_obj.py b/test/test_obj.py
index fbf610c..9083218 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -35,7 +35,7 @@ def test_only_one_obj_hook():
         unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x)
 
 def test_bad_hook():
-    with raises(ValueError):
+    with raises(TypeError):
         packed = packb([3, 1+2j], default=lambda o: o)
         unpacked = unpackb(packed, use_list=1)
 

From 0d5c58bd517caddd6b62a8931f6833e2a3add283 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 20 Oct 2013 23:06:02 +0900
Subject: [PATCH 1210/1648] cleanup

---
 msgpack/unpack_template.h | 2 +-
 setup.py                  | 1 -
 test/test_sequnpack.py    | 3 +--
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 7646896..d34eced 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -322,7 +322,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
             case CS_MAP_32:
                 /* FIXME security guard */
                 start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
-            
+
             default:
                 goto _failed;
             }
diff --git a/setup.py b/setup.py
index 83ae79f..1055a61 100644
--- a/setup.py
+++ b/setup.py
@@ -92,7 +92,6 @@ if not hasattr(sys, 'pypy_version_info'):
                                  libraries=libraries,
                                  include_dirs=['.'],
                                  define_macros=macros,
-                                 extra_compile_args=['-O0'],
                                  ))
 del libraries, macros
 
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index af66b78..f541207 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,10 +1,9 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-import py
 import six
 from msgpack import Unpacker, BufferFull
-from msgpack.exceptions import OutOfData, ExtraData, UnpackValueError
+from msgpack.exceptions import OutOfData
 from pytest import raises
 
 

From 84dc99c894be82b7a8c3708a3554888a6133b33b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 20 Oct 2013 23:27:32 +0900
Subject: [PATCH 1211/1648] Add ext_type example to README.

---
 README.rst | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index 600d7f7..99fb923 100644
--- a/README.rst
+++ b/README.rst
@@ -143,10 +143,27 @@ key-value pairs.
 Extended types
 ^^^^^^^^^^^^^^^
 
-It is also possible to pack/unpack custom data types using the msgpack feature
-of "extended types". For example, msgpack-pypy uses it to provide very fast serialization of int/float lists on top of PyPy (experimental for now):
+It is also possible to pack/unpack custom data types using the msgpack 2.0 feature.
 
-https://bitbucket.org/antocuni/msgpack-pypy/src/default/msgpack_pypy.py
+    >>> import msgpack
+    >>> import array
+    >>> def default(obj):
+    ...     if isinstance(obj, array.array) and obj.typecode == 'd':
+    ...         return msgpack.ExtType(42, obj.tostring())
+    ...     raise TypeError("Unknown type: %r" % (obj,))
+    ...
+    >>> def ext_hook(code, data):
+    ...     if code == 42:
+    ...         a = array.array('d')
+    ...         a.fromstring(data)
+    ...         return a
+    ...     return ExtType(code, data)
+    ...
+    >>> data = array.array('d', [1.2, 3.4])
+    >>> packed = msgpack.packb(data, default=default)
+    >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook)
+    >>> data == unpacked
+    True
 
 
 Advanced unpacking control

From cb789596787592f4ec6bf7dcc0c646e8976b3f16 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 21 Oct 2013 00:01:47 +0900
Subject: [PATCH 1212/1648] Update README.

---
 README.rst | 38 +++++++++++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index 99fb923..c51e518 100644
--- a/README.rst
+++ b/README.rst
@@ -3,8 +3,8 @@ MessagePack for Python
 =======================
 
 :author: INADA Naoki
-:version: 0.3.0
-:date: 2012-12-07
+:version: 0.4.0
+:date: 2013-10-21
 
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
    :target: https://travis-ci.org/#!/msgpack/msgpack-python
@@ -39,8 +39,40 @@ amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.)
 
 Without extension, using pure python implementation on CPython runs slowly.
 
+Notes
+-----
+
+Note for msgpack 2.0 support
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+msgpack 2.0 adds two types: *bin* and *ext*.
+
+*raw* was bytes or string type like Python 2's ``str``.
+To distinguish string and bytes, msgpack 2.0 adds *bin*.
+It is non-string binary like Python 3's ``bytes``.
+
+To use *bin* type for packing ``bytes``, pass ``use_bin_type=True`` to
+packer argument.
+
+    >>> import msgpack
+    >>> packed = msgpack.packb([b'spam', u'egg'], use_bin_type=True)
+    >>> msgpack.unpackb(packed, encoding='utf-8')
+    ['spam', u'egg']
+
+You shoud use it carefully. When you use ``use_bin_type=True``, packed
+binary can be unpacked by unpackers supporting msgpack-2.0.
+
+To use *ext* type, pass ``msgpack.ExtType`` object to packer.
+
+    >>> import msgpack
+    >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy'))
+    >>> msgpack.unpackb(packed)
+    ExtType(code=42, data='xyzzy')
+
+You can use it with ``default`` and ``ext_hook``. See below.
+
 Note for msgpack 0.2.x users
-----------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The msgpack 0.3 have some incompatible changes.
 

From 37c2ad63af8a6e5cb6944f80d931fedbc6b49e7d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 21 Oct 2013 00:29:05 +0900
Subject: [PATCH 1213/1648] Add tests and bugfix.

---
 msgpack/pack_template.h |  3 ++-
 msgpack/unpack.h        |  4 ++--
 test/test_newspec.py    | 23 +++++++++++++++++++++--
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 8b91619..2879bbd 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -705,7 +705,8 @@ static inline int msgpack_pack_bin(msgpack_packer *x, size_t l)
 
 static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t l)
 {
-    msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
+    if (l > 0) msgpack_pack_append_buffer(x, (const unsigned char*)b, l);
+    return 0;
 }
 
 /*
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index c733b24..aced40b 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -157,7 +157,7 @@ static inline int unpack_callback_array_item(unpack_user* u, unsigned int curren
 static inline int unpack_callback_array_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->list_hook) {
-        PyObject *new_c = PyObject_CallFunction(u->list_hook, "(O)", *c);
+        PyObject *new_c = PyObject_CallFunctionObjArgs(u->list_hook, *c, NULL);
         if (!new_c)
             return -1;
         Py_DECREF(*c);
@@ -203,7 +203,7 @@ static inline int unpack_callback_map_item(unpack_user* u, unsigned int current,
 static inline int unpack_callback_map_end(unpack_user* u, msgpack_unpack_object* c)
 {
     if (u->object_hook) {
-        PyObject *new_c = PyObject_CallFunction(u->object_hook, "(O)", *c);
+        PyObject *new_c = PyObject_CallFunctionObjArgs(u->object_hook, *c, NULL);
         if (!new_c)
             return -1;
 
diff --git a/test/test_newspec.py b/test/test_newspec.py
index 8bc2cfe..ab05029 100644
--- a/test/test_newspec.py
+++ b/test/test_newspec.py
@@ -1,6 +1,6 @@
 # coding: utf-8
 
-from msgpack import packb, unpackb
+from msgpack import packb, unpackb, ExtType
 
 
 def test_str8():
@@ -66,4 +66,23 @@ def test_bin32():
     assert b[5:] == data
     assert unpackb(b) == data
 
-
+def test_ext():
+    def check(ext, packed):
+        assert packb(ext) == packed
+        assert unpackb(packed) == ext
+    check(ExtType(0x42, b'Z'), b'\xd4\x42Z') # fixext 1
+    check(ExtType(0x42, b'ZZ'), b'\xd5\x42ZZ') # fixext 2
+    check(ExtType(0x42, b'Z'*4), b'\xd6\x42' + b'Z'*4) # fixext 4
+    check(ExtType(0x42, b'Z'*8), b'\xd7\x42' + b'Z'*8) # fixext 8
+    check(ExtType(0x42, b'Z'*16), b'\xd8\x42' + b'Z'*16) # fixext 16
+    # ext 8
+    check(ExtType(0x42, b''), b'\xc7\x00\x42')
+    check(ExtType(0x42, b'Z'*255), b'\xc7\xff\x42' + b'Z'*255)
+    # ext 16
+    check(ExtType(0x42, b'Z'*256), b'\xc8\x01\x00\x42' + b'Z'*256)
+    check(ExtType(0x42, b'Z'*0xffff), b'\xc8\xff\xff\x42' + b'Z'*0xffff)
+    # ext 32
+    check(ExtType(0x42, b'Z'*0x10000), b'\xc9\x00\x01\x00\x00\x42' + b'Z'*0x10000)
+    # needs large memory
+    #check(ExtType(0x42, b'Z'*0xffffffff),
+    #              b'\xc9\xff\xff\xff\xff\x42' + b'Z'*0xffffffff)

From e3fee4db5fbf1ead4a98fff6c8843574480c3c2a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 21 Oct 2013 00:59:22 +0900
Subject: [PATCH 1214/1648] fallback: support packing ExtType

---
 msgpack/__init__.py |  26 ++++++-
 msgpack/_packer.pyx |   5 +-
 msgpack/fallback.py | 181 +++++++++++++++++++++++++-------------------
 3 files changed, 131 insertions(+), 81 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index a7b47b1..56a0b36 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -2,9 +2,31 @@
 from msgpack._version import version
 from msgpack.exceptions import *
 
-from collections import namedtuple
 
-ExtType = namedtuple('ExtType', 'code data')
+class ExtType(object):
+    __slots__ = ('code', 'data')
+
+    def __init__(self, code, data):
+        if not isinstance(code, int):
+            raise TypeError("code must be int")
+        if not isinstance(data, bytes):
+            raise TypeError("data must be bytes")
+        if not 0 <= code <= 127:
+            raise ValueError("code must be 0~127")
+        self.code = code
+        self.data = data
+
+    def __eq__(self, other):
+        if not isinstance(other, ExtType):
+            return NotImplemented
+        return self.code == other.code and self.data == other.data
+
+    def __hash__(self):
+        return self.code ^ hash(self.data)
+
+    def __repr__(self):
+        return "msgpack.ExtType(%r, %r)" % (self.code, self.data)
+
 
 import os
 if os.environ.get('MSGPACK_PUREPYTHON'):
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index f63667c..f2d058e 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -37,7 +37,6 @@ cdef extern from "pack.h":
 cdef int DEFAULT_RECURSE_LIMIT=511
 
 
-
 cdef class Packer(object):
     """
     MessagePack Packer
@@ -185,8 +184,8 @@ cdef class Packer(object):
                         if ret != 0: break
             elif isinstance(o, ExtType):
                 # This should be before Tuple because ExtType is namedtuple.
-                longval = o[0]
-                rawval = o[1]
+                longval = o.code
+                rawval = o.data
                 L = len(o[1])
                 ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 0b29700..bf5b1c2 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -506,82 +506,111 @@ class Packer(object):
         self._default = default
 
     def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance):
-        if nest_limit < 0:
-            raise PackValueError("recursion limit exceeded")
-        if obj is None:
-            return self._buffer.write(b"\xc0")
-        if isinstance(obj, bool):
-            if obj:
-                return self._buffer.write(b"\xc3")
-            return self._buffer.write(b"\xc2")
-        if isinstance(obj, int_types):
-            if 0 <= obj < 0x80:
-                return self._buffer.write(struct.pack("B", obj))
-            if -0x20 <= obj < 0:
-                return self._buffer.write(struct.pack("b", obj))
-            if 0x80 <= obj <= 0xff:
-                return self._buffer.write(struct.pack("BB", 0xcc, obj))
-            if -0x80 <= obj < 0:
-                return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
-            if 0xff < obj <= 0xffff:
-                return self._buffer.write(struct.pack(">BH", 0xcd, obj))
-            if -0x8000 <= obj < -0x80:
-                return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
-            if 0xffff < obj <= 0xffffffff:
-                return self._buffer.write(struct.pack(">BI", 0xce, obj))
-            if -0x80000000 <= obj < -0x8000:
-                return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
-            if 0xffffffff < obj <= 0xffffffffffffffff:
-                return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
-            if -0x8000000000000000 <= obj < -0x80000000:
-                return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
-            raise PackValueError("Integer value out of range")
-        if self._use_bin_type and isinstance(obj, bytes):
-            n = len(obj)
-            if n <= 0xff:
-                self._buffer.write(struct.pack('>BB', 0xc4, n))
-            elif n <= 0xffff:
-                self._buffer.write(struct.pack(">BH", 0xc5, n))
-            elif n <= 0xffffffff:
-                self._buffer.write(struct.pack(">BI", 0xc6, n))
-            else:
-                raise PackValueError("Bytes is too large")
-            return self._buffer.write(obj)
-        if isinstance(obj, (Unicode, bytes)):
-            if isinstance(obj, Unicode):
-                if self._encoding is None:
-                    raise TypeError(
-                        "Can't encode unicode string: "
-                        "no encoding is specified")
-                obj = obj.encode(self._encoding, self._unicode_errors)
-            n = len(obj)
-            if n <= 0x1f:
-                self._buffer.write(struct.pack('B', 0xa0 + n))
-            elif self._use_bin_type and n <= 0xff:
-                self._buffer.write(struct.pack('>BB', 0xd9, n))
-            elif n <= 0xffff:
-                self._buffer.write(struct.pack(">BH", 0xda, n))
-            elif n <= 0xffffffff:
-                self._buffer.write(struct.pack(">BI", 0xdb, n))
-            else:
-                raise PackValueError("String is too large")
-            return self._buffer.write(obj)
-        if isinstance(obj, float):
-            if self._use_float:
-                return self._buffer.write(struct.pack(">Bf", 0xca, obj))
-            return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
-        if isinstance(obj, (list, tuple)):
-            n = len(obj)
-            self._fb_pack_array_header(n)
-            for i in xrange(n):
-                self._pack(obj[i], nest_limit - 1)
-            return
-        if isinstance(obj, dict):
-            return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj),
-                                           nest_limit - 1)
-        if self._default is not None:
-            return self._pack(self._default(obj), nest_limit - 1)
-        raise TypeError("Cannot serialize %r" % obj)
+        default_used = False
+        while True:
+            if nest_limit < 0:
+                raise PackValueError("recursion limit exceeded")
+            if obj is None:
+                return self._buffer.write(b"\xc0")
+            if isinstance(obj, bool):
+                if obj:
+                    return self._buffer.write(b"\xc3")
+                return self._buffer.write(b"\xc2")
+            if isinstance(obj, int_types):
+                if 0 <= obj < 0x80:
+                    return self._buffer.write(struct.pack("B", obj))
+                if -0x20 <= obj < 0:
+                    return self._buffer.write(struct.pack("b", obj))
+                if 0x80 <= obj <= 0xff:
+                    return self._buffer.write(struct.pack("BB", 0xcc, obj))
+                if -0x80 <= obj < 0:
+                    return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
+                if 0xff < obj <= 0xffff:
+                    return self._buffer.write(struct.pack(">BH", 0xcd, obj))
+                if -0x8000 <= obj < -0x80:
+                    return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
+                if 0xffff < obj <= 0xffffffff:
+                    return self._buffer.write(struct.pack(">BI", 0xce, obj))
+                if -0x80000000 <= obj < -0x8000:
+                    return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
+                if 0xffffffff < obj <= 0xffffffffffffffff:
+                    return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
+                if -0x8000000000000000 <= obj < -0x80000000:
+                    return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
+                raise PackValueError("Integer value out of range")
+            if self._use_bin_type and isinstance(obj, bytes):
+                n = len(obj)
+                if n <= 0xff:
+                    self._buffer.write(struct.pack('>BB', 0xc4, n))
+                elif n <= 0xffff:
+                    self._buffer.write(struct.pack(">BH", 0xc5, n))
+                elif n <= 0xffffffff:
+                    self._buffer.write(struct.pack(">BI", 0xc6, n))
+                else:
+                    raise PackValueError("Bytes is too large")
+                return self._buffer.write(obj)
+            if isinstance(obj, (Unicode, bytes)):
+                if isinstance(obj, Unicode):
+                    if self._encoding is None:
+                        raise TypeError(
+                            "Can't encode unicode string: "
+                            "no encoding is specified")
+                    obj = obj.encode(self._encoding, self._unicode_errors)
+                n = len(obj)
+                if n <= 0x1f:
+                    self._buffer.write(struct.pack('B', 0xa0 + n))
+                elif self._use_bin_type and n <= 0xff:
+                    self._buffer.write(struct.pack('>BB', 0xd9, n))
+                elif n <= 0xffff:
+                    self._buffer.write(struct.pack(">BH", 0xda, n))
+                elif n <= 0xffffffff:
+                    self._buffer.write(struct.pack(">BI", 0xdb, n))
+                else:
+                    raise PackValueError("String is too large")
+                return self._buffer.write(obj)
+            if isinstance(obj, float):
+                if self._use_float:
+                    return self._buffer.write(struct.pack(">Bf", 0xca, obj))
+                return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
+            if isinstance(obj, ExtType):
+                code = obj.code
+                data = obj.data
+                assert isinstance(code, int)
+                assert isinstance(data, bytes)
+                L = len(data)
+                if L == 1:
+                    self._buffer.write(b'\xd4')
+                elif L == 2:
+                    self._buffer.write(b'\xd5')
+                elif L == 4:
+                    self._buffer.write(b'\xd6')
+                elif L == 8:
+                    self._buffer.write(b'\xd7')
+                elif L == 16:
+                    self._buffer.write(b'\xd8')
+                elif L <= 0xff:
+                    self._buffer.write(struct.pack(">BB", 0xc7, L))
+                elif L <= 0xffff:
+                    self._buffer.write(struct.pack(">BH", 0xc8, L))
+                else:
+                    self._buffer.write(struct.pack(">BI", 0xc9, L))
+                self._buffer.write(struct.pack("b", code))
+                self._buffer.write(data)
+                return
+            if isinstance(obj, (list, tuple)):
+                n = len(obj)
+                self._fb_pack_array_header(n)
+                for i in xrange(n):
+                    self._pack(obj[i], nest_limit - 1)
+                return
+            if isinstance(obj, dict):
+                return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj),
+                                               nest_limit - 1)
+            if not default_used and self._default is not None:
+                obj = self._default(obj)
+                default_used = 1
+                continue
+            raise TypeError("Cannot serialize %r" % obj)
 
     def pack(self, obj):
         self._pack(obj)

From d84a403bc0bbbb36c4a5833e00269eef6c4a91ae Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 21 Oct 2013 01:12:57 +0900
Subject: [PATCH 1215/1648] fix bugs.

---
 msgpack/__init__.py | 20 ++++----------------
 msgpack/_packer.pyx |  2 +-
 2 files changed, 5 insertions(+), 17 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 56a0b36..a958025 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -2,30 +2,18 @@
 from msgpack._version import version
 from msgpack.exceptions import *
 
+from collections import namedtuple
 
-class ExtType(object):
-    __slots__ = ('code', 'data')
 
-    def __init__(self, code, data):
+class ExtType(namedtuple('ExtType', 'code data')):
+    def __new__(cls, code, data):
         if not isinstance(code, int):
             raise TypeError("code must be int")
         if not isinstance(data, bytes):
             raise TypeError("data must be bytes")
         if not 0 <= code <= 127:
             raise ValueError("code must be 0~127")
-        self.code = code
-        self.data = data
-
-    def __eq__(self, other):
-        if not isinstance(other, ExtType):
-            return NotImplemented
-        return self.code == other.code and self.data == other.data
-
-    def __hash__(self):
-        return self.code ^ hash(self.data)
-
-    def __repr__(self):
-        return "msgpack.ExtType(%r, %r)" % (self.code, self.data)
+        return super(ExtType, cls).__new__(cls, code, data)
 
 
 import os
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index f2d058e..f261f08 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -186,7 +186,7 @@ cdef class Packer(object):
                 # This should be before Tuple because ExtType is namedtuple.
                 longval = o.code
                 rawval = o.data
-                L = len(o[1])
+                L = len(o.data)
                 ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyTuple_Check(o) or PyList_Check(o):

From 1d0096b998abca1276b88ae6960316f8cff273b9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 21 Oct 2013 01:20:13 +0900
Subject: [PATCH 1216/1648] 0.4.0

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index 5999ede..e7a935c 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 3, 0)
+version = (0, 4, 0)

From f31a4403a1715ea5b48a12e1aa58df0e74b7e85f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 21 Oct 2013 01:47:54 +0900
Subject: [PATCH 1217/1648] Document update.

---
 Makefile            | 3 +++
 docs/api.rst        | 2 ++
 docs/conf.py        | 2 +-
 msgpack/__init__.py | 1 +
 4 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 71be6af..2e53d08 100644
--- a/Makefile
+++ b/Makefile
@@ -9,6 +9,9 @@ doc-serve: all
 doc:
 	cd docs && make zip
 
+upload-doc:
+	python setup.py upload_docs --upload-dir docs/_build/html
+
 cython:
 	cython --cplus msgpack/*.pyx
 
diff --git a/docs/api.rst b/docs/api.rst
index 50a84c4..841c134 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -25,6 +25,8 @@ API reference
 .. autoclass:: Unpacker
     :members:
 
+.. autoclass:: ExtType
+
 exceptions
 -----------
 
diff --git a/docs/conf.py b/docs/conf.py
index fba09b7..0f19fcc 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -49,7 +49,7 @@ copyright = u'2013, INADA Naoki'
 #
 # The short X.Y version.
 # The full version, including alpha/beta/rc tags.
-version = release = '0.3'
+version = release = '0.4'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index a958025..6c5ae53 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -6,6 +6,7 @@ from collections import namedtuple
 
 
 class ExtType(namedtuple('ExtType', 'code data')):
+    """ExtType represents ext type in msgpack."""
     def __new__(cls, code, data):
         if not isinstance(code, int):
             raise TypeError("code must be int")

From d5436c28197f3ccdd473e1c54838133574948dab Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 21 Oct 2013 02:33:58 +0900
Subject: [PATCH 1218/1648] Update ChangeLog

---
 ChangeLog.rst | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index fe64ff8..c5cff15 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,20 @@
+0.4.0
+=====
+:release date: 2013-10-21
+
+Inconpatible Changes
+--------------------
+
+* Raises TypeError instead of ValueError when packer receives unsupported type.
+
+Changes
+-------
+
+* Support New msgpack spec.
+
+
 0.3.0
 =====
-:release date: in development
 
 Inconpatible Changes
 --------------------

From 48ca2d700d019bfec9abe3242a8eb3073c803951 Mon Sep 17 00:00:00 2001
From: Sergey Zhuravlev 
Date: Sun, 15 Dec 2013 16:22:39 +0000
Subject: [PATCH 1219/1648] Added support of bytearrays to msgpack/fallback.py

---
 msgpack/fallback.py | 2 ++
 test/test_buffer.py | 7 +++++++
 2 files changed, 9 insertions(+)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index bf5b1c2..3ef1341 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -193,6 +193,8 @@ class Unpacker(object):
     def feed(self, next_bytes):
         if isinstance(next_bytes, array.array):
             next_bytes = next_bytes.tostring()
+        elif isinstance(next_bytes, bytearray):
+            next_bytes = str(next_bytes)
         assert self._fb_feeding
         if self._fb_buf_n + len(next_bytes) > self._max_buffer_size:
             raise BufferFull
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 04cc02d..5ae87ac 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -11,3 +11,10 @@ def test_unpack_buffer():
     obj = unpackb(buf, use_list=1)
     assert [b'foo', b'bar'] == obj
 
+
+def test_unpack_bytearray():
+    buf = bytearray(packb(('foo', 'bar')))
+    obj = unpackb(buf, use_list=1)
+    assert [b'foo', b'bar'] == obj
+    assert all(type(s)==str for s in obj)
+

From 77046b839da20feb528d303c8858ab6f34013ce7 Mon Sep 17 00:00:00 2001
From: Wouter Bolsterlee 
Date: Tue, 11 Feb 2014 20:34:23 +0100
Subject: [PATCH 1220/1648] Always raise TypeError for wrong argument types

The code that checks whether hooks are callable() (and some other type
checks) should always raise TypeError on failure. Before this change,
both ValueError and TypeError were used in an inconsistent way (C
extension and Python implementation were not the same).
---
 msgpack/_unpacker.pyx |  4 ++--
 msgpack/fallback.py   | 14 +++++++-------
 test/test_obj.py      |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index d5aa46e..18592f4 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -52,7 +52,7 @@ cdef inline init_ctx(unpack_context *ctx,
     ctx.user.object_hook = ctx.user.list_hook = NULL
 
     if object_hook is not None and object_pairs_hook is not None:
-        raise ValueError("object_pairs_hook and object_hook are mutually exclusive.")
+        raise TypeError("object_pairs_hook and object_hook are mutually exclusive.")
 
     if object_hook is not None:
         if not PyCallable_Check(object_hook):
@@ -227,7 +227,7 @@ cdef class Unpacker(object):
         if file_like:
             self.file_like_read = file_like.read
             if not PyCallable_Check(self.file_like_read):
-                raise ValueError("`file_like.read` must be a callable.")
+                raise TypeError("`file_like.read` must be a callable.")
         if not max_buffer_size:
             max_buffer_size = INT_MAX
         if read_size > max_buffer_size:
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index bf5b1c2..cf61023 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -159,7 +159,7 @@ class Unpacker(object):
             self._fb_feeding = True
         else:
             if not callable(file_like.read):
-                raise ValueError("`file_like.read` must be callable")
+                raise TypeError("`file_like.read` must be callable")
             self.file_like = file_like
             self._fb_feeding = False
         self._fb_buffers = []
@@ -179,16 +179,16 @@ class Unpacker(object):
         self._ext_hook = ext_hook
 
         if list_hook is not None and not callable(list_hook):
-            raise ValueError('`list_hook` is not callable')
+            raise TypeError('`list_hook` is not callable')
         if object_hook is not None and not callable(object_hook):
-            raise ValueError('`object_hook` is not callable')
+            raise TypeError('`object_hook` is not callable')
         if object_pairs_hook is not None and not callable(object_pairs_hook):
-            raise ValueError('`object_pairs_hook` is not callable')
+            raise TypeError('`object_pairs_hook` is not callable')
         if object_hook is not None and object_pairs_hook is not None:
-            raise ValueError("object_pairs_hook and object_hook are mutually "
-                             "exclusive")
+            raise TypeError("object_pairs_hook and object_hook are mutually "
+                            "exclusive")
         if not callable(ext_hook):
-            raise ValueError("`ext_hook` is not callable")
+            raise TypeError("`ext_hook` is not callable")
 
     def feed(self, next_bytes):
         if isinstance(next_bytes, array.array):
diff --git a/test/test_obj.py b/test/test_obj.py
index 9083218..390c1b6 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -31,7 +31,7 @@ def test_decode_pairs_hook():
     assert unpacked[1] == prod_sum
 
 def test_only_one_obj_hook():
-    with raises(ValueError):
+    with raises(TypeError):
         unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x)
 
 def test_bad_hook():

From dd65341e0d3a0df843841796050f98ab1520e623 Mon Sep 17 00:00:00 2001
From: Wouter Bolsterlee 
Date: Tue, 11 Feb 2014 21:05:15 +0100
Subject: [PATCH 1221/1648] Cosmetic changes to fix Cython warnings

Put declarations on separate line to avoid warnings like this:

  Non-trivial type declarators in shared declaration (e.g. mix
  of pointers and values). Each pointer declaration should be
  on its own line.
---
 msgpack/_unpacker.pyx | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index d5aa46e..a359507 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -221,7 +221,8 @@ cdef class Unpacker(object):
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  str encoding=None, str unicode_errors='strict', int max_buffer_size=0,
                  object ext_hook=ExtType):
-        cdef char *cenc=NULL, *cerr=NULL
+        cdef char *cenc=NULL,
+        cdef char *cerr=NULL
 
         self.file_like = file_like
         if file_like:

From d2fc8010342512378e01322f8871c10a5974af4f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 13 Feb 2014 01:57:34 +0900
Subject: [PATCH 1222/1648] Fix warning on 64bit environment.

---
 msgpack/unpack.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index aced40b..27e3b62 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -56,9 +56,12 @@ static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpac
 static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
 {
     PyObject *p;
+#if UINT32_MAX > LONG_MAX
     if (d > LONG_MAX) {
         p = PyLong_FromUnsignedLong((unsigned long)d);
-    } else {
+    } else
+#endif
+    {
         p = PyInt_FromLong((long)d);
     }
     if (!p)

From 9d61f243878eeabd2042bb16fe22d4325e441da6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 13 Feb 2014 03:10:51 +0900
Subject: [PATCH 1223/1648] Feed data from file before _unpack()

---
 msgpack/_unpacker.pyx    | 10 ++++++++++
 test/test_unpack_file.py | 19 +++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 test/test_unpack_file.py

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 732adef..16aca5c 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -327,8 +327,18 @@ cdef class Unpacker(object):
         cdef int ret
         cdef object obj
         cdef size_t prev_head
+
+        if self.buf_head >= self.buf_tail and self.file_like is not None:
+            self.read_from_file()
+
         while 1:
             prev_head = self.buf_head
+            if prev_head >= self.buf_tail:
+                if iter:
+                    raise StopIteration("No more data to unpack.")
+                else:
+                    raise OutOfData("No more data to unpack.")
+
             ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
             if write_bytes is not None:
                 write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
diff --git a/test/test_unpack_file.py b/test/test_unpack_file.py
new file mode 100644
index 0000000..1563008
--- /dev/null
+++ b/test/test_unpack_file.py
@@ -0,0 +1,19 @@
+from io import BytesIO
+from msgpack import Unpacker, packb, OutOfData
+from pytest import raises
+
+
+def test_unpack_array_header_from_file():
+    f = BytesIO(packb([1,2,3,4]))
+    unpacker = Unpacker(f)
+    assert unpacker.read_array_header() == 4
+    assert unpacker.unpack() == 1
+    assert unpacker.unpack() == 2
+    assert unpacker.unpack() == 3
+    assert unpacker.unpack() == 4
+    with raises(OutOfData):
+        unpacker.unpack()
+
+
+if __name__ == '__main__':
+    test_unpack_array_header_from_file()

From 11a3b1561a07b28a37ac22637f636c4c5ac24c23 Mon Sep 17 00:00:00 2001
From: Sergey Zhuravlev 
Date: Wed, 12 Feb 2014 23:09:23 +0400
Subject: [PATCH 1224/1648] fixed support of python3

---
 msgpack/fallback.py | 2 +-
 test/test_buffer.py | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 3ef1341..b673222 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -194,7 +194,7 @@ class Unpacker(object):
         if isinstance(next_bytes, array.array):
             next_bytes = next_bytes.tostring()
         elif isinstance(next_bytes, bytearray):
-            next_bytes = str(next_bytes)
+            next_bytes = (bytes if PY3 else str)(next_bytes)
         assert self._fb_feeding
         if self._fb_buf_n + len(next_bytes) > self._max_buffer_size:
             raise BufferFull
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 5ae87ac..7b85e93 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -2,6 +2,7 @@
 # coding: utf-8
 
 from msgpack import packb, unpackb
+import sys
 
 
 def test_unpack_buffer():
@@ -16,5 +17,7 @@ def test_unpack_bytearray():
     buf = bytearray(packb(('foo', 'bar')))
     obj = unpackb(buf, use_list=1)
     assert [b'foo', b'bar'] == obj
-    assert all(type(s)==str for s in obj)
+    expected_type = bytes if sys.version_info[0] == 3 else str
+    assert all(type(s)==expected_type for s in obj)
+
 

From 38cf835c95273dc20dd6cb3376c11a88b72c7fa9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 13 Feb 2014 09:40:12 +0900
Subject: [PATCH 1225/1648] Rename

---
 test/test_unpack.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 test/test_unpack.py

diff --git a/test/test_unpack.py b/test/test_unpack.py
new file mode 100644
index 0000000..1563008
--- /dev/null
+++ b/test/test_unpack.py
@@ -0,0 +1,19 @@
+from io import BytesIO
+from msgpack import Unpacker, packb, OutOfData
+from pytest import raises
+
+
+def test_unpack_array_header_from_file():
+    f = BytesIO(packb([1,2,3,4]))
+    unpacker = Unpacker(f)
+    assert unpacker.read_array_header() == 4
+    assert unpacker.unpack() == 1
+    assert unpacker.unpack() == 2
+    assert unpacker.unpack() == 3
+    assert unpacker.unpack() == 4
+    with raises(OutOfData):
+        unpacker.unpack()
+
+
+if __name__ == '__main__':
+    test_unpack_array_header_from_file()

From 0cab6092e456ffa04834233ffb01acb48d0869c3 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 13 Feb 2014 09:55:17 +0900
Subject: [PATCH 1226/1648] Add refcount check.

---
 test/test_unpack.py      | 30 +++++++++++++++++++++++++++++-
 test/test_unpack_file.py | 19 -------------------
 2 files changed, 29 insertions(+), 20 deletions(-)
 delete mode 100644 test/test_unpack_file.py

diff --git a/test/test_unpack.py b/test/test_unpack.py
index 1563008..d6ca435 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -1,6 +1,7 @@
 from io import BytesIO
+import sys
 from msgpack import Unpacker, packb, OutOfData
-from pytest import raises
+from pytest import raises, mark
 
 
 def test_unpack_array_header_from_file():
@@ -15,5 +16,32 @@ def test_unpack_array_header_from_file():
         unpacker.unpack()
 
 
+@mark.skipif(not hasattr(sys, 'getrefcount'),
+             reason='sys.getrefcount() is needed to pass this test')
+def test_unpacker_hook_refcnt():
+    result = []
+
+    def hook(x):
+        result.append(x)
+        return x
+
+    basecnt = sys.getrefcount(hook)
+
+    up = Unpacker(object_pairs_hook=hook, list_hook=hook)
+
+    assert sys.getrefcount(hook) >= basecnt + 2
+
+    up.feed(packb([{}]))
+    up.feed(packb([{}]))
+    assert up.unpack() == [{}]
+    assert up.unpack() == [{}]
+    assert result == [[{}], [{}]]
+
+    del up
+
+    assert sys.getrefcount(hook) == basecnt
+
+
 if __name__ == '__main__':
     test_unpack_array_header_from_file()
+    test_unpacker_hook_refcnt()
diff --git a/test/test_unpack_file.py b/test/test_unpack_file.py
deleted file mode 100644
index 1563008..0000000
--- a/test/test_unpack_file.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from io import BytesIO
-from msgpack import Unpacker, packb, OutOfData
-from pytest import raises
-
-
-def test_unpack_array_header_from_file():
-    f = BytesIO(packb([1,2,3,4]))
-    unpacker = Unpacker(f)
-    assert unpacker.read_array_header() == 4
-    assert unpacker.unpack() == 1
-    assert unpacker.unpack() == 2
-    assert unpacker.unpack() == 3
-    assert unpacker.unpack() == 4
-    with raises(OutOfData):
-        unpacker.unpack()
-
-
-if __name__ == '__main__':
-    test_unpack_array_header_from_file()

From cf63f19211797261b117227f23066952efebca29 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 13 Feb 2014 09:57:51 +0900
Subject: [PATCH 1227/1648] Fix test

---
 test/test_unpack.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_unpack.py b/test/test_unpack.py
index d6ca435..544cebf 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -27,7 +27,7 @@ def test_unpacker_hook_refcnt():
 
     basecnt = sys.getrefcount(hook)
 
-    up = Unpacker(object_pairs_hook=hook, list_hook=hook)
+    up = Unpacker(object_hook=hook, list_hook=hook)
 
     assert sys.getrefcount(hook) >= basecnt + 2
 
@@ -35,7 +35,7 @@ def test_unpacker_hook_refcnt():
     up.feed(packb([{}]))
     assert up.unpack() == [{}]
     assert up.unpack() == [{}]
-    assert result == [[{}], [{}]]
+    assert result == [{}, [{}], {}, [{}]]
 
     del up
 

From 6d80569b9b7ec6f3756ecc91928e4ce127eb7a4b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 13 Feb 2014 09:58:38 +0900
Subject: [PATCH 1228/1648] Unpacker: maintain refcnt (fix #67).

---
 msgpack/_unpacker.pyx | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 16aca5c..0df6ab3 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -206,7 +206,8 @@ cdef class Unpacker(object):
     cdef object file_like
     cdef object file_like_read
     cdef Py_ssize_t read_size
-    cdef object object_hook
+    # To maintain refcnt.
+    cdef object object_hook, object_pairs_hook, list_hook
     cdef object encoding, unicode_errors
     cdef size_t max_buffer_size
 
@@ -224,6 +225,10 @@ cdef class Unpacker(object):
         cdef char *cenc=NULL,
         cdef char *cerr=NULL
 
+        self.object_hook = object_hook
+        self.object_pairs_hook = object_pairs_hook
+        self.list_hook = list_hook
+
         self.file_like = file_like
         if file_like:
             self.file_like_read = file_like.read

From a5368f62e24dd335bc7bb72980953ca707532693 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 14 Feb 2014 09:01:00 +0900
Subject: [PATCH 1229/1648] travis: Simplify .travis.yml and add PyPy env.

---
 .travis.yml | 10 ++--------
 tox.ini     |  2 +-
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index e536fdc..7cf7f58 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,14 +2,8 @@ language: python
 python:
     - 2.7
 
-env:
-    - PIP_USE_MIRRORS=true
-
 install:
-    - sudo apt-get update -qq
-    - sudo apt-get install -q python3.3-dev
-    - pip install --use-mirrors tox cython
-    - cython --cplus msgpack/_packer.pyx
-    - cython --cplus msgpack/_unpacker.pyx
+    - pip install tox cython
+    - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
 
 script: "tox && MSGPACK_PUREPYTHON=x tox"
diff --git a/tox.ini b/tox.ini
index 1951352..2c5c596 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py26,py27,py32,py33
+envlist = py26,py27,py32,py33,pypy
 
 [testenv]
 deps=

From 213f7888c3edbe828533ec08e7a07bac3a98b482 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 14 Feb 2014 09:26:46 +0900
Subject: [PATCH 1230/1648] Remove too strict type check from test.

---
 test/test_pack.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_pack.py b/test/test_pack.py
index 9dc5ada..0d3ed6d 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -12,7 +12,6 @@ from io import BytesIO
 def check(data, use_list=False):
     re = unpackb(packb(data), use_list=use_list)
     assert re == data
-    assert type(re) == type(data)
 
 def testPack():
     test_data = [

From dee2d87d413eaf9cb38f11c72df10e82a73f452f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 15 Feb 2014 22:16:01 +0900
Subject: [PATCH 1231/1648] six.BytesIO => io.BytesIO

---
 test/test_pack.py       |  6 +++---
 test/test_sequnpack.py  |  4 ++--
 test/test_unpack_raw.py | 12 ++++++------
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/test/test_pack.py b/test/test_pack.py
index 0d3ed6d..2da7089 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -89,7 +89,7 @@ def testPackFloat():
     assert packb(1.0, use_single_float=False) == b'\xcb' + struct.pack('>d', 1.0)
 
 def testArraySize(sizes=[0, 5, 50, 1000]):
-    bio = six.BytesIO()
+    bio = BytesIO()
     packer = Packer()
     for size in sizes:
         bio.write(packer.pack_array_header(size))
@@ -108,7 +108,7 @@ def test_manualreset(sizes=[0, 5, 50, 1000]):
         for i in range(size):
             packer.pack(i)
 
-    bio = six.BytesIO(packer.bytes())
+    bio = BytesIO(packer.bytes())
     unpacker = Unpacker(bio, use_list=1)
     for size in sizes:
         assert unpacker.unpack() == list(range(size))
@@ -117,7 +117,7 @@ def test_manualreset(sizes=[0, 5, 50, 1000]):
     assert packer.bytes() == b''
 
 def testMapSize(sizes=[0, 5, 50, 1000]):
-    bio = six.BytesIO()
+    bio = BytesIO()
     packer = Packer()
     for size in sizes:
         bio.write(packer.pack_map_header(size))
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index f541207..5d37698 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-import six
+import io
 from msgpack import Unpacker, BufferFull
 from msgpack.exceptions import OutOfData
 from pytest import raises
@@ -79,7 +79,7 @@ def test_readbytes():
     assert unpacker.unpack() == ord(b'r')
 
     # Test buffer refill
-    unpacker = Unpacker(six.BytesIO(b'foobar'), read_size=3)
+    unpacker = Unpacker(io.BytesIO(b'foobar'), read_size=3)
     assert unpacker.unpack() == ord(b'f')
     assert unpacker.read_bytes(3) == b'oob'
     assert unpacker.unpack() == ord(b'a')
diff --git a/test/test_unpack_raw.py b/test/test_unpack_raw.py
index 9f3784c..7002601 100644
--- a/test/test_unpack_raw.py
+++ b/test/test_unpack_raw.py
@@ -1,19 +1,19 @@
 """Tests for cases where the user seeks to obtain packed msgpack objects"""
 
-import six
+import io
 from msgpack import Unpacker, packb
 
 
 def test_write_bytes():
     unpacker = Unpacker()
     unpacker.feed(b'abc')
-    f = six.BytesIO()
+    f = io.BytesIO()
     assert unpacker.unpack(f.write) == ord('a')
     assert f.getvalue() == b'a'
-    f = six.BytesIO()
+    f = io.BytesIO()
     assert unpacker.skip(f.write) is None
     assert f.getvalue() == b'b'
-    f = six.BytesIO()
+    f = io.BytesIO()
     assert unpacker.skip() is None
     assert f.getvalue() == b''
 
@@ -21,9 +21,9 @@ def test_write_bytes():
 def test_write_bytes_multi_buffer():
     long_val = (5) * 100
     expected = packb(long_val)
-    unpacker = Unpacker(six.BytesIO(expected), read_size=3, max_buffer_size=3)
+    unpacker = Unpacker(io.BytesIO(expected), read_size=3, max_buffer_size=3)
 
-    f = six.BytesIO()
+    f = io.BytesIO()
     unpacked = unpacker.unpack(f.write)
     assert unpacked == long_val
     assert f.getvalue() == expected

From 63eab502dff62bf7adbc25c75e8bc5bffa6ee234 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 15 Feb 2014 22:20:57 +0900
Subject: [PATCH 1232/1648] Remove six.b()

---
 test/test_seq.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/test/test_seq.py b/test/test_seq.py
index af719b0..fed9ff4 100644
--- a/test/test_seq.py
+++ b/test/test_seq.py
@@ -1,16 +1,15 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-import six
 import io
 import msgpack
 
-binarydata = [chr(i) for i in range(256)]
-binarydata = six.b("".join(binarydata))
+
+binarydata = bytes(bytearray(range(256)))
 
 def gen_binary_data(idx):
-    data = binarydata[:idx % 300]
-    return data
+    return binarydata[:idx % 300]
+
 
 def test_exceeding_unpacker_read_size():
     dumpf = io.BytesIO()

From 0c22e775c9ea71242eec237e2e032aca749e909c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 15 Feb 2014 22:36:52 +0900
Subject: [PATCH 1233/1648] Remove six completely.

---
 test/test_pack.py | 30 ++++++++++++++----------------
 tox.ini           |  1 -
 2 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/test/test_pack.py b/test/test_pack.py
index 2da7089..762ccf5 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # coding: utf-8
+from __future__ import absolute_import, division, print_function, unicode_literals
 
-import six
 import struct
 from pytest import raises, xfail
 
@@ -28,24 +28,22 @@ def testPack():
         check(td)
 
 def testPackUnicode():
-    test_data = [
-        six.u(""), six.u("abcd"), [six.u("defgh")], six.u("РуÑÑкий текÑÑ‚"),
-        ]
+    test_data = ["", "abcd", ["defgh"], "РуÑÑкий текÑÑ‚"]
     for td in test_data:
         re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8')
         assert re == td
         packer = Packer(encoding='utf-8')
         data = packer.pack(td)
-        re = Unpacker(BytesIO(data), encoding='utf-8', use_list=1).unpack()
+        re = Unpacker(BytesIO(data), encoding=str('utf-8'), use_list=1).unpack()
         assert re == td
 
 def testPackUTF32():
     try:
         test_data = [
-            six.u(""),
-            six.u("abcd"),
-            [six.u("defgh")],
-            six.u("РуÑÑкий текÑÑ‚"),
+            "",
+            "abcd",
+            ["defgh"],
+            "РуÑÑкий текÑÑ‚",
             ]
         for td in test_data:
             re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
@@ -70,23 +68,23 @@ def testStrictUnicodeUnpack():
 
 def testStrictUnicodePack():
     with raises(UnicodeEncodeError):
-        packb(six.u("abc\xeddef"), encoding='ascii', unicode_errors='strict')
+        packb("abc\xeddef", encoding='ascii', unicode_errors='strict')
 
 def testIgnoreErrorsPack():
-    re = unpackb(packb(six.u("abcФФФdef"), encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1)
-    assert re == six.u("abcdef")
+    re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1)
+    assert re == "abcdef"
 
 def testNoEncoding():
     with raises(TypeError):
-        packb(six.u("abc"), encoding=None)
+        packb("abc", encoding=None)
 
 def testDecodeBinary():
-    re = unpackb(packb("abc"), encoding=None, use_list=1)
+    re = unpackb(packb(b"abc"), encoding=None, use_list=1)
     assert re == b"abc"
 
 def testPackFloat():
-    assert packb(1.0, use_single_float=True)  == b'\xca' + struct.pack('>f', 1.0)
-    assert packb(1.0, use_single_float=False) == b'\xcb' + struct.pack('>d', 1.0)
+    assert packb(1.0, use_single_float=True)  == b'\xca' + struct.pack(str('>f'), 1.0)
+    assert packb(1.0, use_single_float=False) == b'\xcb' + struct.pack(str('>d'), 1.0)
 
 def testArraySize(sizes=[0, 5, 50, 1000]):
     bio = BytesIO()
diff --git a/tox.ini b/tox.ini
index 2c5c596..2413630 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,6 +4,5 @@ envlist = py26,py27,py32,py33,pypy
 [testenv]
 deps=
     pytest
-    six
 
 commands=py.test test

From 7effb4aac64f1c44241e0a88ab19fa6c8405c3fa Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 17 Feb 2014 04:05:04 +0900
Subject: [PATCH 1234/1648] fix

---
 msgpack/fallback.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index b673222..0c5b50f 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -194,7 +194,7 @@ class Unpacker(object):
         if isinstance(next_bytes, array.array):
             next_bytes = next_bytes.tostring()
         elif isinstance(next_bytes, bytearray):
-            next_bytes = (bytes if PY3 else str)(next_bytes)
+            next_bytes = bytes(next_bytes)
         assert self._fb_feeding
         if self._fb_buf_n + len(next_bytes) > self._max_buffer_size:
             raise BufferFull

From 518f886b111808d7d9f29b34c50baddb01610338 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 17 Feb 2014 04:06:58 +0900
Subject: [PATCH 1235/1648] fix

---
 test/test_buffer.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/test/test_buffer.py b/test/test_buffer.py
index 7b85e93..6cb2295 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -8,7 +8,7 @@ import sys
 def test_unpack_buffer():
     from array import array
     buf = array('b')
-    buf.fromstring(packb(('foo', 'bar')))
+    buf.fromstring(packb((b'foo', b'bar')))
     obj = unpackb(buf, use_list=1)
     assert [b'foo', b'bar'] == obj
 
@@ -17,7 +17,5 @@ def test_unpack_bytearray():
     buf = bytearray(packb(('foo', 'bar')))
     obj = unpackb(buf, use_list=1)
     assert [b'foo', b'bar'] == obj
-    expected_type = bytes if sys.version_info[0] == 3 else str
-    assert all(type(s)==expected_type for s in obj)
-
-
+    expected_type = bytes
+    assert all(type(s) == expected_type for s in obj)

From c567cf478b63baa3226e031bf988b6db2bf6af11 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 17 Feb 2014 09:54:55 +0900
Subject: [PATCH 1236/1648] Remove unused import.

---
 test/test_buffer.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_buffer.py b/test/test_buffer.py
index 6cb2295..5a71f90 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -2,7 +2,6 @@
 # coding: utf-8
 
 from msgpack import packb, unpackb
-import sys
 
 
 def test_unpack_buffer():

From 1ca3c27a81eeb5b97fcd817cf4465923eeceea93 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 17 Feb 2014 10:03:36 +0900
Subject: [PATCH 1237/1648] 0.4.1

---
 ChangeLog.rst       | 18 ++++++++++++++++++
 msgpack/_version.py |  2 +-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index c5cff15..080f5c8 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,21 @@
+0.4.1
+=====
+:release date: 2014-02-17
+
+Inconpatible Changes
+--------------------
+
+Changes
+-------
+
+* fallback.Unpacker.feed() supports bytearray.
+
+Bugs fixed
+----------
+
+* Unpacker doesn't increment refcount of hooks. Hooks may be GCed while unpacking.
+* Unpacker may read unfilled internal buffer.
+
 0.4.0
 =====
 :release date: 2013-10-21
diff --git a/msgpack/_version.py b/msgpack/_version.py
index e7a935c..c6ebcf8 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 4, 0)
+version = (0, 4, 1)

From 55eab8b4d64ac50f7a448e64bbc0cb092c0ed18d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 17 Feb 2014 10:06:39 +0900
Subject: [PATCH 1238/1648] Update README

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index c51e518..f99ed48 100644
--- a/README.rst
+++ b/README.rst
@@ -3,8 +3,8 @@ MessagePack for Python
 =======================
 
 :author: INADA Naoki
-:version: 0.4.0
-:date: 2013-10-21
+:version: 0.4.1
+:date: 2014-02-17
 
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
    :target: https://travis-ci.org/#!/msgpack/msgpack-python

From e9de6b7f391a4b5e692af37505c959eaf02943e0 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 19 Mar 2014 10:42:05 +0900
Subject: [PATCH 1239/1648] travis: Add Python 3.4 to testing.

---
 .travis.yml | 4 ++++
 tox.ini     | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 7cf7f58..1c1efc3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,10 @@ language: python
 python:
     - 2.7
 
+before_install:
+    - sudo apt-get update -qq -y
+    - sudo apt-get install python3.4
+
 install:
     - pip install tox cython
     - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
diff --git a/tox.ini b/tox.ini
index 2413630..13d6b8d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py26,py27,py32,py33,pypy
+envlist = py26,py27,py32,py33,py34,pypy
 
 [testenv]
 deps=

From d850e56dd0e2d392bb6ead388c0f0a59f68e1bd2 Mon Sep 17 00:00:00 2001
From: Alexey Popravka 
Date: Mon, 24 Mar 2014 15:31:06 +0200
Subject: [PATCH 1240/1648] Unpacker's ext_hook fixed + tests

---
 msgpack/_unpacker.pyx |  3 ++-
 test/test_unpack.py   | 25 ++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 0df6ab3..16de40f 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -207,7 +207,7 @@ cdef class Unpacker(object):
     cdef object file_like_read
     cdef Py_ssize_t read_size
     # To maintain refcnt.
-    cdef object object_hook, object_pairs_hook, list_hook
+    cdef object object_hook, object_pairs_hook, list_hook, ext_hook
     cdef object encoding, unicode_errors
     cdef size_t max_buffer_size
 
@@ -228,6 +228,7 @@ cdef class Unpacker(object):
         self.object_hook = object_hook
         self.object_pairs_hook = object_pairs_hook
         self.list_hook = list_hook
+        self.ext_hook = ext_hook
 
         self.file_like = file_like
         if file_like:
diff --git a/test/test_unpack.py b/test/test_unpack.py
index 544cebf..275f124 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -1,6 +1,6 @@
 from io import BytesIO
 import sys
-from msgpack import Unpacker, packb, OutOfData
+from msgpack import Unpacker, packb, OutOfData, ExtType
 from pytest import raises, mark
 
 
@@ -42,6 +42,29 @@ def test_unpacker_hook_refcnt():
     assert sys.getrefcount(hook) == basecnt
 
 
+def test_unpacker_ext_hook():
+
+    class MyUnpacker(Unpacker):
+
+        def __init__(self):
+            super().__init__(ext_hook=self._hook, encoding='utf-8')
+
+        def _hook(self, code, data):
+            if code == 1:
+                return int(data)
+            else:
+                return ExtType(code, data)
+
+    unpacker = MyUnpacker()
+    unpacker.feed(packb({'a': 1}, encoding='utf-8'))
+    assert unpacker.unpack() == {'a': 1}
+    unpacker.feed(packb({'a': ExtType(1, b'123')}, encoding='utf-8'))
+    assert unpacker.unpack() == {'a': 123}
+    unpacker.feed(packb({'a': ExtType(2, b'321')}, encoding='utf-8'))
+    assert unpacker.unpack() == {'a': ExtType(2, b'321')}
+
+
 if __name__ == '__main__':
     test_unpack_array_header_from_file()
     test_unpacker_hook_refcnt()
+    test_unpacker_ext_hook()

From ee38505db59c55d2d96516274030eed712028039 Mon Sep 17 00:00:00 2001
From: Alexey Popravka 
Date: Mon, 24 Mar 2014 15:42:16 +0200
Subject: [PATCH 1241/1648] fixed super() for python2

---
 test/test_unpack.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/test_unpack.py b/test/test_unpack.py
index 275f124..8d0d949 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -47,7 +47,8 @@ def test_unpacker_ext_hook():
     class MyUnpacker(Unpacker):
 
         def __init__(self):
-            super().__init__(ext_hook=self._hook, encoding='utf-8')
+            super(MyUnpacker, self).__init__(ext_hook=self._hook,
+                                             encoding='utf-8')
 
         def _hook(self, code, data):
             if code == 1:

From 6c0c306f966c5cb1caeb2d6b0712ae3d0edd4d82 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 26 Mar 2014 02:49:03 +0900
Subject: [PATCH 1242/1648] Add tests for limits.

---
 test/test_limits.py | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 test/test_limits.py

diff --git a/test/test_limits.py b/test/test_limits.py
new file mode 100644
index 0000000..970f722
--- /dev/null
+++ b/test/test_limits.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# coding: utf-8
+import pytest
+
+from msgpack import packb, unpackb
+
+
+def test_integer():
+    x = -(2 ** 63)
+    assert unpackb(packb(x)) == x
+    with pytest.raises(OverflowError):
+        packb(x-1)
+
+    x = 2 ** 64 - 1
+    assert unpackb(packb(x)) == x
+    with pytest.raises(OverflowError):
+        packb(x+1)
+
+@pytest.mark.skipif(True, "Requires very large memory.")
+def test_binary():
+    x = b'x' * (2**32 - 1)
+    assert unpackb(packb(x)) == x
+    x += b'y'
+    with pytest.raises(ValueError):
+        packb(x)
+
+
+@pytest.mark.skipif(True, "Requires very large memory.")
+def test_string():
+    x = u'x' * (2**32 - 1)
+    assert unpackb(packb(x)) == x
+    x += u'y'
+    with pytest.raises(ValueError):
+        packb(x)
+
+
+@pytest.mark.skipif(True, "Requires very large memory.")
+def test_array():
+    x = [0] * (2**32 - 1)
+    assert unpackb(packb(x)) == x
+    x.append(0)
+    with pytest.raises(ValueError):
+        packb(x)

From e7f87d9d41532b0956ebcd9f7be52df979842466 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 26 Mar 2014 03:00:47 +0900
Subject: [PATCH 1243/1648] More limit check.

---
 test/test_limits.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/test/test_limits.py b/test/test_limits.py
index 970f722..2879c0a 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -2,7 +2,7 @@
 # coding: utf-8
 import pytest
 
-from msgpack import packb, unpackb
+from msgpack import packb, unpackb, Packer
 
 
 def test_integer():
@@ -16,11 +16,27 @@ def test_integer():
     with pytest.raises(OverflowError):
         packb(x+1)
 
+
+def test_array_header():
+    packer = Packer()
+    packer.pack_array_header(2**32-1)
+    with pytest.raises(ValueError):
+        packer.pack_array_header(2**32)
+
+
+def test_map_header():
+    packer = Packer()
+    packer.pack_map_header(2**32-1)
+    with pytest.raises(ValueError):
+        packer.pack_array_header(2**32)
+
+
 @pytest.mark.skipif(True, "Requires very large memory.")
 def test_binary():
     x = b'x' * (2**32 - 1)
     assert unpackb(packb(x)) == x
-    x += b'y'
+    del x
+    x = b'x' * (2**32)
     with pytest.raises(ValueError):
         packb(x)
 

From c60ab28ee72e1b179e21f5fbc6efd0f96d6e6467 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 26 Mar 2014 03:03:18 +0900
Subject: [PATCH 1244/1648] Add check for format limits.

---
 msgpack/_packer.pyx | 30 ++++++++++++++++++++++++++----
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index f261f08..204beaa 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -135,6 +135,8 @@ cdef class Packer(object):
                 else:
                     ret = msgpack_pack_false(&self.pk)
             elif PyLong_Check(o):
+                # PyInt_Check(long) is True for Python 3.
+                # Sow we should test long before int.
                 if o > 0:
                     ullval = o
                     ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
@@ -152,8 +154,10 @@ cdef class Packer(object):
                    dval = o
                    ret = msgpack_pack_double(&self.pk, dval)
             elif PyBytes_Check(o):
-                rawval = o
                 L = len(o)
+                if L > (2**32)-1:
+                    raise ValueError("bytes is too large")
+                rawval = o
                 ret = msgpack_pack_bin(&self.pk, L)
                 if ret == 0:
                     ret = msgpack_pack_raw_body(&self.pk, rawval, L)
@@ -161,13 +165,19 @@ cdef class Packer(object):
                 if not self.encoding:
                     raise TypeError("Can't encode unicode string: no encoding is specified")
                 o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
+                L = len(o)
+                if L > (2**32)-1:
+                    raise ValueError("dict is too large")
                 rawval = o
                 ret = msgpack_pack_raw(&self.pk, len(o))
                 if ret == 0:
                     ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
             elif PyDict_CheckExact(o):
                 d = o
-                ret = msgpack_pack_map(&self.pk, len(d))
+                L = len(d)
+                if L > (2**32)-1:
+                    raise ValueError("dict is too large")
+                ret = msgpack_pack_map(&self.pk, L)
                 if ret == 0:
                     for k, v in d.iteritems():
                         ret = self._pack(k, nest_limit-1)
@@ -175,7 +185,10 @@ cdef class Packer(object):
                         ret = self._pack(v, nest_limit-1)
                         if ret != 0: break
             elif PyDict_Check(o):
-                ret = msgpack_pack_map(&self.pk, len(o))
+                L = len(o)
+                if L > (2**32)-1:
+                    raise ValueError("dict is too large")
+                ret = msgpack_pack_map(&self.pk, L)
                 if ret == 0:
                     for k, v in o.items():
                         ret = self._pack(k, nest_limit-1)
@@ -187,10 +200,15 @@ cdef class Packer(object):
                 longval = o.code
                 rawval = o.data
                 L = len(o.data)
+                if L > (2**32)-1:
+                    raise ValueError("EXT data is too large")
                 ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyTuple_Check(o) or PyList_Check(o):
-                ret = msgpack_pack_array(&self.pk, len(o))
+                L = len(o)
+                if L > (2**32)-1:
+                    raise ValueError("list is too large")
+                ret = msgpack_pack_array(&self.pk, L)
                 if ret == 0:
                     for v in o:
                         ret = self._pack(v, nest_limit-1)
@@ -220,6 +238,8 @@ cdef class Packer(object):
         msgpack_pack_raw_body(&self.pk, data, len(data))
 
     def pack_array_header(self, size_t size):
+        if size >= (2**32-1):
+            raise ValueError
         cdef int ret = msgpack_pack_array(&self.pk, size)
         if ret == -1:
             raise MemoryError
@@ -231,6 +251,8 @@ cdef class Packer(object):
             return buf
 
     def pack_map_header(self, size_t size):
+        if size >= (2**32-1):
+            raise ValueError
         cdef int ret = msgpack_pack_map(&self.pk, size)
         if ret == -1:
             raise MemoryError

From e99331d1ab793105da2e0a4139059169e5c835b3 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 26 Mar 2014 11:01:44 +0900
Subject: [PATCH 1245/1648] Fix skipif marking.

---
 test/test_limits.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/test_limits.py b/test/test_limits.py
index 2879c0a..5d285e7 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -31,7 +31,7 @@ def test_map_header():
         packer.pack_array_header(2**32)
 
 
-@pytest.mark.skipif(True, "Requires very large memory.")
+@pytest.mark.skipif(True, reason="Requires very large memory.")
 def test_binary():
     x = b'x' * (2**32 - 1)
     assert unpackb(packb(x)) == x
@@ -41,7 +41,7 @@ def test_binary():
         packb(x)
 
 
-@pytest.mark.skipif(True, "Requires very large memory.")
+@pytest.mark.skipif(True, reason="Requires very large memory.")
 def test_string():
     x = u'x' * (2**32 - 1)
     assert unpackb(packb(x)) == x
@@ -50,7 +50,7 @@ def test_string():
         packb(x)
 
 
-@pytest.mark.skipif(True, "Requires very large memory.")
+@pytest.mark.skipif(True, reason="Requires very large memory.")
 def test_array():
     x = [0] * (2**32 - 1)
     assert unpackb(packb(x)) == x

From ef5d93d4eaaaf107094776c215f07efd0faae3e9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 26 Mar 2014 11:05:53 +0900
Subject: [PATCH 1246/1648] Fix size limit on pack_array_header and
 pack_map_header.

---
 msgpack/_packer.pyx | 4 ++--
 msgpack/fallback.py | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 204beaa..82e4a63 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -238,7 +238,7 @@ cdef class Packer(object):
         msgpack_pack_raw_body(&self.pk, data, len(data))
 
     def pack_array_header(self, size_t size):
-        if size >= (2**32-1):
+        if size > (2**32-1):
             raise ValueError
         cdef int ret = msgpack_pack_array(&self.pk, size)
         if ret == -1:
@@ -251,7 +251,7 @@ cdef class Packer(object):
             return buf
 
     def pack_map_header(self, size_t size):
-        if size >= (2**32-1):
+        if size > (2**32-1):
             raise ValueError
         cdef int ret = msgpack_pack_map(&self.pk, size)
         if ret == -1:
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 55de72c..49323e6 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -633,6 +633,8 @@ class Packer(object):
         return ret
 
     def pack_array_header(self, n):
+        if n >= 2**32:
+            raise ValueError
         self._fb_pack_array_header(n)
         ret = self._buffer.getvalue()
         if self._autoreset:
@@ -642,6 +644,8 @@ class Packer(object):
         return ret
 
     def pack_map_header(self, n):
+        if n >= 2**32:
+            raise ValueError
         self._fb_pack_map_header(n)
         ret = self._buffer.getvalue()
         if self._autoreset:

From 7d0e145e91700ee58c5accaeb5dff6a9ec39a18d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 26 Mar 2014 12:50:28 +0900
Subject: [PATCH 1247/1648] Allow ValueError for packing integer overs format
 limit.

---
 test/test_limits.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_limits.py b/test/test_limits.py
index 5d285e7..413d651 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -8,12 +8,12 @@ from msgpack import packb, unpackb, Packer
 def test_integer():
     x = -(2 ** 63)
     assert unpackb(packb(x)) == x
-    with pytest.raises(OverflowError):
+    with pytest.raises((OverflowError, ValueError)):
         packb(x-1)
 
     x = 2 ** 64 - 1
     assert unpackb(packb(x)) == x
-    with pytest.raises(OverflowError):
+    with pytest.raises((OverflowError, ValueError)):
         packb(x+1)
 
 

From 5fb9d8a7fd09893afdd1e5844288313bb347856b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 26 Mar 2014 13:32:28 +0900
Subject: [PATCH 1248/1648] Fix Python 3.2

---
 test/test_limits.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_limits.py b/test/test_limits.py
index 413d651..7bc8f4c 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
+from __future__ import absolute_import, division, print_function, unicode_literals
 import pytest
 
 from msgpack import packb, unpackb, Packer
@@ -43,9 +44,9 @@ def test_binary():
 
 @pytest.mark.skipif(True, reason="Requires very large memory.")
 def test_string():
-    x = u'x' * (2**32 - 1)
+    x = 'x' * (2**32 - 1)
     assert unpackb(packb(x)) == x
-    x += u'y'
+    x += 'y'
     with pytest.raises(ValueError):
         packb(x)
 

From a72e75d7c81486bc2775b957b22d3b5e13742589 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 26 Mar 2014 15:12:28 +0900
Subject: [PATCH 1249/1648] Fix PyPy fail.

---
 test/test_limits.py | 54 +++++++++++++++++++++++----------------------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/test/test_limits.py b/test/test_limits.py
index 7bc8f4c..da8cd2b 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -32,29 +32,31 @@ def test_map_header():
         packer.pack_array_header(2**32)
 
 
-@pytest.mark.skipif(True, reason="Requires very large memory.")
-def test_binary():
-    x = b'x' * (2**32 - 1)
-    assert unpackb(packb(x)) == x
-    del x
-    x = b'x' * (2**32)
-    with pytest.raises(ValueError):
-        packb(x)
-
-
-@pytest.mark.skipif(True, reason="Requires very large memory.")
-def test_string():
-    x = 'x' * (2**32 - 1)
-    assert unpackb(packb(x)) == x
-    x += 'y'
-    with pytest.raises(ValueError):
-        packb(x)
-
-
-@pytest.mark.skipif(True, reason="Requires very large memory.")
-def test_array():
-    x = [0] * (2**32 - 1)
-    assert unpackb(packb(x)) == x
-    x.append(0)
-    with pytest.raises(ValueError):
-        packb(x)
+# PyPy fails following tests because of constant folding?
+# https://bugs.pypy.org/issue1721
+#@pytest.mark.skipif(True, reason="Requires very large memory.")
+#def test_binary():
+#    x = b'x' * (2**32 - 1)
+#    assert unpackb(packb(x)) == x
+#    del x
+#    x = b'x' * (2**32)
+#    with pytest.raises(ValueError):
+#        packb(x)
+#
+#
+#@pytest.mark.skipif(True, reason="Requires very large memory.")
+#def test_string():
+#    x = 'x' * (2**32 - 1)
+#    assert unpackb(packb(x)) == x
+#    x += 'y'
+#    with pytest.raises(ValueError):
+#        packb(x)
+#
+#
+#@pytest.mark.skipif(True, reason="Requires very large memory.")
+#def test_array():
+#    x = [0] * (2**32 - 1)
+#    assert unpackb(packb(x)) == x
+#    x.append(0)
+#    with pytest.raises(ValueError):
+#        packb(x)

From 803684b90debbd9b4f8a2631611c613e138cd7df Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 26 Mar 2014 15:52:03 +0900
Subject: [PATCH 1250/1648] 0.4.2

---
 ChangeLog.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 080f5c8..610b026 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,19 @@
+0.4.2
+=====
+:release date: 2014-03-26
+
+Inconpatible Changes
+--------------------
+
+Changes
+-------
+
+Bugs fixed
+----------
+
+* Unpacker doesn't increment refcount of ExtType hook.
+* Packer raises no exception for inputs doesn't fit to msgpack format.
+
 0.4.1
 =====
 :release date: 2014-02-17

From 61bac2f586e82313a0e618093bfed2435cd18983 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 26 Mar 2014 17:20:51 +0900
Subject: [PATCH 1251/1648] 0.4.2

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index c6ebcf8..dddfe49 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 4, 1)
+version = (0, 4, 2)

From 3b933f0966b1e53ea50418970950de294ebbea76 Mon Sep 17 00:00:00 2001
From: faerot 
Date: Thu, 22 May 2014 11:32:54 +0300
Subject: [PATCH 1252/1648] added distinguish_tuple argument to Packer

This will make precise python types serialization possible.
---
 msgpack/_packer.pyx | 12 ++++++++++--
 msgpack/fallback.py | 11 +++++++++--
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 82e4a63..86e460f 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -56,6 +56,10 @@ cdef class Packer(object):
         Convert unicode to bytes with this encoding. (default: 'utf-8')
     :param str unicode_errors:
         Error handler for encoding unicode. (default: 'strict')
+    :param bool distinguish_tuple:
+        If set to true, tuples will not be serialized as lists
+        and will be treated as unsupported type. This is useful when trying
+        to implement accurate serialization for python types.
     :param bool use_single_float:
         Use single precision float type for float. (default: False)
     :param bool autoreset:
@@ -71,6 +75,7 @@ cdef class Packer(object):
     cdef object _berrors
     cdef char *encoding
     cdef char *unicode_errors
+    cdef bool distinguish_tuple
     cdef bool use_float
     cdef bint autoreset
 
@@ -83,10 +88,12 @@ cdef class Packer(object):
         self.pk.length = 0
 
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
-                 use_single_float=False, bint autoreset=1, bint use_bin_type=0):
+                 distinguish_tuple=False, use_single_float=False, bint autoreset=1,
+                 bint use_bin_type=0):
         """
         """
         self.use_float = use_single_float
+        self.distinguish_tuple = distinguish_tuple
         self.autoreset = autoreset
         self.pk.use_bin_type = use_bin_type
         if default is not None:
@@ -122,6 +129,7 @@ cdef class Packer(object):
         cdef dict d
         cdef size_t L
         cdef int default_used = 0
+        cdef bool distinguish_tuple = self.distinguish_tuple
 
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded.")
@@ -204,7 +212,7 @@ cdef class Packer(object):
                     raise ValueError("EXT data is too large")
                 ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
-            elif PyTuple_Check(o) or PyList_Check(o):
+            elif (PyTuple_Check(o) and not distinguish_tuple) or PyList_Check(o):
                 L = len(o)
                 if L > (2**32)-1:
                     raise ValueError("list is too large")
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 49323e6..1d668c2 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -485,6 +485,10 @@ class Packer(object):
             Convert unicode to bytes with this encoding. (default: 'utf-8')
     :param str unicode_errors:
         Error handler for encoding unicode. (default: 'strict')
+    :param bool distinguish_tuple:
+        If set to true, tuples will not be serialized as lists
+        and will be treated as unsupported type. This is useful when trying
+        to implement accurate serialization for python types.
     :param bool use_single_float:
         Use single precision float type for float. (default: False)
     :param bool autoreset:
@@ -495,7 +499,9 @@ class Packer(object):
         It also enable str8 type for unicode.
     """
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
-                 use_single_float=False, autoreset=True, use_bin_type=False):
+                 distinguish_tuple=False, use_single_float=False, autoreset=True,
+                 use_bin_type=False):
+        self._distinguish_tuple = distinguish_tuple
         self._use_float = use_single_float
         self._autoreset = autoreset
         self._use_bin_type = use_bin_type
@@ -509,6 +515,7 @@ class Packer(object):
 
     def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance):
         default_used = False
+        list_type = list if self._distinguish_tuple else (list, tuple)
         while True:
             if nest_limit < 0:
                 raise PackValueError("recursion limit exceeded")
@@ -599,7 +606,7 @@ class Packer(object):
                 self._buffer.write(struct.pack("b", code))
                 self._buffer.write(data)
                 return
-            if isinstance(obj, (list, tuple)):
+            if isinstance(obj, list_type):
                 n = len(obj)
                 self._fb_pack_array_header(n)
                 for i in xrange(n):

From b877ce2afadd4a4c96d7c0542f7b29836785de71 Mon Sep 17 00:00:00 2001
From: faerot 
Date: Thu, 22 May 2014 16:45:26 +0300
Subject: [PATCH 1253/1648] precise_mode instead of distinguish_tuple

When precise_mode flag is set, serialization will be as precise as
possible - type checks will be exact (type(..) is ... instead of
isinstance(..., ...) and tuple will be treated as undefined type. This
mode is to make accurate object serialization possible.
---
 msgpack/_packer.pyx | 37 ++++++++++++++++++----------------
 msgpack/fallback.py | 48 +++++++++++++++++++++++++++++----------------
 2 files changed, 51 insertions(+), 34 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 86e460f..ec34cd8 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -56,10 +56,13 @@ cdef class Packer(object):
         Convert unicode to bytes with this encoding. (default: 'utf-8')
     :param str unicode_errors:
         Error handler for encoding unicode. (default: 'strict')
-    :param bool distinguish_tuple:
-        If set to true, tuples will not be serialized as lists
-        and will be treated as unsupported type. This is useful when trying
-        to implement accurate serialization for python types.
+    :param bool precise_mode:
+        If set to true, types will be checked to be exact. Derived classes
+        from serializeable types will not be serialized and will be
+        treated as unsupported type and forwarded to default.
+        Additionally tuples will not be serialized as lists.
+        This is useful when trying to implement accurate serialization 
+        for python types.
     :param bool use_single_float:
         Use single precision float type for float. (default: False)
     :param bool autoreset:
@@ -75,7 +78,7 @@ cdef class Packer(object):
     cdef object _berrors
     cdef char *encoding
     cdef char *unicode_errors
-    cdef bool distinguish_tuple
+    cdef bint precise_mode
     cdef bool use_float
     cdef bint autoreset
 
@@ -88,12 +91,12 @@ cdef class Packer(object):
         self.pk.length = 0
 
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
-                 distinguish_tuple=False, use_single_float=False, bint autoreset=1,
-                 bint use_bin_type=0):
+                 use_single_float=False, bint autoreset=1, bint use_bin_type=0,
+                 bint precise_mode=0):
         """
         """
         self.use_float = use_single_float
-        self.distinguish_tuple = distinguish_tuple
+        self.precise_mode = precise_mode
         self.autoreset = autoreset
         self.pk.use_bin_type = use_bin_type
         if default is not None:
@@ -129,7 +132,7 @@ cdef class Packer(object):
         cdef dict d
         cdef size_t L
         cdef int default_used = 0
-        cdef bool distinguish_tuple = self.distinguish_tuple
+        cdef bint precise = self.precise_mode
 
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded.")
@@ -137,12 +140,12 @@ cdef class Packer(object):
         while True:
             if o is None:
                 ret = msgpack_pack_nil(&self.pk)
-            elif isinstance(o, bool):
+            elif PyBool_Check(o) if precise else isinstance(o, bool):
                 if o:
                     ret = msgpack_pack_true(&self.pk)
                 else:
                     ret = msgpack_pack_false(&self.pk)
-            elif PyLong_Check(o):
+            elif PyLong_CheckExact(o) if precise else PyLong_Check(o):
                 # PyInt_Check(long) is True for Python 3.
                 # Sow we should test long before int.
                 if o > 0:
@@ -151,17 +154,17 @@ cdef class Packer(object):
                 else:
                     llval = o
                     ret = msgpack_pack_long_long(&self.pk, llval)
-            elif PyInt_Check(o):
+            elif PyInt_CheckExact(o) if precise else PyInt_Check(o):
                 longval = o
                 ret = msgpack_pack_long(&self.pk, longval)
-            elif PyFloat_Check(o):
+            elif PyFloat_CheckExact(o) if precise else PyFloat_Check(o):
                 if self.use_float:
                    fval = o
                    ret = msgpack_pack_float(&self.pk, fval)
                 else:
                    dval = o
                    ret = msgpack_pack_double(&self.pk, dval)
-            elif PyBytes_Check(o):
+            elif PyBytes_CheckExact(o) if precise else PyBytes_Check(o):
                 L = len(o)
                 if L > (2**32)-1:
                     raise ValueError("bytes is too large")
@@ -169,7 +172,7 @@ cdef class Packer(object):
                 ret = msgpack_pack_bin(&self.pk, L)
                 if ret == 0:
                     ret = msgpack_pack_raw_body(&self.pk, rawval, L)
-            elif PyUnicode_Check(o):
+            elif PyUnicode_CheckExact(o) if precise else PyUnicode_Check(o):
                 if not self.encoding:
                     raise TypeError("Can't encode unicode string: no encoding is specified")
                 o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
@@ -192,7 +195,7 @@ cdef class Packer(object):
                         if ret != 0: break
                         ret = self._pack(v, nest_limit-1)
                         if ret != 0: break
-            elif PyDict_Check(o):
+            elif not precise and PyDict_Check(o):
                 L = len(o)
                 if L > (2**32)-1:
                     raise ValueError("dict is too large")
@@ -212,7 +215,7 @@ cdef class Packer(object):
                     raise ValueError("EXT data is too large")
                 ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
-            elif (PyTuple_Check(o) and not distinguish_tuple) or PyList_Check(o):
+            elif PyList_CheckExact(o) if precise else (PyTuple_Check(o) or PyList_Check(o)):
                 L = len(o)
                 if L > (2**32)-1:
                     raise ValueError("list is too large")
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 1d668c2..77922f7 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -485,10 +485,13 @@ class Packer(object):
             Convert unicode to bytes with this encoding. (default: 'utf-8')
     :param str unicode_errors:
         Error handler for encoding unicode. (default: 'strict')
-    :param bool distinguish_tuple:
-        If set to true, tuples will not be serialized as lists
-        and will be treated as unsupported type. This is useful when trying
-        to implement accurate serialization for python types.
+    :param bool precise_mode:
+        If set to true, types will be checked to be exact. Derived classes
+        from serializeable types will not be serialized and will be
+        treated as unsupported type and forwarded to default.
+        Additionally tuples will not be serialized as lists.
+        This is useful when trying to implement accurate serialization 
+        for python types.
     :param bool use_single_float:
         Use single precision float type for float. (default: False)
     :param bool autoreset:
@@ -499,9 +502,9 @@ class Packer(object):
         It also enable str8 type for unicode.
     """
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
-                 distinguish_tuple=False, use_single_float=False, autoreset=True,
+                 precise_mode=False, use_single_float=False, autoreset=True,
                  use_bin_type=False):
-        self._distinguish_tuple = distinguish_tuple
+        self._precise_mode = precise_mode
         self._use_float = use_single_float
         self._autoreset = autoreset
         self._use_bin_type = use_bin_type
@@ -513,19 +516,30 @@ class Packer(object):
                 raise TypeError("default must be callable")
         self._default = default
 
-    def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, isinstance=isinstance):
+    def _check_precise(obj, t, type=type, tuple=tuple):
+        if type(t) is tuple:
+            return type(obj) in t
+        else:
+            return type(obj) is t
+
+    def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT,
+              check=isinstance, check_precise=_check_precise):
         default_used = False
-        list_type = list if self._distinguish_tuple else (list, tuple)
+        if self._precise_mode:
+            check = check_precise
+            list_types = list
+        else:
+            list_types = (list, tuple)
         while True:
             if nest_limit < 0:
                 raise PackValueError("recursion limit exceeded")
             if obj is None:
                 return self._buffer.write(b"\xc0")
-            if isinstance(obj, bool):
+            if check(obj, bool):
                 if obj:
                     return self._buffer.write(b"\xc3")
                 return self._buffer.write(b"\xc2")
-            if isinstance(obj, int_types):
+            if check(obj, int_types):
                 if 0 <= obj < 0x80:
                     return self._buffer.write(struct.pack("B", obj))
                 if -0x20 <= obj < 0:
@@ -547,7 +561,7 @@ class Packer(object):
                 if -0x8000000000000000 <= obj < -0x80000000:
                     return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
                 raise PackValueError("Integer value out of range")
-            if self._use_bin_type and isinstance(obj, bytes):
+            if self._use_bin_type and check(obj, bytes):
                 n = len(obj)
                 if n <= 0xff:
                     self._buffer.write(struct.pack('>BB', 0xc4, n))
@@ -558,8 +572,8 @@ class Packer(object):
                 else:
                     raise PackValueError("Bytes is too large")
                 return self._buffer.write(obj)
-            if isinstance(obj, (Unicode, bytes)):
-                if isinstance(obj, Unicode):
+            if check(obj, (Unicode, bytes)):
+                if check(obj, Unicode):
                     if self._encoding is None:
                         raise TypeError(
                             "Can't encode unicode string: "
@@ -577,11 +591,11 @@ class Packer(object):
                 else:
                     raise PackValueError("String is too large")
                 return self._buffer.write(obj)
-            if isinstance(obj, float):
+            if check(obj, float):
                 if self._use_float:
                     return self._buffer.write(struct.pack(">Bf", 0xca, obj))
                 return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
-            if isinstance(obj, ExtType):
+            if check(obj, ExtType):
                 code = obj.code
                 data = obj.data
                 assert isinstance(code, int)
@@ -606,13 +620,13 @@ class Packer(object):
                 self._buffer.write(struct.pack("b", code))
                 self._buffer.write(data)
                 return
-            if isinstance(obj, list_type):
+            if check(obj, list_types):
                 n = len(obj)
                 self._fb_pack_array_header(n)
                 for i in xrange(n):
                     self._pack(obj[i], nest_limit - 1)
                 return
-            if isinstance(obj, dict):
+            if check(obj, dict):
                 return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj),
                                                nest_limit - 1)
             if not default_used and self._default is not None:

From 7f623c09060fd2561efaa6e5b4ef40da9747ec9e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 26 May 2014 01:17:53 +0900
Subject: [PATCH 1254/1648] Fix unpacking uint32 on 32bit or LLP64.

---
 msgpack/unpack.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 27e3b62..71142c6 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -62,7 +62,7 @@ static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unp
     } else
 #endif
     {
-        p = PyInt_FromLong((long)d);
+        p = PyInt_FromUnsignedLong((long)d);
     }
     if (!p)
         return -1;

From 8f1c0504f15440ba029ca2a069da17db7cb31f65 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 26 May 2014 01:28:30 +0900
Subject: [PATCH 1255/1648] Travis preinstall Python 3.4

---
 .travis.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 1c1efc3..7cf7f58 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,10 +2,6 @@ language: python
 python:
     - 2.7
 
-before_install:
-    - sudo apt-get update -qq -y
-    - sudo apt-get install python3.4
-
 install:
     - pip install tox cython
     - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx

From 4d4a0cc4425b38b1a928ef80ee8963a6b259c316 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 26 May 2014 15:21:14 +0900
Subject: [PATCH 1256/1648] Add changelog

---
 ChangeLog.rst | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 610b026..a7b9800 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,18 @@
+0.4.3
+=====
+:release date: TBD
+
+Inconpatible Changes
+--------------------
+
+Changes
+-------
+
+Bugs fixed
+----------
+
+* Unpacker may unpack wrong uint32 value on 32bit or LLP64 environment. (#101)
+
 0.4.2
 =====
 :release date: 2014-03-26

From 67391fd60e2285067a6cbd7392ef0a603875cc59 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Sat, 14 Jun 2014 18:25:34 +0200
Subject: [PATCH 1257/1648] fallback: add missing update of _fb_buf_n

---
 msgpack/fallback.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 49323e6..d326885 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -252,6 +252,7 @@ class Unpacker(object):
                 if not tmp:
                     break
                 buffs.append(tmp)
+                self._fb_buf_n += len(tmp)
                 continue
             sliced = n - len(ret)
             ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced]

From 56cf3841593733dafe0f84a0e8cbc16c3dc7250d Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Sat, 14 Jun 2014 18:25:57 +0200
Subject: [PATCH 1258/1648] fallback: set default read_size to 4096

---
 msgpack/fallback.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index d326885..294a97e 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -169,7 +169,7 @@ class Unpacker(object):
         self._max_buffer_size = max_buffer_size or 2**31-1
         if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")
-        self._read_size = read_size or min(self._max_buffer_size, 2048)
+        self._read_size = read_size or min(self._max_buffer_size, 4096)
         self._encoding = encoding
         self._unicode_errors = unicode_errors
         self._use_list = use_list

From ba8cf1c402fe68f283480c9abbf7707d004a8583 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Sat, 14 Jun 2014 18:26:30 +0200
Subject: [PATCH 1259/1648] fallback: _fb_consume: improve performance with
 pypy

---
 msgpack/fallback.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 294a97e..0e0557f 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -202,12 +202,17 @@ class Unpacker(object):
         self._fb_buffers.append(next_bytes)
 
     def _fb_consume(self):
-        self._fb_buffers = self._fb_buffers[self._fb_buf_i:]
+        if self._fb_buf_i:
+            for i in xrange(self._fb_buf_i):
+                self._fb_buf_n -=  len(self._fb_buffers[i])
+            self._fb_buffers = self._fb_buffers[self._fb_buf_i:]
+            self._fb_buf_i = 0
         if self._fb_buffers:
             self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:]
+            self._fb_buf_n -= self._fb_buf_o
+        else:
+            self._fb_buf_n = 0
         self._fb_buf_o = 0
-        self._fb_buf_i = 0
-        self._fb_buf_n = sum(map(len, self._fb_buffers))
 
     def _fb_got_extradata(self):
         if self._fb_buf_i != len(self._fb_buffers):

From 7eb371f8278941fb2323e0c2333ed89c88ab822b Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Sat, 14 Jun 2014 18:30:38 +0200
Subject: [PATCH 1260/1648] fallback: do not reset the buffer completely in
 between of iterations

---
 msgpack/fallback.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 0e0557f..d838e55 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -201,7 +201,16 @@ class Unpacker(object):
         self._fb_buf_n += len(next_bytes)
         self._fb_buffers.append(next_bytes)
 
+    def _fb_sloppy_consume(self):
+        """ Gets rid of some of the used parts of the buffer. """
+        if self._fb_buf_i:
+            for i in xrange(self._fb_buf_i):
+                self._fb_buf_n -=  len(self._fb_buffers[i])
+            self._fb_buffers = self._fb_buffers[self._fb_buf_i:]
+            self._fb_buf_i = 0
+
     def _fb_consume(self):
+        """ Gets rid of the used parts of the buffer. """
         if self._fb_buf_i:
             for i in xrange(self._fb_buf_i):
                 self._fb_buf_n -=  len(self._fb_buffers[i])
@@ -446,9 +455,10 @@ class Unpacker(object):
     def next(self):
         try:
             ret = self._fb_unpack(EX_CONSTRUCT, None)
-            self._fb_consume()
+            self._fb_sloppy_consume()
             return ret
         except OutOfData:
+            self._fb_consume()
             raise StopIteration
     __next__ = next
 

From 952eb9fc5331fb7134cc70ce6084b30eff4ccda2 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Sat, 14 Jun 2014 18:34:17 +0200
Subject: [PATCH 1261/1648] fallback: add some comments to _fb_read

---
 msgpack/fallback.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index d838e55..fbd87ba 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -252,11 +252,15 @@ class Unpacker(object):
 
     def _fb_read(self, n, write_bytes=None):
         buffs = self._fb_buffers
+        # We have a redundant codepath for the most common case, such that
+        # pypy optimizes it properly.  This is the case that the read fits
+        # in the current buffer.
         if (write_bytes is None and self._fb_buf_i < len(buffs) and
                 self._fb_buf_o + n < len(buffs[self._fb_buf_i])):
             self._fb_buf_o += n
             return buffs[self._fb_buf_i][self._fb_buf_o - n:self._fb_buf_o]
 
+        # The remaining cases.
         ret = b''
         while len(ret) != n:
             if self._fb_buf_i == len(buffs):

From b334d441c3ed4e8bcd01bb1c2440dc975c377e56 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Sat, 14 Jun 2014 18:42:02 +0200
Subject: [PATCH 1262/1648] fallback: _fb_read: do a big read, when we need a
 big read

---
 msgpack/fallback.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index fbd87ba..c422fb5 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -263,16 +263,19 @@ class Unpacker(object):
         # The remaining cases.
         ret = b''
         while len(ret) != n:
+            sliced = n - len(ret)
             if self._fb_buf_i == len(buffs):
                 if self._fb_feeding:
                     break
-                tmp = self.file_like.read(self._read_size)
+                to_read = sliced
+                if self._read_size > to_read:
+                    to_read = self._read_size
+                tmp = self.file_like.read(to_read)
                 if not tmp:
                     break
                 buffs.append(tmp)
                 self._fb_buf_n += len(tmp)
                 continue
-            sliced = n - len(ret)
             ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced]
             self._fb_buf_o += sliced
             if self._fb_buf_o >= len(buffs[self._fb_buf_i]):

From 0532ea87fb9b7f8d3b8544ff116fd589ad2a93b2 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Sun, 15 Jun 2014 22:45:30 +0200
Subject: [PATCH 1263/1648] fallback: fix BufferFull with sloppy consume

---
 msgpack/fallback.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index c422fb5..8ab3ad2 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -166,6 +166,11 @@ class Unpacker(object):
         self._fb_buf_o = 0
         self._fb_buf_i = 0
         self._fb_buf_n = 0
+        # When Unpacker is used as an iterable, between the calls to next(),
+        # the buffer is not "consumed" completely, for efficiency sake.
+        # Instead, it is done sloppily.  To make sure we raise BufferFull at
+        # the correct moments, we have to keep track of how sloppy we were.
+        self._fb_sloppiness = 0
         self._max_buffer_size = max_buffer_size or 2**31-1
         if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")
@@ -196,7 +201,8 @@ class Unpacker(object):
         elif isinstance(next_bytes, bytearray):
             next_bytes = bytes(next_bytes)
         assert self._fb_feeding
-        if self._fb_buf_n + len(next_bytes) > self._max_buffer_size:
+        if (self._fb_buf_n + len(next_bytes) - self._fb_sloppiness
+                        > self._max_buffer_size):
             raise BufferFull
         self._fb_buf_n += len(next_bytes)
         self._fb_buffers.append(next_bytes)
@@ -208,6 +214,10 @@ class Unpacker(object):
                 self._fb_buf_n -=  len(self._fb_buffers[i])
             self._fb_buffers = self._fb_buffers[self._fb_buf_i:]
             self._fb_buf_i = 0
+        if self._fb_buffers:
+            self._fb_sloppiness = self._fb_buf_o
+        else:
+            self._fb_sloppiness = 0
 
     def _fb_consume(self):
         """ Gets rid of the used parts of the buffer. """
@@ -222,6 +232,7 @@ class Unpacker(object):
         else:
             self._fb_buf_n = 0
         self._fb_buf_o = 0
+        self._fb_sloppiness = 0
 
     def _fb_got_extradata(self):
         if self._fb_buf_i != len(self._fb_buffers):

From d6c773dc4d19e4e8bfe2931d8daf251a8b302f25 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 31 Aug 2014 02:29:05 +0900
Subject: [PATCH 1264/1648] Fix build and tests.

---
 msgpack/unpack.h    | 10 +---------
 test/test_limits.py |  4 ++--
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 71142c6..24045d5 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -55,15 +55,7 @@ static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpac
 
 static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
 {
-    PyObject *p;
-#if UINT32_MAX > LONG_MAX
-    if (d > LONG_MAX) {
-        p = PyLong_FromUnsignedLong((unsigned long)d);
-    } else
-#endif
-    {
-        p = PyInt_FromUnsignedLong((long)d);
-    }
+    PyObject *p = PyInt_FromSize_t((size_t)d);
     if (!p)
         return -1;
     *o = p;
diff --git a/test/test_limits.py b/test/test_limits.py
index da8cd2b..1cfa2d6 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -21,14 +21,14 @@ def test_integer():
 def test_array_header():
     packer = Packer()
     packer.pack_array_header(2**32-1)
-    with pytest.raises(ValueError):
+    with pytest.raises((OverflowError, ValueError)):
         packer.pack_array_header(2**32)
 
 
 def test_map_header():
     packer = Packer()
     packer.pack_map_header(2**32-1)
-    with pytest.raises(ValueError):
+    with pytest.raises((OverflowError, ValueError)):
         packer.pack_array_header(2**32)
 
 

From 6948dd51209f0f6be33d94fc3d44d20345ff9951 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 31 Aug 2014 03:03:48 +0900
Subject: [PATCH 1265/1648] Fix benchmark

---
 benchmark/benchmark.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py
index 1439289..80819c6 100644
--- a/benchmark/benchmark.py
+++ b/benchmark/benchmark.py
@@ -15,10 +15,12 @@ def profile(name, func):
 
 def simple(name, data):
     if has_ext:
-        profile("packing %s (ext)" % name, lambda: _packer.packb(data))
-    profile('packing %s (fallback)' % name, lambda: fallback.packb(data))
+        packer = _packer.Packer()
+        profile("packing %s (ext)" % name, lambda: packer.pack(data))
+    packer = fallback.Packer()
+    profile('packing %s (fallback)' % name, lambda: packer.pack(data))
 
-    data = fallback.packb(data)
+    data = packer.pack(data)
     if has_ext:
         profile('unpacking %s (ext)' % name, lambda: _unpacker.unpackb(data))
     profile('unpacking %s (fallback)' % name, lambda: fallback.unpackb(data))

From d5e9ac93162e3ea846be8862b54267fe9616f72b Mon Sep 17 00:00:00 2001
From: Xiaojie Lin 
Date: Fri, 14 Nov 2014 14:47:03 +1100
Subject: [PATCH 1266/1648] add support for pypy3

---
 msgpack/fallback.py | 5 ++++-
 tox.ini             | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 8ab3ad2..71fa7be 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -23,7 +23,10 @@ if hasattr(sys, 'pypy_version_info'):
     # cStringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
     # StringBuilder is fastest.
     from __pypy__ import newlist_hint
-    from __pypy__.builders import StringBuilder
+    try:
+        from __pypy__.builders import BytesBuilder as StringBuilder
+    except ImportError:
+        from __pypy__.builders import StringBuilder
     USING_STRINGBUILDER = True
     class StringIO(object):
         def __init__(self, s=b''):
diff --git a/tox.ini b/tox.ini
index 13d6b8d..892684d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py26,py27,py32,py33,py34,pypy
+envlist = py26,py27,py32,py33,py34,pypy,pypy3
 
 [testenv]
 deps=

From 0be3e874c63cfbdd647f6800e44a049657aef12b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 18 Dec 2014 13:17:49 +0900
Subject: [PATCH 1267/1648] Use new container based Travis-CI

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 7cf7f58..29f1b7d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,3 +1,4 @@
+sudo: false
 language: python
 python:
     - 2.7

From 593887025ea73acdd94fed93480011fb26e0aca8 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 2 Jan 2015 12:12:09 +0900
Subject: [PATCH 1268/1648] Fix README reST

---
 README.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index f99ed48..705ad75 100644
--- a/README.rst
+++ b/README.rst
@@ -110,7 +110,7 @@ msgpack provides ``dumps`` and ``loads`` as alias for compatibility with
    >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False)
    (1, 2, 3)
 
-You should always pass the ``use_list`` keyword argument. See performance issues relating to use_list_ below.
+You should always pass the ``use_list`` keyword argument. See performance issues relating to `use_list option`_ below.
 
 Read the docstring for other options.
 
@@ -237,8 +237,8 @@ CPython's GC starts when growing allocated object.
 This means unpacking may cause useless GC.
 You can use ``gc.disable()`` when unpacking large message.
 
-`use_list` option
-^^^^^^^^^^^^^^^^^^
+use_list option
+^^^^^^^^^^^^^^^^
 List is the default sequence type of Python.
 But tuple is lighter than list.
 You can use ``use_list=False`` while unpacking when performance is important.

From 9624a2aca31ae94a92e2fc7225a12bd4875e3591 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 7 Jan 2015 12:10:42 +0900
Subject: [PATCH 1269/1648] Fix build failuer for Python 2.7 on Windows.

Remove int8_t usage.
---
 msgpack/_packer.pyx | 3 +--
 msgpack/pack.h      | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 82e4a63..fcd20a7 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -5,7 +5,6 @@ from cpython cimport *
 from libc.stdlib cimport *
 from libc.string cimport *
 from libc.limits cimport *
-from libc.stdint cimport int8_t
 
 from msgpack.exceptions import PackValueError
 from msgpack import ExtType
@@ -32,7 +31,7 @@ cdef extern from "pack.h":
     int msgpack_pack_raw(msgpack_packer* pk, size_t l)
     int msgpack_pack_bin(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
-    int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l)
+    int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l)
 
 cdef int DEFAULT_RECURSE_LIMIT=511
 
diff --git a/msgpack/pack.h b/msgpack/pack.h
index a71c87b..971065c 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -71,7 +71,7 @@ static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
 static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l);
 static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
 
-static inline int msgpack_pack_ext(msgpack_packer* pk, int8_t typecode, size_t l);
+static inline int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l);
 
 static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)
 {

From 5bc685973d133a6c47f28437d52040ee1ae0f1e5 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 7 Jan 2015 15:59:35 +0900
Subject: [PATCH 1270/1648] 0.4.3

---
 ChangeLog.rst       |  3 ++-
 README.rst          | 12 +++++-------
 msgpack/_version.py |  2 +-
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index a7b9800..44e832f 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,6 @@
 0.4.3
 =====
-:release date: TBD
+:release date: 2015-01-07
 
 Inconpatible Changes
 --------------------
@@ -12,6 +12,7 @@ Bugs fixed
 ----------
 
 * Unpacker may unpack wrong uint32 value on 32bit or LLP64 environment. (#101)
+* Build failed on Windows Python 2.7.
 
 0.4.2
 =====
diff --git a/README.rst b/README.rst
index 705ad75..fec8aea 100644
--- a/README.rst
+++ b/README.rst
@@ -3,8 +3,8 @@ MessagePack for Python
 =======================
 
 :author: INADA Naoki
-:version: 0.4.1
-:date: 2014-02-17
+:version: 0.4.3
+:date: 2015-01-07
 
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
    :target: https://travis-ci.org/#!/msgpack/msgpack-python
@@ -18,17 +18,15 @@ writing MessagePack data.
 
 Install
 ---------
-You can use ``pip`` or ``easy_install`` to install msgpack::
 
-   $ easy_install msgpack-python
-     or
+::
+
    $ pip install msgpack-python
 
 PyPy
 ^^^^^
 
-msgpack-python provides pure python implementation.
-PyPy can use this.
+msgpack-python provides pure python implementation.  PyPy can use this.
 
 Windows
 ^^^^^^^
diff --git a/msgpack/_version.py b/msgpack/_version.py
index dddfe49..367086f 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 4, 2)
+version = (0, 4, 3)

From c25c8d724670ceb38c0c7a75274ccfe1be18ec02 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 9 Jan 2015 03:41:52 +0900
Subject: [PATCH 1271/1648] Check extension module was compiled.

---
 .travis.yml |  2 +-
 tox.ini     | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 29f1b7d..1bf6d4f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,4 +7,4 @@ install:
     - pip install tox cython
     - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
 
-script: "tox && MSGPACK_PUREPYTHON=x tox"
+script: tox
diff --git a/tox.ini b/tox.ini
index 892684d..96b9fcb 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,8 +1,14 @@
 [tox]
-envlist = py26,py27,py32,py33,py34,pypy,pypy3
+envlist = {py26,py27,py32,py33,py34}-{c,pure},{pypy,pypy3}-pure
+
+[variants:pure]
+setenv=
+    MSGPACK_PUREPYTHON=x
 
 [testenv]
 deps=
     pytest
 
-commands=py.test test
+commands=
+    c: python -c 'from msgpack import _packer, _unpacker'
+    pure: py.test test

From ee0e435535800e51004e64d827d66bd8d30a1735 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 9 Jan 2015 04:10:25 +0900
Subject: [PATCH 1272/1648] Fix compile error.

---
 msgpack/pack_template.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 2879bbd..5d1088f 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -712,7 +712,7 @@ static inline int msgpack_pack_raw_body(msgpack_packer* x, const void* b, size_t
 /*
  * Ext
  */
-static inline int msgpack_pack_ext(msgpack_packer* x, int8_t typecode, size_t l)
+static inline int msgpack_pack_ext(msgpack_packer* x, char typecode, size_t l)
 {
     if (l == 1) {
         unsigned char buf[2];

From 715fcac6c6e2bf95a3d074ba186bd6fb106ee312 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 9 Jan 2015 04:19:34 +0900
Subject: [PATCH 1273/1648] Fix tox

---
 tox.ini | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 96b9fcb..8f10b15 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,6 +9,7 @@ setenv=
 deps=
     pytest
 
+changedir=test
 commands=
-    c: python -c 'from msgpack import _packer, _unpacker'
-    pure: py.test test
+    c: python -c 'from msgpack import _packer, _unpacker' && py.test
+    pure: py.test

From ca9768771dc63d428c3a33c046b403119e664106 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 9 Jan 2015 04:29:06 +0900
Subject: [PATCH 1274/1648] Use wheelhouse for cython

---
 .travis.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 1bf6d4f..b9d19c1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,10 +1,17 @@
 sudo: false
+cache:
+    directories:
+        - wheelhouse
+
 language: python
 python:
     - 2.7
 
 install:
-    - pip install tox cython
+    - pip install wheel tox
+    - ls -la wheelhouse
+    - if [ ! -f wheelhouse/Cython-0.21.2-cp27-none-linux_x86_64.whl ] ; then pip wheel cython ; fi
+    - pip install wheelhouse/Cython-0.21.2-cp27-none-linux_x86_64.whl
     - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
 
 script: tox

From b6055ce47ea40c5dfcac96b3ff82063a849e31ce Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 9 Jan 2015 09:35:31 +0900
Subject: [PATCH 1275/1648] Fix tox.ini

---
 tox.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 8f10b15..7971dc7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,5 +11,6 @@ deps=
 
 changedir=test
 commands=
-    c: python -c 'from msgpack import _packer, _unpacker' && py.test
+    c: python -c 'from msgpack import _packer, _unpacker'
+    c: py.test
     pure: py.test

From 3445e43d72193a95bc8ff5e8ca1dd406d2b592a4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 9 Jan 2015 09:53:27 +0900
Subject: [PATCH 1276/1648] 0.4.4

---
 ChangeLog.rst | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 44e832f..21d2b16 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,18 @@
+0.4.4
+=====
+:release date: 2015-01-09
+
+Inconpatible Changes
+--------------------
+
+Changes
+-------
+
+Bugs fixed
+----------
+
+* Fix compile error.
+
 0.4.3
 =====
 :release date: 2015-01-07

From deb8094e1d5fac491b770b6a0ec9580a665b83bc Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 9 Jan 2015 09:53:44 +0900
Subject: [PATCH 1277/1648] 0.4.4

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index 367086f..337c9ce 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 4, 3)
+version = (0, 4, 4)

From 87b493b2d84166b678aa493d12f3634e7610240e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 9 Jan 2015 09:54:30 +0900
Subject: [PATCH 1278/1648] Update README

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index fec8aea..ae19984 100644
--- a/README.rst
+++ b/README.rst
@@ -3,8 +3,8 @@ MessagePack for Python
 =======================
 
 :author: INADA Naoki
-:version: 0.4.3
-:date: 2015-01-07
+:version: 0.4.4
+:date: 2015-01-09
 
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
    :target: https://travis-ci.org/#!/msgpack/msgpack-python

From 2b4a815e5a36bd8d989df35504886127cc4d7743 Mon Sep 17 00:00:00 2001
From: Marc Abramowitz 
Date: Thu, 8 Jan 2015 20:37:31 -0800
Subject: [PATCH 1279/1648] README.rst: Add code-blocks

For syntax highlighting
---
 README.rst | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/README.rst b/README.rst
index ae19984..f4268ad 100644
--- a/README.rst
+++ b/README.rst
@@ -52,6 +52,8 @@ It is non-string binary like Python 3's ``bytes``.
 To use *bin* type for packing ``bytes``, pass ``use_bin_type=True`` to
 packer argument.
 
+.. code-block:: pycon
+
     >>> import msgpack
     >>> packed = msgpack.packb([b'spam', u'egg'], use_bin_type=True)
     >>> msgpack.unpackb(packed, encoding='utf-8')
@@ -62,6 +64,8 @@ binary can be unpacked by unpackers supporting msgpack-2.0.
 
 To use *ext* type, pass ``msgpack.ExtType`` object to packer.
 
+.. code-block:: pycon
+
     >>> import msgpack
     >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy'))
     >>> msgpack.unpackb(packed)
@@ -95,7 +99,7 @@ msgpack provides ``dumps`` and ``loads`` as alias for compatibility with
 ``pack`` and ``dump`` packs to file-like object.
 ``unpack`` and ``load`` unpacks from file-like object.
 
-::
+.. code-block:: pycon
 
    >>> import msgpack
    >>> msgpack.packb([1, 2, 3])
@@ -103,7 +107,9 @@ msgpack provides ``dumps`` and ``loads`` as alias for compatibility with
    >>> msgpack.unpackb(_)
    [1, 2, 3]
 
-``unpack`` unpacks msgpack's array to Python's list, but can unpack to tuple::
+``unpack`` unpacks msgpack's array to Python's list, but can unpack to tuple:
+
+.. code-block:: pycon
 
    >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False)
    (1, 2, 3)
@@ -119,7 +125,7 @@ Streaming unpacking
 ``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one
 stream (or from bytes provided through its ``feed`` method).
 
-::
+.. code-block:: python
 
    import msgpack
    from io import BytesIO
@@ -141,7 +147,7 @@ Packing/unpacking of custom data type
 It is also possible to pack/unpack custom data types. Here is an example for
 ``datetime.datetime``.
 
-::
+.. code-block:: python
 
     import datetime
 
@@ -175,6 +181,8 @@ Extended types
 
 It is also possible to pack/unpack custom data types using the msgpack 2.0 feature.
 
+.. code-block:: pycon
+
     >>> import msgpack
     >>> import array
     >>> def default(obj):
@@ -209,7 +217,7 @@ in a map, can be unpacked or skipped individually.
 Each of these methods may optionally write the packed data it reads to a
 callback function:
 
-::
+.. code-block:: python
 
     from io import BytesIO
 

From e7f505119de8eccda1476ded8087a4ea40660511 Mon Sep 17 00:00:00 2001
From: Marc Abramowitz 
Date: Thu, 8 Jan 2015 22:05:27 -0800
Subject: [PATCH 1280/1648] README.rst: Add PyPI badge

---
 README.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.rst b/README.rst
index f4268ad..1481681 100644
--- a/README.rst
+++ b/README.rst
@@ -8,6 +8,10 @@ MessagePack for Python
 
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
    :target: https://travis-ci.org/#!/msgpack/msgpack-python
+   
+.. image:: https://pypip.in/version/msgpack-python/badge.svg
+    :target: https://pypi.python.org/pypi/msgpack-python/
+    :alt: Latest Version
 
 What's this
 ------------

From d14a7885c773ddf05e4431f3b43e506eb77de593 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 14 Jan 2015 10:02:04 +0900
Subject: [PATCH 1281/1648] Fix typo in ChangeLog.

Fixes #121.
Thanks @dmick
---
 ChangeLog.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 21d2b16..9f959a9 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -2,7 +2,7 @@
 =====
 :release date: 2015-01-09
 
-Inconpatible Changes
+Incompatible Changes
 --------------------
 
 Changes
@@ -17,7 +17,7 @@ Bugs fixed
 =====
 :release date: 2015-01-07
 
-Inconpatible Changes
+Incompatible Changes
 --------------------
 
 Changes
@@ -33,7 +33,7 @@ Bugs fixed
 =====
 :release date: 2014-03-26
 
-Inconpatible Changes
+Incompatible Changes
 --------------------
 
 Changes
@@ -49,7 +49,7 @@ Bugs fixed
 =====
 :release date: 2014-02-17
 
-Inconpatible Changes
+Incompatible Changes
 --------------------
 
 Changes
@@ -67,7 +67,7 @@ Bugs fixed
 =====
 :release date: 2013-10-21
 
-Inconpatible Changes
+Incompatible Changes
 --------------------
 
 * Raises TypeError instead of ValueError when packer receives unsupported type.
@@ -81,7 +81,7 @@ Changes
 0.3.0
 =====
 
-Inconpatible Changes
+Incompatible Changes
 --------------------
 
 * Default value of ``use_list`` is ``True`` for now. (It was ``False`` for 0.2.x)

From f40fdf523a545035d3f6ee36a31d50993feac14b Mon Sep 17 00:00:00 2001
From: Ken Dreyer 
Date: Fri, 23 Jan 2015 12:22:28 -0700
Subject: [PATCH 1282/1648] tests: add pytest 2.3 compatibility

Adjust the skipif conditional to use the older pytest 2.3 syntax.

(This allows the tests to pass with the system pytest package on RHEL
7.0, since RHEL 7.0 ships pytest 2.3.5.)
---
 test/test_unpack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_unpack.py b/test/test_unpack.py
index 8d0d949..c0d711c 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -16,7 +16,7 @@ def test_unpack_array_header_from_file():
         unpacker.unpack()
 
 
-@mark.skipif(not hasattr(sys, 'getrefcount'),
+@mark.skipif("not hasattr(sys, 'getrefcount') == True",
              reason='sys.getrefcount() is needed to pass this test')
 def test_unpacker_hook_refcnt():
     result = []

From 75ce78dd1512460712f2600ffd927bedeeb02fbc Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 23 Jun 2014 22:46:08 +0900
Subject: [PATCH 1283/1648] Add max__len option to unpacker. (fixes #97).

Fix build error on 32bit environment (fixes #102).
---
 msgpack/_unpacker.pyx |  60 ++++++++++++++++----
 msgpack/fallback.py   | 127 ++++++++++++++++++++++++++++++++----------
 msgpack/unpack.h      |  33 +++++++++--
 test/test_limits.py   |  73 +++++++++++++++++++++++-
 4 files changed, 246 insertions(+), 47 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 16de40f..a9770d4 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -28,6 +28,11 @@ cdef extern from "unpack.h":
         PyObject* ext_hook
         char *encoding
         char *unicode_errors
+        Py_ssize_t max_str_len
+        Py_ssize_t max_bin_len
+        Py_ssize_t max_array_len
+        Py_ssize_t max_map_len
+        Py_ssize_t max_ext_len
 
     ctypedef struct unpack_context:
         msgpack_user user
@@ -46,10 +51,18 @@ cdef extern from "unpack.h":
 cdef inline init_ctx(unpack_context *ctx,
                      object object_hook, object object_pairs_hook,
                      object list_hook, object ext_hook,
-                     bint use_list, char* encoding, char* unicode_errors):
+                     bint use_list, char* encoding, char* unicode_errors,
+                     Py_ssize_t max_str_len, Py_ssize_t max_bin_len,
+                     Py_ssize_t max_array_len, Py_ssize_t max_map_len,
+                     Py_ssize_t max_ext_len):
     unpack_init(ctx)
     ctx.user.use_list = use_list
     ctx.user.object_hook = ctx.user.list_hook = NULL
+    ctx.user.max_str_len = max_str_len
+    ctx.user.max_bin_len = max_bin_len
+    ctx.user.max_array_len = max_array_len
+    ctx.user.max_map_len = max_map_len
+    ctx.user.max_ext_len = max_ext_len
 
     if object_hook is not None and object_pairs_hook is not None:
         raise TypeError("object_pairs_hook and object_hook are mutually exclusive.")
@@ -85,7 +98,12 @@ def default_read_extended_type(typecode, data):
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=1, encoding=None, unicode_errors="strict",
-            object_pairs_hook=None, ext_hook=ExtType):
+            object_pairs_hook=None, ext_hook=ExtType,
+            Py_ssize_t max_str_len=2147483647, # 2**32-1
+            Py_ssize_t max_bin_len=2147483647,
+            Py_ssize_t max_array_len=2147483647,
+            Py_ssize_t max_map_len=2147483647,
+            Py_ssize_t max_ext_len=2147483647):
     """
     Unpack packed_bytes to object. Returns an unpacked object.
 
@@ -115,7 +133,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         cerr = PyBytes_AsString(unicode_errors)
 
     init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
-             use_list, cenc, cerr)
+             use_list, cenc, cerr,
+             max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
     ret = unpack_construct(&ctx, buf, buf_len, &off)
     if ret == 1:
         obj = unpack_data(&ctx)
@@ -144,8 +163,7 @@ def unpack(object stream, object object_hook=None, object list_hook=None,
 
 
 cdef class Unpacker(object):
-    """
-    Streaming unpacker.
+    """Streaming unpacker.
 
     arguments:
 
@@ -183,6 +201,19 @@ cdef class Unpacker(object):
         Raises `BufferFull` exception when it is insufficient.
         You shoud set this parameter when unpacking data from untrasted source.
 
+    :param int max_str_len:
+        Limits max length of str. (default: 2**31-1)
+
+    :param int max_bin_len:
+        Limits max length of bin. (default: 2**31-1)
+
+    :param int max_array_len:
+        Limits max length of array. (default: 2**31-1)
+
+    :param int max_map_len:
+        Limits max length of map. (default: 2**31-1)
+
+
     example of streaming deserialize from file-like object::
 
         unpacker = Unpacker(file_like)
@@ -221,7 +252,12 @@ cdef class Unpacker(object):
     def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  str encoding=None, str unicode_errors='strict', int max_buffer_size=0,
-                 object ext_hook=ExtType):
+                 object ext_hook=ExtType,
+                 Py_ssize_t max_str_len=2147483647, # 2**32-1
+                 Py_ssize_t max_bin_len=2147483647,
+                 Py_ssize_t max_array_len=2147483647,
+                 Py_ssize_t max_map_len=2147483647,
+                 Py_ssize_t max_ext_len=2147483647):
         cdef char *cenc=NULL,
         cdef char *cerr=NULL
 
@@ -265,7 +301,9 @@ cdef class Unpacker(object):
             cerr = PyBytes_AsString(self.unicode_errors)
 
         init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
-                 ext_hook, use_list, cenc, cerr)
+                 ext_hook, use_list, cenc, cerr,
+                 max_str_len, max_bin_len, max_array_len,
+                 max_map_len, max_ext_len)
 
     def feed(self, object next_bytes):
         """Append `next_bytes` to internal buffer."""
@@ -365,7 +403,7 @@ cdef class Unpacker(object):
                 raise ValueError("Unpack failed: error = %d" % (ret,))
 
     def read_bytes(self, Py_ssize_t nbytes):
-        """read a specified number of raw bytes from the stream"""
+        """Read a specified number of raw bytes from the stream"""
         cdef size_t nread
         nread = min(self.buf_tail - self.buf_head, nbytes)
         ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread)
@@ -375,8 +413,7 @@ cdef class Unpacker(object):
         return ret
 
     def unpack(self, object write_bytes=None):
-        """
-        unpack one object
+        """Unpack one object
 
         If write_bytes is not None, it will be called with parts of the raw
         message as it is unpacked.
@@ -386,8 +423,7 @@ cdef class Unpacker(object):
         return self._unpack(unpack_construct, write_bytes)
 
     def skip(self, object write_bytes=None):
-        """
-        read and ignore one object, returning None
+        """Read and ignore one object, returning None
 
         If write_bytes is not None, it will be called with parts of the raw
         message as it is unpacked.
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 71fa7be..d1f39d1 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -102,62 +102,84 @@ def unpackb(packed, **kwargs):
 
 
 class Unpacker(object):
-    """
-    Streaming unpacker.
+    """Streaming unpacker.
 
-    `file_like` is a file-like object having a `.read(n)` method.
-    When `Unpacker` is initialized with a `file_like`, `.feed()` is not
-    usable.
+    arguments:
 
-    `read_size` is used for `file_like.read(read_size)`.
+    :param file_like:
+        File-like object having `.read(n)` method.
+        If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
 
-    If `use_list` is True (default), msgpack lists are deserialized to Python
-    lists.  Otherwise they are deserialized to tuples.
+    :param int read_size:
+        Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`)
 
-    `object_hook` is the same as in simplejson.  If it is not None, it should
-    be callable and Unpacker calls it with a dict argument after deserializing
-    a map.
+    :param bool use_list:
+        If true, unpack msgpack array to Python list.
+        Otherwise, unpack to Python tuple. (default: True)
 
-    `object_pairs_hook` is the same as in simplejson.  If it is not None, it
-    should be callable and Unpacker calls it with a list of key-value pairs
-    after deserializing a map.
+    :param callable object_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a dict argument after unpacking msgpack map.
+        (See also simplejson)
 
-    `ext_hook` is callback for ext (User defined) type. It called with two
-    arguments: (code, bytes). default: `msgpack.ExtType`
+    :param callable object_pairs_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
+        (See also simplejson)
 
-    `encoding` is the encoding used for decoding msgpack bytes.  If it is
-    None (default), msgpack bytes are deserialized to Python bytes.
+    :param str encoding:
+        Encoding used for decoding msgpack raw.
+        If it is None (default), msgpack raw is deserialized to Python bytes.
 
-    `unicode_errors` is used for decoding bytes.
+    :param str unicode_errors:
+        Used for decoding msgpack raw with *encoding*.
+        (default: `'strict'`)
 
-    `max_buffer_size` limits the buffer size.  0 means INT_MAX (default).
+    :param int max_buffer_size:
+        Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
+        Raises `BufferFull` exception when it is insufficient.
+        You shoud set this parameter when unpacking data from untrasted source.
 
-    Raises `BufferFull` exception when it is unsufficient.
+    :param int max_str_len:
+        Limits max length of str. (default: 2**31-1)
 
-    You should set this parameter when unpacking data from an untrustred source.
+    :param int max_bin_len:
+        Limits max length of bin. (default: 2**31-1)
 
-    example of streaming deserialization from file-like object::
+    :param int max_array_len:
+        Limits max length of array. (default: 2**31-1)
+
+    :param int max_map_len:
+        Limits max length of map. (default: 2**31-1)
+
+
+    example of streaming deserialize from file-like object::
 
         unpacker = Unpacker(file_like)
         for o in unpacker:
-            do_something(o)
+            process(o)
 
-    example of streaming deserialization from socket::
+    example of streaming deserialize from socket::
 
         unpacker = Unpacker()
-        while 1:
-            buf = sock.recv(1024*2)
+        while True:
+            buf = sock.recv(1024**2)
             if not buf:
                 break
             unpacker.feed(buf)
             for o in unpacker:
-                do_something(o)
+                process(o)
     """
 
     def __init__(self, file_like=None, read_size=0, use_list=True,
                  object_hook=None, object_pairs_hook=None, list_hook=None,
                  encoding=None, unicode_errors='strict', max_buffer_size=0,
-                 ext_hook=ExtType):
+                 ext_hook=ExtType,
+                 max_str_len=2147483647, # 2**32-1
+                 max_bin_len=2147483647,
+                 max_array_len=2147483647,
+                 max_map_len=2147483647,
+                 max_ext_len=2147483647):
         if file_like is None:
             self._fb_feeding = True
         else:
@@ -185,6 +207,11 @@ class Unpacker(object):
         self._object_hook = object_hook
         self._object_pairs_hook = object_pairs_hook
         self._ext_hook = ext_hook
+        self._max_str_len = max_str_len
+        self._max_bin_len = max_bin_len
+        self._max_array_len = max_array_len
+        self._max_map_len = max_map_len
+        self._max_ext_len = max_ext_len
 
         if list_hook is not None and not callable(list_hook):
             raise TypeError('`list_hook` is not callable')
@@ -316,12 +343,18 @@ class Unpacker(object):
             n = b & 0b00011111
             obj = self._fb_read(n, write_bytes)
             typ = TYPE_RAW
+            if n > self._max_str_len:
+                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
         elif b & 0b11110000 == 0b10010000:
             n = b & 0b00001111
             typ = TYPE_ARRAY
+            if n > self._max_array_len:
+                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
         elif b & 0b11110000 == 0b10000000:
             n = b & 0b00001111
             typ = TYPE_MAP
+            if n > self._max_map_len:
+                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
         elif b == 0xc0:
             obj = None
         elif b == 0xc2:
@@ -331,26 +364,38 @@ class Unpacker(object):
         elif b == 0xc4:
             typ = TYPE_BIN
             n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            if n > self._max_bin_len:
+                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._fb_read(n, write_bytes)
         elif b == 0xc5:
             typ = TYPE_BIN
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            if n > self._max_bin_len:
+                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._fb_read(n, write_bytes)
         elif b == 0xc6:
             typ = TYPE_BIN
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            if n > self._max_bin_len:
+                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._fb_read(n, write_bytes)
         elif b == 0xc7:  # ext 8
             typ = TYPE_EXT
             L, n = struct.unpack('Bb', self._fb_read(2, write_bytes))
+            if L > self._max_ext_len:
+                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._fb_read(L, write_bytes)
         elif b == 0xc8:  # ext 16
             typ = TYPE_EXT
             L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes))
+            if L > self._max_ext_len:
+                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._fb_read(L, write_bytes)
         elif b == 0xc9:  # ext 32
             typ = TYPE_EXT
             L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes))
+            if L > self._max_ext_len:
+                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._fb_read(L, write_bytes)
         elif b == 0xca:
             obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0]
@@ -374,42 +419,66 @@ class Unpacker(object):
             obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0]
         elif b == 0xd4:  # fixext 1
             typ = TYPE_EXT
+            if self._max_ext_len < 1:
+                raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
             n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes))
         elif b == 0xd5:  # fixext 2
             typ = TYPE_EXT
+            if self._max_ext_len < 2:
+                raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
             n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes))
         elif b == 0xd6:  # fixext 4
             typ = TYPE_EXT
+            if self._max_ext_len < 4:
+                raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
             n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes))
         elif b == 0xd7:  # fixext 8
             typ = TYPE_EXT
+            if self._max_ext_len < 8:
+                raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
             n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes))
         elif b == 0xd8:  # fixext 16
             typ = TYPE_EXT
+            if self._max_ext_len < 16:
+                raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
             n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes))
         elif b == 0xd9:
             typ = TYPE_RAW
             n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            if n > self._max_str_len:
+                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._fb_read(n, write_bytes)
         elif b == 0xda:
             typ = TYPE_RAW
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            if n > self._max_str_len:
+                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._fb_read(n, write_bytes)
         elif b == 0xdb:
             typ = TYPE_RAW
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            if n > self._max_str_len:
+                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._fb_read(n, write_bytes)
         elif b == 0xdc:
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            if n > self._max_array_len:
+                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
             typ = TYPE_ARRAY
         elif b == 0xdd:
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            if n > self._max_array_len:
+                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
             typ = TYPE_ARRAY
         elif b == 0xde:
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            if n > self._max_map_len:
+                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         elif b == 0xdf:
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            if n > self._max_map_len:
+                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         else:
             raise UnpackValueError("Unknown header: 0x%x" % b)
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 24045d5..5deb7cd 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -27,6 +27,7 @@ typedef struct unpack_user {
     PyObject *ext_hook;
     const char *encoding;
     const char *unicode_errors;
+    Py_ssize_t max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len;
 } unpack_user;
 
 typedef PyObject* msgpack_unpack_object;
@@ -68,7 +69,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp
     if (d > LONG_MAX) {
         p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d);
     } else {
-        p = PyInt_FromLong((long)d);
+        p = PyInt_FromSize_t((size_t)d);
     }
     if (!p)
         return -1;
@@ -132,6 +133,10 @@ static inline int unpack_callback_false(unpack_user* u, msgpack_unpack_object* o
 
 static inline int unpack_callback_array(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
 {
+    if (n > u->max_array_len) {
+        PyErr_Format(PyExc_ValueError, "%u exceeds max_array_len(%zd)", n, u->max_array_len);
+        return -1;
+    }
     PyObject *p = u->use_list ? PyList_New(n) : PyTuple_New(n);
 
     if (!p)
@@ -163,6 +168,10 @@ static inline int unpack_callback_array_end(unpack_user* u, msgpack_unpack_objec
 
 static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_unpack_object* o)
 {
+    if (n > u->max_map_len) {
+        PyErr_Format(PyExc_ValueError, "%u exceeds max_map_len(%zd)", n, u->max_map_len);
+        return -1;
+    }
     PyObject *p;
     if (u->has_pairs_hook) {
         p = PyList_New(n); // Or use tuple?
@@ -210,6 +219,11 @@ static inline int unpack_callback_map_end(unpack_user* u, msgpack_unpack_object*
 
 static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
 {
+    if (l > u->max_str_len) {
+        PyErr_Format(PyExc_ValueError, "%u exceeds max_str_len(%zd)", l, u->max_str_len);
+        return -1;
+    }
+
     PyObject *py;
     if(u->encoding) {
         py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors);
@@ -224,6 +238,11 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
 
 static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int l, msgpack_unpack_object* o)
 {
+    if (l > u->max_bin_len) {
+        PyErr_Format(PyExc_ValueError, "%u exceeds max_bin_len(%zd)", l, u->max_bin_len);
+        return -1;
+    }
+
     PyObject *py = PyBytes_FromStringAndSize(p, l);
     if (!py)
         return -1;
@@ -232,7 +251,7 @@ static inline int unpack_callback_bin(unpack_user* u, const char* b, const char*
 }
 
 static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
-                                      unsigned int lenght, msgpack_unpack_object* o)
+                                      unsigned int length, msgpack_unpack_object* o)
 {
     PyObject *py;
     int8_t typecode = (int8_t)*pos++;
@@ -240,11 +259,15 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch
         PyErr_SetString(PyExc_AssertionError, "u->ext_hook cannot be NULL");
         return -1;
     }
-    // length also includes the typecode, so the actual data is lenght-1
+    if (length-1 > u->max_ext_len) {
+        PyErr_Format(PyExc_ValueError, "%u exceeds max_ext_len(%zd)", length, u->max_ext_len);
+        return -1;
+    }
+    // length also includes the typecode, so the actual data is length-1
 #if PY_MAJOR_VERSION == 2
-    py = PyObject_CallFunction(u->ext_hook, "(is#)", typecode, pos, lenght-1);
+    py = PyObject_CallFunction(u->ext_hook, "(is#)", typecode, pos, length-1);
 #else
-    py = PyObject_CallFunction(u->ext_hook, "(iy#)", typecode, pos, lenght-1);
+    py = PyObject_CallFunction(u->ext_hook, "(iy#)", typecode, pos, length-1);
 #endif
     if (!py)
         return -1;
diff --git a/test/test_limits.py b/test/test_limits.py
index 1cfa2d6..3c1cf2a 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -3,7 +3,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 import pytest
 
-from msgpack import packb, unpackb, Packer
+from msgpack import packb, unpackb, Packer, Unpacker, ExtType
 
 
 def test_integer():
@@ -32,6 +32,77 @@ def test_map_header():
         packer.pack_array_header(2**32)
 
 
+def test_max_str_len():
+    d = 'x' * 3
+    packed = packb(d)
+
+    unpacker = Unpacker(max_str_len=3, encoding='utf-8')
+    unpacker.feed(packed)
+    assert unpacker.unpack() == d
+
+    unpacker = Unpacker(max_str_len=2, encoding='utf-8')
+    with pytest.raises(ValueError):
+        unpacker.feed(packed)
+        unpacker.unpack()
+
+
+def test_max_bin_len():
+    d = b'x' * 3
+    packed = packb(d, use_bin_type=True)
+
+    unpacker = Unpacker(max_bin_len=3)
+    unpacker.feed(packed)
+    assert unpacker.unpack() == d
+
+    unpacker = Unpacker(max_bin_len=2)
+    with pytest.raises(ValueError):
+        unpacker.feed(packed)
+        unpacker.unpack()
+
+
+def test_max_array_len():
+    d = [1,2,3]
+    packed = packb(d)
+
+    unpacker = Unpacker(max_array_len=3)
+    unpacker.feed(packed)
+    assert unpacker.unpack() == d
+
+    unpacker = Unpacker(max_array_len=2)
+    with pytest.raises(ValueError):
+        unpacker.feed(packed)
+        unpacker.unpack()
+
+
+def test_max_map_len():
+    d = {1: 2, 3: 4, 5: 6}
+    packed = packb(d)
+
+    unpacker = Unpacker(max_map_len=3)
+    unpacker.feed(packed)
+    assert unpacker.unpack() == d
+
+    unpacker = Unpacker(max_map_len=2)
+    with pytest.raises(ValueError):
+        unpacker.feed(packed)
+        unpacker.unpack()
+
+
+def test_max_ext_len():
+    d = ExtType(42, b"abc")
+    packed = packb(d)
+
+    unpacker = Unpacker(max_ext_len=3)
+    unpacker.feed(packed)
+    assert unpacker.unpack() == d
+
+    unpacker = Unpacker(max_ext_len=2)
+    with pytest.raises(ValueError):
+        unpacker.feed(packed)
+        unpacker.unpack()
+
+
+
 # PyPy fails following tests because of constant folding?
 # https://bugs.pypy.org/issue1721
 #@pytest.mark.skipif(True, reason="Requires very large memory.")

From 2985f4d8651982b07e2cfa7037e7a8c3530a127b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 25 Jan 2015 02:24:53 +0900
Subject: [PATCH 1284/1648] Fix error when use unicode_literal in Python 2

---
 .travis.yml           |  7 +++++++
 msgpack/_unpacker.pyx | 10 +++++++---
 tox.ini               | 29 ++++++++++++++++++++++++++---
 3 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b9d19c1..dad7e87 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,13 @@ language: python
 python:
     - 2.7
 
+env:
+    - TOXENV=py26-c,py27-c
+    - TOXENV=py32-c,py33-c,py34-c
+    - TOXENV=py26-pure,py27-pure
+    - TOXENV=py32-pure,py33-pure,py34-pure
+    - TOXENV=pypy-pure,pypy3-pure
+
 install:
     - pip install wheel tox
     - ls -la wheelhouse
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index a9770d4..f5e7d95 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -251,7 +251,7 @@ cdef class Unpacker(object):
 
     def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
-                 str encoding=None, str unicode_errors='strict', int max_buffer_size=0,
+                 encoding=None, unicode_errors='strict', int max_buffer_size=0,
                  object ext_hook=ExtType,
                  Py_ssize_t max_str_len=2147483647, # 2**32-1
                  Py_ssize_t max_bin_len=2147483647,
@@ -289,15 +289,19 @@ cdef class Unpacker(object):
         if encoding is not None:
             if isinstance(encoding, unicode):
                 self.encoding = encoding.encode('ascii')
-            else:
+            elif isinstance(encoding, bytes):
                 self.encoding = encoding
+            else:
+                raise TypeError("encoding should be bytes or unicode")
             cenc = PyBytes_AsString(self.encoding)
 
         if unicode_errors is not None:
             if isinstance(unicode_errors, unicode):
                 self.unicode_errors = unicode_errors.encode('ascii')
-            else:
+            elif isinstance(unicode_errors, bytes):
                 self.unicode_errors = unicode_errors
+            else:
+                raise TypeError("unicode_errors should be bytes or unicode")
             cerr = PyBytes_AsString(self.unicode_errors)
 
         init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
diff --git a/tox.ini b/tox.ini
index 7971dc7..15feb51 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {py26,py27,py32,py33,py34}-{c,pure},{pypy,pypy3}-pure
+envlist = {py26,py27,py32,py33,py34}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
 
 [variants:pure]
 setenv=
@@ -11,6 +11,29 @@ deps=
 
 changedir=test
 commands=
-    c: python -c 'from msgpack import _packer, _unpacker'
-    c: py.test
+    c,x86: python -c 'from msgpack import _packer, _unpacker'
+    c,x86: py.test
     pure: py.test
+
+[testenv:py27-x86]
+basepython=python2.7-x86
+deps=
+    pytest
+
+changedir=test
+commands=
+    python -c 'import sys; print(hex(sys.maxsize))'
+    python -c 'from msgpack import _packer, _unpacker'
+    py.test
+
+[testenv:py34-x86]
+basepython=python3.4-x86
+deps=
+    pytest
+
+changedir=test
+commands=
+    python -c 'import sys; print(hex(sys.maxsize))'
+    python -c 'from msgpack import _packer, _unpacker'
+    py.test
+

From 630c046bf2dcd9f77c09efb65a51ec1b3d915975 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 26 Jan 2015 00:38:36 +0900
Subject: [PATCH 1285/1648] 0.4.5

---
 ChangeLog.rst       | 18 ++++++++++++++++++
 ROADMAP.md          |  7 -------
 msgpack/_version.py |  2 +-
 3 files changed, 19 insertions(+), 8 deletions(-)
 delete mode 100644 ROADMAP.md

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 9f959a9..797fa66 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,21 @@
+0.4.5
+=====
+:release date: 2015-01-25
+
+Incompatible Changes
+--------------------
+
+Changes
+-------
+
+Bugs fixed
+----------
+
+* Fix test failure on pytest 2.3.  (by @ktdreyer)
+* Fix typos in ChangeLog.  (Thanks to @dmick)
+* Improve README.rst  (by @msabramo)
+
+
 0.4.4
 =====
 :release date: 2015-01-09
diff --git a/ROADMAP.md b/ROADMAP.md
deleted file mode 100644
index 5245cc0..0000000
--- a/ROADMAP.md
+++ /dev/null
@@ -1,7 +0,0 @@
-0.2 series
-==========
-Improve compatibility to simplejson.
-
-0.3 series
-==========
-Add features msgpack-ruby has.
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 337c9ce..ac12f4e 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 4, 4)
+version = (0, 4, 5)

From 68b029446569bdaf77518ceda99da822b0b26f21 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 26 Jan 2015 00:39:50 +0900
Subject: [PATCH 1286/1648] update README

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 1481681..3c1957d 100644
--- a/README.rst
+++ b/README.rst
@@ -3,8 +3,8 @@ MessagePack for Python
 =======================
 
 :author: INADA Naoki
-:version: 0.4.4
-:date: 2015-01-09
+:version: 0.4.5
+:date: 2015-01-25
 
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
    :target: https://travis-ci.org/#!/msgpack/msgpack-python

From 83404945c06baa3124d7e20a2d4e93652ecf90c5 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Mon, 26 Jan 2015 20:31:03 +0100
Subject: [PATCH 1287/1648] Add test for issue #124

---
 test/test_sequnpack.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 5d37698..518b905 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -84,3 +84,15 @@ def test_readbytes():
     assert unpacker.read_bytes(3) == b'oob'
     assert unpacker.unpack() == ord(b'a')
     assert unpacker.unpack() == ord(b'r')
+
+def test_issue124():
+    unpacker = Unpacker()
+    unpacker.feed('\xa1?\xa1!')
+    assert tuple(unpacker) == ('?', '!')
+    assert tuple(unpacker) == ()
+    unpacker.feed("\xa1?\xa1")
+    assert tuple(unpacker) == ('?',)
+    assert tuple(unpacker) == ()
+    unpacker.feed("!")
+    assert tuple(unpacker) == ('!',)
+    assert tuple(unpacker) == ()

From a71a24d86aca4747a954ac642a9173a05cedd257 Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Mon, 26 Jan 2015 20:34:31 +0100
Subject: [PATCH 1288/1648] Fix #124

When using Unpacker as an iterator, after each yield, the internal
buffer (_fb_buffer) was compacted by reallocation (done by _fb_consume).
When dealing with a lot of small objects, this is very ineffecient.
Thus in commit 7eb371f8278941fb2323e0c2333ed89c88ab822b the pure python
fallback only reallocated the complete buffer when the iteration stops.
When halfway there happens to be data missing in the buffer, we rollback
the buffer to the state before this failed call, and raise an OutOfData.
This rollback, done by _fb_rollback, did not consider the possibility
that the buffer was *not* reallocated.  This commit corrects that.
---
 msgpack/fallback.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index d1f39d1..bd99a15 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -195,6 +195,9 @@ class Unpacker(object):
         # the buffer is not "consumed" completely, for efficiency sake.
         # Instead, it is done sloppily.  To make sure we raise BufferFull at
         # the correct moments, we have to keep track of how sloppy we were.
+        # Furthermore, when the buffer is incomplete (that is: in the case
+        # we raise an OutOfData) we need to rollback the buffer to the correct
+        # state, which _fb_slopiness records.
         self._fb_sloppiness = 0
         self._max_buffer_size = max_buffer_size or 2**31-1
         if read_size > self._max_buffer_size:
@@ -283,7 +286,7 @@ class Unpacker(object):
 
     def _fb_rollback(self):
         self._fb_buf_i = 0
-        self._fb_buf_o = 0
+        self._fb_buf_o = self._fb_sloppiness
 
     def _fb_get_extradata(self):
         bufs = self._fb_buffers[self._fb_buf_i:]

From c5d621853d305418feb3ef2616a05f5eb12440bb Mon Sep 17 00:00:00 2001
From: Bas Westerbaan 
Date: Mon, 26 Jan 2015 20:55:23 +0100
Subject: [PATCH 1289/1648] test_sequnpack: python3 literals

---
 test/test_sequnpack.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 518b905..45f4cc7 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -87,12 +87,12 @@ def test_readbytes():
 
 def test_issue124():
     unpacker = Unpacker()
-    unpacker.feed('\xa1?\xa1!')
-    assert tuple(unpacker) == ('?', '!')
+    unpacker.feed(b'\xa1?\xa1!')
+    assert tuple(unpacker) == (b'?', b'!')
     assert tuple(unpacker) == ()
-    unpacker.feed("\xa1?\xa1")
-    assert tuple(unpacker) == ('?',)
+    unpacker.feed(b"\xa1?\xa1")
+    assert tuple(unpacker) == (b'?',)
     assert tuple(unpacker) == ()
-    unpacker.feed("!")
-    assert tuple(unpacker) == ('!',)
+    unpacker.feed(b"!")
+    assert tuple(unpacker) == (b'!',)
     assert tuple(unpacker) == ()

From 4576b94b6c9c641bfa29a1d161968822d6e67d19 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 27 Jan 2015 14:04:32 +0900
Subject: [PATCH 1290/1648] fallback: Add some comment to Unpacker members.

---
 msgpack/fallback.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index bd99a15..235c201 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -187,10 +187,16 @@ class Unpacker(object):
                 raise TypeError("`file_like.read` must be callable")
             self.file_like = file_like
             self._fb_feeding = False
+
+        #: array of bytes feeded.
         self._fb_buffers = []
-        self._fb_buf_o = 0
+        #: Which buffer we currently reads
         self._fb_buf_i = 0
+        #: Which position we currently reads
+        self._fb_buf_o = 0
+        #: Total size of _fb_bufferes
         self._fb_buf_n = 0
+
         # When Unpacker is used as an iterable, between the calls to next(),
         # the buffer is not "consumed" completely, for efficiency sake.
         # Instead, it is done sloppily.  To make sure we raise BufferFull at
@@ -199,6 +205,7 @@ class Unpacker(object):
         # we raise an OutOfData) we need to rollback the buffer to the correct
         # state, which _fb_slopiness records.
         self._fb_sloppiness = 0
+
         self._max_buffer_size = max_buffer_size or 2**31-1
         if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")

From 9fe19cc4089467fff185399c659ffe72f2f52995 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Mar 2015 03:51:14 +0900
Subject: [PATCH 1291/1648] 0.4.6

---
 ChangeLog.rst       | 11 +++++++++++
 msgpack/_version.py |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 797fa66..34f4cd4 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,14 @@
+0.4.6
+=====
+:release date: 2015-03-13
+
+Bugs fixed
+----------
+
+* fallback.Unpacker: Fix Data corruption when OutOfData.
+  This bug only affects "Streaming unpacking."
+
+
 0.4.5
 =====
 :release date: 2015-01-25
diff --git a/msgpack/_version.py b/msgpack/_version.py
index ac12f4e..2c1c96c 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 4, 5)
+version = (0, 4, 6)

From b19e336108ed86ba344eeaccc9476244848e4dd4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Mar 2015 04:05:44 +0900
Subject: [PATCH 1292/1648] travis: Cython 0.22

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index dad7e87..ad54ee0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,8 +17,8 @@ env:
 install:
     - pip install wheel tox
     - ls -la wheelhouse
-    - if [ ! -f wheelhouse/Cython-0.21.2-cp27-none-linux_x86_64.whl ] ; then pip wheel cython ; fi
-    - pip install wheelhouse/Cython-0.21.2-cp27-none-linux_x86_64.whl
+    - if [ ! -f wheelhouse/Cython-0.22-cp27-none-linux_x86_64.whl ] ; then pip wheel cython==0.22 ; fi
+    - pip install wheelhouse/Cython-0.22-cp27-none-linux_x86_64.whl
     - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
 
 script: tox

From 2dda8fc4a58dd9be0c8d6f472342fd777d92886d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Mar 2015 04:18:10 +0900
Subject: [PATCH 1293/1648] travis: Build only master

---
 .travis.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index ad54ee0..ddd5254 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,10 @@ language: python
 python:
     - 2.7
 
+branches:
+    only:
+        - master
+
 env:
     - TOXENV=py26-c,py27-c
     - TOXENV=py32-c,py33-c,py34-c

From b7806a6e6eceb13153df6907b744af3be0e1075e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Mar 2015 04:23:04 +0900
Subject: [PATCH 1294/1648] README: Update version

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 3c1957d..8cee306 100644
--- a/README.rst
+++ b/README.rst
@@ -3,8 +3,8 @@ MessagePack for Python
 =======================
 
 :author: INADA Naoki
-:version: 0.4.5
-:date: 2015-01-25
+:version: 0.4.6
+:date: 2015-03-13
 
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
    :target: https://travis-ci.org/#!/msgpack/msgpack-python

From 2d05b40b030cb6b5da0913e72b59a91b09faccab Mon Sep 17 00:00:00 2001
From: Pramukta Kumar 
Date: Tue, 17 Mar 2015 15:02:40 -0400
Subject: [PATCH 1295/1648] Test to demonstrate that the default function isn't
 always called (#133)

---
 test/test_extension.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/test/test_extension.py b/test/test_extension.py
index 2f85ce3..c552498 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -55,3 +55,22 @@ def test_extension_type():
     s = msgpack.packb(obj, default=default)
     obj2 = msgpack.unpackb(s, ext_hook=ext_hook)
     assert obj == obj2
+
+import sys
+if sys.version > '3':
+    long = int
+
+def test_overriding_hooks():
+    def default(obj):
+        if isinstance(obj, long):
+            return {"__type__": "long", "__data__": str(obj)}
+        else:
+            return obj
+
+    obj = {"testval": long(1823746192837461928374619)}
+    refobj = {"testval": default(obj["testval"])}
+    refout = msgpack.packb(refobj)
+    assert isinstance(refout, (str, bytes))
+    testout = msgpack.packb(obj, default=default)
+
+    assert refout == testout

From 10cd2d2ebf6390e844c2bf59e9efd765f9b60e40 Mon Sep 17 00:00:00 2001
From: Pramukta Kumar 
Date: Tue, 17 Mar 2015 15:05:04 -0400
Subject: [PATCH 1296/1648] calling the default function upon integer overflow
 in the fallback routine

---
 msgpack/fallback.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 235c201..eb20002 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -655,6 +655,10 @@ class Packer(object):
                     return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
                 if -0x8000000000000000 <= obj < -0x80000000:
                     return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
+                if not default_used and self._default is not None:
+                    obj = self._default(obj)
+                    default_used = True
+                    continue
                 raise PackValueError("Integer value out of range")
             if self._use_bin_type and isinstance(obj, bytes):
                 n = len(obj)

From 6f02d252e1dc66d67861b45c5bead8392ed822d4 Mon Sep 17 00:00:00 2001
From: Pramukta Kumar 
Date: Tue, 17 Mar 2015 15:16:17 -0400
Subject: [PATCH 1297/1648] corresponding change to cython implementation

---
 msgpack/_packer.pyx | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index fcd20a7..7129208 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -136,12 +136,20 @@ cdef class Packer(object):
             elif PyLong_Check(o):
                 # PyInt_Check(long) is True for Python 3.
                 # Sow we should test long before int.
-                if o > 0:
-                    ullval = o
-                    ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
-                else:
-                    llval = o
-                    ret = msgpack_pack_long_long(&self.pk, llval)
+                try:
+                    if o > 0:
+                        ullval = o
+                        ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
+                    else:
+                        llval = o
+                        ret = msgpack_pack_long_long(&self.pk, llval)
+                except OverflowError, oe:
+                    if not default_used and self._default is not None:
+                        o = self._default(o)
+                        default_used = True
+                        continue
+                    else:
+                        raise
             elif PyInt_Check(o):
                 longval = o
                 ret = msgpack_pack_long(&self.pk, longval)

From 734cb71dac66b9760cd9704667df92d0a097c1a1 Mon Sep 17 00:00:00 2001
From: tbeu 
Date: Sun, 22 Mar 2015 21:35:21 +0100
Subject: [PATCH 1298/1648] Update README.rst

Fix typo
---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 8cee306..ba2c1b5 100644
--- a/README.rst
+++ b/README.rst
@@ -37,7 +37,7 @@ Windows
 
 When you can't use binary distribution, you need to install Visual Studio
 or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support
-amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.)
+amd64. Windows SDK is recommended way to build amd64 msgpack without any fee.)
 
 Without extension, using pure python implementation on CPython runs slowly.
 

From 4eb4c7a9940a5424ded190c1e0b9a0bb466047fb Mon Sep 17 00:00:00 2001
From: Johannes Dollinger 
Date: Mon, 27 Jul 2015 20:29:43 +0200
Subject: [PATCH 1299/1648] Accept ext_hook for unpack()

---
 msgpack/_unpacker.pyx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index f5e7d95..d53f724 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -147,7 +147,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
 def unpack(object stream, object object_hook=None, object list_hook=None,
            bint use_list=1, encoding=None, unicode_errors="strict",
-           object_pairs_hook=None,
+           object_pairs_hook=None, ext_hook=ExtType
            ):
     """
     Unpack an object from `stream`.
@@ -158,7 +158,7 @@ def unpack(object stream, object object_hook=None, object list_hook=None,
     """
     return unpackb(stream.read(), use_list=use_list,
                    object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
-                   encoding=encoding, unicode_errors=unicode_errors,
+                   encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook
                    )
 
 

From c3a3f9b0a5c3a380018aa42e2f81b0c6752afcf0 Mon Sep 17 00:00:00 2001
From: TW 
Date: Fri, 30 Oct 2015 00:36:12 +0100
Subject: [PATCH 1300/1648] fix typo in setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 1055a61..37729bd 100644
--- a/setup.py
+++ b/setup.py
@@ -49,7 +49,7 @@ class BuildExt(build_ext):
         try:
             return build_ext.build_extension(self, ext)
         except Exception as e:
-            print("WARNING: Failed to compile extensiom modules.")
+            print("WARNING: Failed to compile extension modules.")
             print("msgpack uses fallback pure python implementation.")
             print(e)
 

From 672b220a3ff0c84eec1c8b39abcdedcd1e4cb329 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 7 Nov 2015 13:17:28 +0900
Subject: [PATCH 1301/1648] remove unused bat file

---
 upload_windows.bat | 4 ----
 1 file changed, 4 deletions(-)
 delete mode 100644 upload_windows.bat

diff --git a/upload_windows.bat b/upload_windows.bat
deleted file mode 100644
index 5cd9a7c..0000000
--- a/upload_windows.bat
+++ /dev/null
@@ -1,4 +0,0 @@
-c:\Python27\python       setup.py bdist_egg bdist_wininst upload
-c:\Python33\python       setup.py bdist_egg bdist_wininst upload
-c:\Python27_amd64\python setup.py bdist_egg bdist_wininst upload
-c:\Python33_amd64\python setup.py bdist_egg bdist_wininst upload

From 52a38c6e9d6cb5206732231c50d1b9a5bd0be586 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 7 Nov 2015 14:26:14 +0900
Subject: [PATCH 1302/1648] remove unused bat file

---
 build_windows.bat | 24 ------------------------
 1 file changed, 24 deletions(-)
 delete mode 100644 build_windows.bat

diff --git a/build_windows.bat b/build_windows.bat
deleted file mode 100644
index a71c0e0..0000000
--- a/build_windows.bat
+++ /dev/null
@@ -1,24 +0,0 @@
-set MSSdk=1
-set DISTUTILS_USE_SDK=1
-
-rem Python27 x86
-rem call "C:\Program Files\Microsoft SDKs\Windows\v6.1\Bin\SetEnv.cmd" /Release /x86 /xp
-call "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars32.bat"
-c:\Python27\python setup.py build_ext -f build install
-pause
-
-rem Python27 amd64
-rem call "C:\Program Files\Microsoft SDKs\Windows\v6.1\Bin\SetEnv.cmd" /Release /x64 /xp
-call "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat"
-c:\Python27_amd64\python setup.py build_ext -f build install
-pause
-
-rem Python33 x86
-call "C:\Program Files\Microsoft SDKs\Windows\v7.1\bin\SetEnv.cmd" /Release /x86 /xp
-c:\Python33\python setup.py build_ext -f build install
-pause
-
-rem Python33 amd64
-call "C:\Program Files\Microsoft SDKs\Windows\v7.1\bin\SetEnv.cmd" /Release /x64 /xp
-c:\Python33_amd64\python setup.py build_ext -f build install
-pause

From c102e6cee58df543d352e36f6d2d0bdd595e1063 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 7 Nov 2015 14:30:05 +0900
Subject: [PATCH 1303/1648] executable setup.py

---
 setup.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 setup.py

diff --git a/setup.py b/setup.py
old mode 100644
new mode 100755

From ab359e333044fc89559b946cfec8efb83c2c6122 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 7 Nov 2015 16:45:30 +0900
Subject: [PATCH 1304/1648] Update travis setting

---
 .travis.yml | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index ddd5254..7695184 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,8 +1,5 @@
 sudo: false
-cache:
-    directories:
-        - wheelhouse
-
+cache: pip
 language: python
 python:
     - 2.7
@@ -19,10 +16,8 @@ env:
     - TOXENV=pypy-pure,pypy3-pure
 
 install:
-    - pip install wheel tox
-    - ls -la wheelhouse
-    - if [ ! -f wheelhouse/Cython-0.22-cp27-none-linux_x86_64.whl ] ; then pip wheel cython==0.22 ; fi
-    - pip install wheelhouse/Cython-0.22-cp27-none-linux_x86_64.whl
+    - pip install tox
+    - pip install cython --install-option=--cython-with-refnanny --install-option=--no-cython-compile
     - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
 
 script: tox

From e9ab4d8824fddd070bac7cedca332130bf2028b0 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 7 Nov 2015 16:30:18 +0900
Subject: [PATCH 1305/1648] Fix warnings

fixes #146
---
 msgpack/_unpacker.pyx     | 24 ++++++++++++------------
 msgpack/unpack.h          |  2 +-
 msgpack/unpack_template.h | 10 +++++-----
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index d53f724..997979c 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -37,10 +37,10 @@ cdef extern from "unpack.h":
     ctypedef struct unpack_context:
         msgpack_user user
         PyObject* obj
-        size_t count
+        Py_ssize_t count
 
     ctypedef int (*execute_fn)(unpack_context* ctx, const char* data,
-                               size_t len, size_t* off) except? -1
+                               Py_ssize_t len, Py_ssize_t* off) except? -1
     execute_fn unpack_construct
     execute_fn unpack_skip
     execute_fn read_array_header
@@ -112,7 +112,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     See :class:`Unpacker` for options.
     """
     cdef unpack_context ctx
-    cdef size_t off = 0
+    cdef Py_ssize_t off = 0
     cdef int ret
 
     cdef char* buf
@@ -142,7 +142,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
             raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
         return obj
     else:
-        raise UnpackValueError("Unpack failed: error = %d" % (ret,))
+        raise UnpackValueError("Unpack failed: error = %s" % (ret,))
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None,
@@ -233,14 +233,14 @@ cdef class Unpacker(object):
     """
     cdef unpack_context ctx
     cdef char* buf
-    cdef size_t buf_size, buf_head, buf_tail
+    cdef Py_ssize_t buf_size, buf_head, buf_tail
     cdef object file_like
     cdef object file_like_read
     cdef Py_ssize_t read_size
     # To maintain refcnt.
     cdef object object_hook, object_pairs_hook, list_hook, ext_hook
     cdef object encoding, unicode_errors
-    cdef size_t max_buffer_size
+    cdef Py_ssize_t max_buffer_size
 
     def __cinit__(self):
         self.buf = NULL
@@ -325,10 +325,10 @@ cdef class Unpacker(object):
         cdef:
             char* buf = self.buf
             char* new_buf
-            size_t head = self.buf_head
-            size_t tail = self.buf_tail
-            size_t buf_size = self.buf_size
-            size_t new_size
+            Py_ssize_t head = self.buf_head
+            Py_ssize_t tail = self.buf_tail
+            Py_ssize_t buf_size = self.buf_size
+            Py_ssize_t new_size
 
         if tail + _buf_len > buf_size:
             if ((tail - head) + _buf_len) <= buf_size:
@@ -374,7 +374,7 @@ cdef class Unpacker(object):
     cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0):
         cdef int ret
         cdef object obj
-        cdef size_t prev_head
+        cdef Py_ssize_t prev_head
 
         if self.buf_head >= self.buf_tail and self.file_like is not None:
             self.read_from_file()
@@ -408,7 +408,7 @@ cdef class Unpacker(object):
 
     def read_bytes(self, Py_ssize_t nbytes):
         """Read a specified number of raw bytes from the stream"""
-        cdef size_t nread
+        cdef Py_ssize_t nread
         nread = min(self.buf_tail - self.buf_head, nbytes)
         ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread)
         self.buf_head += nread
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 5deb7cd..297bc93 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -33,7 +33,7 @@ typedef struct unpack_user {
 typedef PyObject* msgpack_unpack_object;
 struct unpack_context;
 typedef struct unpack_context unpack_context;
-typedef int (*execute_fn)(unpack_context *ctx, const char* data, size_t len, size_t* off);
+typedef int (*execute_fn)(unpack_context *ctx, const char* data, Py_ssize_t len, Py_ssize_t* off);
 
 static inline msgpack_unpack_object unpack_callback_root(unpack_user* u)
 {
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index d34eced..5b389b8 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -24,8 +24,8 @@
 
 typedef struct unpack_stack {
     PyObject* obj;
-    size_t size;
-    size_t count;
+    Py_ssize_t size;
+    Py_ssize_t count;
     unsigned int ct;
     PyObject* map_key;
 } unpack_stack;
@@ -72,7 +72,7 @@ static inline PyObject* unpack_data(unpack_context* ctx)
 
 
 template 
-static inline int unpack_execute(unpack_context* ctx, const char* data, size_t len, size_t* off)
+static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off)
 {
     assert(len >= *off);
 
@@ -89,7 +89,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, size_t l
     */
     unpack_user* user = &ctx->user;
 
-    PyObject* obj;
+    PyObject* obj = NULL;
     unpack_stack* c = NULL;
 
     int ret;
@@ -409,7 +409,7 @@ _end:
 #undef start_container
 
 template 
-static inline int unpack_container_header(unpack_context* ctx, const char* data, size_t len, size_t* off)
+static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off)
 {
     assert(len >= *off);
     uint32_t size;

From 35a69ac9c2fbf6b68b970352791f6d98fbd74963 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 8 Nov 2015 12:38:38 +0900
Subject: [PATCH 1306/1648] Decrease refcnt when error happend while unpacking

Fixes #152
---
 msgpack/_unpacker.pyx     | 5 +++--
 msgpack/unpack_template.h | 4 ++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 997979c..d359e57 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -47,6 +47,7 @@ cdef extern from "unpack.h":
     execute_fn read_map_header
     void unpack_init(unpack_context* ctx)
     object unpack_data(unpack_context* ctx)
+    void unpack_clear(unpack_context* ctx)
 
 cdef inline init_ctx(unpack_context *ctx,
                      object object_hook, object object_pairs_hook,
@@ -141,8 +142,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         if off < buf_len:
             raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
         return obj
-    else:
-        raise UnpackValueError("Unpack failed: error = %s" % (ret,))
+    unpack_clear(&ctx)
+    raise UnpackValueError("Unpack failed: error = %d" % (ret,))
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None,
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 5b389b8..6b83d3e 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -70,6 +70,10 @@ static inline PyObject* unpack_data(unpack_context* ctx)
     return (ctx)->stack[0].obj;
 }
 
+static inline PyObject* unpack_clear(unpack_context *ctx)
+{
+    Py_CLEAR(ctx->stack[0].obj);
+}
 
 template 
 static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off)

From 02611afd5f32aa173f9ba8a777ba49fa6b4f67c5 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 8 Nov 2015 17:29:09 +0900
Subject: [PATCH 1307/1648] Drpo pypip.in badge

It downs so long
---
 README.rst | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/README.rst b/README.rst
index ba2c1b5..baa825a 100644
--- a/README.rst
+++ b/README.rst
@@ -9,9 +9,6 @@ MessagePack for Python
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
    :target: https://travis-ci.org/#!/msgpack/msgpack-python
    
-.. image:: https://pypip.in/version/msgpack-python/badge.svg
-    :target: https://pypi.python.org/pypi/msgpack-python/
-    :alt: Latest Version
 
 What's this
 ------------

From 6f208abbc7c9567145bf8c17bc41ba7c30ade0be Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 8 Nov 2015 17:34:52 +0900
Subject: [PATCH 1308/1648] Update Windows compiler information

---
 README.rst | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index baa825a..456de2b 100644
--- a/README.rst
+++ b/README.rst
@@ -33,11 +33,15 @@ Windows
 ^^^^^^^
 
 When you can't use binary distribution, you need to install Visual Studio
-or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support
-amd64. Windows SDK is recommended way to build amd64 msgpack without any fee.)
-
+or Windows SDK on Windows.
 Without extension, using pure python implementation on CPython runs slowly.
 
+For Python 2.7, [Microsoft Visual C++ Compiler for Python 2.7](https://www.microsoft.com/en-us/download/details.aspx?id=44266)
+is recommended solution.
+
+For Python 3.5, [Microsoft Visual Studio 2015](https://www.visualstudio.com/en-us/products/vs-2015-product-editions.aspx)
+Community Edition or Express Edition can be used to build extension module.
+
 Notes
 -----
 

From 53fcd9b9df1c1f84719dfc9744217a1c3a025ef5 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 8 Nov 2015 19:37:40 +0900
Subject: [PATCH 1309/1648] Update gitignore

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index 1bd68b4..70f5746 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,6 @@ dist/*
 msgpack/__version__.py
 msgpack/*.cpp
 *.egg-info
+/venv
+/tags
+/docs/_build

From e601ef4c23c6bfa64f86dcb91d28f370e77b17bc Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 9 Nov 2015 00:43:52 +0900
Subject: [PATCH 1310/1648] Remove `msgpack 2.0` from README

There are no versio in spec.
---
 README.rst | 109 +++++++++++++++++++++++++++++------------------------
 1 file changed, 59 insertions(+), 50 deletions(-)

diff --git a/README.rst b/README.rst
index 456de2b..30453bd 100644
--- a/README.rst
+++ b/README.rst
@@ -42,54 +42,6 @@ is recommended solution.
 For Python 3.5, [Microsoft Visual Studio 2015](https://www.visualstudio.com/en-us/products/vs-2015-product-editions.aspx)
 Community Edition or Express Edition can be used to build extension module.
 
-Notes
------
-
-Note for msgpack 2.0 support
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-msgpack 2.0 adds two types: *bin* and *ext*.
-
-*raw* was bytes or string type like Python 2's ``str``.
-To distinguish string and bytes, msgpack 2.0 adds *bin*.
-It is non-string binary like Python 3's ``bytes``.
-
-To use *bin* type for packing ``bytes``, pass ``use_bin_type=True`` to
-packer argument.
-
-.. code-block:: pycon
-
-    >>> import msgpack
-    >>> packed = msgpack.packb([b'spam', u'egg'], use_bin_type=True)
-    >>> msgpack.unpackb(packed, encoding='utf-8')
-    ['spam', u'egg']
-
-You shoud use it carefully. When you use ``use_bin_type=True``, packed
-binary can be unpacked by unpackers supporting msgpack-2.0.
-
-To use *ext* type, pass ``msgpack.ExtType`` object to packer.
-
-.. code-block:: pycon
-
-    >>> import msgpack
-    >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy'))
-    >>> msgpack.unpackb(packed)
-    ExtType(code=42, data='xyzzy')
-
-You can use it with ``default`` and ``ext_hook``. See below.
-
-Note for msgpack 0.2.x users
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The msgpack 0.3 have some incompatible changes.
-
-The default value of ``use_list`` keyword argument is ``True`` from 0.3.
-You should pass the argument explicitly for backward compatibility.
-
-`Unpacker.unpack()` and some unpack methods now raises `OutOfData`
-instead of `StopIteration`.
-`StopIteration` is used for iterator protocol only.
-
 
 How to use
 -----------
@@ -184,7 +136,7 @@ key-value pairs.
 Extended types
 ^^^^^^^^^^^^^^^
 
-It is also possible to pack/unpack custom data types using the msgpack 2.0 feature.
+It is also possible to pack/unpack custom data types using the **ext** type.
 
 .. code-block:: pycon
 
@@ -238,6 +190,58 @@ callback function:
             unpacker.skip(bytestream.write)
             worker.send(bytestream.getvalue())
 
+
+Notes
+-----
+
+string and binary type
+^^^^^^^^^^^^^^^^^^^^^^
+
+In old days, msgpack doesn't distinguish string and binary types like Python 1.
+The type for represent string and binary types is named **raw**.
+
+msgpack can distinguish string and binary type for now.  But it is not like Python 2.
+Python 2 added unicode string.  But msgpack renamed **raw** to **str** and added **bin** type.
+It is because keep compatibility with data created by old libs. **raw** was used for text more than binary.
+
+Currently, while msgpack-python supports new **bin** type, default setting doesn't use it and
+decodes **raw** as `bytes` instead of `unicode` (`str` in Python 3).
+
+You can change this by using `use_bin_type=True` option in Packer and `encoding="utf-8"` option in Unpacker.
+
+.. code-block:: pycon
+
+    >>> import msgpack
+    >>> packed = msgpack.packb([b'spam', u'egg'], use_bin_type=True)
+    >>> msgpack.unpackb(packed, encoding='utf-8')
+    ['spam', u'egg']
+
+ext type
+^^^^^^^^
+
+To use **ext** type, pass ``msgpack.ExtType`` object to packer.
+
+.. code-block:: pycon
+
+    >>> import msgpack
+    >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy'))
+    >>> msgpack.unpackb(packed)
+    ExtType(code=42, data='xyzzy')
+
+You can use it with ``default`` and ``ext_hook``. See below.
+
+Note for msgpack-python 0.2.x users
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The msgpack-python 0.3 have some incompatible changes.
+
+The default value of ``use_list`` keyword argument is ``True`` from 0.3.
+You should pass the argument explicitly for backward compatibility.
+
+`Unpacker.unpack()` and some unpack methods now raises `OutOfData`
+instead of `StopIteration`.
+`StopIteration` is used for iterator protocol only.
+
 Note about performance
 ------------------------
 
@@ -259,12 +263,17 @@ Python's dict can't use list as key and MessagePack allows array for key of mapp
 Another way to unpacking such object is using ``object_pairs_hook``.
 
 
+Development
+------------
+
 Test
-----
+^^^^
+
 MessagePack uses `pytest` for testing.
 Run test with following command:
 
     $ py.test
 
+
 ..
     vim: filetype=rst

From 8aadc5c380d0d135273729333ace91d3f689702d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 9 Nov 2015 00:50:07 +0900
Subject: [PATCH 1311/1648] readme: Fix markup

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 30453bd..d32ec1d 100644
--- a/README.rst
+++ b/README.rst
@@ -36,10 +36,10 @@ When you can't use binary distribution, you need to install Visual Studio
 or Windows SDK on Windows.
 Without extension, using pure python implementation on CPython runs slowly.
 
-For Python 2.7, [Microsoft Visual C++ Compiler for Python 2.7](https://www.microsoft.com/en-us/download/details.aspx?id=44266)
+For Python 2.7, `Microsoft Visual C++ Compiler for Python 2.7 `_
 is recommended solution.
 
-For Python 3.5, [Microsoft Visual Studio 2015](https://www.visualstudio.com/en-us/products/vs-2015-product-editions.aspx)
+For Python 3.5, `Microsoft Visual Studio 2015 `_
 Community Edition or Express Edition can be used to build extension module.
 
 

From 3cef27b69b250f9abf126c85578a821c2d21e72e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 9 Nov 2015 00:54:06 +0900
Subject: [PATCH 1312/1648] Update ChangeLog

---
 ChangeLog.rst | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 34f4cd4..31a64d9 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,20 @@
+0.4.7
+=====
+:release date: TBD
+
+Bugs fixed
+----------
+
+* Memory leak when unpack is failed
+
+Changes
+-------
+
+* Reduce compiler warnings while building extension module
+* unpack() now accepts ext_hook argument like Unpacker and unpackb()
+* Update Cython version to 0.23.4
+
+
 0.4.6
 =====
 :release date: 2015-03-13

From de3c2b99f78d134c326bc375f19f54b7c851797a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 9 Nov 2015 01:50:40 +0900
Subject: [PATCH 1313/1648] refactor C code

fixes #137
---
 msgpack/pack.h            | 34 ----------------------------------
 msgpack/unpack.h          |  4 ++--
 msgpack/unpack_template.h |  2 +-
 3 files changed, 3 insertions(+), 37 deletions(-)

diff --git a/msgpack/pack.h b/msgpack/pack.h
index 971065c..a75bdb0 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -39,40 +39,6 @@ typedef struct msgpack_packer {
 
 typedef struct Packer Packer;
 
-static inline int msgpack_pack_int(msgpack_packer* pk, int d);
-static inline int msgpack_pack_long(msgpack_packer* pk, long d);
-static inline int msgpack_pack_long_long(msgpack_packer* pk, long long d);
-static inline int msgpack_pack_unsigned_short(msgpack_packer* pk, unsigned short d);
-static inline int msgpack_pack_unsigned_int(msgpack_packer* pk, unsigned int d);
-static inline int msgpack_pack_unsigned_long(msgpack_packer* pk, unsigned long d);
-//static inline int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d);
-
-static inline int msgpack_pack_uint8(msgpack_packer* pk, uint8_t d);
-static inline int msgpack_pack_uint16(msgpack_packer* pk, uint16_t d);
-static inline int msgpack_pack_uint32(msgpack_packer* pk, uint32_t d);
-static inline int msgpack_pack_uint64(msgpack_packer* pk, uint64_t d);
-static inline int msgpack_pack_int8(msgpack_packer* pk, int8_t d);
-static inline int msgpack_pack_int16(msgpack_packer* pk, int16_t d);
-static inline int msgpack_pack_int32(msgpack_packer* pk, int32_t d);
-static inline int msgpack_pack_int64(msgpack_packer* pk, int64_t d);
-
-static inline int msgpack_pack_float(msgpack_packer* pk, float d);
-static inline int msgpack_pack_double(msgpack_packer* pk, double d);
-
-static inline int msgpack_pack_nil(msgpack_packer* pk);
-static inline int msgpack_pack_true(msgpack_packer* pk);
-static inline int msgpack_pack_false(msgpack_packer* pk);
-
-static inline int msgpack_pack_array(msgpack_packer* pk, unsigned int n);
-
-static inline int msgpack_pack_map(msgpack_packer* pk, unsigned int n);
-
-static inline int msgpack_pack_raw(msgpack_packer* pk, size_t l);
-static inline int msgpack_pack_bin(msgpack_packer* pk, size_t l);
-static inline int msgpack_pack_raw_body(msgpack_packer* pk, const void* b, size_t l);
-
-static inline int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l);
-
 static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_t l)
 {
     char* buf = pk->buf;
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 297bc93..92f4f11 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -69,7 +69,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp
     if (d > LONG_MAX) {
         p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d);
     } else {
-        p = PyInt_FromSize_t((size_t)d);
+        p = PyInt_FromLong((long)d);
     }
     if (!p)
         return -1;
@@ -100,7 +100,7 @@ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpac
 {
     PyObject *p;
     if (d > LONG_MAX || d < LONG_MIN) {
-        p = PyLong_FromLongLong((unsigned PY_LONG_LONG)d);
+        p = PyLong_FromLongLong((PY_LONG_LONG)d);
     } else {
         p = PyInt_FromLong((long)d);
     }
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 6b83d3e..e1e08fe 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -70,7 +70,7 @@ static inline PyObject* unpack_data(unpack_context* ctx)
     return (ctx)->stack[0].obj;
 }
 
-static inline PyObject* unpack_clear(unpack_context *ctx)
+static inline void unpack_clear(unpack_context *ctx)
 {
     Py_CLEAR(ctx->stack[0].obj);
 }

From f7d3715f2cedb09babbcdd1950ecc50f4d673fbe Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 9 Nov 2015 02:00:48 +0900
Subject: [PATCH 1314/1648] Add missing params to unpack()

---
 msgpack/_unpacker.pyx | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index d359e57..36cb78e 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -148,7 +148,12 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
 def unpack(object stream, object object_hook=None, object list_hook=None,
            bint use_list=1, encoding=None, unicode_errors="strict",
-           object_pairs_hook=None, ext_hook=ExtType
+           object_pairs_hook=None, ext_hook=ExtType,
+           Py_ssize_t max_str_len=2147483647, # 2**32-1
+           Py_ssize_t max_bin_len=2147483647,
+           Py_ssize_t max_array_len=2147483647,
+           Py_ssize_t max_map_len=2147483647,
+           Py_ssize_t max_ext_len=2147483647):
            ):
     """
     Unpack an object from `stream`.
@@ -159,7 +164,12 @@ def unpack(object stream, object object_hook=None, object list_hook=None,
     """
     return unpackb(stream.read(), use_list=use_list,
                    object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
-                   encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook
+                   encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook,
+                   max_str_len=max_str_len,
+                   max_bin_len=max_bin_len,
+                   max_array_len=max_array_len,
+                   max_map_len=max_map_len,
+                   max_ext_len=max_ext_len,
                    )
 
 

From 7d900371c8f13fa64f06aaf336b6ae65c705bf2c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 9 Nov 2015 02:09:39 +0900
Subject: [PATCH 1315/1648] Fix compile error

---
 msgpack/_unpacker.pyx | 1 -
 1 file changed, 1 deletion(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 36cb78e..aec3b7d 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -154,7 +154,6 @@ def unpack(object stream, object object_hook=None, object list_hook=None,
            Py_ssize_t max_array_len=2147483647,
            Py_ssize_t max_map_len=2147483647,
            Py_ssize_t max_ext_len=2147483647):
-           ):
     """
     Unpack an object from `stream`.
 

From a1317b604f7eb83575c4a0221db028c1929e8026 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 9 Nov 2015 02:34:17 +0900
Subject: [PATCH 1316/1648] refactor

---
 msgpack/_packer.pyx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 7129208..0a6513c 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -135,7 +135,7 @@ cdef class Packer(object):
                     ret = msgpack_pack_false(&self.pk)
             elif PyLong_Check(o):
                 # PyInt_Check(long) is True for Python 3.
-                # Sow we should test long before int.
+                # So we should test long before int.
                 try:
                     if o > 0:
                         ullval = o
@@ -143,7 +143,7 @@ cdef class Packer(object):
                     else:
                         llval = o
                         ret = msgpack_pack_long_long(&self.pk, llval)
-                except OverflowError, oe:
+                except OverflowError as oe:
                     if not default_used and self._default is not None:
                         o = self._default(o)
                         default_used = True

From 29266b024ec82c281fa63bf2b73622088a8c188c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 9 Nov 2015 02:34:28 +0900
Subject: [PATCH 1317/1648] Update ChangeLog

---
 ChangeLog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 31a64d9..35535b4 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -13,6 +13,7 @@ Changes
 * Reduce compiler warnings while building extension module
 * unpack() now accepts ext_hook argument like Unpacker and unpackb()
 * Update Cython version to 0.23.4
+* default function is called when integer overflow
 
 
 0.4.6

From cbdf3c339a2c4c6b40a7e75dd96e457a024937f6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 10 Nov 2015 03:30:11 +0900
Subject: [PATCH 1318/1648] s/precise_mode/strict_types/

---
 msgpack/_packer.pyx | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index c015fcb..50d19ff 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -55,7 +55,7 @@ cdef class Packer(object):
         Convert unicode to bytes with this encoding. (default: 'utf-8')
     :param str unicode_errors:
         Error handler for encoding unicode. (default: 'strict')
-    :param bool precise_mode:
+    :param bool strict_types:
         If set to true, types will be checked to be exact. Derived classes
         from serializeable types will not be serialized and will be
         treated as unsupported type and forwarded to default.
@@ -77,7 +77,7 @@ cdef class Packer(object):
     cdef object _berrors
     cdef char *encoding
     cdef char *unicode_errors
-    cdef bint precise_mode
+    cdef bint strict_types
     cdef bool use_float
     cdef bint autoreset
 
@@ -91,11 +91,11 @@ cdef class Packer(object):
 
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
                  use_single_float=False, bint autoreset=1, bint use_bin_type=0,
-                 bint precise_mode=0):
+                 bint strict_types=0):
         """
         """
         self.use_float = use_single_float
-        self.precise_mode = precise_mode
+        self.strict_types = strict_types
         self.autoreset = autoreset
         self.pk.use_bin_type = use_bin_type
         if default is not None:
@@ -131,7 +131,7 @@ cdef class Packer(object):
         cdef dict d
         cdef size_t L
         cdef int default_used = 0
-        cdef bint precise = self.precise_mode
+        cdef bint strict_types = self.strict_types
 
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded.")
@@ -139,12 +139,12 @@ cdef class Packer(object):
         while True:
             if o is None:
                 ret = msgpack_pack_nil(&self.pk)
-            elif PyBool_Check(o) if precise else isinstance(o, bool):
+            elif PyBool_Check(o) if strict_types else isinstance(o, bool):
                 if o:
                     ret = msgpack_pack_true(&self.pk)
                 else:
                     ret = msgpack_pack_false(&self.pk)
-            elif PyLong_CheckExact(o) if precise else PyLong_Check(o):
+            elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o):
                 # PyInt_Check(long) is True for Python 3.
                 # So we should test long before int.
                 try:
@@ -161,17 +161,17 @@ cdef class Packer(object):
                         continue
                     else:
                         raise
-            elif PyInt_CheckExact(o) if precise else PyInt_Check(o):
+            elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o):
                 longval = o
                 ret = msgpack_pack_long(&self.pk, longval)
-            elif PyFloat_CheckExact(o) if precise else PyFloat_Check(o):
+            elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o):
                 if self.use_float:
                    fval = o
                    ret = msgpack_pack_float(&self.pk, fval)
                 else:
                    dval = o
                    ret = msgpack_pack_double(&self.pk, dval)
-            elif PyBytes_CheckExact(o) if precise else PyBytes_Check(o):
+            elif PyBytes_CheckExact(o) if strict_types else PyBytes_Check(o):
                 L = len(o)
                 if L > (2**32)-1:
                     raise ValueError("bytes is too large")
@@ -179,7 +179,7 @@ cdef class Packer(object):
                 ret = msgpack_pack_bin(&self.pk, L)
                 if ret == 0:
                     ret = msgpack_pack_raw_body(&self.pk, rawval, L)
-            elif PyUnicode_CheckExact(o) if precise else PyUnicode_Check(o):
+            elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o):
                 if not self.encoding:
                     raise TypeError("Can't encode unicode string: no encoding is specified")
                 o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
@@ -202,7 +202,7 @@ cdef class Packer(object):
                         if ret != 0: break
                         ret = self._pack(v, nest_limit-1)
                         if ret != 0: break
-            elif not precise and PyDict_Check(o):
+            elif not strict_types and PyDict_Check(o):
                 L = len(o)
                 if L > (2**32)-1:
                     raise ValueError("dict is too large")
@@ -222,7 +222,7 @@ cdef class Packer(object):
                     raise ValueError("EXT data is too large")
                 ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
-            elif PyList_CheckExact(o) if precise else (PyTuple_Check(o) or PyList_Check(o)):
+            elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
                 L = len(o)
                 if L > (2**32)-1:
                     raise ValueError("list is too large")

From 1032ef9bf2baef73b04f209181e42978ab4c71fe Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 10 Nov 2015 03:33:50 +0900
Subject: [PATCH 1319/1648] fallback unpacker: precise => strict

---
 msgpack/fallback.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 0e37f74..cffecca 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -69,6 +69,13 @@ TYPE_EXT                = 5
 DEFAULT_RECURSE_LIMIT = 511
 
 
+def _check_type_strict(obj, t, type=type, tuple=tuple):
+    if type(t) is tuple:
+        return type(obj) in t
+    else:
+        return type(obj) is t
+
+
 def unpack(stream, **kwargs):
     """
     Unpack an object from `stream`.
@@ -601,7 +608,7 @@ class Packer(object):
             Convert unicode to bytes with this encoding. (default: 'utf-8')
     :param str unicode_errors:
         Error handler for encoding unicode. (default: 'strict')
-    :param bool precise_mode:
+    :param bool strict_types:
         If set to true, types will be checked to be exact. Derived classes
         from serializeable types will not be serialized and will be
         treated as unsupported type and forwarded to default.
@@ -618,9 +625,9 @@ class Packer(object):
         It also enable str8 type for unicode.
     """
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
-                 precise_mode=False, use_single_float=False, autoreset=True,
+                 strict_types=False, use_single_float=False, autoreset=True,
                  use_bin_type=False):
-        self._precise_mode = precise_mode
+        self._strict_types = strict_types
         self._use_float = use_single_float
         self._autoreset = autoreset
         self._use_bin_type = use_bin_type
@@ -632,17 +639,11 @@ class Packer(object):
                 raise TypeError("default must be callable")
         self._default = default
 
-    def _check_precise(obj, t, type=type, tuple=tuple):
-        if type(t) is tuple:
-            return type(obj) in t
-        else:
-            return type(obj) is t
-
     def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT,
-              check=isinstance, check_precise=_check_precise):
+              check=isinstance, check_type_strict=_check_type_strict):
         default_used = False
-        if self._precise_mode:
-            check = check_precise
+        if self._strict_types:
+            check = check_type_strict
             list_types = list
         else:
             list_types = (list, tuple)

From 9b673279d36468e3334b513b5e86d40cba4c4acc Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 10 Nov 2015 03:37:54 +0900
Subject: [PATCH 1320/1648] strict_types should be last argument

---
 msgpack/_packer.pyx | 14 +++++++-------
 msgpack/fallback.py | 18 +++++++++---------
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 50d19ff..c8d4fd1 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -55,13 +55,6 @@ cdef class Packer(object):
         Convert unicode to bytes with this encoding. (default: 'utf-8')
     :param str unicode_errors:
         Error handler for encoding unicode. (default: 'strict')
-    :param bool strict_types:
-        If set to true, types will be checked to be exact. Derived classes
-        from serializeable types will not be serialized and will be
-        treated as unsupported type and forwarded to default.
-        Additionally tuples will not be serialized as lists.
-        This is useful when trying to implement accurate serialization 
-        for python types.
     :param bool use_single_float:
         Use single precision float type for float. (default: False)
     :param bool autoreset:
@@ -70,6 +63,13 @@ cdef class Packer(object):
     :param bool use_bin_type:
         Use bin type introduced in msgpack spec 2.0 for bytes.
         It also enable str8 type for unicode.
+    :param bool strict_types:
+        If set to true, types will be checked to be exact. Derived classes
+        from serializeable types will not be serialized and will be
+        treated as unsupported type and forwarded to default.
+        Additionally tuples will not be serialized as lists.
+        This is useful when trying to implement accurate serialization
+        for python types.
     """
     cdef msgpack_packer pk
     cdef object _default
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index cffecca..11280ed 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -608,13 +608,6 @@ class Packer(object):
             Convert unicode to bytes with this encoding. (default: 'utf-8')
     :param str unicode_errors:
         Error handler for encoding unicode. (default: 'strict')
-    :param bool strict_types:
-        If set to true, types will be checked to be exact. Derived classes
-        from serializeable types will not be serialized and will be
-        treated as unsupported type and forwarded to default.
-        Additionally tuples will not be serialized as lists.
-        This is useful when trying to implement accurate serialization 
-        for python types.
     :param bool use_single_float:
         Use single precision float type for float. (default: False)
     :param bool autoreset:
@@ -623,10 +616,17 @@ class Packer(object):
     :param bool use_bin_type:
         Use bin type introduced in msgpack spec 2.0 for bytes.
         It also enable str8 type for unicode.
+    :param bool strict_types:
+        If set to true, types will be checked to be exact. Derived classes
+        from serializeable types will not be serialized and will be
+        treated as unsupported type and forwarded to default.
+        Additionally tuples will not be serialized as lists.
+        This is useful when trying to implement accurate serialization
+        for python types.
     """
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
-                 strict_types=False, use_single_float=False, autoreset=True,
-                 use_bin_type=False):
+                 use_single_float=False, autoreset=True, use_bin_type=False,
+                 strict_types=False):
         self._strict_types = strict_types
         self._use_float = use_single_float
         self._autoreset = autoreset

From 628c5191873148a8372bae4ee99454ad13b7b492 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 10 Nov 2015 03:41:09 +0900
Subject: [PATCH 1321/1648] strict type check for ext type

---
 msgpack/_packer.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index c8d4fd1..7c1e53d 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -213,7 +213,7 @@ cdef class Packer(object):
                         if ret != 0: break
                         ret = self._pack(v, nest_limit-1)
                         if ret != 0: break
-            elif isinstance(o, ExtType):
+            elif type(o) is ExtType if strict_types else isinstance(o, ExtType):
                 # This should be before Tuple because ExtType is namedtuple.
                 longval = o.code
                 rawval = o.data

From 6f38bf7dd4c45523c3b0e9e8af0e69d5af35708b Mon Sep 17 00:00:00 2001
From: Omer Katz 
Date: Tue, 17 Nov 2015 15:31:36 +0200
Subject: [PATCH 1322/1648] Added python 3.5 to tox.ini.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 15feb51..56ded49 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {py26,py27,py32,py33,py34}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
+envlist = {py26,py27,py32,py33,py34,py35}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
 
 [variants:pure]
 setenv=

From 4d9684db0a7b2187f54f7a81a4ea64e981fa11ca Mon Sep 17 00:00:00 2001
From: Omer Katz 
Date: Tue, 17 Nov 2015 15:32:34 +0200
Subject: [PATCH 1323/1648] Added Python 3.5 to the build matrix.

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7695184..22f19dc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,9 +10,9 @@ branches:
 
 env:
     - TOXENV=py26-c,py27-c
-    - TOXENV=py32-c,py33-c,py34-c
+    - TOXENV=py32-c,py33-c,py34-c,py35-c
     - TOXENV=py26-pure,py27-pure
-    - TOXENV=py32-pure,py33-pure,py34-pure
+    - TOXENV=py32-pure,py33-pure,py34-pure,py35-pure
     - TOXENV=pypy-pure,pypy3-pure
 
 install:

From 81177caff7ddc9b193da9fcfd27fcd6807da7fe1 Mon Sep 17 00:00:00 2001
From: Omer Katz 
Date: Tue, 17 Nov 2015 16:57:25 +0200
Subject: [PATCH 1324/1648] Run the build with 3.5 since it's still not
 available by default in travis.

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 22f19dc..7a80cdf 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,7 +2,7 @@ sudo: false
 cache: pip
 language: python
 python:
-    - 2.7
+    - 3.5
 
 branches:
     only:

From e4aa43d76935ac44aecccecd30fb51d00854ffac Mon Sep 17 00:00:00 2001
From: Omer Katz 
Date: Tue, 17 Nov 2015 17:08:04 +0200
Subject: [PATCH 1325/1648] Travis will now cache dependencies despite having a
 custom install step.

---
 .travis.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 7695184..bc0efba 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,7 @@
 sudo: false
-cache: pip
+cache:
+  directories:
+    - $HOME/.cache/pip
 language: python
 python:
     - 2.7

From 9c6584ee101d300fdf0621cac44c3dcb9df817ba Mon Sep 17 00:00:00 2001
From: Thomas Waldmann 
Date: Wed, 9 Dec 2015 13:53:42 +0100
Subject: [PATCH 1326/1648] fix typos

---
 msgpack/_unpacker.pyx | 2 +-
 msgpack/fallback.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index d53f724..b98957e 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -199,7 +199,7 @@ cdef class Unpacker(object):
     :param int max_buffer_size:
         Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
         Raises `BufferFull` exception when it is insufficient.
-        You shoud set this parameter when unpacking data from untrasted source.
+        You shoud set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
         Limits max length of str. (default: 2**31-1)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 235c201..5b5085e 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -138,7 +138,7 @@ class Unpacker(object):
     :param int max_buffer_size:
         Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
         Raises `BufferFull` exception when it is insufficient.
-        You shoud set this parameter when unpacking data from untrasted source.
+        You shoud set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
         Limits max length of str. (default: 2**31-1)

From 83424bd7b3c4a417fe1dca4523adc177cee1bef9 Mon Sep 17 00:00:00 2001
From: Sadayuki Furuhashi 
Date: Mon, 11 Jan 2016 13:57:33 -0800
Subject: [PATCH 1327/1648] Fix wrong 'dict is too large' on unicode string

---
 msgpack/_packer.pyx | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 0a6513c..6392655 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -174,11 +174,11 @@ cdef class Packer(object):
                 o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
                 L = len(o)
                 if L > (2**32)-1:
-                    raise ValueError("dict is too large")
+                    raise ValueError("unicode string is too large")
                 rawval = o
-                ret = msgpack_pack_raw(&self.pk, len(o))
+                ret = msgpack_pack_raw(&self.pk, L)
                 if ret == 0:
-                    ret = msgpack_pack_raw_body(&self.pk, rawval, len(o))
+                    ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyDict_CheckExact(o):
                 d = o
                 L = len(d)

From 1f8240eaf65b28e93621a8e35f1078d4292047f1 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 25 Jan 2016 01:10:50 +0900
Subject: [PATCH 1328/1648] 0.4.7

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index 2c1c96c..37c172d 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 4, 6)
+version = (0, 4, 7)

From 3a8bb070f76af211356fef9e0395e1429e81a9b2 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 25 Jan 2016 01:12:56 +0900
Subject: [PATCH 1329/1648] Update ChangeLog

---
 ChangeLog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 35535b4..f20bb75 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,6 @@
 0.4.7
 =====
-:release date: TBD
+:release date: 2016-01-25
 
 Bugs fixed
 ----------

From 005739388d4a611979ae029fab2a1982a366a285 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 25 Jan 2016 01:15:51 +0900
Subject: [PATCH 1330/1648] Drop Python 2.6, 3.2 support

---
 .travis.yml | 6 ++----
 tox.ini     | 3 +--
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2ba2caa..eced353 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,10 +11,8 @@ branches:
         - master
 
 env:
-    - TOXENV=py26-c,py27-c
-    - TOXENV=py32-c,py33-c,py34-c,py35-c
-    - TOXENV=py26-pure,py27-pure
-    - TOXENV=py32-pure,py33-pure,py34-pure,py35-pure
+    - TOXENV=py27-c,py33-c,py34-c,py35-c
+    - TOXENV=py27-pure,py33-pure,py34-pure,py35-pure
     - TOXENV=pypy-pure,pypy3-pure
 
 install:
diff --git a/tox.ini b/tox.ini
index 56ded49..b6e7a7f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {py26,py27,py32,py33,py34,py35}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
+envlist = {py27,py33,py34,py35}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
 
 [variants:pure]
 setenv=
@@ -36,4 +36,3 @@ commands=
     python -c 'import sys; print(hex(sys.maxsize))'
     python -c 'from msgpack import _packer, _unpacker'
     py.test
-

From a779b79b47529f84cd71593f284788d939226d66 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 25 Jan 2016 02:18:25 +0900
Subject: [PATCH 1331/1648] Add test for strict_types option

---
 test/test_stricttype.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 test/test_stricttype.py

diff --git a/test/test_stricttype.py b/test/test_stricttype.py
new file mode 100644
index 0000000..a20b5eb
--- /dev/null
+++ b/test/test_stricttype.py
@@ -0,0 +1,15 @@
+# coding: utf-8
+
+from collections import namedtuple
+from msgpack import packb, unpackb
+
+
+def test_namedtuple():
+    T = namedtuple('T', "foo bar")
+    def default(o):
+        if isinstance(o, T):
+            return dict(o._asdict())
+        raise TypeError('Unsupported type %s' % (type(o),))
+    packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default)
+    unpacked = unpackb(packed, encoding='utf-8')
+    assert unpacked == {'foo': 1, 'bar': 42}

From 31adc5a3c09a5f3506db192e1fb8b7ca4b72d974 Mon Sep 17 00:00:00 2001
From: folz 
Date: Thu, 12 Nov 2015 11:49:19 +0100
Subject: [PATCH 1332/1648] Support packing memoryview objects

---
 msgpack/_packer.pyx     | 17 +++++++++++++++++
 msgpack/fallback.py     |  6 ++++--
 test/test_memoryview.py | 12 ++++++++++++
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 test/test_memoryview.py

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index c3ef1a4..b19d462 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -10,6 +10,11 @@ from msgpack.exceptions import PackValueError
 from msgpack import ExtType
 
 
+cdef extern from "Python.h":
+
+    int PyMemoryView_Check(object obj)
+
+
 cdef extern from "pack.h":
     struct msgpack_packer:
         char* buf
@@ -132,6 +137,7 @@ cdef class Packer(object):
         cdef size_t L
         cdef int default_used = 0
         cdef bint strict_types = self.strict_types
+        cdef Py_buffer view
 
         if nest_limit < 0:
             raise PackValueError("recursion limit exceeded.")
@@ -231,6 +237,17 @@ cdef class Packer(object):
                     for v in o:
                         ret = self._pack(v, nest_limit-1)
                         if ret != 0: break
+            elif PyMemoryView_Check(o):
+                if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0:
+                    raise ValueError("could not get buffer for memoryview")
+                L = view.len
+                if L > (2**32)-1:
+                    PyBuffer_Release(&view);
+                    raise ValueError("memoryview is too large")
+                ret = msgpack_pack_bin(&self.pk, L)
+                if ret == 0:
+                    ret = msgpack_pack_raw_body(&self.pk, view.buf, L)
+                PyBuffer_Release(&view);
             elif not default_used and self._default:
                 o = self._default(o)
                 default_used = 1
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 40c54a8..348e017 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -36,6 +36,8 @@ if hasattr(sys, 'pypy_version_info'):
             else:
                 self.builder = StringBuilder()
         def write(self, s):
+            if isinstance(s, memoryview):
+                s = s.tobytes()
             self.builder.append(s)
         def getvalue(self):
             return self.builder.build()
@@ -682,7 +684,7 @@ class Packer(object):
                     default_used = True
                     continue
                 raise PackValueError("Integer value out of range")
-            if self._use_bin_type and check(obj, bytes):
+            if self._use_bin_type and check(obj, (bytes, memoryview)):
                 n = len(obj)
                 if n <= 0xff:
                     self._buffer.write(struct.pack('>BB', 0xc4, n))
@@ -693,7 +695,7 @@ class Packer(object):
                 else:
                     raise PackValueError("Bytes is too large")
                 return self._buffer.write(obj)
-            if check(obj, (Unicode, bytes)):
+            if check(obj, (Unicode, bytes, memoryview)):
                 if check(obj, Unicode):
                     if self._encoding is None:
                         raise TypeError(
diff --git a/test/test_memoryview.py b/test/test_memoryview.py
new file mode 100644
index 0000000..aed5069
--- /dev/null
+++ b/test/test_memoryview.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+
+from msgpack import packb, unpackb
+
+
+def test_pack_memoryview():
+    data = bytearray(range(256))
+    view = memoryview(data)
+    unpacked = unpackb(packb(view))
+    assert data == unpacked

From 7d2d46effce37f9fbf394fac74d380aaa7c95f02 Mon Sep 17 00:00:00 2001
From: palaviv 
Date: Fri, 12 Feb 2016 11:00:39 +0200
Subject: [PATCH 1333/1648] msgpack pack and unpack throws only exception that
 inherit from MsgpackBaseException. cython and fallback throws same exceptions

---
 msgpack/_packer.pyx   | 14 ++++++------
 msgpack/exceptions.py | 12 ++++++++--
 msgpack/fallback.py   | 51 ++++++++++++++++++++++---------------------
 test/test_limits.py   | 46 +++++++++++++++++++++++---------------
 4 files changed, 71 insertions(+), 52 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index b19d462..8216813 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -6,7 +6,7 @@ from libc.stdlib cimport *
 from libc.string cimport *
 from libc.limits cimport *
 
-from msgpack.exceptions import PackValueError
+from msgpack.exceptions import PackValueError, PackOverflowError
 from msgpack import ExtType
 
 
@@ -166,7 +166,7 @@ cdef class Packer(object):
                         default_used = True
                         continue
                     else:
-                        raise
+                        raise PackOverflowError("Integer value out of range")
             elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o):
                 longval = o
                 ret = msgpack_pack_long(&self.pk, longval)
@@ -180,7 +180,7 @@ cdef class Packer(object):
             elif PyBytes_CheckExact(o) if strict_types else PyBytes_Check(o):
                 L = len(o)
                 if L > (2**32)-1:
-                    raise ValueError("bytes is too large")
+                    raise PackValueError("bytes is too large")
                 rawval = o
                 ret = msgpack_pack_bin(&self.pk, L)
                 if ret == 0:
@@ -191,7 +191,7 @@ cdef class Packer(object):
                 o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
                 L = len(o)
                 if L > (2**32)-1:
-                    raise ValueError("unicode string is too large")
+                    raise PackValueError("unicode string is too large")
                 rawval = o
                 ret = msgpack_pack_raw(&self.pk, L)
                 if ret == 0:
@@ -211,7 +211,7 @@ cdef class Packer(object):
             elif not strict_types and PyDict_Check(o):
                 L = len(o)
                 if L > (2**32)-1:
-                    raise ValueError("dict is too large")
+                    raise PackValueError("dict is too large")
                 ret = msgpack_pack_map(&self.pk, L)
                 if ret == 0:
                     for k, v in o.items():
@@ -225,13 +225,13 @@ cdef class Packer(object):
                 rawval = o.data
                 L = len(o.data)
                 if L > (2**32)-1:
-                    raise ValueError("EXT data is too large")
+                    raise PackValueError("EXT data is too large")
                 ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
                 L = len(o)
                 if L > (2**32)-1:
-                    raise ValueError("list is too large")
+                    raise PackValueError("list is too large")
                 ret = msgpack_pack_array(&self.pk, L)
                 if ret == 0:
                     for v in o:
diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py
index f7678f1..e0d5b5f 100644
--- a/msgpack/exceptions.py
+++ b/msgpack/exceptions.py
@@ -1,4 +1,8 @@
-class UnpackException(Exception):
+class MsgpackBaseException(Exception):
+    pass
+
+
+class UnpackException(MsgpackBaseException):
     pass
 
 
@@ -22,8 +26,12 @@ class ExtraData(ValueError):
     def __str__(self):
         return "unpack(b) received extra data."
 
-class PackException(Exception):
+class PackException(MsgpackBaseException):
     pass
 
 class PackValueError(PackException, ValueError):
     pass
+
+
+class PackOverflowError(PackValueError, OverflowError):
+    pass
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 348e017..d8c5d73 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -51,6 +51,7 @@ from msgpack.exceptions import (
     OutOfData,
     UnpackValueError,
     PackValueError,
+    PackOverflowError,
     ExtraData)
 
 from msgpack import ExtType
@@ -363,17 +364,17 @@ class Unpacker(object):
             obj = self._fb_read(n, write_bytes)
             typ = TYPE_RAW
             if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
         elif b & 0b11110000 == 0b10010000:
             n = b & 0b00001111
             typ = TYPE_ARRAY
             if n > self._max_array_len:
-                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+                raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
         elif b & 0b11110000 == 0b10000000:
             n = b & 0b00001111
             typ = TYPE_MAP
             if n > self._max_map_len:
-                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+                raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
         elif b == 0xc0:
             obj = None
         elif b == 0xc2:
@@ -384,37 +385,37 @@ class Unpacker(object):
             typ = TYPE_BIN
             n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
             if n > self._max_bin_len:
-                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
+                raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._fb_read(n, write_bytes)
         elif b == 0xc5:
             typ = TYPE_BIN
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
             if n > self._max_bin_len:
-                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
+                raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._fb_read(n, write_bytes)
         elif b == 0xc6:
             typ = TYPE_BIN
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             if n > self._max_bin_len:
-                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
+                raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._fb_read(n, write_bytes)
         elif b == 0xc7:  # ext 8
             typ = TYPE_EXT
             L, n = struct.unpack('Bb', self._fb_read(2, write_bytes))
             if L > self._max_ext_len:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._fb_read(L, write_bytes)
         elif b == 0xc8:  # ext 16
             typ = TYPE_EXT
             L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes))
             if L > self._max_ext_len:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._fb_read(L, write_bytes)
         elif b == 0xc9:  # ext 32
             typ = TYPE_EXT
             L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes))
             if L > self._max_ext_len:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._fb_read(L, write_bytes)
         elif b == 0xca:
             obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0]
@@ -439,65 +440,65 @@ class Unpacker(object):
         elif b == 0xd4:  # fixext 1
             typ = TYPE_EXT
             if self._max_ext_len < 1:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
             n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes))
         elif b == 0xd5:  # fixext 2
             typ = TYPE_EXT
             if self._max_ext_len < 2:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
             n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes))
         elif b == 0xd6:  # fixext 4
             typ = TYPE_EXT
             if self._max_ext_len < 4:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
             n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes))
         elif b == 0xd7:  # fixext 8
             typ = TYPE_EXT
             if self._max_ext_len < 8:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
             n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes))
         elif b == 0xd8:  # fixext 16
             typ = TYPE_EXT
             if self._max_ext_len < 16:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
+                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
             n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes))
         elif b == 0xd9:
             typ = TYPE_RAW
             n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
             if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._fb_read(n, write_bytes)
         elif b == 0xda:
             typ = TYPE_RAW
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
             if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._fb_read(n, write_bytes)
         elif b == 0xdb:
             typ = TYPE_RAW
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._fb_read(n, write_bytes)
         elif b == 0xdc:
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
             if n > self._max_array_len:
-                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+                raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
             typ = TYPE_ARRAY
         elif b == 0xdd:
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             if n > self._max_array_len:
-                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+                raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
             typ = TYPE_ARRAY
         elif b == 0xde:
             n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
             if n > self._max_map_len:
-                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+                raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         elif b == 0xdf:
             n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
             if n > self._max_map_len:
-                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+                raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         else:
             raise UnpackValueError("Unknown header: 0x%x" % b)
@@ -683,7 +684,7 @@ class Packer(object):
                     obj = self._default(obj)
                     default_used = True
                     continue
-                raise PackValueError("Integer value out of range")
+                raise PackOverflowError("Integer value out of range")
             if self._use_bin_type and check(obj, (bytes, memoryview)):
                 n = len(obj)
                 if n <= 0xff:
@@ -778,7 +779,7 @@ class Packer(object):
 
     def pack_array_header(self, n):
         if n >= 2**32:
-            raise ValueError
+            raise PackValueError
         self._fb_pack_array_header(n)
         ret = self._buffer.getvalue()
         if self._autoreset:
@@ -789,7 +790,7 @@ class Packer(object):
 
     def pack_map_header(self, n):
         if n >= 2**32:
-            raise ValueError
+            raise PackValueError
         self._fb_pack_map_header(n)
         ret = self._buffer.getvalue()
         if self._autoreset:
@@ -807,7 +808,7 @@ class Packer(object):
             raise TypeError("data must have bytes type")
         L = len(data)
         if L > 0xffffffff:
-            raise ValueError("Too large data")
+            raise PackValueError("Too large data")
         if L == 1:
             self._buffer.write(b'\xd4')
         elif L == 2:
diff --git a/test/test_limits.py b/test/test_limits.py
index 3c1cf2a..34acf7c 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -3,36 +3,42 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 import pytest
 
-from msgpack import packb, unpackb, Packer, Unpacker, ExtType
+from msgpack import packb, unpackb, Packer, Unpacker, ExtType, PackException, PackOverflowError, PackValueError
+from msgpack import UnpackValueError, UnpackException, MsgpackBaseException
 
 
-def test_integer():
+@pytest.mark.parametrize("expected_exception", [OverflowError, ValueError, PackOverflowError,
+                                                PackException, PackValueError, MsgpackBaseException])
+def test_integer(expected_exception):
     x = -(2 ** 63)
     assert unpackb(packb(x)) == x
-    with pytest.raises((OverflowError, ValueError)):
+    with pytest.raises(expected_exception):
         packb(x-1)
 
     x = 2 ** 64 - 1
     assert unpackb(packb(x)) == x
-    with pytest.raises((OverflowError, ValueError)):
+    with pytest.raises(expected_exception):
         packb(x+1)
 
 
-def test_array_header():
+@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError, MsgpackBaseException])
+def test_array_header(expected_exception):
     packer = Packer()
     packer.pack_array_header(2**32-1)
-    with pytest.raises((OverflowError, ValueError)):
+    with pytest.raises(expected_exception):
         packer.pack_array_header(2**32)
 
 
-def test_map_header():
+@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError, MsgpackBaseException])
+def test_map_header(expected_exception):
     packer = Packer()
     packer.pack_map_header(2**32-1)
-    with pytest.raises((OverflowError, ValueError)):
+    with pytest.raises(expected_exception):
         packer.pack_array_header(2**32)
 
 
-def test_max_str_len():
+@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException])
+def test_max_str_len(expected_exception):
     d = 'x' * 3
     packed = packb(d)
 
@@ -41,12 +47,13 @@ def test_max_str_len():
     assert unpacker.unpack() == d
 
     unpacker = Unpacker(max_str_len=2, encoding='utf-8')
-    with pytest.raises(ValueError):
+    with pytest.raises(expected_exception):
         unpacker.feed(packed)
         unpacker.unpack()
 
 
-def test_max_bin_len():
+@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException])
+def test_max_bin_len(expected_exception):
     d = b'x' * 3
     packed = packb(d, use_bin_type=True)
 
@@ -55,12 +62,13 @@ def test_max_bin_len():
     assert unpacker.unpack() == d
 
     unpacker = Unpacker(max_bin_len=2)
-    with pytest.raises(ValueError):
+    with pytest.raises(expected_exception):
         unpacker.feed(packed)
         unpacker.unpack()
 
 
-def test_max_array_len():
+@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException])
+def test_max_array_len(expected_exception):
     d = [1,2,3]
     packed = packb(d)
 
@@ -69,12 +77,13 @@ def test_max_array_len():
     assert unpacker.unpack() == d
 
     unpacker = Unpacker(max_array_len=2)
-    with pytest.raises(ValueError):
+    with pytest.raises(expected_exception):
         unpacker.feed(packed)
         unpacker.unpack()
 
 
-def test_max_map_len():
+@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException])
+def test_max_map_len(expected_exception):
     d = {1: 2, 3: 4, 5: 6}
     packed = packb(d)
 
@@ -83,12 +92,13 @@ def test_max_map_len():
     assert unpacker.unpack() == d
 
     unpacker = Unpacker(max_map_len=2)
-    with pytest.raises(ValueError):
+    with pytest.raises(expected_exception):
         unpacker.feed(packed)
         unpacker.unpack()
 
 
-def test_max_ext_len():
+@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException])
+def test_max_ext_len(expected_exception):
     d = ExtType(42, b"abc")
     packed = packb(d)
 
@@ -97,7 +107,7 @@ def test_max_ext_len():
     assert unpacker.unpack() == d
 
     unpacker = Unpacker(max_ext_len=2)
-    with pytest.raises(ValueError):
+    with pytest.raises(expected_exception):
         unpacker.feed(packed)
         unpacker.unpack()
 

From d44063119bf11fa5c4b559f9e246df60058bfe31 Mon Sep 17 00:00:00 2001
From: palaviv 
Date: Fri, 12 Feb 2016 15:36:48 +0200
Subject: [PATCH 1334/1648] changed more ValueErrors to PackValueError

---
 msgpack/_packer.pyx | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 8216813..e923895 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -200,7 +200,7 @@ cdef class Packer(object):
                 d = o
                 L = len(d)
                 if L > (2**32)-1:
-                    raise ValueError("dict is too large")
+                    raise PackValueError("dict is too large")
                 ret = msgpack_pack_map(&self.pk, L)
                 if ret == 0:
                     for k, v in d.iteritems():
@@ -239,11 +239,11 @@ cdef class Packer(object):
                         if ret != 0: break
             elif PyMemoryView_Check(o):
                 if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0:
-                    raise ValueError("could not get buffer for memoryview")
+                    raise PackValueError("could not get buffer for memoryview")
                 L = view.len
                 if L > (2**32)-1:
                     PyBuffer_Release(&view);
-                    raise ValueError("memoryview is too large")
+                    raise PackValueError("memoryview is too large")
                 ret = msgpack_pack_bin(&self.pk, L)
                 if ret == 0:
                     ret = msgpack_pack_raw_body(&self.pk, view.buf, L)
@@ -274,7 +274,7 @@ cdef class Packer(object):
 
     def pack_array_header(self, size_t size):
         if size > (2**32-1):
-            raise ValueError
+            raise PackValueError
         cdef int ret = msgpack_pack_array(&self.pk, size)
         if ret == -1:
             raise MemoryError
@@ -287,7 +287,7 @@ cdef class Packer(object):
 
     def pack_map_header(self, size_t size):
         if size > (2**32-1):
-            raise ValueError
+            raise PackValueError
         cdef int ret = msgpack_pack_map(&self.pk, size)
         if ret == -1:
             raise MemoryError

From 1183eff688189b0e94ea9e15c5ae13c2f757d745 Mon Sep 17 00:00:00 2001
From: palaviv 
Date: Fri, 12 Feb 2016 15:37:39 +0200
Subject: [PATCH 1335/1648] reraising ValueError from unpack.h as
 UnpackValueError

---
 msgpack/_unpacker.pyx | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 1aefc64..90ebf7d 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -397,24 +397,27 @@ cdef class Unpacker(object):
                 else:
                     raise OutOfData("No more data to unpack.")
 
-            ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
-            if write_bytes is not None:
-                write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
+            try:
+                ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+                if write_bytes is not None:
+                    write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
 
-            if ret == 1:
-                obj = unpack_data(&self.ctx)
-                unpack_init(&self.ctx)
-                return obj
-            elif ret == 0:
-                if self.file_like is not None:
-                    self.read_from_file()
-                    continue
-                if iter:
-                    raise StopIteration("No more data to unpack.")
+                if ret == 1:
+                    obj = unpack_data(&self.ctx)
+                    unpack_init(&self.ctx)
+                    return obj
+                elif ret == 0:
+                    if self.file_like is not None:
+                        self.read_from_file()
+                        continue
+                    if iter:
+                        raise StopIteration("No more data to unpack.")
+                    else:
+                        raise OutOfData("No more data to unpack.")
                 else:
-                    raise OutOfData("No more data to unpack.")
-            else:
-                raise ValueError("Unpack failed: error = %d" % (ret,))
+                    raise UnpackValueError("Unpack failed: error = %d" % (ret,))
+            except ValueError as e:
+                raise UnpackValueError(e)
 
     def read_bytes(self, Py_ssize_t nbytes):
         """Read a specified number of raw bytes from the stream"""

From e15085db0362899520f714e3959c37721c839cef Mon Sep 17 00:00:00 2001
From: palaviv 
Date: Fri, 12 Feb 2016 15:39:50 +0200
Subject: [PATCH 1336/1648] removed MsgpackBaseException

---
 msgpack/exceptions.py | 10 ++++------
 test/test_limits.py   | 18 +++++++++---------
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py
index e0d5b5f..e982079 100644
--- a/msgpack/exceptions.py
+++ b/msgpack/exceptions.py
@@ -1,8 +1,4 @@
-class MsgpackBaseException(Exception):
-    pass
-
-
-class UnpackException(MsgpackBaseException):
+class UnpackException(Exception):
     pass
 
 
@@ -26,9 +22,11 @@ class ExtraData(ValueError):
     def __str__(self):
         return "unpack(b) received extra data."
 
-class PackException(MsgpackBaseException):
+
+class PackException(Exception):
     pass
 
+
 class PackValueError(PackException, ValueError):
     pass
 
diff --git a/test/test_limits.py b/test/test_limits.py
index 34acf7c..e9bc9df 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -4,11 +4,11 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import pytest
 
 from msgpack import packb, unpackb, Packer, Unpacker, ExtType, PackException, PackOverflowError, PackValueError
-from msgpack import UnpackValueError, UnpackException, MsgpackBaseException
+from msgpack import UnpackValueError, UnpackException
 
 
 @pytest.mark.parametrize("expected_exception", [OverflowError, ValueError, PackOverflowError,
-                                                PackException, PackValueError, MsgpackBaseException])
+                                                PackException, PackValueError])
 def test_integer(expected_exception):
     x = -(2 ** 63)
     assert unpackb(packb(x)) == x
@@ -21,7 +21,7 @@ def test_integer(expected_exception):
         packb(x+1)
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError, MsgpackBaseException])
+@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError])
 def test_array_header(expected_exception):
     packer = Packer()
     packer.pack_array_header(2**32-1)
@@ -29,7 +29,7 @@ def test_array_header(expected_exception):
         packer.pack_array_header(2**32)
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError, MsgpackBaseException])
+@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError])
 def test_map_header(expected_exception):
     packer = Packer()
     packer.pack_map_header(2**32-1)
@@ -37,7 +37,7 @@ def test_map_header(expected_exception):
         packer.pack_array_header(2**32)
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException])
+@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException])
 def test_max_str_len(expected_exception):
     d = 'x' * 3
     packed = packb(d)
@@ -52,7 +52,7 @@ def test_max_str_len(expected_exception):
         unpacker.unpack()
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException])
+@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException])
 def test_max_bin_len(expected_exception):
     d = b'x' * 3
     packed = packb(d, use_bin_type=True)
@@ -67,7 +67,7 @@ def test_max_bin_len(expected_exception):
         unpacker.unpack()
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException])
+@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException])
 def test_max_array_len(expected_exception):
     d = [1,2,3]
     packed = packb(d)
@@ -82,7 +82,7 @@ def test_max_array_len(expected_exception):
         unpacker.unpack()
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException])
+@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException])
 def test_max_map_len(expected_exception):
     d = {1: 2, 3: 4, 5: 6}
     packed = packb(d)
@@ -97,7 +97,7 @@ def test_max_map_len(expected_exception):
         unpacker.unpack()
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException, MsgpackBaseException])
+@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException])
 def test_max_ext_len(expected_exception):
     d = ExtType(42, b"abc")
     packed = packb(d)

From d90008d4f57ec83a15b84cf2db2edc36e9504ac1 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 14 Feb 2016 11:46:28 +0900
Subject: [PATCH 1337/1648] ExtraData should be UnpackValueError

---
 msgpack/exceptions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py
index e982079..bb12264 100644
--- a/msgpack/exceptions.py
+++ b/msgpack/exceptions.py
@@ -14,7 +14,7 @@ class UnpackValueError(UnpackException, ValueError):
     pass
 
 
-class ExtraData(ValueError):
+class ExtraData(UnpackValueError):
     def __init__(self, unpacked, extra):
         self.unpacked = unpacked
         self.extra = extra

From 3dad39811d93a58c5d3de874193290b935da1446 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 14 Feb 2016 11:54:01 +0900
Subject: [PATCH 1338/1648] Deprecate PackExceptions

---
 msgpack/_unpacker.pyx | 10 +++++-----
 msgpack/exceptions.py | 12 +++++++++---
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 90ebf7d..0443505 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -11,11 +11,11 @@ from libc.string cimport *
 from libc.limits cimport *
 
 from msgpack.exceptions import (
-        BufferFull,
-        OutOfData,
-        UnpackValueError,
-        ExtraData,
-        )
+    BufferFull,
+    OutOfData,
+    UnpackValueError,
+    ExtraData,
+)
 from msgpack import ExtType
 
 
diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py
index bb12264..73010b7 100644
--- a/msgpack/exceptions.py
+++ b/msgpack/exceptions.py
@@ -24,12 +24,18 @@ class ExtraData(UnpackValueError):
 
 
 class PackException(Exception):
-    pass
+    """Deprecated.  Use Exception instead to catch all exception during packing."""
 
 
 class PackValueError(PackException, ValueError):
-    pass
+    """PackValueError is raised when type of input data is supported but it's value is unsupported.
+
+    Deprecated.  Use ValueError instead.
+    """
 
 
 class PackOverflowError(PackValueError, OverflowError):
-    pass
+    """PackOverflowError is raised when integer value is out of range of msgpack support [-2**31, 2**32).
+
+    Deprecated.  Use ValueError instead.
+    """

From 6e364762394fdb06d0453411a5f020ee594c06b0 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 14 Feb 2016 11:58:56 +0900
Subject: [PATCH 1339/1648] remove too much parameterized tests

---
 test/test_limits.py | 49 +++++++++++++++++++--------------------------
 1 file changed, 21 insertions(+), 28 deletions(-)

diff --git a/test/test_limits.py b/test/test_limits.py
index e9bc9df..197ef46 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -3,42 +3,39 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 import pytest
 
-from msgpack import packb, unpackb, Packer, Unpacker, ExtType, PackException, PackOverflowError, PackValueError
-from msgpack import UnpackValueError, UnpackException
+from msgpack import (
+    packb, unpackb, Packer, Unpacker, ExtType,
+    PackOverflowError, PackValueError, UnpackValueError,
+)
 
 
-@pytest.mark.parametrize("expected_exception", [OverflowError, ValueError, PackOverflowError,
-                                                PackException, PackValueError])
-def test_integer(expected_exception):
+def test_integer():
     x = -(2 ** 63)
     assert unpackb(packb(x)) == x
-    with pytest.raises(expected_exception):
+    with pytest.raises(PackOverflowError):
         packb(x-1)
 
     x = 2 ** 64 - 1
     assert unpackb(packb(x)) == x
-    with pytest.raises(expected_exception):
+    with pytest.raises(PackOverflowError):
         packb(x+1)
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError])
-def test_array_header(expected_exception):
+def test_array_header():
     packer = Packer()
     packer.pack_array_header(2**32-1)
-    with pytest.raises(expected_exception):
+    with pytest.raises(PackValueError):
         packer.pack_array_header(2**32)
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, PackException, PackValueError])
-def test_map_header(expected_exception):
+def test_map_header():
     packer = Packer()
     packer.pack_map_header(2**32-1)
-    with pytest.raises(expected_exception):
+    with pytest.raises(PackValueError):
         packer.pack_array_header(2**32)
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException])
-def test_max_str_len(expected_exception):
+def test_max_str_len():
     d = 'x' * 3
     packed = packb(d)
 
@@ -47,13 +44,12 @@ def test_max_str_len(expected_exception):
     assert unpacker.unpack() == d
 
     unpacker = Unpacker(max_str_len=2, encoding='utf-8')
-    with pytest.raises(expected_exception):
+    with pytest.raises(UnpackValueError):
         unpacker.feed(packed)
         unpacker.unpack()
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException])
-def test_max_bin_len(expected_exception):
+def test_max_bin_len():
     d = b'x' * 3
     packed = packb(d, use_bin_type=True)
 
@@ -62,13 +58,12 @@ def test_max_bin_len(expected_exception):
     assert unpacker.unpack() == d
 
     unpacker = Unpacker(max_bin_len=2)
-    with pytest.raises(expected_exception):
+    with pytest.raises(UnpackValueError):
         unpacker.feed(packed)
         unpacker.unpack()
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException])
-def test_max_array_len(expected_exception):
+def test_max_array_len():
     d = [1,2,3]
     packed = packb(d)
 
@@ -77,13 +72,12 @@ def test_max_array_len(expected_exception):
     assert unpacker.unpack() == d
 
     unpacker = Unpacker(max_array_len=2)
-    with pytest.raises(expected_exception):
+    with pytest.raises(UnpackValueError):
         unpacker.feed(packed)
         unpacker.unpack()
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException])
-def test_max_map_len(expected_exception):
+def test_max_map_len():
     d = {1: 2, 3: 4, 5: 6}
     packed = packb(d)
 
@@ -92,13 +86,12 @@ def test_max_map_len(expected_exception):
     assert unpacker.unpack() == d
 
     unpacker = Unpacker(max_map_len=2)
-    with pytest.raises(expected_exception):
+    with pytest.raises(UnpackValueError):
         unpacker.feed(packed)
         unpacker.unpack()
 
 
-@pytest.mark.parametrize("expected_exception", [ValueError, UnpackValueError, UnpackException])
-def test_max_ext_len(expected_exception):
+def test_max_ext_len():
     d = ExtType(42, b"abc")
     packed = packb(d)
 
@@ -107,7 +100,7 @@ def test_max_ext_len(expected_exception):
     assert unpacker.unpack() == d
 
     unpacker = Unpacker(max_ext_len=2)
-    with pytest.raises(expected_exception):
+    with pytest.raises(UnpackValueError):
         unpacker.feed(packed)
         unpacker.unpack()
 

From b2a8ce6cbdbef80d1a89d02fa483f56862cf1efa Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 14 Feb 2016 14:32:11 +0900
Subject: [PATCH 1340/1648] Deprecate more useless exceptions

---
 msgpack/exceptions.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py
index 73010b7..9766881 100644
--- a/msgpack/exceptions.py
+++ b/msgpack/exceptions.py
@@ -1,5 +1,5 @@
 class UnpackException(Exception):
-    pass
+    """Deprecated.  Use Exception instead to catch all exception during unpacking."""
 
 
 class BufferFull(UnpackException):
@@ -11,7 +11,7 @@ class OutOfData(UnpackException):
 
 
 class UnpackValueError(UnpackException, ValueError):
-    pass
+    """Deprecated.  Use ValueError instead."""
 
 
 class ExtraData(UnpackValueError):

From 2192310bc4a7af32a628191ededd1feeec624845 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 16 Apr 2016 02:03:18 +0900
Subject: [PATCH 1341/1648] Use manylinux1 wheel for Cython (#179)

* Use manylinux1 wheel for Cython
* Use newer pip
---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index eced353..e7e9b63 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,8 +16,8 @@ env:
     - TOXENV=pypy-pure,pypy3-pure
 
 install:
-    - pip install tox
-    - pip install cython --install-option=--cython-with-refnanny --install-option=--no-cython-compile
+    - pip install -U pip
+    - pip install tox cython
     - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
 
 script: tox

From 40ee322440a018c9e09634aa2c190d1747d7f0bd Mon Sep 17 00:00:00 2001
From: Timothy Cyrus 
Date: Fri, 29 Apr 2016 11:18:27 -0400
Subject: [PATCH 1342/1648] Update README.rst (#184)

Change PNG Badges to SVG
---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index d32ec1d..8136ccb 100644
--- a/README.rst
+++ b/README.rst
@@ -6,7 +6,7 @@ MessagePack for Python
 :version: 0.4.6
 :date: 2015-03-13
 
-.. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png
+.. image:: https://secure.travis-ci.org/msgpack/msgpack-python.svg
    :target: https://travis-ci.org/#!/msgpack/msgpack-python
    
 

From 6b113a6fb37ffb969e92429b06aab9ea9b8eeb4a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 30 Apr 2016 17:07:14 +0900
Subject: [PATCH 1343/1648] Use Python's memory API (#185)

---
 msgpack/_packer.pyx   |  7 +++----
 msgpack/_unpacker.pyx | 23 ++++++++++++++++++-----
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index e923895..d491cc1 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -88,7 +88,7 @@ cdef class Packer(object):
 
     def __cinit__(self):
         cdef int buf_size = 1024*1024
-        self.pk.buf =  malloc(buf_size);
+        self.pk.buf =  PyMem_Malloc(buf_size)
         if self.pk.buf == NULL:
             raise MemoryError("Unable to allocate internal buffer.")
         self.pk.buf_size = buf_size
@@ -97,8 +97,6 @@ cdef class Packer(object):
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
                  use_single_float=False, bint autoreset=1, bint use_bin_type=0,
                  bint strict_types=0):
-        """
-        """
         self.use_float = use_single_float
         self.strict_types = strict_types
         self.autoreset = autoreset
@@ -123,7 +121,8 @@ cdef class Packer(object):
             self.unicode_errors = PyBytes_AsString(self._berrors)
 
     def __dealloc__(self):
-        free(self.pk.buf);
+        PyMem_Free(self.pk.buf)
+        self.pk.buf = NULL
 
     cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1:
         cdef long long llval
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 0443505..23f6478 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -1,7 +1,20 @@
 # coding: utf-8
 #cython: embedsignature=True
 
-from cpython cimport *
+from cpython.bytes cimport (
+    PyBytes_AsString,
+    PyBytes_FromStringAndSize,
+    PyBytes_Size,
+)
+from cpython.buffer cimport (
+    Py_buffer,
+    PyBuffer_Release,
+    PyObject_GetBuffer,
+    PyBUF_SIMPLE,
+)
+from cpython.mem cimport PyMem_Malloc, PyMem_Free
+from cpython.object cimport PyCallable_Check
+
 cdef extern from "Python.h":
     ctypedef struct PyObject
     cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1
@@ -256,7 +269,7 @@ cdef class Unpacker(object):
         self.buf = NULL
 
     def __dealloc__(self):
-        free(self.buf)
+        PyMem_Free(self.buf)
         self.buf = NULL
 
     def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
@@ -289,7 +302,7 @@ cdef class Unpacker(object):
             read_size = min(max_buffer_size, 1024**2)
         self.max_buffer_size = max_buffer_size
         self.read_size = read_size
-        self.buf = malloc(read_size)
+        self.buf = PyMem_Malloc(read_size)
         if self.buf == NULL:
             raise MemoryError("Unable to allocate internal buffer.")
         self.buf_size = read_size
@@ -352,13 +365,13 @@ cdef class Unpacker(object):
                 if new_size > self.max_buffer_size:
                     raise BufferFull
                 new_size = min(new_size*2, self.max_buffer_size)
-                new_buf = malloc(new_size)
+                new_buf = PyMem_Malloc(new_size)
                 if new_buf == NULL:
                     # self.buf still holds old buffer and will be freed during
                     # obj destruction
                     raise MemoryError("Unable to enlarge internal buffer.")
                 memcpy(new_buf, buf + head, tail - head)
-                free(buf)
+                PyMem_Free(buf)
 
                 buf = new_buf
                 buf_size = new_size

From ceb9635a3f4d7f3dd4874b98773ca6f7db9296a7 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 3 May 2016 11:58:28 +0900
Subject: [PATCH 1344/1648] Use AppVeyor to build windows wheel (#188)

* Add AppVeyor support to build windows wheel
* Fix test_limits on 32bit environments
* Ignore Python35-x64 test fail for now
  Should be fixed in next version.
---
 appveyor.yml        | 57 +++++++++++++++++++++++++++++++++++++++++++++
 build.cmd           | 21 +++++++++++++++++
 msgpack/_packer.pyx |  4 ++--
 3 files changed, 80 insertions(+), 2 deletions(-)
 create mode 100644 appveyor.yml
 create mode 100644 build.cmd

diff --git a/appveyor.yml b/appveyor.yml
new file mode 100644
index 0000000..02b4461
--- /dev/null
+++ b/appveyor.yml
@@ -0,0 +1,57 @@
+environment:
+
+  matrix:
+
+    # For Python versions available on Appveyor, see
+    # http://www.appveyor.com/docs/installed-software#python
+    # The list here is complete (excluding Python 2.6, which
+    # isn't covered by this document) at the time of writing.
+
+    - PYTHON: "C:\\Python27"
+    - PYTHON: "C:\\Python34"
+    - PYTHON: "C:\\Python35"
+    - PYTHON: "C:\\Python27-x64"
+    - PYTHON: "C:\\Python34-x64"
+      DISTUTILS_USE_SDK: "1"
+
+    # Python35-x64 test fails with MemoryError
+    # TODO: investigate it
+    #- PYTHON: "C:\\Python35-x64"
+
+install:
+  # We need wheel installed to build wheels
+  - "%PYTHON%\\python.exe -m pip install -U pip wheel pytest cython"
+
+build: off
+
+test_script:
+  # Put your test command here.
+  # If you don't need to build C extensions on 64-bit Python 3.3 or 3.4,
+  # you can remove "build.cmd" from the front of the command, as it's
+  # only needed to support those cases.
+  # Note that you must use the environment variable %PYTHON% to refer to
+  # the interpreter you're using - Appveyor does not do anything special
+  # to put the Python evrsion you want to use on PATH.
+  - "build.cmd %PYTHON%\\python.exe setup.py build_ext -i"
+  - "build.cmd %PYTHON%\\python.exe setup.py install"
+  - "%PYTHON%\\python.exe -c \"import sys; print(hex(sys.maxsize))\""
+  - "%PYTHON%\\python.exe -c \"from msgpack import _packer, _unpacker\""
+  - "%PYTHON%\\Scripts\\py.test test"
+  - "build.cmd %PYTHON%\\python.exe setup.py bdist_wheel"
+
+after_test:
+  # This step builds your wheels.
+  # Again, you only need build.cmd if you're building C extensions for
+  # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct
+  # interpreter
+
+artifacts:
+  # bdist_wheel puts your built wheel in the dist directory
+  - path: dist\*
+
+#on_success:
+#  You can use this step to upload your artifacts to a public website.
+#  See Appveyor's documentation for more details. Or you can simply
+#  access your wheels from the Appveyor "artifacts" tab for your build.
+
+# vim: set shiftwidth=2
diff --git a/build.cmd b/build.cmd
new file mode 100644
index 0000000..243dc9a
--- /dev/null
+++ b/build.cmd
@@ -0,0 +1,21 @@
+@echo off
+:: To build extensions for 64 bit Python 3, we need to configure environment
+:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 4
+::
+:: More details at:
+:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
+
+IF "%DISTUTILS_USE_SDK%"=="1" (
+    ECHO Configuring environment to build with MSVC on a 64bit architecture
+    ECHO Using Windows SDK 7.1
+    "C:\Program Files\Microsoft SDKs\Windows\v7.1\Setup\WindowsSdkVer.exe" -q -version:v7.1
+    CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release
+    SET MSSdk=1
+    REM Need the following to allow tox to see the SDK compiler
+    SET TOX_TESTENV_PASSENV=DISTUTILS_USE_SDK MSSdk INCLUDE LIB
+) ELSE (
+    ECHO Using default MSVC build environment
+)
+
+CALL %*
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index d491cc1..b1a912b 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -271,7 +271,7 @@ cdef class Packer(object):
         msgpack_pack_ext(&self.pk, typecode, len(data))
         msgpack_pack_raw_body(&self.pk, data, len(data))
 
-    def pack_array_header(self, size_t size):
+    def pack_array_header(self, long long size):
         if size > (2**32-1):
             raise PackValueError
         cdef int ret = msgpack_pack_array(&self.pk, size)
@@ -284,7 +284,7 @@ cdef class Packer(object):
             self.pk.length = 0
             return buf
 
-    def pack_map_header(self, size_t size):
+    def pack_map_header(self, long long size):
         if size > (2**32-1):
             raise PackValueError
         cdef int ret = msgpack_pack_map(&self.pk, size)

From 0ec2e3534f9b7751be484bd2f1344e24c49bb24f Mon Sep 17 00:00:00 2001
From: folz 
Date: Thu, 28 Apr 2016 15:08:28 +0200
Subject: [PATCH 1345/1648] fix problems associated with packing memoryviews

fix wrong length when packing multibyte memoryviews in fallback
add tests for memoryviews of different types and sizes and check contents of packed data
---
 msgpack/_packer.pyx     |  23 +++----
 msgpack/fallback.py     |  26 +++++++-
 test/test_memoryview.py | 133 ++++++++++++++++++++++++++++++++++++++--
 3 files changed, 164 insertions(+), 18 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index b1a912b..e07b194 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -39,6 +39,7 @@ cdef extern from "pack.h":
     int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l)
 
 cdef int DEFAULT_RECURSE_LIMIT=511
+cdef size_t ITEM_LIMIT = (2**32)-1
 
 
 cdef class Packer(object):
@@ -178,7 +179,7 @@ cdef class Packer(object):
                    ret = msgpack_pack_double(&self.pk, dval)
             elif PyBytes_CheckExact(o) if strict_types else PyBytes_Check(o):
                 L = len(o)
-                if L > (2**32)-1:
+                if L > ITEM_LIMIT:
                     raise PackValueError("bytes is too large")
                 rawval = o
                 ret = msgpack_pack_bin(&self.pk, L)
@@ -189,7 +190,7 @@ cdef class Packer(object):
                     raise TypeError("Can't encode unicode string: no encoding is specified")
                 o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
                 L = len(o)
-                if L > (2**32)-1:
+                if L > ITEM_LIMIT:
                     raise PackValueError("unicode string is too large")
                 rawval = o
                 ret = msgpack_pack_raw(&self.pk, L)
@@ -198,7 +199,7 @@ cdef class Packer(object):
             elif PyDict_CheckExact(o):
                 d = o
                 L = len(d)
-                if L > (2**32)-1:
+                if L > ITEM_LIMIT:
                     raise PackValueError("dict is too large")
                 ret = msgpack_pack_map(&self.pk, L)
                 if ret == 0:
@@ -209,7 +210,7 @@ cdef class Packer(object):
                         if ret != 0: break
             elif not strict_types and PyDict_Check(o):
                 L = len(o)
-                if L > (2**32)-1:
+                if L > ITEM_LIMIT:
                     raise PackValueError("dict is too large")
                 ret = msgpack_pack_map(&self.pk, L)
                 if ret == 0:
@@ -223,13 +224,13 @@ cdef class Packer(object):
                 longval = o.code
                 rawval = o.data
                 L = len(o.data)
-                if L > (2**32)-1:
+                if L > ITEM_LIMIT:
                     raise PackValueError("EXT data is too large")
                 ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
                 L = len(o)
-                if L > (2**32)-1:
+                if L > ITEM_LIMIT:
                     raise PackValueError("list is too large")
                 ret = msgpack_pack_array(&self.pk, L)
                 if ret == 0:
@@ -240,7 +241,7 @@ cdef class Packer(object):
                 if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0:
                     raise PackValueError("could not get buffer for memoryview")
                 L = view.len
-                if L > (2**32)-1:
+                if L > ITEM_LIMIT:
                     PyBuffer_Release(&view);
                     raise PackValueError("memoryview is too large")
                 ret = msgpack_pack_bin(&self.pk, L)
@@ -271,8 +272,8 @@ cdef class Packer(object):
         msgpack_pack_ext(&self.pk, typecode, len(data))
         msgpack_pack_raw_body(&self.pk, data, len(data))
 
-    def pack_array_header(self, long long size):
-        if size > (2**32-1):
+    def pack_array_header(self, size_t size):
+        if size > ITEM_LIMIT:
             raise PackValueError
         cdef int ret = msgpack_pack_array(&self.pk, size)
         if ret == -1:
@@ -284,8 +285,8 @@ cdef class Packer(object):
             self.pk.length = 0
             return buf
 
-    def pack_map_header(self, long long size):
-        if size > (2**32-1):
+    def pack_map_header(self, size_t size):
+        if size > ITEM_LIMIT:
             raise PackValueError
         cdef int ret = msgpack_pack_map(&self.pk, size)
         if ret == -1:
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index d8c5d73..db47d5c 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -685,7 +685,7 @@ class Packer(object):
                     default_used = True
                     continue
                 raise PackOverflowError("Integer value out of range")
-            if self._use_bin_type and check(obj, (bytes, memoryview)):
+            if self._use_bin_type and check(obj, bytes):
                 n = len(obj)
                 if n <= 0xff:
                     self._buffer.write(struct.pack('>BB', 0xc4, n))
@@ -696,7 +696,7 @@ class Packer(object):
                 else:
                     raise PackValueError("Bytes is too large")
                 return self._buffer.write(obj)
-            if check(obj, (Unicode, bytes, memoryview)):
+            if check(obj, (Unicode, bytes)):
                 if check(obj, Unicode):
                     if self._encoding is None:
                         raise TypeError(
@@ -715,6 +715,28 @@ class Packer(object):
                 else:
                     raise PackValueError("String is too large")
                 return self._buffer.write(obj)
+            if check(obj, memoryview):
+                n = len(obj) * obj.itemsize
+                if self._use_bin_type:
+                    if n <= 0xff:
+                        self._buffer.write(struct.pack('>BB', 0xc4, n))
+                    elif n <= 0xffff:
+                        self._buffer.write(struct.pack(">BH", 0xc5, n))
+                    elif n <= 0xffffffff:
+                        self._buffer.write(struct.pack(">BI", 0xc6, n))
+                    else:
+                        raise PackValueError("memoryview is too large")
+                    return self._buffer.write(obj)
+                else:
+                    if n <= 0x1f:
+                        self._buffer.write(struct.pack('B', 0xa0 + n))
+                    elif n <= 0xffff:
+                        self._buffer.write(struct.pack(">BH", 0xda, n))
+                    elif n <= 0xffffffff:
+                        self._buffer.write(struct.pack(">BI", 0xdb, n))
+                    else:
+                        raise PackValueError("memoryview is too large")
+                    return self._buffer.write(obj)
             if check(obj, float):
                 if self._use_float:
                     return self._buffer.write(struct.pack(">Bf", 0xca, obj))
diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index aed5069..f555c5b 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -2,11 +2,134 @@
 # coding: utf-8
 
 
+from array import array
 from msgpack import packb, unpackb
+import sys
 
 
-def test_pack_memoryview():
-    data = bytearray(range(256))
-    view = memoryview(data)
-    unpacked = unpackb(packb(view))
-    assert data == unpacked
+# For Python < 3:
+#  - array type only supports old buffer interface
+#  - array.frombytes is not available, must use deprecated array.fromstring
+if sys.version_info[0] < 3:
+    def __memoryview(obj):
+        return memoryview(buffer(obj))
+
+    def __make_array(f, data):
+        a = array(f)
+        a.fromstring(data)
+        return a
+
+    def __get_data(a):
+        return a.tostring()
+else:
+    __memoryview = memoryview
+
+    def __make_array(f, data):
+        a = array(f)
+        a.frombytes(data)
+        return a
+
+    def __get_data(a):
+        return a.tobytes()
+
+
+def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type):
+    # create a new array
+    original_array = array(format)
+    original_array.fromlist([255] * (nbytes // original_array.itemsize))
+    original_data = __get_data(original_array)
+    view = __memoryview(original_array)
+
+    # pack, unpack, and reconstruct array
+    packed = packb(view, use_bin_type=use_bin_type)
+    unpacked = unpackb(packed)
+    reconstructed_array = __make_array(format, unpacked)
+
+    # check that we got the right amount of data
+    assert len(original_data) == nbytes
+    # check packed header
+    assert packed[:1] == expected_header
+    # check packed length prefix, if any
+    assert packed[1:1+len(expected_prefix)] == expected_prefix
+    # check packed data
+    assert packed[1+len(expected_prefix):] == original_data
+    # check array unpacked correctly
+    assert original_array == reconstructed_array
+
+
+# -----------
+# test fixstr
+# -----------
+
+
+def test_memoryview_byte_fixstr():
+    __run_test('B', 31, b'\xbf', b'', False)
+
+
+def test_memoryview_float_fixstr():
+    __run_test('f', 28, b'\xbc', b'', False)
+
+
+# ----------
+# test str16
+# ----------
+
+
+def test_memoryview_byte_str16():
+    __run_test('B', 2**8, b'\xda', b'\x01\x00', False)
+
+
+def test_memoryview_float_str16():
+    __run_test('f', 2**8, b'\xda', b'\x01\x00', False)
+
+
+# ----------
+# test str32
+# ----------
+
+
+def test_memoryview_byte_str32():
+    __run_test('B', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
+
+
+def test_memoryview_float_str32():
+    __run_test('f', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
+
+
+# ---------
+# test bin8
+# ---------
+
+
+def test_memoryview_byte_bin8():
+    __run_test('B', 1, b'\xc4', b'\x01', True)
+
+
+def test_memoryview_float_bin8():
+    __run_test('f', 4, b'\xc4', b'\x04', True)
+
+
+# ----------
+# test bin16
+# ----------
+
+
+def test_memoryview_byte_bin16():
+    __run_test('B', 2**8, b'\xc5', b'\x01\x00', True)
+
+
+def test_memoryview_float_bin16():
+    __run_test('f', 2**8, b'\xc5', b'\x01\x00', True)
+
+
+# ----------
+# test bin32
+# ----------
+
+
+def test_memoryview_byte_bin32():
+    __run_test('B', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)
+
+
+def test_memoryview_float_bin32():
+    __run_test('f', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)

From 0b55989f0b045f1a77d4230bea3b6da70eb3d840 Mon Sep 17 00:00:00 2001
From: folz 
Date: Wed, 4 May 2016 10:04:09 +0200
Subject: [PATCH 1346/1648] more descriptive test names

---
 test/test_memoryview.py | 72 ++++++++++++-----------------------------
 1 file changed, 21 insertions(+), 51 deletions(-)

diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index f555c5b..2768867 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -11,25 +11,25 @@ import sys
 #  - array type only supports old buffer interface
 #  - array.frombytes is not available, must use deprecated array.fromstring
 if sys.version_info[0] < 3:
-    def __memoryview(obj):
+    def make_memoryview(obj):
         return memoryview(buffer(obj))
 
-    def __make_array(f, data):
+    def make_array(f, data):
         a = array(f)
         a.fromstring(data)
         return a
 
-    def __get_data(a):
+    def get_data(a):
         return a.tostring()
 else:
-    __memoryview = memoryview
+    make_memoryview = memoryview
 
-    def __make_array(f, data):
+    def make_array(f, data):
         a = array(f)
         a.frombytes(data)
         return a
 
-    def __get_data(a):
+    def get_data(a):
         return a.tobytes()
 
 
@@ -37,13 +37,13 @@ def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type):
     # create a new array
     original_array = array(format)
     original_array.fromlist([255] * (nbytes // original_array.itemsize))
-    original_data = __get_data(original_array)
-    view = __memoryview(original_array)
+    original_data = get_data(original_array)
+    view = make_memoryview(original_array)
 
     # pack, unpack, and reconstruct array
     packed = packb(view, use_bin_type=use_bin_type)
     unpacked = unpackb(packed)
-    reconstructed_array = __make_array(format, unpacked)
+    reconstructed_array = make_array(format, unpacked)
 
     # check that we got the right amount of data
     assert len(original_data) == nbytes
@@ -57,79 +57,49 @@ def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type):
     assert original_array == reconstructed_array
 
 
-# -----------
-# test fixstr
-# -----------
-
-
-def test_memoryview_byte_fixstr():
+def test_fixstr_from_byte():
     __run_test('B', 31, b'\xbf', b'', False)
 
 
-def test_memoryview_float_fixstr():
+def test_fixstr_from_float():
     __run_test('f', 28, b'\xbc', b'', False)
 
 
-# ----------
-# test str16
-# ----------
-
-
-def test_memoryview_byte_str16():
+def test_str16_from_byte():
     __run_test('B', 2**8, b'\xda', b'\x01\x00', False)
 
 
-def test_memoryview_float_str16():
+def test_str16_from_float():
     __run_test('f', 2**8, b'\xda', b'\x01\x00', False)
 
 
-# ----------
-# test str32
-# ----------
-
-
-def test_memoryview_byte_str32():
+def test_str32_from_byte():
     __run_test('B', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
 
 
-def test_memoryview_float_str32():
+def test_str32_from_float():
     __run_test('f', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
 
 
-# ---------
-# test bin8
-# ---------
-
-
-def test_memoryview_byte_bin8():
+def test_bin8_from_byte():
     __run_test('B', 1, b'\xc4', b'\x01', True)
 
 
-def test_memoryview_float_bin8():
+def test_bin8_from_float():
     __run_test('f', 4, b'\xc4', b'\x04', True)
 
 
-# ----------
-# test bin16
-# ----------
-
-
-def test_memoryview_byte_bin16():
+def test_bin16_from_byte():
     __run_test('B', 2**8, b'\xc5', b'\x01\x00', True)
 
 
-def test_memoryview_float_bin16():
+def test_bin16_from_float():
     __run_test('f', 2**8, b'\xc5', b'\x01\x00', True)
 
 
-# ----------
-# test bin32
-# ----------
-
-
-def test_memoryview_byte_bin32():
+def test_bin32_from_byte():
     __run_test('B', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)
 
 
-def test_memoryview_float_bin32():
+def test_bin32_from_float():
     __run_test('f', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)

From 5860af953ae1c3f459ddc589cd815ec195db46a9 Mon Sep 17 00:00:00 2001
From: folz 
Date: Wed, 4 May 2016 11:01:27 +0200
Subject: [PATCH 1347/1648] refactor header packing for str and bin types

---
 msgpack/fallback.py | 83 +++++++++++++++++++++------------------------
 1 file changed, 39 insertions(+), 44 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index db47d5c..abed3d9 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -685,58 +685,29 @@ class Packer(object):
                     default_used = True
                     continue
                 raise PackOverflowError("Integer value out of range")
-            if self._use_bin_type and check(obj, bytes):
+            if check(obj, bytes):
                 n = len(obj)
-                if n <= 0xff:
-                    self._buffer.write(struct.pack('>BB', 0xc4, n))
-                elif n <= 0xffff:
-                    self._buffer.write(struct.pack(">BH", 0xc5, n))
-                elif n <= 0xffffffff:
-                    self._buffer.write(struct.pack(">BI", 0xc6, n))
-                else:
+                if n >= 2**32:
                     raise PackValueError("Bytes is too large")
+                self._fb_pack_bin_header(n)
                 return self._buffer.write(obj)
-            if check(obj, (Unicode, bytes)):
-                if check(obj, Unicode):
-                    if self._encoding is None:
-                        raise TypeError(
-                            "Can't encode unicode string: "
-                            "no encoding is specified")
-                    obj = obj.encode(self._encoding, self._unicode_errors)
+            if check(obj, Unicode):
+                if self._encoding is None:
+                    raise TypeError(
+                        "Can't encode unicode string: "
+                        "no encoding is specified")
+                obj = obj.encode(self._encoding, self._unicode_errors)
                 n = len(obj)
-                if n <= 0x1f:
-                    self._buffer.write(struct.pack('B', 0xa0 + n))
-                elif self._use_bin_type and n <= 0xff:
-                    self._buffer.write(struct.pack('>BB', 0xd9, n))
-                elif n <= 0xffff:
-                    self._buffer.write(struct.pack(">BH", 0xda, n))
-                elif n <= 0xffffffff:
-                    self._buffer.write(struct.pack(">BI", 0xdb, n))
-                else:
+                if n >= 2**32:
                     raise PackValueError("String is too large")
+                self._fb_pack_raw_header(n)
                 return self._buffer.write(obj)
             if check(obj, memoryview):
                 n = len(obj) * obj.itemsize
-                if self._use_bin_type:
-                    if n <= 0xff:
-                        self._buffer.write(struct.pack('>BB', 0xc4, n))
-                    elif n <= 0xffff:
-                        self._buffer.write(struct.pack(">BH", 0xc5, n))
-                    elif n <= 0xffffffff:
-                        self._buffer.write(struct.pack(">BI", 0xc6, n))
-                    else:
-                        raise PackValueError("memoryview is too large")
-                    return self._buffer.write(obj)
-                else:
-                    if n <= 0x1f:
-                        self._buffer.write(struct.pack('B', 0xa0 + n))
-                    elif n <= 0xffff:
-                        self._buffer.write(struct.pack(">BH", 0xda, n))
-                    elif n <= 0xffffffff:
-                        self._buffer.write(struct.pack(">BI", 0xdb, n))
-                    else:
-                        raise PackValueError("memoryview is too large")
-                    return self._buffer.write(obj)
+                if n >= 2**32:
+                    raise PackValueError("Memoryview is too large")
+                self._fb_pack_bin_header(n)
+                return self._buffer.write(obj)
             if check(obj, float):
                 if self._use_float:
                     return self._buffer.write(struct.pack(">Bf", 0xca, obj))
@@ -874,6 +845,30 @@ class Packer(object):
             self._pack(k, nest_limit - 1)
             self._pack(v, nest_limit - 1)
 
+    def _fb_pack_raw_header(self, n):
+        if n <= 0x1f:
+            self._buffer.write(struct.pack('B', 0xa0 + n))
+        elif self._use_bin_type and n <= 0xff:
+            self._buffer.write(struct.pack('>BB', 0xd9, n))
+        elif n <= 0xffff:
+            self._buffer.write(struct.pack(">BH", 0xda, n))
+        elif n <= 0xffffffff:
+            self._buffer.write(struct.pack(">BI", 0xdb, n))
+        else:
+            raise PackValueError('Raw is too large')
+
+    def _fb_pack_bin_header(self, n):
+        if not self._use_bin_type:
+            return self._fb_pack_raw_header(n)
+        elif n <= 0xff:
+            return self._buffer.write(struct.pack('>BB', 0xc4, n))
+        elif n <= 0xffff:
+            return self._buffer.write(struct.pack(">BH", 0xc5, n))
+        elif n <= 0xffffffff:
+            return self._buffer.write(struct.pack(">BI", 0xc6, n))
+        else:
+            raise PackValueError('Bin is too large')
+
     def bytes(self):
         return self._buffer.getvalue()
 

From a91d5c538ea5bbee0f00ff180a8e72d27df6cfc1 Mon Sep 17 00:00:00 2001
From: folz 
Date: Wed, 4 May 2016 12:03:37 +0200
Subject: [PATCH 1348/1648] add lower bound tests for memoryviews

---
 test/test_memoryview.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index 2768867..7ce6bfc 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -58,19 +58,23 @@ def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type):
 
 
 def test_fixstr_from_byte():
+    __run_test('B', 1, b'\xa1', b'', False)
     __run_test('B', 31, b'\xbf', b'', False)
 
 
 def test_fixstr_from_float():
+    __run_test('f', 4, b'\xa4', b'', False)
     __run_test('f', 28, b'\xbc', b'', False)
 
 
 def test_str16_from_byte():
     __run_test('B', 2**8, b'\xda', b'\x01\x00', False)
+    __run_test('B', 2**16-1, b'\xda', b'\xff\xff', False)
 
 
 def test_str16_from_float():
     __run_test('f', 2**8, b'\xda', b'\x01\x00', False)
+    __run_test('f', 2**16-4, b'\xda', b'\xff\xfc', False)
 
 
 def test_str32_from_byte():
@@ -83,18 +87,22 @@ def test_str32_from_float():
 
 def test_bin8_from_byte():
     __run_test('B', 1, b'\xc4', b'\x01', True)
+    __run_test('B', 2**8-1, b'\xc4', b'\xff', True)
 
 
 def test_bin8_from_float():
     __run_test('f', 4, b'\xc4', b'\x04', True)
+    __run_test('f', 2**8-4, b'\xc4', b'\xfc', True)
 
 
 def test_bin16_from_byte():
     __run_test('B', 2**8, b'\xc5', b'\x01\x00', True)
+    __run_test('B', 2**16-1, b'\xc5', b'\xff\xff', True)
 
 
 def test_bin16_from_float():
     __run_test('f', 2**8, b'\xc5', b'\x01\x00', True)
+    __run_test('f', 2**16-4, b'\xc5', b'\xff\xfc', True)
 
 
 def test_bin32_from_byte():

From 53f47ef55d8d93e276ecf9041a9a8b43fc041aef Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 5 May 2016 00:49:48 +0900
Subject: [PATCH 1349/1648] Remove double underscore prefix

---
 test/test_memoryview.py | 43 ++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index 7ce6bfc..f6d74ed 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-
 from array import array
 from msgpack import packb, unpackb
 import sys
@@ -33,7 +32,7 @@ else:
         return a.tobytes()
 
 
-def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type):
+def _runtest(format, nbytes, expected_header, expected_prefix, use_bin_type):
     # create a new array
     original_array = array(format)
     original_array.fromlist([255] * (nbytes // original_array.itemsize))
@@ -58,56 +57,56 @@ def __run_test(format, nbytes, expected_header, expected_prefix, use_bin_type):
 
 
 def test_fixstr_from_byte():
-    __run_test('B', 1, b'\xa1', b'', False)
-    __run_test('B', 31, b'\xbf', b'', False)
+    _runtest('B', 1, b'\xa1', b'', False)
+    _runtest('B', 31, b'\xbf', b'', False)
 
 
 def test_fixstr_from_float():
-    __run_test('f', 4, b'\xa4', b'', False)
-    __run_test('f', 28, b'\xbc', b'', False)
+    _runtest('f', 4, b'\xa4', b'', False)
+    _runtest('f', 28, b'\xbc', b'', False)
 
 
 def test_str16_from_byte():
-    __run_test('B', 2**8, b'\xda', b'\x01\x00', False)
-    __run_test('B', 2**16-1, b'\xda', b'\xff\xff', False)
+    _runtest('B', 2**8, b'\xda', b'\x01\x00', False)
+    _runtest('B', 2**16-1, b'\xda', b'\xff\xff', False)
 
 
 def test_str16_from_float():
-    __run_test('f', 2**8, b'\xda', b'\x01\x00', False)
-    __run_test('f', 2**16-4, b'\xda', b'\xff\xfc', False)
+    _runtest('f', 2**8, b'\xda', b'\x01\x00', False)
+    _runtest('f', 2**16-4, b'\xda', b'\xff\xfc', False)
 
 
 def test_str32_from_byte():
-    __run_test('B', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
+    _runtest('B', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
 
 
 def test_str32_from_float():
-    __run_test('f', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
+    _runtest('f', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
 
 
 def test_bin8_from_byte():
-    __run_test('B', 1, b'\xc4', b'\x01', True)
-    __run_test('B', 2**8-1, b'\xc4', b'\xff', True)
+    _runtest('B', 1, b'\xc4', b'\x01', True)
+    _runtest('B', 2**8-1, b'\xc4', b'\xff', True)
 
 
 def test_bin8_from_float():
-    __run_test('f', 4, b'\xc4', b'\x04', True)
-    __run_test('f', 2**8-4, b'\xc4', b'\xfc', True)
+    _runtest('f', 4, b'\xc4', b'\x04', True)
+    _runtest('f', 2**8-4, b'\xc4', b'\xfc', True)
 
 
 def test_bin16_from_byte():
-    __run_test('B', 2**8, b'\xc5', b'\x01\x00', True)
-    __run_test('B', 2**16-1, b'\xc5', b'\xff\xff', True)
+    _runtest('B', 2**8, b'\xc5', b'\x01\x00', True)
+    _runtest('B', 2**16-1, b'\xc5', b'\xff\xff', True)
 
 
 def test_bin16_from_float():
-    __run_test('f', 2**8, b'\xc5', b'\x01\x00', True)
-    __run_test('f', 2**16-4, b'\xc5', b'\xff\xfc', True)
+    _runtest('f', 2**8, b'\xc5', b'\x01\x00', True)
+    _runtest('f', 2**16-4, b'\xc5', b'\xff\xfc', True)
 
 
 def test_bin32_from_byte():
-    __run_test('B', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)
+    _runtest('B', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)
 
 
 def test_bin32_from_float():
-    __run_test('f', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)
+    _runtest('f', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)

From 63e23d37f9f3646f0fc3b327ddf1f3e1f200baf5 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 5 May 2016 02:07:46 +0900
Subject: [PATCH 1350/1648] travis: Use docker to test 32bit environment (#189)

* travis: testing matrix.include feature to use docker
* Add test script for 32bit
* Fix OverflowError in 32bit Environment
---
 .travis.yml         | 38 +++++++++++++++++++++++++++-----------
 docker/runtests.sh  | 14 ++++++++++++++
 msgpack/_packer.pyx |  4 ++--
 3 files changed, 43 insertions(+), 13 deletions(-)
 create mode 100755 docker/runtests.sh

diff --git a/.travis.yml b/.travis.yml
index e7e9b63..b4396cb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,23 +1,39 @@
 sudo: false
+language: python
+python: 3.5
 cache:
   directories:
     - $HOME/.cache/pip
-language: python
-python:
-    - 3.5
 
 branches:
-    only:
-        - master
+  only:
+    - master
 
 env:
-    - TOXENV=py27-c,py33-c,py34-c,py35-c
-    - TOXENV=py27-pure,py33-pure,py34-pure,py35-pure
-    - TOXENV=pypy-pure,pypy3-pure
+  - TOXENV=py27-c,py33-c,py34-c,py35-c
+  - TOXENV=py27-pure,py33-pure,py34-pure,py35-pure
+  - TOXENV=pypy-pure,pypy3-pure
+
+matrix:
+  include:
+    - sudo: required
+      services:
+        - docker
+      env:
+        - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686
+      install:
+        - pip install -U pip
+        - pip install cython
+        - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
+        - docker pull $DOCKER_IMAGE
+      script:
+        - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
 
 install:
-    - pip install -U pip
-    - pip install tox cython
-    - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
+  - pip install -U pip
+  - pip install tox cython
+  - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
 
 script: tox
+
+# vim: sw=2 ts=2
diff --git a/docker/runtests.sh b/docker/runtests.sh
new file mode 100755
index 0000000..0d74802
--- /dev/null
+++ b/docker/runtests.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e -x
+
+for V in cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do
+    PYBIN=/opt/python/$V/bin
+    $PYBIN/python setup.py install
+    rm -rf build/       # Avoid lib build by narrow Python is used by wide python
+    $PYBIN/pip install pytest
+    pushd test          # prevent importing msgpack package in current directory.
+    $PYBIN/python -c 'import sys; print(hex(sys.maxsize))'
+    $PYBIN/python -c 'from msgpack import _packer, _unpacker'
+    $PYBIN/py.test -v
+    popd
+done
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index e07b194..3981f20 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -272,7 +272,7 @@ cdef class Packer(object):
         msgpack_pack_ext(&self.pk, typecode, len(data))
         msgpack_pack_raw_body(&self.pk, data, len(data))
 
-    def pack_array_header(self, size_t size):
+    def pack_array_header(self, long long size):
         if size > ITEM_LIMIT:
             raise PackValueError
         cdef int ret = msgpack_pack_array(&self.pk, size)
@@ -285,7 +285,7 @@ cdef class Packer(object):
             self.pk.length = 0
             return buf
 
-    def pack_map_header(self, size_t size):
+    def pack_map_header(self, long long size):
         if size > ITEM_LIMIT:
             raise PackValueError
         cdef int ret = msgpack_pack_map(&self.pk, size)

From 5c052264bc52c4bbdb5e5736f4f24834af46b8d6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 5 May 2016 02:31:03 +0900
Subject: [PATCH 1351/1648] Update ChangeLog

---
 ChangeLog.rst | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index f20bb75..396ccb7 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,29 @@
+0.5.0
+=====
+:release date: TBD
+
+0.5 is important step toward 1.0.  There are some deprecations.
+Please read changes carefully.
+
+Changes
+-------
+
+* Drop Python 2.6 and 3.2 support
+
+* Deprecate useless custom exceptions.  Use ValueError instead of PackValueError,
+  Exception instead of PackException and UnpackException, etc...
+  See msgpack/exceptions.py
+
+* Add `strict_types` option to packer.  It can be used to serialize subclass of
+  builtin types.  For example, when packing object which type is subclass of dict,
+  `default()` is called.
+
+* Pure Python implementation supports packing memoryview object.
+
+Bugs fixed
+----------
+
+
 0.4.7
 =====
 :release date: 2016-01-25

From a5c8bafad4ae29b0173f20dc1d7027219396f6aa Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 5 May 2016 02:46:10 +0900
Subject: [PATCH 1352/1648] Remove unused import (#190)

---
 msgpack/_packer.pyx | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 3981f20..5c950ce 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -2,9 +2,6 @@
 #cython: embedsignature=True
 
 from cpython cimport *
-from libc.stdlib cimport *
-from libc.string cimport *
-from libc.limits cimport *
 
 from msgpack.exceptions import PackValueError, PackOverflowError
 from msgpack import ExtType

From c6c4e59f4cb2ccb6de697de5d55a52e57f6a9a4e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 8 May 2016 16:31:52 +0900
Subject: [PATCH 1353/1648] s/realloc/PyMem_Realloc/ (#193)

---
 msgpack/pack.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/pack.h b/msgpack/pack.h
index a75bdb0..d3aeff7 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -47,7 +47,7 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_
 
     if (len + l > bs) {
         bs = (len + l) * 2;
-        buf = (char*)realloc(buf, bs);
+        buf = (char*)PyMem_Realloc(buf, bs);
         if (!buf) return -1;
     }
     memcpy(buf + len, data, l);

From 318ddfc0527ef3ddf1ad3467ece10c26efa8d741 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 May 2016 09:35:02 +0900
Subject: [PATCH 1354/1648] Remove wrong download_url from package metadata

---
 Makefile | 6 ++++++
 setup.py | 7 +++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 2e53d08..01d8092 100644
--- a/Makefile
+++ b/Makefile
@@ -20,3 +20,9 @@ python3: cython
 
 test:
 	py.test test
+
+.PHONY: clean
+clean:
+	rm -rf build
+	rm msgpack/*.so
+	rm -rf msgpack/__pycache__
diff --git a/setup.py b/setup.py
index 37729bd..d62c8f2 100755
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
+import io
 import os
 import sys
 from glob import glob
@@ -97,9 +98,8 @@ del libraries, macros
 
 
 desc = 'MessagePack (de)serializer.'
-f = open('README.rst')
-long_desc = f.read()
-f.close()
+with io.open('README.rst', encoding='utf-8') as f:
+    long_desc = f.read()
 del f
 
 setup(name='msgpack-python',
@@ -112,7 +112,6 @@ setup(name='msgpack-python',
       description=desc,
       long_description=long_desc,
       url='http://msgpack.org/',
-      download_url='http://pypi.python.org/pypi/msgpack/',
       classifiers=[
           'Programming Language :: Python :: 2',
           'Programming Language :: Python :: 3',

From f421f59a287ae26b7fa83a0cac18650d0dd09c03 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 19 May 2016 22:35:42 +0900
Subject: [PATCH 1355/1648] fallback: Rewrite buffer from array of bytes to
 bytearray

---
 msgpack/fallback.py | 169 ++++++++++++++++++--------------------------
 1 file changed, 69 insertions(+), 100 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index abed3d9..181d7e2 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -86,11 +86,8 @@ def unpack(stream, **kwargs):
     Raises `ExtraData` when `packed` contains extra bytes.
     See :class:`Unpacker` for options.
     """
-    unpacker = Unpacker(stream, **kwargs)
-    ret = unpacker._fb_unpack()
-    if unpacker._fb_got_extradata():
-        raise ExtraData(ret, unpacker._fb_get_extradata())
-    return ret
+    data = stream.read()
+    return unpackb(data, **kwargs)
 
 
 def unpackb(packed, **kwargs):
@@ -121,7 +118,7 @@ class Unpacker(object):
         If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
 
     :param int read_size:
-        Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`)
+        Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
 
     :param bool use_list:
         If true, unpack msgpack array to Python list.
@@ -199,13 +196,9 @@ class Unpacker(object):
             self._fb_feeding = False
 
         #: array of bytes feeded.
-        self._fb_buffers = []
-        #: Which buffer we currently reads
-        self._fb_buf_i = 0
+        self._buffer = b""
         #: Which position we currently reads
-        self._fb_buf_o = 0
-        #: Total size of _fb_bufferes
-        self._fb_buf_n = 0
+        self._buff_i = 0
 
         # When Unpacker is used as an iterable, between the calls to next(),
         # the buffer is not "consumed" completely, for efficiency sake.
@@ -213,13 +206,13 @@ class Unpacker(object):
         # the correct moments, we have to keep track of how sloppy we were.
         # Furthermore, when the buffer is incomplete (that is: in the case
         # we raise an OutOfData) we need to rollback the buffer to the correct
-        # state, which _fb_slopiness records.
-        self._fb_sloppiness = 0
+        # state, which _buf_checkpoint records.
+        self._buf_checkpoint = 0
 
         self._max_buffer_size = max_buffer_size or 2**31-1
         if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")
-        self._read_size = read_size or min(self._max_buffer_size, 4096)
+        self._read_size = read_size or min(self._max_buffer_size, 16*1024)
         self._encoding = encoding
         self._unicode_errors = unicode_errors
         self._use_list = use_list
@@ -248,103 +241,75 @@ class Unpacker(object):
     def feed(self, next_bytes):
         if isinstance(next_bytes, array.array):
             next_bytes = next_bytes.tostring()
-        elif isinstance(next_bytes, bytearray):
-            next_bytes = bytes(next_bytes)
+        if not isinstance(next_bytes, (bytes, bytearray)):
+            raise TypeError("next_bytes should be bytes, bytearray or array.array")
         assert self._fb_feeding
-        if (self._fb_buf_n + len(next_bytes) - self._fb_sloppiness
-                        > self._max_buffer_size):
-            raise BufferFull
-        self._fb_buf_n += len(next_bytes)
-        self._fb_buffers.append(next_bytes)
 
-    def _fb_sloppy_consume(self):
-        """ Gets rid of some of the used parts of the buffer. """
-        if self._fb_buf_i:
-            for i in xrange(self._fb_buf_i):
-                self._fb_buf_n -=  len(self._fb_buffers[i])
-            self._fb_buffers = self._fb_buffers[self._fb_buf_i:]
-            self._fb_buf_i = 0
-        if self._fb_buffers:
-            self._fb_sloppiness = self._fb_buf_o
-        else:
-            self._fb_sloppiness = 0
+        if (len(self._buffer) - self._buff_i + len(next_bytes) > self._max_buffer_size):
+            raise BufferFull
+        # bytes + bytearray -> bytearray
+        # So cast before append
+        self._buffer += bytes(next_bytes)
 
     def _fb_consume(self):
         """ Gets rid of the used parts of the buffer. """
-        if self._fb_buf_i:
-            for i in xrange(self._fb_buf_i):
-                self._fb_buf_n -=  len(self._fb_buffers[i])
-            self._fb_buffers = self._fb_buffers[self._fb_buf_i:]
-            self._fb_buf_i = 0
-        if self._fb_buffers:
-            self._fb_buffers[0] = self._fb_buffers[0][self._fb_buf_o:]
-            self._fb_buf_n -= self._fb_buf_o
-        else:
-            self._fb_buf_n = 0
-        self._fb_buf_o = 0
-        self._fb_sloppiness = 0
+        self._buf_checkpoint = self._buff_i
 
     def _fb_got_extradata(self):
-        if self._fb_buf_i != len(self._fb_buffers):
-            return True
-        if self._fb_feeding:
-            return False
-        if not self.file_like:
-            return False
-        if self.file_like.read(1):
-            return True
-        return False
+        return self._buff_i < len(self._buffer)
 
-    def __iter__(self):
-        return self
+    def _fb_get_extradata(self):
+        return self._buffer[self._buff_i:]
 
     def read_bytes(self, n):
         return self._fb_read(n)
 
-    def _fb_rollback(self):
-        self._fb_buf_i = 0
-        self._fb_buf_o = self._fb_sloppiness
-
-    def _fb_get_extradata(self):
-        bufs = self._fb_buffers[self._fb_buf_i:]
-        if bufs:
-            bufs[0] = bufs[0][self._fb_buf_o:]
-        return b''.join(bufs)
-
     def _fb_read(self, n, write_bytes=None):
-        buffs = self._fb_buffers
-        # We have a redundant codepath for the most common case, such that
-        # pypy optimizes it properly.  This is the case that the read fits
-        # in the current buffer.
-        if (write_bytes is None and self._fb_buf_i < len(buffs) and
-                self._fb_buf_o + n < len(buffs[self._fb_buf_i])):
-            self._fb_buf_o += n
-            return buffs[self._fb_buf_i][self._fb_buf_o - n:self._fb_buf_o]
+        # (int, Optional[Callable]) -> bytearray
+        remain_bytes = len(self._buffer) - self._buff_i - n
 
-        # The remaining cases.
-        ret = b''
-        while len(ret) != n:
-            sliced = n - len(ret)
-            if self._fb_buf_i == len(buffs):
-                if self._fb_feeding:
-                    break
-                to_read = sliced
-                if self._read_size > to_read:
-                    to_read = self._read_size
-                tmp = self.file_like.read(to_read)
-                if not tmp:
-                    break
-                buffs.append(tmp)
-                self._fb_buf_n += len(tmp)
-                continue
-            ret += buffs[self._fb_buf_i][self._fb_buf_o:self._fb_buf_o + sliced]
-            self._fb_buf_o += sliced
-            if self._fb_buf_o >= len(buffs[self._fb_buf_i]):
-                self._fb_buf_o = 0
-                self._fb_buf_i += 1
-        if len(ret) != n:
-            self._fb_rollback()
+        # Fast path: buffer has n bytes already
+        if remain_bytes >= 0:
+            ret = self._buffer[self._buff_i:self._buff_i+n]
+            self._buff_i += n
+            if write_bytes is not None:
+                write_bytes(ret)
+            return ret
+
+        if self._fb_feeding:
+            self._buff_i = self._buf_checkpoint
             raise OutOfData
+
+        # Strip buffer before checkpoint before reading file.
+        if self._buf_checkpoint > 0:
+            self._buffer = self._buffer[self._buf_checkpoint:]
+            self._buff_i -= self._buf_checkpoint
+            self._buf_checkpoint = 0
+
+        # Read from file
+        remain_bytes = -remain_bytes
+        while remain_bytes > 0:
+            to_read_bytes = max(self._read_size, remain_bytes)
+            read_data = self.file_like.read(to_read_bytes)
+            if not read_data:
+                break
+            assert isinstance(read_data, bytes)
+            self._buffer += read_data
+            remain_bytes -= len(read_data)
+
+        if len(self._buffer) < n + self._buff_i:
+            self._buff_i = 0  # rollback
+            raise OutOfData
+
+        if len(self._buffer) == n:
+            # checkpoint == 0
+            ret = self._buffer
+            self._buffer = b""
+            self._buff_i = 0
+        else:
+            ret = self._buffer[self._buff_i:self._buff_i+n]
+            self._buff_i += n
+
         if write_bytes is not None:
             write_bytes(ret)
         return ret
@@ -562,15 +527,19 @@ class Unpacker(object):
         assert typ == TYPE_IMMEDIATE
         return obj
 
-    def next(self):
+    def __iter__(self):
+        return self
+
+    def __next__(self):
         try:
             ret = self._fb_unpack(EX_CONSTRUCT, None)
-            self._fb_sloppy_consume()
+            self._fb_consume()
             return ret
         except OutOfData:
             self._fb_consume()
             raise StopIteration
-    __next__ = next
+
+    next = __next__
 
     def skip(self, write_bytes=None):
         self._fb_unpack(EX_SKIP, write_bytes)

From 3322a769890458ec8df5a365f9303510e59c3efb Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 22 May 2016 11:08:20 +0900
Subject: [PATCH 1356/1648] Remove _fb_ prefix

---
 msgpack/fallback.py | 172 ++++++++++++++++++++++----------------------
 1 file changed, 86 insertions(+), 86 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 181d7e2..95be713 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -100,11 +100,11 @@ def unpackb(packed, **kwargs):
     unpacker = Unpacker(None, **kwargs)
     unpacker.feed(packed)
     try:
-        ret = unpacker._fb_unpack()
+        ret = unpacker._unpack()
     except OutOfData:
         raise UnpackValueError("Data is not enough.")
-    if unpacker._fb_got_extradata():
-        raise ExtraData(ret, unpacker._fb_get_extradata())
+    if unpacker._got_extradata():
+        raise ExtraData(ret, unpacker._get_extradata())
     return ret
 
 
@@ -188,12 +188,12 @@ class Unpacker(object):
                  max_map_len=2147483647,
                  max_ext_len=2147483647):
         if file_like is None:
-            self._fb_feeding = True
+            self._feeding = True
         else:
             if not callable(file_like.read):
                 raise TypeError("`file_like.read` must be callable")
             self.file_like = file_like
-            self._fb_feeding = False
+            self._feeding = False
 
         #: array of bytes feeded.
         self._buffer = b""
@@ -243,7 +243,7 @@ class Unpacker(object):
             next_bytes = next_bytes.tostring()
         if not isinstance(next_bytes, (bytes, bytearray)):
             raise TypeError("next_bytes should be bytes, bytearray or array.array")
-        assert self._fb_feeding
+        assert self._feeding
 
         if (len(self._buffer) - self._buff_i + len(next_bytes) > self._max_buffer_size):
             raise BufferFull
@@ -251,20 +251,20 @@ class Unpacker(object):
         # So cast before append
         self._buffer += bytes(next_bytes)
 
-    def _fb_consume(self):
+    def _consume(self):
         """ Gets rid of the used parts of the buffer. """
         self._buf_checkpoint = self._buff_i
 
-    def _fb_got_extradata(self):
+    def _got_extradata(self):
         return self._buff_i < len(self._buffer)
 
-    def _fb_get_extradata(self):
+    def _get_extradata(self):
         return self._buffer[self._buff_i:]
 
     def read_bytes(self, n):
-        return self._fb_read(n)
+        return self._read(n)
 
-    def _fb_read(self, n, write_bytes=None):
+    def _read(self, n, write_bytes=None):
         # (int, Optional[Callable]) -> bytearray
         remain_bytes = len(self._buffer) - self._buff_i - n
 
@@ -276,7 +276,7 @@ class Unpacker(object):
                 write_bytes(ret)
             return ret
 
-        if self._fb_feeding:
+        if self._feeding:
             self._buff_i = self._buf_checkpoint
             raise OutOfData
 
@@ -318,7 +318,7 @@ class Unpacker(object):
         typ = TYPE_IMMEDIATE
         n = 0
         obj = None
-        c = self._fb_read(1, write_bytes)
+        c = self._read(1, write_bytes)
         b = ord(c)
         if   b & 0b10000000 == 0:
             obj = b
@@ -326,7 +326,7 @@ class Unpacker(object):
             obj = struct.unpack("b", c)[0]
         elif b & 0b11100000 == 0b10100000:
             n = b & 0b00011111
-            obj = self._fb_read(n, write_bytes)
+            obj = self._read(n, write_bytes)
             typ = TYPE_RAW
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
@@ -348,120 +348,120 @@ class Unpacker(object):
             obj = True
         elif b == 0xc4:
             typ = TYPE_BIN
-            n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            n = struct.unpack("B", self._read(1, write_bytes))[0]
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
-            obj = self._fb_read(n, write_bytes)
+            obj = self._read(n, write_bytes)
         elif b == 0xc5:
             typ = TYPE_BIN
-            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            n = struct.unpack(">H", self._read(2, write_bytes))[0]
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
-            obj = self._fb_read(n, write_bytes)
+            obj = self._read(n, write_bytes)
         elif b == 0xc6:
             typ = TYPE_BIN
-            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            n = struct.unpack(">I", self._read(4, write_bytes))[0]
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
-            obj = self._fb_read(n, write_bytes)
+            obj = self._read(n, write_bytes)
         elif b == 0xc7:  # ext 8
             typ = TYPE_EXT
-            L, n = struct.unpack('Bb', self._fb_read(2, write_bytes))
+            L, n = struct.unpack('Bb', self._read(2, write_bytes))
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
-            obj = self._fb_read(L, write_bytes)
+            obj = self._read(L, write_bytes)
         elif b == 0xc8:  # ext 16
             typ = TYPE_EXT
-            L, n = struct.unpack('>Hb', self._fb_read(3, write_bytes))
+            L, n = struct.unpack('>Hb', self._read(3, write_bytes))
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
-            obj = self._fb_read(L, write_bytes)
+            obj = self._read(L, write_bytes)
         elif b == 0xc9:  # ext 32
             typ = TYPE_EXT
-            L, n = struct.unpack('>Ib', self._fb_read(5, write_bytes))
+            L, n = struct.unpack('>Ib', self._read(5, write_bytes))
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
-            obj = self._fb_read(L, write_bytes)
+            obj = self._read(L, write_bytes)
         elif b == 0xca:
-            obj = struct.unpack(">f", self._fb_read(4, write_bytes))[0]
+            obj = struct.unpack(">f", self._read(4, write_bytes))[0]
         elif b == 0xcb:
-            obj = struct.unpack(">d", self._fb_read(8, write_bytes))[0]
+            obj = struct.unpack(">d", self._read(8, write_bytes))[0]
         elif b == 0xcc:
-            obj = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            obj = struct.unpack("B", self._read(1, write_bytes))[0]
         elif b == 0xcd:
-            obj = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            obj = struct.unpack(">H", self._read(2, write_bytes))[0]
         elif b == 0xce:
-            obj = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            obj = struct.unpack(">I", self._read(4, write_bytes))[0]
         elif b == 0xcf:
-            obj = struct.unpack(">Q", self._fb_read(8, write_bytes))[0]
+            obj = struct.unpack(">Q", self._read(8, write_bytes))[0]
         elif b == 0xd0:
-            obj = struct.unpack("b", self._fb_read(1, write_bytes))[0]
+            obj = struct.unpack("b", self._read(1, write_bytes))[0]
         elif b == 0xd1:
-            obj = struct.unpack(">h", self._fb_read(2, write_bytes))[0]
+            obj = struct.unpack(">h", self._read(2, write_bytes))[0]
         elif b == 0xd2:
-            obj = struct.unpack(">i", self._fb_read(4, write_bytes))[0]
+            obj = struct.unpack(">i", self._read(4, write_bytes))[0]
         elif b == 0xd3:
-            obj = struct.unpack(">q", self._fb_read(8, write_bytes))[0]
+            obj = struct.unpack(">q", self._read(8, write_bytes))[0]
         elif b == 0xd4:  # fixext 1
             typ = TYPE_EXT
             if self._max_ext_len < 1:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
-            n, obj = struct.unpack('b1s', self._fb_read(2, write_bytes))
+            n, obj = struct.unpack('b1s', self._read(2, write_bytes))
         elif b == 0xd5:  # fixext 2
             typ = TYPE_EXT
             if self._max_ext_len < 2:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
-            n, obj = struct.unpack('b2s', self._fb_read(3, write_bytes))
+            n, obj = struct.unpack('b2s', self._read(3, write_bytes))
         elif b == 0xd6:  # fixext 4
             typ = TYPE_EXT
             if self._max_ext_len < 4:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
-            n, obj = struct.unpack('b4s', self._fb_read(5, write_bytes))
+            n, obj = struct.unpack('b4s', self._read(5, write_bytes))
         elif b == 0xd7:  # fixext 8
             typ = TYPE_EXT
             if self._max_ext_len < 8:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
-            n, obj = struct.unpack('b8s', self._fb_read(9, write_bytes))
+            n, obj = struct.unpack('b8s', self._read(9, write_bytes))
         elif b == 0xd8:  # fixext 16
             typ = TYPE_EXT
             if self._max_ext_len < 16:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
-            n, obj = struct.unpack('b16s', self._fb_read(17, write_bytes))
+            n, obj = struct.unpack('b16s', self._read(17, write_bytes))
         elif b == 0xd9:
             typ = TYPE_RAW
-            n = struct.unpack("B", self._fb_read(1, write_bytes))[0]
+            n = struct.unpack("B", self._read(1, write_bytes))[0]
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
-            obj = self._fb_read(n, write_bytes)
+            obj = self._read(n, write_bytes)
         elif b == 0xda:
             typ = TYPE_RAW
-            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            n = struct.unpack(">H", self._read(2, write_bytes))[0]
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
-            obj = self._fb_read(n, write_bytes)
+            obj = self._read(n, write_bytes)
         elif b == 0xdb:
             typ = TYPE_RAW
-            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            n = struct.unpack(">I", self._read(4, write_bytes))[0]
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
-            obj = self._fb_read(n, write_bytes)
+            obj = self._read(n, write_bytes)
         elif b == 0xdc:
-            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            n = struct.unpack(">H", self._read(2, write_bytes))[0]
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
             typ = TYPE_ARRAY
         elif b == 0xdd:
-            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            n = struct.unpack(">I", self._read(4, write_bytes))[0]
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
             typ = TYPE_ARRAY
         elif b == 0xde:
-            n = struct.unpack(">H", self._fb_read(2, write_bytes))[0]
+            n = struct.unpack(">H", self._read(2, write_bytes))[0]
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         elif b == 0xdf:
-            n = struct.unpack(">I", self._fb_read(4, write_bytes))[0]
+            n = struct.unpack(">I", self._read(4, write_bytes))[0]
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
@@ -469,7 +469,7 @@ class Unpacker(object):
             raise UnpackValueError("Unknown header: 0x%x" % b)
         return typ, n, obj
 
-    def _fb_unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
+    def _unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
         typ, n, obj = self._read_header(execute, write_bytes)
 
         if execute == EX_READ_ARRAY_HEADER:
@@ -485,11 +485,11 @@ class Unpacker(object):
             if execute == EX_SKIP:
                 for i in xrange(n):
                     # TODO check whether we need to call `list_hook`
-                    self._fb_unpack(EX_SKIP, write_bytes)
+                    self._unpack(EX_SKIP, write_bytes)
                 return
             ret = newlist_hint(n)
             for i in xrange(n):
-                ret.append(self._fb_unpack(EX_CONSTRUCT, write_bytes))
+                ret.append(self._unpack(EX_CONSTRUCT, write_bytes))
             if self._list_hook is not None:
                 ret = self._list_hook(ret)
             # TODO is the interaction between `list_hook` and `use_list` ok?
@@ -498,19 +498,19 @@ class Unpacker(object):
             if execute == EX_SKIP:
                 for i in xrange(n):
                     # TODO check whether we need to call hooks
-                    self._fb_unpack(EX_SKIP, write_bytes)
-                    self._fb_unpack(EX_SKIP, write_bytes)
+                    self._unpack(EX_SKIP, write_bytes)
+                    self._unpack(EX_SKIP, write_bytes)
                 return
             if self._object_pairs_hook is not None:
                 ret = self._object_pairs_hook(
-                    (self._fb_unpack(EX_CONSTRUCT, write_bytes),
-                     self._fb_unpack(EX_CONSTRUCT, write_bytes))
+                    (self._unpack(EX_CONSTRUCT, write_bytes),
+                     self._unpack(EX_CONSTRUCT, write_bytes))
                     for _ in xrange(n))
             else:
                 ret = {}
                 for _ in xrange(n):
-                    key = self._fb_unpack(EX_CONSTRUCT, write_bytes)
-                    ret[key] = self._fb_unpack(EX_CONSTRUCT, write_bytes)
+                    key = self._unpack(EX_CONSTRUCT, write_bytes)
+                    ret[key] = self._unpack(EX_CONSTRUCT, write_bytes)
                 if self._object_hook is not None:
                     ret = self._object_hook(ret)
             return ret
@@ -532,32 +532,32 @@ class Unpacker(object):
 
     def __next__(self):
         try:
-            ret = self._fb_unpack(EX_CONSTRUCT, None)
-            self._fb_consume()
+            ret = self._unpack(EX_CONSTRUCT, None)
+            self._consume()
             return ret
         except OutOfData:
-            self._fb_consume()
+            self._consume()
             raise StopIteration
 
     next = __next__
 
     def skip(self, write_bytes=None):
-        self._fb_unpack(EX_SKIP, write_bytes)
-        self._fb_consume()
+        self._unpack(EX_SKIP, write_bytes)
+        self._consume()
 
     def unpack(self, write_bytes=None):
-        ret = self._fb_unpack(EX_CONSTRUCT, write_bytes)
-        self._fb_consume()
+        ret = self._unpack(EX_CONSTRUCT, write_bytes)
+        self._consume()
         return ret
 
     def read_array_header(self, write_bytes=None):
-        ret = self._fb_unpack(EX_READ_ARRAY_HEADER, write_bytes)
-        self._fb_consume()
+        ret = self._unpack(EX_READ_ARRAY_HEADER, write_bytes)
+        self._consume()
         return ret
 
     def read_map_header(self, write_bytes=None):
-        ret = self._fb_unpack(EX_READ_MAP_HEADER, write_bytes)
-        self._fb_consume()
+        ret = self._unpack(EX_READ_MAP_HEADER, write_bytes)
+        self._consume()
         return ret
 
 
@@ -658,7 +658,7 @@ class Packer(object):
                 n = len(obj)
                 if n >= 2**32:
                     raise PackValueError("Bytes is too large")
-                self._fb_pack_bin_header(n)
+                self._pack_bin_header(n)
                 return self._buffer.write(obj)
             if check(obj, Unicode):
                 if self._encoding is None:
@@ -669,13 +669,13 @@ class Packer(object):
                 n = len(obj)
                 if n >= 2**32:
                     raise PackValueError("String is too large")
-                self._fb_pack_raw_header(n)
+                self._pack_raw_header(n)
                 return self._buffer.write(obj)
             if check(obj, memoryview):
                 n = len(obj) * obj.itemsize
                 if n >= 2**32:
                     raise PackValueError("Memoryview is too large")
-                self._fb_pack_bin_header(n)
+                self._pack_bin_header(n)
                 return self._buffer.write(obj)
             if check(obj, float):
                 if self._use_float:
@@ -708,12 +708,12 @@ class Packer(object):
                 return
             if check(obj, list_types):
                 n = len(obj)
-                self._fb_pack_array_header(n)
+                self._pack_array_header(n)
                 for i in xrange(n):
                     self._pack(obj[i], nest_limit - 1)
                 return
             if check(obj, dict):
-                return self._fb_pack_map_pairs(len(obj), dict_iteritems(obj),
+                return self._pack_map_pairs(len(obj), dict_iteritems(obj),
                                                nest_limit - 1)
             if not default_used and self._default is not None:
                 obj = self._default(obj)
@@ -731,7 +731,7 @@ class Packer(object):
         return ret
 
     def pack_map_pairs(self, pairs):
-        self._fb_pack_map_pairs(len(pairs), pairs)
+        self._pack_map_pairs(len(pairs), pairs)
         ret = self._buffer.getvalue()
         if self._autoreset:
             self._buffer = StringIO()
@@ -742,7 +742,7 @@ class Packer(object):
     def pack_array_header(self, n):
         if n >= 2**32:
             raise PackValueError
-        self._fb_pack_array_header(n)
+        self._pack_array_header(n)
         ret = self._buffer.getvalue()
         if self._autoreset:
             self._buffer = StringIO()
@@ -753,7 +753,7 @@ class Packer(object):
     def pack_map_header(self, n):
         if n >= 2**32:
             raise PackValueError
-        self._fb_pack_map_header(n)
+        self._pack_map_header(n)
         ret = self._buffer.getvalue()
         if self._autoreset:
             self._buffer = StringIO()
@@ -790,7 +790,7 @@ class Packer(object):
         self._buffer.write(struct.pack('B', typecode))
         self._buffer.write(data)
 
-    def _fb_pack_array_header(self, n):
+    def _pack_array_header(self, n):
         if n <= 0x0f:
             return self._buffer.write(struct.pack('B', 0x90 + n))
         if n <= 0xffff:
@@ -799,7 +799,7 @@ class Packer(object):
             return self._buffer.write(struct.pack(">BI", 0xdd, n))
         raise PackValueError("Array is too large")
 
-    def _fb_pack_map_header(self, n):
+    def _pack_map_header(self, n):
         if n <= 0x0f:
             return self._buffer.write(struct.pack('B', 0x80 + n))
         if n <= 0xffff:
@@ -808,13 +808,13 @@ class Packer(object):
             return self._buffer.write(struct.pack(">BI", 0xdf, n))
         raise PackValueError("Dict is too large")
 
-    def _fb_pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
-        self._fb_pack_map_header(n)
+    def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
+        self._pack_map_header(n)
         for (k, v) in pairs:
             self._pack(k, nest_limit - 1)
             self._pack(v, nest_limit - 1)
 
-    def _fb_pack_raw_header(self, n):
+    def _pack_raw_header(self, n):
         if n <= 0x1f:
             self._buffer.write(struct.pack('B', 0xa0 + n))
         elif self._use_bin_type and n <= 0xff:
@@ -826,9 +826,9 @@ class Packer(object):
         else:
             raise PackValueError('Raw is too large')
 
-    def _fb_pack_bin_header(self, n):
+    def _pack_bin_header(self, n):
         if not self._use_bin_type:
-            return self._fb_pack_raw_header(n)
+            return self._pack_raw_header(n)
         elif n <= 0xff:
             return self._buffer.write(struct.pack('>BB', 0xc4, n))
         elif n <= 0xffff:

From e9c42fa523b51c184b581d6eab85f0ad40dff620 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 22 May 2016 13:31:01 +0900
Subject: [PATCH 1357/1648] fallback: simplify write_bytes callback
 implementation

---
 msgpack/fallback.py | 124 +++++++++++++++++++++++---------------------
 1 file changed, 64 insertions(+), 60 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 95be713..ecdbec4 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -264,7 +264,7 @@ class Unpacker(object):
     def read_bytes(self, n):
         return self._read(n)
 
-    def _read(self, n, write_bytes=None):
+    def _read(self, n):
         # (int, Optional[Callable]) -> bytearray
         remain_bytes = len(self._buffer) - self._buff_i - n
 
@@ -272,8 +272,6 @@ class Unpacker(object):
         if remain_bytes >= 0:
             ret = self._buffer[self._buff_i:self._buff_i+n]
             self._buff_i += n
-            if write_bytes is not None:
-                write_bytes(ret)
             return ret
 
         if self._feeding:
@@ -310,15 +308,13 @@ class Unpacker(object):
             ret = self._buffer[self._buff_i:self._buff_i+n]
             self._buff_i += n
 
-        if write_bytes is not None:
-            write_bytes(ret)
         return ret
 
-    def _read_header(self, execute=EX_CONSTRUCT, write_bytes=None):
+    def _read_header(self, execute=EX_CONSTRUCT):
         typ = TYPE_IMMEDIATE
         n = 0
         obj = None
-        c = self._read(1, write_bytes)
+        c = self._read(1)
         b = ord(c)
         if   b & 0b10000000 == 0:
             obj = b
@@ -326,7 +322,7 @@ class Unpacker(object):
             obj = struct.unpack("b", c)[0]
         elif b & 0b11100000 == 0b10100000:
             n = b & 0b00011111
-            obj = self._read(n, write_bytes)
+            obj = self._read(n)
             typ = TYPE_RAW
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
@@ -348,120 +344,120 @@ class Unpacker(object):
             obj = True
         elif b == 0xc4:
             typ = TYPE_BIN
-            n = struct.unpack("B", self._read(1, write_bytes))[0]
+            n = struct.unpack("B", self._read(1))[0]
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
-            obj = self._read(n, write_bytes)
+            obj = self._read(n)
         elif b == 0xc5:
             typ = TYPE_BIN
-            n = struct.unpack(">H", self._read(2, write_bytes))[0]
+            n = struct.unpack(">H", self._read(2))[0]
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
-            obj = self._read(n, write_bytes)
+            obj = self._read(n)
         elif b == 0xc6:
             typ = TYPE_BIN
-            n = struct.unpack(">I", self._read(4, write_bytes))[0]
+            n = struct.unpack(">I", self._read(4))[0]
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
-            obj = self._read(n, write_bytes)
+            obj = self._read(n)
         elif b == 0xc7:  # ext 8
             typ = TYPE_EXT
-            L, n = struct.unpack('Bb', self._read(2, write_bytes))
+            L, n = struct.unpack('Bb', self._read(2))
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
-            obj = self._read(L, write_bytes)
+            obj = self._read(L)
         elif b == 0xc8:  # ext 16
             typ = TYPE_EXT
-            L, n = struct.unpack('>Hb', self._read(3, write_bytes))
+            L, n = struct.unpack('>Hb', self._read(3))
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
-            obj = self._read(L, write_bytes)
+            obj = self._read(L)
         elif b == 0xc9:  # ext 32
             typ = TYPE_EXT
-            L, n = struct.unpack('>Ib', self._read(5, write_bytes))
+            L, n = struct.unpack('>Ib', self._read(5))
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
-            obj = self._read(L, write_bytes)
+            obj = self._read(L)
         elif b == 0xca:
-            obj = struct.unpack(">f", self._read(4, write_bytes))[0]
+            obj = struct.unpack(">f", self._read(4))[0]
         elif b == 0xcb:
-            obj = struct.unpack(">d", self._read(8, write_bytes))[0]
+            obj = struct.unpack(">d", self._read(8))[0]
         elif b == 0xcc:
-            obj = struct.unpack("B", self._read(1, write_bytes))[0]
+            obj = struct.unpack("B", self._read(1))[0]
         elif b == 0xcd:
-            obj = struct.unpack(">H", self._read(2, write_bytes))[0]
+            obj = struct.unpack(">H", self._read(2))[0]
         elif b == 0xce:
-            obj = struct.unpack(">I", self._read(4, write_bytes))[0]
+            obj = struct.unpack(">I", self._read(4))[0]
         elif b == 0xcf:
-            obj = struct.unpack(">Q", self._read(8, write_bytes))[0]
+            obj = struct.unpack(">Q", self._read(8))[0]
         elif b == 0xd0:
-            obj = struct.unpack("b", self._read(1, write_bytes))[0]
+            obj = struct.unpack("b", self._read(1))[0]
         elif b == 0xd1:
-            obj = struct.unpack(">h", self._read(2, write_bytes))[0]
+            obj = struct.unpack(">h", self._read(2))[0]
         elif b == 0xd2:
-            obj = struct.unpack(">i", self._read(4, write_bytes))[0]
+            obj = struct.unpack(">i", self._read(4))[0]
         elif b == 0xd3:
-            obj = struct.unpack(">q", self._read(8, write_bytes))[0]
+            obj = struct.unpack(">q", self._read(8))[0]
         elif b == 0xd4:  # fixext 1
             typ = TYPE_EXT
             if self._max_ext_len < 1:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
-            n, obj = struct.unpack('b1s', self._read(2, write_bytes))
+            n, obj = struct.unpack('b1s', self._read(2))
         elif b == 0xd5:  # fixext 2
             typ = TYPE_EXT
             if self._max_ext_len < 2:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
-            n, obj = struct.unpack('b2s', self._read(3, write_bytes))
+            n, obj = struct.unpack('b2s', self._read(3))
         elif b == 0xd6:  # fixext 4
             typ = TYPE_EXT
             if self._max_ext_len < 4:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
-            n, obj = struct.unpack('b4s', self._read(5, write_bytes))
+            n, obj = struct.unpack('b4s', self._read(5))
         elif b == 0xd7:  # fixext 8
             typ = TYPE_EXT
             if self._max_ext_len < 8:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
-            n, obj = struct.unpack('b8s', self._read(9, write_bytes))
+            n, obj = struct.unpack('b8s', self._read(9))
         elif b == 0xd8:  # fixext 16
             typ = TYPE_EXT
             if self._max_ext_len < 16:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
-            n, obj = struct.unpack('b16s', self._read(17, write_bytes))
+            n, obj = struct.unpack('b16s', self._read(17))
         elif b == 0xd9:
             typ = TYPE_RAW
-            n = struct.unpack("B", self._read(1, write_bytes))[0]
+            n = struct.unpack("B", self._read(1))[0]
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
-            obj = self._read(n, write_bytes)
+            obj = self._read(n)
         elif b == 0xda:
             typ = TYPE_RAW
-            n = struct.unpack(">H", self._read(2, write_bytes))[0]
+            n = struct.unpack(">H", self._read(2))[0]
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
-            obj = self._read(n, write_bytes)
+            obj = self._read(n)
         elif b == 0xdb:
             typ = TYPE_RAW
-            n = struct.unpack(">I", self._read(4, write_bytes))[0]
+            n = struct.unpack(">I", self._read(4))[0]
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
-            obj = self._read(n, write_bytes)
+            obj = self._read(n)
         elif b == 0xdc:
-            n = struct.unpack(">H", self._read(2, write_bytes))[0]
+            n = struct.unpack(">H", self._read(2))[0]
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
             typ = TYPE_ARRAY
         elif b == 0xdd:
-            n = struct.unpack(">I", self._read(4, write_bytes))[0]
+            n = struct.unpack(">I", self._read(4))[0]
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
             typ = TYPE_ARRAY
         elif b == 0xde:
-            n = struct.unpack(">H", self._read(2, write_bytes))[0]
+            n = struct.unpack(">H", self._read(2))[0]
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         elif b == 0xdf:
-            n = struct.unpack(">I", self._read(4, write_bytes))[0]
+            n = struct.unpack(">I", self._read(4))[0]
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
@@ -469,8 +465,8 @@ class Unpacker(object):
             raise UnpackValueError("Unknown header: 0x%x" % b)
         return typ, n, obj
 
-    def _unpack(self, execute=EX_CONSTRUCT, write_bytes=None):
-        typ, n, obj = self._read_header(execute, write_bytes)
+    def _unpack(self, execute=EX_CONSTRUCT):
+        typ, n, obj = self._read_header(execute)
 
         if execute == EX_READ_ARRAY_HEADER:
             if typ != TYPE_ARRAY:
@@ -485,11 +481,11 @@ class Unpacker(object):
             if execute == EX_SKIP:
                 for i in xrange(n):
                     # TODO check whether we need to call `list_hook`
-                    self._unpack(EX_SKIP, write_bytes)
+                    self._unpack(EX_SKIP)
                 return
             ret = newlist_hint(n)
             for i in xrange(n):
-                ret.append(self._unpack(EX_CONSTRUCT, write_bytes))
+                ret.append(self._unpack(EX_CONSTRUCT))
             if self._list_hook is not None:
                 ret = self._list_hook(ret)
             # TODO is the interaction between `list_hook` and `use_list` ok?
@@ -498,19 +494,19 @@ class Unpacker(object):
             if execute == EX_SKIP:
                 for i in xrange(n):
                     # TODO check whether we need to call hooks
-                    self._unpack(EX_SKIP, write_bytes)
-                    self._unpack(EX_SKIP, write_bytes)
+                    self._unpack(EX_SKIP)
+                    self._unpack(EX_SKIP)
                 return
             if self._object_pairs_hook is not None:
                 ret = self._object_pairs_hook(
-                    (self._unpack(EX_CONSTRUCT, write_bytes),
-                     self._unpack(EX_CONSTRUCT, write_bytes))
+                    (self._unpack(EX_CONSTRUCT),
+                     self._unpack(EX_CONSTRUCT))
                     for _ in xrange(n))
             else:
                 ret = {}
                 for _ in xrange(n):
-                    key = self._unpack(EX_CONSTRUCT, write_bytes)
-                    ret[key] = self._unpack(EX_CONSTRUCT, write_bytes)
+                    key = self._unpack(EX_CONSTRUCT)
+                    ret[key] = self._unpack(EX_CONSTRUCT)
                 if self._object_hook is not None:
                     ret = self._object_hook(ret)
             return ret
@@ -532,7 +528,7 @@ class Unpacker(object):
 
     def __next__(self):
         try:
-            ret = self._unpack(EX_CONSTRUCT, None)
+            ret = self._unpack(EX_CONSTRUCT)
             self._consume()
             return ret
         except OutOfData:
@@ -542,21 +538,29 @@ class Unpacker(object):
     next = __next__
 
     def skip(self, write_bytes=None):
-        self._unpack(EX_SKIP, write_bytes)
+        self._unpack(EX_SKIP)
+        if write_bytes is not None:
+            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
 
     def unpack(self, write_bytes=None):
-        ret = self._unpack(EX_CONSTRUCT, write_bytes)
+        ret = self._unpack(EX_CONSTRUCT)
+        if write_bytes is not None:
+            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
         return ret
 
     def read_array_header(self, write_bytes=None):
-        ret = self._unpack(EX_READ_ARRAY_HEADER, write_bytes)
+        ret = self._unpack(EX_READ_ARRAY_HEADER)
+        if write_bytes is not None:
+            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
         return ret
 
     def read_map_header(self, write_bytes=None):
-        ret = self._unpack(EX_READ_MAP_HEADER, write_bytes)
+        ret = self._unpack(EX_READ_MAP_HEADER)
+        if write_bytes is not None:
+            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
         return ret
 

From 6b8919355d6acdda74acc568dd348598e552f003 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 24 May 2016 02:46:29 +0900
Subject: [PATCH 1358/1648] fallback: Use struct.unpack_from when possible

---
 msgpack/fallback.py | 150 +++++++++++++++++++++++++++++---------------
 1 file changed, 100 insertions(+), 50 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index ecdbec4..9b32610 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -265,14 +265,18 @@ class Unpacker(object):
         return self._read(n)
 
     def _read(self, n):
-        # (int, Optional[Callable]) -> bytearray
+        # (int) -> bytearray
+        self._reserve(n)
+        i = self._buff_i
+        self._buff_i = i+n
+        return self._buffer[i:i+n]
+
+    def _reserve(self, n):
         remain_bytes = len(self._buffer) - self._buff_i - n
 
         # Fast path: buffer has n bytes already
         if remain_bytes >= 0:
-            ret = self._buffer[self._buff_i:self._buff_i+n]
-            self._buff_i += n
-            return ret
+            return
 
         if self._feeding:
             self._buff_i = self._buf_checkpoint
@@ -299,33 +303,23 @@ class Unpacker(object):
             self._buff_i = 0  # rollback
             raise OutOfData
 
-        if len(self._buffer) == n:
-            # checkpoint == 0
-            ret = self._buffer
-            self._buffer = b""
-            self._buff_i = 0
-        else:
-            ret = self._buffer[self._buff_i:self._buff_i+n]
-            self._buff_i += n
-
-        return ret
-
     def _read_header(self, execute=EX_CONSTRUCT):
         typ = TYPE_IMMEDIATE
         n = 0
         obj = None
-        c = self._read(1)
-        b = ord(c)
-        if   b & 0b10000000 == 0:
+        self._reserve(1)
+        b = struct.unpack_from("B", self._buffer, self._buff_i)[0]
+        self._buff_i += 1
+        if b & 0b10000000 == 0:
             obj = b
         elif b & 0b11100000 == 0b11100000:
-            obj = struct.unpack("b", c)[0]
+            obj = -1 - (b ^ 0xff)
         elif b & 0b11100000 == 0b10100000:
             n = b & 0b00011111
-            obj = self._read(n)
             typ = TYPE_RAW
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+            obj = self._read(n)
         elif b & 0b11110000 == 0b10010000:
             n = b & 0b00001111
             typ = TYPE_ARRAY
@@ -344,120 +338,176 @@ class Unpacker(object):
             obj = True
         elif b == 0xc4:
             typ = TYPE_BIN
-            n = struct.unpack("B", self._read(1))[0]
+            self._reserve(1)
+            n = struct.unpack_from("B", self._buffer, self._buff_i)[0]
+            self._buff_i += 1
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
         elif b == 0xc5:
             typ = TYPE_BIN
-            n = struct.unpack(">H", self._read(2))[0]
+            self._reserve(2)
+            n = struct.unpack_from(">H", self._buffer, self._buff_i)[0]
+            self._buff_i += 2
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
         elif b == 0xc6:
             typ = TYPE_BIN
-            n = struct.unpack(">I", self._read(4))[0]
+            self._reserve(4)
+            n = struct.unpack_from(">I", self._buffer, self._buff_i)[0]
+            self._buff_i += 4
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
         elif b == 0xc7:  # ext 8
             typ = TYPE_EXT
-            L, n = struct.unpack('Bb', self._read(2))
+            self._reserve(2)
+            L, n = struct.unpack_from('Bb', self._buffer, self._buff_i)
+            self._buff_i += 2
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
         elif b == 0xc8:  # ext 16
             typ = TYPE_EXT
-            L, n = struct.unpack('>Hb', self._read(3))
+            self._reserve(3)
+            L, n = struct.unpack_from('>Hb', self._buffer, self._buff_i)
+            self._buff_i += 3
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
         elif b == 0xc9:  # ext 32
             typ = TYPE_EXT
-            L, n = struct.unpack('>Ib', self._read(5))
+            self._reserve(5)
+            L, n = struct.unpack_from('>Ib', self._buffer, self._buff_i)
+            self._buff_i += 5
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
         elif b == 0xca:
-            obj = struct.unpack(">f", self._read(4))[0]
+            self._reserve(4)
+            obj = struct.unpack_from(">f", self._buffer, self._buff_i)[0]
+            self._buff_i += 4
         elif b == 0xcb:
-            obj = struct.unpack(">d", self._read(8))[0]
+            self._reserve(8)
+            obj = struct.unpack_from(">d", self._buffer, self._buff_i)[0]
+            self._buff_i += 8
         elif b == 0xcc:
-            obj = struct.unpack("B", self._read(1))[0]
+            self._reserve(1)
+            obj = struct.unpack_from("B", self._buffer, self._buff_i)[0]
+            self._buff_i += 1
         elif b == 0xcd:
-            obj = struct.unpack(">H", self._read(2))[0]
+            self._reserve(2)
+            obj = struct.unpack_from(">H", self._buffer, self._buff_i)[0]
+            self._buff_i += 2
         elif b == 0xce:
-            obj = struct.unpack(">I", self._read(4))[0]
+            self._reserve(4)
+            obj = struct.unpack_from(">I", self._buffer, self._buff_i)[0]
+            self._buff_i += 4
         elif b == 0xcf:
-            obj = struct.unpack(">Q", self._read(8))[0]
+            self._reserve(8)
+            obj = struct.unpack_from(">Q", self._buffer, self._buff_i)[0]
+            self._buff_i += 8
         elif b == 0xd0:
-            obj = struct.unpack("b", self._read(1))[0]
+            self._reserve(1)
+            obj = struct.unpack_from("b", self._buffer, self._buff_i)[0]
+            self._buff_i += 1
         elif b == 0xd1:
-            obj = struct.unpack(">h", self._read(2))[0]
+            self._reserve(2)
+            obj = struct.unpack_from(">h", self._buffer, self._buff_i)[0]
+            self._buff_i += 2
         elif b == 0xd2:
-            obj = struct.unpack(">i", self._read(4))[0]
+            self._reserve(4)
+            obj = struct.unpack_from(">i", self._buffer, self._buff_i)[0]
+            self._buff_i += 4
         elif b == 0xd3:
-            obj = struct.unpack(">q", self._read(8))[0]
+            self._reserve(8)
+            obj = struct.unpack_from(">q", self._buffer, self._buff_i)[0]
+            self._buff_i += 8
         elif b == 0xd4:  # fixext 1
             typ = TYPE_EXT
             if self._max_ext_len < 1:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
-            n, obj = struct.unpack('b1s', self._read(2))
+            self._reserve(2)
+            n, obj = struct.unpack_from("b1s", self._buffer, self._buff_i)
+            self._buff_i += 2
         elif b == 0xd5:  # fixext 2
             typ = TYPE_EXT
             if self._max_ext_len < 2:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
-            n, obj = struct.unpack('b2s', self._read(3))
+            self._reserve(3)
+            n, obj = struct.unpack_from("b2s", self._buffer, self._buff_i)
+            self._buff_i += 3
         elif b == 0xd6:  # fixext 4
             typ = TYPE_EXT
             if self._max_ext_len < 4:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
-            n, obj = struct.unpack('b4s', self._read(5))
+            self._reserve(5)
+            n, obj = struct.unpack_from("b4s", self._buffer, self._buff_i)
+            self._buff_i += 5
         elif b == 0xd7:  # fixext 8
             typ = TYPE_EXT
             if self._max_ext_len < 8:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
-            n, obj = struct.unpack('b8s', self._read(9))
+            self._reserve(9)
+            n, obj = struct.unpack_from("b8s", self._buffer, self._buff_i)
+            self._buff_i += 9
         elif b == 0xd8:  # fixext 16
             typ = TYPE_EXT
             if self._max_ext_len < 16:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
-            n, obj = struct.unpack('b16s', self._read(17))
+            self._reserve(17)
+            n, obj = struct.unpack_from("b16s", self._buffer, self._buff_i)
+            self._buff_i += 17
         elif b == 0xd9:
             typ = TYPE_RAW
-            n = struct.unpack("B", self._read(1))[0]
+            self._reserve(1)
+            n, = struct.unpack_from("B", self._buffer, self._buff_i)
+            self._buff_i += 1
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
         elif b == 0xda:
             typ = TYPE_RAW
-            n = struct.unpack(">H", self._read(2))[0]
+            self._reserve(2)
+            n, = struct.unpack_from(">H", self._buffer, self._buff_i)
+            self._buff_i += 2
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
         elif b == 0xdb:
             typ = TYPE_RAW
-            n = struct.unpack(">I", self._read(4))[0]
+            self._reserve(4)
+            n, = struct.unpack_from(">I", self._buffer, self._buff_i)
+            self._buff_i += 4
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
         elif b == 0xdc:
-            n = struct.unpack(">H", self._read(2))[0]
+            typ = TYPE_ARRAY
+            self._reserve(2)
+            n, = struct.unpack_from(">H", self._buffer, self._buff_i)
+            self._buff_i += 2
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
-            typ = TYPE_ARRAY
         elif b == 0xdd:
-            n = struct.unpack(">I", self._read(4))[0]
+            typ = TYPE_ARRAY
+            self._reserve(4)
+            n, = struct.unpack_from(">I", self._buffer, self._buff_i)
+            self._buff_i += 4
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
-            typ = TYPE_ARRAY
         elif b == 0xde:
-            n = struct.unpack(">H", self._read(2))[0]
+            self._reserve(2)
+            n, = struct.unpack_from(">H", self._buffer, self._buff_i)
+            self._buff_i += 2
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         elif b == 0xdf:
-            n = struct.unpack(">I", self._read(4))[0]
+            self._reserve(4)
+            n, = struct.unpack_from(">I", self._buffer, self._buff_i)
+            self._buff_i += 4
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP

From c16a1c6bdf667d5cfe314d09d5613808f1243a8f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 24 May 2016 07:32:30 +0900
Subject: [PATCH 1359/1648] fallback: Use bytearray as buffer

---
 msgpack/fallback.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 9b32610..a23ad8c 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -196,7 +196,7 @@ class Unpacker(object):
             self._feeding = False
 
         #: array of bytes feeded.
-        self._buffer = b""
+        self._buffer = bytearray()
         #: Which position we currently reads
         self._buff_i = 0
 
@@ -249,7 +249,7 @@ class Unpacker(object):
             raise BufferFull
         # bytes + bytearray -> bytearray
         # So cast before append
-        self._buffer += bytes(next_bytes)
+        self._buffer += next_bytes
 
     def _consume(self):
         """ Gets rid of the used parts of the buffer. """
@@ -284,7 +284,7 @@ class Unpacker(object):
 
         # Strip buffer before checkpoint before reading file.
         if self._buf_checkpoint > 0:
-            self._buffer = self._buffer[self._buf_checkpoint:]
+            del self._buffer[:self._buf_checkpoint]
             self._buff_i -= self._buf_checkpoint
             self._buf_checkpoint = 0
 
@@ -308,7 +308,8 @@ class Unpacker(object):
         n = 0
         obj = None
         self._reserve(1)
-        b = struct.unpack_from("B", self._buffer, self._buff_i)[0]
+        #b = struct.unpack_from("B", self._buffer, self._buff_i)[0]
+        b = self._buffer[self._buff_i]
         self._buff_i += 1
         if b & 0b10000000 == 0:
             obj = b
@@ -339,7 +340,8 @@ class Unpacker(object):
         elif b == 0xc4:
             typ = TYPE_BIN
             self._reserve(1)
-            n = struct.unpack_from("B", self._buffer, self._buff_i)[0]
+            #n = struct.unpack_from("B", self._buffer, self._buff_i)[0]
+            n = self._buffer[self._buff_i]
             self._buff_i += 1
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
@@ -394,7 +396,8 @@ class Unpacker(object):
             self._buff_i += 8
         elif b == 0xcc:
             self._reserve(1)
-            obj = struct.unpack_from("B", self._buffer, self._buff_i)[0]
+            #obj = struct.unpack_from("B", self._buffer, self._buff_i)[0]
+            obj = self._buffer[self._buff_i]
             self._buff_i += 1
         elif b == 0xcd:
             self._reserve(2)
@@ -462,7 +465,8 @@ class Unpacker(object):
         elif b == 0xd9:
             typ = TYPE_RAW
             self._reserve(1)
-            n, = struct.unpack_from("B", self._buffer, self._buff_i)
+            #n, = struct.unpack_from("B", self._buffer, self._buff_i)
+            n = self._buffer[self._buff_i]
             self._buff_i += 1
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
@@ -565,11 +569,13 @@ class Unpacker(object):
         if typ == TYPE_RAW:
             if self._encoding is not None:
                 obj = obj.decode(self._encoding, self._unicode_errors)
+            else:
+                obj = bytes(obj)
             return obj
         if typ == TYPE_EXT:
-            return self._ext_hook(n, obj)
+            return self._ext_hook(n, bytes(obj))
         if typ == TYPE_BIN:
-            return obj
+            return bytes(obj)
         assert typ == TYPE_IMMEDIATE
         return obj
 

From 2b63e9fbbb3440d73d6638ec8af6315aeb8ecd97 Mon Sep 17 00:00:00 2001
From: folz 
Date: Sat, 7 May 2016 15:18:20 +0200
Subject: [PATCH 1360/1648] enable unpacking from memoryview

---
 msgpack/_unpacker.pyx | 92 ++++++++++++++++++++++++++++++++++---------
 msgpack/fallback.py   | 37 ++++++++++-------
 test/test_buffer.py   |  9 +++++
 3 files changed, 106 insertions(+), 32 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 23f6478..f6e06b0 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -8,16 +8,23 @@ from cpython.bytes cimport (
 )
 from cpython.buffer cimport (
     Py_buffer,
-    PyBuffer_Release,
+    PyObject_CheckBuffer,
     PyObject_GetBuffer,
+    PyBuffer_Release,
+    PyBuffer_IsContiguous,
+    PyBUF_READ,
     PyBUF_SIMPLE,
+    PyBUF_FULL_RO,
 )
 from cpython.mem cimport PyMem_Malloc, PyMem_Free
 from cpython.object cimport PyCallable_Check
+from cpython.ref cimport Py_DECREF
+from cpython.exc cimport PyErr_WarnEx
 
 cdef extern from "Python.h":
     ctypedef struct PyObject
     cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1
+    object PyMemoryView_GetContiguous(object obj, int buffertype, char order)
 
 from libc.stdlib cimport *
 from libc.string cimport *
@@ -110,6 +117,42 @@ cdef inline init_ctx(unpack_context *ctx,
 def default_read_extended_type(typecode, data):
     raise NotImplementedError("Cannot decode extended type with typecode=%d" % typecode)
 
+cdef inline int get_data_from_buffer(object obj,
+                                     Py_buffer *view,
+                                     char **buf,
+                                     Py_ssize_t *buffer_len,
+                                     int *new_protocol) except 0:
+    cdef object contiguous
+    cdef Py_buffer tmp
+    if PyObject_CheckBuffer(obj):
+        new_protocol[0] = 1
+        if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1:
+            raise
+        if view.itemsize != 1:
+            PyBuffer_Release(view)
+            raise BufferError("cannot unpack from multi-byte object")
+        if PyBuffer_IsContiguous(view, 'A') == 0:
+            PyBuffer_Release(view)
+            # create a contiguous copy and get buffer
+            contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C')
+            PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE)
+            # view must hold the only reference to contiguous,
+            # so memory is freed when view is released
+            Py_DECREF(contiguous)
+        buffer_len[0] = view.len
+        buf[0] =  view.buf
+        return 1
+    else:
+        new_protocol[0] = 0
+        if PyObject_AsReadBuffer(obj,  buf, buffer_len) == -1:
+            raise BufferError("could not get memoryview")
+        PyErr_WarnEx(RuntimeWarning,
+                     "using old buffer interface to unpack %s; "
+                     "this leads to unpacking errors if slicing is used and "
+                     "will be removed in a future version" % type(obj),
+                     1)
+        return 1
+
 def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=1, encoding=None, unicode_errors="strict",
             object_pairs_hook=None, ext_hook=ExtType,
@@ -129,27 +172,34 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef Py_ssize_t off = 0
     cdef int ret
 
-    cdef char* buf
+    cdef Py_buffer view
+    cdef char* buf = NULL
     cdef Py_ssize_t buf_len
     cdef char* cenc = NULL
     cdef char* cerr = NULL
+    cdef int new_protocol = 0
 
-    PyObject_AsReadBuffer(packed, &buf, &buf_len)
+    get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol)
 
-    if encoding is not None:
-        if isinstance(encoding, unicode):
-            encoding = encoding.encode('ascii')
-        cenc = PyBytes_AsString(encoding)
+    try:
+        if encoding is not None:
+            if isinstance(encoding, unicode):
+                encoding = encoding.encode('ascii')
+            cenc = PyBytes_AsString(encoding)
 
-    if unicode_errors is not None:
-        if isinstance(unicode_errors, unicode):
-            unicode_errors = unicode_errors.encode('ascii')
-        cerr = PyBytes_AsString(unicode_errors)
+        if unicode_errors is not None:
+            if isinstance(unicode_errors, unicode):
+                unicode_errors = unicode_errors.encode('ascii')
+            cerr = PyBytes_AsString(unicode_errors)
+
+        init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
+                 use_list, cenc, cerr,
+                 max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
+        ret = unpack_construct(&ctx, buf, buf_len, &off)
+    finally:
+        if new_protocol:
+            PyBuffer_Release(&view);
 
-    init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
-             use_list, cenc, cerr,
-             max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
-    ret = unpack_construct(&ctx, buf, buf_len, &off)
     if ret == 1:
         obj = unpack_data(&ctx)
         if off < buf_len:
@@ -335,14 +385,20 @@ cdef class Unpacker(object):
     def feed(self, object next_bytes):
         """Append `next_bytes` to internal buffer."""
         cdef Py_buffer pybuff
+        cdef int new_protocol = 0
+        cdef char* buf
+        cdef Py_ssize_t buf_len
+
         if self.file_like is not None:
             raise AssertionError(
                     "unpacker.feed() is not be able to use with `file_like`.")
-        PyObject_GetBuffer(next_bytes, &pybuff, PyBUF_SIMPLE)
+
+        get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len, &new_protocol)
         try:
-            self.append_buffer(pybuff.buf, pybuff.len)
+            self.append_buffer(buf, buf_len)
         finally:
-            PyBuffer_Release(&pybuff)
+            if new_protocol:
+                PyBuffer_Release(&pybuff)
 
     cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
         cdef:
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index a23ad8c..11087eb 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -1,8 +1,8 @@
 """Fallback pure Python implementation of msgpack"""
 
 import sys
-import array
 import struct
+import warnings
 
 if sys.version_info[0] == 3:
     PY3 = True
@@ -46,6 +46,7 @@ else:
     from io import BytesIO as StringIO
     newlist_hint = lambda size: []
 
+
 from msgpack.exceptions import (
     BufferFull,
     OutOfData,
@@ -79,6 +80,24 @@ def _check_type_strict(obj, t, type=type, tuple=tuple):
         return type(obj) is t
 
 
+def _get_data_from_buffer(obj):
+    try:
+        view = memoryview(obj)
+    except TypeError:
+        # try to use legacy buffer protocol if 2.7, otherwise re-raise
+        if not PY3:
+            view = memoryview(buffer(obj))
+            warnings.warn("using old buffer interface to unpack %s; "
+                          "this leads to unpacking errors if slicing is used and "
+                          "will be removed in a future version" % type(obj),
+                          RuntimeWarning)
+        else:
+            raise
+    if view.itemsize != 1:
+        raise ValueError("cannot unpack from multi-byte object")
+    return view
+
+
 def unpack(stream, **kwargs):
     """
     Unpack an object from `stream`.
@@ -239,17 +258,11 @@ class Unpacker(object):
             raise TypeError("`ext_hook` is not callable")
 
     def feed(self, next_bytes):
-        if isinstance(next_bytes, array.array):
-            next_bytes = next_bytes.tostring()
-        if not isinstance(next_bytes, (bytes, bytearray)):
-            raise TypeError("next_bytes should be bytes, bytearray or array.array")
         assert self._feeding
-
-        if (len(self._buffer) - self._buff_i + len(next_bytes) > self._max_buffer_size):
+        view = _get_data_from_buffer(next_bytes)
+        if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size):
             raise BufferFull
-        # bytes + bytearray -> bytearray
-        # So cast before append
-        self._buffer += next_bytes
+        self._buffer += view
 
     def _consume(self):
         """ Gets rid of the used parts of the buffer. """
@@ -308,7 +321,6 @@ class Unpacker(object):
         n = 0
         obj = None
         self._reserve(1)
-        #b = struct.unpack_from("B", self._buffer, self._buff_i)[0]
         b = self._buffer[self._buff_i]
         self._buff_i += 1
         if b & 0b10000000 == 0:
@@ -340,7 +352,6 @@ class Unpacker(object):
         elif b == 0xc4:
             typ = TYPE_BIN
             self._reserve(1)
-            #n = struct.unpack_from("B", self._buffer, self._buff_i)[0]
             n = self._buffer[self._buff_i]
             self._buff_i += 1
             if n > self._max_bin_len:
@@ -396,7 +407,6 @@ class Unpacker(object):
             self._buff_i += 8
         elif b == 0xcc:
             self._reserve(1)
-            #obj = struct.unpack_from("B", self._buffer, self._buff_i)[0]
             obj = self._buffer[self._buff_i]
             self._buff_i += 1
         elif b == 0xcd:
@@ -465,7 +475,6 @@ class Unpacker(object):
         elif b == 0xd9:
             typ = TYPE_RAW
             self._reserve(1)
-            #n, = struct.unpack_from("B", self._buffer, self._buff_i)
             n = self._buffer[self._buff_i]
             self._buff_i += 1
             if n > self._max_str_len:
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 5a71f90..87f359f 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -18,3 +18,12 @@ def test_unpack_bytearray():
     assert [b'foo', b'bar'] == obj
     expected_type = bytes
     assert all(type(s) == expected_type for s in obj)
+
+
+def test_unpack_memoryview():
+    buf = bytearray(packb(('foo', 'bar')))
+    view = memoryview(buf)
+    obj = unpackb(view, use_list=1)
+    assert [b'foo', b'bar'] == obj
+    expected_type = bytes
+    assert all(type(s) == expected_type for s in obj)

From d6254abc8a3ebec6a135b923951767bd97557de4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 3 May 2016 11:58:28 +0900
Subject: [PATCH 1361/1648] Use AppVeyor to build windows wheel (#188)

* Add AppVeyor support to build windows wheel
* Fix test_limits on 32bit environments
* Ignore Python35-x64 test fail for now
  Should be fixed in next version.
---
 appveyor.yml        | 57 +++++++++++++++++++++++++++++++++++++++++++++
 build.cmd           | 21 +++++++++++++++++
 msgpack/_packer.pyx |  4 ++--
 3 files changed, 80 insertions(+), 2 deletions(-)
 create mode 100644 appveyor.yml
 create mode 100644 build.cmd

diff --git a/appveyor.yml b/appveyor.yml
new file mode 100644
index 0000000..02b4461
--- /dev/null
+++ b/appveyor.yml
@@ -0,0 +1,57 @@
+environment:
+
+  matrix:
+
+    # For Python versions available on Appveyor, see
+    # http://www.appveyor.com/docs/installed-software#python
+    # The list here is complete (excluding Python 2.6, which
+    # isn't covered by this document) at the time of writing.
+
+    - PYTHON: "C:\\Python27"
+    - PYTHON: "C:\\Python34"
+    - PYTHON: "C:\\Python35"
+    - PYTHON: "C:\\Python27-x64"
+    - PYTHON: "C:\\Python34-x64"
+      DISTUTILS_USE_SDK: "1"
+
+    # Python35-x64 test fails with MemoryError
+    # TODO: investigate it
+    #- PYTHON: "C:\\Python35-x64"
+
+install:
+  # We need wheel installed to build wheels
+  - "%PYTHON%\\python.exe -m pip install -U pip wheel pytest cython"
+
+build: off
+
+test_script:
+  # Put your test command here.
+  # If you don't need to build C extensions on 64-bit Python 3.3 or 3.4,
+  # you can remove "build.cmd" from the front of the command, as it's
+  # only needed to support those cases.
+  # Note that you must use the environment variable %PYTHON% to refer to
+  # the interpreter you're using - Appveyor does not do anything special
+  # to put the Python evrsion you want to use on PATH.
+  - "build.cmd %PYTHON%\\python.exe setup.py build_ext -i"
+  - "build.cmd %PYTHON%\\python.exe setup.py install"
+  - "%PYTHON%\\python.exe -c \"import sys; print(hex(sys.maxsize))\""
+  - "%PYTHON%\\python.exe -c \"from msgpack import _packer, _unpacker\""
+  - "%PYTHON%\\Scripts\\py.test test"
+  - "build.cmd %PYTHON%\\python.exe setup.py bdist_wheel"
+
+after_test:
+  # This step builds your wheels.
+  # Again, you only need build.cmd if you're building C extensions for
+  # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct
+  # interpreter
+
+artifacts:
+  # bdist_wheel puts your built wheel in the dist directory
+  - path: dist\*
+
+#on_success:
+#  You can use this step to upload your artifacts to a public website.
+#  See Appveyor's documentation for more details. Or you can simply
+#  access your wheels from the Appveyor "artifacts" tab for your build.
+
+# vim: set shiftwidth=2
diff --git a/build.cmd b/build.cmd
new file mode 100644
index 0000000..243dc9a
--- /dev/null
+++ b/build.cmd
@@ -0,0 +1,21 @@
+@echo off
+:: To build extensions for 64 bit Python 3, we need to configure environment
+:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
+:: MS Windows SDK for Windows 7 and .NET Framework 4
+::
+:: More details at:
+:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
+
+IF "%DISTUTILS_USE_SDK%"=="1" (
+    ECHO Configuring environment to build with MSVC on a 64bit architecture
+    ECHO Using Windows SDK 7.1
+    "C:\Program Files\Microsoft SDKs\Windows\v7.1\Setup\WindowsSdkVer.exe" -q -version:v7.1
+    CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release
+    SET MSSdk=1
+    REM Need the following to allow tox to see the SDK compiler
+    SET TOX_TESTENV_PASSENV=DISTUTILS_USE_SDK MSSdk INCLUDE LIB
+) ELSE (
+    ECHO Using default MSVC build environment
+)
+
+CALL %*
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 6392655..872465b 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -244,7 +244,7 @@ cdef class Packer(object):
         msgpack_pack_ext(&self.pk, typecode, len(data))
         msgpack_pack_raw_body(&self.pk, data, len(data))
 
-    def pack_array_header(self, size_t size):
+    def pack_array_header(self, long long size):
         if size > (2**32-1):
             raise ValueError
         cdef int ret = msgpack_pack_array(&self.pk, size)
@@ -257,7 +257,7 @@ cdef class Packer(object):
             self.pk.length = 0
             return buf
 
-    def pack_map_header(self, size_t size):
+    def pack_map_header(self, long long size):
         if size > (2**32-1):
             raise ValueError
         cdef int ret = msgpack_pack_map(&self.pk, size)

From 334dbe2a9652f43abdf27d978d9f4cdaf3f2a34d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 21 Jul 2016 19:19:32 +0900
Subject: [PATCH 1362/1648] Enable Python35-x64 in AppVeyor

---
 appveyor.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 02b4461..9e766c5 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -13,10 +13,7 @@ environment:
     - PYTHON: "C:\\Python27-x64"
     - PYTHON: "C:\\Python34-x64"
       DISTUTILS_USE_SDK: "1"
-
-    # Python35-x64 test fails with MemoryError
-    # TODO: investigate it
-    #- PYTHON: "C:\\Python35-x64"
+    - PYTHON: "C:\\Python35-x64"
 
 install:
   # We need wheel installed to build wheels

From b911b3c652e190e6942a610fb3389aaaf2ccf3cc Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 21 Jul 2016 19:32:00 +0900
Subject: [PATCH 1363/1648] Fix ext_hook call (#203)

fixes #202
---
 msgpack/unpack.h | 4 ++--
 setup.py         | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 92f4f11..da2cfb6 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -265,9 +265,9 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch
     }
     // length also includes the typecode, so the actual data is length-1
 #if PY_MAJOR_VERSION == 2
-    py = PyObject_CallFunction(u->ext_hook, "(is#)", typecode, pos, length-1);
+    py = PyObject_CallFunction(u->ext_hook, "(is#)", (int)typecode, pos, (Py_ssize_t)length-1);
 #else
-    py = PyObject_CallFunction(u->ext_hook, "(iy#)", typecode, pos, length-1);
+    py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1);
 #endif
     if (!py)
         return -1;
diff --git a/setup.py b/setup.py
index 37729bd..1363586 100755
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@ except ImportError:
 
 def cythonize(src):
     sys.stderr.write("cythonize: %r\n" % (src,))
-    cython_compiler.compile([src], cplus=True, emit_linenums=True)
+    cython_compiler.compile([src], cplus=True)
 
 def ensure_source(src):
     pyx = os.path.splitext(src)[0] + '.pyx'

From ff208ad7d0b288f2d94d3160bf8a4cddeebc987d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 29 Jul 2016 22:25:05 +0900
Subject: [PATCH 1364/1648] 0.4.8

---
 ChangeLog.rst       | 11 ++++++++++-
 msgpack/_version.py |  2 +-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 35535b4..2151736 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,6 +1,15 @@
+0.4.8
+=====
+:release date: 2016-07-29
+
+Bugs fixed
+----------
+
+* Calling ext_hook with wrong length. (Only on Windows, maybe. #203)
+
 0.4.7
 =====
-:release date: TBD
+:release date: 2016-01-25
 
 Bugs fixed
 ----------
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 37c172d..76bd8fb 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 4, 7)
+version = (0, 4, 8)

From a9f4dad4dcde4db148f22720c694e5b5e0cb6f2d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 30 Jul 2016 11:35:26 +0900
Subject: [PATCH 1365/1648] Make manylinux1 wheels

---
 Makefile             |  4 ++++
 docker/buildwheel.sh | 11 +++++++++++
 2 files changed, 15 insertions(+)
 create mode 100644 docker/buildwheel.sh

diff --git a/Makefile b/Makefile
index 2e53d08..6eaef53 100644
--- a/Makefile
+++ b/Makefile
@@ -20,3 +20,7 @@ python3: cython
 
 test:
 	py.test test
+
+build-manylinux1-wheel:
+	docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686   bash docker/buildwheel.sh
+	docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh
diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh
new file mode 100644
index 0000000..b654e45
--- /dev/null
+++ b/docker/buildwheel.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e -x
+
+ARCH=`uname -p`
+echo "arch=$ARCH"
+
+for V in cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do
+    PYBIN=/opt/python/$V/bin
+    rm -rf build/       # Avoid lib build by narrow Python is used by wide python
+    $PYBIN/python setup.py bdist_wheel -p manylinux1_${ARCH}
+done

From e3fea94509767047a8ff45aa07cd58a9ba9694e7 Mon Sep 17 00:00:00 2001
From: TW 
Date: Wed, 11 Jan 2017 04:04:23 +0100
Subject: [PATCH 1366/1648] fix typos and other cosmetic issues (#214)

cosmetic issues:
- reST headlines' underline length needs to match the headline length
  (looks like somebody is / was using a proportional font)
- Cython code lines do not need to be terminated with a semicolon
- always use triple-double-quotes for docstrings
---
 ChangeLog.rst          | 46 +++++++++++++++++++++---------------------
 README.rst             | 29 +++++++++++++-------------
 appveyor.yml           |  2 +-
 docs/api.rst           |  2 +-
 docs/index.rst         |  2 +-
 msgpack/_packer.pyx    | 10 ++++-----
 msgpack/_unpacker.pyx  |  2 +-
 msgpack/fallback.py    | 16 +++++++--------
 test/test_extension.py |  2 +-
 test/test_pack.py      |  2 +-
 10 files changed, 57 insertions(+), 56 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 2151736..30e6c5f 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -160,7 +160,7 @@ Changes
 
 
 0.2.4
-=======
+=====
 :release date: 2012-12-22
 
 Bugs fixed
@@ -169,7 +169,7 @@ Bugs fixed
 * Fix SEGV when object_hook or object_pairs_hook raise Exception. (#39)
 
 0.2.3
-=======
+=====
 :release date: 2012-12-11
 
 Changes
@@ -177,11 +177,11 @@ Changes
 * Warn when use_list is not specified. It's default value will be changed in 0.3.
 
 Bugs fixed
------------
+----------
 * Can't pack subclass of dict.
 
 0.2.2
-=======
+=====
 :release date: 2012-09-21
 
 Changes
@@ -190,7 +190,7 @@ Changes
   object in single precision format.
 
 Bugs fixed
------------
+----------
 * ``unpack()`` didn't restores gc state when it called with gc disabled.
   ``unpack()`` doesn't control gc now instead of restoring gc state collectly.
   User can control gc state when gc cause performance issue.
@@ -198,7 +198,7 @@ Bugs fixed
 * ``Unpacker``'s ``read_size`` option didn't used.
 
 0.2.1
-=======
+=====
 :release date: 2012-08-20
 
 Changes
@@ -206,8 +206,8 @@ Changes
 * Add ``max_buffer_size`` parameter to Unpacker. It limits internal buffer size
   and allows unpack data from untrusted source safely.
 
-* Unpacker's buffer reallocation algorithm is less greedy now. It cause perforamce
-  derease in rare case but memory efficient and don't allocate than ``max_buffer_size``.
+* Unpacker's buffer reallocation algorithm is less greedy now. It cause performance
+  decrease in rare case but memory efficient and don't allocate than ``max_buffer_size``.
 
 Bugs fixed
 ----------
@@ -217,7 +217,7 @@ Bugs fixed
 
 
 0.2.0
-=======
+=====
 :release date: 2012-06-27
 
 Changes
@@ -232,16 +232,16 @@ Bugs fixed
 
 
 0.1.13
-=======
+======
 :release date: 2012-04-21
 
 New
-----
+---
 * Don't accept subtype of list and tuple as msgpack list. (Steeve Morin)
   It allows customize how it serialized with ``default`` argument.
 
 Bugs fixed
------------
+----------
 * Fix wrong error message. (David Wolever)
 * Fix memory leak while unpacking when ``object_hook`` or ``list_hook`` is used.
   (Steeve Morin)
@@ -253,21 +253,21 @@ Other changes
 
 
 0.1.12
-=======
+======
 :release date: 2011-12-27
 
 Bugs fixed
--------------
+----------
 
 * Re-enable packs/unpacks removed at 0.1.11. It will be removed when 0.2 is released.
 
 
 0.1.11
-=======
+======
 :release date: 2011-12-26
 
 Bugs fixed
--------------
+----------
 
 * Include test code for Python3 to sdist. (Johan Bergström)
 * Fix compilation error on MSVC. (davidgaleano)
@@ -285,7 +285,7 @@ New feature
 
 
 0.1.9
-======
+=====
 :release date: 2011-01-29
 
 New feature
@@ -299,16 +299,16 @@ Bugs fixed
 * Add MemoryError check.
 
 0.1.8
-======
+=====
 :release date: 2011-01-10
 
 New feature
-------------
+-----------
 * Support ``loads`` and ``dumps`` aliases for API compatibility with
   simplejson and pickle.
 
 * Add *object_hook* and *list_hook* option to unpacker. It allows you to
-  hook unpacing mapping type and array type.
+  hook unpacking mapping type and array type.
 
 * Add *default* option to packer. It allows you to pack unsupported types.
 
@@ -320,13 +320,13 @@ Bugs fixed
 
 
 0.1.7
-======
+=====
 :release date: 2010-11-02
 
 New feature
-------------
+-----------
 * Add *object_hook* and *list_hook* option to unpacker. It allows you to
-  hook unpacing mapping type and array type.
+  hook unpacking mapping type and array type.
 
 * Add *default* option to packer. It allows you to pack unsupported types.
 
diff --git a/README.rst b/README.rst
index d32ec1d..e37c61d 100644
--- a/README.rst
+++ b/README.rst
@@ -1,6 +1,6 @@
-=======================
+======================
 MessagePack for Python
-=======================
+======================
 
 :author: INADA Naoki
 :version: 0.4.6
@@ -11,21 +11,22 @@ MessagePack for Python
    
 
 What's this
-------------
+-----------
 
-`MessagePack `_ is a fast, compact binary serialization format, suitable for
-similar data to JSON. This package provides CPython bindings for reading and
-writing MessagePack data.
+`MessagePack `_ is an efficient binary serialization format.
+It lets you exchange data among multiple languages like JSON.
+But it's faster and smaller.
+This package provides CPython bindings for reading and writing MessagePack data.
 
 Install
----------
+-------
 
 ::
 
    $ pip install msgpack-python
 
 PyPy
-^^^^^
+^^^^
 
 msgpack-python provides pure python implementation.  PyPy can use this.
 
@@ -44,7 +45,7 @@ Community Edition or Express Edition can be used to build extension module.
 
 
 How to use
------------
+----------
 
 One-shot pack & unpack
 ^^^^^^^^^^^^^^^^^^^^^^
@@ -134,7 +135,7 @@ It is also possible to pack/unpack custom data types. Here is an example for
 key-value pairs.
 
 Extended types
-^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^
 
 It is also possible to pack/unpack custom data types using the **ext** type.
 
@@ -166,7 +167,7 @@ Advanced unpacking control
 
 As an alternative to iteration, ``Unpacker`` objects provide ``unpack``,
 ``skip``, ``read_array_header`` and ``read_map_header`` methods. The former two
-read an entire message from the stream, respectively deserialising and returning
+read an entire message from the stream, respectively de-serialising and returning
 the result, or ignoring it. The latter two methods return the number of elements
 in the upcoming container, so that each element in an array, or key-value pair
 in a map, can be unpacked or skipped individually.
@@ -243,7 +244,7 @@ instead of `StopIteration`.
 `StopIteration` is used for iterator protocol only.
 
 Note about performance
-------------------------
+----------------------
 
 GC
 ^^
@@ -253,7 +254,7 @@ This means unpacking may cause useless GC.
 You can use ``gc.disable()`` when unpacking large message.
 
 use_list option
-^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^
 List is the default sequence type of Python.
 But tuple is lighter than list.
 You can use ``use_list=False`` while unpacking when performance is important.
@@ -264,7 +265,7 @@ Another way to unpacking such object is using ``object_pairs_hook``.
 
 
 Development
-------------
+-----------
 
 Test
 ^^^^
diff --git a/appveyor.yml b/appveyor.yml
index 9e766c5..a8a2352 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -28,7 +28,7 @@ test_script:
   # only needed to support those cases.
   # Note that you must use the environment variable %PYTHON% to refer to
   # the interpreter you're using - Appveyor does not do anything special
-  # to put the Python evrsion you want to use on PATH.
+  # to put the Python version you want to use on PATH.
   - "build.cmd %PYTHON%\\python.exe setup.py build_ext -i"
   - "build.cmd %PYTHON%\\python.exe setup.py install"
   - "%PYTHON%\\python.exe -c \"import sys; print(hex(sys.maxsize))\""
diff --git a/docs/api.rst b/docs/api.rst
index 841c134..6336793 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -28,7 +28,7 @@ API reference
 .. autoclass:: ExtType
 
 exceptions
------------
+----------
 
 These exceptions are accessible via `msgpack` package.
 (For example, `msgpack.OutOfData` is shortcut for `msgpack.exceptions.OutOfData`)
diff --git a/docs/index.rst b/docs/index.rst
index 72d4499..dcdab4f 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -1,5 +1,5 @@
 msgpack document
-==================
+================
 
 `MessagePack `_ is a efficient format for inter
 language data exchange.
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 872465b..7a12853 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -58,11 +58,11 @@ cdef class Packer(object):
     :param bool use_single_float:
         Use single precision float type for float. (default: False)
     :param bool autoreset:
-        Reset buffer after each pack and return it's content as `bytes`. (default: True).
+        Reset buffer after each pack and return its content as `bytes`. (default: True).
         If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
     :param bool use_bin_type:
         Use bin type introduced in msgpack spec 2.0 for bytes.
-        It also enable str8 type for unicode.
+        It also enables str8 type for unicode.
     """
     cdef msgpack_packer pk
     cdef object _default
@@ -75,7 +75,7 @@ cdef class Packer(object):
 
     def __cinit__(self):
         cdef int buf_size = 1024*1024
-        self.pk.buf =  malloc(buf_size);
+        self.pk.buf =  malloc(buf_size)
         if self.pk.buf == NULL:
             raise MemoryError("Unable to allocate internal buffer.")
         self.pk.buf_size = buf_size
@@ -108,7 +108,7 @@ cdef class Packer(object):
             self.unicode_errors = PyBytes_AsString(self._berrors)
 
     def __dealloc__(self):
-        free(self.pk.buf);
+        free(self.pk.buf)
 
     cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1:
         cdef long long llval
@@ -274,7 +274,7 @@ cdef class Packer(object):
         """
         Pack *pairs* as msgpack map type.
 
-        *pairs* should sequence of pair.
+        *pairs* should be a sequence of pairs.
         (`len(pairs)` and `for k, v in pairs:` should be supported.)
         """
         cdef int ret = msgpack_pack_map(&self.pk, len(pairs))
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 1aefc64..2a13903 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -209,7 +209,7 @@ cdef class Unpacker(object):
     :param int max_buffer_size:
         Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
         Raises `BufferFull` exception when it is insufficient.
-        You shoud set this parameter when unpacking data from untrusted source.
+        You should set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
         Limits max length of str. (default: 2**31-1)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index f682611..fefabb8 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -138,7 +138,7 @@ class Unpacker(object):
     :param int max_buffer_size:
         Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
         Raises `BufferFull` exception when it is insufficient.
-        You shoud set this parameter when unpacking data from untrusted source.
+        You should set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
         Limits max length of str. (default: 2**31-1)
@@ -188,13 +188,13 @@ class Unpacker(object):
             self.file_like = file_like
             self._fb_feeding = False
 
-        #: array of bytes feeded.
+        #: array of bytes fed.
         self._fb_buffers = []
         #: Which buffer we currently reads
         self._fb_buf_i = 0
         #: Which position we currently reads
         self._fb_buf_o = 0
-        #: Total size of _fb_bufferes
+        #: Total size of _fb_buffers
         self._fb_buf_n = 0
 
         # When Unpacker is used as an iterable, between the calls to next(),
@@ -203,7 +203,7 @@ class Unpacker(object):
         # the correct moments, we have to keep track of how sloppy we were.
         # Furthermore, when the buffer is incomplete (that is: in the case
         # we raise an OutOfData) we need to rollback the buffer to the correct
-        # state, which _fb_slopiness records.
+        # state, which _fb_sloppiness records.
         self._fb_sloppiness = 0
 
         self._max_buffer_size = max_buffer_size or 2**31-1
@@ -303,7 +303,7 @@ class Unpacker(object):
 
     def _fb_read(self, n, write_bytes=None):
         buffs = self._fb_buffers
-        # We have a redundant codepath for the most common case, such that
+        # We have a redundant code path for the most common case, such that
         # pypy optimizes it properly.  This is the case that the read fits
         # in the current buffer.
         if (write_bytes is None and self._fb_buf_i < len(buffs) and
@@ -598,17 +598,17 @@ class Packer(object):
         Convert user type to builtin type that Packer supports.
         See also simplejson's document.
     :param str encoding:
-            Convert unicode to bytes with this encoding. (default: 'utf-8')
+        Convert unicode to bytes with this encoding. (default: 'utf-8')
     :param str unicode_errors:
         Error handler for encoding unicode. (default: 'strict')
     :param bool use_single_float:
         Use single precision float type for float. (default: False)
     :param bool autoreset:
-        Reset buffer after each pack and return it's content as `bytes`. (default: True).
+        Reset buffer after each pack and return its content as `bytes`. (default: True).
         If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
     :param bool use_bin_type:
         Use bin type introduced in msgpack spec 2.0 for bytes.
-        It also enable str8 type for unicode.
+        It also enables str8 type for unicode.
     """
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
                  use_single_float=False, autoreset=True, use_bin_type=False):
diff --git a/test/test_extension.py b/test/test_extension.py
index c552498..d05d7ab 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -42,7 +42,7 @@ def test_extension_type():
             typecode = 123 # application specific typecode
             data = obj.tostring()
             return ExtType(typecode, data)
-        raise TypeError("Unknwon type object %r" % (obj,))
+        raise TypeError("Unknown type object %r" % (obj,))
 
     def ext_hook(code, data):
         print('ext_hook called', code, data)
diff --git a/test/test_pack.py b/test/test_pack.py
index 762ccf5..e945902 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -130,7 +130,7 @@ def testMapSize(sizes=[0, 5, 50, 1000]):
 
 
 class odict(dict):
-    '''Reimplement OrderedDict to run test on Python 2.6'''
+    """Reimplement OrderedDict to run test on Python 2.6"""
     def __init__(self, seq):
         self._seq = seq
         dict.__init__(self, seq)

From f985ee8a665daa298282ab87c488e81b7d9814a7 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Jan 2017 19:57:04 +0900
Subject: [PATCH 1367/1648] Remove version and date from README

---
 README.rst | 2 --
 1 file changed, 2 deletions(-)

diff --git a/README.rst b/README.rst
index 74f5fc7..da625b4 100644
--- a/README.rst
+++ b/README.rst
@@ -3,8 +3,6 @@ MessagePack for Python
 ======================
 
 :author: INADA Naoki
-:version: 0.4.6
-:date: 2015-03-13
 
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.svg
    :target: https://travis-ci.org/#!/msgpack/msgpack-python

From 12845692b5a3703bc88e85f492143ca1a9985902 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Jan 2017 20:41:33 +0900
Subject: [PATCH 1368/1648] Add requirements.txt for Read the Docs

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 requirements.txt

diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..87f04da
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+Cython==0.25.2

From b328f3ecffd22e7f0db76e81774727fd171bf303 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Jan 2017 20:48:48 +0900
Subject: [PATCH 1369/1648] Add badge for Read the Docs

---
 README.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index da625b4..0e1ddd7 100644
--- a/README.rst
+++ b/README.rst
@@ -2,11 +2,12 @@
 MessagePack for Python
 ======================
 
-:author: INADA Naoki
-
 .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.svg
    :target: https://travis-ci.org/#!/msgpack/msgpack-python
-   
+
+.. image:: https://readthedocs.org/projects/msgpack-python/badge/?version=latest
+   :target: http://msgpack-python.readthedocs.io/en/latest/?badge=latest
+   :alt: Documentation Status
 
 What's this
 -----------

From 3388e4a6ee6adea56789d97cc05ed610a4e5b4fc Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Jan 2017 21:46:31 +0900
Subject: [PATCH 1370/1648] travis and appveyor update (#217)

travis:

* stop using tox
* Add Python 3.6 and 3.7-dev
* Stop pypy3 (until PyPy3.5 is released)

appveyor:

* Drop Python 3.4 and add 3.6
---
 .travis.yml        | 34 +++++++++++++++++++++++-----------
 appveyor.yml       |  8 ++++----
 docker/runtests.sh |  2 +-
 3 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b4396cb..0170360 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,22 +1,23 @@
 sudo: false
 language: python
-python: 3.5
-cache:
-  directories:
-    - $HOME/.cache/pip
+cache: pip
+
+python:
+  - "2.7"
+  - "3.3"
+  - "3.4"
+  - "3.5"
+  - "3.6"
+  - "3.7-dev"
 
 branches:
   only:
     - master
 
-env:
-  - TOXENV=py27-c,py33-c,py34-c,py35-c
-  - TOXENV=py27-pure,py33-pure,py34-pure,py35-pure
-  - TOXENV=pypy-pure,pypy3-pure
-
 matrix:
   include:
     - sudo: required
+      language: c
       services:
         - docker
       env:
@@ -28,12 +29,23 @@ matrix:
         - docker pull $DOCKER_IMAGE
       script:
         - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
+    - python: "pypy"
+      install:
+        - pip install -e .
+      script:
+        - py.test -v test
+
 
 install:
   - pip install -U pip
-  - pip install tox cython
+  - pip install cython
   - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
+  - pip install -e .
 
-script: tox
+script:
+  - python -c 'import sys; print(hex(sys.maxsize))'
+  - python -c 'from msgpack import _packer, _unpacker'
+  - py.test -v test
+  - MSGPACK_PUREPYTHON=x py.test -v test
 
 # vim: sw=2 ts=2
diff --git a/appveyor.yml b/appveyor.yml
index a8a2352..e63423d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -8,16 +8,16 @@ environment:
     # isn't covered by this document) at the time of writing.
 
     - PYTHON: "C:\\Python27"
-    - PYTHON: "C:\\Python34"
-    - PYTHON: "C:\\Python35"
     - PYTHON: "C:\\Python27-x64"
-    - PYTHON: "C:\\Python34-x64"
-      DISTUTILS_USE_SDK: "1"
+    - PYTHON: "C:\\Python35"
     - PYTHON: "C:\\Python35-x64"
+    - PYTHON: "C:\\Python36"
+    - PYTHON: "C:\\Python36-x64"
 
 install:
   # We need wheel installed to build wheels
   - "%PYTHON%\\python.exe -m pip install -U pip wheel pytest cython"
+  - "%PYTHON%\\Scripts\\cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx"
 
 build: off
 
diff --git a/docker/runtests.sh b/docker/runtests.sh
index 0d74802..0eea715 100755
--- a/docker/runtests.sh
+++ b/docker/runtests.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e -x
 
-for V in cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do
+for V in cp36-cp36m cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do
     PYBIN=/opt/python/$V/bin
     $PYBIN/python setup.py install
     rm -rf build/       # Avoid lib build by narrow Python is used by wide python

From a8d9162ca6cff6101c1f6b9547e94749c6acae96 Mon Sep 17 00:00:00 2001
From: jfolz 
Date: Sat, 29 Apr 2017 19:33:20 +0200
Subject: [PATCH 1371/1648] Unpacker: add tell() (#227)

---
 msgpack/_unpacker.pyx  |  7 +++++++
 msgpack/fallback.py    |  5 +++++
 test/test_sequnpack.py | 20 ++++++++++++++++++++
 3 files changed, 32 insertions(+)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index a9801ee..dabc5f7 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -29,6 +29,7 @@ cdef extern from "Python.h":
 from libc.stdlib cimport *
 from libc.string cimport *
 from libc.limits cimport *
+ctypedef unsigned long long uint64_t
 
 from msgpack.exceptions import (
     BufferFull,
@@ -314,6 +315,7 @@ cdef class Unpacker(object):
     cdef object object_hook, object_pairs_hook, list_hook, ext_hook
     cdef object encoding, unicode_errors
     cdef Py_ssize_t max_buffer_size
+    cdef uint64_t stream_offset
 
     def __cinit__(self):
         self.buf = NULL
@@ -358,6 +360,7 @@ cdef class Unpacker(object):
         self.buf_size = read_size
         self.buf_head = 0
         self.buf_tail = 0
+        self.stream_offset = 0
 
         if encoding is not None:
             if isinstance(encoding, unicode):
@@ -468,6 +471,7 @@ cdef class Unpacker(object):
 
             try:
                 ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+                self.stream_offset += self.buf_head - prev_head
                 if write_bytes is not None:
                     write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
 
@@ -534,6 +538,9 @@ cdef class Unpacker(object):
         """
         return self._unpack(read_map_header, write_bytes)
 
+    def tell(self):
+        return self.stream_offset
+
     def __iter__(self):
         return self
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index d2eb9f4..508fd06 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -244,6 +244,7 @@ class Unpacker(object):
         self._max_array_len = max_array_len
         self._max_map_len = max_map_len
         self._max_ext_len = max_ext_len
+        self._stream_offset = 0
 
         if list_hook is not None and not callable(list_hook):
             raise TypeError('`list_hook` is not callable')
@@ -266,6 +267,7 @@ class Unpacker(object):
 
     def _consume(self):
         """ Gets rid of the used parts of the buffer. """
+        self._stream_offset += self._buff_i - self._buf_checkpoint
         self._buf_checkpoint = self._buff_i
 
     def _got_extradata(self):
@@ -629,6 +631,9 @@ class Unpacker(object):
         self._consume()
         return ret
 
+    def tell(self):
+        return self._stream_offset
+
 
 class Packer(object):
     """
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 45f4cc7..59718f5 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -3,6 +3,7 @@
 
 import io
 from msgpack import Unpacker, BufferFull
+from msgpack import pack
 from msgpack.exceptions import OutOfData
 from pytest import raises
 
@@ -96,3 +97,22 @@ def test_issue124():
     unpacker.feed(b"!")
     assert tuple(unpacker) == (b'!',)
     assert tuple(unpacker) == ()
+
+
+def test_unpack_tell():
+    stream = io.BytesIO()
+    messages = [2**i-1 for i in range(65)]
+    messages += [-(2**i) for i in range(1, 64)]
+    messages += [b'hello', b'hello'*1000, list(range(20)),
+                 {i: bytes(i)*i for i in range(10)},
+                 {i: bytes(i)*i for i in range(32)}]
+    offsets = []
+    for m in messages:
+        pack(m, stream)
+        offsets.append(stream.tell())
+    stream.seek(0)
+    unpacker = Unpacker(stream)
+    for m, o in zip(messages, offsets):
+        m2 = next(unpacker)
+        assert m == m2
+        assert o == unpacker.tell()

From f0f2c0b39703e0129d2352c71ec9811a8f275cc8 Mon Sep 17 00:00:00 2001
From: jfolz 
Date: Thu, 18 May 2017 13:03:15 +0200
Subject: [PATCH 1372/1648] Packer accepts bytearray objects (#229)

---
 msgpack/_packer.pyx | 14 ++++++++++++--
 msgpack/fallback.py |  6 ++++--
 test/test_pack.py   |  7 +++++++
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index f24aa70..5a81709 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -10,6 +10,8 @@ from msgpack import ExtType
 cdef extern from "Python.h":
 
     int PyMemoryView_Check(object obj)
+    int PyByteArray_Check(object obj)
+    int PyByteArray_CheckExact(object obj)
 
 
 cdef extern from "pack.h":
@@ -39,6 +41,14 @@ cdef int DEFAULT_RECURSE_LIMIT=511
 cdef size_t ITEM_LIMIT = (2**32)-1
 
 
+cdef inline int PyBytesLike_Check(object o):
+    return PyBytes_Check(o) or PyByteArray_Check(o)
+
+
+cdef inline int PyBytesLike_CheckExact(object o):
+    return PyBytes_CheckExact(o) or PyByteArray_CheckExact(o)
+
+
 cdef class Packer(object):
     """
     MessagePack Packer
@@ -174,10 +184,10 @@ cdef class Packer(object):
                 else:
                    dval = o
                    ret = msgpack_pack_double(&self.pk, dval)
-            elif PyBytes_CheckExact(o) if strict_types else PyBytes_Check(o):
+            elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o):
                 L = len(o)
                 if L > ITEM_LIMIT:
-                    raise PackValueError("bytes is too large")
+                    raise PackValueError("%s is too large" % type(o).__name__)
                 rawval = o
                 ret = msgpack_pack_bin(&self.pk, L)
                 if ret == 0:
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 508fd06..a02cbe1 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -38,6 +38,8 @@ if hasattr(sys, 'pypy_version_info'):
         def write(self, s):
             if isinstance(s, memoryview):
                 s = s.tobytes()
+            elif isinstance(s, bytearray):
+                s = bytes(s)
             self.builder.append(s)
         def getvalue(self):
             return self.builder.build()
@@ -728,10 +730,10 @@ class Packer(object):
                     default_used = True
                     continue
                 raise PackOverflowError("Integer value out of range")
-            if check(obj, bytes):
+            if check(obj, (bytes, bytearray)):
                 n = len(obj)
                 if n >= 2**32:
-                    raise PackValueError("Bytes is too large")
+                    raise PackValueError("%s is too large" % type(obj).__name__)
                 self._pack_bin_header(n)
                 return self._buffer.write(obj)
             if check(obj, Unicode):
diff --git a/test/test_pack.py b/test/test_pack.py
index e945902..a704fdb 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -58,6 +58,13 @@ def testPackBytes():
     for td in test_data:
         check(td)
 
+def testPackByteArrays():
+    test_data = [
+        bytearray(b""), bytearray(b"abcd"), (bytearray(b"defgh"),),
+        ]
+    for td in test_data:
+        check(td)
+
 def testIgnoreUnicodeErrors():
     re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
     assert re == "abcdef"

From deeda31a8840cee334f05f15bd2308af13dc9c64 Mon Sep 17 00:00:00 2001
From: Lorenzo Bolla 
Date: Sat, 30 Sep 2017 08:23:55 +0100
Subject: [PATCH 1373/1648] Add unittests to document serialisation of tuples
 (#246)

Also, fix formatting of error message in case of tuple.
See https://github.com/msgpack/msgpack-python/issues/245
---
 msgpack/fallback.py     |  2 +-
 test/test_stricttype.py | 49 ++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index a02cbe1..28478ca 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -795,7 +795,7 @@ class Packer(object):
                 obj = self._default(obj)
                 default_used = 1
                 continue
-            raise TypeError("Cannot serialize %r" % obj)
+            raise TypeError("Cannot serialize %r" % (obj, ))
 
     def pack(self, obj):
         self._pack(obj)
diff --git a/test/test_stricttype.py b/test/test_stricttype.py
index a20b5eb..0f865c8 100644
--- a/test/test_stricttype.py
+++ b/test/test_stricttype.py
@@ -1,7 +1,7 @@
 # coding: utf-8
 
 from collections import namedtuple
-from msgpack import packb, unpackb
+from msgpack import packb, unpackb, ExtType
 
 
 def test_namedtuple():
@@ -13,3 +13,50 @@ def test_namedtuple():
     packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default)
     unpacked = unpackb(packed, encoding='utf-8')
     assert unpacked == {'foo': 1, 'bar': 42}
+
+
+def test_tuple():
+    t = ('one', 2, b'three', (4, ))
+
+    def default(o):
+        if isinstance(o, tuple):
+            return {
+                '__type__': 'tuple',
+                'value': list(o),
+                }
+        raise TypeError('Unsupported type %s' % (type(o),))
+
+    def convert(o):
+        if o.get('__type__') == 'tuple':
+            return tuple(o['value'])
+        return o
+
+    data = packb(t, strict_types=True, use_bin_type=True, default=default)
+    expected = unpackb(data, encoding='utf-8', object_hook=convert)
+
+    assert expected == t
+
+
+def test_tuple_ext():
+    t = ('one', 2, b'three', (4, ))
+
+    MSGPACK_EXT_TYPE_TUPLE = 0
+
+    def default(o):
+        if isinstance(o, tuple):
+            # Convert to list and pack
+            payload = packb(
+                list(o), strict_types=True, use_bin_type=True, default=default)
+            return ExtType(MSGPACK_EXT_TYPE_TUPLE, payload)
+        raise TypeError(repr(o))
+
+    def convert(code, payload):
+        if code == MSGPACK_EXT_TYPE_TUPLE:
+            # Unpack and convert to tuple
+            return tuple(unpackb(payload, encoding='utf-8', ext_hook=convert))
+        raise ValueError('Unknown Ext code {}'.format(code))
+
+    data = packb(t, strict_types=True, use_bin_type=True, default=default)
+    expected = unpackb(data, encoding='utf-8', ext_hook=convert)
+
+    assert expected == t

From b57106c246867b5beec62874a239c87d94dafba5 Mon Sep 17 00:00:00 2001
From: Hugo 
Date: Wed, 11 Oct 2017 20:49:02 +0300
Subject: [PATCH 1374/1648] Update badges (#247)

---
 README.rst | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/README.rst b/README.rst
index 0e1ddd7..ea1499b 100644
--- a/README.rst
+++ b/README.rst
@@ -2,17 +2,18 @@
 MessagePack for Python
 ======================
 
-.. image:: https://secure.travis-ci.org/msgpack/msgpack-python.svg
-   :target: https://travis-ci.org/#!/msgpack/msgpack-python
+.. image:: https://travis-ci.org/msgpack/msgpack-python.svg?branch=master
+   :target: https://travis-ci.org/msgpack/msgpack-python
+   :alt: Build Status
 
 .. image:: https://readthedocs.org/projects/msgpack-python/badge/?version=latest
-   :target: http://msgpack-python.readthedocs.io/en/latest/?badge=latest
+   :target: https://msgpack-python.readthedocs.io/en/latest/?badge=latest
    :alt: Documentation Status
 
 What's this
 -----------
 
-`MessagePack `_ is an efficient binary serialization format.
+`MessagePack `_ is an efficient binary serialization format.
 It lets you exchange data among multiple languages like JSON.
 But it's faster and smaller.
 This package provides CPython bindings for reading and writing MessagePack data.

From 54aa47b2dd489297d894c0639811653fd6ff7bfa Mon Sep 17 00:00:00 2001
From: Hugo 
Date: Thu, 12 Oct 2017 03:26:34 +0300
Subject: [PATCH 1375/1648] Update supported versions in classifiers (#248)

---
 setup.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/setup.py b/setup.py
index 5f0f08c..5c88397 100755
--- a/setup.py
+++ b/setup.py
@@ -114,7 +114,13 @@ setup(name='msgpack-python',
       url='http://msgpack.org/',
       classifiers=[
           'Programming Language :: Python :: 2',
+          'Programming Language :: Python :: 2.7',
           'Programming Language :: Python :: 3',
+          'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
+          'Programming Language :: Python :: 3.7',
+          'Programming Language :: Python :: Implementation :: CPython',
+          'Programming Language :: Python :: Implementation :: PyPy',
           'Intended Audience :: Developers',
           'License :: OSI Approved :: Apache Software License',
           ]

From 6fd1890be4692cca1a0c53cc160ac5da83e1d272 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Jan 2017 21:47:26 +0900
Subject: [PATCH 1376/1648] Add py36 to tox.ini

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index b6e7a7f..b2ac336 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {py27,py33,py34,py35}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
+envlist = {py27,py33,py34,py35,py36}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
 
 [variants:pure]
 setenv=

From 3d7ebc47b4fed0cc06f652013c34f32107728c98 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 12 Oct 2017 15:28:40 +0900
Subject: [PATCH 1377/1648] travis: Remove "only: master" restriction

---
 .travis.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0170360..f5141d6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,10 +10,6 @@ python:
   - "3.6"
   - "3.7-dev"
 
-branches:
-  only:
-    - master
-
 matrix:
   include:
     - sudo: required

From a70ce0c3d7bb25646302fd2649ba916178cf4a69 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 12 Oct 2017 16:26:58 +0900
Subject: [PATCH 1378/1648] Fix travis fail (#251)

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index f5141d6..54e0c62 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,7 +13,7 @@ python:
 matrix:
   include:
     - sudo: required
-      language: c
+      language: python
       services:
         - docker
       env:

From 0fc4ee98be498f39a320eff501ba30c49c31482d Mon Sep 17 00:00:00 2001
From: Hugo 
Date: Thu, 12 Oct 2017 10:27:39 +0300
Subject: [PATCH 1379/1648] Remove code and tests for unsupported Python 2.6
 (#250)

---
 .gitignore        |  1 +
 test/test_pack.py | 18 ++----------------
 2 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/.gitignore b/.gitignore
index 70f5746..800f1c2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ msgpack/*.cpp
 /venv
 /tags
 /docs/_build
+.cache
diff --git a/test/test_pack.py b/test/test_pack.py
index a704fdb..ac93103 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -7,6 +7,7 @@ from pytest import raises, xfail
 
 from msgpack import packb, unpackb, Unpacker, Packer
 
+from collections import OrderedDict
 from io import BytesIO
 
 def check(data, use_list=False):
@@ -136,24 +137,9 @@ def testMapSize(sizes=[0, 5, 50, 1000]):
         assert unpacker.unpack() == dict((i, i * 2) for i in range(size))
 
 
-class odict(dict):
-    """Reimplement OrderedDict to run test on Python 2.6"""
-    def __init__(self, seq):
-        self._seq = seq
-        dict.__init__(self, seq)
-
-    def items(self):
-        return self._seq[:]
-
-    def iteritems(self):
-        return iter(self._seq)
-
-    def keys(self):
-        return [x[0] for x in self._seq]
-
 def test_odict():
     seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
-    od = odict(seq)
+    od = OrderedDict(seq)
     assert unpackb(packb(od), use_list=1) == dict(seq)
     def pair_hook(seq):
         return list(seq)

From 1985eb7618296e6a93c8abc4a697c7b00fda72f8 Mon Sep 17 00:00:00 2001
From: Martin Braun 
Date: Mon, 16 Oct 2017 20:30:55 -0700
Subject: [PATCH 1380/1648] Clarify README, fix grammar, update section on byte
 arrays (#253)

---
 README.rst | 59 ++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 37 insertions(+), 22 deletions(-)

diff --git a/README.rst b/README.rst
index ea1499b..3c16078 100644
--- a/README.rst
+++ b/README.rst
@@ -28,14 +28,14 @@ Install
 PyPy
 ^^^^
 
-msgpack-python provides pure python implementation.  PyPy can use this.
+msgpack-python provides a pure Python implementation.  PyPy can use this.
 
 Windows
 ^^^^^^^
 
-When you can't use binary distribution, you need to install Visual Studio
+When you can't use a binary distribution, you need to install Visual Studio
 or Windows SDK on Windows.
-Without extension, using pure python implementation on CPython runs slowly.
+Without extension, using pure Python implementation on CPython runs slowly.
 
 For Python 2.7, `Microsoft Visual C++ Compiler for Python 2.7 `_
 is recommended solution.
@@ -51,11 +51,11 @@ One-shot pack & unpack
 ^^^^^^^^^^^^^^^^^^^^^^
 
 Use ``packb`` for packing and ``unpackb`` for unpacking.
-msgpack provides ``dumps`` and ``loads`` as alias for compatibility with
+msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with
 ``json`` and ``pickle``.
 
-``pack`` and ``dump`` packs to file-like object.
-``unpack`` and ``load`` unpacks from file-like object.
+``pack`` and ``dump`` packs to a file-like object.
+``unpack`` and ``load`` unpacks from a file-like object.
 
 .. code-block:: pycon
 
@@ -65,14 +65,15 @@ msgpack provides ``dumps`` and ``loads`` as alias for compatibility with
    >>> msgpack.unpackb(_)
    [1, 2, 3]
 
-``unpack`` unpacks msgpack's array to Python's list, but can unpack to tuple:
+``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple:
 
 .. code-block:: pycon
 
    >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False)
    (1, 2, 3)
 
-You should always pass the ``use_list`` keyword argument. See performance issues relating to `use_list option`_ below.
+You should always specify the ``use_list`` keyword argument for backward compatibility.
+See performance issues relating to `use_list option`_ below.
 
 Read the docstring for other options.
 
@@ -198,29 +199,43 @@ Notes
 string and binary type
 ^^^^^^^^^^^^^^^^^^^^^^
 
-In old days, msgpack doesn't distinguish string and binary types like Python 1.
-The type for represent string and binary types is named **raw**.
+Early versions of msgpack didn't distinguish string and binary types (like Python 1).
+The type for representing both string and binary types was named **raw**.
 
-msgpack can distinguish string and binary type for now.  But it is not like Python 2.
-Python 2 added unicode string.  But msgpack renamed **raw** to **str** and added **bin** type.
-It is because keep compatibility with data created by old libs. **raw** was used for text more than binary.
+For backward compatibility reasons, msgpack-python will still default all
+strings to byte strings, unless you specify the `use_bin_type=True` option in
+the packer. If you do so, it will use a non-standard type called **bin** to
+serialize byte arrays, and **raw** becomes to mean **str**. If you want to
+distinguish **bin** and **raw** in the unpacker, specify `encoding='utf-8'`.
 
-Currently, while msgpack-python supports new **bin** type, default setting doesn't use it and
-decodes **raw** as `bytes` instead of `unicode` (`str` in Python 3).
-
-You can change this by using `use_bin_type=True` option in Packer and `encoding="utf-8"` option in Unpacker.
+Note that Python 2 defaults to byte-arrays over Unicode strings:
 
 .. code-block:: pycon
 
     >>> import msgpack
-    >>> packed = msgpack.packb([b'spam', u'egg'], use_bin_type=True)
-    >>> msgpack.unpackb(packed, encoding='utf-8')
-    ['spam', u'egg']
+    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
+    ['spam', 'eggs']
+    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
+                        encoding='utf-8')
+    ['spam', u'eggs']
+
+This is the same code in Python 3 (same behaviour, but Python 3 has a
+different default):
+
+.. code-block:: pycon
+
+    >>> import msgpack
+    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
+    [b'spam', b'eggs']
+    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
+                        encoding='utf-8')
+    [b'spam', 'eggs']
+
 
 ext type
 ^^^^^^^^
 
-To use **ext** type, pass ``msgpack.ExtType`` object to packer.
+To use the **ext** type, pass ``msgpack.ExtType`` object to packer.
 
 .. code-block:: pycon
 
@@ -234,7 +249,7 @@ You can use it with ``default`` and ``ext_hook``. See below.
 Note for msgpack-python 0.2.x users
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The msgpack-python 0.3 have some incompatible changes.
+The msgpack-python release 0.3 has some incompatible changes.
 
 The default value of ``use_list`` keyword argument is ``True`` from 0.3.
 You should pass the argument explicitly for backward compatibility.

From 3a098851bea500ef1ffde856a60d80ddab230dee Mon Sep 17 00:00:00 2001
From: Hugo 
Date: Thu, 2 Nov 2017 11:06:15 +0200
Subject: [PATCH 1381/1648] Remove code and tests for unsupported Python 3.3
 and 3.4 (#249)

---
 .travis.yml        |  2 --
 appveyor.yml       | 13 ++++---------
 build.cmd          | 21 ---------------------
 docker/runtests.sh |  2 +-
 tox.ini            |  2 +-
 5 files changed, 6 insertions(+), 34 deletions(-)
 delete mode 100644 build.cmd

diff --git a/.travis.yml b/.travis.yml
index 54e0c62..7aac664 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,8 +4,6 @@ cache: pip
 
 python:
   - "2.7"
-  - "3.3"
-  - "3.4"
   - "3.5"
   - "3.6"
   - "3.7-dev"
diff --git a/appveyor.yml b/appveyor.yml
index e63423d..d581839 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -23,24 +23,19 @@ build: off
 
 test_script:
   # Put your test command here.
-  # If you don't need to build C extensions on 64-bit Python 3.3 or 3.4,
-  # you can remove "build.cmd" from the front of the command, as it's
-  # only needed to support those cases.
   # Note that you must use the environment variable %PYTHON% to refer to
   # the interpreter you're using - Appveyor does not do anything special
   # to put the Python version you want to use on PATH.
-  - "build.cmd %PYTHON%\\python.exe setup.py build_ext -i"
-  - "build.cmd %PYTHON%\\python.exe setup.py install"
+  - "%PYTHON%\\python.exe setup.py build_ext -i"
+  - "%PYTHON%\\python.exe setup.py install"
   - "%PYTHON%\\python.exe -c \"import sys; print(hex(sys.maxsize))\""
   - "%PYTHON%\\python.exe -c \"from msgpack import _packer, _unpacker\""
   - "%PYTHON%\\Scripts\\py.test test"
-  - "build.cmd %PYTHON%\\python.exe setup.py bdist_wheel"
+  - "%PYTHON%\\python.exe setup.py bdist_wheel"
 
 after_test:
   # This step builds your wheels.
-  # Again, you only need build.cmd if you're building C extensions for
-  # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct
-  # interpreter
+  # Again, you need to use %PYTHON% to get the correct interpreter
 
 artifacts:
   # bdist_wheel puts your built wheel in the dist directory
diff --git a/build.cmd b/build.cmd
deleted file mode 100644
index 243dc9a..0000000
--- a/build.cmd
+++ /dev/null
@@ -1,21 +0,0 @@
-@echo off
-:: To build extensions for 64 bit Python 3, we need to configure environment
-:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
-:: MS Windows SDK for Windows 7 and .NET Framework 4
-::
-:: More details at:
-:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
-
-IF "%DISTUTILS_USE_SDK%"=="1" (
-    ECHO Configuring environment to build with MSVC on a 64bit architecture
-    ECHO Using Windows SDK 7.1
-    "C:\Program Files\Microsoft SDKs\Windows\v7.1\Setup\WindowsSdkVer.exe" -q -version:v7.1
-    CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 /release
-    SET MSSdk=1
-    REM Need the following to allow tox to see the SDK compiler
-    SET TOX_TESTENV_PASSENV=DISTUTILS_USE_SDK MSSdk INCLUDE LIB
-) ELSE (
-    ECHO Using default MSVC build environment
-)
-
-CALL %*
diff --git a/docker/runtests.sh b/docker/runtests.sh
index 0eea715..11ef9f4 100755
--- a/docker/runtests.sh
+++ b/docker/runtests.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e -x
 
-for V in cp36-cp36m cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do
+for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do
     PYBIN=/opt/python/$V/bin
     $PYBIN/python setup.py install
     rm -rf build/       # Avoid lib build by narrow Python is used by wide python
diff --git a/tox.ini b/tox.ini
index b2ac336..68a2f53 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {py27,py33,py34,py35,py36}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
+envlist = {py27,py35,py36}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
 
 [variants:pure]
 setenv=

From 99341035f2f7f7b9d708c73f59e92277579abb0c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 21 Dec 2017 20:46:14 +0900
Subject: [PATCH 1382/1648] fix zero length raw can't be decoded. (#236)

fix #234
---
 msgpack/unpack_template.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index e1e08fe..525dea2 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -82,7 +82,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
 
     const unsigned char* p = (unsigned char*)data + *off;
     const unsigned char* const pe = (unsigned char*)data + len;
-    const void* n = NULL;
+    const void* n = p;
 
     unsigned int trail = ctx->trail;
     unsigned int cs = ctx->cs;

From 2eb6e75db1d4b4e30997aa88f9e904dc462a28da Mon Sep 17 00:00:00 2001
From: aaron jheng 
Date: Sun, 31 Dec 2017 10:52:50 +0800
Subject: [PATCH 1383/1648] add license info to metadata (#260)

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 5c88397..5a26241 100755
--- a/setup.py
+++ b/setup.py
@@ -112,6 +112,7 @@ setup(name='msgpack-python',
       description=desc,
       long_description=long_desc,
       url='http://msgpack.org/',
+      license='Apache 2.0',
       classifiers=[
           'Programming Language :: Python :: 2',
           'Programming Language :: Python :: 2.7',

From 0e2021d3a3d1218ca191f4e802df0af3bbfaa51f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 5 Jan 2018 19:16:14 +0900
Subject: [PATCH 1384/1648] Update changelog

---
 ChangeLog.rst | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 73ffc30..ed0f92e 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,24 +1,33 @@
 0.5.0
 ======
 
-0.5 is important step toward 1.0.  There are some deprecations.
-Please read changes carefully.
+There are some deprecations.  Please read changes carefully.
 
 Changes
 -------
 
-* Drop Python 2.6 and 3.2 support
+* Drop Python 2.6 and ~3.4 support.  Python 2.7 and 3.5+ are supported.
 
 * Deprecate useless custom exceptions.  Use ValueError instead of PackValueError,
   Exception instead of PackException and UnpackException, etc...
   See msgpack/exceptions.py
 
-* Add `strict_types` option to packer.  It can be used to serialize subclass of
+* Add *strict_types* option to packer.  It can be used to serialize subclass of
   builtin types.  For example, when packing object which type is subclass of dict,
-  `default()` is called.
+  ``default()`` is called.  ``default()`` is called for tuple too.
 
 * Pure Python implementation supports packing memoryview object.
 
+* Support packing bytearray.
+
+* Add ``Unpacker.tell()``.  And ``write_bytes`` option is deprecated.
+
+
+Bugs fixed
+----------
+
+* Fixed zero length raw can't be decoded when encoding is specified. (#236)
+
 
 0.4.8
 =====

From 43137d6bd2cc841af775a9c8132e72d284b119e3 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 5 Jan 2018 20:19:04 +0900
Subject: [PATCH 1385/1648] Deprecate write_bytes option in Unpacker. (#262)

Fixes #197
---
 msgpack/_unpacker.pyx | 3 +++
 msgpack/fallback.py   | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index dabc5f7..564749e 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -458,6 +458,9 @@ cdef class Unpacker(object):
         cdef object obj
         cdef Py_ssize_t prev_head
 
+        if write_bytes is not None:
+            PyErr_WarnEx(DeprecationWarning, "`write_bytes` option is deprecated. Use `.tell()` instead.", 1)
+
         if self.buf_head >= self.buf_tail and self.file_like is not None:
             self.read_from_file()
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 28478ca..3c9c3b8 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -609,12 +609,14 @@ class Unpacker(object):
     def skip(self, write_bytes=None):
         self._unpack(EX_SKIP)
         if write_bytes is not None:
+            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
             write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
 
     def unpack(self, write_bytes=None):
         ret = self._unpack(EX_CONSTRUCT)
         if write_bytes is not None:
+            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
             write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
         return ret
@@ -622,6 +624,7 @@ class Unpacker(object):
     def read_array_header(self, write_bytes=None):
         ret = self._unpack(EX_READ_ARRAY_HEADER)
         if write_bytes is not None:
+            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
             write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
         return ret
@@ -629,6 +632,7 @@ class Unpacker(object):
     def read_map_header(self, write_bytes=None):
         ret = self._unpack(EX_READ_MAP_HEADER)
         if write_bytes is not None:
+            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
             write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
         return ret

From 1979722ba2de84e68ae5992d33bc39461aa7b4b2 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 5 Jan 2018 20:58:14 +0900
Subject: [PATCH 1386/1648] Raise MemoryError when failed to grow buffer (#263)

---
 msgpack/pack.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/msgpack/pack.h b/msgpack/pack.h
index d3aeff7..3bc21ea 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -48,7 +48,10 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_
     if (len + l > bs) {
         bs = (len + l) * 2;
         buf = (char*)PyMem_Realloc(buf, bs);
-        if (!buf) return -1;
+        if (!buf) {
+            PyErr_NoMemory();
+            return -1;
+        }
     }
     memcpy(buf + len, data, l);
     len += l;

From d0d3a403892106dfb809693f5e006a546cb55b83 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 6 Jan 2018 02:07:39 +0900
Subject: [PATCH 1387/1648] Warn about future use_bin_type change (#264)

---
 README.rst          | 13 ++++++++-----
 msgpack/_packer.pyx | 12 ++++++++++--
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/README.rst b/README.rst
index 3c16078..20353f5 100644
--- a/README.rst
+++ b/README.rst
@@ -60,7 +60,7 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with
 .. code-block:: pycon
 
    >>> import msgpack
-   >>> msgpack.packb([1, 2, 3])
+   >>> msgpack.packb([1, 2, 3], use_bin_type=True)
    '\x93\x01\x02\x03'
    >>> msgpack.unpackb(_)
    [1, 2, 3]
@@ -91,13 +91,13 @@ stream (or from bytes provided through its ``feed`` method).
 
    buf = BytesIO()
    for i in range(100):
-      buf.write(msgpack.packb(range(i)))
+      buf.write(msgpack.packb(range(i), use_bin_type=True))
 
    buf.seek(0)
 
    unpacker = msgpack.Unpacker(buf)
    for unpacked in unpacker:
-       print unpacked
+       print(unpacked)
 
 
 Packing/unpacking of custom data type
@@ -109,7 +109,6 @@ It is also possible to pack/unpack custom data types. Here is an example for
 .. code-block:: python
 
     import datetime
-
     import msgpack
 
     useful_dict = {
@@ -128,7 +127,7 @@ It is also possible to pack/unpack custom data types. Here is an example for
         return obj
 
 
-    packed_dict = msgpack.packb(useful_dict, default=encode_datetime)
+    packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True)
     this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime)
 
 ``Unpacker``'s ``object_hook`` callback receives a dict; the
@@ -208,6 +207,10 @@ the packer. If you do so, it will use a non-standard type called **bin** to
 serialize byte arrays, and **raw** becomes to mean **str**. If you want to
 distinguish **bin** and **raw** in the unpacker, specify `encoding='utf-8'`.
 
+**In future version, default value of ``use_bin_type`` will be changed to ``False``.
+To avoid this change will break your code, you must specify it explicitly
+even when you want to use old format.**
+
 Note that Python 2 defaults to byte-arrays over Unicode strings:
 
 .. code-block:: pycon
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 5a81709..ebaeb65 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -2,6 +2,7 @@
 #cython: embedsignature=True
 
 from cpython cimport *
+from cpython.exc cimport PyErr_WarnEx
 
 from msgpack.exceptions import PackValueError, PackOverflowError
 from msgpack import ExtType
@@ -76,6 +77,8 @@ cdef class Packer(object):
     :param bool use_bin_type:
         Use bin type introduced in msgpack spec 2.0 for bytes.
         It also enables str8 type for unicode.
+        Current default value is false, but it will be changed to true
+        in future version.  You should specify it explicitly.
     :param bool strict_types:
         If set to true, types will be checked to be exact. Derived classes
         from serializeable types will not be serialized and will be
@@ -103,12 +106,17 @@ cdef class Packer(object):
         self.pk.length = 0
 
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
-                 use_single_float=False, bint autoreset=1, bint use_bin_type=0,
+                 use_single_float=False, bint autoreset=1, use_bin_type=None,
                  bint strict_types=0):
+        if use_bin_type is None:
+            PyErr_WarnEx(
+                FutureWarning,
+                "use_bin_type option is not specified. Default value of the option will be changed in future version.",
+                1)
         self.use_float = use_single_float
         self.strict_types = strict_types
         self.autoreset = autoreset
-        self.pk.use_bin_type = use_bin_type
+        self.pk.use_bin_type = use_bin_type
         if default is not None:
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")

From 89e4f8b7b3a43fc84ea0dd278a4b8b54b1fac4bd Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 7 Jan 2018 01:57:47 +0900
Subject: [PATCH 1388/1648] Rename package name to msgpack

---
 setup.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 5a26241..419482b 100755
--- a/setup.py
+++ b/setup.py
@@ -102,7 +102,7 @@ with io.open('README.rst', encoding='utf-8') as f:
     long_desc = f.read()
 del f
 
-setup(name='msgpack-python',
+setup(name='msgpack',
       author='INADA Naoki',
       author_email='songofacandy@gmail.com',
       version=version_str,
@@ -124,5 +124,5 @@ setup(name='msgpack-python',
           'Programming Language :: Python :: Implementation :: PyPy',
           'Intended Audience :: Developers',
           'License :: OSI Approved :: Apache Software License',
-          ]
-      )
+      ],
+)

From d720c42468c2e9cad2c04e28c23917187fa64412 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 7 Jan 2018 01:58:01 +0900
Subject: [PATCH 1389/1648] prepare 0.5

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index 76bd8fb..f90cdc1 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 4, 8)
+version = (0, 5, 0)

From 9f4c12f29ce7a6d62423311ab12a031dc79c458a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 7 Jan 2018 01:59:14 +0900
Subject: [PATCH 1390/1648] Add transition package

---
 dummy/README   |  2 ++
 dummy/setup.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 dummy/README
 create mode 100644 dummy/setup.py

diff --git a/dummy/README b/dummy/README
new file mode 100644
index 0000000..bf1eeb6
--- /dev/null
+++ b/dummy/README
@@ -0,0 +1,2 @@
+This is dummy transition package for msgpak.
+Install msgpack instead of msgpack-python.
diff --git a/dummy/setup.py b/dummy/setup.py
new file mode 100644
index 0000000..0f1e6e7
--- /dev/null
+++ b/dummy/setup.py
@@ -0,0 +1,31 @@
+from setuptools import setup, Extension
+
+long_desc = """\
+msgpack-python is renamed to just msgpack.
+
+Install msgpack by ``pip install msgpack``.
+"""
+
+
+setup(name='msgpack-python',
+      author='INADA Naoki',
+      author_email='songofacandy@gmail.com',
+      version="0.5.0",
+      description="Transition package for msgpack",
+      long_description=long_desc,
+      install_requires=["msgpack>=0.5"],
+      url='http://msgpack.org/',
+      license='Apache 2.0',
+      classifiers=[
+          'Programming Language :: Python :: 2',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3',
+          'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
+          'Programming Language :: Python :: 3.7',
+          'Programming Language :: Python :: Implementation :: CPython',
+          'Programming Language :: Python :: Implementation :: PyPy',
+          'Intended Audience :: Developers',
+          'License :: OSI Approved :: Apache Software License',
+      ],
+)

From 35fc29797036b9bf8619b726f7c7b231c508439e Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 7 Jan 2018 02:01:20 +0900
Subject: [PATCH 1391/1648] Update README

---
 README.rst | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/README.rst b/README.rst
index 20353f5..f5fd233 100644
--- a/README.rst
+++ b/README.rst
@@ -23,12 +23,12 @@ Install
 
 ::
 
-   $ pip install msgpack-python
+   $ pip install msgpack
 
 PyPy
 ^^^^
 
-msgpack-python provides a pure Python implementation.  PyPy can use this.
+msgpack provides a pure Python implementation.  PyPy can use this.
 
 Windows
 ^^^^^^^
@@ -249,17 +249,6 @@ To use the **ext** type, pass ``msgpack.ExtType`` object to packer.
 
 You can use it with ``default`` and ``ext_hook``. See below.
 
-Note for msgpack-python 0.2.x users
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The msgpack-python release 0.3 has some incompatible changes.
-
-The default value of ``use_list`` keyword argument is ``True`` from 0.3.
-You should pass the argument explicitly for backward compatibility.
-
-`Unpacker.unpack()` and some unpack methods now raises `OutOfData`
-instead of `StopIteration`.
-`StopIteration` is used for iterator protocol only.
 
 Note about performance
 ----------------------

From dbb827815aef9e26c54467436bc267c23b2eff81 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sun, 7 Jan 2018 02:04:49 +0900
Subject: [PATCH 1392/1648] Update Cython version

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 87f04da..cd54e6d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1 @@
-Cython==0.25.2
+Cython==0.27.3

From 7c22d983f4aad34d612e6fb4dc84676fcb2bece9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 9 Jan 2018 13:17:47 +0900
Subject: [PATCH 1393/1648] Update README

---
 README.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/README.rst b/README.rst
index f5fd233..fed1dfa 100644
--- a/README.rst
+++ b/README.rst
@@ -10,6 +10,20 @@ MessagePack for Python
    :target: https://msgpack-python.readthedocs.io/en/latest/?badge=latest
    :alt: Documentation Status
 
+Upgrading from msgpack-0.4
+--------------------------
+
+TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`.
+Do `pip uninstall msgpack-python; pip install msgpack` instead.
+
+Package name on PyPI was changed to msgpack from 0.5.
+I upload transitional package (msgpack-python 0.5 which depending on msgpack)
+for smooth transition from msgpack-python to msgpack.
+
+Sadly, this doesn't work for upgrade install.  After `pip install -U msgpack-python`,
+msgpack is removed and `import msgpack` fail.
+
+
 What's this
 -----------
 

From 45c1a53d5aeab5799f76cfae02f064ba89098f7f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 9 Jan 2018 17:58:32 +0900
Subject: [PATCH 1394/1648] Update AppVeyor build (#267)

---
 appveyor.yml    | 32 ++++++++++++++------------------
 ci/runtests.bat |  7 +++++++
 2 files changed, 21 insertions(+), 18 deletions(-)
 create mode 100644 ci/runtests.bat

diff --git a/appveyor.yml b/appveyor.yml
index d581839..72b334a 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -1,22 +1,12 @@
 environment:
-
   matrix:
-
     # For Python versions available on Appveyor, see
     # http://www.appveyor.com/docs/installed-software#python
-    # The list here is complete (excluding Python 2.6, which
-    # isn't covered by this document) at the time of writing.
-
-    - PYTHON: "C:\\Python27"
-    - PYTHON: "C:\\Python27-x64"
-    - PYTHON: "C:\\Python35"
-    - PYTHON: "C:\\Python35-x64"
     - PYTHON: "C:\\Python36"
-    - PYTHON: "C:\\Python36-x64"
 
 install:
   # We need wheel installed to build wheels
-  - "%PYTHON%\\python.exe -m pip install -U pip wheel pytest cython"
+  - "%PYTHON%\\python.exe -m pip install -U cython"
   - "%PYTHON%\\Scripts\\cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx"
 
 build: off
@@ -26,12 +16,18 @@ test_script:
   # Note that you must use the environment variable %PYTHON% to refer to
   # the interpreter you're using - Appveyor does not do anything special
   # to put the Python version you want to use on PATH.
-  - "%PYTHON%\\python.exe setup.py build_ext -i"
-  - "%PYTHON%\\python.exe setup.py install"
-  - "%PYTHON%\\python.exe -c \"import sys; print(hex(sys.maxsize))\""
-  - "%PYTHON%\\python.exe -c \"from msgpack import _packer, _unpacker\""
-  - "%PYTHON%\\Scripts\\py.test test"
-  - "%PYTHON%\\python.exe setup.py bdist_wheel"
+  - set PYTHON="C:\\Python27"
+  - ci\\runtests.bat
+  - set PYTHON="C:\\Python27-x64"
+  - ci\\runtests.bat
+  - set PYTHON="C:\\Python35"
+  - ci\\runtests.bat
+  - set PYTHON="C:\\Python35-x64"
+  - ci\\runtests.bat
+  - set PYTHON="C:\\Python36"
+  - ci\\runtests.bat
+  - set PYTHON="C:\\Python36-x64"
+  - ci\\runtests.bat
 
 after_test:
   # This step builds your wheels.
@@ -39,7 +35,7 @@ after_test:
 
 artifacts:
   # bdist_wheel puts your built wheel in the dist directory
-  - path: dist\*
+  - path: dist\*.whl
 
 #on_success:
 #  You can use this step to upload your artifacts to a public website.
diff --git a/ci/runtests.bat b/ci/runtests.bat
new file mode 100644
index 0000000..9efea00
--- /dev/null
+++ b/ci/runtests.bat
@@ -0,0 +1,7 @@
+%PYTHON%\python.exe -m pip install -U pip wheel pytest
+%PYTHON%\python.exe setup.py build_ext -i
+%PYTHON%\python.exe setup.py install
+%PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))"
+%PYTHON%\python.exe -c "from msgpack import _packer, _unpacker"
+%PYTHON%\python.exe -m pytest -v test
+%PYTHON%\python.exe setup.py bdist_wheel

From 676bbcd0eee2a6e8aece5239e380c12ea633375a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 9 Jan 2018 19:00:42 +0900
Subject: [PATCH 1395/1648] manylinux1: Add 3.6 and remove 3.4

---
 docker/buildwheel.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh
index b654e45..f586a8d 100644
--- a/docker/buildwheel.sh
+++ b/docker/buildwheel.sh
@@ -4,7 +4,7 @@ set -e -x
 ARCH=`uname -p`
 echo "arch=$ARCH"
 
-for V in cp35-cp35m cp34-cp34m cp27-cp27m cp27-cp27mu; do
+for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do
     PYBIN=/opt/python/$V/bin
     rm -rf build/       # Avoid lib build by narrow Python is used by wide python
     $PYBIN/python setup.py bdist_wheel -p manylinux1_${ARCH}

From e0934355c6534690d3c80ea8659d51c65a55ee0f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 9 Jan 2018 20:48:45 +0900
Subject: [PATCH 1396/1648] Update Makefile

---
 Makefile | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/Makefile b/Makefile
index f833bbc..84decd8 100644
--- a/Makefile
+++ b/Makefile
@@ -1,32 +1,27 @@
-.PHONY: test all python3
-
+.PHONY: all
 all: cython
 	python setup.py build_ext -i -f
 
-doc-serve: all
-	cd docs && make serve
-
-doc:
-	cd docs && make zip
-
-upload-doc:
-	python setup.py upload_docs --upload-dir docs/_build/html
-
+.PHONY: cython
 cython:
 	cython --cplus msgpack/*.pyx
 
-python3: cython
-	python3 setup.py build_ext -i -f
-
+.PHONY: test
 test:
-	py.test test
+	py.test -v test
+
+.PHONY: serve-doc
+serve-doc: all
+	cd docs && make serve
 
 .PHONY: clean
 clean:
 	rm -rf build
 	rm msgpack/*.so
 	rm -rf msgpack/__pycache__
+	rm -rf test/__pycache__
 
-build-manylinux1-wheel:
+.PHONY: linux-wheel
+linux-wheel:
 	docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686   bash docker/buildwheel.sh
 	docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh

From ab66c272b03805fd16f0346238e8b7d1233b96c4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 9 Jan 2018 22:03:06 +0900
Subject: [PATCH 1397/1648] Update README

---
 README.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index fed1dfa..d863328 100644
--- a/README.rst
+++ b/README.rst
@@ -148,6 +148,7 @@ It is also possible to pack/unpack custom data types. Here is an example for
 ``object_pairs_hook`` callback may instead be used to receive a list of
 key-value pairs.
 
+
 Extended types
 ^^^^^^^^^^^^^^
 
@@ -170,7 +171,7 @@ It is also possible to pack/unpack custom data types using the **ext** type.
     ...     return ExtType(code, data)
     ...
     >>> data = array.array('d', [1.2, 3.4])
-    >>> packed = msgpack.packb(data, default=default)
+    >>> packed = msgpack.packb(data, default=default, use_bin_type=True)
     >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook)
     >>> data == unpacked
     True
@@ -294,7 +295,7 @@ Test
 MessagePack uses `pytest` for testing.
 Run test with following command:
 
-    $ py.test
+    $ pytest -v test
 
 
 ..

From 5be93786404d4e95de933d1bc64640402c3f2696 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 10 Jan 2018 02:48:08 +0900
Subject: [PATCH 1398/1648] Make msgpack-python deprecated clone of msgpack.

---
 README.rst     |  4 ++--
 dummy/README   |  2 --
 dummy/setup.py | 31 -------------------------------
 setup.py       | 11 ++++++++++-
 4 files changed, 12 insertions(+), 36 deletions(-)
 delete mode 100644 dummy/README
 delete mode 100644 dummy/setup.py

diff --git a/README.rst b/README.rst
index d863328..42758b8 100644
--- a/README.rst
+++ b/README.rst
@@ -10,8 +10,8 @@ MessagePack for Python
    :target: https://msgpack-python.readthedocs.io/en/latest/?badge=latest
    :alt: Documentation Status
 
-Upgrading from msgpack-0.4
---------------------------
+IMPORTANT: Upgrading from msgpack-0.4
+--------------------------------------
 
 TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`.
 Do `pip uninstall msgpack-python; pip install msgpack` instead.
diff --git a/dummy/README b/dummy/README
deleted file mode 100644
index bf1eeb6..0000000
--- a/dummy/README
+++ /dev/null
@@ -1,2 +0,0 @@
-This is dummy transition package for msgpak.
-Install msgpack instead of msgpack-python.
diff --git a/dummy/setup.py b/dummy/setup.py
deleted file mode 100644
index 0f1e6e7..0000000
--- a/dummy/setup.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from setuptools import setup, Extension
-
-long_desc = """\
-msgpack-python is renamed to just msgpack.
-
-Install msgpack by ``pip install msgpack``.
-"""
-
-
-setup(name='msgpack-python',
-      author='INADA Naoki',
-      author_email='songofacandy@gmail.com',
-      version="0.5.0",
-      description="Transition package for msgpack",
-      long_description=long_desc,
-      install_requires=["msgpack>=0.5"],
-      url='http://msgpack.org/',
-      license='Apache 2.0',
-      classifiers=[
-          'Programming Language :: Python :: 2',
-          'Programming Language :: Python :: 2.7',
-          'Programming Language :: Python :: 3',
-          'Programming Language :: Python :: 3.5',
-          'Programming Language :: Python :: 3.6',
-          'Programming Language :: Python :: 3.7',
-          'Programming Language :: Python :: Implementation :: CPython',
-          'Programming Language :: Python :: Implementation :: PyPy',
-          'Intended Audience :: Developers',
-          'License :: OSI Approved :: Apache Software License',
-      ],
-)
diff --git a/setup.py b/setup.py
index 419482b..6108587 100755
--- a/setup.py
+++ b/setup.py
@@ -9,6 +9,9 @@ from setuptools import setup, Extension
 
 from distutils.command.build_ext import build_ext
 
+# for building transitional package.
+TRANSITIONAL = False
+
 class NoCython(Exception):
     pass
 
@@ -102,7 +105,13 @@ with io.open('README.rst', encoding='utf-8') as f:
     long_desc = f.read()
 del f
 
-setup(name='msgpack',
+name = 'msgpack'
+
+if TRANSITIONAL:
+    name = 'msgpack-python'
+    long_desc = "This package is deprecated.  Install msgpack instead."
+
+setup(name=name,
       author='INADA Naoki',
       author_email='songofacandy@gmail.com',
       version=version_str,

From e0f2fd3af348e26b269d7eb90c74876d908aafca Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 10 Jan 2018 02:49:50 +0900
Subject: [PATCH 1399/1648] Fix README

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 42758b8..01a8b2a 100644
--- a/README.rst
+++ b/README.rst
@@ -222,7 +222,7 @@ the packer. If you do so, it will use a non-standard type called **bin** to
 serialize byte arrays, and **raw** becomes to mean **str**. If you want to
 distinguish **bin** and **raw** in the unpacker, specify `encoding='utf-8'`.
 
-**In future version, default value of ``use_bin_type`` will be changed to ``False``.
+**In future version, default value of ``use_bin_type`` will be changed to ``True``.
 To avoid this change will break your code, you must specify it explicitly
 even when you want to use old format.**
 

From 0112957bcff8e16dddd6cbc474bfe8a49f418fad Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 10 Jan 2018 02:54:59 +0900
Subject: [PATCH 1400/1648] Remove FutureWarning about use_bin_type option
 (#271)

---
 msgpack/_packer.pyx | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index ebaeb65..13a18f6 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -2,7 +2,7 @@
 #cython: embedsignature=True
 
 from cpython cimport *
-from cpython.exc cimport PyErr_WarnEx
+#from cpython.exc cimport PyErr_WarnEx
 
 from msgpack.exceptions import PackValueError, PackOverflowError
 from msgpack import ExtType
@@ -65,20 +65,20 @@ cdef class Packer(object):
     :param callable default:
         Convert user type to builtin type that Packer supports.
         See also simplejson's document.
-    :param str encoding:
-        Convert unicode to bytes with this encoding. (default: 'utf-8')
-    :param str unicode_errors:
-        Error handler for encoding unicode. (default: 'strict')
+
     :param bool use_single_float:
         Use single precision float type for float. (default: False)
+
     :param bool autoreset:
         Reset buffer after each pack and return its content as `bytes`. (default: True).
         If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+
     :param bool use_bin_type:
         Use bin type introduced in msgpack spec 2.0 for bytes.
         It also enables str8 type for unicode.
         Current default value is false, but it will be changed to true
         in future version.  You should specify it explicitly.
+
     :param bool strict_types:
         If set to true, types will be checked to be exact. Derived classes
         from serializeable types will not be serialized and will be
@@ -86,6 +86,11 @@ cdef class Packer(object):
         Additionally tuples will not be serialized as lists.
         This is useful when trying to implement accurate serialization
         for python types.
+
+    :param str encoding:
+        (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
+    :param str unicode_errors:
+        (deprecated) Error handler for encoding unicode. (default: 'strict')
     """
     cdef msgpack_packer pk
     cdef object _default
@@ -106,17 +111,12 @@ cdef class Packer(object):
         self.pk.length = 0
 
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
-                 use_single_float=False, bint autoreset=1, use_bin_type=None,
-                 bint strict_types=0):
-        if use_bin_type is None:
-            PyErr_WarnEx(
-                FutureWarning,
-                "use_bin_type option is not specified. Default value of the option will be changed in future version.",
-                1)
+                 bint use_single_float=False, bint autoreset=True, bint use_bin_type=False,
+                 bint strict_types=False):
         self.use_float = use_single_float
         self.strict_types = strict_types
         self.autoreset = autoreset
-        self.pk.use_bin_type = use_bin_type
+        self.pk.use_bin_type = use_bin_type
         if default is not None:
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")

From fc09da997ca323cb1a545478e9c1563d8db37ab1 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 10 Jan 2018 02:58:55 +0900
Subject: [PATCH 1401/1648] fallback: Update docstring.

---
 msgpack/_version.py |  2 +-
 msgpack/fallback.py | 16 +++++++++++-----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index f90cdc1..ecba3d8 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 5, 0)
+version = (0, 5, 1)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 3c9c3b8..5447b53 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -160,7 +160,7 @@ class Unpacker(object):
         If it is None (default), msgpack raw is deserialized to Python bytes.
 
     :param str unicode_errors:
-        Used for decoding msgpack raw with *encoding*.
+        (deprecated) Used for decoding msgpack raw with *encoding*.
         (default: `'strict'`)
 
     :param int max_buffer_size:
@@ -656,18 +656,18 @@ class Packer(object):
     :param callable default:
         Convert user type to builtin type that Packer supports.
         See also simplejson's document.
-    :param str encoding:
-        Convert unicode to bytes with this encoding. (default: 'utf-8')
-    :param str unicode_errors:
-        Error handler for encoding unicode. (default: 'strict')
+
     :param bool use_single_float:
         Use single precision float type for float. (default: False)
+
     :param bool autoreset:
         Reset buffer after each pack and return its content as `bytes`. (default: True).
         If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+
     :param bool use_bin_type:
         Use bin type introduced in msgpack spec 2.0 for bytes.
         It also enables str8 type for unicode.
+
     :param bool strict_types:
         If set to true, types will be checked to be exact. Derived classes
         from serializeable types will not be serialized and will be
@@ -675,6 +675,12 @@ class Packer(object):
         Additionally tuples will not be serialized as lists.
         This is useful when trying to implement accurate serialization
         for python types.
+
+    :param str encoding:
+        (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
+
+    :param str unicode_errors:
+        (deprecated) Error handler for encoding unicode. (default: 'strict')
     """
     def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
                  use_single_float=False, autoreset=True, use_bin_type=False,

From 50ea49c86f5aaff8bb1cd37778b50b13df83ba8f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 10 Jan 2018 03:04:54 +0900
Subject: [PATCH 1402/1648] Update doc

---
 docs/Makefile | 2 +-
 docs/conf.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/Makefile b/docs/Makefile
index 0869604..b09d884 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -153,7 +153,7 @@ doctest:
 	      "results in $(BUILDDIR)/doctest/output.txt."
 
 serve: html
-	cd _build/html && python3.3 -m http.server
+	cd _build/html && python3 -m http.server
 
 zip: html
 	cd _build/html && zip -r ../../../msgpack-doc.zip .
diff --git a/docs/conf.py b/docs/conf.py
index 0f19fcc..47d745a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -49,7 +49,7 @@ copyright = u'2013, INADA Naoki'
 #
 # The short X.Y version.
 # The full version, including alpha/beta/rc tags.
-version = release = '0.4'
+version = release = '0.5'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

From 5534d0c7af0114db3d27f7b96c82a7fe22ce1e40 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 11 Jan 2018 17:02:41 +0900
Subject: [PATCH 1403/1648] Add raw_as_bytes option to Unpacker. (#265)

---
 Makefile                |  3 +-
 README.rst              | 78 ++++++++++++++++++++++++++++-----------
 ci/runtests.bat         |  4 +-
 msgpack/_packer.pyx     | 18 ++++++---
 msgpack/_unpacker.pyx   | 81 ++++++++++++++++++++++++++---------------
 msgpack/fallback.py     | 52 +++++++++++++++++++++++---
 msgpack/unpack.h        | 12 ++++--
 test/test_limits.py     |  4 +-
 test/test_pack.py       | 22 +++++------
 test/test_stricttype.py |  8 ++--
 test/test_unpack.py     | 10 ++---
 11 files changed, 199 insertions(+), 93 deletions(-)

diff --git a/Makefile b/Makefile
index 84decd8..6a9906c 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,8 @@ cython:
 
 .PHONY: test
 test:
-	py.test -v test
+	pytest -v test
+	MSGPACK_PUREPYTHON=1 pytest -v test
 
 .PHONY: serve-doc
 serve-doc: all
diff --git a/README.rst b/README.rst
index 01a8b2a..a5038db 100644
--- a/README.rst
+++ b/README.rst
@@ -10,8 +10,21 @@ MessagePack for Python
    :target: https://msgpack-python.readthedocs.io/en/latest/?badge=latest
    :alt: Documentation Status
 
-IMPORTANT: Upgrading from msgpack-0.4
---------------------------------------
+
+What's this
+-----------
+
+`MessagePack `_ is an efficient binary serialization format.
+It lets you exchange data among multiple languages like JSON.
+But it's faster and smaller.
+This package provides CPython bindings for reading and writing MessagePack data.
+
+
+Very important notes for existing users
+---------------------------------------
+
+PyPI package name
+^^^^^^^^^^^^^^^^^
 
 TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`.
 Do `pip uninstall msgpack-python; pip install msgpack` instead.
@@ -24,13 +37,37 @@ Sadly, this doesn't work for upgrade install.  After `pip install -U msgpack-pyt
 msgpack is removed and `import msgpack` fail.
 
 
-What's this
------------
+Deprecating encoding option
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+encoding and unicode_errors options are deprecated.
+
+In case of packer, use UTF-8 always.  Storing other than UTF-8 is not recommended.
+
+For backward compatibility, you can use ``use_bin_type=False`` and pack ``bytes``
+object into msgpack raw type.
+
+In case of unpacker, there is new ``raw_as_bytes`` option.  It is ``True`` by default
+for backward compatibility, but it is changed to ``False`` in near future.
+You can use ``raw_as_bytes=False`` instead of ``encoding='utf-8'``.
+
+Planned backward incompatible changes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When msgpack 1.0, I planning these breaking changes:
+
+* packer and unpacker: Remove ``encoding`` and ``unicode_errors`` option.
+* packer: Change default of ``use_bin_type`` option from False to True.
+* unpacker: Change default of ``raw_as_bytes`` option from True to False.
+* unpacker: Reduce all ``max_xxx_len`` options for typical usage.
+* unpacker: Remove ``write_bytes`` option from all methods.
+
+To avoid these breaking changes breaks your application, please:
+
+* Don't use deprecated options.
+* Pass ``use_bin_type`` and ``raw_as_bytes`` options explicitly.
+* If your application handle large (>1MB) data, specify ``max_xxx_len`` options too.
 
-`MessagePack `_ is an efficient binary serialization format.
-It lets you exchange data among multiple languages like JSON.
-But it's faster and smaller.
-This package provides CPython bindings for reading and writing MessagePack data.
 
 Install
 -------
@@ -76,14 +113,14 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with
    >>> import msgpack
    >>> msgpack.packb([1, 2, 3], use_bin_type=True)
    '\x93\x01\x02\x03'
-   >>> msgpack.unpackb(_)
+   >>> msgpack.unpackb(_, raw_as_bytes=False)
    [1, 2, 3]
 
 ``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple:
 
 .. code-block:: pycon
 
-   >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False)
+   >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw_as_bytes=False)
    (1, 2, 3)
 
 You should always specify the ``use_list`` keyword argument for backward compatibility.
@@ -109,7 +146,7 @@ stream (or from bytes provided through its ``feed`` method).
 
    buf.seek(0)
 
-   unpacker = msgpack.Unpacker(buf)
+   unpacker = msgpack.Unpacker(buf, raw_as_bytes=False)
    for unpacked in unpacker:
        print(unpacked)
 
@@ -142,7 +179,7 @@ It is also possible to pack/unpack custom data types. Here is an example for
 
 
     packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True)
-    this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime)
+    this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw_as_bytes=False)
 
 ``Unpacker``'s ``object_hook`` callback receives a dict; the
 ``object_pairs_hook`` callback may instead be used to receive a list of
@@ -172,7 +209,7 @@ It is also possible to pack/unpack custom data types using the **ext** type.
     ...
     >>> data = array.array('d', [1.2, 3.4])
     >>> packed = msgpack.packb(data, default=default, use_bin_type=True)
-    >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook)
+    >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw_as_bytes=False)
     >>> data == unpacked
     True
 
@@ -217,14 +254,10 @@ Early versions of msgpack didn't distinguish string and binary types (like Pytho
 The type for representing both string and binary types was named **raw**.
 
 For backward compatibility reasons, msgpack-python will still default all
-strings to byte strings, unless you specify the `use_bin_type=True` option in
+strings to byte strings, unless you specify the ``use_bin_type=True`` option in
 the packer. If you do so, it will use a non-standard type called **bin** to
 serialize byte arrays, and **raw** becomes to mean **str**. If you want to
-distinguish **bin** and **raw** in the unpacker, specify `encoding='utf-8'`.
-
-**In future version, default value of ``use_bin_type`` will be changed to ``True``.
-To avoid this change will break your code, you must specify it explicitly
-even when you want to use old format.**
+distinguish **bin** and **raw** in the unpacker, specify ``raw_as_bytes=False``.
 
 Note that Python 2 defaults to byte-arrays over Unicode strings:
 
@@ -234,7 +267,7 @@ Note that Python 2 defaults to byte-arrays over Unicode strings:
     >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
     ['spam', 'eggs']
     >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
-                        encoding='utf-8')
+                        raw_as_bytes=False)
     ['spam', u'eggs']
 
 This is the same code in Python 3 (same behaviour, but Python 3 has a
@@ -246,7 +279,7 @@ different default):
     >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
     [b'spam', b'eggs']
     >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
-                        encoding='utf-8')
+                        raw_as_bytes=False)
     [b'spam', 'eggs']
 
 
@@ -277,6 +310,7 @@ You can use ``gc.disable()`` when unpacking large message.
 
 use_list option
 ^^^^^^^^^^^^^^^
+
 List is the default sequence type of Python.
 But tuple is lighter than list.
 You can use ``use_list=False`` while unpacking when performance is important.
@@ -295,7 +329,7 @@ Test
 MessagePack uses `pytest` for testing.
 Run test with following command:
 
-    $ pytest -v test
+    $ make test
 
 
 ..
diff --git a/ci/runtests.bat b/ci/runtests.bat
index 9efea00..0240467 100644
--- a/ci/runtests.bat
+++ b/ci/runtests.bat
@@ -3,5 +3,7 @@
 %PYTHON%\python.exe setup.py install
 %PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))"
 %PYTHON%\python.exe -c "from msgpack import _packer, _unpacker"
-%PYTHON%\python.exe -m pytest -v test
 %PYTHON%\python.exe setup.py bdist_wheel
+%PYTHON%\python.exe -m pytest -v test
+SET EL=%ERRORLEVEL%
+exit /b %EL%
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 13a18f6..39da91b 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -2,7 +2,7 @@
 #cython: embedsignature=True
 
 from cpython cimport *
-#from cpython.exc cimport PyErr_WarnEx
+from cpython.exc cimport PyErr_WarnEx
 
 from msgpack.exceptions import PackValueError, PackOverflowError
 from msgpack import ExtType
@@ -39,7 +39,7 @@ cdef extern from "pack.h":
     int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l)
 
 cdef int DEFAULT_RECURSE_LIMIT=511
-cdef size_t ITEM_LIMIT = (2**32)-1
+cdef long long ITEM_LIMIT = (2**32)-1
 
 
 cdef inline int PyBytesLike_Check(object o):
@@ -110,9 +110,13 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
-    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
+    def __init__(self, default=None, encoding=None, unicode_errors=None,
                  bint use_single_float=False, bint autoreset=True, bint use_bin_type=False,
                  bint strict_types=False):
+        if encoding is not None:
+            PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated.", 1)
+        if unicode_errors is not None:
+            PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated.", 1)
         self.use_float = use_single_float
         self.strict_types = strict_types
         self.autoreset = autoreset
@@ -122,7 +126,7 @@ cdef class Packer(object):
                 raise TypeError("default must be a callable.")
         self._default = default
         if encoding is None:
-            self.encoding = NULL
+            self.encoding = 'utf_8'
             self.unicode_errors = NULL
         else:
             if isinstance(encoding, unicode):
@@ -134,7 +138,8 @@ cdef class Packer(object):
                 self._berrors = unicode_errors.encode('ascii')
             else:
                 self._berrors = unicode_errors
-            self.unicode_errors = PyBytes_AsString(self._berrors)
+            if self._berrors is not None:
+                self.unicode_errors = PyBytes_AsString(self._berrors)
 
     def __dealloc__(self):
         PyMem_Free(self.pk.buf)
@@ -149,7 +154,7 @@ cdef class Packer(object):
         cdef char* rawval
         cdef int ret
         cdef dict d
-        cdef size_t L
+        cdef Py_ssize_t L
         cdef int default_used = 0
         cdef bint strict_types = self.strict_types
         cdef Py_buffer view
@@ -203,6 +208,7 @@ cdef class Packer(object):
             elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o):
                 if not self.encoding:
                     raise TypeError("Can't encode unicode string: no encoding is specified")
+                #TODO: Use faster API for UTF-8
                 o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
                 L = len(o)
                 if L > ITEM_LIMIT:
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 564749e..b796d04 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -43,8 +43,9 @@ from msgpack import ExtType
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
         bint use_list
-        PyObject* object_hook
+        bint raw_as_bytes
         bint has_pairs_hook # call object_hook with k-v pairs
+        PyObject* object_hook
         PyObject* list_hook
         PyObject* ext_hook
         char *encoding
@@ -73,12 +74,14 @@ cdef extern from "unpack.h":
 cdef inline init_ctx(unpack_context *ctx,
                      object object_hook, object object_pairs_hook,
                      object list_hook, object ext_hook,
-                     bint use_list, char* encoding, char* unicode_errors,
+                     bint use_list, bint raw_as_bytes,
+                     char* encoding, char* unicode_errors,
                      Py_ssize_t max_str_len, Py_ssize_t max_bin_len,
                      Py_ssize_t max_array_len, Py_ssize_t max_map_len,
                      Py_ssize_t max_ext_len):
     unpack_init(ctx)
     ctx.user.use_list = use_list
+    ctx.user.raw_as_bytes = raw_as_bytes
     ctx.user.object_hook = ctx.user.list_hook = NULL
     ctx.user.max_str_len = max_str_len
     ctx.user.max_bin_len = max_bin_len
@@ -155,7 +158,8 @@ cdef inline int get_data_from_buffer(object obj,
         return 1
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
-            bint use_list=1, encoding=None, unicode_errors="strict",
+            bint use_list=True, bint raw_as_bytes=True,
+            encoding=None, unicode_errors="strict",
             object_pairs_hook=None, ext_hook=ExtType,
             Py_ssize_t max_str_len=2147483647, # 2**32-1
             Py_ssize_t max_bin_len=2147483647,
@@ -180,21 +184,26 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef char* cerr = NULL
     cdef int new_protocol = 0
 
+    if encoding is not None:
+        PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1)
+        if isinstance(encoding, unicode):
+            encoding = encoding.encode('ascii')
+        elif not isinstance(encoding, bytes):
+            raise TypeError("encoding should be bytes or unicode")
+        cenc = PyBytes_AsString(encoding)
+
+    if unicode_errors is not None:
+        PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated", 1)
+        if isinstance(unicode_errors, unicode):
+            unicode_errors = unicode_errors.encode('ascii')
+        elif not isinstance(unicode_errors, bytes):
+            raise TypeError("unicode_errors should be bytes or unicode")
+        cerr = PyBytes_AsString(unicode_errors)
+
     get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol)
-
     try:
-        if encoding is not None:
-            if isinstance(encoding, unicode):
-                encoding = encoding.encode('ascii')
-            cenc = PyBytes_AsString(encoding)
-
-        if unicode_errors is not None:
-            if isinstance(unicode_errors, unicode):
-                unicode_errors = unicode_errors.encode('ascii')
-            cerr = PyBytes_AsString(unicode_errors)
-
         init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
-                 use_list, cenc, cerr,
+                 use_list, raw_as_bytes, cenc, cerr,
                  max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
         ret = unpack_construct(&ctx, buf, buf_len, &off)
     finally:
@@ -252,6 +261,16 @@ cdef class Unpacker(object):
         If true, unpack msgpack array to Python list.
         Otherwise, unpack to Python tuple. (default: True)
 
+    :param bool raw_as_bytes:
+        If true, unpack msgpack raw to Python bytes (default).
+        Otherwise, unpack to Python str (or unicode on Python 2) by decoding
+        with UTF-8 encoding (recommended).
+        Currently, the default is true, but it will be changed to false in
+        near future.  So you must specify it explicitly for keeping backward
+        compatibility.
+
+        *encoding* option which is deprecated overrides this option.
+
     :param callable object_hook:
         When specified, it should be callable.
         Unpacker calls it with a dict argument after unpacking msgpack map.
@@ -262,14 +281,6 @@ cdef class Unpacker(object):
         Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
         (See also simplejson)
 
-    :param str encoding:
-        Encoding used for decoding msgpack raw.
-        If it is None (default), msgpack raw is deserialized to Python bytes.
-
-    :param str unicode_errors:
-        Used for decoding msgpack raw with *encoding*.
-        (default: `'strict'`)
-
     :param int max_buffer_size:
         Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
         Raises `BufferFull` exception when it is insufficient.
@@ -287,16 +298,25 @@ cdef class Unpacker(object):
     :param int max_map_len:
         Limits max length of map. (default: 2**31-1)
 
+    :param str encoding:
+        Deprecated, use raw_as_bytes instead.
+        Encoding used for decoding msgpack raw.
+        If it is None (default), msgpack raw is deserialized to Python bytes.
 
-    example of streaming deserialize from file-like object::
+    :param str unicode_errors:
+        Deprecated. Used for decoding msgpack raw with *encoding*.
+        (default: `'strict'`)
 
-        unpacker = Unpacker(file_like)
+
+    Example of streaming deserialize from file-like object::
+
+        unpacker = Unpacker(file_like, raw_as_bytes=False)
         for o in unpacker:
             process(o)
 
-    example of streaming deserialize from socket::
+    Example of streaming deserialize from socket::
 
-        unpacker = Unpacker()
+        unpacker = Unpacker(raw_as_bytes=False)
         while True:
             buf = sock.recv(1024**2)
             if not buf:
@@ -324,7 +344,8 @@ cdef class Unpacker(object):
         PyMem_Free(self.buf)
         self.buf = NULL
 
-    def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1,
+    def __init__(self, file_like=None, Py_ssize_t read_size=0,
+                 bint use_list=True, bint raw_as_bytes=True,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict', int max_buffer_size=0,
                  object ext_hook=ExtType,
@@ -363,6 +384,7 @@ cdef class Unpacker(object):
         self.stream_offset = 0
 
         if encoding is not None:
+            PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1)
             if isinstance(encoding, unicode):
                 self.encoding = encoding.encode('ascii')
             elif isinstance(encoding, bytes):
@@ -372,6 +394,7 @@ cdef class Unpacker(object):
             cenc = PyBytes_AsString(self.encoding)
 
         if unicode_errors is not None:
+            PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated", 1)
             if isinstance(unicode_errors, unicode):
                 self.unicode_errors = unicode_errors.encode('ascii')
             elif isinstance(unicode_errors, bytes):
@@ -381,7 +404,7 @@ cdef class Unpacker(object):
             cerr = PyBytes_AsString(self.unicode_errors)
 
         init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
-                 ext_hook, use_list, cenc, cerr,
+                 ext_hook, use_list, raw_as_bytes, cenc, cerr,
                  max_str_len, max_bin_len, max_array_len,
                  max_map_len, max_ext_len)
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 5447b53..d95f621 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -145,6 +145,16 @@ class Unpacker(object):
         If true, unpack msgpack array to Python list.
         Otherwise, unpack to Python tuple. (default: True)
 
+    :param bool raw_as_bytes:
+        If true, unpack msgpack raw to Python bytes (default).
+        Otherwise, unpack to Python str (or unicode on Python 2) by decoding
+        with UTF-8 encoding (recommended).
+        Currently, the default is true, but it will be changed to false in
+        near future.  So you must specify it explicitly for keeping backward
+        compatibility.
+
+        *encoding* option which is deprecated overrides this option.
+
     :param callable object_hook:
         When specified, it should be callable.
         Unpacker calls it with a dict argument after unpacking msgpack map.
@@ -183,13 +193,13 @@ class Unpacker(object):
 
     example of streaming deserialize from file-like object::
 
-        unpacker = Unpacker(file_like)
+        unpacker = Unpacker(file_like, raw_as_bytes=False)
         for o in unpacker:
             process(o)
 
     example of streaming deserialize from socket::
 
-        unpacker = Unpacker()
+        unpacker = Unpacker(raw_as_bytes=False)
         while True:
             buf = sock.recv(1024**2)
             if not buf:
@@ -199,15 +209,28 @@ class Unpacker(object):
                 process(o)
     """
 
-    def __init__(self, file_like=None, read_size=0, use_list=True,
+    def __init__(self, file_like=None, read_size=0, use_list=True, raw_as_bytes=True,
                  object_hook=None, object_pairs_hook=None, list_hook=None,
-                 encoding=None, unicode_errors='strict', max_buffer_size=0,
+                 encoding=None, unicode_errors=None, max_buffer_size=0,
                  ext_hook=ExtType,
                  max_str_len=2147483647, # 2**32-1
                  max_bin_len=2147483647,
                  max_array_len=2147483647,
                  max_map_len=2147483647,
                  max_ext_len=2147483647):
+
+        if encoding is not None:
+            warnings.warn(
+                "encoding is deprecated, Use raw_as_bytes=False instead.",
+                PendingDeprecationWarning)
+
+        if unicode_errors is not None:
+            warnings.warn(
+                "unicode_errors is deprecated.",
+                PendingDeprecationWarning)
+        else:
+            unicode_errors = 'strict'
+
         if file_like is None:
             self._feeding = True
         else:
@@ -234,6 +257,7 @@ class Unpacker(object):
         if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")
         self._read_size = read_size or min(self._max_buffer_size, 16*1024)
+        self._raw_as_bytes = bool(raw_as_bytes)
         self._encoding = encoding
         self._unicode_errors = unicode_errors
         self._use_list = use_list
@@ -582,8 +606,10 @@ class Unpacker(object):
         if typ == TYPE_RAW:
             if self._encoding is not None:
                 obj = obj.decode(self._encoding, self._unicode_errors)
-            else:
+            elif self._raw_as_bytes:
                 obj = bytes(obj)
+            else:
+                obj = obj.decode('utf_8')
             return obj
         if typ == TYPE_EXT:
             return self._ext_hook(n, bytes(obj))
@@ -682,9 +708,23 @@ class Packer(object):
     :param str unicode_errors:
         (deprecated) Error handler for encoding unicode. (default: 'strict')
     """
-    def __init__(self, default=None, encoding='utf-8', unicode_errors='strict',
+    def __init__(self, default=None, encoding=None, unicode_errors=None,
                  use_single_float=False, autoreset=True, use_bin_type=False,
                  strict_types=False):
+        if encoding is None:
+            encoding = 'utf_8'
+        else:
+            warnings.warn(
+                "encoding is deprecated, Use raw_as_bytes=False instead.",
+                PendingDeprecationWarning)
+
+        if unicode_errors is None:
+            unicode_errors = 'strict'
+        else:
+            warnings.warn(
+                "unicode_errors is deprecated.",
+                PendingDeprecationWarning)
+
         self._strict_types = strict_types
         self._use_float = use_single_float
         self._autoreset = autoreset
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index da2cfb6..8c2fc46 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -20,9 +20,10 @@
 #include "unpack_define.h"
 
 typedef struct unpack_user {
-    int use_list;
-    PyObject *object_hook;
+    bool use_list;
+    bool raw_as_bytes;
     bool has_pairs_hook;
+    PyObject *object_hook;
     PyObject *list_hook;
     PyObject *ext_hook;
     const char *encoding;
@@ -225,10 +226,13 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
     }
 
     PyObject *py;
-    if(u->encoding) {
+
+    if (u->encoding) {
         py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors);
-    } else {
+    } else if (u->raw_as_bytes) {
         py = PyBytes_FromStringAndSize(p, l);
+    } else {
+        py = PyUnicode_DecodeUTF8(p, l, NULL);
     }
     if (!py)
         return -1;
diff --git a/test/test_limits.py b/test/test_limits.py
index 197ef46..3febc30 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -39,11 +39,11 @@ def test_max_str_len():
     d = 'x' * 3
     packed = packb(d)
 
-    unpacker = Unpacker(max_str_len=3, encoding='utf-8')
+    unpacker = Unpacker(max_str_len=3, raw_as_bytes=False)
     unpacker.feed(packed)
     assert unpacker.unpack() == d
 
-    unpacker = Unpacker(max_str_len=2, encoding='utf-8')
+    unpacker = Unpacker(max_str_len=2, raw_as_bytes=False)
     with pytest.raises(UnpackValueError):
         unpacker.feed(packed)
         unpacker.unpack()
diff --git a/test/test_pack.py b/test/test_pack.py
index ac93103..29f5887 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -31,14 +31,14 @@ def testPack():
 def testPackUnicode():
     test_data = ["", "abcd", ["defgh"], "РуÑÑкий текÑÑ‚"]
     for td in test_data:
-        re = unpackb(packb(td, encoding='utf-8'), use_list=1, encoding='utf-8')
+        re = unpackb(packb(td), use_list=1, raw_as_bytes=False)
         assert re == td
-        packer = Packer(encoding='utf-8')
+        packer = Packer()
         data = packer.pack(td)
-        re = Unpacker(BytesIO(data), encoding=str('utf-8'), use_list=1).unpack()
+        re = Unpacker(BytesIO(data), raw_as_bytes=False, use_list=1).unpack()
         assert re == td
 
-def testPackUTF32():
+def testPackUTF32():  # deprecated
     try:
         test_data = [
             "",
@@ -66,26 +66,22 @@ def testPackByteArrays():
     for td in test_data:
         check(td)
 
-def testIgnoreUnicodeErrors():
+def testIgnoreUnicodeErrors(): # deprecated
     re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
     assert re == "abcdef"
 
 def testStrictUnicodeUnpack():
     with raises(UnicodeDecodeError):
-        unpackb(packb(b'abc\xeddef'), encoding='utf-8', use_list=1)
+        unpackb(packb(b'abc\xeddef'), raw_as_bytes=False, use_list=1)
 
-def testStrictUnicodePack():
+def testStrictUnicodePack():  # deprecated
     with raises(UnicodeEncodeError):
         packb("abc\xeddef", encoding='ascii', unicode_errors='strict')
 
-def testIgnoreErrorsPack():
-    re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), encoding='utf-8', use_list=1)
+def testIgnoreErrorsPack():  # deprecated
+    re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw_as_bytes=False, use_list=1)
     assert re == "abcdef"
 
-def testNoEncoding():
-    with raises(TypeError):
-        packb("abc", encoding=None)
-
 def testDecodeBinary():
     re = unpackb(packb(b"abc"), encoding=None, use_list=1)
     assert re == b"abc"
diff --git a/test/test_stricttype.py b/test/test_stricttype.py
index 0f865c8..13239f1 100644
--- a/test/test_stricttype.py
+++ b/test/test_stricttype.py
@@ -11,7 +11,7 @@ def test_namedtuple():
             return dict(o._asdict())
         raise TypeError('Unsupported type %s' % (type(o),))
     packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default)
-    unpacked = unpackb(packed, encoding='utf-8')
+    unpacked = unpackb(packed, raw_as_bytes=False)
     assert unpacked == {'foo': 1, 'bar': 42}
 
 
@@ -32,7 +32,7 @@ def test_tuple():
         return o
 
     data = packb(t, strict_types=True, use_bin_type=True, default=default)
-    expected = unpackb(data, encoding='utf-8', object_hook=convert)
+    expected = unpackb(data, raw_as_bytes=False, object_hook=convert)
 
     assert expected == t
 
@@ -53,10 +53,10 @@ def test_tuple_ext():
     def convert(code, payload):
         if code == MSGPACK_EXT_TYPE_TUPLE:
             # Unpack and convert to tuple
-            return tuple(unpackb(payload, encoding='utf-8', ext_hook=convert))
+            return tuple(unpackb(payload, raw_as_bytes=False, ext_hook=convert))
         raise ValueError('Unknown Ext code {}'.format(code))
 
     data = packb(t, strict_types=True, use_bin_type=True, default=default)
-    expected = unpackb(data, encoding='utf-8', ext_hook=convert)
+    expected = unpackb(data, raw_as_bytes=False, ext_hook=convert)
 
     assert expected == t
diff --git a/test/test_unpack.py b/test/test_unpack.py
index c0d711c..143f999 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -47,8 +47,8 @@ def test_unpacker_ext_hook():
     class MyUnpacker(Unpacker):
 
         def __init__(self):
-            super(MyUnpacker, self).__init__(ext_hook=self._hook,
-                                             encoding='utf-8')
+            super(MyUnpacker, self).__init__(
+                ext_hook=self._hook, raw_as_bytes=False)
 
         def _hook(self, code, data):
             if code == 1:
@@ -57,11 +57,11 @@ def test_unpacker_ext_hook():
                 return ExtType(code, data)
 
     unpacker = MyUnpacker()
-    unpacker.feed(packb({'a': 1}, encoding='utf-8'))
+    unpacker.feed(packb({'a': 1}))
     assert unpacker.unpack() == {'a': 1}
-    unpacker.feed(packb({'a': ExtType(1, b'123')}, encoding='utf-8'))
+    unpacker.feed(packb({'a': ExtType(1, b'123')}))
     assert unpacker.unpack() == {'a': 123}
-    unpacker.feed(packb({'a': ExtType(2, b'321')}, encoding='utf-8'))
+    unpacker.feed(packb({'a': ExtType(2, b'321')}))
     assert unpacker.unpack() == {'a': ExtType(2, b'321')}
 
 

From 60ef3879d792ec92480cf9d6d610951657c2e8c7 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 11 Jan 2018 19:41:05 +0900
Subject: [PATCH 1404/1648] packer: Use PyUnicode_AsUTF8AndSize() for utf-8
 (#272)

---
 docker/runtests.sh  |  2 +-
 msgpack/_packer.pyx | 36 +++++++++++++++++++++-------------
 msgpack/pack.h      | 47 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 71 insertions(+), 14 deletions(-)

diff --git a/docker/runtests.sh b/docker/runtests.sh
index 11ef9f4..113b630 100755
--- a/docker/runtests.sh
+++ b/docker/runtests.sh
@@ -9,6 +9,6 @@ for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do
     pushd test          # prevent importing msgpack package in current directory.
     $PYBIN/python -c 'import sys; print(hex(sys.maxsize))'
     $PYBIN/python -c 'from msgpack import _packer, _unpacker'
-    $PYBIN/py.test -v
+    $PYBIN/pytest -v .
     popd
 done
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 39da91b..a4913ab 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -13,6 +13,7 @@ cdef extern from "Python.h":
     int PyMemoryView_Check(object obj)
     int PyByteArray_Check(object obj)
     int PyByteArray_CheckExact(object obj)
+    char* PyUnicode_AsUTF8AndSize(object obj, Py_ssize_t *l) except NULL
 
 
 cdef extern from "pack.h":
@@ -37,6 +38,7 @@ cdef extern from "pack.h":
     int msgpack_pack_bin(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
     int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l)
+    int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit)
 
 cdef int DEFAULT_RECURSE_LIMIT=511
 cdef long long ITEM_LIMIT = (2**32)-1
@@ -126,8 +128,12 @@ cdef class Packer(object):
                 raise TypeError("default must be a callable.")
         self._default = default
         if encoding is None:
-            self.encoding = 'utf_8'
-            self.unicode_errors = NULL
+            if unicode_errors is None:
+                self.encoding = NULL
+                self.unicode_errors = NULL
+            else:
+                self.encoding = "utf_8"
+                self.unicode_errors = unicode_errors
         else:
             if isinstance(encoding, unicode):
                 self._bencoding = encoding.encode('ascii')
@@ -140,6 +146,8 @@ cdef class Packer(object):
                 self._berrors = unicode_errors
             if self._berrors is not None:
                 self.unicode_errors = PyBytes_AsString(self._berrors)
+            else:
+                self.unicode_errors = NULL
 
     def __dealloc__(self):
         PyMem_Free(self.pk.buf)
@@ -206,17 +214,19 @@ cdef class Packer(object):
                 if ret == 0:
                     ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o):
-                if not self.encoding:
-                    raise TypeError("Can't encode unicode string: no encoding is specified")
-                #TODO: Use faster API for UTF-8
-                o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
-                L = len(o)
-                if L > ITEM_LIMIT:
-                    raise PackValueError("unicode string is too large")
-                rawval = o
-                ret = msgpack_pack_raw(&self.pk, L)
-                if ret == 0:
-                    ret = msgpack_pack_raw_body(&self.pk, rawval, L)
+                if self.encoding == NULL:
+                    ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT);
+                    if ret == -2:
+                        raise PackValueError("unicode string is too large")
+                else:
+                    o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
+                    L = len(o)
+                    if L > ITEM_LIMIT:
+                        raise PackValueError("unicode string is too large")
+                    ret = msgpack_pack_raw(&self.pk, L)
+                    if ret == 0:
+                        rawval = o
+                        ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyDict_CheckExact(o):
                 d = o
                 L = len(d)
diff --git a/msgpack/pack.h b/msgpack/pack.h
index 3bc21ea..4f3ce1d 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -67,6 +67,53 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_
 
 #include "pack_template.h"
 
+// return -2 when o is too long
+static inline int
+msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit)
+{
+#if PY_MAJOR_VERSION >= 3
+    assert(PyUnicode_Check(o));
+
+    Py_ssize_t len;
+    const char* buf = PyUnicode_AsUTF8AndSize(o, &len);
+    if (buf == NULL)
+        return -1;
+
+    if (len > limit) {
+        return -2;
+    }
+
+    int ret = msgpack_pack_raw(pk, len);
+    if (ret) return ret;
+
+    return msgpack_pack_raw_body(pk, buf, len);
+#else
+    PyObject *bytes;
+    Py_ssize_t len;
+    int ret;
+
+    // py2
+    bytes = PyUnicode_AsUTF8String(o);
+    if (bytes == NULL)
+        return -1;
+
+    len = PyString_GET_SIZE(bytes);
+    if (len > limit) {
+        Py_DECREF(bytes);
+        return -2;
+    }
+
+    ret = msgpack_pack_raw(pk, len);
+    if (ret) {
+        Py_DECREF(bytes);
+        return -1;
+    }
+    ret = msgpack_pack_raw_body(pk, PyString_AS_STRING(bytes), len);
+    Py_DECREF(bytes);
+    return ret;
+#endif
+}
+
 #ifdef __cplusplus
 }
 #endif

From d9ec8fc905fc9ed37c86700f794adeb883b4f5ea Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 11 Jan 2018 23:50:41 +0900
Subject: [PATCH 1405/1648] Packer.pack() reset buffer on exception (#274)

fixes #210
---
 msgpack/_packer.pyx | 12 +++++++-----
 msgpack/fallback.py |  6 +++++-
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index a4913ab..35e5a9d 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -289,11 +289,13 @@ cdef class Packer(object):
 
     cpdef pack(self, object obj):
         cdef int ret
-        ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
-        if ret == -1:
-            raise MemoryError
-        elif ret:  # should not happen.
-            raise TypeError
+        try:
+            ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
+        except:
+            self.pk.length = 0
+            raise
+        if ret:  # should not happen.
+            raise RuntimeError("internal error")
         if self.autoreset:
             buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
             self.pk.length = 0
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index d95f621..675ee8a 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -848,7 +848,11 @@ class Packer(object):
             raise TypeError("Cannot serialize %r" % (obj, ))
 
     def pack(self, obj):
-        self._pack(obj)
+        try:
+            self._pack(obj)
+        except:
+            self._buffer = StringIO()  # force reset
+            raise
         ret = self._buffer.getvalue()
         if self._autoreset:
             self._buffer = StringIO()

From 5569a4efcdc913d343eaff4e55c9b19fafde4268 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 12 Jan 2018 19:22:36 +0900
Subject: [PATCH 1406/1648] s/raw_as_bytes/raw/g (#276)

fixes #273
---
 Makefile                |  3 ++-
 README.rst              | 24 ++++++++++++------------
 msgpack/_unpacker.pyx   | 26 +++++++++++++-------------
 msgpack/fallback.py     | 16 ++++++++--------
 msgpack/unpack.h        |  4 ++--
 test/test_limits.py     |  4 ++--
 test/test_pack.py       |  8 ++++----
 test/test_stricttype.py |  8 ++++----
 test/test_unpack.py     |  2 +-
 9 files changed, 48 insertions(+), 47 deletions(-)

diff --git a/Makefile b/Makefile
index 6a9906c..4030080 100644
--- a/Makefile
+++ b/Makefile
@@ -18,7 +18,8 @@ serve-doc: all
 .PHONY: clean
 clean:
 	rm -rf build
-	rm msgpack/*.so
+	rm -f msgpack/_packer.cpp
+	rm -f msgpack/_unpacker.cpp
 	rm -rf msgpack/__pycache__
 	rm -rf test/__pycache__
 
diff --git a/README.rst b/README.rst
index a5038db..8925a65 100644
--- a/README.rst
+++ b/README.rst
@@ -47,9 +47,9 @@ In case of packer, use UTF-8 always.  Storing other than UTF-8 is not recommende
 For backward compatibility, you can use ``use_bin_type=False`` and pack ``bytes``
 object into msgpack raw type.
 
-In case of unpacker, there is new ``raw_as_bytes`` option.  It is ``True`` by default
+In case of unpacker, there is new ``raw`` option.  It is ``True`` by default
 for backward compatibility, but it is changed to ``False`` in near future.
-You can use ``raw_as_bytes=False`` instead of ``encoding='utf-8'``.
+You can use ``raw=False`` instead of ``encoding='utf-8'``.
 
 Planned backward incompatible changes
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -58,14 +58,14 @@ When msgpack 1.0, I planning these breaking changes:
 
 * packer and unpacker: Remove ``encoding`` and ``unicode_errors`` option.
 * packer: Change default of ``use_bin_type`` option from False to True.
-* unpacker: Change default of ``raw_as_bytes`` option from True to False.
+* unpacker: Change default of ``raw`` option from True to False.
 * unpacker: Reduce all ``max_xxx_len`` options for typical usage.
 * unpacker: Remove ``write_bytes`` option from all methods.
 
 To avoid these breaking changes breaks your application, please:
 
 * Don't use deprecated options.
-* Pass ``use_bin_type`` and ``raw_as_bytes`` options explicitly.
+* Pass ``use_bin_type`` and ``raw`` options explicitly.
 * If your application handle large (>1MB) data, specify ``max_xxx_len`` options too.
 
 
@@ -113,14 +113,14 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with
    >>> import msgpack
    >>> msgpack.packb([1, 2, 3], use_bin_type=True)
    '\x93\x01\x02\x03'
-   >>> msgpack.unpackb(_, raw_as_bytes=False)
+   >>> msgpack.unpackb(_, raw=False)
    [1, 2, 3]
 
 ``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple:
 
 .. code-block:: pycon
 
-   >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw_as_bytes=False)
+   >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False)
    (1, 2, 3)
 
 You should always specify the ``use_list`` keyword argument for backward compatibility.
@@ -146,7 +146,7 @@ stream (or from bytes provided through its ``feed`` method).
 
    buf.seek(0)
 
-   unpacker = msgpack.Unpacker(buf, raw_as_bytes=False)
+   unpacker = msgpack.Unpacker(buf, raw=False)
    for unpacked in unpacker:
        print(unpacked)
 
@@ -179,7 +179,7 @@ It is also possible to pack/unpack custom data types. Here is an example for
 
 
     packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True)
-    this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw_as_bytes=False)
+    this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False)
 
 ``Unpacker``'s ``object_hook`` callback receives a dict; the
 ``object_pairs_hook`` callback may instead be used to receive a list of
@@ -209,7 +209,7 @@ It is also possible to pack/unpack custom data types using the **ext** type.
     ...
     >>> data = array.array('d', [1.2, 3.4])
     >>> packed = msgpack.packb(data, default=default, use_bin_type=True)
-    >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw_as_bytes=False)
+    >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False)
     >>> data == unpacked
     True
 
@@ -257,7 +257,7 @@ For backward compatibility reasons, msgpack-python will still default all
 strings to byte strings, unless you specify the ``use_bin_type=True`` option in
 the packer. If you do so, it will use a non-standard type called **bin** to
 serialize byte arrays, and **raw** becomes to mean **str**. If you want to
-distinguish **bin** and **raw** in the unpacker, specify ``raw_as_bytes=False``.
+distinguish **bin** and **raw** in the unpacker, specify ``raw=False``.
 
 Note that Python 2 defaults to byte-arrays over Unicode strings:
 
@@ -267,7 +267,7 @@ Note that Python 2 defaults to byte-arrays over Unicode strings:
     >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
     ['spam', 'eggs']
     >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
-                        raw_as_bytes=False)
+                        raw=False)
     ['spam', u'eggs']
 
 This is the same code in Python 3 (same behaviour, but Python 3 has a
@@ -279,7 +279,7 @@ different default):
     >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
     [b'spam', b'eggs']
     >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
-                        raw_as_bytes=False)
+                        raw=False)
     [b'spam', 'eggs']
 
 
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index b796d04..806be4f 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -43,7 +43,7 @@ from msgpack import ExtType
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
         bint use_list
-        bint raw_as_bytes
+        bint raw
         bint has_pairs_hook # call object_hook with k-v pairs
         PyObject* object_hook
         PyObject* list_hook
@@ -74,14 +74,14 @@ cdef extern from "unpack.h":
 cdef inline init_ctx(unpack_context *ctx,
                      object object_hook, object object_pairs_hook,
                      object list_hook, object ext_hook,
-                     bint use_list, bint raw_as_bytes,
+                     bint use_list, bint raw,
                      char* encoding, char* unicode_errors,
                      Py_ssize_t max_str_len, Py_ssize_t max_bin_len,
                      Py_ssize_t max_array_len, Py_ssize_t max_map_len,
                      Py_ssize_t max_ext_len):
     unpack_init(ctx)
     ctx.user.use_list = use_list
-    ctx.user.raw_as_bytes = raw_as_bytes
+    ctx.user.raw = raw
     ctx.user.object_hook = ctx.user.list_hook = NULL
     ctx.user.max_str_len = max_str_len
     ctx.user.max_bin_len = max_bin_len
@@ -158,7 +158,7 @@ cdef inline int get_data_from_buffer(object obj,
         return 1
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
-            bint use_list=True, bint raw_as_bytes=True,
+            bint use_list=True, bint raw=True,
             encoding=None, unicode_errors="strict",
             object_pairs_hook=None, ext_hook=ExtType,
             Py_ssize_t max_str_len=2147483647, # 2**32-1
@@ -185,7 +185,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef int new_protocol = 0
 
     if encoding is not None:
-        PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1)
+        PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
         if isinstance(encoding, unicode):
             encoding = encoding.encode('ascii')
         elif not isinstance(encoding, bytes):
@@ -203,7 +203,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol)
     try:
         init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
-                 use_list, raw_as_bytes, cenc, cerr,
+                 use_list, raw, cenc, cerr,
                  max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
         ret = unpack_construct(&ctx, buf, buf_len, &off)
     finally:
@@ -261,7 +261,7 @@ cdef class Unpacker(object):
         If true, unpack msgpack array to Python list.
         Otherwise, unpack to Python tuple. (default: True)
 
-    :param bool raw_as_bytes:
+    :param bool raw:
         If true, unpack msgpack raw to Python bytes (default).
         Otherwise, unpack to Python str (or unicode on Python 2) by decoding
         with UTF-8 encoding (recommended).
@@ -299,7 +299,7 @@ cdef class Unpacker(object):
         Limits max length of map. (default: 2**31-1)
 
     :param str encoding:
-        Deprecated, use raw_as_bytes instead.
+        Deprecated, use raw instead.
         Encoding used for decoding msgpack raw.
         If it is None (default), msgpack raw is deserialized to Python bytes.
 
@@ -310,13 +310,13 @@ cdef class Unpacker(object):
 
     Example of streaming deserialize from file-like object::
 
-        unpacker = Unpacker(file_like, raw_as_bytes=False)
+        unpacker = Unpacker(file_like, raw=False)
         for o in unpacker:
             process(o)
 
     Example of streaming deserialize from socket::
 
-        unpacker = Unpacker(raw_as_bytes=False)
+        unpacker = Unpacker(raw=False)
         while True:
             buf = sock.recv(1024**2)
             if not buf:
@@ -345,7 +345,7 @@ cdef class Unpacker(object):
         self.buf = NULL
 
     def __init__(self, file_like=None, Py_ssize_t read_size=0,
-                 bint use_list=True, bint raw_as_bytes=True,
+                 bint use_list=True, bint raw=True,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  encoding=None, unicode_errors='strict', int max_buffer_size=0,
                  object ext_hook=ExtType,
@@ -384,7 +384,7 @@ cdef class Unpacker(object):
         self.stream_offset = 0
 
         if encoding is not None:
-            PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw_as_bytes=False instead.", 1)
+            PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
             if isinstance(encoding, unicode):
                 self.encoding = encoding.encode('ascii')
             elif isinstance(encoding, bytes):
@@ -404,7 +404,7 @@ cdef class Unpacker(object):
             cerr = PyBytes_AsString(self.unicode_errors)
 
         init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
-                 ext_hook, use_list, raw_as_bytes, cenc, cerr,
+                 ext_hook, use_list, raw, cenc, cerr,
                  max_str_len, max_bin_len, max_array_len,
                  max_map_len, max_ext_len)
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 675ee8a..bf6c9a6 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -145,7 +145,7 @@ class Unpacker(object):
         If true, unpack msgpack array to Python list.
         Otherwise, unpack to Python tuple. (default: True)
 
-    :param bool raw_as_bytes:
+    :param bool raw:
         If true, unpack msgpack raw to Python bytes (default).
         Otherwise, unpack to Python str (or unicode on Python 2) by decoding
         with UTF-8 encoding (recommended).
@@ -193,13 +193,13 @@ class Unpacker(object):
 
     example of streaming deserialize from file-like object::
 
-        unpacker = Unpacker(file_like, raw_as_bytes=False)
+        unpacker = Unpacker(file_like, raw=False)
         for o in unpacker:
             process(o)
 
     example of streaming deserialize from socket::
 
-        unpacker = Unpacker(raw_as_bytes=False)
+        unpacker = Unpacker(raw=False)
         while True:
             buf = sock.recv(1024**2)
             if not buf:
@@ -209,7 +209,7 @@ class Unpacker(object):
                 process(o)
     """
 
-    def __init__(self, file_like=None, read_size=0, use_list=True, raw_as_bytes=True,
+    def __init__(self, file_like=None, read_size=0, use_list=True, raw=True,
                  object_hook=None, object_pairs_hook=None, list_hook=None,
                  encoding=None, unicode_errors=None, max_buffer_size=0,
                  ext_hook=ExtType,
@@ -221,7 +221,7 @@ class Unpacker(object):
 
         if encoding is not None:
             warnings.warn(
-                "encoding is deprecated, Use raw_as_bytes=False instead.",
+                "encoding is deprecated, Use raw=False instead.",
                 PendingDeprecationWarning)
 
         if unicode_errors is not None:
@@ -257,7 +257,7 @@ class Unpacker(object):
         if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")
         self._read_size = read_size or min(self._max_buffer_size, 16*1024)
-        self._raw_as_bytes = bool(raw_as_bytes)
+        self._raw = bool(raw)
         self._encoding = encoding
         self._unicode_errors = unicode_errors
         self._use_list = use_list
@@ -606,7 +606,7 @@ class Unpacker(object):
         if typ == TYPE_RAW:
             if self._encoding is not None:
                 obj = obj.decode(self._encoding, self._unicode_errors)
-            elif self._raw_as_bytes:
+            elif self._raw:
                 obj = bytes(obj)
             else:
                 obj = obj.decode('utf_8')
@@ -715,7 +715,7 @@ class Packer(object):
             encoding = 'utf_8'
         else:
             warnings.warn(
-                "encoding is deprecated, Use raw_as_bytes=False instead.",
+                "encoding is deprecated, Use raw=False instead.",
                 PendingDeprecationWarning)
 
         if unicode_errors is None:
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 8c2fc46..d7b5e00 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -21,7 +21,7 @@
 
 typedef struct unpack_user {
     bool use_list;
-    bool raw_as_bytes;
+    bool raw;
     bool has_pairs_hook;
     PyObject *object_hook;
     PyObject *list_hook;
@@ -229,7 +229,7 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
 
     if (u->encoding) {
         py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors);
-    } else if (u->raw_as_bytes) {
+    } else if (u->raw) {
         py = PyBytes_FromStringAndSize(p, l);
     } else {
         py = PyUnicode_DecodeUTF8(p, l, NULL);
diff --git a/test/test_limits.py b/test/test_limits.py
index 3febc30..74e48c1 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -39,11 +39,11 @@ def test_max_str_len():
     d = 'x' * 3
     packed = packb(d)
 
-    unpacker = Unpacker(max_str_len=3, raw_as_bytes=False)
+    unpacker = Unpacker(max_str_len=3, raw=False)
     unpacker.feed(packed)
     assert unpacker.unpack() == d
 
-    unpacker = Unpacker(max_str_len=2, raw_as_bytes=False)
+    unpacker = Unpacker(max_str_len=2, raw=False)
     with pytest.raises(UnpackValueError):
         unpacker.feed(packed)
         unpacker.unpack()
diff --git a/test/test_pack.py b/test/test_pack.py
index 29f5887..b447f9c 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -31,11 +31,11 @@ def testPack():
 def testPackUnicode():
     test_data = ["", "abcd", ["defgh"], "РуÑÑкий текÑÑ‚"]
     for td in test_data:
-        re = unpackb(packb(td), use_list=1, raw_as_bytes=False)
+        re = unpackb(packb(td), use_list=1, raw=False)
         assert re == td
         packer = Packer()
         data = packer.pack(td)
-        re = Unpacker(BytesIO(data), raw_as_bytes=False, use_list=1).unpack()
+        re = Unpacker(BytesIO(data), raw=False, use_list=1).unpack()
         assert re == td
 
 def testPackUTF32():  # deprecated
@@ -72,14 +72,14 @@ def testIgnoreUnicodeErrors(): # deprecated
 
 def testStrictUnicodeUnpack():
     with raises(UnicodeDecodeError):
-        unpackb(packb(b'abc\xeddef'), raw_as_bytes=False, use_list=1)
+        unpackb(packb(b'abc\xeddef'), raw=False, use_list=1)
 
 def testStrictUnicodePack():  # deprecated
     with raises(UnicodeEncodeError):
         packb("abc\xeddef", encoding='ascii', unicode_errors='strict')
 
 def testIgnoreErrorsPack():  # deprecated
-    re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw_as_bytes=False, use_list=1)
+    re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1)
     assert re == "abcdef"
 
 def testDecodeBinary():
diff --git a/test/test_stricttype.py b/test/test_stricttype.py
index 13239f1..87e7c1c 100644
--- a/test/test_stricttype.py
+++ b/test/test_stricttype.py
@@ -11,7 +11,7 @@ def test_namedtuple():
             return dict(o._asdict())
         raise TypeError('Unsupported type %s' % (type(o),))
     packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default)
-    unpacked = unpackb(packed, raw_as_bytes=False)
+    unpacked = unpackb(packed, raw=False)
     assert unpacked == {'foo': 1, 'bar': 42}
 
 
@@ -32,7 +32,7 @@ def test_tuple():
         return o
 
     data = packb(t, strict_types=True, use_bin_type=True, default=default)
-    expected = unpackb(data, raw_as_bytes=False, object_hook=convert)
+    expected = unpackb(data, raw=False, object_hook=convert)
 
     assert expected == t
 
@@ -53,10 +53,10 @@ def test_tuple_ext():
     def convert(code, payload):
         if code == MSGPACK_EXT_TYPE_TUPLE:
             # Unpack and convert to tuple
-            return tuple(unpackb(payload, raw_as_bytes=False, ext_hook=convert))
+            return tuple(unpackb(payload, raw=False, ext_hook=convert))
         raise ValueError('Unknown Ext code {}'.format(code))
 
     data = packb(t, strict_types=True, use_bin_type=True, default=default)
-    expected = unpackb(data, raw_as_bytes=False, ext_hook=convert)
+    expected = unpackb(data, raw=False, ext_hook=convert)
 
     assert expected == t
diff --git a/test/test_unpack.py b/test/test_unpack.py
index 143f999..00a1061 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -48,7 +48,7 @@ def test_unpacker_ext_hook():
 
         def __init__(self):
             super(MyUnpacker, self).__init__(
-                ext_hook=self._hook, raw_as_bytes=False)
+                ext_hook=self._hook, raw=False)
 
         def _hook(self, code, data):
             if code == 1:

From 52fb85a2c5776590599df3a5839117b88bc49980 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 2 Feb 2018 19:43:42 +0900
Subject: [PATCH 1407/1648] 0.5.2

---
 ChangeLog.rst       | 13 +++++++++++++
 msgpack/_version.py |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index ed0f92e..ffdc910 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,16 @@
+0.5.2
+======
+
+* Add ``raw`` option to Unpacker.  It is preferred way than ``encoding`` option.
+
+* Packer.pack() reset buffer on exception (#274)
+
+
+0.5.1
+======
+
+* Remove FutureWarning about use_bin_type option (#271)
+
 0.5.0
 ======
 
diff --git a/msgpack/_version.py b/msgpack/_version.py
index ecba3d8..4e3b29f 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 5, 1)
+version = (0, 5, 2)

From a0ba076c3527bb474cd8ec820ae0fd6c8293d4af Mon Sep 17 00:00:00 2001
From: Andrew Rabert <6550543+nvllsvm@users.noreply.github.com>
Date: Fri, 2 Feb 2018 20:34:42 -0500
Subject: [PATCH 1408/1648] Fix encoding and unicode_errors (#277)

Previously, unicode_errors was either set to NULL or to
the result of PyBytes_AsString. This restores that behavior while also
keeping the existing NULL default behavior.
Original defaults were restored to keep API compatibility until these
deprecated options are finally removed.
---
 msgpack/_packer.pyx | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 35e5a9d..c49e719 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -127,27 +127,26 @@ cdef class Packer(object):
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")
         self._default = default
-        if encoding is None:
-            if unicode_errors is None:
-                self.encoding = NULL
-                self.unicode_errors = NULL
-            else:
-                self.encoding = "utf_8"
-                self.unicode_errors = unicode_errors
+        if encoding is None and unicode_errors is None:
+            self.encoding = NULL
+            self.unicode_errors = NULL
         else:
-            if isinstance(encoding, unicode):
-                self._bencoding = encoding.encode('ascii')
+            if encoding is None:
+                self.encoding = 'utf-8'
             else:
-                self._bencoding = encoding
-            self.encoding = PyBytes_AsString(self._bencoding)
-            if isinstance(unicode_errors, unicode):
-                self._berrors = unicode_errors.encode('ascii')
+                if isinstance(encoding, unicode):
+                    self._bencoding = encoding.encode('ascii')
+                else:
+                    self._bencoding = encoding
+                self.encoding = PyBytes_AsString(self._bencoding)
+            if unicode_errors is None:
+                self.unicode_errors = 'strict'
             else:
-                self._berrors = unicode_errors
-            if self._berrors is not None:
+                if isinstance(unicode_errors, unicode):
+                    self._berrors = unicode_errors.encode('ascii')
+                else:
+                    self._berrors = unicode_errors
                 self.unicode_errors = PyBytes_AsString(self._berrors)
-            else:
-                self.unicode_errors = NULL
 
     def __dealloc__(self):
         PyMem_Free(self.pk.buf)

From 618b2cb027e326b315edf5b856ed3b2011c8dad9 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Sat, 3 Feb 2018 10:54:21 +0900
Subject: [PATCH 1409/1648] 0.5.3

---
 ChangeLog.rst       | 5 +++++
 msgpack/_version.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index ffdc910..4a5c599 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,8 @@
+0.5.3
+======
+
+* Fixed regression when passing ``unicode_errors`` to Packer but not ``encoding``. (#277)
+
 0.5.2
 ======
 
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 4e3b29f..217bfce 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 5, 2)
+version = (0, 5, 3)

From 9fdb83719d3544322f0687a9b1637276c1443eda Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 5 Feb 2018 02:19:48 +0900
Subject: [PATCH 1410/1648] Undeprecate unicode_errors (#278)

---
 msgpack/_packer.pyx   | 7 +++----
 msgpack/_unpacker.pyx | 9 +++------
 msgpack/unpack.h      | 2 +-
 3 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index c49e719..b987aee 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -89,10 +89,11 @@ cdef class Packer(object):
         This is useful when trying to implement accurate serialization
         for python types.
 
+    :param str unicode_errors:
+        Error handler for encoding unicode. (default: 'strict')
+
     :param str encoding:
         (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
-    :param str unicode_errors:
-        (deprecated) Error handler for encoding unicode. (default: 'strict')
     """
     cdef msgpack_packer pk
     cdef object _default
@@ -117,8 +118,6 @@ cdef class Packer(object):
                  bint strict_types=False):
         if encoding is not None:
             PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated.", 1)
-        if unicode_errors is not None:
-            PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated.", 1)
         self.use_float = use_single_float
         self.strict_types = strict_types
         self.autoreset = autoreset
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 806be4f..25fdcd9 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -159,7 +159,7 @@ cdef inline int get_data_from_buffer(object obj,
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=True, bint raw=True,
-            encoding=None, unicode_errors="strict",
+            encoding=None, unicode_errors=None,
             object_pairs_hook=None, ext_hook=ExtType,
             Py_ssize_t max_str_len=2147483647, # 2**32-1
             Py_ssize_t max_bin_len=2147483647,
@@ -193,7 +193,6 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         cenc = PyBytes_AsString(encoding)
 
     if unicode_errors is not None:
-        PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated", 1)
         if isinstance(unicode_errors, unicode):
             unicode_errors = unicode_errors.encode('ascii')
         elif not isinstance(unicode_errors, bytes):
@@ -304,8 +303,7 @@ cdef class Unpacker(object):
         If it is None (default), msgpack raw is deserialized to Python bytes.
 
     :param str unicode_errors:
-        Deprecated. Used for decoding msgpack raw with *encoding*.
-        (default: `'strict'`)
+        Error handler used for decoding str type.  (default: `'strict'`)
 
 
     Example of streaming deserialize from file-like object::
@@ -347,7 +345,7 @@ cdef class Unpacker(object):
     def __init__(self, file_like=None, Py_ssize_t read_size=0,
                  bint use_list=True, bint raw=True,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
-                 encoding=None, unicode_errors='strict', int max_buffer_size=0,
+                 encoding=None, unicode_errors=None, int max_buffer_size=0,
                  object ext_hook=ExtType,
                  Py_ssize_t max_str_len=2147483647, # 2**32-1
                  Py_ssize_t max_bin_len=2147483647,
@@ -394,7 +392,6 @@ cdef class Unpacker(object):
             cenc = PyBytes_AsString(self.encoding)
 
         if unicode_errors is not None:
-            PyErr_WarnEx(PendingDeprecationWarning, "unicode_errors is deprecated", 1)
             if isinstance(unicode_errors, unicode):
                 self.unicode_errors = unicode_errors.encode('ascii')
             elif isinstance(unicode_errors, bytes):
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index d7b5e00..63e5543 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -232,7 +232,7 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
     } else if (u->raw) {
         py = PyBytes_FromStringAndSize(p, l);
     } else {
-        py = PyUnicode_DecodeUTF8(p, l, NULL);
+        py = PyUnicode_DecodeUTF8(p, l, u->unicode_errors);
     }
     if (!py)
         return -1;

From 351023946fa92fc78bdd5247f64d3a62fa233e95 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 5 Feb 2018 02:25:12 +0900
Subject: [PATCH 1411/1648] 0.5.4

---
 ChangeLog.rst       | 5 +++++
 msgpack/_version.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 4a5c599..67ee24c 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,8 @@
+0.5.4
+======
+
+* Undeprecate ``unicode_errors`` option. (#278)
+
 0.5.3
 ======
 
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 217bfce..e7435d9 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 5, 3)
+version = (0, 5, 4)

From 2644cbdcb7e1f2a8373de5bd30d71c907827caf6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 5 Feb 2018 11:44:17 +0900
Subject: [PATCH 1412/1648] Use cython's cast for converting encoding and
 errors (#279)

It is little faster on Python 3 because we can skip temporary bytes object
---
 msgpack/_packer.pyx   | 40 +++++++++++++++++---------------------
 msgpack/_unpacker.pyx | 45 ++++++++++++++-----------------------------
 2 files changed, 32 insertions(+), 53 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index b987aee..225f24a 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -1,7 +1,8 @@
 # coding: utf-8
-#cython: embedsignature=True
+#cython: embedsignature=True, c_string_encoding=ascii
 
 from cpython cimport *
+from cpython.version cimport PY_MAJOR_VERSION
 from cpython.exc cimport PyErr_WarnEx
 
 from msgpack.exceptions import PackValueError, PackOverflowError
@@ -99,8 +100,8 @@ cdef class Packer(object):
     cdef object _default
     cdef object _bencoding
     cdef object _berrors
-    cdef char *encoding
-    cdef char *unicode_errors
+    cdef const char *encoding
+    cdef const char *unicode_errors
     cdef bint strict_types
     cdef bool use_float
     cdef bint autoreset
@@ -126,26 +127,21 @@ cdef class Packer(object):
             if not PyCallable_Check(default):
                 raise TypeError("default must be a callable.")
         self._default = default
-        if encoding is None and unicode_errors is None:
-            self.encoding = NULL
-            self.unicode_errors = NULL
-        else:
-            if encoding is None:
+
+        self._bencoding = encoding
+        if encoding is None:
+            if PY_MAJOR_VERSION < 3:
                 self.encoding = 'utf-8'
             else:
-                if isinstance(encoding, unicode):
-                    self._bencoding = encoding.encode('ascii')
-                else:
-                    self._bencoding = encoding
-                self.encoding = PyBytes_AsString(self._bencoding)
-            if unicode_errors is None:
-                self.unicode_errors = 'strict'
-            else:
-                if isinstance(unicode_errors, unicode):
-                    self._berrors = unicode_errors.encode('ascii')
-                else:
-                    self._berrors = unicode_errors
-                self.unicode_errors = PyBytes_AsString(self._berrors)
+                self.encoding = NULL
+        else:
+            self.encoding = self._bencoding
+
+        self._berrors = unicode_errors
+        if unicode_errors is None:
+            self.unicode_errors = NULL
+        else:
+            self.unicode_errors = self._berrors
 
     def __dealloc__(self):
         PyMem_Free(self.pk.buf)
@@ -212,7 +208,7 @@ cdef class Packer(object):
                 if ret == 0:
                     ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o):
-                if self.encoding == NULL:
+                if self.encoding == NULL and self.unicode_errors == NULL:
                     ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT);
                     if ret == -2:
                         raise PackValueError("unicode string is too large")
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 25fdcd9..fb58490 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -1,6 +1,7 @@
 # coding: utf-8
-#cython: embedsignature=True
+#cython: embedsignature=True, c_string_encoding=ascii
 
+from cpython.version cimport PY_MAJOR_VERSION
 from cpython.bytes cimport (
     PyBytes_AsString,
     PyBytes_FromStringAndSize,
@@ -75,7 +76,7 @@ cdef inline init_ctx(unpack_context *ctx,
                      object object_hook, object object_pairs_hook,
                      object list_hook, object ext_hook,
                      bint use_list, bint raw,
-                     char* encoding, char* unicode_errors,
+                     const char* encoding, const char* unicode_errors,
                      Py_ssize_t max_str_len, Py_ssize_t max_bin_len,
                      Py_ssize_t max_array_len, Py_ssize_t max_map_len,
                      Py_ssize_t max_ext_len):
@@ -180,24 +181,16 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef Py_buffer view
     cdef char* buf = NULL
     cdef Py_ssize_t buf_len
-    cdef char* cenc = NULL
-    cdef char* cerr = NULL
+    cdef const char* cenc = NULL
+    cdef const char* cerr = NULL
     cdef int new_protocol = 0
 
     if encoding is not None:
         PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
-        if isinstance(encoding, unicode):
-            encoding = encoding.encode('ascii')
-        elif not isinstance(encoding, bytes):
-            raise TypeError("encoding should be bytes or unicode")
-        cenc = PyBytes_AsString(encoding)
+        cenc = encoding
 
     if unicode_errors is not None:
-        if isinstance(unicode_errors, unicode):
-            unicode_errors = unicode_errors.encode('ascii')
-        elif not isinstance(unicode_errors, bytes):
-            raise TypeError("unicode_errors should be bytes or unicode")
-        cerr = PyBytes_AsString(unicode_errors)
+        cerr = unicode_errors
 
     get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol)
     try:
@@ -219,7 +212,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
 
 def unpack(object stream, object object_hook=None, object list_hook=None,
-           bint use_list=1, encoding=None, unicode_errors="strict",
+           bint use_list=1, encoding=None, unicode_errors=None,
            object_pairs_hook=None, ext_hook=ExtType,
            Py_ssize_t max_str_len=2147483647, # 2**32-1
            Py_ssize_t max_bin_len=2147483647,
@@ -352,8 +345,8 @@ cdef class Unpacker(object):
                  Py_ssize_t max_array_len=2147483647,
                  Py_ssize_t max_map_len=2147483647,
                  Py_ssize_t max_ext_len=2147483647):
-        cdef char *cenc=NULL,
-        cdef char *cerr=NULL
+        cdef const char *cenc=NULL,
+        cdef const char *cerr=NULL
 
         self.object_hook = object_hook
         self.object_pairs_hook = object_pairs_hook
@@ -383,22 +376,12 @@ cdef class Unpacker(object):
 
         if encoding is not None:
             PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
-            if isinstance(encoding, unicode):
-                self.encoding = encoding.encode('ascii')
-            elif isinstance(encoding, bytes):
-                self.encoding = encoding
-            else:
-                raise TypeError("encoding should be bytes or unicode")
-            cenc = PyBytes_AsString(self.encoding)
+            self.encoding = encoding
+            cenc = encoding
 
         if unicode_errors is not None:
-            if isinstance(unicode_errors, unicode):
-                self.unicode_errors = unicode_errors.encode('ascii')
-            elif isinstance(unicode_errors, bytes):
-                self.unicode_errors = unicode_errors
-            else:
-                raise TypeError("unicode_errors should be bytes or unicode")
-            cerr = PyBytes_AsString(self.unicode_errors)
+            self.unicode_errors = unicode_errors
+            cerr = unicode_errors
 
         init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
                  ext_hook, use_list, raw, cenc, cerr,

From 4b72b6177321ef24b6c7af2354fb980db69e2aec Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 5 Feb 2018 15:08:19 +0900
Subject: [PATCH 1413/1648] Add Makefile target for updating docker image

---
 Makefile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Makefile b/Makefile
index 4030080..124f243 100644
--- a/Makefile
+++ b/Makefile
@@ -23,6 +23,11 @@ clean:
 	rm -rf msgpack/__pycache__
 	rm -rf test/__pycache__
 
+.PHONY: update-docker
+update-docker:
+	docker pull quay.io/pypa/manylinux1_i686
+	docker pull quay.io/pypa/manylinux1_x86_64
+
 .PHONY: linux-wheel
 linux-wheel:
 	docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686   bash docker/buildwheel.sh

From ae8d4694829d5b58d613c588c30e29dd29860c4f Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 16 Feb 2018 16:35:22 +0900
Subject: [PATCH 1414/1648] Fix memory leak in pure Python Unpacker.feed()
 (#284)

fixes #283
---
 msgpack/fallback.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index bf6c9a6..7e40686 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -289,6 +289,8 @@ class Unpacker(object):
         view = _get_data_from_buffer(next_bytes)
         if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size):
             raise BufferFull
+        del self._buffer[:self._buff_i]
+        self._buff_i = 0
         self._buffer += view
 
     def _consume(self):

From da902f9c1d996fb461f1efef6487ef40d32d365a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 22 Feb 2018 00:55:32 +0900
Subject: [PATCH 1415/1648] Move unpack() from each implementation to __init__.
 (#286)

Fixes #285
---
 msgpack/__init__.py   | 17 ++++++++++++++---
 msgpack/_unpacker.pyx | 26 --------------------------
 msgpack/fallback.py   | 10 ----------
 3 files changed, 14 insertions(+), 39 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 6c5ae53..70de97f 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -19,13 +19,13 @@ class ExtType(namedtuple('ExtType', 'code data')):
 
 import os
 if os.environ.get('MSGPACK_PUREPYTHON'):
-    from msgpack.fallback import Packer, unpack, unpackb, Unpacker
+    from msgpack.fallback import Packer, unpackb, Unpacker
 else:
     try:
         from msgpack._packer import Packer
-        from msgpack._unpacker import unpack, unpackb, Unpacker
+        from msgpack._unpacker import unpackb, Unpacker
     except ImportError:
-        from msgpack.fallback import Packer, unpack, unpackb, Unpacker
+        from msgpack.fallback import Packer, unpackb, Unpacker
 
 
 def pack(o, stream, **kwargs):
@@ -46,6 +46,17 @@ def packb(o, **kwargs):
     """
     return Packer(**kwargs).pack(o)
 
+
+def unpack(stream, **kwargs):
+    """
+    Unpack an object from `stream`.
+
+    Raises `ExtraData` when `packed` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    return unpackb(stream.read(), **kwargs)
+
+
 # alias for compatibility to simplejson/marshal/pickle.
 load = unpack
 loads = unpackb
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index fb58490..25a7401 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -211,32 +211,6 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     raise UnpackValueError("Unpack failed: error = %d" % (ret,))
 
 
-def unpack(object stream, object object_hook=None, object list_hook=None,
-           bint use_list=1, encoding=None, unicode_errors=None,
-           object_pairs_hook=None, ext_hook=ExtType,
-           Py_ssize_t max_str_len=2147483647, # 2**32-1
-           Py_ssize_t max_bin_len=2147483647,
-           Py_ssize_t max_array_len=2147483647,
-           Py_ssize_t max_map_len=2147483647,
-           Py_ssize_t max_ext_len=2147483647):
-    """
-    Unpack an object from `stream`.
-
-    Raises `ValueError` when `stream` has extra bytes.
-
-    See :class:`Unpacker` for options.
-    """
-    return unpackb(stream.read(), use_list=use_list,
-                   object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
-                   encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook,
-                   max_str_len=max_str_len,
-                   max_bin_len=max_bin_len,
-                   max_array_len=max_array_len,
-                   max_map_len=max_map_len,
-                   max_ext_len=max_ext_len,
-                   )
-
-
 cdef class Unpacker(object):
     """Streaming unpacker.
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 7e40686..3609fd8 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -100,16 +100,6 @@ def _get_data_from_buffer(obj):
     return view
 
 
-def unpack(stream, **kwargs):
-    """
-    Unpack an object from `stream`.
-
-    Raises `ExtraData` when `packed` contains extra bytes.
-    See :class:`Unpacker` for options.
-    """
-    data = stream.read()
-    return unpackb(data, **kwargs)
-
 
 def unpackb(packed, **kwargs):
     """

From 02c881c7cb402b37418a9bd9a3fa56daf673a71b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 22 Feb 2018 17:55:39 +0900
Subject: [PATCH 1416/1648] 0.5.5

---
 ChangeLog.rst       | 6 ++++++
 msgpack/_version.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 67ee24c..263d1f3 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,9 @@
+0.5.5
+======
+
+* Fix memory leak in pure Python Unpacker.feed() (#283)
+* Fix unpack() didn't support `raw` option (#285)
+
 0.5.4
 ======
 
diff --git a/msgpack/_version.py b/msgpack/_version.py
index e7435d9..4035588 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 5, 4)
+version = (0, 5, 5)

From 9bf38105f7dfd7e9885d8faee81c8bd188b4de4d Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 23 Feb 2018 11:32:26 +0900
Subject: [PATCH 1417/1648] Revert "0.5.5"

This reverts commit 02c881c7cb402b37418a9bd9a3fa56daf673a71b.

From 9455fccc5283abe59868c55ee3f4cedd5bf2d14b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 23 Feb 2018 11:32:26 +0900
Subject: [PATCH 1418/1648] Revert "Move unpack() from each implementation to
 __init__. (#286)"

This reverts commit da902f9c1d996fb461f1efef6487ef40d32d365a.
---
 msgpack/__init__.py   | 17 +++--------------
 msgpack/_unpacker.pyx | 26 ++++++++++++++++++++++++++
 msgpack/fallback.py   | 10 ++++++++++
 3 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 70de97f..6c5ae53 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -19,13 +19,13 @@ class ExtType(namedtuple('ExtType', 'code data')):
 
 import os
 if os.environ.get('MSGPACK_PUREPYTHON'):
-    from msgpack.fallback import Packer, unpackb, Unpacker
+    from msgpack.fallback import Packer, unpack, unpackb, Unpacker
 else:
     try:
         from msgpack._packer import Packer
-        from msgpack._unpacker import unpackb, Unpacker
+        from msgpack._unpacker import unpack, unpackb, Unpacker
     except ImportError:
-        from msgpack.fallback import Packer, unpackb, Unpacker
+        from msgpack.fallback import Packer, unpack, unpackb, Unpacker
 
 
 def pack(o, stream, **kwargs):
@@ -46,17 +46,6 @@ def packb(o, **kwargs):
     """
     return Packer(**kwargs).pack(o)
 
-
-def unpack(stream, **kwargs):
-    """
-    Unpack an object from `stream`.
-
-    Raises `ExtraData` when `packed` contains extra bytes.
-    See :class:`Unpacker` for options.
-    """
-    return unpackb(stream.read(), **kwargs)
-
-
 # alias for compatibility to simplejson/marshal/pickle.
 load = unpack
 loads = unpackb
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 25a7401..fb58490 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -211,6 +211,32 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     raise UnpackValueError("Unpack failed: error = %d" % (ret,))
 
 
+def unpack(object stream, object object_hook=None, object list_hook=None,
+           bint use_list=1, encoding=None, unicode_errors=None,
+           object_pairs_hook=None, ext_hook=ExtType,
+           Py_ssize_t max_str_len=2147483647, # 2**32-1
+           Py_ssize_t max_bin_len=2147483647,
+           Py_ssize_t max_array_len=2147483647,
+           Py_ssize_t max_map_len=2147483647,
+           Py_ssize_t max_ext_len=2147483647):
+    """
+    Unpack an object from `stream`.
+
+    Raises `ValueError` when `stream` has extra bytes.
+
+    See :class:`Unpacker` for options.
+    """
+    return unpackb(stream.read(), use_list=use_list,
+                   object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
+                   encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook,
+                   max_str_len=max_str_len,
+                   max_bin_len=max_bin_len,
+                   max_array_len=max_array_len,
+                   max_map_len=max_map_len,
+                   max_ext_len=max_ext_len,
+                   )
+
+
 cdef class Unpacker(object):
     """Streaming unpacker.
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 3609fd8..7e40686 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -100,6 +100,16 @@ def _get_data_from_buffer(obj):
     return view
 
 
+def unpack(stream, **kwargs):
+    """
+    Unpack an object from `stream`.
+
+    Raises `ExtraData` when `packed` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    data = stream.read()
+    return unpackb(data, **kwargs)
+
 
 def unpackb(packed, **kwargs):
     """

From fbaa1360be67672f855dd6337e4137f2edf7ade6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 23 Feb 2018 11:35:09 +0900
Subject: [PATCH 1419/1648] Fix #285 again

---
 msgpack/_unpacker.pyx | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index fb58490..3843e92 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -211,30 +211,15 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     raise UnpackValueError("Unpack failed: error = %d" % (ret,))
 
 
-def unpack(object stream, object object_hook=None, object list_hook=None,
-           bint use_list=1, encoding=None, unicode_errors=None,
-           object_pairs_hook=None, ext_hook=ExtType,
-           Py_ssize_t max_str_len=2147483647, # 2**32-1
-           Py_ssize_t max_bin_len=2147483647,
-           Py_ssize_t max_array_len=2147483647,
-           Py_ssize_t max_map_len=2147483647,
-           Py_ssize_t max_ext_len=2147483647):
+def unpack(object stream, **kwargs):
     """
     Unpack an object from `stream`.
 
     Raises `ValueError` when `stream` has extra bytes.
-
     See :class:`Unpacker` for options.
     """
-    return unpackb(stream.read(), use_list=use_list,
-                   object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
-                   encoding=encoding, unicode_errors=unicode_errors, ext_hook=ext_hook,
-                   max_str_len=max_str_len,
-                   max_bin_len=max_bin_len,
-                   max_array_len=max_array_len,
-                   max_map_len=max_map_len,
-                   max_ext_len=max_ext_len,
-                   )
+    data = stream.read()
+    return unpackb(data, **kwargs)
 
 
 cdef class Unpacker(object):

From f38c1a3674b7623080cb774b56ede21383bde04a Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 23 Feb 2018 11:52:48 +0900
Subject: [PATCH 1420/1648] Fix Unpacker.feed() drops unused data in buffer.
 (#289)

Fixes #287
---
 msgpack/fallback.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 7e40686..e9108d2 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -289,8 +289,13 @@ class Unpacker(object):
         view = _get_data_from_buffer(next_bytes)
         if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size):
             raise BufferFull
-        del self._buffer[:self._buff_i]
-        self._buff_i = 0
+
+        # Strip buffer before checkpoint before reading file.
+        if self._buf_checkpoint > 0:
+            del self._buffer[:self._buf_checkpoint]
+            self._buff_i -= self._buf_checkpoint
+            self._buf_checkpoint = 0
+
         self._buffer += view
 
     def _consume(self):

From ae3a6ba0b04487d7108360d1d504332aed079556 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 23 Feb 2018 15:41:21 +0900
Subject: [PATCH 1421/1648] Deprecate implementation module's unpack() (#290)

---
 msgpack/__init__.py   | 18 +++++++++++++++---
 msgpack/_unpacker.pyx |  9 +++------
 msgpack/fallback.py   | 21 +++++----------------
 3 files changed, 23 insertions(+), 25 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 6c5ae53..3955a41 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -19,13 +19,13 @@ class ExtType(namedtuple('ExtType', 'code data')):
 
 import os
 if os.environ.get('MSGPACK_PUREPYTHON'):
-    from msgpack.fallback import Packer, unpack, unpackb, Unpacker
+    from msgpack.fallback import Packer, unpackb, Unpacker
 else:
     try:
         from msgpack._packer import Packer
-        from msgpack._unpacker import unpack, unpackb, Unpacker
+        from msgpack._unpacker import unpackb, Unpacker
     except ImportError:
-        from msgpack.fallback import Packer, unpack, unpackb, Unpacker
+        from msgpack.fallback import Packer, unpackb, Unpacker
 
 
 def pack(o, stream, **kwargs):
@@ -46,6 +46,18 @@ def packb(o, **kwargs):
     """
     return Packer(**kwargs).pack(o)
 
+
+def unpack(stream, **kwargs):
+    """
+    Unpack an object from `stream`.
+
+    Raises `ExtraData` when `stream` contains extra bytes.
+    See :class:`Unpacker` for options.
+    """
+    data = stream.read()
+    return unpackb(data, **kwargs)
+
+
 # alias for compatibility to simplejson/marshal/pickle.
 load = unpack
 loads = unpackb
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 3843e92..d7fa5bc 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -212,12 +212,9 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
 
 def unpack(object stream, **kwargs):
-    """
-    Unpack an object from `stream`.
-
-    Raises `ValueError` when `stream` has extra bytes.
-    See :class:`Unpacker` for options.
-    """
+    PyErr_WarnEx(
+        PendingDeprecationWarning,
+        "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1)
     data = stream.read()
     return unpackb(data, **kwargs)
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index e9108d2..c0e5fd6 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -101,12 +101,9 @@ def _get_data_from_buffer(obj):
 
 
 def unpack(stream, **kwargs):
-    """
-    Unpack an object from `stream`.
-
-    Raises `ExtraData` when `packed` contains extra bytes.
-    See :class:`Unpacker` for options.
-    """
+    warnings.warn(
+        "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
+        PendingDeprecationWarning)
     data = stream.read()
     return unpackb(data, **kwargs)
 
@@ -224,11 +221,7 @@ class Unpacker(object):
                 "encoding is deprecated, Use raw=False instead.",
                 PendingDeprecationWarning)
 
-        if unicode_errors is not None:
-            warnings.warn(
-                "unicode_errors is deprecated.",
-                PendingDeprecationWarning)
-        else:
+        if unicode_errors is None:
             unicode_errors = 'strict'
 
         if file_like is None:
@@ -713,7 +706,7 @@ class Packer(object):
         (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
 
     :param str unicode_errors:
-        (deprecated) Error handler for encoding unicode. (default: 'strict')
+        Error handler for encoding unicode. (default: 'strict')
     """
     def __init__(self, default=None, encoding=None, unicode_errors=None,
                  use_single_float=False, autoreset=True, use_bin_type=False,
@@ -727,10 +720,6 @@ class Packer(object):
 
         if unicode_errors is None:
             unicode_errors = 'strict'
-        else:
-            warnings.warn(
-                "unicode_errors is deprecated.",
-                PendingDeprecationWarning)
 
         self._strict_types = strict_types
         self._use_float = use_single_float

From d4675bee6c83d42cecda9a84e6716021eb2ad679 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 23 Feb 2018 15:45:34 +0900
Subject: [PATCH 1422/1648] 0.5.6

---
 ChangeLog.rst       | 7 +++++++
 msgpack/_version.py | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 263d1f3..b6158c3 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,10 @@
+0.5.6
+======
+
+* Fix fallback.Unpacker.feed() dropped unused data from buffer (#287)
+* Resurrect fallback.unpack() and _unpacker.unpack().
+  They were removed at 0.5.5 but it breaks backward compatibility. (#288, #290)
+
 0.5.5
 ======
 
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 4035588..d28f0de 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 5, 5)
+version = (0, 5, 6)

From 984116bd1805c52fb4e7c07bfd9635ed902850c3 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Apr 2018 23:41:01 +0900
Subject: [PATCH 1423/1648] Update setup()

---
 setup.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6108587..c252d81 100755
--- a/setup.py
+++ b/setup.py
@@ -120,7 +120,13 @@ setup(name=name,
       packages=['msgpack'],
       description=desc,
       long_description=long_desc,
-      url='http://msgpack.org/',
+      long_description_content_type="text/x-rst",
+      url='https://msgpack.org/',
+      project_urls = {
+          'Documentation': 'https://msgpack-python.readthedocs.io/',
+          'Source': 'https://github.com/msgpack/msgpack-python',
+          'Tracker': 'https://github.com/msgpack/msgpack-python/issues',
+      },
       license='Apache 2.0',
       classifiers=[
           'Programming Language :: Python :: 2',

From b10cf78f54a5daab866b19c32e45e207d838f52b Mon Sep 17 00:00:00 2001
From: Alex Gaynor 
Date: Sun, 15 Apr 2018 23:18:35 -0400
Subject: [PATCH 1424/1648] Fix TypeError in fallback.unpack() on H", self._buffer, self._buff_i)[0]
+            n = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0]
             self._buff_i += 2
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
@@ -396,7 +402,7 @@ class Unpacker(object):
         elif b == 0xc6:
             typ = TYPE_BIN
             self._reserve(4)
-            n = struct.unpack_from(">I", self._buffer, self._buff_i)[0]
+            n = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0]
             self._buff_i += 4
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
@@ -404,7 +410,7 @@ class Unpacker(object):
         elif b == 0xc7:  # ext 8
             typ = TYPE_EXT
             self._reserve(2)
-            L, n = struct.unpack_from('Bb', self._buffer, self._buff_i)
+            L, n = struct.unpack_from('Bb', self._buffer_view, self._buff_i)
             self._buff_i += 2
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
@@ -412,7 +418,7 @@ class Unpacker(object):
         elif b == 0xc8:  # ext 16
             typ = TYPE_EXT
             self._reserve(3)
-            L, n = struct.unpack_from('>Hb', self._buffer, self._buff_i)
+            L, n = struct.unpack_from('>Hb', self._buffer_view, self._buff_i)
             self._buff_i += 3
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
@@ -420,18 +426,18 @@ class Unpacker(object):
         elif b == 0xc9:  # ext 32
             typ = TYPE_EXT
             self._reserve(5)
-            L, n = struct.unpack_from('>Ib', self._buffer, self._buff_i)
+            L, n = struct.unpack_from('>Ib', self._buffer_view, self._buff_i)
             self._buff_i += 5
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
         elif b == 0xca:
             self._reserve(4)
-            obj = struct.unpack_from(">f", self._buffer, self._buff_i)[0]
+            obj = struct.unpack_from(">f", self._buffer_view, self._buff_i)[0]
             self._buff_i += 4
         elif b == 0xcb:
             self._reserve(8)
-            obj = struct.unpack_from(">d", self._buffer, self._buff_i)[0]
+            obj = struct.unpack_from(">d", self._buffer_view, self._buff_i)[0]
             self._buff_i += 8
         elif b == 0xcc:
             self._reserve(1)
@@ -439,66 +445,66 @@ class Unpacker(object):
             self._buff_i += 1
         elif b == 0xcd:
             self._reserve(2)
-            obj = struct.unpack_from(">H", self._buffer, self._buff_i)[0]
+            obj = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0]
             self._buff_i += 2
         elif b == 0xce:
             self._reserve(4)
-            obj = struct.unpack_from(">I", self._buffer, self._buff_i)[0]
+            obj = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0]
             self._buff_i += 4
         elif b == 0xcf:
             self._reserve(8)
-            obj = struct.unpack_from(">Q", self._buffer, self._buff_i)[0]
+            obj = struct.unpack_from(">Q", self._buffer_view, self._buff_i)[0]
             self._buff_i += 8
         elif b == 0xd0:
             self._reserve(1)
-            obj = struct.unpack_from("b", self._buffer, self._buff_i)[0]
+            obj = struct.unpack_from("b", self._buffer_view, self._buff_i)[0]
             self._buff_i += 1
         elif b == 0xd1:
             self._reserve(2)
-            obj = struct.unpack_from(">h", self._buffer, self._buff_i)[0]
+            obj = struct.unpack_from(">h", self._buffer_view, self._buff_i)[0]
             self._buff_i += 2
         elif b == 0xd2:
             self._reserve(4)
-            obj = struct.unpack_from(">i", self._buffer, self._buff_i)[0]
+            obj = struct.unpack_from(">i", self._buffer_view, self._buff_i)[0]
             self._buff_i += 4
         elif b == 0xd3:
             self._reserve(8)
-            obj = struct.unpack_from(">q", self._buffer, self._buff_i)[0]
+            obj = struct.unpack_from(">q", self._buffer_view, self._buff_i)[0]
             self._buff_i += 8
         elif b == 0xd4:  # fixext 1
             typ = TYPE_EXT
             if self._max_ext_len < 1:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
             self._reserve(2)
-            n, obj = struct.unpack_from("b1s", self._buffer, self._buff_i)
+            n, obj = struct.unpack_from("b1s", self._buffer_view, self._buff_i)
             self._buff_i += 2
         elif b == 0xd5:  # fixext 2
             typ = TYPE_EXT
             if self._max_ext_len < 2:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
             self._reserve(3)
-            n, obj = struct.unpack_from("b2s", self._buffer, self._buff_i)
+            n, obj = struct.unpack_from("b2s", self._buffer_view, self._buff_i)
             self._buff_i += 3
         elif b == 0xd6:  # fixext 4
             typ = TYPE_EXT
             if self._max_ext_len < 4:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
             self._reserve(5)
-            n, obj = struct.unpack_from("b4s", self._buffer, self._buff_i)
+            n, obj = struct.unpack_from("b4s", self._buffer_view, self._buff_i)
             self._buff_i += 5
         elif b == 0xd7:  # fixext 8
             typ = TYPE_EXT
             if self._max_ext_len < 8:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
             self._reserve(9)
-            n, obj = struct.unpack_from("b8s", self._buffer, self._buff_i)
+            n, obj = struct.unpack_from("b8s", self._buffer_view, self._buff_i)
             self._buff_i += 9
         elif b == 0xd8:  # fixext 16
             typ = TYPE_EXT
             if self._max_ext_len < 16:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
             self._reserve(17)
-            n, obj = struct.unpack_from("b16s", self._buffer, self._buff_i)
+            n, obj = struct.unpack_from("b16s", self._buffer_view, self._buff_i)
             self._buff_i += 17
         elif b == 0xd9:
             typ = TYPE_RAW
@@ -511,7 +517,7 @@ class Unpacker(object):
         elif b == 0xda:
             typ = TYPE_RAW
             self._reserve(2)
-            n, = struct.unpack_from(">H", self._buffer, self._buff_i)
+            n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
             self._buff_i += 2
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
@@ -519,7 +525,7 @@ class Unpacker(object):
         elif b == 0xdb:
             typ = TYPE_RAW
             self._reserve(4)
-            n, = struct.unpack_from(">I", self._buffer, self._buff_i)
+            n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
             self._buff_i += 4
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
@@ -527,27 +533,27 @@ class Unpacker(object):
         elif b == 0xdc:
             typ = TYPE_ARRAY
             self._reserve(2)
-            n, = struct.unpack_from(">H", self._buffer, self._buff_i)
+            n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
             self._buff_i += 2
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
         elif b == 0xdd:
             typ = TYPE_ARRAY
             self._reserve(4)
-            n, = struct.unpack_from(">I", self._buffer, self._buff_i)
+            n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
             self._buff_i += 4
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
         elif b == 0xde:
             self._reserve(2)
-            n, = struct.unpack_from(">H", self._buffer, self._buff_i)
+            n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
             self._buff_i += 2
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         elif b == 0xdf:
             self._reserve(4)
-            n, = struct.unpack_from(">I", self._buffer, self._buff_i)
+            n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
             self._buff_i += 4
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)

From 5f684aed82d0d08079b9aa74e1d41cc2a367833d Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 27 Jun 2018 01:27:31 +0900
Subject: [PATCH 1425/1648] fallback: Fix error on Jython

Fixes #303
---
 msgpack/fallback.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index c8c8c78..20ad4c9 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -100,6 +100,13 @@ def _get_data_from_buffer(obj):
     return view
 
 
+# Jython's memoryview support is incomplete
+# See https://github.com/msgpack/msgpack-python/issues/303
+_is_jython = sys.platform.startswith('java')
+if _is_jython:
+    _get_data_from_buffer = bytes
+
+
 def unpack(stream, **kwargs):
     warnings.warn(
         "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",

From d1060de29309717355b78fe9092f7995897b4f0c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 13 Jul 2018 19:54:44 +0900
Subject: [PATCH 1426/1648] travis: Run test on Python 3.4 (#307)

Python 3.4 is not supported officially.
But keep running test for a while, to know when msgpack-python
stop working on Python 3.4 actually.
---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 7aac664..823c8b1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,7 @@ cache: pip
 
 python:
   - "2.7"
+  - "3.4"
   - "3.5"
   - "3.6"
   - "3.7-dev"

From 70b5f21b34b28cd50125ee337b5a399001759b73 Mon Sep 17 00:00:00 2001
From: Raymond E Ferguson 
Date: Tue, 2 Oct 2018 11:20:06 +0000
Subject: [PATCH 1427/1648] Alternate fixes for jython and legacy CPython
 (#310)

Python 3.4 is not supported officially.
But keep running test for a while, to know when msgpack-python
stop working on Python 3.4 actually.

The current patches did not work under jython-2.7.1 where implicit
casting of buffer or memoryview doesn't work. It may also be the
jython is a little pickier about string casting non string bytes
due to the underlying strong typing of java.

See issues #303 & #304.
---
 msgpack/fallback.py | 74 +++++++++++++++++++++------------------------
 1 file changed, 35 insertions(+), 39 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 20ad4c9..0b60ba2 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -100,13 +100,6 @@ def _get_data_from_buffer(obj):
     return view
 
 
-# Jython's memoryview support is incomplete
-# See https://github.com/msgpack/msgpack-python/issues/303
-_is_jython = sys.platform.startswith('java')
-if _is_jython:
-    _get_data_from_buffer = bytes
-
-
 def unpack(stream, **kwargs):
     warnings.warn(
         "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
@@ -133,6 +126,14 @@ def unpackb(packed, **kwargs):
     return ret
 
 
+if sys.version_info < (2, 7, 6):
+    def _unpack_from(f, b, o=0):
+        """Explicit typcast for legacy struct.unpack_from"""
+        return struct.unpack_from(f, bytes(b), o)
+else:
+    _unpack_from = struct.unpack_from
+
+
 class Unpacker(object):
     """Streaming unpacker.
 
@@ -241,12 +242,6 @@ class Unpacker(object):
 
         #: array of bytes fed.
         self._buffer = bytearray()
-        # Some very old pythons don't support `struct.unpack_from()` with a
-        # `bytearray`. So we wrap it in a `buffer()` there.
-        if sys.version_info < (2, 7, 6):
-            self._buffer_view = buffer(self._buffer)
-        else:
-            self._buffer_view = self._buffer
         #: Which position we currently reads
         self._buff_i = 0
 
@@ -302,7 +297,8 @@ class Unpacker(object):
             self._buff_i -= self._buf_checkpoint
             self._buf_checkpoint = 0
 
-        self._buffer += view
+        # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython
+        self._buffer.extend(view)
 
     def _consume(self):
         """ Gets rid of the used parts of the buffer. """
@@ -401,7 +397,7 @@ class Unpacker(object):
         elif b == 0xc5:
             typ = TYPE_BIN
             self._reserve(2)
-            n = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0]
+            n = _unpack_from(">H", self._buffer, self._buff_i)[0]
             self._buff_i += 2
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
@@ -409,7 +405,7 @@ class Unpacker(object):
         elif b == 0xc6:
             typ = TYPE_BIN
             self._reserve(4)
-            n = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0]
+            n = _unpack_from(">I", self._buffer, self._buff_i)[0]
             self._buff_i += 4
             if n > self._max_bin_len:
                 raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
@@ -417,7 +413,7 @@ class Unpacker(object):
         elif b == 0xc7:  # ext 8
             typ = TYPE_EXT
             self._reserve(2)
-            L, n = struct.unpack_from('Bb', self._buffer_view, self._buff_i)
+            L, n = _unpack_from('Bb', self._buffer, self._buff_i)
             self._buff_i += 2
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
@@ -425,7 +421,7 @@ class Unpacker(object):
         elif b == 0xc8:  # ext 16
             typ = TYPE_EXT
             self._reserve(3)
-            L, n = struct.unpack_from('>Hb', self._buffer_view, self._buff_i)
+            L, n = _unpack_from('>Hb', self._buffer, self._buff_i)
             self._buff_i += 3
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
@@ -433,18 +429,18 @@ class Unpacker(object):
         elif b == 0xc9:  # ext 32
             typ = TYPE_EXT
             self._reserve(5)
-            L, n = struct.unpack_from('>Ib', self._buffer_view, self._buff_i)
+            L, n = _unpack_from('>Ib', self._buffer, self._buff_i)
             self._buff_i += 5
             if L > self._max_ext_len:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
         elif b == 0xca:
             self._reserve(4)
-            obj = struct.unpack_from(">f", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">f", self._buffer, self._buff_i)[0]
             self._buff_i += 4
         elif b == 0xcb:
             self._reserve(8)
-            obj = struct.unpack_from(">d", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">d", self._buffer, self._buff_i)[0]
             self._buff_i += 8
         elif b == 0xcc:
             self._reserve(1)
@@ -452,66 +448,66 @@ class Unpacker(object):
             self._buff_i += 1
         elif b == 0xcd:
             self._reserve(2)
-            obj = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">H", self._buffer, self._buff_i)[0]
             self._buff_i += 2
         elif b == 0xce:
             self._reserve(4)
-            obj = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">I", self._buffer, self._buff_i)[0]
             self._buff_i += 4
         elif b == 0xcf:
             self._reserve(8)
-            obj = struct.unpack_from(">Q", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">Q", self._buffer, self._buff_i)[0]
             self._buff_i += 8
         elif b == 0xd0:
             self._reserve(1)
-            obj = struct.unpack_from("b", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from("b", self._buffer, self._buff_i)[0]
             self._buff_i += 1
         elif b == 0xd1:
             self._reserve(2)
-            obj = struct.unpack_from(">h", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">h", self._buffer, self._buff_i)[0]
             self._buff_i += 2
         elif b == 0xd2:
             self._reserve(4)
-            obj = struct.unpack_from(">i", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">i", self._buffer, self._buff_i)[0]
             self._buff_i += 4
         elif b == 0xd3:
             self._reserve(8)
-            obj = struct.unpack_from(">q", self._buffer_view, self._buff_i)[0]
+            obj = _unpack_from(">q", self._buffer, self._buff_i)[0]
             self._buff_i += 8
         elif b == 0xd4:  # fixext 1
             typ = TYPE_EXT
             if self._max_ext_len < 1:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
             self._reserve(2)
-            n, obj = struct.unpack_from("b1s", self._buffer_view, self._buff_i)
+            n, obj = _unpack_from("b1s", self._buffer, self._buff_i)
             self._buff_i += 2
         elif b == 0xd5:  # fixext 2
             typ = TYPE_EXT
             if self._max_ext_len < 2:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
             self._reserve(3)
-            n, obj = struct.unpack_from("b2s", self._buffer_view, self._buff_i)
+            n, obj = _unpack_from("b2s", self._buffer, self._buff_i)
             self._buff_i += 3
         elif b == 0xd6:  # fixext 4
             typ = TYPE_EXT
             if self._max_ext_len < 4:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
             self._reserve(5)
-            n, obj = struct.unpack_from("b4s", self._buffer_view, self._buff_i)
+            n, obj = _unpack_from("b4s", self._buffer, self._buff_i)
             self._buff_i += 5
         elif b == 0xd7:  # fixext 8
             typ = TYPE_EXT
             if self._max_ext_len < 8:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
             self._reserve(9)
-            n, obj = struct.unpack_from("b8s", self._buffer_view, self._buff_i)
+            n, obj = _unpack_from("b8s", self._buffer, self._buff_i)
             self._buff_i += 9
         elif b == 0xd8:  # fixext 16
             typ = TYPE_EXT
             if self._max_ext_len < 16:
                 raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
             self._reserve(17)
-            n, obj = struct.unpack_from("b16s", self._buffer_view, self._buff_i)
+            n, obj = _unpack_from("b16s", self._buffer, self._buff_i)
             self._buff_i += 17
         elif b == 0xd9:
             typ = TYPE_RAW
@@ -524,7 +520,7 @@ class Unpacker(object):
         elif b == 0xda:
             typ = TYPE_RAW
             self._reserve(2)
-            n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
@@ -532,7 +528,7 @@ class Unpacker(object):
         elif b == 0xdb:
             typ = TYPE_RAW
             self._reserve(4)
-            n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_str_len:
                 raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
@@ -540,27 +536,27 @@ class Unpacker(object):
         elif b == 0xdc:
             typ = TYPE_ARRAY
             self._reserve(2)
-            n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
         elif b == 0xdd:
             typ = TYPE_ARRAY
             self._reserve(4)
-            n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_array_len:
                 raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
         elif b == 0xde:
             self._reserve(2)
-            n, = struct.unpack_from(">H", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         elif b == 0xdf:
             self._reserve(4)
-            n, = struct.unpack_from(">I", self._buffer_view, self._buff_i)
+            n, = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_map_len:
                 raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)

From 205f7d39b26d7441f3cce86e93449e50bd71a6fa Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Oct 2018 21:06:20 +0900
Subject: [PATCH 1428/1648] Start 0.6 development

---
 ChangeLog.rst       | 6 ++++++
 msgpack/_version.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index b6158c3..18a9917 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,9 @@
+0.6.0
+======
+
+Release Date: TBD
+
+
 0.5.6
 ======
 
diff --git a/msgpack/_version.py b/msgpack/_version.py
index d28f0de..0952ec6 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 5, 6)
+version = (0, 6, 0, 'dev')

From b077a21f89881df8af56a05cc41d4bdebea19105 Mon Sep 17 00:00:00 2001
From: Marat Sharafutdinov 
Date: Sun, 4 Nov 2018 19:14:11 +0300
Subject: [PATCH 1429/1648] Fix stream unpacking example in README (#317)

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 8925a65..94a4bb2 100644
--- a/README.rst
+++ b/README.rst
@@ -142,7 +142,7 @@ stream (or from bytes provided through its ``feed`` method).
 
    buf = BytesIO()
    for i in range(100):
-      buf.write(msgpack.packb(range(i), use_bin_type=True))
+      buf.write(msgpack.packb(i, use_bin_type=True))
 
    buf.seek(0)
 

From 91ec9e1daf5cc915a47e2b356a7b1dd9662573a3 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 7 Nov 2018 23:02:35 +0900
Subject: [PATCH 1430/1648] Update travis.yml

---
 .travis.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 823c8b1..fd6125d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
-sudo: false
+dist: xenial
 language: python
 cache: pip
 
@@ -7,7 +7,8 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
-  - "3.7-dev"
+  - "3.7"
+  - "nightly"
 
 matrix:
   include:
@@ -24,7 +25,12 @@ matrix:
         - docker pull $DOCKER_IMAGE
       script:
         - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
-    - python: "pypy"
+    - python: "pypy2.7-5.10.0"
+      install:
+        - pip install -e .
+      script:
+        - py.test -v test
+    - python: "pypy3.5"
       install:
         - pip install -e .
       script:

From f6f95972492bcb83d8fe4c63be3b96d46e47bab7 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Oct 2018 20:55:51 +0900
Subject: [PATCH 1431/1648] Merge extension module

There were `_packer.so` and `_unpacker.so`.
But single module is simpler than double module.

Merge extension module into single `_msgpack.so`.
---
 Makefile             |  5 ++---
 msgpack/__init__.py  |  3 +--
 msgpack/_msgpack.pyx |  4 ++++
 setup.py             | 13 +++----------
 4 files changed, 10 insertions(+), 15 deletions(-)
 create mode 100644 msgpack/_msgpack.pyx

diff --git a/Makefile b/Makefile
index 124f243..ff9a482 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ all: cython
 
 .PHONY: cython
 cython:
-	cython --cplus msgpack/*.pyx
+	cython --cplus msgpack/_msgpack.pyx
 
 .PHONY: test
 test:
@@ -18,8 +18,7 @@ serve-doc: all
 .PHONY: clean
 clean:
 	rm -rf build
-	rm -f msgpack/_packer.cpp
-	rm -f msgpack/_unpacker.cpp
+	rm -f msgpack/_msgpack.cpp
 	rm -rf msgpack/__pycache__
 	rm -rf test/__pycache__
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 3955a41..7c5d4c0 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -22,8 +22,7 @@ if os.environ.get('MSGPACK_PUREPYTHON'):
     from msgpack.fallback import Packer, unpackb, Unpacker
 else:
     try:
-        from msgpack._packer import Packer
-        from msgpack._unpacker import unpackb, Unpacker
+        from msgpack._msgpack import Packer, unpackb, Unpacker
     except ImportError:
         from msgpack.fallback import Packer, unpackb, Unpacker
 
diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
new file mode 100644
index 0000000..4381394
--- /dev/null
+++ b/msgpack/_msgpack.pyx
@@ -0,0 +1,4 @@
+# coding: utf-8
+#cython: embedsignature=True, c_string_encoding=ascii
+include "_packer.pyx"
+include "_unpacker.pyx"
diff --git a/setup.py b/setup.py
index c252d81..8b8f7bd 100755
--- a/setup.py
+++ b/setup.py
@@ -68,8 +68,7 @@ if len(version) > 3 and version[3] != 'final':
 if have_cython:
     class Sdist(sdist):
         def __init__(self, *args, **kwargs):
-            for src in glob('msgpack/*.pyx'):
-                cythonize(src)
+            cythonize('msgpack/_msgpack.pyx')
             sdist.__init__(self, *args, **kwargs)
 else:
     Sdist = sdist
@@ -85,14 +84,8 @@ else:
 
 ext_modules = []
 if not hasattr(sys, 'pypy_version_info'):
-    ext_modules.append(Extension('msgpack._packer',
-                                 sources=['msgpack/_packer.cpp'],
-                                 libraries=libraries,
-                                 include_dirs=['.'],
-                                 define_macros=macros,
-                                 ))
-    ext_modules.append(Extension('msgpack._unpacker',
-                                 sources=['msgpack/_unpacker.cpp'],
+    ext_modules.append(Extension('msgpack._msgpack',
+                                 sources=['msgpack/_msgpack.cpp'],
                                  libraries=libraries,
                                  include_dirs=['.'],
                                  define_macros=macros,

From 6c8e539eec5bfbda752337c34fb701980f061859 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Oct 2018 21:09:45 +0900
Subject: [PATCH 1432/1648] Update travis config

---
 .travis.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index fd6125d..822ca9a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,7 +21,7 @@ matrix:
       install:
         - pip install -U pip
         - pip install cython
-        - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
+        - make cython
         - docker pull $DOCKER_IMAGE
       script:
         - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
@@ -34,19 +34,19 @@ matrix:
       install:
         - pip install -e .
       script:
-        - py.test -v test
+        - pytest -v test
 
 
 install:
   - pip install -U pip
   - pip install cython
-  - cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx
+  - make cython
   - pip install -e .
 
 script:
   - python -c 'import sys; print(hex(sys.maxsize))'
-  - python -c 'from msgpack import _packer, _unpacker'
-  - py.test -v test
-  - MSGPACK_PUREPYTHON=x py.test -v test
+  - python -c 'from msgpack import _msgpack'
+  - pytest -v test
+  - MSGPACK_PUREPYTHON=x pytest -v test
 
 # vim: sw=2 ts=2

From 9d11249d89c67fd87acaeb16184414b5ef0b5aa4 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 3 Oct 2018 21:44:06 +0900
Subject: [PATCH 1433/1648] Update docker/runtests

---
 docker/runtests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/runtests.sh b/docker/runtests.sh
index 113b630..f6c2c68 100755
--- a/docker/runtests.sh
+++ b/docker/runtests.sh
@@ -8,7 +8,7 @@ for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do
     $PYBIN/pip install pytest
     pushd test          # prevent importing msgpack package in current directory.
     $PYBIN/python -c 'import sys; print(hex(sys.maxsize))'
-    $PYBIN/python -c 'from msgpack import _packer, _unpacker'
+    $PYBIN/python -c 'from msgpack import _msgpack'  # Ensure extension is available
     $PYBIN/pytest -v .
     popd
 done

From ae90b26c3015e090a80a880b73895daa35f048fa Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 8 Nov 2018 22:21:05 +0900
Subject: [PATCH 1434/1648] Update ChangeLog

---
 ChangeLog.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 18a9917..b135745 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -3,6 +3,10 @@
 
 Release Date: TBD
 
+Extension modules are merged.  There is ``msgpack._msgpack`` instead of
+``msgpack._packer`` and ``msgpack._unpacker``. (#314)
+
+
 
 0.5.6
 ======

From 3b80233592674d18c8db7a62fa56504a5a285296 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Thu, 8 Nov 2018 22:21:44 +0900
Subject: [PATCH 1435/1648] unpacker: Make default size limit smaller (#319)

To avoid DoS attack, make default size limit smaller.

Fixes #295
---
 msgpack/_unpacker.pyx | 31 +++++++++++++++++--------------
 msgpack/fallback.py   | 20 +++++++++++---------
 2 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index d7fa5bc..cc9e7f0 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -162,11 +162,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=True, bint raw=True,
             encoding=None, unicode_errors=None,
             object_pairs_hook=None, ext_hook=ExtType,
-            Py_ssize_t max_str_len=2147483647, # 2**32-1
-            Py_ssize_t max_bin_len=2147483647,
-            Py_ssize_t max_array_len=2147483647,
-            Py_ssize_t max_map_len=2147483647,
-            Py_ssize_t max_ext_len=2147483647):
+            Py_ssize_t max_str_len=1024*1024,
+            Py_ssize_t max_bin_len=1024*1024,
+            Py_ssize_t max_array_len=128*1024,
+            Py_ssize_t max_map_len=32*1024,
+            Py_ssize_t max_ext_len=1024*1024):
     """
     Unpack packed_bytes to object. Returns an unpacked object.
 
@@ -261,16 +261,19 @@ cdef class Unpacker(object):
         You should set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
-        Limits max length of str. (default: 2**31-1)
+        Limits max length of str. (default: 1024*1024)
 
     :param int max_bin_len:
-        Limits max length of bin. (default: 2**31-1)
+        Limits max length of bin. (default: 1024*1024)
 
     :param int max_array_len:
-        Limits max length of array. (default: 2**31-1)
+        Limits max length of array. (default: 128*1024)
 
     :param int max_map_len:
-        Limits max length of map. (default: 2**31-1)
+        Limits max length of map. (default: 32*1024)
+
+    :param int max_ext_len:
+        Limits max length of map. (default: 1024*1024)
 
     :param str encoding:
         Deprecated, use raw instead.
@@ -322,11 +325,11 @@ cdef class Unpacker(object):
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  encoding=None, unicode_errors=None, int max_buffer_size=0,
                  object ext_hook=ExtType,
-                 Py_ssize_t max_str_len=2147483647, # 2**32-1
-                 Py_ssize_t max_bin_len=2147483647,
-                 Py_ssize_t max_array_len=2147483647,
-                 Py_ssize_t max_map_len=2147483647,
-                 Py_ssize_t max_ext_len=2147483647):
+                 Py_ssize_t max_str_len=1024*1024,
+                 Py_ssize_t max_bin_len=1024*1024,
+                 Py_ssize_t max_array_len=128*1024,
+                 Py_ssize_t max_map_len=32*1024,
+                 Py_ssize_t max_ext_len=1024*1024):
         cdef const char *cenc=NULL,
         cdef const char *cerr=NULL
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 0b60ba2..895864e 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -184,17 +184,19 @@ class Unpacker(object):
         You should set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
-        Limits max length of str. (default: 2**31-1)
+        Limits max length of str. (default: 1024*1024)
 
     :param int max_bin_len:
-        Limits max length of bin. (default: 2**31-1)
+        Limits max length of bin. (default: 1024*1024)
 
     :param int max_array_len:
-        Limits max length of array. (default: 2**31-1)
+        Limits max length of array. (default: 128*1024)
 
     :param int max_map_len:
-        Limits max length of map. (default: 2**31-1)
+        Limits max length of map. (default: 32*1024)
 
+    :param int max_ext_len:
+        Limits max length of map. (default: 1024*1024)
 
     example of streaming deserialize from file-like object::
 
@@ -218,11 +220,11 @@ class Unpacker(object):
                  object_hook=None, object_pairs_hook=None, list_hook=None,
                  encoding=None, unicode_errors=None, max_buffer_size=0,
                  ext_hook=ExtType,
-                 max_str_len=2147483647, # 2**32-1
-                 max_bin_len=2147483647,
-                 max_array_len=2147483647,
-                 max_map_len=2147483647,
-                 max_ext_len=2147483647):
+                 max_str_len=1024*1024,
+                 max_bin_len=1024*1024,
+                 max_array_len=128*1024,
+                 max_map_len=32*1024,
+                 max_ext_len=1024*1024):
 
         if encoding is not None:
             warnings.warn(

From a8b3e97fe588a2411a8e869b52be1946ed9f0f86 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 8 Nov 2018 22:25:05 +0900
Subject: [PATCH 1436/1648] Update changelog

---
 ChangeLog.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index b135745..d71e0f5 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -3,9 +3,21 @@
 
 Release Date: TBD
 
+
+Important changes
+------------------
+
 Extension modules are merged.  There is ``msgpack._msgpack`` instead of
 ``msgpack._packer`` and ``msgpack._unpacker``. (#314)
 
+unpacker: Default size limits is smaller than before to avoid DoS attack.
+If you need to handle large data, you need to specify limits manually.
+
+
+
+Other changes
+--------------
+
 
 
 0.5.6

From 9e210bfc1a922031db67bf42e508b1b4550814c6 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 9 Nov 2018 20:55:13 +0900
Subject: [PATCH 1437/1648] Add Packer.buffer() (#320)

---
 ChangeLog.rst            |  2 ++
 docs/Makefile            |  2 +-
 docs/advanced.rst        | 32 ++++++++++++++++++++++++++++++++
 docs/index.rst           |  1 +
 msgpack/_packer.pyx      | 14 ++++++++++++--
 msgpack/buff_converter.h | 28 ++++++++++++++++++++++++++++
 msgpack/fallback.py      | 36 ++++++++++++++++++++----------------
 test/test_pack.py        | 12 +++++++++++-
 8 files changed, 107 insertions(+), 20 deletions(-)
 create mode 100644 docs/advanced.rst
 create mode 100644 msgpack/buff_converter.h

diff --git a/ChangeLog.rst b/ChangeLog.rst
index d71e0f5..cc6b5e4 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -18,6 +18,8 @@ If you need to handle large data, you need to specify limits manually.
 Other changes
 --------------
 
+Add ``Unpacker.getbuffer()`` method.
+
 
 
 0.5.6
diff --git a/docs/Makefile b/docs/Makefile
index b09d884..831a6a7 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -153,7 +153,7 @@ doctest:
 	      "results in $(BUILDDIR)/doctest/output.txt."
 
 serve: html
-	cd _build/html && python3 -m http.server
+	python3 -m http.server -d _build/html
 
 zip: html
 	cd _build/html && zip -r ../../../msgpack-doc.zip .
diff --git a/docs/advanced.rst b/docs/advanced.rst
new file mode 100644
index 0000000..3837008
--- /dev/null
+++ b/docs/advanced.rst
@@ -0,0 +1,32 @@
+Advanced usage
+===============
+
+Packer
+------
+
+autoreset
+~~~~~~~~~
+
+When you used ``autoreset=False`` option of :class:`~msgpack.Packer`, 
+``pack()`` method doesn't return packed ``bytes``.
+
+You can use :meth:`~msgpack.Packer.bytes` or :meth:`~msgpack.Packer.getbuffer` to
+get packed data.
+
+``bytes()`` returns ``bytes`` object.  ``getbuffer()`` returns some bytes-like
+object.  It's concrete type is implement detail and it will be changed in future
+versions.
+
+You can reduce temporary bytes object by using ``Unpacker.getbuffer()``.
+
+.. code-block:: python
+
+    packer = Packer(use_bin_type=True, autoreset=False)
+
+    packer.pack([1, 2])
+    packer.pack([3, 4])
+
+    with open('data.bin', 'wb') as f:
+        f.write(packer.getbuffer())
+
+    packer.reset()  # reset internal buffer
diff --git a/docs/index.rst b/docs/index.rst
index dcdab4f..e9c2ce8 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -8,3 +8,4 @@ language data exchange.
    :maxdepth: 1
 
    api
+   advanced
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 225f24a..fd05ae0 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -41,6 +41,9 @@ cdef extern from "pack.h":
     int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l)
     int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit)
 
+cdef extern from "buff_converter.h":
+    object buff_to_buff(char *, Py_ssize_t)
+
 cdef int DEFAULT_RECURSE_LIMIT=511
 cdef long long ITEM_LIMIT = (2**32)-1
 
@@ -349,9 +352,16 @@ cdef class Packer(object):
             return buf
 
     def reset(self):
-        """Clear internal buffer."""
+        """Reset internal buffer.
+
+        This method is usaful only when autoreset=False.
+        """
         self.pk.length = 0
 
     def bytes(self):
-        """Return buffer content."""
+        """Return internal buffer contents as bytes object"""
         return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
+
+    def getbuffer(self):
+        """Return view of internal buffer."""
+        return buff_to_buff(self.pk.buf, self.pk.length)
diff --git a/msgpack/buff_converter.h b/msgpack/buff_converter.h
new file mode 100644
index 0000000..bc7227a
--- /dev/null
+++ b/msgpack/buff_converter.h
@@ -0,0 +1,28 @@
+#include "Python.h"
+
+/* cython does not support this preprocessor check => write it in raw C */
+#if PY_MAJOR_VERSION == 2
+static PyObject *
+buff_to_buff(char *buff, Py_ssize_t size)
+{
+    return PyBuffer_FromMemory(buff, size);
+}
+
+#elif (PY_MAJOR_VERSION == 3) && (PY_MINOR_VERSION >= 3)
+static PyObject *
+buff_to_buff(char *buff, Py_ssize_t size)
+{
+    return PyMemoryView_FromMemory(buff, size, PyBUF_READ);
+}
+#else
+static PyObject *
+buff_to_buff(char *buff, Py_ssize_t size)
+{
+    Py_buffer pybuf;
+    if (PyBuffer_FillInfo(&pybuf, NULL, buff, size, 1, PyBUF_FULL_RO) == -1) {
+        return NULL;
+    }
+
+    return PyMemoryView_FromBuffer(&pybuf);
+}
+#endif
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 895864e..5b4d6ce 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -860,43 +860,35 @@ class Packer(object):
         except:
             self._buffer = StringIO()  # force reset
             raise
-        ret = self._buffer.getvalue()
         if self._autoreset:
+            ret = self._buffer.getvalue()
             self._buffer = StringIO()
-        elif USING_STRINGBUILDER:
-            self._buffer = StringIO(ret)
-        return ret
+            return ret
 
     def pack_map_pairs(self, pairs):
         self._pack_map_pairs(len(pairs), pairs)
-        ret = self._buffer.getvalue()
         if self._autoreset:
+            ret = self._buffer.getvalue()
             self._buffer = StringIO()
-        elif USING_STRINGBUILDER:
-            self._buffer = StringIO(ret)
-        return ret
+            return ret
 
     def pack_array_header(self, n):
         if n >= 2**32:
             raise PackValueError
         self._pack_array_header(n)
-        ret = self._buffer.getvalue()
         if self._autoreset:
+            ret = self._buffer.getvalue()
             self._buffer = StringIO()
-        elif USING_STRINGBUILDER:
-            self._buffer = StringIO(ret)
-        return ret
+            return ret
 
     def pack_map_header(self, n):
         if n >= 2**32:
             raise PackValueError
         self._pack_map_header(n)
-        ret = self._buffer.getvalue()
         if self._autoreset:
+            ret = self._buffer.getvalue()
             self._buffer = StringIO()
-        elif USING_STRINGBUILDER:
-            self._buffer = StringIO(ret)
-        return ret
+            return ret
 
     def pack_ext_type(self, typecode, data):
         if not isinstance(typecode, int):
@@ -976,7 +968,19 @@ class Packer(object):
             raise PackValueError('Bin is too large')
 
     def bytes(self):
+        """Return internal buffer contents as bytes object"""
         return self._buffer.getvalue()
 
     def reset(self):
+        """Reset internal buffer.
+
+        This method is usaful only when autoreset=False.
+        """
         self._buffer = StringIO()
+
+    def getbuffer(self):
+        """Return view of internal buffer."""
+        if USING_STRINGBUILDER or not PY3:
+            return memoryview(self.bytes())
+        else:
+            return self._buffer.getbuffer()
diff --git a/test/test_pack.py b/test/test_pack.py
index b447f9c..4608083 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import struct
 from pytest import raises, xfail
 
-from msgpack import packb, unpackb, Unpacker, Packer
+from msgpack import packb, unpackb, Unpacker, Packer, pack
 
 from collections import OrderedDict
 from io import BytesIO
@@ -148,3 +148,13 @@ def test_pairlist():
     packed = packer.pack_map_pairs(pairlist)
     unpacked = unpackb(packed, object_pairs_hook=list)
     assert pairlist == unpacked
+
+def test_get_buffer():
+    packer = Packer(autoreset=0, use_bin_type=True)
+    packer.pack([1, 2])
+    strm = BytesIO()
+    strm.write(packer.getbuffer())
+    written = strm.getvalue()
+
+    expected = packb([1, 2], use_bin_type=True)
+    assert written == expected

From 1bf62ba6f8f94ab8a7dd135e0039ee3b10e0e96c Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Fri, 9 Nov 2018 21:39:25 +0900
Subject: [PATCH 1438/1648] PendingDeprecationWarning -> DeprecationWarning
 (#321)

---
 msgpack/_packer.pyx   | 2 +-
 msgpack/_unpacker.pyx | 6 +++---
 msgpack/fallback.py   | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index fd05ae0..6a6d917 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -121,7 +121,7 @@ cdef class Packer(object):
                  bint use_single_float=False, bint autoreset=True, bint use_bin_type=False,
                  bint strict_types=False):
         if encoding is not None:
-            PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated.", 1)
+            PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1)
         self.use_float = use_single_float
         self.strict_types = strict_types
         self.autoreset = autoreset
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index cc9e7f0..85c404a 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -186,7 +186,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef int new_protocol = 0
 
     if encoding is not None:
-        PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
+        PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
         cenc = encoding
 
     if unicode_errors is not None:
@@ -213,7 +213,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
 def unpack(object stream, **kwargs):
     PyErr_WarnEx(
-        PendingDeprecationWarning,
+        DeprecationWarning,
         "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1)
     data = stream.read()
     return unpackb(data, **kwargs)
@@ -360,7 +360,7 @@ cdef class Unpacker(object):
         self.stream_offset = 0
 
         if encoding is not None:
-            PyErr_WarnEx(PendingDeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
+            PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
             self.encoding = encoding
             cenc = encoding
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 5b4d6ce..197b6d2 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -103,7 +103,7 @@ def _get_data_from_buffer(obj):
 def unpack(stream, **kwargs):
     warnings.warn(
         "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
-        PendingDeprecationWarning)
+        DeprecationWarning)
     data = stream.read()
     return unpackb(data, **kwargs)
 
@@ -229,7 +229,7 @@ class Unpacker(object):
         if encoding is not None:
             warnings.warn(
                 "encoding is deprecated, Use raw=False instead.",
-                PendingDeprecationWarning)
+                DeprecationWarning)
 
         if unicode_errors is None:
             unicode_errors = 'strict'
@@ -727,7 +727,7 @@ class Packer(object):
         else:
             warnings.warn(
                 "encoding is deprecated, Use raw=False instead.",
-                PendingDeprecationWarning)
+                DeprecationWarning)
 
         if unicode_errors is None:
             unicode_errors = 'strict'

From 07f0beeabb71828377f481ff83746997b3babf23 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 12 Nov 2018 02:19:01 +0900
Subject: [PATCH 1439/1648] Remove deprecated exception classes (#323)

---
 msgpack/_packer.pyx   | 27 ++++++++-------
 msgpack/_unpacker.pyx | 40 ++++++++++------------
 msgpack/exceptions.py | 35 +++++++++-----------
 msgpack/fallback.py   | 77 +++++++++++++++++++++----------------------
 4 files changed, 84 insertions(+), 95 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 6a6d917..2643f85 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -5,7 +5,6 @@ from cpython cimport *
 from cpython.version cimport PY_MAJOR_VERSION
 from cpython.exc cimport PyErr_WarnEx
 
-from msgpack.exceptions import PackValueError, PackOverflowError
 from msgpack import ExtType
 
 
@@ -165,7 +164,7 @@ cdef class Packer(object):
         cdef Py_buffer view
 
         if nest_limit < 0:
-            raise PackValueError("recursion limit exceeded.")
+            raise ValueError("recursion limit exceeded.")
 
         while True:
             if o is None:
@@ -191,7 +190,7 @@ cdef class Packer(object):
                         default_used = True
                         continue
                     else:
-                        raise PackOverflowError("Integer value out of range")
+                        raise OverflowError("Integer value out of range")
             elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o):
                 longval = o
                 ret = msgpack_pack_long(&self.pk, longval)
@@ -205,7 +204,7 @@ cdef class Packer(object):
             elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o):
                 L = len(o)
                 if L > ITEM_LIMIT:
-                    raise PackValueError("%s is too large" % type(o).__name__)
+                    raise ValueError("%s is too large" % type(o).__name__)
                 rawval = o
                 ret = msgpack_pack_bin(&self.pk, L)
                 if ret == 0:
@@ -214,12 +213,12 @@ cdef class Packer(object):
                 if self.encoding == NULL and self.unicode_errors == NULL:
                     ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT);
                     if ret == -2:
-                        raise PackValueError("unicode string is too large")
+                        raise ValueError("unicode string is too large")
                 else:
                     o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
                     L = len(o)
                     if L > ITEM_LIMIT:
-                        raise PackValueError("unicode string is too large")
+                        raise ValueError("unicode string is too large")
                     ret = msgpack_pack_raw(&self.pk, L)
                     if ret == 0:
                         rawval = o
@@ -228,7 +227,7 @@ cdef class Packer(object):
                 d = o
                 L = len(d)
                 if L > ITEM_LIMIT:
-                    raise PackValueError("dict is too large")
+                    raise ValueError("dict is too large")
                 ret = msgpack_pack_map(&self.pk, L)
                 if ret == 0:
                     for k, v in d.iteritems():
@@ -239,7 +238,7 @@ cdef class Packer(object):
             elif not strict_types and PyDict_Check(o):
                 L = len(o)
                 if L > ITEM_LIMIT:
-                    raise PackValueError("dict is too large")
+                    raise ValueError("dict is too large")
                 ret = msgpack_pack_map(&self.pk, L)
                 if ret == 0:
                     for k, v in o.items():
@@ -253,13 +252,13 @@ cdef class Packer(object):
                 rawval = o.data
                 L = len(o.data)
                 if L > ITEM_LIMIT:
-                    raise PackValueError("EXT data is too large")
+                    raise ValueError("EXT data is too large")
                 ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
                 L = len(o)
                 if L > ITEM_LIMIT:
-                    raise PackValueError("list is too large")
+                    raise ValueError("list is too large")
                 ret = msgpack_pack_array(&self.pk, L)
                 if ret == 0:
                     for v in o:
@@ -267,11 +266,11 @@ cdef class Packer(object):
                         if ret != 0: break
             elif PyMemoryView_Check(o):
                 if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0:
-                    raise PackValueError("could not get buffer for memoryview")
+                    raise ValueError("could not get buffer for memoryview")
                 L = view.len
                 if L > ITEM_LIMIT:
                     PyBuffer_Release(&view);
-                    raise PackValueError("memoryview is too large")
+                    raise ValueError("memoryview is too large")
                 ret = msgpack_pack_bin(&self.pk, L)
                 if ret == 0:
                     ret = msgpack_pack_raw_body(&self.pk, view.buf, L)
@@ -304,7 +303,7 @@ cdef class Packer(object):
 
     def pack_array_header(self, long long size):
         if size > ITEM_LIMIT:
-            raise PackValueError
+            raise ValueError
         cdef int ret = msgpack_pack_array(&self.pk, size)
         if ret == -1:
             raise MemoryError
@@ -317,7 +316,7 @@ cdef class Packer(object):
 
     def pack_map_header(self, long long size):
         if size > ITEM_LIMIT:
-            raise PackValueError
+            raise ValueError
         cdef int ret = msgpack_pack_map(&self.pk, size)
         if ret == -1:
             raise MemoryError
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 85c404a..2f99019 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -35,7 +35,6 @@ ctypedef unsigned long long uint64_t
 from msgpack.exceptions import (
     BufferFull,
     OutOfData,
-    UnpackValueError,
     ExtraData,
 )
 from msgpack import ExtType
@@ -208,7 +207,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
             raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
         return obj
     unpack_clear(&ctx)
-    raise UnpackValueError("Unpack failed: error = %d" % (ret,))
+    raise ValueError("Unpack failed: error = %d" % (ret,))
 
 
 def unpack(object stream, **kwargs):
@@ -460,28 +459,25 @@ cdef class Unpacker(object):
                 else:
                     raise OutOfData("No more data to unpack.")
 
-            try:
-                ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
-                self.stream_offset += self.buf_head - prev_head
-                if write_bytes is not None:
-                    write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
+            ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+            self.stream_offset += self.buf_head - prev_head
+            if write_bytes is not None:
+                write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
 
-                if ret == 1:
-                    obj = unpack_data(&self.ctx)
-                    unpack_init(&self.ctx)
-                    return obj
-                elif ret == 0:
-                    if self.file_like is not None:
-                        self.read_from_file()
-                        continue
-                    if iter:
-                        raise StopIteration("No more data to unpack.")
-                    else:
-                        raise OutOfData("No more data to unpack.")
+            if ret == 1:
+                obj = unpack_data(&self.ctx)
+                unpack_init(&self.ctx)
+                return obj
+            elif ret == 0:
+                if self.file_like is not None:
+                    self.read_from_file()
+                    continue
+                if iter:
+                    raise StopIteration("No more data to unpack.")
                 else:
-                    raise UnpackValueError("Unpack failed: error = %d" % (ret,))
-            except ValueError as e:
-                raise UnpackValueError(e)
+                    raise OutOfData("No more data to unpack.")
+            else:
+                raise ValueError("Unpack failed: error = %d" % (ret,))
 
     def read_bytes(self, Py_ssize_t nbytes):
         """Read a specified number of raw bytes from the stream"""
diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py
index 9766881..5bee5b2 100644
--- a/msgpack/exceptions.py
+++ b/msgpack/exceptions.py
@@ -1,6 +1,10 @@
 class UnpackException(Exception):
-    """Deprecated.  Use Exception instead to catch all exception during unpacking."""
+    """Base class for some exceptions raised while unpacking.
 
+    NOTE: unpack may raise exception other than subclass of
+    UnpackException.  If you want to catch all error, catch
+    Exception instead.
+    """
 
 class BufferFull(UnpackException):
     pass
@@ -10,11 +14,16 @@ class OutOfData(UnpackException):
     pass
 
 
-class UnpackValueError(UnpackException, ValueError):
-    """Deprecated.  Use ValueError instead."""
+# Deprecated.  Use ValueError instead
+UnpackValueError = ValueError
 
 
 class ExtraData(UnpackValueError):
+    """ExtraData is raised when there is trailing data.
+
+    This exception is raised while only one-shot (not streaming)
+    unpack.
+    """
     def __init__(self, unpacked, extra):
         self.unpacked = unpacked
         self.extra = extra
@@ -23,19 +32,7 @@ class ExtraData(UnpackValueError):
         return "unpack(b) received extra data."
 
 
-class PackException(Exception):
-    """Deprecated.  Use Exception instead to catch all exception during packing."""
-
-
-class PackValueError(PackException, ValueError):
-    """PackValueError is raised when type of input data is supported but it's value is unsupported.
-
-    Deprecated.  Use ValueError instead.
-    """
-
-
-class PackOverflowError(PackValueError, OverflowError):
-    """PackOverflowError is raised when integer value is out of range of msgpack support [-2**31, 2**32).
-
-    Deprecated.  Use ValueError instead.
-    """
+#Deprecated.  Use Exception instead to catch all exception during packing.
+PackException = Exception
+PackValueError = ValueError
+PackOverflowError = OverflowError
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 197b6d2..9d46171 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -52,9 +52,6 @@ else:
 from msgpack.exceptions import (
     BufferFull,
     OutOfData,
-    UnpackValueError,
-    PackValueError,
-    PackOverflowError,
     ExtraData)
 
 from msgpack import ExtType
@@ -120,7 +117,7 @@ def unpackb(packed, **kwargs):
     try:
         ret = unpacker._unpack()
     except OutOfData:
-        raise UnpackValueError("Data is not enough.")
+        raise ValueError("Data is not enough.")
     if unpacker._got_extradata():
         raise ExtraData(ret, unpacker._get_extradata())
     return ret
@@ -370,18 +367,18 @@ class Unpacker(object):
             n = b & 0b00011111
             typ = TYPE_RAW
             if n > self._max_str_len:
-                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
         elif b & 0b11110000 == 0b10010000:
             n = b & 0b00001111
             typ = TYPE_ARRAY
             if n > self._max_array_len:
-                raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
         elif b & 0b11110000 == 0b10000000:
             n = b & 0b00001111
             typ = TYPE_MAP
             if n > self._max_map_len:
-                raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
         elif b == 0xc0:
             obj = None
         elif b == 0xc2:
@@ -394,7 +391,7 @@ class Unpacker(object):
             n = self._buffer[self._buff_i]
             self._buff_i += 1
             if n > self._max_bin_len:
-                raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
+                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
         elif b == 0xc5:
             typ = TYPE_BIN
@@ -402,7 +399,7 @@ class Unpacker(object):
             n = _unpack_from(">H", self._buffer, self._buff_i)[0]
             self._buff_i += 2
             if n > self._max_bin_len:
-                raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
+                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
         elif b == 0xc6:
             typ = TYPE_BIN
@@ -410,7 +407,7 @@ class Unpacker(object):
             n = _unpack_from(">I", self._buffer, self._buff_i)[0]
             self._buff_i += 4
             if n > self._max_bin_len:
-                raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
+                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
         elif b == 0xc7:  # ext 8
             typ = TYPE_EXT
@@ -418,7 +415,7 @@ class Unpacker(object):
             L, n = _unpack_from('Bb', self._buffer, self._buff_i)
             self._buff_i += 2
             if L > self._max_ext_len:
-                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
+                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
         elif b == 0xc8:  # ext 16
             typ = TYPE_EXT
@@ -426,7 +423,7 @@ class Unpacker(object):
             L, n = _unpack_from('>Hb', self._buffer, self._buff_i)
             self._buff_i += 3
             if L > self._max_ext_len:
-                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
+                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
         elif b == 0xc9:  # ext 32
             typ = TYPE_EXT
@@ -434,7 +431,7 @@ class Unpacker(object):
             L, n = _unpack_from('>Ib', self._buffer, self._buff_i)
             self._buff_i += 5
             if L > self._max_ext_len:
-                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
+                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
         elif b == 0xca:
             self._reserve(4)
@@ -479,35 +476,35 @@ class Unpacker(object):
         elif b == 0xd4:  # fixext 1
             typ = TYPE_EXT
             if self._max_ext_len < 1:
-                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
+                raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
             self._reserve(2)
             n, obj = _unpack_from("b1s", self._buffer, self._buff_i)
             self._buff_i += 2
         elif b == 0xd5:  # fixext 2
             typ = TYPE_EXT
             if self._max_ext_len < 2:
-                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
+                raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
             self._reserve(3)
             n, obj = _unpack_from("b2s", self._buffer, self._buff_i)
             self._buff_i += 3
         elif b == 0xd6:  # fixext 4
             typ = TYPE_EXT
             if self._max_ext_len < 4:
-                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
+                raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
             self._reserve(5)
             n, obj = _unpack_from("b4s", self._buffer, self._buff_i)
             self._buff_i += 5
         elif b == 0xd7:  # fixext 8
             typ = TYPE_EXT
             if self._max_ext_len < 8:
-                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
+                raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
             self._reserve(9)
             n, obj = _unpack_from("b8s", self._buffer, self._buff_i)
             self._buff_i += 9
         elif b == 0xd8:  # fixext 16
             typ = TYPE_EXT
             if self._max_ext_len < 16:
-                raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
+                raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
             self._reserve(17)
             n, obj = _unpack_from("b16s", self._buffer, self._buff_i)
             self._buff_i += 17
@@ -517,7 +514,7 @@ class Unpacker(object):
             n = self._buffer[self._buff_i]
             self._buff_i += 1
             if n > self._max_str_len:
-                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
         elif b == 0xda:
             typ = TYPE_RAW
@@ -525,7 +522,7 @@ class Unpacker(object):
             n, = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_str_len:
-                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
         elif b == 0xdb:
             typ = TYPE_RAW
@@ -533,7 +530,7 @@ class Unpacker(object):
             n, = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_str_len:
-                raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
         elif b == 0xdc:
             typ = TYPE_ARRAY
@@ -541,30 +538,30 @@ class Unpacker(object):
             n, = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_array_len:
-                raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
         elif b == 0xdd:
             typ = TYPE_ARRAY
             self._reserve(4)
             n, = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_array_len:
-                raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
         elif b == 0xde:
             self._reserve(2)
             n, = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_map_len:
-                raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         elif b == 0xdf:
             self._reserve(4)
             n, = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_map_len:
-                raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         else:
-            raise UnpackValueError("Unknown header: 0x%x" % b)
+            raise ValueError("Unknown header: 0x%x" % b)
         return typ, n, obj
 
     def _unpack(self, execute=EX_CONSTRUCT):
@@ -572,11 +569,11 @@ class Unpacker(object):
 
         if execute == EX_READ_ARRAY_HEADER:
             if typ != TYPE_ARRAY:
-                raise UnpackValueError("Expected array")
+                raise ValueError("Expected array")
             return n
         if execute == EX_READ_MAP_HEADER:
             if typ != TYPE_MAP:
-                raise UnpackValueError("Expected map")
+                raise ValueError("Expected map")
             return n
         # TODO should we eliminate the recursion?
         if typ == TYPE_ARRAY:
@@ -754,7 +751,7 @@ class Packer(object):
             list_types = (list, tuple)
         while True:
             if nest_limit < 0:
-                raise PackValueError("recursion limit exceeded")
+                raise ValueError("recursion limit exceeded")
             if obj is None:
                 return self._buffer.write(b"\xc0")
             if check(obj, bool):
@@ -786,11 +783,11 @@ class Packer(object):
                     obj = self._default(obj)
                     default_used = True
                     continue
-                raise PackOverflowError("Integer value out of range")
+                raise OverflowError("Integer value out of range")
             if check(obj, (bytes, bytearray)):
                 n = len(obj)
                 if n >= 2**32:
-                    raise PackValueError("%s is too large" % type(obj).__name__)
+                    raise ValueError("%s is too large" % type(obj).__name__)
                 self._pack_bin_header(n)
                 return self._buffer.write(obj)
             if check(obj, Unicode):
@@ -801,13 +798,13 @@ class Packer(object):
                 obj = obj.encode(self._encoding, self._unicode_errors)
                 n = len(obj)
                 if n >= 2**32:
-                    raise PackValueError("String is too large")
+                    raise ValueError("String is too large")
                 self._pack_raw_header(n)
                 return self._buffer.write(obj)
             if check(obj, memoryview):
                 n = len(obj) * obj.itemsize
                 if n >= 2**32:
-                    raise PackValueError("Memoryview is too large")
+                    raise ValueError("Memoryview is too large")
                 self._pack_bin_header(n)
                 return self._buffer.write(obj)
             if check(obj, float):
@@ -874,7 +871,7 @@ class Packer(object):
 
     def pack_array_header(self, n):
         if n >= 2**32:
-            raise PackValueError
+            raise ValueError
         self._pack_array_header(n)
         if self._autoreset:
             ret = self._buffer.getvalue()
@@ -883,7 +880,7 @@ class Packer(object):
 
     def pack_map_header(self, n):
         if n >= 2**32:
-            raise PackValueError
+            raise ValueError
         self._pack_map_header(n)
         if self._autoreset:
             ret = self._buffer.getvalue()
@@ -899,7 +896,7 @@ class Packer(object):
             raise TypeError("data must have bytes type")
         L = len(data)
         if L > 0xffffffff:
-            raise PackValueError("Too large data")
+            raise ValueError("Too large data")
         if L == 1:
             self._buffer.write(b'\xd4')
         elif L == 2:
@@ -926,7 +923,7 @@ class Packer(object):
             return self._buffer.write(struct.pack(">BH", 0xdc, n))
         if n <= 0xffffffff:
             return self._buffer.write(struct.pack(">BI", 0xdd, n))
-        raise PackValueError("Array is too large")
+        raise ValueError("Array is too large")
 
     def _pack_map_header(self, n):
         if n <= 0x0f:
@@ -935,7 +932,7 @@ class Packer(object):
             return self._buffer.write(struct.pack(">BH", 0xde, n))
         if n <= 0xffffffff:
             return self._buffer.write(struct.pack(">BI", 0xdf, n))
-        raise PackValueError("Dict is too large")
+        raise ValueError("Dict is too large")
 
     def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
         self._pack_map_header(n)
@@ -953,7 +950,7 @@ class Packer(object):
         elif n <= 0xffffffff:
             self._buffer.write(struct.pack(">BI", 0xdb, n))
         else:
-            raise PackValueError('Raw is too large')
+            raise ValueError('Raw is too large')
 
     def _pack_bin_header(self, n):
         if not self._use_bin_type:
@@ -965,7 +962,7 @@ class Packer(object):
         elif n <= 0xffffffff:
             return self._buffer.write(struct.pack(">BI", 0xc6, n))
         else:
-            raise PackValueError('Bin is too large')
+            raise ValueError('Bin is too large')
 
     def bytes(self):
         """Return internal buffer contents as bytes object"""

From 39f8aa78c7bc6b6b18a1c814a0c296f55242f028 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 12 Nov 2018 02:33:31 +0900
Subject: [PATCH 1440/1648] Remove deprecated write_bytes option (#322)

---
 msgpack/_unpacker.pyx   | 35 ++++++++++++-----------------------
 msgpack/fallback.py     | 20 ++++----------------
 test/test_unpack_raw.py | 29 -----------------------------
 3 files changed, 16 insertions(+), 68 deletions(-)
 delete mode 100644 test/test_unpack_raw.py

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 2f99019..e168587 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -135,10 +135,10 @@ cdef inline int get_data_from_buffer(object obj,
         if view.itemsize != 1:
             PyBuffer_Release(view)
             raise BufferError("cannot unpack from multi-byte object")
-        if PyBuffer_IsContiguous(view, 'A') == 0:
+        if PyBuffer_IsContiguous(view, b'A') == 0:
             PyBuffer_Release(view)
             # create a contiguous copy and get buffer
-            contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C')
+            contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C')
             PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE)
             # view must hold the only reference to contiguous,
             # so memory is freed when view is released
@@ -440,14 +440,11 @@ cdef class Unpacker(object):
         else:
             self.file_like = None
 
-    cdef object _unpack(self, execute_fn execute, object write_bytes, bint iter=0):
+    cdef object _unpack(self, execute_fn execute, bint iter=0):
         cdef int ret
         cdef object obj
         cdef Py_ssize_t prev_head
 
-        if write_bytes is not None:
-            PyErr_WarnEx(DeprecationWarning, "`write_bytes` option is deprecated. Use `.tell()` instead.", 1)
-
         if self.buf_head >= self.buf_tail and self.file_like is not None:
             self.read_from_file()
 
@@ -461,8 +458,6 @@ cdef class Unpacker(object):
 
             ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
             self.stream_offset += self.buf_head - prev_head
-            if write_bytes is not None:
-                write_bytes(PyBytes_FromStringAndSize(self.buf + prev_head, self.buf_head - prev_head))
 
             if ret == 1:
                 obj = unpack_data(&self.ctx)
@@ -489,41 +484,35 @@ cdef class Unpacker(object):
             ret += self.file_like.read(nbytes - len(ret))
         return ret
 
-    def unpack(self, object write_bytes=None):
+    def unpack(self):
         """Unpack one object
 
-        If write_bytes is not None, it will be called with parts of the raw
-        message as it is unpacked.
-
         Raises `OutOfData` when there are no more bytes to unpack.
         """
-        return self._unpack(unpack_construct, write_bytes)
+        return self._unpack(unpack_construct)
 
-    def skip(self, object write_bytes=None):
+    def skip(self):
         """Read and ignore one object, returning None
 
-        If write_bytes is not None, it will be called with parts of the raw
-        message as it is unpacked.
-
         Raises `OutOfData` when there are no more bytes to unpack.
         """
-        return self._unpack(unpack_skip, write_bytes)
+        return self._unpack(unpack_skip)
 
-    def read_array_header(self, object write_bytes=None):
+    def read_array_header(self):
         """assuming the next object is an array, return its size n, such that
         the next n unpack() calls will iterate over its contents.
 
         Raises `OutOfData` when there are no more bytes to unpack.
         """
-        return self._unpack(read_array_header, write_bytes)
+        return self._unpack(read_array_header)
 
-    def read_map_header(self, object write_bytes=None):
+    def read_map_header(self):
         """assuming the next object is a map, return its size n, such that the
         next n * 2 unpack() calls will iterate over its key-value pairs.
 
         Raises `OutOfData` when there are no more bytes to unpack.
         """
-        return self._unpack(read_map_header, write_bytes)
+        return self._unpack(read_map_header)
 
     def tell(self):
         return self.stream_offset
@@ -532,7 +521,7 @@ cdef class Unpacker(object):
         return self
 
     def __next__(self):
-        return self._unpack(unpack_construct, None, 1)
+        return self._unpack(unpack_construct, 1)
 
     # for debug.
     #def _buf(self):
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 9d46171..b9ef296 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -640,34 +640,22 @@ class Unpacker(object):
 
     next = __next__
 
-    def skip(self, write_bytes=None):
+    def skip(self):
         self._unpack(EX_SKIP)
-        if write_bytes is not None:
-            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
-            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
 
-    def unpack(self, write_bytes=None):
+    def unpack(self):
         ret = self._unpack(EX_CONSTRUCT)
-        if write_bytes is not None:
-            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
-            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
         return ret
 
-    def read_array_header(self, write_bytes=None):
+    def read_array_header(self):
         ret = self._unpack(EX_READ_ARRAY_HEADER)
-        if write_bytes is not None:
-            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
-            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
         return ret
 
-    def read_map_header(self, write_bytes=None):
+    def read_map_header(self):
         ret = self._unpack(EX_READ_MAP_HEADER)
-        if write_bytes is not None:
-            warnings.warn("`write_bytes` option is deprecated.  Use `.tell()` instead.", DeprecationWarning)
-            write_bytes(self._buffer[self._buf_checkpoint:self._buff_i])
         self._consume()
         return ret
 
diff --git a/test/test_unpack_raw.py b/test/test_unpack_raw.py
deleted file mode 100644
index 7002601..0000000
--- a/test/test_unpack_raw.py
+++ /dev/null
@@ -1,29 +0,0 @@
-"""Tests for cases where the user seeks to obtain packed msgpack objects"""
-
-import io
-from msgpack import Unpacker, packb
-
-
-def test_write_bytes():
-    unpacker = Unpacker()
-    unpacker.feed(b'abc')
-    f = io.BytesIO()
-    assert unpacker.unpack(f.write) == ord('a')
-    assert f.getvalue() == b'a'
-    f = io.BytesIO()
-    assert unpacker.skip(f.write) is None
-    assert f.getvalue() == b'b'
-    f = io.BytesIO()
-    assert unpacker.skip() is None
-    assert f.getvalue() == b''
-
-
-def test_write_bytes_multi_buffer():
-    long_val = (5) * 100
-    expected = packb(long_val)
-    unpacker = Unpacker(io.BytesIO(expected), read_size=3, max_buffer_size=3)
-
-    f = io.BytesIO()
-    unpacked = unpacker.unpack(f.write)
-    assert unpacked == long_val
-    assert f.getvalue() == expected

From 2b5f59166beeccde0ee230c8673cf50932c8daba Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 14 Nov 2018 16:34:51 +0900
Subject: [PATCH 1441/1648] fallback: Fix warning stacklevel (#327)

---
 msgpack/fallback.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index b9ef296..04fb5b9 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -89,7 +89,7 @@ def _get_data_from_buffer(obj):
             warnings.warn("using old buffer interface to unpack %s; "
                           "this leads to unpacking errors if slicing is used and "
                           "will be removed in a future version" % type(obj),
-                          RuntimeWarning)
+                          RuntimeWarning, stacklevel=3)
         else:
             raise
     if view.itemsize != 1:
@@ -100,7 +100,7 @@ def _get_data_from_buffer(obj):
 def unpack(stream, **kwargs):
     warnings.warn(
         "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
-        DeprecationWarning)
+        DeprecationWarning, stacklevel=2)
     data = stream.read()
     return unpackb(data, **kwargs)
 
@@ -226,7 +226,7 @@ class Unpacker(object):
         if encoding is not None:
             warnings.warn(
                 "encoding is deprecated, Use raw=False instead.",
-                DeprecationWarning)
+                DeprecationWarning, stacklevel=2)
 
         if unicode_errors is None:
             unicode_errors = 'strict'
@@ -712,7 +712,7 @@ class Packer(object):
         else:
             warnings.warn(
                 "encoding is deprecated, Use raw=False instead.",
-                DeprecationWarning)
+                DeprecationWarning, stacklevel=2)
 
         if unicode_errors is None:
             unicode_errors = 'strict'

From d782464c9150e448ab3a8d81197ff335e1ac2c2b Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 14 Nov 2018 16:35:37 +0900
Subject: [PATCH 1442/1648] Refactor Cython code (#328)

_msgpack -> _cmsgpack
---
 .travis.yml           |  2 +-
 Makefile              |  2 +-
 appveyor.yml          |  3 ++-
 docker/runtests.sh    |  2 +-
 msgpack/__init__.py   |  2 +-
 msgpack/_cmsgpack.pyx |  4 ++++
 msgpack/_msgpack.pyx  |  4 ----
 msgpack/_packer.pyx   | 10 +++-------
 msgpack/_unpacker.pyx | 22 +---------------------
 setup.py              |  6 +++---
 10 files changed, 17 insertions(+), 40 deletions(-)
 create mode 100644 msgpack/_cmsgpack.pyx
 delete mode 100644 msgpack/_msgpack.pyx

diff --git a/.travis.yml b/.travis.yml
index 822ca9a..1adbdc2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -45,7 +45,7 @@ install:
 
 script:
   - python -c 'import sys; print(hex(sys.maxsize))'
-  - python -c 'from msgpack import _msgpack'
+  - python -c 'from msgpack import _cmsgpack'
   - pytest -v test
   - MSGPACK_PUREPYTHON=x pytest -v test
 
diff --git a/Makefile b/Makefile
index ff9a482..b65aa85 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ all: cython
 
 .PHONY: cython
 cython:
-	cython --cplus msgpack/_msgpack.pyx
+	cython --cplus msgpack/_cmsgpack.pyx
 
 .PHONY: test
 test:
diff --git a/appveyor.yml b/appveyor.yml
index 72b334a..f0e21fc 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -6,8 +6,9 @@ environment:
 
 install:
   # We need wheel installed to build wheels
+  - "%PYTHON%\\python.exe -m pip install -U pip"
   - "%PYTHON%\\python.exe -m pip install -U cython"
-  - "%PYTHON%\\Scripts\\cython --cplus msgpack/_packer.pyx msgpack/_unpacker.pyx"
+  - "%PYTHON%\\Scripts\\cython --cplus msgpack/_cmsgpack.pyx"
 
 build: off
 
diff --git a/docker/runtests.sh b/docker/runtests.sh
index f6c2c68..c6bbf60 100755
--- a/docker/runtests.sh
+++ b/docker/runtests.sh
@@ -8,7 +8,7 @@ for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do
     $PYBIN/pip install pytest
     pushd test          # prevent importing msgpack package in current directory.
     $PYBIN/python -c 'import sys; print(hex(sys.maxsize))'
-    $PYBIN/python -c 'from msgpack import _msgpack'  # Ensure extension is available
+    $PYBIN/python -c 'from msgpack import _cmsgpack'  # Ensure extension is available
     $PYBIN/pytest -v .
     popd
 done
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 7c5d4c0..7493c4c 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -22,7 +22,7 @@ if os.environ.get('MSGPACK_PUREPYTHON'):
     from msgpack.fallback import Packer, unpackb, Unpacker
 else:
     try:
-        from msgpack._msgpack import Packer, unpackb, Unpacker
+        from msgpack._cmsgpack import Packer, unpackb, Unpacker
     except ImportError:
         from msgpack.fallback import Packer, unpackb, Unpacker
 
diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx
new file mode 100644
index 0000000..a48d5b5
--- /dev/null
+++ b/msgpack/_cmsgpack.pyx
@@ -0,0 +1,4 @@
+# coding: utf-8
+#cython: embedsignature=True, c_string_encoding=ascii, language_level=2
+include "_packer.pyx"
+include "_unpacker.pyx"
diff --git a/msgpack/_msgpack.pyx b/msgpack/_msgpack.pyx
deleted file mode 100644
index 4381394..0000000
--- a/msgpack/_msgpack.pyx
+++ /dev/null
@@ -1,4 +0,0 @@
-# coding: utf-8
-#cython: embedsignature=True, c_string_encoding=ascii
-include "_packer.pyx"
-include "_unpacker.pyx"
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 2643f85..3be593f 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -1,9 +1,7 @@
 # coding: utf-8
-#cython: embedsignature=True, c_string_encoding=ascii
 
 from cpython cimport *
-from cpython.version cimport PY_MAJOR_VERSION
-from cpython.exc cimport PyErr_WarnEx
+from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact
 
 from msgpack import ExtType
 
@@ -11,8 +9,6 @@ from msgpack import ExtType
 cdef extern from "Python.h":
 
     int PyMemoryView_Check(object obj)
-    int PyByteArray_Check(object obj)
-    int PyByteArray_CheckExact(object obj)
     char* PyUnicode_AsUTF8AndSize(object obj, Py_ssize_t *l) except NULL
 
 
@@ -204,7 +200,7 @@ cdef class Packer(object):
             elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o):
                 L = len(o)
                 if L > ITEM_LIMIT:
-                    raise ValueError("%s is too large" % type(o).__name__)
+                    PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name)
                 rawval = o
                 ret = msgpack_pack_bin(&self.pk, L)
                 if ret == 0:
@@ -280,7 +276,7 @@ cdef class Packer(object):
                 default_used = 1
                 continue
             else:
-                raise TypeError("can't serialize %r" % (o,))
+                PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name)
             return ret
 
     cpdef pack(self, object obj):
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index e168587..aeebe2a 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -1,26 +1,6 @@
 # coding: utf-8
-#cython: embedsignature=True, c_string_encoding=ascii
 
-from cpython.version cimport PY_MAJOR_VERSION
-from cpython.bytes cimport (
-    PyBytes_AsString,
-    PyBytes_FromStringAndSize,
-    PyBytes_Size,
-)
-from cpython.buffer cimport (
-    Py_buffer,
-    PyObject_CheckBuffer,
-    PyObject_GetBuffer,
-    PyBuffer_Release,
-    PyBuffer_IsContiguous,
-    PyBUF_READ,
-    PyBUF_SIMPLE,
-    PyBUF_FULL_RO,
-)
-from cpython.mem cimport PyMem_Malloc, PyMem_Free
-from cpython.object cimport PyCallable_Check
-from cpython.ref cimport Py_DECREF
-from cpython.exc cimport PyErr_WarnEx
+from cpython cimport *
 
 cdef extern from "Python.h":
     ctypedef struct PyObject
diff --git a/setup.py b/setup.py
index 8b8f7bd..eb9403f 100755
--- a/setup.py
+++ b/setup.py
@@ -68,7 +68,7 @@ if len(version) > 3 and version[3] != 'final':
 if have_cython:
     class Sdist(sdist):
         def __init__(self, *args, **kwargs):
-            cythonize('msgpack/_msgpack.pyx')
+            cythonize('msgpack/_cmsgpack.pyx')
             sdist.__init__(self, *args, **kwargs)
 else:
     Sdist = sdist
@@ -84,8 +84,8 @@ else:
 
 ext_modules = []
 if not hasattr(sys, 'pypy_version_info'):
-    ext_modules.append(Extension('msgpack._msgpack',
-                                 sources=['msgpack/_msgpack.cpp'],
+    ext_modules.append(Extension('msgpack._cmsgpack',
+                                 sources=['msgpack/_cmsgpack.cpp'],
                                  libraries=libraries,
                                  include_dirs=['.'],
                                  define_macros=macros,

From 2f808b6e012bdc506ca83a33c2f53af1b255a069 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 14 Nov 2018 20:04:22 +0900
Subject: [PATCH 1443/1648] Try language_level=3 (#329)

---
 msgpack/_cmsgpack.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx
index a48d5b5..8ebdbf5 100644
--- a/msgpack/_cmsgpack.pyx
+++ b/msgpack/_cmsgpack.pyx
@@ -1,4 +1,4 @@
 # coding: utf-8
-#cython: embedsignature=True, c_string_encoding=ascii, language_level=2
+#cython: embedsignature=True, c_string_encoding=ascii, language_level=3
 include "_packer.pyx"
 include "_unpacker.pyx"

From 8b6ce53cce40e528af7cce89f358f7dde1a09289 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Wed, 14 Nov 2018 21:06:16 +0900
Subject: [PATCH 1444/1648] s/iteritems/items/g (#330)

---
 msgpack/_packer.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 3be593f..bfde043 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -226,7 +226,7 @@ cdef class Packer(object):
                     raise ValueError("dict is too large")
                 ret = msgpack_pack_map(&self.pk, L)
                 if ret == 0:
-                    for k, v in d.iteritems():
+                    for k, v in d.items():
                         ret = self._pack(k, nest_limit-1)
                         if ret != 0: break
                         ret = self._pack(v, nest_limit-1)

From 44254dd35e8aa3cfd6706e14effab117d7f22c25 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 20 Nov 2018 13:12:49 +0900
Subject: [PATCH 1445/1648] Add StackError and FormatError (#331)

---
 ChangeLog.rst             | 22 ++++++++++++--------
 Makefile                  |  3 ++-
 msgpack/_unpacker.pyx     | 26 +++++++++++++++++++++--
 msgpack/exceptions.py     | 12 ++++++++++-
 msgpack/fallback.py       | 43 ++++++++++++++++++++++++++++++++++-----
 msgpack/unpack_template.h | 29 +++++---------------------
 test/test_except.py       | 33 ++++++++++++++++++++++++------
 7 files changed, 120 insertions(+), 48 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index cc6b5e4..d39e76b 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -5,21 +5,25 @@ Release Date: TBD
 
 
 Important changes
-------------------
-
-Extension modules are merged.  There is ``msgpack._msgpack`` instead of
-``msgpack._packer`` and ``msgpack._unpacker``. (#314)
-
-unpacker: Default size limits is smaller than before to avoid DoS attack.
-If you need to handle large data, you need to specify limits manually.
+-----------------
 
+* unpacker: Default size limits is smaller than before to avoid DoS attack.
+  If you need to handle large data, you need to specify limits manually. (#319)
 
 
 Other changes
---------------
+-------------
 
-Add ``Unpacker.getbuffer()`` method.
+* Extension modules are merged.  There is ``msgpack._msgpack`` instead of
+  ``msgpack._packer`` and ``msgpack._unpacker``. (#314)
 
+* Add ``Unpacker.getbuffer()`` method. (#320)
+
+* unpacker: ``msgpack.StackError`` is raised when input data contains too
+  nested data. (#331)
+
+* unpacker: ``msgpack.FormatError`` is raised when input data is not valid
+  msgpack format. (#331)
 
 
 0.5.6
diff --git a/Makefile b/Makefile
index b65aa85..5828ed4 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,8 @@ cython:
 	cython --cplus msgpack/_cmsgpack.pyx
 
 .PHONY: test
-test:
+test: cython
+	pip install -e .
 	pytest -v test
 	MSGPACK_PUREPYTHON=1 pytest -v test
 
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index aeebe2a..69330d3 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -16,6 +16,8 @@ from msgpack.exceptions import (
     BufferFull,
     OutOfData,
     ExtraData,
+    FormatError,
+    StackError,
 )
 from msgpack import ExtType
 
@@ -149,7 +151,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     """
     Unpack packed_bytes to object. Returns an unpacked object.
 
-    Raises `ValueError` when `packed` contains extra bytes.
+    Raises ``ExtraData`` when *packed* contains extra bytes.
+    Raises ``ValueError`` when *packed* is incomplete.
+    Raises ``FormatError`` when *packed* is not valid msgpack.
+    Raises ``StackError`` when *packed* contains too nested.
+    Other exceptions can be raised during unpacking.
 
     See :class:`Unpacker` for options.
     """
@@ -187,6 +193,12 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
             raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off))
         return obj
     unpack_clear(&ctx)
+    if ret == 0:
+        raise ValueError("Unpack failed: incomplete input")
+    elif ret == -2:
+        raise FormatError
+    elif ret == -3:
+        raise StackError
     raise ValueError("Unpack failed: error = %d" % (ret,))
 
 
@@ -201,7 +213,7 @@ def unpack(object stream, **kwargs):
 cdef class Unpacker(object):
     """Streaming unpacker.
 
-    arguments:
+    Arguments:
 
     :param file_like:
         File-like object having `.read(n)` method.
@@ -279,6 +291,12 @@ cdef class Unpacker(object):
             unpacker.feed(buf)
             for o in unpacker:
                 process(o)
+
+    Raises ``ExtraData`` when *packed* contains extra bytes.
+    Raises ``OutOfData`` when *packed* is incomplete.
+    Raises ``FormatError`` when *packed* is not valid msgpack.
+    Raises ``StackError`` when *packed* contains too nested.
+    Other exceptions can be raised during unpacking.
     """
     cdef unpack_context ctx
     cdef char* buf
@@ -451,6 +469,10 @@ cdef class Unpacker(object):
                     raise StopIteration("No more data to unpack.")
                 else:
                     raise OutOfData("No more data to unpack.")
+            elif ret == -2:
+                raise FormatError
+            elif ret == -3:
+                raise StackError
             else:
                 raise ValueError("Unpack failed: error = %d" % (ret,))
 
diff --git a/msgpack/exceptions.py b/msgpack/exceptions.py
index 5bee5b2..d6d2615 100644
--- a/msgpack/exceptions.py
+++ b/msgpack/exceptions.py
@@ -6,6 +6,7 @@ class UnpackException(Exception):
     Exception instead.
     """
 
+
 class BufferFull(UnpackException):
     pass
 
@@ -14,6 +15,14 @@ class OutOfData(UnpackException):
     pass
 
 
+class FormatError(ValueError, UnpackException):
+    """Invalid msgpack format"""
+
+
+class StackError(ValueError, UnpackException):
+    """Too nested"""
+
+
 # Deprecated.  Use ValueError instead
 UnpackValueError = ValueError
 
@@ -24,6 +33,7 @@ class ExtraData(UnpackValueError):
     This exception is raised while only one-shot (not streaming)
     unpack.
     """
+
     def __init__(self, unpacked, extra):
         self.unpacked = unpacked
         self.extra = extra
@@ -32,7 +42,7 @@ class ExtraData(UnpackValueError):
         return "unpack(b) received extra data."
 
 
-#Deprecated.  Use Exception instead to catch all exception during packing.
+# Deprecated.  Use Exception instead to catch all exception during packing.
 PackException = Exception
 PackValueError = ValueError
 PackOverflowError = OverflowError
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 04fb5b9..9c767a7 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -18,6 +18,16 @@ else:
     def dict_iteritems(d):
         return d.iteritems()
 
+if sys.version_info < (3, 5):
+    # Ugly hack...
+    RecursionError = RuntimeError
+
+    def _is_recursionerror(e):
+        return len(e.args) == 1 and isinstance(e.args[0], str) and \
+            e.args[0].startswith('maximum recursion depth exceeded')
+else:
+    def _is_recursionerror(e):
+        return True
 
 if hasattr(sys, 'pypy_version_info'):
     # cStringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
@@ -52,7 +62,10 @@ else:
 from msgpack.exceptions import (
     BufferFull,
     OutOfData,
-    ExtraData)
+    ExtraData,
+    FormatError,
+    StackError,
+)
 
 from msgpack import ExtType
 
@@ -109,7 +122,12 @@ def unpackb(packed, **kwargs):
     """
     Unpack an object from `packed`.
 
-    Raises `ExtraData` when `packed` contains extra bytes.
+    Raises ``ExtraData`` when *packed* contains extra bytes.
+    Raises ``ValueError`` when *packed* is incomplete.
+    Raises ``FormatError`` when *packed* is not valid msgpack.
+    Raises ``StackError`` when *packed* contains too nested.
+    Other exceptions can be raised during unpacking.
+
     See :class:`Unpacker` for options.
     """
     unpacker = Unpacker(None, **kwargs)
@@ -117,7 +135,11 @@ def unpackb(packed, **kwargs):
     try:
         ret = unpacker._unpack()
     except OutOfData:
-        raise ValueError("Data is not enough.")
+        raise ValueError("Unpack failed: incomplete input")
+    except RecursionError as e:
+        if _is_recursionerror(e):
+            raise StackError
+        raise
     if unpacker._got_extradata():
         raise ExtraData(ret, unpacker._get_extradata())
     return ret
@@ -211,6 +233,12 @@ class Unpacker(object):
             unpacker.feed(buf)
             for o in unpacker:
                 process(o)
+
+    Raises ``ExtraData`` when *packed* contains extra bytes.
+    Raises ``OutOfData`` when *packed* is incomplete.
+    Raises ``FormatError`` when *packed* is not valid msgpack.
+    Raises ``StackError`` when *packed* contains too nested.
+    Other exceptions can be raised during unpacking.
     """
 
     def __init__(self, file_like=None, read_size=0, use_list=True, raw=True,
@@ -561,7 +589,7 @@ class Unpacker(object):
                 raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
         else:
-            raise ValueError("Unknown header: 0x%x" % b)
+            raise FormatError("Unknown header: 0x%x" % b)
         return typ, n, obj
 
     def _unpack(self, execute=EX_CONSTRUCT):
@@ -637,6 +665,8 @@ class Unpacker(object):
         except OutOfData:
             self._consume()
             raise StopIteration
+        except RecursionError:
+            raise StackError
 
     next = __next__
 
@@ -645,7 +675,10 @@ class Unpacker(object):
         self._consume()
 
     def unpack(self):
-        ret = self._unpack(EX_CONSTRUCT)
+        try:
+            ret = self._unpack(EX_CONSTRUCT)
+        except RecursionError:
+            raise StackError
         self._consume()
         return ret
 
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 525dea2..a78b7fa 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -123,7 +123,7 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
     goto _fixed_trail_again
 
 #define start_container(func, count_, ct_) \
-    if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
+    if(top >= MSGPACK_EMBED_STACK_SIZE) { ret = -3; goto _end; } \
     if(construct_cb(func)(user, count_, &stack[top].obj) < 0) { goto _failed; } \
     if((count_) == 0) { obj = stack[top].obj; \
         if (construct_cb(func##_end)(user, &obj) < 0) { goto _failed; } \
@@ -132,27 +132,6 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
     stack[top].size  = count_; \
     stack[top].count = 0; \
     ++top; \
-    /*printf("container %d count %d stack %d\n",stack[top].obj,count_,top);*/ \
-    /*printf("stack push %d\n", top);*/ \
-    /* FIXME \
-    if(top >= stack_size) { \
-        if(stack_size == MSGPACK_EMBED_STACK_SIZE) { \
-            size_t csize = sizeof(unpack_stack) * MSGPACK_EMBED_STACK_SIZE; \
-            size_t nsize = csize * 2; \
-            unpack_stack* tmp = (unpack_stack*)malloc(nsize); \
-            if(tmp == NULL) { goto _failed; } \
-            memcpy(tmp, ctx->stack, csize); \
-            ctx->stack = stack = tmp; \
-            ctx->stack_size = stack_size = MSGPACK_EMBED_STACK_SIZE * 2; \
-        } else { \
-            size_t nsize = sizeof(unpack_stack) * ctx->stack_size * 2; \
-            unpack_stack* tmp = (unpack_stack*)realloc(ctx->stack, nsize); \
-            if(tmp == NULL) { goto _failed; } \
-            ctx->stack = stack = tmp; \
-            ctx->stack_size = stack_size = stack_size * 2; \
-        } \
-    } \
-    */ \
     goto _header_again
 
 #define NEXT_CS(p)  ((unsigned int)*p & 0x1f)
@@ -229,7 +208,8 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
                 case 0xdf:  // map 32
                     again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
                 default:
-                    goto _failed;
+                    ret = -2;
+                    goto _end;
                 }
             SWITCH_RANGE(0xa0, 0xbf)  // FixRaw
                 again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
@@ -239,7 +219,8 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
                 start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
 
             SWITCH_RANGE_DEFAULT
-                goto _failed;
+                ret = -2;
+                goto _end;
             SWITCH_RANGE_END
             // end CS_HEADER
 
diff --git a/test/test_except.py b/test/test_except.py
index 361d4ea..626c8be 100644
--- a/test/test_except.py
+++ b/test/test_except.py
@@ -2,7 +2,7 @@
 # coding: utf-8
 
 from pytest import raises
-from msgpack import packb, unpackb
+from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData
 
 import datetime
 
@@ -19,13 +19,34 @@ def test_raise_on_find_unsupported_value():
 def test_raise_from_object_hook():
     def hook(obj):
         raise DummyException
+
     raises(DummyException, unpackb, packb({}), object_hook=hook)
-    raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_hook=hook)
-    raises(DummyException, unpackb, packb({'fizz': 'buzz'}), object_pairs_hook=hook)
-    raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_hook=hook)
-    raises(DummyException, unpackb, packb({'fizz': {'buzz': 'spam'}}), object_pairs_hook=hook)
+    raises(DummyException, unpackb, packb({"fizz": "buzz"}), object_hook=hook)
+    raises(DummyException, unpackb, packb({"fizz": "buzz"}), object_pairs_hook=hook)
+    raises(DummyException, unpackb, packb({"fizz": {"buzz": "spam"}}), object_hook=hook)
+    raises(
+        DummyException,
+        unpackb,
+        packb({"fizz": {"buzz": "spam"}}),
+        object_pairs_hook=hook,
+    )
 
 
 def test_invalidvalue():
+    incomplete = b"\xd9\x97#DL_"  # raw8 - length=0x97
     with raises(ValueError):
-        unpackb(b'\xd9\x97#DL_')
+        unpackb(incomplete)
+
+    with raises(OutOfData):
+        unpacker = Unpacker()
+        unpacker.feed(incomplete)
+        unpacker.unpack()
+
+    with raises(FormatError):
+        unpackb(b"\xc1")  # (undefined tag)
+
+    with raises(FormatError):
+        unpackb(b"\x91\xc1")  # fixarray(len=1) [ (undefined tag) ]
+
+    with raises(StackError):
+        unpackb(b"\x91" * 3000)  # nested fixarray(len=1)

From ab2415eaa0cdbe8e5b6e248d447cf5e66e858eb2 Mon Sep 17 00:00:00 2001
From: jkorvin 
Date: Tue, 20 Nov 2018 09:24:35 +0300
Subject: [PATCH 1446/1648] Unpacker: allow to use buffer with size greater
 than 2 GB (#332)

---
 msgpack/_unpacker.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 69330d3..a5403d8 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -320,7 +320,7 @@ cdef class Unpacker(object):
     def __init__(self, file_like=None, Py_ssize_t read_size=0,
                  bint use_list=True, bint raw=True,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
-                 encoding=None, unicode_errors=None, int max_buffer_size=0,
+                 encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0,
                  object ext_hook=ExtType,
                  Py_ssize_t max_str_len=1024*1024,
                  Py_ssize_t max_bin_len=1024*1024,

From 3c9c6edbc88908fceb3c69ff3d6455be8b5914c8 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Tue, 20 Nov 2018 15:48:44 +0900
Subject: [PATCH 1447/1648] Update README

---
 ChangeLog.rst | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index d39e76b..8bc80e6 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -7,15 +7,21 @@ Release Date: TBD
 Important changes
 -----------------
 
-* unpacker: Default size limits is smaller than before to avoid DoS attack.
+* unpacker: Default value of input limits are smaller than before to avoid DoS attack.
   If you need to handle large data, you need to specify limits manually. (#319)
 
+* Unpacker doesn't wrap underlaying ``ValueError`` (including ``UnicodeError``) into
+  ``UnpackValueError``.  If you want to catch all exception during unpack, you need
+  to use ``try ... except Exception`` with minimum try code block. (#323, #233)
+
+* ``PackValueError`` and ``PackOverflowError`` are also removed.  You need to catch
+  normal ``ValueError`` and ``OverflowError``. (#323, #233)
 
 Other changes
 -------------
 
-* Extension modules are merged.  There is ``msgpack._msgpack`` instead of
-  ``msgpack._packer`` and ``msgpack._unpacker``. (#314)
+* Extension modules are merged.  There is ``msgpack._cmsgpack`` instead of
+  ``msgpack._packer`` and ``msgpack._unpacker``. (#314, #328)
 
 * Add ``Unpacker.getbuffer()`` method. (#320)
 

From e9086a34e4b3d64df78314339f152c800e79c8e1 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 29 Nov 2018 22:29:38 +0900
Subject: [PATCH 1448/1648] Add strict_map_key option to unpacker

---
 msgpack/_unpacker.pyx | 17 ++++++++++++-----
 msgpack/unpack.h      |  5 +++++
 test/test_except.py   | 11 +++++++++++
 3 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index a5403d8..2163425 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -27,6 +27,7 @@ cdef extern from "unpack.h":
         bint use_list
         bint raw
         bint has_pairs_hook # call object_hook with k-v pairs
+        bint strict_map_key
         PyObject* object_hook
         PyObject* list_hook
         PyObject* ext_hook
@@ -56,7 +57,7 @@ cdef extern from "unpack.h":
 cdef inline init_ctx(unpack_context *ctx,
                      object object_hook, object object_pairs_hook,
                      object list_hook, object ext_hook,
-                     bint use_list, bint raw,
+                     bint use_list, bint raw, bint strict_map_key,
                      const char* encoding, const char* unicode_errors,
                      Py_ssize_t max_str_len, Py_ssize_t max_bin_len,
                      Py_ssize_t max_array_len, Py_ssize_t max_map_len,
@@ -64,6 +65,7 @@ cdef inline init_ctx(unpack_context *ctx,
     unpack_init(ctx)
     ctx.user.use_list = use_list
     ctx.user.raw = raw
+    ctx.user.strict_map_key = strict_map_key
     ctx.user.object_hook = ctx.user.list_hook = NULL
     ctx.user.max_str_len = max_str_len
     ctx.user.max_bin_len = max_bin_len
@@ -140,7 +142,7 @@ cdef inline int get_data_from_buffer(object obj,
         return 1
 
 def unpackb(object packed, object object_hook=None, object list_hook=None,
-            bint use_list=True, bint raw=True,
+            bint use_list=True, bint raw=True, bint strict_map_key=False,
             encoding=None, unicode_errors=None,
             object_pairs_hook=None, ext_hook=ExtType,
             Py_ssize_t max_str_len=1024*1024,
@@ -180,7 +182,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol)
     try:
         init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
-                 use_list, raw, cenc, cerr,
+                 use_list, raw, strict_map_key, cenc, cerr,
                  max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
         ret = unpack_construct(&ctx, buf, buf_len, &off)
     finally:
@@ -236,6 +238,11 @@ cdef class Unpacker(object):
 
         *encoding* option which is deprecated overrides this option.
 
+    :param bool strict_map_key:
+        If true, only str or bytes are accepted for map (dict) keys.
+        It's False by default for backward-compatibility.
+        But it will be True from msgpack 1.0.
+
     :param callable object_hook:
         When specified, it should be callable.
         Unpacker calls it with a dict argument after unpacking msgpack map.
@@ -318,7 +325,7 @@ cdef class Unpacker(object):
         self.buf = NULL
 
     def __init__(self, file_like=None, Py_ssize_t read_size=0,
-                 bint use_list=True, bint raw=True,
+                 bint use_list=True, bint raw=True, bint strict_map_key=False,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0,
                  object ext_hook=ExtType,
@@ -366,7 +373,7 @@ cdef class Unpacker(object):
             cerr = unicode_errors
 
         init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
-                 ext_hook, use_list, raw, cenc, cerr,
+                 ext_hook, use_list, raw, strict_map_key, cenc, cerr,
                  max_str_len, max_bin_len, max_array_len,
                  max_map_len, max_ext_len)
 
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 63e5543..85dbbed 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -23,6 +23,7 @@ typedef struct unpack_user {
     bool use_list;
     bool raw;
     bool has_pairs_hook;
+    bool strict_map_key;
     PyObject *object_hook;
     PyObject *list_hook;
     PyObject *ext_hook;
@@ -188,6 +189,10 @@ static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_un
 
 static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
 {
+    if (u->strict_map_key && !PyUnicode_CheckExact(k) && !PyBytes_CheckExact(k)) {
+        PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key", Py_TYPE(k)->tp_name);
+        return -1;
+    }
     if (u->has_pairs_hook) {
         msgpack_unpack_object item = PyTuple_Pack(2, k, v);
         if (!item)
diff --git a/test/test_except.py b/test/test_except.py
index 626c8be..40ca3ee 100644
--- a/test/test_except.py
+++ b/test/test_except.py
@@ -50,3 +50,14 @@ def test_invalidvalue():
 
     with raises(StackError):
         unpackb(b"\x91" * 3000)  # nested fixarray(len=1)
+
+
+def test_strict_map_key():
+    valid = {u"unicode": 1, b"bytes": 2}
+    packed = packb(valid, use_bin_type=True)
+    assert valid == unpackb(packed, raw=True)
+
+    invalid = {42: 1}
+    packed = packb(invalid, use_bin_type=True)
+    with raises(ValueError):
+        unpackb(packed, raw=True)

From dc1b9930793ea57aa4e2a9773a23582b5483c53a Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 29 Nov 2018 22:35:12 +0900
Subject: [PATCH 1449/1648] Implement strict_map_key to fallback unpacker.

---
 msgpack/fallback.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 9c767a7..be60cca 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -179,6 +179,11 @@ class Unpacker(object):
 
         *encoding* option which is deprecated overrides this option.
 
+    :param bool strict_map_key:
+        If true, only str or bytes are accepted for map (dict) keys.
+        It's False by default for backward-compatibility.
+        But it will be True from msgpack 1.0.
+
     :param callable object_hook:
         When specified, it should be callable.
         Unpacker calls it with a dict argument after unpacking msgpack map.
@@ -241,7 +246,7 @@ class Unpacker(object):
     Other exceptions can be raised during unpacking.
     """
 
-    def __init__(self, file_like=None, read_size=0, use_list=True, raw=True,
+    def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, strict_map_key=False,
                  object_hook=None, object_pairs_hook=None, list_hook=None,
                  encoding=None, unicode_errors=None, max_buffer_size=0,
                  ext_hook=ExtType,
@@ -286,6 +291,7 @@ class Unpacker(object):
             raise ValueError("read_size must be smaller than max_buffer_size")
         self._read_size = read_size or min(self._max_buffer_size, 16*1024)
         self._raw = bool(raw)
+        self._strict_map_key = bool(strict_map_key)
         self._encoding = encoding
         self._unicode_errors = unicode_errors
         self._use_list = use_list
@@ -633,6 +639,8 @@ class Unpacker(object):
                 ret = {}
                 for _ in xrange(n):
                     key = self._unpack(EX_CONSTRUCT)
+                    if self._strict_map_key and type(key) not in (unicode, bytes):
+                        raise ValueError("%s is not allowed for map key" % str(type(key)))
                     ret[key] = self._unpack(EX_CONSTRUCT)
                 if self._object_hook is not None:
                     ret = self._object_hook(ret)

From e76091a82c04d092a363b0e1f7307dc70bf784f5 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 29 Nov 2018 22:38:22 +0900
Subject: [PATCH 1450/1648] Fix test

---
 test/test_except.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_except.py b/test/test_except.py
index 40ca3ee..01961d2 100644
--- a/test/test_except.py
+++ b/test/test_except.py
@@ -55,9 +55,9 @@ def test_invalidvalue():
 def test_strict_map_key():
     valid = {u"unicode": 1, b"bytes": 2}
     packed = packb(valid, use_bin_type=True)
-    assert valid == unpackb(packed, raw=True)
+    assert valid == unpackb(packed, raw=True, strict_map_key=True)
 
     invalid = {42: 1}
     packed = packb(invalid, use_bin_type=True)
     with raises(ValueError):
-        unpackb(packed, raw=True)
+        unpackb(packed, raw=True, strict_map_key=True)

From ab789813b8e5786f404e9132ff1c6f2647dc4afa Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 30 Nov 2018 11:36:15 +0900
Subject: [PATCH 1451/1648] Fix test

---
 test/test_except.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_except.py b/test/test_except.py
index 01961d2..5544f2b 100644
--- a/test/test_except.py
+++ b/test/test_except.py
@@ -55,9 +55,9 @@ def test_invalidvalue():
 def test_strict_map_key():
     valid = {u"unicode": 1, b"bytes": 2}
     packed = packb(valid, use_bin_type=True)
-    assert valid == unpackb(packed, raw=True, strict_map_key=True)
+    assert valid == unpackb(packed, raw=False, strict_map_key=True)
 
     invalid = {42: 1}
     packed = packb(invalid, use_bin_type=True)
     with raises(ValueError):
-        unpackb(packed, raw=True, strict_map_key=True)
+        unpackb(packed, raw=False, strict_map_key=True)

From 8ae6320072e746fad29bc14a095569811e009695 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 30 Nov 2018 11:42:51 +0900
Subject: [PATCH 1452/1648] Fix fallback

---
 msgpack/fallback.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index be60cca..ae2fcfc 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -639,7 +639,7 @@ class Unpacker(object):
                 ret = {}
                 for _ in xrange(n):
                     key = self._unpack(EX_CONSTRUCT)
-                    if self._strict_map_key and type(key) not in (unicode, bytes):
+                    if self._strict_map_key and type(key) not in (Unicode, bytes):
                         raise ValueError("%s is not allowed for map key" % str(type(key)))
                     ret[key] = self._unpack(EX_CONSTRUCT)
                 if self._object_hook is not None:

From 04cf8fc7f4b9e8dd32d809cd2c45b05b83d7f913 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 30 Nov 2018 14:04:18 +0900
Subject: [PATCH 1453/1648] Update ChangeLog

---
 ChangeLog.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 8bc80e6..806007e 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -17,6 +17,11 @@ Important changes
 * ``PackValueError`` and ``PackOverflowError`` are also removed.  You need to catch
   normal ``ValueError`` and ``OverflowError``. (#323, #233)
 
+* Unpacker has ``strict_map_key`` option now.  When it is true, only bytes and str
+  (unicode in Python 2) are allowed for map keys.  It is recommended to avoid
+  hashdos.  Default value of this option is False for backward compatibility reason.
+  But it will be changed True in 1.0. (#296, #334)
+
 Other changes
 -------------
 

From 93b5953eae11fa3a8668de16a3ccbf20d7bf0fd9 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 30 Nov 2018 16:05:31 +0900
Subject: [PATCH 1454/1648] Update tox.ini

---
 tox.ini | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index 68a2f53..0945a6d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {py27,py35,py36}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
+envlist = {py27,py35,py36,py37}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
 
 [variants:pure]
 setenv=
@@ -11,7 +11,7 @@ deps=
 
 changedir=test
 commands=
-    c,x86: python -c 'from msgpack import _packer, _unpacker'
+    c,x86: python -c 'from msgpack import _cmsgpack'
     c,x86: py.test
     pure: py.test
 
@@ -23,7 +23,7 @@ deps=
 changedir=test
 commands=
     python -c 'import sys; print(hex(sys.maxsize))'
-    python -c 'from msgpack import _packer, _unpacker'
+    python -c 'from msgpack import _cmsgpack'
     py.test
 
 [testenv:py34-x86]
@@ -34,5 +34,5 @@ deps=
 changedir=test
 commands=
     python -c 'import sys; print(hex(sys.maxsize))'
-    python -c 'from msgpack import _packer, _unpacker'
+    python -c 'from msgpack import _cmsgpack'
     py.test

From bbdfd4d92e54e89604a2ebf6af86ced4ae5ae05d Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 30 Nov 2018 16:28:41 +0900
Subject: [PATCH 1455/1648] cleanup

---
 msgpack/fallback.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index ae2fcfc..57b436a 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -4,19 +4,19 @@ import sys
 import struct
 import warnings
 
-if sys.version_info[0] == 3:
-    PY3 = True
+
+if sys.version_info[0] == 2:
+    PY2 = True
+    int_types = (int, long)
+    def dict_iteritems(d):
+        return d.iteritems()
+else:
+    PY2 = False
     int_types = int
-    Unicode = str
+    unicode = str
     xrange = range
     def dict_iteritems(d):
         return d.items()
-else:
-    PY3 = False
-    int_types = (int, long)
-    Unicode = unicode
-    def dict_iteritems(d):
-        return d.iteritems()
 
 if sys.version_info < (3, 5):
     # Ugly hack...
@@ -97,7 +97,7 @@ def _get_data_from_buffer(obj):
         view = memoryview(obj)
     except TypeError:
         # try to use legacy buffer protocol if 2.7, otherwise re-raise
-        if not PY3:
+        if PY2:
             view = memoryview(buffer(obj))
             warnings.warn("using old buffer interface to unpack %s; "
                           "this leads to unpacking errors if slicing is used and "
@@ -639,7 +639,7 @@ class Unpacker(object):
                 ret = {}
                 for _ in xrange(n):
                     key = self._unpack(EX_CONSTRUCT)
-                    if self._strict_map_key and type(key) not in (Unicode, bytes):
+                    if self._strict_map_key and type(key) not in (unicode, bytes):
                         raise ValueError("%s is not allowed for map key" % str(type(key)))
                     ret[key] = self._unpack(EX_CONSTRUCT)
                 if self._object_hook is not None:
@@ -819,7 +819,7 @@ class Packer(object):
                     raise ValueError("%s is too large" % type(obj).__name__)
                 self._pack_bin_header(n)
                 return self._buffer.write(obj)
-            if check(obj, Unicode):
+            if check(obj, unicode):
                 if self._encoding is None:
                     raise TypeError(
                         "Can't encode unicode string: "
@@ -1006,7 +1006,7 @@ class Packer(object):
 
     def getbuffer(self):
         """Return view of internal buffer."""
-        if USING_STRINGBUILDER or not PY3:
+        if USING_STRINGBUILDER or PY2:
             return memoryview(self.bytes())
         else:
             return self._buffer.getbuffer()

From cc7fd5722b779d438f7d226a9c7f61115764b39c Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 30 Nov 2018 19:03:44 +0900
Subject: [PATCH 1456/1648] 0.6.0

---
 ChangeLog.rst       | 3 ++-
 msgpack/_version.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 806007e..651ba62 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,8 +1,9 @@
 0.6.0
 ======
 
-Release Date: TBD
+Release Date: 2018-11-30
 
+This release contains some backward incompatible changes for security reason (DoS).
 
 Important changes
 -----------------
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 0952ec6..49a4103 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 6, 0, 'dev')
+version = (0, 6, 0)

From b1d658e7a0aed4e723f9b6b238cc9f2f876e54d7 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 30 Nov 2018 19:25:14 +0900
Subject: [PATCH 1457/1648] AppVeyor: Add Python 3.7 and remove 3.6

---
 appveyor.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index f0e21fc..bd0800a 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -21,14 +21,14 @@ test_script:
   - ci\\runtests.bat
   - set PYTHON="C:\\Python27-x64"
   - ci\\runtests.bat
-  - set PYTHON="C:\\Python35"
-  - ci\\runtests.bat
-  - set PYTHON="C:\\Python35-x64"
-  - ci\\runtests.bat
   - set PYTHON="C:\\Python36"
   - ci\\runtests.bat
   - set PYTHON="C:\\Python36-x64"
   - ci\\runtests.bat
+  - set PYTHON="C:\\Python37"
+  - ci\\runtests.bat
+  - set PYTHON="C:\\Python37-x64"
+  - ci\\runtests.bat
 
 after_test:
   # This step builds your wheels.

From b8bf3c950c4474d3af82a6b6bda6326b2e197a5e Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 4 Dec 2018 17:18:34 +0900
Subject: [PATCH 1458/1648] Build linux wheel for Python 3.7

---
 docker/buildwheel.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh
index f586a8d..d8c74cb 100644
--- a/docker/buildwheel.sh
+++ b/docker/buildwheel.sh
@@ -4,7 +4,7 @@ set -e -x
 ARCH=`uname -p`
 echo "arch=$ARCH"
 
-for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do
+for V in cp37-cp37m cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do
     PYBIN=/opt/python/$V/bin
     rm -rf build/       # Avoid lib build by narrow Python is used by wide python
     $PYBIN/python setup.py bdist_wheel -p manylinux1_${ARCH}

From 197e30723a242e3ba31f9634b3263ae4cb2937b1 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 4 Dec 2018 20:10:21 +0900
Subject: [PATCH 1459/1648] Fix docstring

---
 msgpack/_unpacker.pyx | 2 +-
 msgpack/fallback.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 2163425..4ea0545 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -271,7 +271,7 @@ cdef class Unpacker(object):
         Limits max length of map. (default: 32*1024)
 
     :param int max_ext_len:
-        Limits max length of map. (default: 1024*1024)
+        Limits max size of ext type. (default: 1024*1024)
 
     :param str encoding:
         Deprecated, use raw instead.
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 57b436a..4567e2d 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -220,7 +220,7 @@ class Unpacker(object):
         Limits max length of map. (default: 32*1024)
 
     :param int max_ext_len:
-        Limits max length of map. (default: 1024*1024)
+        Limits max size of ext type. (default: 1024*1024)
 
     example of streaming deserialize from file-like object::
 

From f46523b1af7ff2d408da8500ea36a4f9f2abe915 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 7 Jan 2019 21:10:40 +0900
Subject: [PATCH 1460/1648] use _PyFloat APIs to (de)serialize (#340)

---
 msgpack/pack_template.h   | 13 +++----------
 msgpack/unpack_template.h | 14 ++++----------
 2 files changed, 7 insertions(+), 20 deletions(-)

diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 5d1088f..69982f4 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -566,24 +566,17 @@ if(sizeof(unsigned long long) == 2) {
 
 static inline int msgpack_pack_float(msgpack_packer* x, float d)
 {
-    union { float f; uint32_t i; } mem;
-    mem.f = d;
     unsigned char buf[5];
-    buf[0] = 0xca; _msgpack_store32(&buf[1], mem.i);
+    buf[0] = 0xca;
+    _PyFloat_Pack4(d, &buf[1], 0);
     msgpack_pack_append_buffer(x, buf, 5);
 }
 
 static inline int msgpack_pack_double(msgpack_packer* x, double d)
 {
-    union { double f; uint64_t i; } mem;
-    mem.f = d;
     unsigned char buf[9];
     buf[0] = 0xcb;
-#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
-    // https://github.com/msgpack/msgpack-perl/pull/1
-    mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
-#endif
-    _msgpack_store64(&buf[1], mem.i);
+    _PyFloat_Pack8(d, &buf[1], 0);
     msgpack_pack_append_buffer(x, buf, 9);
 }
 
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index a78b7fa..9924b9c 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -243,17 +243,11 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
                                           _msgpack_load32(uint32_t,n)+1,
                                           _ext_zero);
             case CS_FLOAT: {
-                    union { uint32_t i; float f; } mem;
-                    mem.i = _msgpack_load32(uint32_t,n);
-                    push_fixed_value(_float, mem.f); }
+                    double f = _PyFloat_Unpack4((unsigned char*)n, 0);
+                    push_fixed_value(_float, f); }
             case CS_DOUBLE: {
-                    union { uint64_t i; double f; } mem;
-                    mem.i = _msgpack_load64(uint64_t,n);
-#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
-                    // https://github.com/msgpack/msgpack-perl/pull/1
-                    mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
-#endif
-                    push_fixed_value(_double, mem.f); }
+                    double f = _PyFloat_Unpack8((unsigned char*)n, 0);
+                    push_fixed_value(_double, f); }
             case CS_UINT_8:
                 push_fixed_value(_uint8, *(uint8_t*)n);
             case CS_UINT_16:

From 28b5f46a34933cc177aca333203d1344b5e3639a Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 24 Jan 2019 18:46:39 +0900
Subject: [PATCH 1461/1648] Auto limit configuration (#342)

---
 msgpack/_unpacker.pyx | 56 +++++++++++++++++++++++++++++++------------
 msgpack/fallback.py   | 39 ++++++++++++++++++++----------
 test/test_limits.py   | 25 ++++++++++++++++++-
 3 files changed, 92 insertions(+), 28 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 4ea0545..38119c0 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -145,11 +145,11 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
             bint use_list=True, bint raw=True, bint strict_map_key=False,
             encoding=None, unicode_errors=None,
             object_pairs_hook=None, ext_hook=ExtType,
-            Py_ssize_t max_str_len=1024*1024,
-            Py_ssize_t max_bin_len=1024*1024,
-            Py_ssize_t max_array_len=128*1024,
-            Py_ssize_t max_map_len=32*1024,
-            Py_ssize_t max_ext_len=1024*1024):
+            Py_ssize_t max_str_len=-1,
+            Py_ssize_t max_bin_len=-1,
+            Py_ssize_t max_array_len=-1,
+            Py_ssize_t max_map_len=-1,
+            Py_ssize_t max_ext_len=-1):
     """
     Unpack packed_bytes to object. Returns an unpacked object.
 
@@ -160,6 +160,8 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     Other exceptions can be raised during unpacking.
 
     See :class:`Unpacker` for options.
+
+    *max_xxx_len* options are configured automatically from ``len(packed)``.
     """
     cdef unpack_context ctx
     cdef Py_ssize_t off = 0
@@ -180,6 +182,18 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
         cerr = unicode_errors
 
     get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol)
+
+    if max_str_len == -1:
+        max_str_len = buf_len
+    if max_bin_len == -1:
+        max_bin_len = buf_len
+    if max_array_len == -1:
+        max_array_len = buf_len
+    if max_map_len == -1:
+        max_map_len = buf_len//2
+    if max_ext_len == -1:
+        max_ext_len = buf_len
+
     try:
         init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
                  use_list, raw, strict_map_key, cenc, cerr,
@@ -259,19 +273,19 @@ cdef class Unpacker(object):
         You should set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
-        Limits max length of str. (default: 1024*1024)
+        Limits max length of str. (default: max_buffer_size or 1024*1024)
 
     :param int max_bin_len:
-        Limits max length of bin. (default: 1024*1024)
+        Limits max length of bin. (default: max_buffer_size or 1024*1024)
 
     :param int max_array_len:
-        Limits max length of array. (default: 128*1024)
+        Limits max length of array. (default: max_buffer_size or 128*1024)
 
     :param int max_map_len:
-        Limits max length of map. (default: 32*1024)
+        Limits max length of map. (default: max_buffer_size//2 or 32*1024)
 
     :param int max_ext_len:
-        Limits max size of ext type. (default: 1024*1024)
+        Limits max size of ext type. (default: max_buffer_size or 1024*1024)
 
     :param str encoding:
         Deprecated, use raw instead.
@@ -329,11 +343,11 @@ cdef class Unpacker(object):
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0,
                  object ext_hook=ExtType,
-                 Py_ssize_t max_str_len=1024*1024,
-                 Py_ssize_t max_bin_len=1024*1024,
-                 Py_ssize_t max_array_len=128*1024,
-                 Py_ssize_t max_map_len=32*1024,
-                 Py_ssize_t max_ext_len=1024*1024):
+                 Py_ssize_t max_str_len=-1,
+                 Py_ssize_t max_bin_len=-1,
+                 Py_ssize_t max_array_len=-1,
+                 Py_ssize_t max_map_len=-1,
+                 Py_ssize_t max_ext_len=-1):
         cdef const char *cenc=NULL,
         cdef const char *cerr=NULL
 
@@ -347,6 +361,18 @@ cdef class Unpacker(object):
             self.file_like_read = file_like.read
             if not PyCallable_Check(self.file_like_read):
                 raise TypeError("`file_like.read` must be a callable.")
+
+        if max_str_len == -1:
+            max_str_len = max_buffer_size or 1024*1024
+        if max_bin_len == -1:
+            max_bin_len = max_buffer_size or 1024*1024
+        if max_array_len == -1:
+            max_array_len = max_buffer_size or 128*1024
+        if max_map_len == -1:
+            max_map_len = max_buffer_size//2 or 32*1024
+        if max_ext_len == -1:
+            max_ext_len = max_buffer_size or 1024*1024
+
         if not max_buffer_size:
             max_buffer_size = INT_MAX
         if read_size > max_buffer_size:
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 4567e2d..7524448 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -130,7 +130,7 @@ def unpackb(packed, **kwargs):
 
     See :class:`Unpacker` for options.
     """
-    unpacker = Unpacker(None, **kwargs)
+    unpacker = Unpacker(None, max_buffer_size=len(packed), **kwargs)
     unpacker.feed(packed)
     try:
         ret = unpacker._unpack()
@@ -208,19 +208,24 @@ class Unpacker(object):
         You should set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
-        Limits max length of str. (default: 1024*1024)
+        (deprecated) Limits max length of str.
+        (default: max_buffer_size or 1024*1024)
 
     :param int max_bin_len:
-        Limits max length of bin. (default: 1024*1024)
+        (deprecated) Limits max length of bin.
+        (default: max_buffer_size or 1024*1024)
 
     :param int max_array_len:
-        Limits max length of array. (default: 128*1024)
+        Limits max length of array.
+        (default: max_buffer_size or 128*1024)
 
     :param int max_map_len:
-        Limits max length of map. (default: 32*1024)
+        Limits max length of map.
+        (default: max_buffer_size//2 or 32*1024)
 
     :param int max_ext_len:
-        Limits max size of ext type. (default: 1024*1024)
+        (deprecated) Limits max size of ext type.
+        (default: max_buffer_size or 1024*1024)
 
     example of streaming deserialize from file-like object::
 
@@ -250,12 +255,11 @@ class Unpacker(object):
                  object_hook=None, object_pairs_hook=None, list_hook=None,
                  encoding=None, unicode_errors=None, max_buffer_size=0,
                  ext_hook=ExtType,
-                 max_str_len=1024*1024,
-                 max_bin_len=1024*1024,
-                 max_array_len=128*1024,
-                 max_map_len=32*1024,
-                 max_ext_len=1024*1024):
-
+                 max_str_len=-1,
+                 max_bin_len=-1,
+                 max_array_len=-1,
+                 max_map_len=-1,
+                 max_ext_len=-1):
         if encoding is not None:
             warnings.warn(
                 "encoding is deprecated, Use raw=False instead.",
@@ -286,6 +290,17 @@ class Unpacker(object):
         # state, which _buf_checkpoint records.
         self._buf_checkpoint = 0
 
+        if max_str_len == -1:
+            max_str_len = max_buffer_size or 1024*1024
+        if max_bin_len == -1:
+            max_bin_len = max_buffer_size or 1024*1024
+        if max_array_len == -1:
+            max_array_len = max_buffer_size or 128*1024
+        if max_map_len == -1:
+            max_map_len = max_buffer_size//2 or 32*1024
+        if max_ext_len == -1:
+            max_ext_len = max_buffer_size or 1024*1024
+
         self._max_buffer_size = max_buffer_size or 2**31-1
         if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")
diff --git a/test/test_limits.py b/test/test_limits.py
index 74e48c1..8c7606f 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -105,7 +105,6 @@ def test_max_ext_len():
         unpacker.unpack()
 
 
-
 # PyPy fails following tests because of constant folding?
 # https://bugs.pypy.org/issue1721
 #@pytest.mark.skipif(True, reason="Requires very large memory.")
@@ -134,3 +133,27 @@ def test_max_ext_len():
 #    x.append(0)
 #    with pytest.raises(ValueError):
 #        packb(x)
+
+
+# auto max len
+
+def test_auto_max_array_len():
+    packed = b'\xde\x00\x06zz'
+    with pytest.raises(UnpackValueError):
+        unpackb(packed, raw=False)
+
+    unpacker = Unpacker(max_buffer_size=5, raw=False)
+    unpacker.feed(packed)
+    with pytest.raises(UnpackValueError):
+        unpacker.unpack()
+
+def test_auto_max_map_len():
+    # len(packed) == 6 -> max_map_len == 3
+    packed = b'\xde\x00\x04zzz'
+    with pytest.raises(UnpackValueError):
+        unpackb(packed, raw=False)
+
+    unpacker = Unpacker(max_buffer_size=6, raw=False)
+    unpacker.feed(packed)
+    with pytest.raises(UnpackValueError):
+        unpacker.unpack()

From 464fe277e1165a5870d4edc040be9c9ac1c1df0c Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 25 Jan 2019 20:52:57 +0900
Subject: [PATCH 1462/1648] Remove pytest warnings

---
 test/test_buffer.py    |  5 ++++-
 test/test_extension.py | 10 ++++++++--
 test/test_pack.py      | 23 +++++++++++++++--------
 3 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/test/test_buffer.py b/test/test_buffer.py
index 87f359f..d723e8d 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -7,7 +7,10 @@ from msgpack import packb, unpackb
 def test_unpack_buffer():
     from array import array
     buf = array('b')
-    buf.fromstring(packb((b'foo', b'bar')))
+    try:
+        buf.frombytes(packb((b'foo', b'bar')))
+    except AttributeError:  # PY2
+        buf.fromstring(packb((b'foo', b'bar')))
     obj = unpackb(buf, use_list=1)
     assert [b'foo', b'bar'] == obj
 
diff --git a/test/test_extension.py b/test/test_extension.py
index d05d7ab..8aa0cbb 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -40,7 +40,10 @@ def test_extension_type():
         print('default called', obj)
         if isinstance(obj, array.array):
             typecode = 123 # application specific typecode
-            data = obj.tostring()
+            try:
+                data = obj.tobytes()
+            except AttributeError:
+                data = obj.tostring()
             return ExtType(typecode, data)
         raise TypeError("Unknown type object %r" % (obj,))
 
@@ -48,7 +51,10 @@ def test_extension_type():
         print('ext_hook called', code, data)
         assert code == 123
         obj = array.array('d')
-        obj.fromstring(data)
+        try:
+            obj.frombytes(data)
+        except AttributeError:  # PY2
+            obj.fromstring(data)
         return obj
 
     obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])]
diff --git a/test/test_pack.py b/test/test_pack.py
index 4608083..3658a97 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -2,13 +2,15 @@
 # coding: utf-8
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+from collections import OrderedDict
+from io import BytesIO
 import struct
+
+import pytest
 from pytest import raises, xfail
 
 from msgpack import packb, unpackb, Unpacker, Packer, pack
 
-from collections import OrderedDict
-from io import BytesIO
 
 def check(data, use_list=False):
     re = unpackb(packb(data), use_list=use_list)
@@ -47,7 +49,8 @@ def testPackUTF32():  # deprecated
             "РуÑÑкий текÑÑ‚",
             ]
         for td in test_data:
-            re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
+            with pytest.deprecated_call():
+                re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
             assert re == td
     except LookupError as e:
         xfail(e)
@@ -67,19 +70,23 @@ def testPackByteArrays():
         check(td)
 
 def testIgnoreUnicodeErrors(): # deprecated
-    re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
+    with pytest.deprecated_call():
+        re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
     assert re == "abcdef"
 
 def testStrictUnicodeUnpack():
-    with raises(UnicodeDecodeError):
-        unpackb(packb(b'abc\xeddef'), raw=False, use_list=1)
+    packed = packb(b'abc\xeddef')
+    with pytest.raises(UnicodeDecodeError):
+        unpackb(packed, raw=False, use_list=1)
 
 def testStrictUnicodePack():  # deprecated
     with raises(UnicodeEncodeError):
-        packb("abc\xeddef", encoding='ascii', unicode_errors='strict')
+        with pytest.deprecated_call():
+            packb("abc\xeddef", encoding='ascii', unicode_errors='strict')
 
 def testIgnoreErrorsPack():  # deprecated
-    re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1)
+    with pytest.deprecated_call():
+        re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1)
     assert re == "abcdef"
 
 def testDecodeBinary():

From 9951b894555e4f9c7120375028e686f7420de92a Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 25 Jan 2019 21:04:14 +0900
Subject: [PATCH 1463/1648] travis: Install new pytest

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 1adbdc2..51917c5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -39,7 +39,7 @@ matrix:
 
 install:
   - pip install -U pip
-  - pip install cython
+  - pip install -U cython pytest
   - make cython
   - pip install -e .
 

From 280308e8ced50322414fd4f7426d56093a57dbf1 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 25 Jan 2019 21:27:46 +0900
Subject: [PATCH 1464/1648] Recommend max_buffer_len instead of
 max_(str|bin|ext)_len

---
 ChangeLog.rst         | 18 ++++++++++++++++++
 msgpack/_unpacker.pyx |  9 ++++++---
 msgpack/fallback.py   | 20 ++++++++++----------
 3 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 651ba62..2c988db 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,21 @@
+0.6.1
+======
+
+Release Date: 2019-01-25
+
+This release is for mitigating pain caused by v0.6.1 reduced max input limits
+for security reason.
+
+* ``unpackb(data)`` configures ``max_*_len`` options from ``len(data)``,
+  instead of static default sizes.
+
+* ``Unpacker(max_buffer_len=N)`` configures ``max_*_len`` options from ``N``,
+  instead of static default sizes.
+
+* ``max_bin_len``, ``max_str_len``, and ``max_ext_len`` are deprecated.
+  Since this is minor release, it's document only deprecation.
+
+
 0.6.0
 ======
 
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 38119c0..3c6d59e 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -273,9 +273,11 @@ cdef class Unpacker(object):
         You should set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
+        Deprecated, use *max_buffer_size* instead.
         Limits max length of str. (default: max_buffer_size or 1024*1024)
 
     :param int max_bin_len:
+        Deprecated, use *max_buffer_size* instead.
         Limits max length of bin. (default: max_buffer_size or 1024*1024)
 
     :param int max_array_len:
@@ -285,10 +287,11 @@ cdef class Unpacker(object):
         Limits max length of map. (default: max_buffer_size//2 or 32*1024)
 
     :param int max_ext_len:
+        Deprecated, use *max_buffer_size* instead.
         Limits max size of ext type. (default: max_buffer_size or 1024*1024)
 
     :param str encoding:
-        Deprecated, use raw instead.
+        Deprecated, use ``raw=False`` instead.
         Encoding used for decoding msgpack raw.
         If it is None (default), msgpack raw is deserialized to Python bytes.
 
@@ -298,13 +301,13 @@ cdef class Unpacker(object):
 
     Example of streaming deserialize from file-like object::
 
-        unpacker = Unpacker(file_like, raw=False)
+        unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024)
         for o in unpacker:
             process(o)
 
     Example of streaming deserialize from socket::
 
-        unpacker = Unpacker(raw=False)
+        unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024)
         while True:
             buf = sock.recv(1024**2)
             if not buf:
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 7524448..1aa3bdf 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -208,12 +208,12 @@ class Unpacker(object):
         You should set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
-        (deprecated) Limits max length of str.
-        (default: max_buffer_size or 1024*1024)
+        Deprecated, use *max_buffer_size* instead.
+        Limits max length of str. (default: max_buffer_size or 1024*1024)
 
     :param int max_bin_len:
-        (deprecated) Limits max length of bin.
-        (default: max_buffer_size or 1024*1024)
+        Deprecated, use *max_buffer_size* instead.
+        Limits max length of bin. (default: max_buffer_size or 1024*1024)
 
     :param int max_array_len:
         Limits max length of array.
@@ -224,18 +224,18 @@ class Unpacker(object):
         (default: max_buffer_size//2 or 32*1024)
 
     :param int max_ext_len:
-        (deprecated) Limits max size of ext type.
-        (default: max_buffer_size or 1024*1024)
+        Deprecated, use *max_buffer_size* instead.
+        Limits max size of ext type.  (default: max_buffer_size or 1024*1024)
 
-    example of streaming deserialize from file-like object::
+    Example of streaming deserialize from file-like object::
 
-        unpacker = Unpacker(file_like, raw=False)
+        unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024)
         for o in unpacker:
             process(o)
 
-    example of streaming deserialize from socket::
+    Example of streaming deserialize from socket::
 
-        unpacker = Unpacker(raw=False)
+        unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024)
         while True:
             buf = sock.recv(1024**2)
             if not buf:

From 8f513af999d4abd39d632fcc8732225a658268ee Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 25 Jan 2019 21:43:28 +0900
Subject: [PATCH 1465/1648] v0.6.1

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index 49a4103..926c5e7 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 6, 0)
+version = (0, 6, 1)

From 381c2eff5f8ee0b8669fd6daf1fd1ecaffe7c931 Mon Sep 17 00:00:00 2001
From: INADA Naoki 
Date: Mon, 4 Feb 2019 12:08:07 +0900
Subject: [PATCH 1466/1648] Fix changelog.

Fixes #343
---
 ChangeLog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 2c988db..727ca9a 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -3,7 +3,7 @@
 
 Release Date: 2019-01-25
 
-This release is for mitigating pain caused by v0.6.1 reduced max input limits
+This release is for mitigating pain caused by v0.6.0 reduced max input limits
 for security reason.
 
 * ``unpackb(data)`` configures ``max_*_len`` options from ``len(data)``,

From 737f08a885dcff32aa1a417a45936d7f7810ee37 Mon Sep 17 00:00:00 2001
From: Hugues 
Date: Wed, 27 Mar 2019 06:37:26 -0700
Subject: [PATCH 1467/1648] Update requirements.txt (#346)

bytearray.pxd is only available starting with Cython 0.29
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index cd54e6d..e08dd4f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1 @@
-Cython==0.27.3
+Cython~=0.29.5

From 05ff11dbcc8181cc781b121e46e76a01258a32af Mon Sep 17 00:00:00 2001
From: Felix Schwarz 
Date: Sun, 12 May 2019 14:44:32 +0200
Subject: [PATCH 1468/1648] use relative imports (#357)

Some applications use msgpack to store persistent data and require a specific
msgpack version (e.g. borgbackup). Bundling helps in case there is an
(incompatible) version of msgpack in a system-wide install.
---
 msgpack/__init__.py   | 10 +++++-----
 msgpack/_packer.pyx   |  2 +-
 msgpack/_unpacker.pyx |  4 ++--
 msgpack/fallback.py   |  4 ++--
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 7493c4c..4ad9c1a 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,6 +1,6 @@
 # coding: utf-8
-from msgpack._version import version
-from msgpack.exceptions import *
+from ._version import version
+from .exceptions import *
 
 from collections import namedtuple
 
@@ -19,12 +19,12 @@ class ExtType(namedtuple('ExtType', 'code data')):
 
 import os
 if os.environ.get('MSGPACK_PUREPYTHON'):
-    from msgpack.fallback import Packer, unpackb, Unpacker
+    from .fallback import Packer, unpackb, Unpacker
 else:
     try:
-        from msgpack._cmsgpack import Packer, unpackb, Unpacker
+        from ._cmsgpack import Packer, unpackb, Unpacker
     except ImportError:
-        from msgpack.fallback import Packer, unpackb, Unpacker
+        from .fallback import Packer, unpackb, Unpacker
 
 
 def pack(o, stream, **kwargs):
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index bfde043..dcee213 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -3,7 +3,7 @@
 from cpython cimport *
 from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact
 
-from msgpack import ExtType
+from . import ExtType
 
 
 cdef extern from "Python.h":
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 3c6d59e..3727f50 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -12,14 +12,14 @@ from libc.string cimport *
 from libc.limits cimport *
 ctypedef unsigned long long uint64_t
 
-from msgpack.exceptions import (
+from .exceptions import (
     BufferFull,
     OutOfData,
     ExtraData,
     FormatError,
     StackError,
 )
-from msgpack import ExtType
+from . import ExtType
 
 
 cdef extern from "unpack.h":
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 1aa3bdf..3836e83 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -59,7 +59,7 @@ else:
     newlist_hint = lambda size: []
 
 
-from msgpack.exceptions import (
+from .exceptions import (
     BufferFull,
     OutOfData,
     ExtraData,
@@ -67,7 +67,7 @@ from msgpack.exceptions import (
     StackError,
 )
 
-from msgpack import ExtType
+from . import ExtType
 
 
 EX_SKIP                 = 0

From b98b8cab99d7b2dbfe2b2211974564b7e22e9412 Mon Sep 17 00:00:00 2001
From: Marty B 
Date: Wed, 18 Sep 2019 18:15:09 +0200
Subject: [PATCH 1469/1648] Avoid calling __Pyx_GetModuleGlobalName for ExtType
 (#363)

---
 msgpack/_packer.pyx | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index dcee213..e275ef2 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -3,6 +3,8 @@
 from cpython cimport *
 from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact
 
+cdef ExtType
+
 from . import ExtType
 
 

From 3146ebd330cbd02d0d7b4f82a94472cb395804ef Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 19 Sep 2019 13:20:57 +0900
Subject: [PATCH 1470/1648] Use Py_SIZE() when it is safe (#369)

---
 msgpack/_packer.pyx | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index e275ef2..2f4d120 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -200,7 +200,7 @@ cdef class Packer(object):
                    dval = o
                    ret = msgpack_pack_double(&self.pk, dval)
             elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o):
-                L = len(o)
+                L = Py_SIZE(o)
                 if L > ITEM_LIMIT:
                     PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name)
                 rawval = o
@@ -214,7 +214,7 @@ cdef class Packer(object):
                         raise ValueError("unicode string is too large")
                 else:
                     o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
-                    L = len(o)
+                    L = Py_SIZE(o)
                     if L > ITEM_LIMIT:
                         raise ValueError("unicode string is too large")
                     ret = msgpack_pack_raw(&self.pk, L)
@@ -254,7 +254,7 @@ cdef class Packer(object):
                 ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
-                L = len(o)
+                L = Py_SIZE(o)
                 if L > ITEM_LIMIT:
                     raise ValueError("list is too large")
                 ret = msgpack_pack_array(&self.pk, L)

From c25e2a0984ec5d092fee38eeb4b74676ada9aef4 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 19 Sep 2019 20:14:33 +0900
Subject: [PATCH 1471/1648] update Cython to 0.29.13 (#370)

---
 .travis.yml      | 5 +++--
 requirements.txt | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 51917c5..43c5259 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,7 +20,7 @@ matrix:
         - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686
       install:
         - pip install -U pip
-        - pip install cython
+        - pip install -r requirements.txt
         - make cython
         - docker pull $DOCKER_IMAGE
       script:
@@ -39,7 +39,8 @@ matrix:
 
 install:
   - pip install -U pip
-  - pip install -U cython pytest
+  - pip install -U pytest
+  - pip install -r requirements.txt
   - make cython
   - pip install -e .
 
diff --git a/requirements.txt b/requirements.txt
index e08dd4f..a2cce25 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1 @@
-Cython~=0.29.5
+Cython~=0.29.13

From fd3f0048633423651772526875611f125dda68f6 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 19 Sep 2019 20:37:19 +0900
Subject: [PATCH 1472/1648] Add Python 3.8 to travis (#371)

---
 .travis.yml | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 43c5259..c80bb37 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,16 +3,20 @@ language: python
 cache: pip
 
 python:
+  # Available Python (PyPy) can be listed by:
+  #
+  #   $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/
   - "2.7"
   - "3.4"
   - "3.5"
   - "3.6"
   - "3.7"
-  - "nightly"
+  - "3.8-dev"
 
 matrix:
   include:
-    - sudo: required
+    - name: 32bit build
+      sudo: required
       language: python
       services:
         - docker
@@ -25,12 +29,14 @@ matrix:
         - docker pull $DOCKER_IMAGE
       script:
         - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
-    - python: "pypy2.7-5.10.0"
+    - name: "pypy2.7"
+      python: "pypy2.7-7.1.1"
       install:
         - pip install -e .
       script:
         - py.test -v test
-    - python: "pypy3.5"
+    - name: "pypy3"
+      python: "pypy3.6-7.1.1"
       install:
         - pip install -e .
       script:

From 144f276e885be867c1545226a60c99957dac04e0 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 20 Sep 2019 16:36:37 +0900
Subject: [PATCH 1473/1648] Update ChangeLog

---
 ChangeLog.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 727ca9a..a295e6b 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,13 @@
+0.6.2
+=====
+
+Release Date: 2019-09-20
+
+* Support Python 3.8.
+* Update Cython to 0.29.13 for support Python 3.8.
+* Some small optimizations.
+
+
 0.6.1
 ======
 

From 997b524f06176aaa6bd255a046a8746e99b4f87d Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 20 Sep 2019 16:37:08 +0900
Subject: [PATCH 1474/1648] 0.6.2

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index 926c5e7..1e73a00 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 6, 1)
+version = (0, 6, 2)

From b458e9a6a2cf69e483fa5994d227382c6a01f3c4 Mon Sep 17 00:00:00 2001
From: Terence Honles 
Date: Fri, 22 Nov 2019 19:58:55 -0800
Subject: [PATCH 1475/1648] update for Python 3.8 (#374)

---
 appveyor.yml         | 4 ++++
 docker/buildwheel.sh | 5 ++++-
 docker/runtests.sh   | 5 ++++-
 docker/shared.env    | 8 ++++++++
 setup.py             | 1 +
 tox.ini              | 2 +-
 6 files changed, 22 insertions(+), 3 deletions(-)
 create mode 100644 docker/shared.env

diff --git a/appveyor.yml b/appveyor.yml
index bd0800a..f338e17 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -29,6 +29,10 @@ test_script:
   - ci\\runtests.bat
   - set PYTHON="C:\\Python37-x64"
   - ci\\runtests.bat
+  - set PYTHON="C:\\Python38"
+  - ci\\runtests.bat
+  - set PYTHON="C:\\Python38-x64"
+  - ci\\runtests.bat
 
 after_test:
   # This step builds your wheels.
diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh
index d8c74cb..c953127 100644
--- a/docker/buildwheel.sh
+++ b/docker/buildwheel.sh
@@ -1,10 +1,13 @@
 #!/bin/bash
+DOCKER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$DOCKER_DIR/shared.env"
+
 set -e -x
 
 ARCH=`uname -p`
 echo "arch=$ARCH"
 
-for V in cp37-cp37m cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do
+for V in "${PYTHON_VERSIONS[@]}"; do
     PYBIN=/opt/python/$V/bin
     rm -rf build/       # Avoid lib build by narrow Python is used by wide python
     $PYBIN/python setup.py bdist_wheel -p manylinux1_${ARCH}
diff --git a/docker/runtests.sh b/docker/runtests.sh
index c6bbf60..fa7e979 100755
--- a/docker/runtests.sh
+++ b/docker/runtests.sh
@@ -1,7 +1,10 @@
 #!/bin/bash
+DOCKER_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$DOCKER_DIR/shared.env"
+
 set -e -x
 
-for V in cp36-cp36m cp35-cp35m cp27-cp27m cp27-cp27mu; do
+for V in "${PYTHON_VERSIONS[@]}"; do
     PYBIN=/opt/python/$V/bin
     $PYBIN/python setup.py install
     rm -rf build/       # Avoid lib build by narrow Python is used by wide python
diff --git a/docker/shared.env b/docker/shared.env
new file mode 100644
index 0000000..b5772e3
--- /dev/null
+++ b/docker/shared.env
@@ -0,0 +1,8 @@
+PYTHON_VERSIONS=(
+    cp38-cp38
+    cp37-cp37m
+    cp36-cp36m
+    cp35-cp35m
+    cp27-cp27m
+    cp27-cp27mu
+)
diff --git a/setup.py b/setup.py
index eb9403f..8b8d7a0 100755
--- a/setup.py
+++ b/setup.py
@@ -128,6 +128,7 @@ setup(name=name,
           'Programming Language :: Python :: 3.5',
           'Programming Language :: Python :: 3.6',
           'Programming Language :: Python :: 3.7',
+          'Programming Language :: Python :: 3.8',
           'Programming Language :: Python :: Implementation :: CPython',
           'Programming Language :: Python :: Implementation :: PyPy',
           'Intended Audience :: Developers',
diff --git a/tox.ini b/tox.ini
index 0945a6d..4b059ff 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {py27,py35,py36,py37}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
+envlist = {py27,py35,py36,py37,py38}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
 
 [variants:pure]
 setenv=

From 891f2d8743857bb75204f96b0469cb2ec90c7f79 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 28 Nov 2019 20:23:34 +0900
Subject: [PATCH 1476/1648] Drop Python 2 support from _cmsgpack (#376)

---
 .travis.yml              | 10 ++++++++--
 README.rst               | 20 +++++++++++---------
 docker/shared.env        |  2 --
 msgpack/__init__.py      |  5 +++--
 msgpack/_packer.pyx      |  5 +----
 msgpack/buff_converter.h | 20 --------------------
 msgpack/fallback.py      |  5 ++---
 msgpack/unpack.h         |  4 ----
 setup.py                 | 20 +++++++++++---------
 9 files changed, 36 insertions(+), 55 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c80bb37..7b298af 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,6 @@ python:
   # Available Python (PyPy) can be listed by:
   #
   #   $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/
-  - "2.7"
   - "3.4"
   - "3.5"
   - "3.6"
@@ -41,7 +40,14 @@ matrix:
         - pip install -e .
       script:
         - pytest -v test
-
+    - name: "Python 2 (fallback)"
+      python: "2.7"
+      install:
+        - pip install -U pip
+        - pip install -U pytest
+        - pip install .
+      script:
+        - pytest -v test
 
 install:
   - pip install -U pip
diff --git a/README.rst b/README.rst
index 94a4bb2..82b6c02 100644
--- a/README.rst
+++ b/README.rst
@@ -76,10 +76,18 @@ Install
 
    $ pip install msgpack
 
-PyPy
-^^^^
+Pure Python implementation
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The extension module in msgpack (``msgpack._cmsgpack``) does not support
+Python 2 and PyPy.
+
+But msgpack provides a pure Python implementation (``msgpack.fallback``)
+for PyPy and Python 2.
+
+Since the [pip](https://pip.pypa.io/) uses the pure Python implementation,
+Python 2 support will not be dropped in foreseeable feature.
 
-msgpack provides a pure Python implementation.  PyPy can use this.
 
 Windows
 ^^^^^^^
@@ -88,12 +96,6 @@ When you can't use a binary distribution, you need to install Visual Studio
 or Windows SDK on Windows.
 Without extension, using pure Python implementation on CPython runs slowly.
 
-For Python 2.7, `Microsoft Visual C++ Compiler for Python 2.7 `_
-is recommended solution.
-
-For Python 3.5, `Microsoft Visual Studio 2015 `_
-Community Edition or Express Edition can be used to build extension module.
-
 
 How to use
 ----------
diff --git a/docker/shared.env b/docker/shared.env
index b5772e3..17abdd8 100644
--- a/docker/shared.env
+++ b/docker/shared.env
@@ -3,6 +3,4 @@ PYTHON_VERSIONS=(
     cp37-cp37m
     cp36-cp36m
     cp35-cp35m
-    cp27-cp27m
-    cp27-cp27mu
 )
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 4ad9c1a..4112a16 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -2,6 +2,8 @@
 from ._version import version
 from .exceptions import *
 
+import os
+import sys
 from collections import namedtuple
 
 
@@ -17,8 +19,7 @@ class ExtType(namedtuple('ExtType', 'code data')):
         return super(ExtType, cls).__new__(cls, code, data)
 
 
-import os
-if os.environ.get('MSGPACK_PUREPYTHON'):
+if os.environ.get('MSGPACK_PUREPYTHON') or sys.version_info[0] == 2:
     from .fallback import Packer, unpackb, Unpacker
 else:
     try:
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 2f4d120..e620914 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -130,10 +130,7 @@ cdef class Packer(object):
 
         self._bencoding = encoding
         if encoding is None:
-            if PY_MAJOR_VERSION < 3:
-                self.encoding = 'utf-8'
-            else:
-                self.encoding = NULL
+            self.encoding = 'utf-8'
         else:
             self.encoding = self._bencoding
 
diff --git a/msgpack/buff_converter.h b/msgpack/buff_converter.h
index bc7227a..86b4196 100644
--- a/msgpack/buff_converter.h
+++ b/msgpack/buff_converter.h
@@ -1,28 +1,8 @@
 #include "Python.h"
 
 /* cython does not support this preprocessor check => write it in raw C */
-#if PY_MAJOR_VERSION == 2
-static PyObject *
-buff_to_buff(char *buff, Py_ssize_t size)
-{
-    return PyBuffer_FromMemory(buff, size);
-}
-
-#elif (PY_MAJOR_VERSION == 3) && (PY_MINOR_VERSION >= 3)
 static PyObject *
 buff_to_buff(char *buff, Py_ssize_t size)
 {
     return PyMemoryView_FromMemory(buff, size, PyBUF_READ);
 }
-#else
-static PyObject *
-buff_to_buff(char *buff, Py_ssize_t size)
-{
-    Py_buffer pybuf;
-    if (PyBuffer_FillInfo(&pybuf, NULL, buff, size, 1, PyBUF_FULL_RO) == -1) {
-        return NULL;
-    }
-
-    return PyMemoryView_FromBuffer(&pybuf);
-}
-#endif
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 3836e83..1ed6e77 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -5,13 +5,12 @@ import struct
 import warnings
 
 
-if sys.version_info[0] == 2:
-    PY2 = True
+PY2 = sys.version_info[0] == 2
+if PY2:
     int_types = (int, long)
     def dict_iteritems(d):
         return d.iteritems()
 else:
-    PY2 = False
     int_types = int
     unicode = str
     xrange = range
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 85dbbed..bbce91c 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -273,11 +273,7 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch
         return -1;
     }
     // length also includes the typecode, so the actual data is length-1
-#if PY_MAJOR_VERSION == 2
-    py = PyObject_CallFunction(u->ext_hook, "(is#)", (int)typecode, pos, (Py_ssize_t)length-1);
-#else
     py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1);
-#endif
     if (!py)
         return -1;
     *o = py;
diff --git a/setup.py b/setup.py
index 8b8d7a0..77b81c6 100755
--- a/setup.py
+++ b/setup.py
@@ -9,6 +9,11 @@ from setuptools import setup, Extension
 
 from distutils.command.build_ext import build_ext
 
+
+PYPY = hasattr(sys, "pypy_version_info")
+PY2 = sys.version_info[0] == 2
+
+
 # for building transitional package.
 TRANSITIONAL = False
 
@@ -64,14 +69,11 @@ version_str = '.'.join(str(x) for x in version[:3])
 if len(version) > 3 and version[3] != 'final':
     version_str += version[3]
 
-# take care of extension modules.
-if have_cython:
-    class Sdist(sdist):
-        def __init__(self, *args, **kwargs):
-            cythonize('msgpack/_cmsgpack.pyx')
-            sdist.__init__(self, *args, **kwargs)
-else:
-    Sdist = sdist
+# Cython is required for sdist
+class Sdist(sdist):
+    def __init__(self, *args, **kwargs):
+        cythonize('msgpack/_cmsgpack.pyx')
+        sdist.__init__(self, *args, **kwargs)
 
 libraries = []
 if sys.platform == 'win32':
@@ -83,7 +85,7 @@ else:
     macros = [('__LITTLE_ENDIAN__', '1')]
 
 ext_modules = []
-if not hasattr(sys, 'pypy_version_info'):
+if not PYPY and not PY2:
     ext_modules.append(Extension('msgpack._cmsgpack',
                                  sources=['msgpack/_cmsgpack.cpp'],
                                  libraries=libraries,

From cc3a8665d6210e933bcfb9120bd0ceb15224f03e Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 3 Dec 2019 17:46:28 +0900
Subject: [PATCH 1477/1648] Use Github Actions for Windows (#377)

---
 .github/workflows/windows.yaml | 70 ++++++++++++++++++++++++++++++++++
 ci/runtests.bat                |  2 +-
 ci/runtests.sh                 |  8 ++++
 3 files changed, 79 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/windows.yaml
 create mode 100644 ci/runtests.sh

diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml
new file mode 100644
index 0000000..cecb825
--- /dev/null
+++ b/.github/workflows/windows.yaml
@@ -0,0 +1,70 @@
+name: Build and test windows wheels
+on:
+  push:
+    branches:
+      - master
+      - test
+  pull_request:
+  create:
+
+jobs:
+  build:
+    runs-on: windows-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Cythonize
+        shell: bash
+        run: |
+          pip install -U Cython
+          make cython
+          #python setup.py sdist
+
+      - name: Python 3.6 (amd64)
+        env:
+          PYTHON: "py -3.6-64"
+        shell: bash
+        run: |
+          ci/runtests.sh
+
+      - name: Python 3.6 (x86)
+        env:
+          PYTHON: "py -3.6-32"
+        shell: bash
+        run: |
+          ci/runtests.sh
+
+      - name: Python 3.7 (amd64)
+        env:
+          PYTHON: "py -3.7-64"
+        shell: bash
+        run: |
+          ci/runtests.sh
+
+      - name: Python 3.7 (x86)
+        env:
+          PYTHON: "py -3.7-32"
+        shell: bash
+        run: |
+          ci/runtests.sh
+
+      - name: Python 3.8 (amd64)
+        env:
+          PYTHON: "py -3.8-64"
+        shell: bash
+        run: |
+          ci/runtests.sh
+
+      - name: Python 3.8 (x86)
+        env:
+          PYTHON: "py -3.8-32"
+        shell: bash
+        run: |
+          ci/runtests.sh
+
+      - name: Upload Wheels
+        uses: actions/upload-artifact@v1
+        with:
+          name: win-wheels
+          path: ./dist
diff --git a/ci/runtests.bat b/ci/runtests.bat
index 0240467..4ae2f70 100644
--- a/ci/runtests.bat
+++ b/ci/runtests.bat
@@ -2,7 +2,7 @@
 %PYTHON%\python.exe setup.py build_ext -i
 %PYTHON%\python.exe setup.py install
 %PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))"
-%PYTHON%\python.exe -c "from msgpack import _packer, _unpacker"
+%PYTHON%\python.exe -c "from msgpack import _cmsgpack"
 %PYTHON%\python.exe setup.py bdist_wheel
 %PYTHON%\python.exe -m pytest -v test
 SET EL=%ERRORLEVEL%
diff --git a/ci/runtests.sh b/ci/runtests.sh
new file mode 100644
index 0000000..5d87f69
--- /dev/null
+++ b/ci/runtests.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+set -ex
+${PYTHON} -VV
+${PYTHON} -m pip install setuptools wheel pytest
+${PYTHON} setup.py build_ext -if
+${PYTHON} -c "from msgpack import _cmsgpack"
+${PYTHON} setup.py bdist_wheel
+${PYTHON} -m pytest -v test

From e1ed0044bf31dc0d6ef951f6298de4f420170968 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 3 Dec 2019 18:54:01 +0900
Subject: [PATCH 1478/1648] Remove encoding/unicode_errors options from Packer
 (#378)

---
 msgpack/_packer.pyx | 46 +++++++--------------------------------------
 msgpack/fallback.py | 26 ++-----------------------
 test/test_pack.py   | 30 -----------------------------
 3 files changed, 9 insertions(+), 93 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index e620914..2e698e1 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -89,19 +89,9 @@ cdef class Packer(object):
         Additionally tuples will not be serialized as lists.
         This is useful when trying to implement accurate serialization
         for python types.
-
-    :param str unicode_errors:
-        Error handler for encoding unicode. (default: 'strict')
-
-    :param str encoding:
-        (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
     """
     cdef msgpack_packer pk
     cdef object _default
-    cdef object _bencoding
-    cdef object _berrors
-    cdef const char *encoding
-    cdef const char *unicode_errors
     cdef bint strict_types
     cdef bool use_float
     cdef bint autoreset
@@ -114,11 +104,11 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
-    def __init__(self, default=None, encoding=None, unicode_errors=None,
-                 bint use_single_float=False, bint autoreset=True, bint use_bin_type=False,
+    def __init__(self, default=None,
+                 bint use_single_float=False,
+                 bint autoreset=True,
+                 bint use_bin_type=False,
                  bint strict_types=False):
-        if encoding is not None:
-            PyErr_WarnEx(DeprecationWarning, "encoding is deprecated.", 1)
         self.use_float = use_single_float
         self.strict_types = strict_types
         self.autoreset = autoreset
@@ -128,18 +118,6 @@ cdef class Packer(object):
                 raise TypeError("default must be a callable.")
         self._default = default
 
-        self._bencoding = encoding
-        if encoding is None:
-            self.encoding = 'utf-8'
-        else:
-            self.encoding = self._bencoding
-
-        self._berrors = unicode_errors
-        if unicode_errors is None:
-            self.unicode_errors = NULL
-        else:
-            self.unicode_errors = self._berrors
-
     def __dealloc__(self):
         PyMem_Free(self.pk.buf)
         self.pk.buf = NULL
@@ -205,19 +183,9 @@ cdef class Packer(object):
                 if ret == 0:
                     ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o):
-                if self.encoding == NULL and self.unicode_errors == NULL:
-                    ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT);
-                    if ret == -2:
-                        raise ValueError("unicode string is too large")
-                else:
-                    o = PyUnicode_AsEncodedString(o, self.encoding, self.unicode_errors)
-                    L = Py_SIZE(o)
-                    if L > ITEM_LIMIT:
-                        raise ValueError("unicode string is too large")
-                    ret = msgpack_pack_raw(&self.pk, L)
-                    if ret == 0:
-                        rawval = o
-                        ret = msgpack_pack_raw_body(&self.pk, rawval, L)
+                ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT);
+                if ret == -2:
+                    raise ValueError("unicode string is too large")
             elif PyDict_CheckExact(o):
                 d = o
                 L = len(d)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 1ed6e77..5dab906 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -752,32 +752,14 @@ class Packer(object):
         Additionally tuples will not be serialized as lists.
         This is useful when trying to implement accurate serialization
         for python types.
-
-    :param str encoding:
-        (deprecated) Convert unicode to bytes with this encoding. (default: 'utf-8')
-
-    :param str unicode_errors:
-        Error handler for encoding unicode. (default: 'strict')
     """
-    def __init__(self, default=None, encoding=None, unicode_errors=None,
+    def __init__(self, default=None,
                  use_single_float=False, autoreset=True, use_bin_type=False,
                  strict_types=False):
-        if encoding is None:
-            encoding = 'utf_8'
-        else:
-            warnings.warn(
-                "encoding is deprecated, Use raw=False instead.",
-                DeprecationWarning, stacklevel=2)
-
-        if unicode_errors is None:
-            unicode_errors = 'strict'
-
         self._strict_types = strict_types
         self._use_float = use_single_float
         self._autoreset = autoreset
         self._use_bin_type = use_bin_type
-        self._encoding = encoding
-        self._unicode_errors = unicode_errors
         self._buffer = StringIO()
         if default is not None:
             if not callable(default):
@@ -834,11 +816,7 @@ class Packer(object):
                 self._pack_bin_header(n)
                 return self._buffer.write(obj)
             if check(obj, unicode):
-                if self._encoding is None:
-                    raise TypeError(
-                        "Can't encode unicode string: "
-                        "no encoding is specified")
-                obj = obj.encode(self._encoding, self._unicode_errors)
+                obj = obj.encode("utf-8")
                 n = len(obj)
                 if n >= 2**32:
                     raise ValueError("String is too large")
diff --git a/test/test_pack.py b/test/test_pack.py
index 3658a97..194b2c9 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -40,21 +40,6 @@ def testPackUnicode():
         re = Unpacker(BytesIO(data), raw=False, use_list=1).unpack()
         assert re == td
 
-def testPackUTF32():  # deprecated
-    try:
-        test_data = [
-            "",
-            "abcd",
-            ["defgh"],
-            "РуÑÑкий текÑÑ‚",
-            ]
-        for td in test_data:
-            with pytest.deprecated_call():
-                re = unpackb(packb(td, encoding='utf-32'), use_list=1, encoding='utf-32')
-            assert re == td
-    except LookupError as e:
-        xfail(e)
-
 def testPackBytes():
     test_data = [
         b"", b"abcd", (b"defgh",),
@@ -69,26 +54,11 @@ def testPackByteArrays():
     for td in test_data:
         check(td)
 
-def testIgnoreUnicodeErrors(): # deprecated
-    with pytest.deprecated_call():
-        re = unpackb(packb(b'abc\xeddef'), encoding='utf-8', unicode_errors='ignore', use_list=1)
-    assert re == "abcdef"
-
 def testStrictUnicodeUnpack():
     packed = packb(b'abc\xeddef')
     with pytest.raises(UnicodeDecodeError):
         unpackb(packed, raw=False, use_list=1)
 
-def testStrictUnicodePack():  # deprecated
-    with raises(UnicodeEncodeError):
-        with pytest.deprecated_call():
-            packb("abc\xeddef", encoding='ascii', unicode_errors='strict')
-
-def testIgnoreErrorsPack():  # deprecated
-    with pytest.deprecated_call():
-        re = unpackb(packb("abcФФФdef", encoding='ascii', unicode_errors='ignore'), raw=False, use_list=1)
-    assert re == "abcdef"
-
 def testDecodeBinary():
     re = unpackb(packb(b"abc"), encoding=None, use_list=1)
     assert re == b"abc"

From a0480c760256b4afc18beaebd5e3c79de1d4ce56 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 3 Dec 2019 18:54:18 +0900
Subject: [PATCH 1479/1648] Update ChangeLog

---
 ChangeLog.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index a295e6b..1352af8 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,13 @@
+1.0.0
+=====
+
+Release Date: TBD
+
+* Remove Python 2 support from the ``msgpack/_cmsgpack``.
+  ``msgpack/fallback`` still supports Python 2.
+* Remove encoding and unicode_errors options from the Packer.
+
+
 0.6.2
 =====
 

From 83ebb63c447a99c81d043eb6808bbfb50697a751 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 3 Dec 2019 20:53:11 +0900
Subject: [PATCH 1480/1648] Ressurect unicode_errors of the Packer. (#379)

---
 ChangeLog.rst       |  2 +-
 msgpack/_packer.pyx | 34 +++++++++++++++++++++++++++-------
 msgpack/fallback.py | 11 ++++++++---
 test/test_pack.py   | 16 ++++++++++++++--
 4 files changed, 50 insertions(+), 13 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 1352af8..1d784af 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -5,7 +5,7 @@ Release Date: TBD
 
 * Remove Python 2 support from the ``msgpack/_cmsgpack``.
   ``msgpack/fallback`` still supports Python 2.
-* Remove encoding and unicode_errors options from the Packer.
+* Remove ``encoding`` option from the Packer.
 
 
 0.6.2
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 2e698e1..8b1a392 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -89,9 +89,15 @@ cdef class Packer(object):
         Additionally tuples will not be serialized as lists.
         This is useful when trying to implement accurate serialization
         for python types.
+
+    :param str unicode_errors:
+        The error handler for encoding unicode. (default: 'strict')
+        DO NOT USE THIS!!  This option is kept for very specific usage.
     """
     cdef msgpack_packer pk
     cdef object _default
+    cdef object _berrors
+    cdef const char *unicode_errors
     cdef bint strict_types
     cdef bool use_float
     cdef bint autoreset
@@ -104,10 +110,8 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
-    def __init__(self, default=None,
-                 bint use_single_float=False,
-                 bint autoreset=True,
-                 bint use_bin_type=False,
+    def __init__(self, *, default=None, unicode_errors=None,
+                 bint use_single_float=False, bint autoreset=True, bint use_bin_type=False,
                  bint strict_types=False):
         self.use_float = use_single_float
         self.strict_types = strict_types
@@ -118,6 +122,12 @@ cdef class Packer(object):
                 raise TypeError("default must be a callable.")
         self._default = default
 
+        self._berrors = unicode_errors
+        if unicode_errors is None:
+            self.unicode_errors = NULL
+        else:
+            self.unicode_errors = self._berrors
+
     def __dealloc__(self):
         PyMem_Free(self.pk.buf)
         self.pk.buf = NULL
@@ -183,9 +193,19 @@ cdef class Packer(object):
                 if ret == 0:
                     ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o):
-                ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT);
-                if ret == -2:
-                    raise ValueError("unicode string is too large")
+                if self.unicode_errors == NULL:
+                    ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT);
+                    if ret == -2:
+                        raise ValueError("unicode string is too large")
+                else:
+                    o = PyUnicode_AsEncodedString(o, NULL, self.unicode_errors)
+                    L = Py_SIZE(o)
+                    if L > ITEM_LIMIT:
+                        raise ValueError("unicode string is too large")
+                    ret = msgpack_pack_raw(&self.pk, L)
+                    if ret == 0:
+                        rawval = o
+                        ret = msgpack_pack_raw_body(&self.pk, rawval, L)
             elif PyDict_CheckExact(o):
                 d = o
                 L = len(d)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 5dab906..0c0c101 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -667,7 +667,7 @@ class Unpacker(object):
             elif self._raw:
                 obj = bytes(obj)
             else:
-                obj = obj.decode('utf_8')
+                obj = obj.decode('utf_8', self._unicode_errors)
             return obj
         if typ == TYPE_EXT:
             return self._ext_hook(n, bytes(obj))
@@ -752,14 +752,19 @@ class Packer(object):
         Additionally tuples will not be serialized as lists.
         This is useful when trying to implement accurate serialization
         for python types.
+
+    :param str unicode_errors:
+        The error handler for encoding unicode. (default: 'strict')
+        DO NOT USE THIS!!  This option is kept for very specific usage.
     """
-    def __init__(self, default=None,
+    def __init__(self, default=None, unicode_errors=None,
                  use_single_float=False, autoreset=True, use_bin_type=False,
                  strict_types=False):
         self._strict_types = strict_types
         self._use_float = use_single_float
         self._autoreset = autoreset
         self._use_bin_type = use_bin_type
+        self._unicode_errors = unicode_errors or "strict"
         self._buffer = StringIO()
         if default is not None:
             if not callable(default):
@@ -816,7 +821,7 @@ class Packer(object):
                 self._pack_bin_header(n)
                 return self._buffer.write(obj)
             if check(obj, unicode):
-                obj = obj.encode("utf-8")
+                obj = obj.encode("utf-8", self._unicode_errors)
                 n = len(obj)
                 if n >= 2**32:
                     raise ValueError("String is too large")
diff --git a/test/test_pack.py b/test/test_pack.py
index 194b2c9..b6752e5 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 from collections import OrderedDict
 from io import BytesIO
 import struct
+import sys
 
 import pytest
 from pytest import raises, xfail
@@ -54,13 +55,24 @@ def testPackByteArrays():
     for td in test_data:
         check(td)
 
+@pytest.mark.skipif(sys.version_info < (3,0), reason="Python 2 passes invalid surrogates")
+def testIgnoreUnicodeErrors():
+    re = unpackb(packb(b'abc\xeddef', use_bin_type=False),
+                 raw=False, unicode_errors='ignore')
+    assert re == "abcdef"
+
 def testStrictUnicodeUnpack():
-    packed = packb(b'abc\xeddef')
+    packed = packb(b'abc\xeddef', use_bin_type=False)
     with pytest.raises(UnicodeDecodeError):
         unpackb(packed, raw=False, use_list=1)
 
+@pytest.mark.skipif(sys.version_info < (3,0), reason="Python 2 passes invalid surrogates")
+def testIgnoreErrorsPack():
+    re = unpackb(packb(u"abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors='ignore'), raw=False, use_list=1)
+    assert re == "abcdef"
+
 def testDecodeBinary():
-    re = unpackb(packb(b"abc"), encoding=None, use_list=1)
+    re = unpackb(packb(b"abc"), use_list=1)
     assert re == b"abc"
 
 def testPackFloat():

From e419cd8e2db6b8226bd681b52b6acfe70d8e6a86 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 3 Dec 2019 21:13:05 +0900
Subject: [PATCH 1481/1648] Remove encoding option from Unpacker. (#380)

---
 ChangeLog.rst         |  2 +-
 msgpack/_unpacker.pyx | 36 ++++++++----------------------------
 msgpack/fallback.py   | 23 +++++------------------
 msgpack/unpack.h      |  5 +----
 4 files changed, 15 insertions(+), 51 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 1d784af..d44b36a 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -5,7 +5,7 @@ Release Date: TBD
 
 * Remove Python 2 support from the ``msgpack/_cmsgpack``.
   ``msgpack/fallback`` still supports Python 2.
-* Remove ``encoding`` option from the Packer.
+* Remove ``encoding`` option from the Packer and Unpacker.
 
 
 0.6.2
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 3727f50..b258686 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -31,7 +31,6 @@ cdef extern from "unpack.h":
         PyObject* object_hook
         PyObject* list_hook
         PyObject* ext_hook
-        char *encoding
         char *unicode_errors
         Py_ssize_t max_str_len
         Py_ssize_t max_bin_len
@@ -58,7 +57,7 @@ cdef inline init_ctx(unpack_context *ctx,
                      object object_hook, object object_pairs_hook,
                      object list_hook, object ext_hook,
                      bint use_list, bint raw, bint strict_map_key,
-                     const char* encoding, const char* unicode_errors,
+                     const char* unicode_errors,
                      Py_ssize_t max_str_len, Py_ssize_t max_bin_len,
                      Py_ssize_t max_array_len, Py_ssize_t max_map_len,
                      Py_ssize_t max_ext_len):
@@ -99,7 +98,6 @@ cdef inline init_ctx(unpack_context *ctx,
             raise TypeError("ext_hook must be a callable.")
         ctx.user.ext_hook = ext_hook
 
-    ctx.user.encoding = encoding
     ctx.user.unicode_errors = unicode_errors
 
 def default_read_extended_type(typecode, data):
@@ -141,9 +139,9 @@ cdef inline int get_data_from_buffer(object obj,
                      1)
         return 1
 
-def unpackb(object packed, object object_hook=None, object list_hook=None,
+def unpackb(object packed, *, object object_hook=None, object list_hook=None,
             bint use_list=True, bint raw=True, bint strict_map_key=False,
-            encoding=None, unicode_errors=None,
+            unicode_errors=None,
             object_pairs_hook=None, ext_hook=ExtType,
             Py_ssize_t max_str_len=-1,
             Py_ssize_t max_bin_len=-1,
@@ -170,14 +168,9 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
     cdef Py_buffer view
     cdef char* buf = NULL
     cdef Py_ssize_t buf_len
-    cdef const char* cenc = NULL
     cdef const char* cerr = NULL
     cdef int new_protocol = 0
 
-    if encoding is not None:
-        PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
-        cenc = encoding
-
     if unicode_errors is not None:
         cerr = unicode_errors
 
@@ -196,7 +189,7 @@ def unpackb(object packed, object object_hook=None, object list_hook=None,
 
     try:
         init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
-                 use_list, raw, strict_map_key, cenc, cerr,
+                 use_list, raw, strict_map_key, cerr,
                  max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
         ret = unpack_construct(&ctx, buf, buf_len, &off)
     finally:
@@ -250,8 +243,6 @@ cdef class Unpacker(object):
         near future.  So you must specify it explicitly for keeping backward
         compatibility.
 
-        *encoding* option which is deprecated overrides this option.
-
     :param bool strict_map_key:
         If true, only str or bytes are accepted for map (dict) keys.
         It's False by default for backward-compatibility.
@@ -290,11 +281,6 @@ cdef class Unpacker(object):
         Deprecated, use *max_buffer_size* instead.
         Limits max size of ext type. (default: max_buffer_size or 1024*1024)
 
-    :param str encoding:
-        Deprecated, use ``raw=False`` instead.
-        Encoding used for decoding msgpack raw.
-        If it is None (default), msgpack raw is deserialized to Python bytes.
-
     :param str unicode_errors:
         Error handler used for decoding str type.  (default: `'strict'`)
 
@@ -330,7 +316,7 @@ cdef class Unpacker(object):
     cdef Py_ssize_t read_size
     # To maintain refcnt.
     cdef object object_hook, object_pairs_hook, list_hook, ext_hook
-    cdef object encoding, unicode_errors
+    cdef object unicode_errors
     cdef Py_ssize_t max_buffer_size
     cdef uint64_t stream_offset
 
@@ -341,17 +327,16 @@ cdef class Unpacker(object):
         PyMem_Free(self.buf)
         self.buf = NULL
 
-    def __init__(self, file_like=None, Py_ssize_t read_size=0,
+    def __init__(self, file_like=None, *, Py_ssize_t read_size=0,
                  bint use_list=True, bint raw=True, bint strict_map_key=False,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
-                 encoding=None, unicode_errors=None, Py_ssize_t max_buffer_size=0,
+                 unicode_errors=None, Py_ssize_t max_buffer_size=0,
                  object ext_hook=ExtType,
                  Py_ssize_t max_str_len=-1,
                  Py_ssize_t max_bin_len=-1,
                  Py_ssize_t max_array_len=-1,
                  Py_ssize_t max_map_len=-1,
                  Py_ssize_t max_ext_len=-1):
-        cdef const char *cenc=NULL,
         cdef const char *cerr=NULL
 
         self.object_hook = object_hook
@@ -392,17 +377,12 @@ cdef class Unpacker(object):
         self.buf_tail = 0
         self.stream_offset = 0
 
-        if encoding is not None:
-            PyErr_WarnEx(DeprecationWarning, "encoding is deprecated, Use raw=False instead.", 1)
-            self.encoding = encoding
-            cenc = encoding
-
         if unicode_errors is not None:
             self.unicode_errors = unicode_errors
             cerr = unicode_errors
 
         init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
-                 ext_hook, use_list, raw, strict_map_key, cenc, cerr,
+                 ext_hook, use_list, raw, strict_map_key, cerr,
                  max_str_len, max_bin_len, max_array_len,
                  max_map_len, max_ext_len)
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 0c0c101..9e31213 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -176,8 +176,6 @@ class Unpacker(object):
         near future.  So you must specify it explicitly for keeping backward
         compatibility.
 
-        *encoding* option which is deprecated overrides this option.
-
     :param bool strict_map_key:
         If true, only str or bytes are accepted for map (dict) keys.
         It's False by default for backward-compatibility.
@@ -193,13 +191,10 @@ class Unpacker(object):
         Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
         (See also simplejson)
 
-    :param str encoding:
-        Encoding used for decoding msgpack raw.
-        If it is None (default), msgpack raw is deserialized to Python bytes.
-
     :param str unicode_errors:
-        (deprecated) Used for decoding msgpack raw with *encoding*.
-        (default: `'strict'`)
+        The error handler for decoding unicode. (default: 'strict')
+        This option should be used only when you have msgpack data which
+        contains invalid UTF-8 string.
 
     :param int max_buffer_size:
         Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
@@ -252,18 +247,13 @@ class Unpacker(object):
 
     def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, strict_map_key=False,
                  object_hook=None, object_pairs_hook=None, list_hook=None,
-                 encoding=None, unicode_errors=None, max_buffer_size=0,
+                 unicode_errors=None, max_buffer_size=0,
                  ext_hook=ExtType,
                  max_str_len=-1,
                  max_bin_len=-1,
                  max_array_len=-1,
                  max_map_len=-1,
                  max_ext_len=-1):
-        if encoding is not None:
-            warnings.warn(
-                "encoding is deprecated, Use raw=False instead.",
-                DeprecationWarning, stacklevel=2)
-
         if unicode_errors is None:
             unicode_errors = 'strict'
 
@@ -306,7 +296,6 @@ class Unpacker(object):
         self._read_size = read_size or min(self._max_buffer_size, 16*1024)
         self._raw = bool(raw)
         self._strict_map_key = bool(strict_map_key)
-        self._encoding = encoding
         self._unicode_errors = unicode_errors
         self._use_list = use_list
         self._list_hook = list_hook
@@ -662,9 +651,7 @@ class Unpacker(object):
         if execute == EX_SKIP:
             return
         if typ == TYPE_RAW:
-            if self._encoding is not None:
-                obj = obj.decode(self._encoding, self._unicode_errors)
-            elif self._raw:
+            if self._raw:
                 obj = bytes(obj)
             else:
                 obj = obj.decode('utf_8', self._unicode_errors)
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index bbce91c..539a991 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -27,7 +27,6 @@ typedef struct unpack_user {
     PyObject *object_hook;
     PyObject *list_hook;
     PyObject *ext_hook;
-    const char *encoding;
     const char *unicode_errors;
     Py_ssize_t max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len;
 } unpack_user;
@@ -232,9 +231,7 @@ static inline int unpack_callback_raw(unpack_user* u, const char* b, const char*
 
     PyObject *py;
 
-    if (u->encoding) {
-        py = PyUnicode_Decode(p, l, u->encoding, u->unicode_errors);
-    } else if (u->raw) {
+    if (u->raw) {
         py = PyBytes_FromStringAndSize(p, l);
     } else {
         py = PyUnicode_DecodeUTF8(p, l, u->unicode_errors);

From 2c6668941f72e3bcb797d096437683eca4e3caf5 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 3 Dec 2019 21:18:17 +0900
Subject: [PATCH 1482/1648] Intern map keys (#381)

Fixes #372.
---
 msgpack/fallback.py | 2 ++
 msgpack/unpack.h    | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 9e31213..9a48b71 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -644,6 +644,8 @@ class Unpacker(object):
                     key = self._unpack(EX_CONSTRUCT)
                     if self._strict_map_key and type(key) not in (unicode, bytes):
                         raise ValueError("%s is not allowed for map key" % str(type(key)))
+                    if not PY2 and type(key) is str:
+                        key = sys.intern(key)
                     ret[key] = self._unpack(EX_CONSTRUCT)
                 if self._object_hook is not None:
                     ret = self._object_hook(ret)
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 539a991..ead5095 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -192,6 +192,9 @@ static inline int unpack_callback_map_item(unpack_user* u, unsigned int current,
         PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key", Py_TYPE(k)->tp_name);
         return -1;
     }
+    if (PyUnicode_CheckExact(k)) {
+        PyUnicode_InternInPlace(&k);
+    }
     if (u->has_pairs_hook) {
         msgpack_unpack_object item = PyTuple_Pack(2, k, v);
         if (!item)

From 641406902efaa8f22f4a7973d04c921a2a35a6be Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 5 Dec 2019 18:29:15 +0900
Subject: [PATCH 1483/1648] Add Timestamp support (#382)

---
 docs/api.rst            |   4 ++
 msgpack/__init__.py     |  14 +----
 msgpack/_packer.pyx     |   9 ++-
 msgpack/_unpacker.pyx   |   5 +-
 msgpack/ext.py          | 136 ++++++++++++++++++++++++++++++++++++++++
 msgpack/fallback.py     |  12 ++--
 msgpack/pack_template.h |  33 ++++++++++
 msgpack/unpack.h        |  44 ++++++++++++-
 test/test_timestamp.py  |  46 ++++++++++++++
 9 files changed, 283 insertions(+), 20 deletions(-)
 create mode 100644 msgpack/ext.py
 create mode 100644 test/test_timestamp.py

diff --git a/docs/api.rst b/docs/api.rst
index 6336793..93827e1 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -27,6 +27,10 @@ API reference
 
 .. autoclass:: ExtType
 
+.. autoclass:: Timestamp
+    :members:
+    :special-members: __init__
+
 exceptions
 ----------
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 4112a16..ff66f46 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,22 +1,10 @@
 # coding: utf-8
 from ._version import version
 from .exceptions import *
+from .ext import ExtType, Timestamp
 
 import os
 import sys
-from collections import namedtuple
-
-
-class ExtType(namedtuple('ExtType', 'code data')):
-    """ExtType represents ext type in msgpack."""
-    def __new__(cls, code, data):
-        if not isinstance(code, int):
-            raise TypeError("code must be int")
-        if not isinstance(data, bytes):
-            raise TypeError("data must be bytes")
-        if not 0 <= code <= 127:
-            raise ValueError("code must be 0~127")
-        return super(ExtType, cls).__new__(cls, code, data)
 
 
 if os.environ.get('MSGPACK_PUREPYTHON') or sys.version_info[0] == 2:
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 8b1a392..f3bde3f 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -4,8 +4,9 @@ from cpython cimport *
 from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact
 
 cdef ExtType
+cdef Timestamp
 
-from . import ExtType
+from .ext import ExtType, Timestamp
 
 
 cdef extern from "Python.h":
@@ -36,6 +37,7 @@ cdef extern from "pack.h":
     int msgpack_pack_bin(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
     int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l)
+    int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds);
     int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit)
 
 cdef extern from "buff_converter.h":
@@ -135,6 +137,7 @@ cdef class Packer(object):
     cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1:
         cdef long long llval
         cdef unsigned long long ullval
+        cdef unsigned long ulval
         cdef long longval
         cdef float fval
         cdef double dval
@@ -238,6 +241,10 @@ cdef class Packer(object):
                     raise ValueError("EXT data is too large")
                 ret = msgpack_pack_ext(&self.pk, longval, L)
                 ret = msgpack_pack_raw_body(&self.pk, rawval, L)
+            elif type(o) is Timestamp:
+                llval = o.seconds
+                ulval = o.nanoseconds
+                ret = msgpack_pack_timestamp(&self.pk, llval, ulval)
             elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
                 L = Py_SIZE(o)
                 if L > ITEM_LIMIT:
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index b258686..6dedd30 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -19,7 +19,7 @@ from .exceptions import (
     FormatError,
     StackError,
 )
-from . import ExtType
+from .ext import ExtType, Timestamp
 
 
 cdef extern from "unpack.h":
@@ -31,6 +31,7 @@ cdef extern from "unpack.h":
         PyObject* object_hook
         PyObject* list_hook
         PyObject* ext_hook
+        PyObject* timestamp_t
         char *unicode_errors
         Py_ssize_t max_str_len
         Py_ssize_t max_bin_len
@@ -98,6 +99,8 @@ cdef inline init_ctx(unpack_context *ctx,
             raise TypeError("ext_hook must be a callable.")
         ctx.user.ext_hook = ext_hook
 
+    # Add Timestamp type to the user object so it may be used in unpack.h
+    ctx.user.timestamp_t = Timestamp
     ctx.user.unicode_errors = unicode_errors
 
 def default_read_extended_type(typecode, data):
diff --git a/msgpack/ext.py b/msgpack/ext.py
new file mode 100644
index 0000000..1a0f8fe
--- /dev/null
+++ b/msgpack/ext.py
@@ -0,0 +1,136 @@
+# coding: utf-8
+from collections import namedtuple
+import sys
+import struct
+
+
+PY2 = sys.version_info[0] == 2
+if not PY2:
+    long = int
+
+
+class ExtType(namedtuple('ExtType', 'code data')):
+    """ExtType represents ext type in msgpack."""
+    def __new__(cls, code, data):
+        if not isinstance(code, int):
+            raise TypeError("code must be int")
+        if not isinstance(data, bytes):
+            raise TypeError("data must be bytes")
+        if code == -1:
+            return Timestamp.from_bytes(data)
+        if not 0 <= code <= 127:
+            raise ValueError("code must be 0~127")
+        return super(ExtType, cls).__new__(cls, code, data)
+
+
+class Timestamp(object):
+    """Timestamp represents the Timestamp extension type in msgpack.
+
+    When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python
+    msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and unpack `Timestamp`.
+    """
+    __slots__ = ["seconds", "nanoseconds"]
+
+    def __init__(self, seconds, nanoseconds=0):
+        """Initialize a Timestamp object.
+
+        :param seconds: Number of seconds since the UNIX epoch (00:00:00 UTC Jan 1 1970, minus leap seconds). May be
+            negative. If :code:`seconds` includes a fractional part, :code:`nanoseconds` must be 0.
+        :type seconds: int or float
+
+        :param nanoseconds: Number of nanoseconds to add to `seconds` to get fractional time. Maximum is 999_999_999.
+            Default is 0.
+        :type nanoseconds: int
+
+        Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns.
+        """
+        if not isinstance(seconds, (int, long, float)):
+            raise TypeError("seconds must be numeric")
+        if not isinstance(nanoseconds, (int, long)):
+            raise TypeError("nanoseconds must be an integer")
+        if nanoseconds:
+            if nanoseconds < 0 or nanoseconds % 1 != 0 or nanoseconds > (1e9 - 1):
+                raise ValueError("nanoseconds must be a non-negative integer less than 999999999.")
+            if not isinstance(seconds, (int, long)):
+                raise ValueError("seconds must be an integer if also providing nanoseconds.")
+            self.nanoseconds = nanoseconds
+        else:
+            # round helps with floating point issues
+            self.nanoseconds = int(round(seconds % 1 * 1e9, 0))
+        self.seconds = int(seconds // 1)
+
+    def __repr__(self):
+        """String representation of Timestamp."""
+        return "Timestamp(seconds={0}, nanoseconds={1})".format(self.seconds, self.nanoseconds)
+
+    def __eq__(self, other):
+        """Check for equality with another Timestamp object"""
+        if type(other) is self.__class__:
+            return self.seconds == other.seconds and self.nanoseconds == other.nanoseconds
+        return False
+
+    def __ne__(self, other):
+        """not-equals method (see :func:`__eq__()`)"""
+        return not self.__eq__(other)
+
+    @staticmethod
+    def from_bytes(b):
+        """Unpack bytes into a `Timestamp` object.
+
+        Used for pure-Python msgpack unpacking.
+
+        :param b: Payload from msgpack ext message with code -1
+        :type b: bytes
+
+        :returns: Timestamp object unpacked from msgpack ext payload
+        :rtype: Timestamp
+        """
+        if len(b) == 4:
+            seconds = struct.unpack("!L", b)[0]
+            nanoseconds = 0
+        elif len(b) == 8:
+            data64 = struct.unpack("!Q", b)[0]
+            seconds = data64 & 0x00000003ffffffff
+            nanoseconds = data64 >> 34
+        elif len(b) == 12:
+            nanoseconds, seconds = struct.unpack("!Iq", b)
+        else:
+            raise ValueError("Timestamp type can only be created from 32, 64, or 96-bit byte objects")
+        return Timestamp(seconds, nanoseconds)
+
+    def to_bytes(self):
+        """Pack this Timestamp object into bytes.
+
+        Used for pure-Python msgpack packing.
+
+        :returns data: Payload for EXT message with code -1 (timestamp type)
+        :rtype: bytes
+        """
+        if (self.seconds >> 34) == 0:  # seconds is non-negative and fits in 34 bits
+            data64 = self.nanoseconds << 34 | self.seconds
+            if data64 & 0xffffffff00000000 == 0:
+                # nanoseconds is zero and seconds < 2**32, so timestamp 32
+                data = struct.pack("!L", data64)
+            else:
+                # timestamp 64
+                data = struct.pack("!Q", data64)
+        else:
+            # timestamp 96
+            data = struct.pack("!Iq", self.nanoseconds, self.seconds)
+        return data
+
+    def to_float_s(self):
+        """Get the timestamp as a floating-point value.
+
+        :returns: posix timestamp
+        :rtype: float
+        """
+        return self.seconds + self.nanoseconds/1e9
+
+    def to_unix_ns(self):
+        """Get the timestamp as a unixtime in nanoseconds.
+
+        :returns: posix timestamp in nanoseconds
+        :rtype: int
+        """
+        return int(self.seconds * 1e9 + self.nanoseconds)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 9a48b71..55e66f5 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -66,7 +66,7 @@ from .exceptions import (
     StackError,
 )
 
-from . import ExtType
+from .ext import ExtType, Timestamp
 
 
 EX_SKIP                 = 0
@@ -826,9 +826,13 @@ class Packer(object):
                 if self._use_float:
                     return self._buffer.write(struct.pack(">Bf", 0xca, obj))
                 return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
-            if check(obj, ExtType):
-                code = obj.code
-                data = obj.data
+            if check(obj, (ExtType, Timestamp)):
+                if check(obj, Timestamp):
+                    code = -1
+                    data = obj.to_bytes()
+                else:
+                    code = obj.code
+                    data = obj.data
                 assert isinstance(code, int)
                 assert isinstance(data, bytes)
                 L = len(data)
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 69982f4..0e940b8 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -759,6 +759,39 @@ static inline int msgpack_pack_ext(msgpack_packer* x, char typecode, size_t l)
 
 }
 
+/*
+ * Pack Timestamp extension type. Follows msgpack-c pack_template.h.
+ */
+static inline int msgpack_pack_timestamp(msgpack_packer* x, int64_t seconds, uint32_t nanoseconds)
+{
+    if ((seconds >> 34) == 0) {
+        /* seconds is unsigned and fits in 34 bits */
+        uint64_t data64 = ((uint64_t)nanoseconds << 34) | (uint64_t)seconds;
+        if ((data64 & 0xffffffff00000000L) == 0) {
+            /* no nanoseconds and seconds is 32bits or smaller. timestamp32. */
+            unsigned char buf[4];
+            uint32_t data32 = (uint32_t)data64;
+            msgpack_pack_ext(x, -1, 4);
+            _msgpack_store32(buf, data32);
+            msgpack_pack_raw_body(x, buf, 4);
+        } else {
+            /* timestamp64 */
+            unsigned char buf[8];
+            msgpack_pack_ext(x, -1, 8);
+            _msgpack_store64(buf, data64);
+            msgpack_pack_raw_body(x, buf, 8);
+
+        }
+    } else {
+       /* seconds is signed or >34bits */
+       unsigned char buf[12];
+       _msgpack_store32(&buf[0], nanoseconds);
+       _msgpack_store64(&buf[4], seconds);
+       msgpack_pack_ext(x, -1, 12);
+       msgpack_pack_raw_body(x, buf, 12);
+    }
+    return 0;
+}
 
 
 #undef msgpack_pack_append_buffer
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index ead5095..4380ec5 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -27,6 +27,7 @@ typedef struct unpack_user {
     PyObject *object_hook;
     PyObject *list_hook;
     PyObject *ext_hook;
+    PyObject *timestamp_t;
     const char *unicode_errors;
     Py_ssize_t max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len;
 } unpack_user;
@@ -259,6 +260,38 @@ static inline int unpack_callback_bin(unpack_user* u, const char* b, const char*
     return 0;
 }
 
+typedef struct msgpack_timestamp {
+    int64_t tv_sec;
+    uint32_t tv_nsec;
+} msgpack_timestamp;
+
+/*
+ * Unpack ext buffer to a timestamp. Pulled from msgpack-c timestamp.h.
+ */
+static inline int unpack_timestamp(const char* buf, unsigned int buflen, msgpack_timestamp* ts) {
+    switch (buflen) {
+    case 4:
+        ts->tv_nsec = 0;
+        {
+            uint32_t v = _msgpack_load32(uint32_t, buf);
+            ts->tv_sec = (int64_t)v;
+        }
+        return 0;
+    case 8: {
+        uint64_t value =_msgpack_load64(uint64_t, buf);
+        ts->tv_nsec = (uint32_t)(value >> 34);
+        ts->tv_sec = value & 0x00000003ffffffffLL;
+        return 0;
+    }
+    case 12:
+        ts->tv_nsec = _msgpack_load32(uint32_t, buf);
+        ts->tv_sec = _msgpack_load64(int64_t, buf + 4);
+        return 0;
+    default:
+        return -1;
+    }
+}
+
 static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
                                       unsigned int length, msgpack_unpack_object* o)
 {
@@ -273,7 +306,16 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch
         return -1;
     }
     // length also includes the typecode, so the actual data is length-1
-    py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1);
+    if (typecode == -1) {
+        msgpack_timestamp ts;
+        if (unpack_timestamp(pos, length-1, &ts) == 0) {
+            py = PyObject_CallFunction(u->timestamp_t, "(Lk)", ts.tv_sec, ts.tv_nsec);
+        } else {
+            py = NULL;
+        }
+    } else {
+        py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1);
+    }
     if (!py)
         return -1;
     *o = py;
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
new file mode 100644
index 0000000..55c2f6d
--- /dev/null
+++ b/test/test_timestamp.py
@@ -0,0 +1,46 @@
+import msgpack
+from msgpack import Timestamp
+
+
+def test_timestamp():
+    # timestamp32
+    ts = Timestamp(2**32 - 1)
+    assert ts.to_bytes() == b"\xff\xff\xff\xff"
+    packed = msgpack.packb(ts)
+    assert packed == b"\xd6\xff" + ts.to_bytes()
+    unpacked = msgpack.unpackb(packed)
+    assert ts == unpacked
+    assert ts.seconds == 2**32 - 1 and ts.nanoseconds == 0
+
+    # timestamp64
+    ts = Timestamp(2**34 - 1, 999999999)
+    assert ts.to_bytes() == b"\xee\x6b\x27\xff\xff\xff\xff\xff"
+    packed = msgpack.packb(ts)
+    assert packed == b"\xd7\xff" + ts.to_bytes()
+    unpacked = msgpack.unpackb(packed)
+    assert ts == unpacked
+    assert ts.seconds == 2**34 - 1 and ts.nanoseconds == 999999999
+
+    # timestamp96
+    ts = Timestamp(2**63 - 1, 999999999)
+    assert ts.to_bytes() == b"\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff"
+    packed = msgpack.packb(ts)
+    assert packed == b"\xc7\x0c\xff" + ts.to_bytes()
+    unpacked = msgpack.unpackb(packed)
+    assert ts == unpacked
+    assert ts.seconds == 2**63 - 1 and ts.nanoseconds == 999999999
+
+    # negative fractional
+    ts = Timestamp(-2.3)  #s: -3, ns: 700000000
+    assert ts.to_bytes() == b"\x29\xb9\x27\x00\xff\xff\xff\xff\xff\xff\xff\xfd"
+    packed = msgpack.packb(ts)
+    assert packed == b"\xc7\x0c\xff" + ts.to_bytes()
+    unpacked = msgpack.unpackb(packed)
+    assert ts == unpacked
+    assert ts.seconds == -3 and ts.nanoseconds == 700000000
+
+
+def test_timestamp_to():
+    t = Timestamp(42, 14)
+    assert t.to_float_s() == 42.000000014
+    assert t.to_unix_ns() == 42000000014

From e557e17cbd4e88622e48547ac52834e9ab95f946 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 5 Dec 2019 18:50:13 +0900
Subject: [PATCH 1484/1648] blacken

---
 msgpack/__init__.py |   2 +-
 msgpack/ext.py      |  30 +++-
 msgpack/fallback.py | 393 +++++++++++++++++++++++++-------------------
 3 files changed, 245 insertions(+), 180 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index ff66f46..d6705e2 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -7,7 +7,7 @@ import os
 import sys
 
 
-if os.environ.get('MSGPACK_PUREPYTHON') or sys.version_info[0] == 2:
+if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2:
     from .fallback import Packer, unpackb, Unpacker
 else:
     try:
diff --git a/msgpack/ext.py b/msgpack/ext.py
index 1a0f8fe..c7efff6 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -9,8 +9,9 @@ if not PY2:
     long = int
 
 
-class ExtType(namedtuple('ExtType', 'code data')):
+class ExtType(namedtuple("ExtType", "code data")):
     """ExtType represents ext type in msgpack."""
+
     def __new__(cls, code, data):
         if not isinstance(code, int):
             raise TypeError("code must be int")
@@ -29,6 +30,7 @@ class Timestamp(object):
     When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python
     msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and unpack `Timestamp`.
     """
+
     __slots__ = ["seconds", "nanoseconds"]
 
     def __init__(self, seconds, nanoseconds=0):
@@ -50,9 +52,13 @@ class Timestamp(object):
             raise TypeError("nanoseconds must be an integer")
         if nanoseconds:
             if nanoseconds < 0 or nanoseconds % 1 != 0 or nanoseconds > (1e9 - 1):
-                raise ValueError("nanoseconds must be a non-negative integer less than 999999999.")
+                raise ValueError(
+                    "nanoseconds must be a non-negative integer less than 999999999."
+                )
             if not isinstance(seconds, (int, long)):
-                raise ValueError("seconds must be an integer if also providing nanoseconds.")
+                raise ValueError(
+                    "seconds must be an integer if also providing nanoseconds."
+                )
             self.nanoseconds = nanoseconds
         else:
             # round helps with floating point issues
@@ -61,12 +67,16 @@ class Timestamp(object):
 
     def __repr__(self):
         """String representation of Timestamp."""
-        return "Timestamp(seconds={0}, nanoseconds={1})".format(self.seconds, self.nanoseconds)
+        return "Timestamp(seconds={0}, nanoseconds={1})".format(
+            self.seconds, self.nanoseconds
+        )
 
     def __eq__(self, other):
         """Check for equality with another Timestamp object"""
         if type(other) is self.__class__:
-            return self.seconds == other.seconds and self.nanoseconds == other.nanoseconds
+            return (
+                self.seconds == other.seconds and self.nanoseconds == other.nanoseconds
+            )
         return False
 
     def __ne__(self, other):
@@ -90,12 +100,14 @@ class Timestamp(object):
             nanoseconds = 0
         elif len(b) == 8:
             data64 = struct.unpack("!Q", b)[0]
-            seconds = data64 & 0x00000003ffffffff
+            seconds = data64 & 0x00000003FFFFFFFF
             nanoseconds = data64 >> 34
         elif len(b) == 12:
             nanoseconds, seconds = struct.unpack("!Iq", b)
         else:
-            raise ValueError("Timestamp type can only be created from 32, 64, or 96-bit byte objects")
+            raise ValueError(
+                "Timestamp type can only be created from 32, 64, or 96-bit byte objects"
+            )
         return Timestamp(seconds, nanoseconds)
 
     def to_bytes(self):
@@ -108,7 +120,7 @@ class Timestamp(object):
         """
         if (self.seconds >> 34) == 0:  # seconds is non-negative and fits in 34 bits
             data64 = self.nanoseconds << 34 | self.seconds
-            if data64 & 0xffffffff00000000 == 0:
+            if data64 & 0xFFFFFFFF00000000 == 0:
                 # nanoseconds is zero and seconds < 2**32, so timestamp 32
                 data = struct.pack("!L", data64)
             else:
@@ -125,7 +137,7 @@ class Timestamp(object):
         :returns: posix timestamp
         :rtype: float
         """
-        return self.seconds + self.nanoseconds/1e9
+        return self.seconds + self.nanoseconds / 1e9
 
     def to_unix_ns(self):
         """Get the timestamp as a unixtime in nanoseconds.
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 55e66f5..577e571 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -8,53 +8,72 @@ import warnings
 PY2 = sys.version_info[0] == 2
 if PY2:
     int_types = (int, long)
+
     def dict_iteritems(d):
         return d.iteritems()
+
+
 else:
     int_types = int
     unicode = str
     xrange = range
+
     def dict_iteritems(d):
         return d.items()
 
+
 if sys.version_info < (3, 5):
     # Ugly hack...
     RecursionError = RuntimeError
 
     def _is_recursionerror(e):
-        return len(e.args) == 1 and isinstance(e.args[0], str) and \
-            e.args[0].startswith('maximum recursion depth exceeded')
+        return (
+            len(e.args) == 1
+            and isinstance(e.args[0], str)
+            and e.args[0].startswith("maximum recursion depth exceeded")
+        )
+
+
 else:
+
     def _is_recursionerror(e):
         return True
 
-if hasattr(sys, 'pypy_version_info'):
+
+if hasattr(sys, "pypy_version_info"):
     # cStringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
     # StringBuilder is fastest.
     from __pypy__ import newlist_hint
+
     try:
         from __pypy__.builders import BytesBuilder as StringBuilder
     except ImportError:
         from __pypy__.builders import StringBuilder
     USING_STRINGBUILDER = True
+
     class StringIO(object):
-        def __init__(self, s=b''):
+        def __init__(self, s=b""):
             if s:
                 self.builder = StringBuilder(len(s))
                 self.builder.append(s)
             else:
                 self.builder = StringBuilder()
+
         def write(self, s):
             if isinstance(s, memoryview):
                 s = s.tobytes()
             elif isinstance(s, bytearray):
                 s = bytes(s)
             self.builder.append(s)
+
         def getvalue(self):
             return self.builder.build()
+
+
 else:
     USING_STRINGBUILDER = False
     from io import BytesIO as StringIO
+
     newlist_hint = lambda size: []
 
 
@@ -69,17 +88,17 @@ from .exceptions import (
 from .ext import ExtType, Timestamp
 
 
-EX_SKIP                 = 0
-EX_CONSTRUCT            = 1
-EX_READ_ARRAY_HEADER    = 2
-EX_READ_MAP_HEADER      = 3
+EX_SKIP = 0
+EX_CONSTRUCT = 1
+EX_READ_ARRAY_HEADER = 2
+EX_READ_MAP_HEADER = 3
 
-TYPE_IMMEDIATE          = 0
-TYPE_ARRAY              = 1
-TYPE_MAP                = 2
-TYPE_RAW                = 3
-TYPE_BIN                = 4
-TYPE_EXT                = 5
+TYPE_IMMEDIATE = 0
+TYPE_ARRAY = 1
+TYPE_MAP = 2
+TYPE_RAW = 3
+TYPE_BIN = 4
+TYPE_EXT = 5
 
 DEFAULT_RECURSE_LIMIT = 511
 
@@ -98,10 +117,13 @@ def _get_data_from_buffer(obj):
         # try to use legacy buffer protocol if 2.7, otherwise re-raise
         if PY2:
             view = memoryview(buffer(obj))
-            warnings.warn("using old buffer interface to unpack %s; "
-                          "this leads to unpacking errors if slicing is used and "
-                          "will be removed in a future version" % type(obj),
-                          RuntimeWarning, stacklevel=3)
+            warnings.warn(
+                "using old buffer interface to unpack %s; "
+                "this leads to unpacking errors if slicing is used and "
+                "will be removed in a future version" % type(obj),
+                RuntimeWarning,
+                stacklevel=3,
+            )
         else:
             raise
     if view.itemsize != 1:
@@ -112,7 +134,9 @@ def _get_data_from_buffer(obj):
 def unpack(stream, **kwargs):
     warnings.warn(
         "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
-        DeprecationWarning, stacklevel=2)
+        DeprecationWarning,
+        stacklevel=2,
+    )
     data = stream.read()
     return unpackb(data, **kwargs)
 
@@ -145,9 +169,12 @@ def unpackb(packed, **kwargs):
 
 
 if sys.version_info < (2, 7, 6):
+
     def _unpack_from(f, b, o=0):
         """Explicit typcast for legacy struct.unpack_from"""
         return struct.unpack_from(f, bytes(b), o)
+
+
 else:
     _unpack_from = struct.unpack_from
 
@@ -245,17 +272,27 @@ class Unpacker(object):
     Other exceptions can be raised during unpacking.
     """
 
-    def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, strict_map_key=False,
-                 object_hook=None, object_pairs_hook=None, list_hook=None,
-                 unicode_errors=None, max_buffer_size=0,
-                 ext_hook=ExtType,
-                 max_str_len=-1,
-                 max_bin_len=-1,
-                 max_array_len=-1,
-                 max_map_len=-1,
-                 max_ext_len=-1):
+    def __init__(
+        self,
+        file_like=None,
+        read_size=0,
+        use_list=True,
+        raw=True,
+        strict_map_key=False,
+        object_hook=None,
+        object_pairs_hook=None,
+        list_hook=None,
+        unicode_errors=None,
+        max_buffer_size=0,
+        ext_hook=ExtType,
+        max_str_len=-1,
+        max_bin_len=-1,
+        max_array_len=-1,
+        max_map_len=-1,
+        max_ext_len=-1,
+    ):
         if unicode_errors is None:
-            unicode_errors = 'strict'
+            unicode_errors = "strict"
 
         if file_like is None:
             self._feeding = True
@@ -280,20 +317,20 @@ class Unpacker(object):
         self._buf_checkpoint = 0
 
         if max_str_len == -1:
-            max_str_len = max_buffer_size or 1024*1024
+            max_str_len = max_buffer_size or 1024 * 1024
         if max_bin_len == -1:
-            max_bin_len = max_buffer_size or 1024*1024
+            max_bin_len = max_buffer_size or 1024 * 1024
         if max_array_len == -1:
-            max_array_len = max_buffer_size or 128*1024
+            max_array_len = max_buffer_size or 128 * 1024
         if max_map_len == -1:
-            max_map_len = max_buffer_size//2 or 32*1024
+            max_map_len = max_buffer_size // 2 or 32 * 1024
         if max_ext_len == -1:
-            max_ext_len = max_buffer_size or 1024*1024
+            max_ext_len = max_buffer_size or 1024 * 1024
 
-        self._max_buffer_size = max_buffer_size or 2**31-1
+        self._max_buffer_size = max_buffer_size or 2 ** 31 - 1
         if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")
-        self._read_size = read_size or min(self._max_buffer_size, 16*1024)
+        self._read_size = read_size or min(self._max_buffer_size, 16 * 1024)
         self._raw = bool(raw)
         self._strict_map_key = bool(strict_map_key)
         self._unicode_errors = unicode_errors
@@ -310,26 +347,27 @@ class Unpacker(object):
         self._stream_offset = 0
 
         if list_hook is not None and not callable(list_hook):
-            raise TypeError('`list_hook` is not callable')
+            raise TypeError("`list_hook` is not callable")
         if object_hook is not None and not callable(object_hook):
-            raise TypeError('`object_hook` is not callable')
+            raise TypeError("`object_hook` is not callable")
         if object_pairs_hook is not None and not callable(object_pairs_hook):
-            raise TypeError('`object_pairs_hook` is not callable')
+            raise TypeError("`object_pairs_hook` is not callable")
         if object_hook is not None and object_pairs_hook is not None:
-            raise TypeError("object_pairs_hook and object_hook are mutually "
-                            "exclusive")
+            raise TypeError(
+                "object_pairs_hook and object_hook are mutually " "exclusive"
+            )
         if not callable(ext_hook):
             raise TypeError("`ext_hook` is not callable")
 
     def feed(self, next_bytes):
         assert self._feeding
         view = _get_data_from_buffer(next_bytes)
-        if (len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size):
+        if len(self._buffer) - self._buff_i + len(view) > self._max_buffer_size:
             raise BufferFull
 
         # Strip buffer before checkpoint before reading file.
         if self._buf_checkpoint > 0:
-            del self._buffer[:self._buf_checkpoint]
+            del self._buffer[: self._buf_checkpoint]
             self._buff_i -= self._buf_checkpoint
             self._buf_checkpoint = 0
 
@@ -345,7 +383,7 @@ class Unpacker(object):
         return self._buff_i < len(self._buffer)
 
     def _get_extradata(self):
-        return self._buffer[self._buff_i:]
+        return self._buffer[self._buff_i :]
 
     def read_bytes(self, n):
         return self._read(n)
@@ -354,8 +392,8 @@ class Unpacker(object):
         # (int) -> bytearray
         self._reserve(n)
         i = self._buff_i
-        self._buff_i = i+n
-        return self._buffer[i:i+n]
+        self._buff_i = i + n
+        return self._buffer[i : i + n]
 
     def _reserve(self, n):
         remain_bytes = len(self._buffer) - self._buff_i - n
@@ -370,7 +408,7 @@ class Unpacker(object):
 
         # Strip buffer before checkpoint before reading file.
         if self._buf_checkpoint > 0:
-            del self._buffer[:self._buf_checkpoint]
+            del self._buffer[: self._buf_checkpoint]
             self._buff_i -= self._buf_checkpoint
             self._buf_checkpoint = 0
 
@@ -399,7 +437,7 @@ class Unpacker(object):
         if b & 0b10000000 == 0:
             obj = b
         elif b & 0b11100000 == 0b11100000:
-            obj = -1 - (b ^ 0xff)
+            obj = -1 - (b ^ 0xFF)
         elif b & 0b11100000 == 0b10100000:
             n = b & 0b00011111
             typ = TYPE_RAW
@@ -416,13 +454,13 @@ class Unpacker(object):
             typ = TYPE_MAP
             if n > self._max_map_len:
                 raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
-        elif b == 0xc0:
+        elif b == 0xC0:
             obj = None
-        elif b == 0xc2:
+        elif b == 0xC2:
             obj = False
-        elif b == 0xc3:
+        elif b == 0xC3:
             obj = True
-        elif b == 0xc4:
+        elif b == 0xC4:
             typ = TYPE_BIN
             self._reserve(1)
             n = self._buffer[self._buff_i]
@@ -430,7 +468,7 @@ class Unpacker(object):
             if n > self._max_bin_len:
                 raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
-        elif b == 0xc5:
+        elif b == 0xC5:
             typ = TYPE_BIN
             self._reserve(2)
             n = _unpack_from(">H", self._buffer, self._buff_i)[0]
@@ -438,7 +476,7 @@ class Unpacker(object):
             if n > self._max_bin_len:
                 raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
-        elif b == 0xc6:
+        elif b == 0xC6:
             typ = TYPE_BIN
             self._reserve(4)
             n = _unpack_from(">I", self._buffer, self._buff_i)[0]
@@ -446,106 +484,106 @@ class Unpacker(object):
             if n > self._max_bin_len:
                 raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
-        elif b == 0xc7:  # ext 8
+        elif b == 0xC7:  # ext 8
             typ = TYPE_EXT
             self._reserve(2)
-            L, n = _unpack_from('Bb', self._buffer, self._buff_i)
+            L, n = _unpack_from("Bb", self._buffer, self._buff_i)
             self._buff_i += 2
             if L > self._max_ext_len:
                 raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
-        elif b == 0xc8:  # ext 16
+        elif b == 0xC8:  # ext 16
             typ = TYPE_EXT
             self._reserve(3)
-            L, n = _unpack_from('>Hb', self._buffer, self._buff_i)
+            L, n = _unpack_from(">Hb", self._buffer, self._buff_i)
             self._buff_i += 3
             if L > self._max_ext_len:
                 raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
-        elif b == 0xc9:  # ext 32
+        elif b == 0xC9:  # ext 32
             typ = TYPE_EXT
             self._reserve(5)
-            L, n = _unpack_from('>Ib', self._buffer, self._buff_i)
+            L, n = _unpack_from(">Ib", self._buffer, self._buff_i)
             self._buff_i += 5
             if L > self._max_ext_len:
                 raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
-        elif b == 0xca:
+        elif b == 0xCA:
             self._reserve(4)
             obj = _unpack_from(">f", self._buffer, self._buff_i)[0]
             self._buff_i += 4
-        elif b == 0xcb:
+        elif b == 0xCB:
             self._reserve(8)
             obj = _unpack_from(">d", self._buffer, self._buff_i)[0]
             self._buff_i += 8
-        elif b == 0xcc:
+        elif b == 0xCC:
             self._reserve(1)
             obj = self._buffer[self._buff_i]
             self._buff_i += 1
-        elif b == 0xcd:
+        elif b == 0xCD:
             self._reserve(2)
             obj = _unpack_from(">H", self._buffer, self._buff_i)[0]
             self._buff_i += 2
-        elif b == 0xce:
+        elif b == 0xCE:
             self._reserve(4)
             obj = _unpack_from(">I", self._buffer, self._buff_i)[0]
             self._buff_i += 4
-        elif b == 0xcf:
+        elif b == 0xCF:
             self._reserve(8)
             obj = _unpack_from(">Q", self._buffer, self._buff_i)[0]
             self._buff_i += 8
-        elif b == 0xd0:
+        elif b == 0xD0:
             self._reserve(1)
             obj = _unpack_from("b", self._buffer, self._buff_i)[0]
             self._buff_i += 1
-        elif b == 0xd1:
+        elif b == 0xD1:
             self._reserve(2)
             obj = _unpack_from(">h", self._buffer, self._buff_i)[0]
             self._buff_i += 2
-        elif b == 0xd2:
+        elif b == 0xD2:
             self._reserve(4)
             obj = _unpack_from(">i", self._buffer, self._buff_i)[0]
             self._buff_i += 4
-        elif b == 0xd3:
+        elif b == 0xD3:
             self._reserve(8)
             obj = _unpack_from(">q", self._buffer, self._buff_i)[0]
             self._buff_i += 8
-        elif b == 0xd4:  # fixext 1
+        elif b == 0xD4:  # fixext 1
             typ = TYPE_EXT
             if self._max_ext_len < 1:
                 raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
             self._reserve(2)
             n, obj = _unpack_from("b1s", self._buffer, self._buff_i)
             self._buff_i += 2
-        elif b == 0xd5:  # fixext 2
+        elif b == 0xD5:  # fixext 2
             typ = TYPE_EXT
             if self._max_ext_len < 2:
                 raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
             self._reserve(3)
             n, obj = _unpack_from("b2s", self._buffer, self._buff_i)
             self._buff_i += 3
-        elif b == 0xd6:  # fixext 4
+        elif b == 0xD6:  # fixext 4
             typ = TYPE_EXT
             if self._max_ext_len < 4:
                 raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
             self._reserve(5)
             n, obj = _unpack_from("b4s", self._buffer, self._buff_i)
             self._buff_i += 5
-        elif b == 0xd7:  # fixext 8
+        elif b == 0xD7:  # fixext 8
             typ = TYPE_EXT
             if self._max_ext_len < 8:
                 raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
             self._reserve(9)
             n, obj = _unpack_from("b8s", self._buffer, self._buff_i)
             self._buff_i += 9
-        elif b == 0xd8:  # fixext 16
+        elif b == 0xD8:  # fixext 16
             typ = TYPE_EXT
             if self._max_ext_len < 16:
                 raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
             self._reserve(17)
             n, obj = _unpack_from("b16s", self._buffer, self._buff_i)
             self._buff_i += 17
-        elif b == 0xd9:
+        elif b == 0xD9:
             typ = TYPE_RAW
             self._reserve(1)
             n = self._buffer[self._buff_i]
@@ -553,46 +591,46 @@ class Unpacker(object):
             if n > self._max_str_len:
                 raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
-        elif b == 0xda:
+        elif b == 0xDA:
             typ = TYPE_RAW
             self._reserve(2)
-            n, = _unpack_from(">H", self._buffer, self._buff_i)
+            (n,) = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_str_len:
                 raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
-        elif b == 0xdb:
+        elif b == 0xDB:
             typ = TYPE_RAW
             self._reserve(4)
-            n, = _unpack_from(">I", self._buffer, self._buff_i)
+            (n,) = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_str_len:
                 raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
-        elif b == 0xdc:
+        elif b == 0xDC:
             typ = TYPE_ARRAY
             self._reserve(2)
-            n, = _unpack_from(">H", self._buffer, self._buff_i)
+            (n,) = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_array_len:
                 raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
-        elif b == 0xdd:
+        elif b == 0xDD:
             typ = TYPE_ARRAY
             self._reserve(4)
-            n, = _unpack_from(">I", self._buffer, self._buff_i)
+            (n,) = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_array_len:
                 raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
-        elif b == 0xde:
+        elif b == 0xDE:
             self._reserve(2)
-            n, = _unpack_from(">H", self._buffer, self._buff_i)
+            (n,) = _unpack_from(">H", self._buffer, self._buff_i)
             self._buff_i += 2
             if n > self._max_map_len:
                 raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
             typ = TYPE_MAP
-        elif b == 0xdf:
+        elif b == 0xDF:
             self._reserve(4)
-            n, = _unpack_from(">I", self._buffer, self._buff_i)
+            (n,) = _unpack_from(">I", self._buffer, self._buff_i)
             self._buff_i += 4
             if n > self._max_map_len:
                 raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
@@ -635,15 +673,17 @@ class Unpacker(object):
                 return
             if self._object_pairs_hook is not None:
                 ret = self._object_pairs_hook(
-                    (self._unpack(EX_CONSTRUCT),
-                     self._unpack(EX_CONSTRUCT))
-                    for _ in xrange(n))
+                    (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT))
+                    for _ in xrange(n)
+                )
             else:
                 ret = {}
                 for _ in xrange(n):
                     key = self._unpack(EX_CONSTRUCT)
                     if self._strict_map_key and type(key) not in (unicode, bytes):
-                        raise ValueError("%s is not allowed for map key" % str(type(key)))
+                        raise ValueError(
+                            "%s is not allowed for map key" % str(type(key))
+                        )
                     if not PY2 and type(key) is str:
                         key = sys.intern(key)
                     ret[key] = self._unpack(EX_CONSTRUCT)
@@ -656,7 +696,7 @@ class Unpacker(object):
             if self._raw:
                 obj = bytes(obj)
             else:
-                obj = obj.decode('utf_8', self._unicode_errors)
+                obj = obj.decode("utf_8", self._unicode_errors)
             return obj
         if typ == TYPE_EXT:
             return self._ext_hook(n, bytes(obj))
@@ -746,9 +786,16 @@ class Packer(object):
         The error handler for encoding unicode. (default: 'strict')
         DO NOT USE THIS!!  This option is kept for very specific usage.
     """
-    def __init__(self, default=None, unicode_errors=None,
-                 use_single_float=False, autoreset=True, use_bin_type=False,
-                 strict_types=False):
+
+    def __init__(
+        self,
+        default=None,
+        unicode_errors=None,
+        use_single_float=False,
+        autoreset=True,
+        use_bin_type=False,
+        strict_types=False,
+    ):
         self._strict_types = strict_types
         self._use_float = use_single_float
         self._autoreset = autoreset
@@ -760,8 +807,13 @@ class Packer(object):
                 raise TypeError("default must be callable")
         self._default = default
 
-    def _pack(self, obj, nest_limit=DEFAULT_RECURSE_LIMIT,
-              check=isinstance, check_type_strict=_check_type_strict):
+    def _pack(
+        self,
+        obj,
+        nest_limit=DEFAULT_RECURSE_LIMIT,
+        check=isinstance,
+        check_type_strict=_check_type_strict,
+    ):
         default_used = False
         if self._strict_types:
             check = check_type_strict
@@ -782,22 +834,22 @@ class Packer(object):
                     return self._buffer.write(struct.pack("B", obj))
                 if -0x20 <= obj < 0:
                     return self._buffer.write(struct.pack("b", obj))
-                if 0x80 <= obj <= 0xff:
-                    return self._buffer.write(struct.pack("BB", 0xcc, obj))
+                if 0x80 <= obj <= 0xFF:
+                    return self._buffer.write(struct.pack("BB", 0xCC, obj))
                 if -0x80 <= obj < 0:
-                    return self._buffer.write(struct.pack(">Bb", 0xd0, obj))
-                if 0xff < obj <= 0xffff:
-                    return self._buffer.write(struct.pack(">BH", 0xcd, obj))
+                    return self._buffer.write(struct.pack(">Bb", 0xD0, obj))
+                if 0xFF < obj <= 0xFFFF:
+                    return self._buffer.write(struct.pack(">BH", 0xCD, obj))
                 if -0x8000 <= obj < -0x80:
-                    return self._buffer.write(struct.pack(">Bh", 0xd1, obj))
-                if 0xffff < obj <= 0xffffffff:
-                    return self._buffer.write(struct.pack(">BI", 0xce, obj))
+                    return self._buffer.write(struct.pack(">Bh", 0xD1, obj))
+                if 0xFFFF < obj <= 0xFFFFFFFF:
+                    return self._buffer.write(struct.pack(">BI", 0xCE, obj))
                 if -0x80000000 <= obj < -0x8000:
-                    return self._buffer.write(struct.pack(">Bi", 0xd2, obj))
-                if 0xffffffff < obj <= 0xffffffffffffffff:
-                    return self._buffer.write(struct.pack(">BQ", 0xcf, obj))
+                    return self._buffer.write(struct.pack(">Bi", 0xD2, obj))
+                if 0xFFFFFFFF < obj <= 0xFFFFFFFFFFFFFFFF:
+                    return self._buffer.write(struct.pack(">BQ", 0xCF, obj))
                 if -0x8000000000000000 <= obj < -0x80000000:
-                    return self._buffer.write(struct.pack(">Bq", 0xd3, obj))
+                    return self._buffer.write(struct.pack(">Bq", 0xD3, obj))
                 if not default_used and self._default is not None:
                     obj = self._default(obj)
                     default_used = True
@@ -805,27 +857,27 @@ class Packer(object):
                 raise OverflowError("Integer value out of range")
             if check(obj, (bytes, bytearray)):
                 n = len(obj)
-                if n >= 2**32:
+                if n >= 2 ** 32:
                     raise ValueError("%s is too large" % type(obj).__name__)
                 self._pack_bin_header(n)
                 return self._buffer.write(obj)
             if check(obj, unicode):
                 obj = obj.encode("utf-8", self._unicode_errors)
                 n = len(obj)
-                if n >= 2**32:
+                if n >= 2 ** 32:
                     raise ValueError("String is too large")
                 self._pack_raw_header(n)
                 return self._buffer.write(obj)
             if check(obj, memoryview):
                 n = len(obj) * obj.itemsize
-                if n >= 2**32:
+                if n >= 2 ** 32:
                     raise ValueError("Memoryview is too large")
                 self._pack_bin_header(n)
                 return self._buffer.write(obj)
             if check(obj, float):
                 if self._use_float:
-                    return self._buffer.write(struct.pack(">Bf", 0xca, obj))
-                return self._buffer.write(struct.pack(">Bd", 0xcb, obj))
+                    return self._buffer.write(struct.pack(">Bf", 0xCA, obj))
+                return self._buffer.write(struct.pack(">Bd", 0xCB, obj))
             if check(obj, (ExtType, Timestamp)):
                 if check(obj, Timestamp):
                     code = -1
@@ -837,21 +889,21 @@ class Packer(object):
                 assert isinstance(data, bytes)
                 L = len(data)
                 if L == 1:
-                    self._buffer.write(b'\xd4')
+                    self._buffer.write(b"\xd4")
                 elif L == 2:
-                    self._buffer.write(b'\xd5')
+                    self._buffer.write(b"\xd5")
                 elif L == 4:
-                    self._buffer.write(b'\xd6')
+                    self._buffer.write(b"\xd6")
                 elif L == 8:
-                    self._buffer.write(b'\xd7')
+                    self._buffer.write(b"\xd7")
                 elif L == 16:
-                    self._buffer.write(b'\xd8')
-                elif L <= 0xff:
-                    self._buffer.write(struct.pack(">BB", 0xc7, L))
-                elif L <= 0xffff:
-                    self._buffer.write(struct.pack(">BH", 0xc8, L))
+                    self._buffer.write(b"\xd8")
+                elif L <= 0xFF:
+                    self._buffer.write(struct.pack(">BB", 0xC7, L))
+                elif L <= 0xFFFF:
+                    self._buffer.write(struct.pack(">BH", 0xC8, L))
                 else:
-                    self._buffer.write(struct.pack(">BI", 0xc9, L))
+                    self._buffer.write(struct.pack(">BI", 0xC9, L))
                 self._buffer.write(struct.pack("b", code))
                 self._buffer.write(data)
                 return
@@ -862,13 +914,14 @@ class Packer(object):
                     self._pack(obj[i], nest_limit - 1)
                 return
             if check(obj, dict):
-                return self._pack_map_pairs(len(obj), dict_iteritems(obj),
-                                               nest_limit - 1)
+                return self._pack_map_pairs(
+                    len(obj), dict_iteritems(obj), nest_limit - 1
+                )
             if not default_used and self._default is not None:
                 obj = self._default(obj)
                 default_used = 1
                 continue
-            raise TypeError("Cannot serialize %r" % (obj, ))
+            raise TypeError("Cannot serialize %r" % (obj,))
 
     def pack(self, obj):
         try:
@@ -889,7 +942,7 @@ class Packer(object):
             return ret
 
     def pack_array_header(self, n):
-        if n >= 2**32:
+        if n >= 2 ** 32:
             raise ValueError
         self._pack_array_header(n)
         if self._autoreset:
@@ -898,7 +951,7 @@ class Packer(object):
             return ret
 
     def pack_map_header(self, n):
-        if n >= 2**32:
+        if n >= 2 ** 32:
             raise ValueError
         self._pack_map_header(n)
         if self._autoreset:
@@ -914,43 +967,43 @@ class Packer(object):
         if not isinstance(data, bytes):
             raise TypeError("data must have bytes type")
         L = len(data)
-        if L > 0xffffffff:
+        if L > 0xFFFFFFFF:
             raise ValueError("Too large data")
         if L == 1:
-            self._buffer.write(b'\xd4')
+            self._buffer.write(b"\xd4")
         elif L == 2:
-            self._buffer.write(b'\xd5')
+            self._buffer.write(b"\xd5")
         elif L == 4:
-            self._buffer.write(b'\xd6')
+            self._buffer.write(b"\xd6")
         elif L == 8:
-            self._buffer.write(b'\xd7')
+            self._buffer.write(b"\xd7")
         elif L == 16:
-            self._buffer.write(b'\xd8')
-        elif L <= 0xff:
-            self._buffer.write(b'\xc7' + struct.pack('B', L))
-        elif L <= 0xffff:
-            self._buffer.write(b'\xc8' + struct.pack('>H', L))
+            self._buffer.write(b"\xd8")
+        elif L <= 0xFF:
+            self._buffer.write(b"\xc7" + struct.pack("B", L))
+        elif L <= 0xFFFF:
+            self._buffer.write(b"\xc8" + struct.pack(">H", L))
         else:
-            self._buffer.write(b'\xc9' + struct.pack('>I', L))
-        self._buffer.write(struct.pack('B', typecode))
+            self._buffer.write(b"\xc9" + struct.pack(">I", L))
+        self._buffer.write(struct.pack("B", typecode))
         self._buffer.write(data)
 
     def _pack_array_header(self, n):
-        if n <= 0x0f:
-            return self._buffer.write(struct.pack('B', 0x90 + n))
-        if n <= 0xffff:
-            return self._buffer.write(struct.pack(">BH", 0xdc, n))
-        if n <= 0xffffffff:
-            return self._buffer.write(struct.pack(">BI", 0xdd, n))
+        if n <= 0x0F:
+            return self._buffer.write(struct.pack("B", 0x90 + n))
+        if n <= 0xFFFF:
+            return self._buffer.write(struct.pack(">BH", 0xDC, n))
+        if n <= 0xFFFFFFFF:
+            return self._buffer.write(struct.pack(">BI", 0xDD, n))
         raise ValueError("Array is too large")
 
     def _pack_map_header(self, n):
-        if n <= 0x0f:
-            return self._buffer.write(struct.pack('B', 0x80 + n))
-        if n <= 0xffff:
-            return self._buffer.write(struct.pack(">BH", 0xde, n))
-        if n <= 0xffffffff:
-            return self._buffer.write(struct.pack(">BI", 0xdf, n))
+        if n <= 0x0F:
+            return self._buffer.write(struct.pack("B", 0x80 + n))
+        if n <= 0xFFFF:
+            return self._buffer.write(struct.pack(">BH", 0xDE, n))
+        if n <= 0xFFFFFFFF:
+            return self._buffer.write(struct.pack(">BI", 0xDF, n))
         raise ValueError("Dict is too large")
 
     def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
@@ -960,28 +1013,28 @@ class Packer(object):
             self._pack(v, nest_limit - 1)
 
     def _pack_raw_header(self, n):
-        if n <= 0x1f:
-            self._buffer.write(struct.pack('B', 0xa0 + n))
-        elif self._use_bin_type and n <= 0xff:
-            self._buffer.write(struct.pack('>BB', 0xd9, n))
-        elif n <= 0xffff:
-            self._buffer.write(struct.pack(">BH", 0xda, n))
-        elif n <= 0xffffffff:
-            self._buffer.write(struct.pack(">BI", 0xdb, n))
+        if n <= 0x1F:
+            self._buffer.write(struct.pack("B", 0xA0 + n))
+        elif self._use_bin_type and n <= 0xFF:
+            self._buffer.write(struct.pack(">BB", 0xD9, n))
+        elif n <= 0xFFFF:
+            self._buffer.write(struct.pack(">BH", 0xDA, n))
+        elif n <= 0xFFFFFFFF:
+            self._buffer.write(struct.pack(">BI", 0xDB, n))
         else:
-            raise ValueError('Raw is too large')
+            raise ValueError("Raw is too large")
 
     def _pack_bin_header(self, n):
         if not self._use_bin_type:
             return self._pack_raw_header(n)
-        elif n <= 0xff:
-            return self._buffer.write(struct.pack('>BB', 0xc4, n))
-        elif n <= 0xffff:
-            return self._buffer.write(struct.pack(">BH", 0xc5, n))
-        elif n <= 0xffffffff:
-            return self._buffer.write(struct.pack(">BI", 0xc6, n))
+        elif n <= 0xFF:
+            return self._buffer.write(struct.pack(">BB", 0xC4, n))
+        elif n <= 0xFFFF:
+            return self._buffer.write(struct.pack(">BH", 0xC5, n))
+        elif n <= 0xFFFFFFFF:
+            return self._buffer.write(struct.pack(">BI", 0xC6, n))
         else:
-            raise ValueError('Bin is too large')
+            raise ValueError("Bin is too large")
 
     def bytes(self):
         """Return internal buffer contents as bytes object"""

From 10e5e39ff9739fa3ce589ad9d451260be0f3842c Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 5 Dec 2019 18:51:45 +0900
Subject: [PATCH 1485/1648] blacken test

---
 test/test_buffer.py     |  17 +++---
 test/test_case.py       | 116 +++++++++++++++++++++++++-------------
 test/test_extension.py  |  55 ++++++++++--------
 test/test_format.py     |  92 ++++++++++++++++++------------
 test/test_limits.py     |  46 ++++++++-------
 test/test_memoryview.py |  47 ++++++++--------
 test/test_newspec.py    |  60 ++++++++++----------
 test/test_obj.py        |  42 +++++++++-----
 test/test_pack.py       | 102 ++++++++++++++++++++++++---------
 test/test_read_size.py  |  49 ++++++++--------
 test/test_seq.py        |  11 ++--
 test/test_sequnpack.py  | 121 ++++++++++++++++++++++------------------
 test/test_stricttype.py |  29 +++++-----
 test/test_subtype.py    |   7 ++-
 test/test_timestamp.py  |  14 ++---
 test/test_unpack.py     |  27 +++++----
 16 files changed, 501 insertions(+), 334 deletions(-)

diff --git a/test/test_buffer.py b/test/test_buffer.py
index d723e8d..64fbdef 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -6,27 +6,28 @@ from msgpack import packb, unpackb
 
 def test_unpack_buffer():
     from array import array
-    buf = array('b')
+
+    buf = array("b")
     try:
-        buf.frombytes(packb((b'foo', b'bar')))
+        buf.frombytes(packb((b"foo", b"bar")))
     except AttributeError:  # PY2
-        buf.fromstring(packb((b'foo', b'bar')))
+        buf.fromstring(packb((b"foo", b"bar")))
     obj = unpackb(buf, use_list=1)
-    assert [b'foo', b'bar'] == obj
+    assert [b"foo", b"bar"] == obj
 
 
 def test_unpack_bytearray():
-    buf = bytearray(packb(('foo', 'bar')))
+    buf = bytearray(packb(("foo", "bar")))
     obj = unpackb(buf, use_list=1)
-    assert [b'foo', b'bar'] == obj
+    assert [b"foo", b"bar"] == obj
     expected_type = bytes
     assert all(type(s) == expected_type for s in obj)
 
 
 def test_unpack_memoryview():
-    buf = bytearray(packb(('foo', 'bar')))
+    buf = bytearray(packb(("foo", "bar")))
     view = memoryview(buf)
     obj = unpackb(view, use_list=1)
-    assert [b'foo', b'bar'] == obj
+    assert [b"foo", b"bar"] == obj
     expected_type = bytes
     assert all(type(s) == expected_type for s in obj)
diff --git a/test/test_case.py b/test/test_case.py
index 5a4bb6c..3bc1b26 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -6,97 +6,133 @@ from msgpack import packb, unpackb
 
 def check(length, obj):
     v = packb(obj)
-    assert len(v) == length, \
-        "%r length should be %r but get %r" % (obj, length, len(v))
+    assert len(v) == length, "%r length should be %r but get %r" % (obj, length, len(v))
     assert unpackb(v, use_list=0) == obj
 
+
 def test_1():
-    for o in [None, True, False, 0, 1, (1 << 6), (1 << 7) - 1, -1,
-              -((1<<5)-1), -(1<<5)]:
+    for o in [
+        None,
+        True,
+        False,
+        0,
+        1,
+        (1 << 6),
+        (1 << 7) - 1,
+        -1,
+        -((1 << 5) - 1),
+        -(1 << 5),
+    ]:
         check(1, o)
 
+
 def test_2():
-    for o in [1 << 7, (1 << 8) - 1,
-              -((1<<5)+1), -(1<<7)
-             ]:
+    for o in [1 << 7, (1 << 8) - 1, -((1 << 5) + 1), -(1 << 7)]:
         check(2, o)
 
+
 def test_3():
-    for o in [1 << 8, (1 << 16) - 1,
-              -((1<<7)+1), -(1<<15)]:
+    for o in [1 << 8, (1 << 16) - 1, -((1 << 7) + 1), -(1 << 15)]:
         check(3, o)
 
+
 def test_5():
-    for o in [1 << 16, (1 << 32) - 1,
-              -((1<<15)+1), -(1<<31)]:
+    for o in [1 << 16, (1 << 32) - 1, -((1 << 15) + 1), -(1 << 31)]:
         check(5, o)
 
+
 def test_9():
-    for o in [1 << 32, (1 << 64) - 1,
-              -((1<<31)+1), -(1<<63),
-              1.0, 0.1, -0.1, -1.0]:
+    for o in [
+        1 << 32,
+        (1 << 64) - 1,
+        -((1 << 31) + 1),
+        -(1 << 63),
+        1.0,
+        0.1,
+        -0.1,
+        -1.0,
+    ]:
         check(9, o)
 
 
 def check_raw(overhead, num):
     check(num + overhead, b" " * num)
 
+
 def test_fixraw():
     check_raw(1, 0)
-    check_raw(1, (1<<5) - 1)
+    check_raw(1, (1 << 5) - 1)
+
 
 def test_raw16():
-    check_raw(3, 1<<5)
-    check_raw(3, (1<<16) - 1)
+    check_raw(3, 1 << 5)
+    check_raw(3, (1 << 16) - 1)
+
 
 def test_raw32():
-    check_raw(5, 1<<16)
+    check_raw(5, 1 << 16)
 
 
 def check_array(overhead, num):
     check(num + overhead, (None,) * num)
 
+
 def test_fixarray():
     check_array(1, 0)
     check_array(1, (1 << 4) - 1)
 
+
 def test_array16():
     check_array(3, 1 << 4)
-    check_array(3, (1<<16)-1)
+    check_array(3, (1 << 16) - 1)
+
 
 def test_array32():
-    check_array(5, (1<<16))
+    check_array(5, (1 << 16))
 
 
 def match(obj, buf):
     assert packb(obj) == buf
     assert unpackb(buf, use_list=0) == obj
 
+
 def test_match():
     cases = [
-        (None, b'\xc0'),
-        (False, b'\xc2'),
-        (True, b'\xc3'),
-        (0, b'\x00'),
-        (127, b'\x7f'),
-        (128, b'\xcc\x80'),
-        (256, b'\xcd\x01\x00'),
-        (-1, b'\xff'),
-        (-33, b'\xd0\xdf'),
-        (-129, b'\xd1\xff\x7f'),
-        ({1:1}, b'\x81\x01\x01'),
+        (None, b"\xc0"),
+        (False, b"\xc2"),
+        (True, b"\xc3"),
+        (0, b"\x00"),
+        (127, b"\x7f"),
+        (128, b"\xcc\x80"),
+        (256, b"\xcd\x01\x00"),
+        (-1, b"\xff"),
+        (-33, b"\xd0\xdf"),
+        (-129, b"\xd1\xff\x7f"),
+        ({1: 1}, b"\x81\x01\x01"),
         (1.0, b"\xcb\x3f\xf0\x00\x00\x00\x00\x00\x00"),
-        ((), b'\x90'),
-        (tuple(range(15)),b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e"),
-        (tuple(range(16)),b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"),
-        ({}, b'\x80'),
-        (dict([(x,x) for x in range(15)]), b'\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e'),
-        (dict([(x,x) for x in range(16)]), b'\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f'),
-        ]
+        ((), b"\x90"),
+        (
+            tuple(range(15)),
+            b"\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e",
+        ),
+        (
+            tuple(range(16)),
+            b"\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+        ),
+        ({}, b"\x80"),
+        (
+            dict([(x, x) for x in range(15)]),
+            b"\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e",
+        ),
+        (
+            dict([(x, x) for x in range(16)]),
+            b"\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f",
+        ),
+    ]
 
     for v, p in cases:
         match(v, p)
 
-def test_unicode():
-    assert unpackb(packb('foobar'), use_list=1) == b'foobar'
 
+def test_unicode():
+    assert unpackb(packb("foobar"), use_list=1) == b"foobar"
diff --git a/test/test_extension.py b/test/test_extension.py
index 8aa0cbb..6b36575 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -9,37 +9,41 @@ def test_pack_ext_type():
         packer = msgpack.Packer()
         packer.pack_ext_type(0x42, s)
         return packer.bytes()
-    assert p(b'A')        == b'\xd4\x42A'          # fixext 1
-    assert p(b'AB')       == b'\xd5\x42AB'         # fixext 2
-    assert p(b'ABCD')     == b'\xd6\x42ABCD'       # fixext 4
-    assert p(b'ABCDEFGH') == b'\xd7\x42ABCDEFGH'   # fixext 8
-    assert p(b'A'*16)     == b'\xd8\x42' + b'A'*16 # fixext 16
-    assert p(b'ABC')      == b'\xc7\x03\x42ABC'        # ext 8
-    assert p(b'A'*0x0123)     == b'\xc8\x01\x23\x42' + b'A'*0x0123 # ext 16
-    assert p(b'A'*0x00012345) == b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345 # ext 32
+
+    assert p(b"A") == b"\xd4\x42A"  # fixext 1
+    assert p(b"AB") == b"\xd5\x42AB"  # fixext 2
+    assert p(b"ABCD") == b"\xd6\x42ABCD"  # fixext 4
+    assert p(b"ABCDEFGH") == b"\xd7\x42ABCDEFGH"  # fixext 8
+    assert p(b"A" * 16) == b"\xd8\x42" + b"A" * 16  # fixext 16
+    assert p(b"ABC") == b"\xc7\x03\x42ABC"  # ext 8
+    assert p(b"A" * 0x0123) == b"\xc8\x01\x23\x42" + b"A" * 0x0123  # ext 16
+    assert (
+        p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345
+    )  # ext 32
 
 
 def test_unpack_ext_type():
     def check(b, expected):
         assert msgpack.unpackb(b) == expected
 
-    check(b'\xd4\x42A',         ExtType(0x42, b'A'))        # fixext 1
-    check(b'\xd5\x42AB',        ExtType(0x42, b'AB'))       # fixext 2
-    check(b'\xd6\x42ABCD',      ExtType(0x42, b'ABCD'))     # fixext 4
-    check(b'\xd7\x42ABCDEFGH',  ExtType(0x42, b'ABCDEFGH')) # fixext 8
-    check(b'\xd8\x42' + b'A'*16, ExtType(0x42, b'A'*16))    # fixext 16
-    check(b'\xc7\x03\x42ABC',   ExtType(0x42, b'ABC'))      # ext 8
-    check(b'\xc8\x01\x23\x42' + b'A'*0x0123,
-          ExtType(0x42, b'A'*0x0123))                        # ext 16
-    check(b'\xc9\x00\x01\x23\x45\x42' + b'A'*0x00012345,
-          ExtType(0x42, b'A'*0x00012345))                   # ext 32
+    check(b"\xd4\x42A", ExtType(0x42, b"A"))  # fixext 1
+    check(b"\xd5\x42AB", ExtType(0x42, b"AB"))  # fixext 2
+    check(b"\xd6\x42ABCD", ExtType(0x42, b"ABCD"))  # fixext 4
+    check(b"\xd7\x42ABCDEFGH", ExtType(0x42, b"ABCDEFGH"))  # fixext 8
+    check(b"\xd8\x42" + b"A" * 16, ExtType(0x42, b"A" * 16))  # fixext 16
+    check(b"\xc7\x03\x42ABC", ExtType(0x42, b"ABC"))  # ext 8
+    check(b"\xc8\x01\x23\x42" + b"A" * 0x0123, ExtType(0x42, b"A" * 0x0123))  # ext 16
+    check(
+        b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345,
+        ExtType(0x42, b"A" * 0x00012345),
+    )  # ext 32
 
 
 def test_extension_type():
     def default(obj):
-        print('default called', obj)
+        print("default called", obj)
         if isinstance(obj, array.array):
-            typecode = 123 # application specific typecode
+            typecode = 123  # application specific typecode
             try:
                 data = obj.tobytes()
             except AttributeError:
@@ -48,24 +52,27 @@ def test_extension_type():
         raise TypeError("Unknown type object %r" % (obj,))
 
     def ext_hook(code, data):
-        print('ext_hook called', code, data)
+        print("ext_hook called", code, data)
         assert code == 123
-        obj = array.array('d')
+        obj = array.array("d")
         try:
             obj.frombytes(data)
         except AttributeError:  # PY2
             obj.fromstring(data)
         return obj
 
-    obj = [42, b'hello', array.array('d', [1.1, 2.2, 3.3])]
+    obj = [42, b"hello", array.array("d", [1.1, 2.2, 3.3])]
     s = msgpack.packb(obj, default=default)
     obj2 = msgpack.unpackb(s, ext_hook=ext_hook)
     assert obj == obj2
 
+
 import sys
-if sys.version > '3':
+
+if sys.version > "3":
     long = int
 
+
 def test_overriding_hooks():
     def default(obj):
         if isinstance(obj, long):
diff --git a/test/test_format.py b/test/test_format.py
index 5fec0c3..c2cdfbd 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -3,68 +3,88 @@
 
 from msgpack import unpackb
 
+
 def check(src, should, use_list=0):
     assert unpackb(src, use_list=use_list) == should
 
+
 def testSimpleValue():
-    check(b"\x93\xc0\xc2\xc3", 
-            (None, False, True,))
+    check(b"\x93\xc0\xc2\xc3", (None, False, True,))
+
 
 def testFixnum():
-    check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff",
-          ((0,64,127,), (-32,-16,-1,),)
-          )
+    check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", ((0, 64, 127,), (-32, -16, -1,),))
+
 
 def testFixArray():
-    check(b"\x92\x90\x91\x91\xc0",
-          ((),((None,),),),
-          )
+    check(
+        b"\x92\x90\x91\x91\xc0", ((), ((None,),),),
+    )
+
 
 def testFixRaw():
-    check(b"\x94\xa0\xa1a\xa2bc\xa3def",
-          (b"", b"a", b"bc", b"def",),
-          )
+    check(
+        b"\x94\xa0\xa1a\xa2bc\xa3def", (b"", b"a", b"bc", b"def",),
+    )
+
 
 def testFixMap():
     check(
-          b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
-          {False: {None: None}, True:{None:{}}},
-          )
+        b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
+        {False: {None: None}, True: {None: {}}},
+    )
+
 
 def testUnsignedInt():
     check(
-          b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
-          b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
-          b"\xce\xff\xff\xff\xff",
-          (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,),
-          )
+        b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
+        b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
+        b"\xce\xff\xff\xff\xff",
+        (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,),
+    )
+
 
 def testSignedInt():
-    check(b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
-          b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
-          b"\xd2\xff\xff\xff\xff",
-          (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,))
+    check(
+        b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
+        b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
+        b"\xd2\xff\xff\xff\xff",
+        (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,),
+    )
+
 
 def testRaw():
-    check(b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
+    check(
+        b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
         b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
-        (b"", b"a", b"ab", b"", b"a", b"ab"))
+        (b"", b"a", b"ab", b"", b"a", b"ab"),
+    )
+
 
 def testArray():
-    check(b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
+    check(
+        b"\x96\xdc\x00\x00\xdc\x00\x01\xc0\xdc\x00\x02\xc2\xc3\xdd\x00"
         b"\x00\x00\x00\xdd\x00\x00\x00\x01\xc0\xdd\x00\x00\x00\x02"
         b"\xc2\xc3",
-        ((), (None,), (False,True), (), (None,), (False,True))
-        )
+        ((), (None,), (False, True), (), (None,), (False, True)),
+    )
+
 
 def testMap():
     check(
         b"\x96"
-            b"\xde\x00\x00"
-            b"\xde\x00\x01\xc0\xc2"
-            b"\xde\x00\x02\xc0\xc2\xc3\xc2"
-            b"\xdf\x00\x00\x00\x00"
-            b"\xdf\x00\x00\x00\x01\xc0\xc2"
-            b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
-        ({}, {None: False}, {True: False, None: False}, {},
-            {None: False}, {True: False, None: False}))
+        b"\xde\x00\x00"
+        b"\xde\x00\x01\xc0\xc2"
+        b"\xde\x00\x02\xc0\xc2\xc3\xc2"
+        b"\xdf\x00\x00\x00\x00"
+        b"\xdf\x00\x00\x00\x01\xc0\xc2"
+        b"\xdf\x00\x00\x00\x02\xc0\xc2\xc3\xc2",
+        (
+            {},
+            {None: False},
+            {True: False, None: False},
+            {},
+            {None: False},
+            {True: False, None: False},
+        ),
+    )
diff --git a/test/test_limits.py b/test/test_limits.py
index 8c7606f..6e85030 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -4,8 +4,14 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import pytest
 
 from msgpack import (
-    packb, unpackb, Packer, Unpacker, ExtType,
-    PackOverflowError, PackValueError, UnpackValueError,
+    packb,
+    unpackb,
+    Packer,
+    Unpacker,
+    ExtType,
+    PackOverflowError,
+    PackValueError,
+    UnpackValueError,
 )
 
 
@@ -13,30 +19,30 @@ def test_integer():
     x = -(2 ** 63)
     assert unpackb(packb(x)) == x
     with pytest.raises(PackOverflowError):
-        packb(x-1)
+        packb(x - 1)
 
     x = 2 ** 64 - 1
     assert unpackb(packb(x)) == x
     with pytest.raises(PackOverflowError):
-        packb(x+1)
+        packb(x + 1)
 
 
 def test_array_header():
     packer = Packer()
-    packer.pack_array_header(2**32-1)
+    packer.pack_array_header(2 ** 32 - 1)
     with pytest.raises(PackValueError):
-        packer.pack_array_header(2**32)
+        packer.pack_array_header(2 ** 32)
 
 
 def test_map_header():
     packer = Packer()
-    packer.pack_map_header(2**32-1)
+    packer.pack_map_header(2 ** 32 - 1)
     with pytest.raises(PackValueError):
-        packer.pack_array_header(2**32)
+        packer.pack_array_header(2 ** 32)
 
 
 def test_max_str_len():
-    d = 'x' * 3
+    d = "x" * 3
     packed = packb(d)
 
     unpacker = Unpacker(max_str_len=3, raw=False)
@@ -50,7 +56,7 @@ def test_max_str_len():
 
 
 def test_max_bin_len():
-    d = b'x' * 3
+    d = b"x" * 3
     packed = packb(d, use_bin_type=True)
 
     unpacker = Unpacker(max_bin_len=3)
@@ -64,7 +70,7 @@ def test_max_bin_len():
 
 
 def test_max_array_len():
-    d = [1,2,3]
+    d = [1, 2, 3]
     packed = packb(d)
 
     unpacker = Unpacker(max_array_len=3)
@@ -107,8 +113,8 @@ def test_max_ext_len():
 
 # PyPy fails following tests because of constant folding?
 # https://bugs.pypy.org/issue1721
-#@pytest.mark.skipif(True, reason="Requires very large memory.")
-#def test_binary():
+# @pytest.mark.skipif(True, reason="Requires very large memory.")
+# def test_binary():
 #    x = b'x' * (2**32 - 1)
 #    assert unpackb(packb(x)) == x
 #    del x
@@ -117,8 +123,8 @@ def test_max_ext_len():
 #        packb(x)
 #
 #
-#@pytest.mark.skipif(True, reason="Requires very large memory.")
-#def test_string():
+# @pytest.mark.skipif(True, reason="Requires very large memory.")
+# def test_string():
 #    x = 'x' * (2**32 - 1)
 #    assert unpackb(packb(x)) == x
 #    x += 'y'
@@ -126,8 +132,8 @@ def test_max_ext_len():
 #        packb(x)
 #
 #
-#@pytest.mark.skipif(True, reason="Requires very large memory.")
-#def test_array():
+# @pytest.mark.skipif(True, reason="Requires very large memory.")
+# def test_array():
 #    x = [0] * (2**32 - 1)
 #    assert unpackb(packb(x)) == x
 #    x.append(0)
@@ -137,8 +143,9 @@ def test_max_ext_len():
 
 # auto max len
 
+
 def test_auto_max_array_len():
-    packed = b'\xde\x00\x06zz'
+    packed = b"\xde\x00\x06zz"
     with pytest.raises(UnpackValueError):
         unpackb(packed, raw=False)
 
@@ -147,9 +154,10 @@ def test_auto_max_array_len():
     with pytest.raises(UnpackValueError):
         unpacker.unpack()
 
+
 def test_auto_max_map_len():
     # len(packed) == 6 -> max_map_len == 3
-    packed = b'\xde\x00\x04zzz'
+    packed = b"\xde\x00\x04zzz"
     with pytest.raises(UnpackValueError):
         unpackb(packed, raw=False)
 
diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index f6d74ed..e1b63b8 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -10,6 +10,7 @@ import sys
 #  - array type only supports old buffer interface
 #  - array.frombytes is not available, must use deprecated array.fromstring
 if sys.version_info[0] < 3:
+
     def make_memoryview(obj):
         return memoryview(buffer(obj))
 
@@ -20,6 +21,8 @@ if sys.version_info[0] < 3:
 
     def get_data(a):
         return a.tostring()
+
+
 else:
     make_memoryview = memoryview
 
@@ -49,64 +52,64 @@ def _runtest(format, nbytes, expected_header, expected_prefix, use_bin_type):
     # check packed header
     assert packed[:1] == expected_header
     # check packed length prefix, if any
-    assert packed[1:1+len(expected_prefix)] == expected_prefix
+    assert packed[1 : 1 + len(expected_prefix)] == expected_prefix
     # check packed data
-    assert packed[1+len(expected_prefix):] == original_data
+    assert packed[1 + len(expected_prefix) :] == original_data
     # check array unpacked correctly
     assert original_array == reconstructed_array
 
 
 def test_fixstr_from_byte():
-    _runtest('B', 1, b'\xa1', b'', False)
-    _runtest('B', 31, b'\xbf', b'', False)
+    _runtest("B", 1, b"\xa1", b"", False)
+    _runtest("B", 31, b"\xbf", b"", False)
 
 
 def test_fixstr_from_float():
-    _runtest('f', 4, b'\xa4', b'', False)
-    _runtest('f', 28, b'\xbc', b'', False)
+    _runtest("f", 4, b"\xa4", b"", False)
+    _runtest("f", 28, b"\xbc", b"", False)
 
 
 def test_str16_from_byte():
-    _runtest('B', 2**8, b'\xda', b'\x01\x00', False)
-    _runtest('B', 2**16-1, b'\xda', b'\xff\xff', False)
+    _runtest("B", 2 ** 8, b"\xda", b"\x01\x00", False)
+    _runtest("B", 2 ** 16 - 1, b"\xda", b"\xff\xff", False)
 
 
 def test_str16_from_float():
-    _runtest('f', 2**8, b'\xda', b'\x01\x00', False)
-    _runtest('f', 2**16-4, b'\xda', b'\xff\xfc', False)
+    _runtest("f", 2 ** 8, b"\xda", b"\x01\x00", False)
+    _runtest("f", 2 ** 16 - 4, b"\xda", b"\xff\xfc", False)
 
 
 def test_str32_from_byte():
-    _runtest('B', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
+    _runtest("B", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False)
 
 
 def test_str32_from_float():
-    _runtest('f', 2**16, b'\xdb', b'\x00\x01\x00\x00', False)
+    _runtest("f", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False)
 
 
 def test_bin8_from_byte():
-    _runtest('B', 1, b'\xc4', b'\x01', True)
-    _runtest('B', 2**8-1, b'\xc4', b'\xff', True)
+    _runtest("B", 1, b"\xc4", b"\x01", True)
+    _runtest("B", 2 ** 8 - 1, b"\xc4", b"\xff", True)
 
 
 def test_bin8_from_float():
-    _runtest('f', 4, b'\xc4', b'\x04', True)
-    _runtest('f', 2**8-4, b'\xc4', b'\xfc', True)
+    _runtest("f", 4, b"\xc4", b"\x04", True)
+    _runtest("f", 2 ** 8 - 4, b"\xc4", b"\xfc", True)
 
 
 def test_bin16_from_byte():
-    _runtest('B', 2**8, b'\xc5', b'\x01\x00', True)
-    _runtest('B', 2**16-1, b'\xc5', b'\xff\xff', True)
+    _runtest("B", 2 ** 8, b"\xc5", b"\x01\x00", True)
+    _runtest("B", 2 ** 16 - 1, b"\xc5", b"\xff\xff", True)
 
 
 def test_bin16_from_float():
-    _runtest('f', 2**8, b'\xc5', b'\x01\x00', True)
-    _runtest('f', 2**16-4, b'\xc5', b'\xff\xfc', True)
+    _runtest("f", 2 ** 8, b"\xc5", b"\x01\x00", True)
+    _runtest("f", 2 ** 16 - 4, b"\xc5", b"\xff\xfc", True)
 
 
 def test_bin32_from_byte():
-    _runtest('B', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)
+    _runtest("B", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True)
 
 
 def test_bin32_from_float():
-    _runtest('f', 2**16, b'\xc6', b'\x00\x01\x00\x00', True)
+    _runtest("f", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True)
diff --git a/test/test_newspec.py b/test/test_newspec.py
index ab05029..f4f2a23 100644
--- a/test/test_newspec.py
+++ b/test/test_newspec.py
@@ -4,85 +4,87 @@ from msgpack import packb, unpackb, ExtType
 
 
 def test_str8():
-    header = b'\xd9'
-    data = b'x' * 32
+    header = b"\xd9"
+    data = b"x" * 32
     b = packb(data.decode(), use_bin_type=True)
     assert len(b) == len(data) + 2
-    assert b[0:2] == header + b'\x20'
+    assert b[0:2] == header + b"\x20"
     assert b[2:] == data
     assert unpackb(b) == data
 
-    data = b'x' * 255
+    data = b"x" * 255
     b = packb(data.decode(), use_bin_type=True)
     assert len(b) == len(data) + 2
-    assert b[0:2] == header + b'\xff'
+    assert b[0:2] == header + b"\xff"
     assert b[2:] == data
     assert unpackb(b) == data
 
 
 def test_bin8():
-    header = b'\xc4'
-    data = b''
+    header = b"\xc4"
+    data = b""
     b = packb(data, use_bin_type=True)
     assert len(b) == len(data) + 2
-    assert b[0:2] == header + b'\x00'
+    assert b[0:2] == header + b"\x00"
     assert b[2:] == data
     assert unpackb(b) == data
 
-    data = b'x' * 255
+    data = b"x" * 255
     b = packb(data, use_bin_type=True)
     assert len(b) == len(data) + 2
-    assert b[0:2] == header + b'\xff'
+    assert b[0:2] == header + b"\xff"
     assert b[2:] == data
     assert unpackb(b) == data
 
 
 def test_bin16():
-    header = b'\xc5'
-    data = b'x' * 256
+    header = b"\xc5"
+    data = b"x" * 256
     b = packb(data, use_bin_type=True)
     assert len(b) == len(data) + 3
     assert b[0:1] == header
-    assert b[1:3] == b'\x01\x00'
+    assert b[1:3] == b"\x01\x00"
     assert b[3:] == data
     assert unpackb(b) == data
 
-    data = b'x' * 65535
+    data = b"x" * 65535
     b = packb(data, use_bin_type=True)
     assert len(b) == len(data) + 3
     assert b[0:1] == header
-    assert b[1:3] == b'\xff\xff'
+    assert b[1:3] == b"\xff\xff"
     assert b[3:] == data
     assert unpackb(b) == data
 
 
 def test_bin32():
-    header = b'\xc6'
-    data = b'x' * 65536
+    header = b"\xc6"
+    data = b"x" * 65536
     b = packb(data, use_bin_type=True)
     assert len(b) == len(data) + 5
     assert b[0:1] == header
-    assert b[1:5] == b'\x00\x01\x00\x00'
+    assert b[1:5] == b"\x00\x01\x00\x00"
     assert b[5:] == data
     assert unpackb(b) == data
 
+
 def test_ext():
     def check(ext, packed):
         assert packb(ext) == packed
         assert unpackb(packed) == ext
-    check(ExtType(0x42, b'Z'), b'\xd4\x42Z') # fixext 1
-    check(ExtType(0x42, b'ZZ'), b'\xd5\x42ZZ') # fixext 2
-    check(ExtType(0x42, b'Z'*4), b'\xd6\x42' + b'Z'*4) # fixext 4
-    check(ExtType(0x42, b'Z'*8), b'\xd7\x42' + b'Z'*8) # fixext 8
-    check(ExtType(0x42, b'Z'*16), b'\xd8\x42' + b'Z'*16) # fixext 16
+
+    check(ExtType(0x42, b"Z"), b"\xd4\x42Z")  # fixext 1
+    check(ExtType(0x42, b"ZZ"), b"\xd5\x42ZZ")  # fixext 2
+    check(ExtType(0x42, b"Z" * 4), b"\xd6\x42" + b"Z" * 4)  # fixext 4
+    check(ExtType(0x42, b"Z" * 8), b"\xd7\x42" + b"Z" * 8)  # fixext 8
+    check(ExtType(0x42, b"Z" * 16), b"\xd8\x42" + b"Z" * 16)  # fixext 16
     # ext 8
-    check(ExtType(0x42, b''), b'\xc7\x00\x42')
-    check(ExtType(0x42, b'Z'*255), b'\xc7\xff\x42' + b'Z'*255)
+    check(ExtType(0x42, b""), b"\xc7\x00\x42")
+    check(ExtType(0x42, b"Z" * 255), b"\xc7\xff\x42" + b"Z" * 255)
     # ext 16
-    check(ExtType(0x42, b'Z'*256), b'\xc8\x01\x00\x42' + b'Z'*256)
-    check(ExtType(0x42, b'Z'*0xffff), b'\xc8\xff\xff\x42' + b'Z'*0xffff)
+    check(ExtType(0x42, b"Z" * 256), b"\xc8\x01\x00\x42" + b"Z" * 256)
+    check(ExtType(0x42, b"Z" * 0xFFFF), b"\xc8\xff\xff\x42" + b"Z" * 0xFFFF)
     # ext 32
-    check(ExtType(0x42, b'Z'*0x10000), b'\xc9\x00\x01\x00\x00\x42' + b'Z'*0x10000)
+    check(ExtType(0x42, b"Z" * 0x10000), b"\xc9\x00\x01\x00\x00\x42" + b"Z" * 0x10000)
     # needs large memory
-    #check(ExtType(0x42, b'Z'*0xffffffff),
+    # check(ExtType(0x42, b'Z'*0xffffffff),
     #              b'\xc9\xff\xff\xff\xff\x42' + b'Z'*0xffffffff)
diff --git a/test/test_obj.py b/test/test_obj.py
index 390c1b6..0b99cea 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -4,64 +4,76 @@
 from pytest import raises
 from msgpack import packb, unpackb
 
+
 def _decode_complex(obj):
-    if b'__complex__' in obj:
-        return complex(obj[b'real'], obj[b'imag'])
+    if b"__complex__" in obj:
+        return complex(obj[b"real"], obj[b"imag"])
     return obj
 
+
 def _encode_complex(obj):
     if isinstance(obj, complex):
-        return {b'__complex__': True, b'real': 1, b'imag': 2}
+        return {b"__complex__": True, b"real": 1, b"imag": 2}
     return obj
 
+
 def test_encode_hook():
-    packed = packb([3, 1+2j], default=_encode_complex)
+    packed = packb([3, 1 + 2j], default=_encode_complex)
     unpacked = unpackb(packed, use_list=1)
-    assert unpacked[1] == {b'__complex__': True, b'real': 1, b'imag': 2}
+    assert unpacked[1] == {b"__complex__": True, b"real": 1, b"imag": 2}
+
 
 def test_decode_hook():
-    packed = packb([3, {b'__complex__': True, b'real': 1, b'imag': 2}])
+    packed = packb([3, {b"__complex__": True, b"real": 1, b"imag": 2}])
     unpacked = unpackb(packed, object_hook=_decode_complex, use_list=1)
-    assert unpacked[1] == 1+2j
+    assert unpacked[1] == 1 + 2j
+
 
 def test_decode_pairs_hook():
     packed = packb([3, {1: 2, 3: 4}])
     prod_sum = 1 * 2 + 3 * 4
-    unpacked = unpackb(packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1)
+    unpacked = unpackb(
+        packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1
+    )
     assert unpacked[1] == prod_sum
 
+
 def test_only_one_obj_hook():
     with raises(TypeError):
-        unpackb(b'', object_hook=lambda x: x, object_pairs_hook=lambda x: x)
+        unpackb(b"", object_hook=lambda x: x, object_pairs_hook=lambda x: x)
+
 
 def test_bad_hook():
     with raises(TypeError):
-        packed = packb([3, 1+2j], default=lambda o: o)
+        packed = packb([3, 1 + 2j], default=lambda o: o)
         unpacked = unpackb(packed, use_list=1)
 
+
 def _arr_to_str(arr):
-    return ''.join(str(c) for c in arr)
+    return "".join(str(c) for c in arr)
+
 
 def test_array_hook():
-    packed = packb([1,2,3])
+    packed = packb([1, 2, 3])
     unpacked = unpackb(packed, list_hook=_arr_to_str, use_list=1)
-    assert unpacked == '123'
+    assert unpacked == "123"
 
 
 class DecodeError(Exception):
     pass
 
+
 def bad_complex_decoder(o):
     raise DecodeError("Ooops!")
 
 
 def test_an_exception_in_objecthook1():
     with raises(DecodeError):
-        packed = packb({1: {'__complex__': True, 'real': 1, 'imag': 2}})
+        packed = packb({1: {"__complex__": True, "real": 1, "imag": 2}})
         unpackb(packed, object_hook=bad_complex_decoder)
 
 
 def test_an_exception_in_objecthook2():
     with raises(DecodeError):
-        packed = packb({1: [{'__complex__': True, 'real': 1, 'imag': 2}]})
+        packed = packb({1: [{"__complex__": True, "real": 1, "imag": 2}]})
         unpackb(packed, list_hook=bad_complex_decoder, use_list=1)
diff --git a/test/test_pack.py b/test/test_pack.py
index b6752e5..de212ef 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -17,20 +17,46 @@ def check(data, use_list=False):
     re = unpackb(packb(data), use_list=use_list)
     assert re == data
 
+
 def testPack():
     test_data = [
-            0, 1, 127, 128, 255, 256, 65535, 65536, 4294967295, 4294967296,
-            -1, -32, -33, -128, -129, -32768, -32769, -4294967296, -4294967297,
-            1.0,
-        b"", b"a", b"a"*31, b"a"*32,
-        None, True, False,
-        (), ((),), ((), None,),
+        0,
+        1,
+        127,
+        128,
+        255,
+        256,
+        65535,
+        65536,
+        4294967295,
+        4294967296,
+        -1,
+        -32,
+        -33,
+        -128,
+        -129,
+        -32768,
+        -32769,
+        -4294967296,
+        -4294967297,
+        1.0,
+        b"",
+        b"a",
+        b"a" * 31,
+        b"a" * 32,
+        None,
+        True,
+        False,
+        (),
+        ((),),
+        ((), None,),
         {None: 0},
-        (1<<23),
-        ]
+        (1 << 23),
+    ]
     for td in test_data:
         check(td)
 
+
 def testPackUnicode():
     test_data = ["", "abcd", ["defgh"], "РуÑÑкий текÑÑ‚"]
     for td in test_data:
@@ -41,43 +67,64 @@ def testPackUnicode():
         re = Unpacker(BytesIO(data), raw=False, use_list=1).unpack()
         assert re == td
 
+
 def testPackBytes():
     test_data = [
-        b"", b"abcd", (b"defgh",),
-        ]
+        b"",
+        b"abcd",
+        (b"defgh",),
+    ]
     for td in test_data:
         check(td)
 
+
 def testPackByteArrays():
     test_data = [
-        bytearray(b""), bytearray(b"abcd"), (bytearray(b"defgh"),),
-        ]
+        bytearray(b""),
+        bytearray(b"abcd"),
+        (bytearray(b"defgh"),),
+    ]
     for td in test_data:
         check(td)
 
-@pytest.mark.skipif(sys.version_info < (3,0), reason="Python 2 passes invalid surrogates")
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates"
+)
 def testIgnoreUnicodeErrors():
-    re = unpackb(packb(b'abc\xeddef', use_bin_type=False),
-                 raw=False, unicode_errors='ignore')
+    re = unpackb(
+        packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore"
+    )
     assert re == "abcdef"
 
+
 def testStrictUnicodeUnpack():
-    packed = packb(b'abc\xeddef', use_bin_type=False)
+    packed = packb(b"abc\xeddef", use_bin_type=False)
     with pytest.raises(UnicodeDecodeError):
         unpackb(packed, raw=False, use_list=1)
 
-@pytest.mark.skipif(sys.version_info < (3,0), reason="Python 2 passes invalid surrogates")
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates"
+)
 def testIgnoreErrorsPack():
-    re = unpackb(packb(u"abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors='ignore'), raw=False, use_list=1)
+    re = unpackb(
+        packb("abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors="ignore"),
+        raw=False,
+        use_list=1,
+    )
     assert re == "abcdef"
 
+
 def testDecodeBinary():
     re = unpackb(packb(b"abc"), use_list=1)
     assert re == b"abc"
 
+
 def testPackFloat():
-    assert packb(1.0, use_single_float=True)  == b'\xca' + struct.pack(str('>f'), 1.0)
-    assert packb(1.0, use_single_float=False) == b'\xcb' + struct.pack(str('>d'), 1.0)
+    assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(str(">f"), 1.0)
+    assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(str(">d"), 1.0)
+
 
 def testArraySize(sizes=[0, 5, 50, 1000]):
     bio = BytesIO()
@@ -92,6 +139,7 @@ def testArraySize(sizes=[0, 5, 50, 1000]):
     for size in sizes:
         assert unpacker.unpack() == list(range(size))
 
+
 def test_manualreset(sizes=[0, 5, 50, 1000]):
     packer = Packer(autoreset=False)
     for size in sizes:
@@ -105,7 +153,8 @@ def test_manualreset(sizes=[0, 5, 50, 1000]):
         assert unpacker.unpack() == list(range(size))
 
     packer.reset()
-    assert packer.bytes() == b''
+    assert packer.bytes() == b""
+
 
 def testMapSize(sizes=[0, 5, 50, 1000]):
     bio = BytesIO()
@@ -113,8 +162,8 @@ def testMapSize(sizes=[0, 5, 50, 1000]):
     for size in sizes:
         bio.write(packer.pack_map_header(size))
         for i in range(size):
-            bio.write(packer.pack(i)) # key
-            bio.write(packer.pack(i * 2)) # value
+            bio.write(packer.pack(i))  # key
+            bio.write(packer.pack(i * 2))  # value
 
     bio.seek(0)
     unpacker = Unpacker(bio)
@@ -123,21 +172,24 @@ def testMapSize(sizes=[0, 5, 50, 1000]):
 
 
 def test_odict():
-    seq = [(b'one', 1), (b'two', 2), (b'three', 3), (b'four', 4)]
+    seq = [(b"one", 1), (b"two", 2), (b"three", 3), (b"four", 4)]
     od = OrderedDict(seq)
     assert unpackb(packb(od), use_list=1) == dict(seq)
+
     def pair_hook(seq):
         return list(seq)
+
     assert unpackb(packb(od), object_pairs_hook=pair_hook, use_list=1) == seq
 
 
 def test_pairlist():
-    pairlist = [(b'a', 1), (2, b'b'), (b'foo', b'bar')]
+    pairlist = [(b"a", 1), (2, b"b"), (b"foo", b"bar")]
     packer = Packer()
     packed = packer.pack_map_pairs(pairlist)
     unpacked = unpackb(packed, object_pairs_hook=list)
     assert pairlist == unpacked
 
+
 def test_get_buffer():
     packer = Packer(autoreset=0, use_bin_type=True)
     packer.pack([1, 2])
diff --git a/test/test_read_size.py b/test/test_read_size.py
index 4e6c2b9..8d8df64 100644
--- a/test/test_read_size.py
+++ b/test/test_read_size.py
@@ -1,66 +1,71 @@
 """Test Unpacker's read_array_header and read_map_header methods"""
 from msgpack import packb, Unpacker, OutOfData
+
 UnexpectedTypeException = ValueError
 
+
 def test_read_array_header():
     unpacker = Unpacker()
-    unpacker.feed(packb(['a', 'b', 'c']))
+    unpacker.feed(packb(["a", "b", "c"]))
     assert unpacker.read_array_header() == 3
-    assert unpacker.unpack() == b'a'
-    assert unpacker.unpack() == b'b'
-    assert unpacker.unpack() == b'c'
+    assert unpacker.unpack() == b"a"
+    assert unpacker.unpack() == b"b"
+    assert unpacker.unpack() == b"c"
     try:
         unpacker.unpack()
-        assert 0, 'should raise exception'
+        assert 0, "should raise exception"
     except OutOfData:
-        assert 1, 'okay'
+        assert 1, "okay"
 
 
 def test_read_map_header():
     unpacker = Unpacker()
-    unpacker.feed(packb({'a': 'A'}))
+    unpacker.feed(packb({"a": "A"}))
     assert unpacker.read_map_header() == 1
-    assert unpacker.unpack() == B'a'
-    assert unpacker.unpack() == B'A'
+    assert unpacker.unpack() == b"a"
+    assert unpacker.unpack() == b"A"
     try:
         unpacker.unpack()
-        assert 0, 'should raise exception'
+        assert 0, "should raise exception"
     except OutOfData:
-        assert 1, 'okay'
+        assert 1, "okay"
+
 
 def test_incorrect_type_array():
     unpacker = Unpacker()
     unpacker.feed(packb(1))
     try:
         unpacker.read_array_header()
-        assert 0, 'should raise exception'
+        assert 0, "should raise exception"
     except UnexpectedTypeException:
-        assert 1, 'okay'
+        assert 1, "okay"
+
 
 def test_incorrect_type_map():
     unpacker = Unpacker()
     unpacker.feed(packb(1))
     try:
         unpacker.read_map_header()
-        assert 0, 'should raise exception'
+        assert 0, "should raise exception"
     except UnexpectedTypeException:
-        assert 1, 'okay'
+        assert 1, "okay"
+
 
 def test_correct_type_nested_array():
     unpacker = Unpacker()
-    unpacker.feed(packb({'a': ['b', 'c', 'd']}))
+    unpacker.feed(packb({"a": ["b", "c", "d"]}))
     try:
         unpacker.read_array_header()
-        assert 0, 'should raise exception'
+        assert 0, "should raise exception"
     except UnexpectedTypeException:
-        assert 1, 'okay'
+        assert 1, "okay"
+
 
 def test_incorrect_type_nested_map():
     unpacker = Unpacker()
-    unpacker.feed(packb([{'a': 'b'}]))
+    unpacker.feed(packb([{"a": "b"}]))
     try:
         unpacker.read_map_header()
-        assert 0, 'should raise exception'
+        assert 0, "should raise exception"
     except UnexpectedTypeException:
-        assert 1, 'okay'
-
+        assert 1, "okay"
diff --git a/test/test_seq.py b/test/test_seq.py
index fed9ff4..0d5d806 100644
--- a/test/test_seq.py
+++ b/test/test_seq.py
@@ -7,8 +7,9 @@ import msgpack
 
 binarydata = bytes(bytearray(range(256)))
 
+
 def gen_binary_data(idx):
-    return binarydata[:idx % 300]
+    return binarydata[: idx % 300]
 
 
 def test_exceeding_unpacker_read_size():
@@ -18,10 +19,10 @@ def test_exceeding_unpacker_read_size():
 
     NUMBER_OF_STRINGS = 6
     read_size = 16
-                                # 5 ok for read_size=16, while 6 glibc detected *** python: double free or corruption (fasttop):
-                                # 20 ok for read_size=256, while 25 segfaults / glibc detected *** python: double free or corruption (!prev)
-                                # 40 ok for read_size=1024, while 50 introduces errors
-                                # 7000 ok for read_size=1024*1024, while 8000 leads to  glibc detected *** python: double free or corruption (!prev):
+    # 5 ok for read_size=16, while 6 glibc detected *** python: double free or corruption (fasttop):
+    # 20 ok for read_size=256, while 25 segfaults / glibc detected *** python: double free or corruption (!prev)
+    # 40 ok for read_size=1024, while 50 introduces errors
+    # 7000 ok for read_size=1024*1024, while 8000 leads to  glibc detected *** python: double free or corruption (!prev):
 
     for idx in range(NUMBER_OF_STRINGS):
         data = gen_binary_data(idx)
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 59718f5..e576571 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -10,102 +10,115 @@ from pytest import raises
 
 def test_partialdata():
     unpacker = Unpacker()
-    unpacker.feed(b'\xa5')
-    with raises(StopIteration): next(iter(unpacker))
-    unpacker.feed(b'h')
-    with raises(StopIteration): next(iter(unpacker))
-    unpacker.feed(b'a')
-    with raises(StopIteration): next(iter(unpacker))
-    unpacker.feed(b'l')
-    with raises(StopIteration): next(iter(unpacker))
-    unpacker.feed(b'l')
-    with raises(StopIteration): next(iter(unpacker))
-    unpacker.feed(b'o')
-    assert next(iter(unpacker)) == b'hallo'
+    unpacker.feed(b"\xa5")
+    with raises(StopIteration):
+        next(iter(unpacker))
+    unpacker.feed(b"h")
+    with raises(StopIteration):
+        next(iter(unpacker))
+    unpacker.feed(b"a")
+    with raises(StopIteration):
+        next(iter(unpacker))
+    unpacker.feed(b"l")
+    with raises(StopIteration):
+        next(iter(unpacker))
+    unpacker.feed(b"l")
+    with raises(StopIteration):
+        next(iter(unpacker))
+    unpacker.feed(b"o")
+    assert next(iter(unpacker)) == b"hallo"
+
 
 def test_foobar():
     unpacker = Unpacker(read_size=3, use_list=1)
-    unpacker.feed(b'foobar')
-    assert unpacker.unpack() == ord(b'f')
-    assert unpacker.unpack() == ord(b'o')
-    assert unpacker.unpack() == ord(b'o')
-    assert unpacker.unpack() == ord(b'b')
-    assert unpacker.unpack() == ord(b'a')
-    assert unpacker.unpack() == ord(b'r')
+    unpacker.feed(b"foobar")
+    assert unpacker.unpack() == ord(b"f")
+    assert unpacker.unpack() == ord(b"o")
+    assert unpacker.unpack() == ord(b"o")
+    assert unpacker.unpack() == ord(b"b")
+    assert unpacker.unpack() == ord(b"a")
+    assert unpacker.unpack() == ord(b"r")
     with raises(OutOfData):
         unpacker.unpack()
 
-    unpacker.feed(b'foo')
-    unpacker.feed(b'bar')
+    unpacker.feed(b"foo")
+    unpacker.feed(b"bar")
 
     k = 0
-    for o, e in zip(unpacker, 'foobarbaz'):
+    for o, e in zip(unpacker, "foobarbaz"):
         assert o == ord(e)
         k += 1
-    assert k == len(b'foobar')
+    assert k == len(b"foobar")
+
 
 def test_foobar_skip():
     unpacker = Unpacker(read_size=3, use_list=1)
-    unpacker.feed(b'foobar')
-    assert unpacker.unpack() == ord(b'f')
+    unpacker.feed(b"foobar")
+    assert unpacker.unpack() == ord(b"f")
     unpacker.skip()
-    assert unpacker.unpack() == ord(b'o')
+    assert unpacker.unpack() == ord(b"o")
     unpacker.skip()
-    assert unpacker.unpack() == ord(b'a')
+    assert unpacker.unpack() == ord(b"a")
     unpacker.skip()
     with raises(OutOfData):
         unpacker.unpack()
 
+
 def test_maxbuffersize():
     with raises(ValueError):
         Unpacker(read_size=5, max_buffer_size=3)
     unpacker = Unpacker(read_size=3, max_buffer_size=3, use_list=1)
-    unpacker.feed(b'fo')
+    unpacker.feed(b"fo")
     with raises(BufferFull):
-        unpacker.feed(b'ob')
-    unpacker.feed(b'o')
-    assert ord('f') == next(unpacker)
-    unpacker.feed(b'b')
-    assert ord('o') == next(unpacker)
-    assert ord('o') == next(unpacker)
-    assert ord('b') == next(unpacker)
+        unpacker.feed(b"ob")
+    unpacker.feed(b"o")
+    assert ord("f") == next(unpacker)
+    unpacker.feed(b"b")
+    assert ord("o") == next(unpacker)
+    assert ord("o") == next(unpacker)
+    assert ord("b") == next(unpacker)
 
 
 def test_readbytes():
     unpacker = Unpacker(read_size=3)
-    unpacker.feed(b'foobar')
-    assert unpacker.unpack() == ord(b'f')
-    assert unpacker.read_bytes(3) == b'oob'
-    assert unpacker.unpack() == ord(b'a')
-    assert unpacker.unpack() == ord(b'r')
+    unpacker.feed(b"foobar")
+    assert unpacker.unpack() == ord(b"f")
+    assert unpacker.read_bytes(3) == b"oob"
+    assert unpacker.unpack() == ord(b"a")
+    assert unpacker.unpack() == ord(b"r")
 
     # Test buffer refill
-    unpacker = Unpacker(io.BytesIO(b'foobar'), read_size=3)
-    assert unpacker.unpack() == ord(b'f')
-    assert unpacker.read_bytes(3) == b'oob'
-    assert unpacker.unpack() == ord(b'a')
-    assert unpacker.unpack() == ord(b'r')
+    unpacker = Unpacker(io.BytesIO(b"foobar"), read_size=3)
+    assert unpacker.unpack() == ord(b"f")
+    assert unpacker.read_bytes(3) == b"oob"
+    assert unpacker.unpack() == ord(b"a")
+    assert unpacker.unpack() == ord(b"r")
+
 
 def test_issue124():
     unpacker = Unpacker()
-    unpacker.feed(b'\xa1?\xa1!')
-    assert tuple(unpacker) == (b'?', b'!')
+    unpacker.feed(b"\xa1?\xa1!")
+    assert tuple(unpacker) == (b"?", b"!")
     assert tuple(unpacker) == ()
     unpacker.feed(b"\xa1?\xa1")
-    assert tuple(unpacker) == (b'?',)
+    assert tuple(unpacker) == (b"?",)
     assert tuple(unpacker) == ()
     unpacker.feed(b"!")
-    assert tuple(unpacker) == (b'!',)
+    assert tuple(unpacker) == (b"!",)
     assert tuple(unpacker) == ()
 
 
 def test_unpack_tell():
     stream = io.BytesIO()
-    messages = [2**i-1 for i in range(65)]
-    messages += [-(2**i) for i in range(1, 64)]
-    messages += [b'hello', b'hello'*1000, list(range(20)),
-                 {i: bytes(i)*i for i in range(10)},
-                 {i: bytes(i)*i for i in range(32)}]
+    messages = [2 ** i - 1 for i in range(65)]
+    messages += [-(2 ** i) for i in range(1, 64)]
+    messages += [
+        b"hello",
+        b"hello" * 1000,
+        list(range(20)),
+        {i: bytes(i) * i for i in range(10)},
+        {i: bytes(i) * i for i in range(32)},
+    ]
     offsets = []
     for m in messages:
         pack(m, stream)
diff --git a/test/test_stricttype.py b/test/test_stricttype.py
index 87e7c1c..78e1723 100644
--- a/test/test_stricttype.py
+++ b/test/test_stricttype.py
@@ -5,30 +5,32 @@ from msgpack import packb, unpackb, ExtType
 
 
 def test_namedtuple():
-    T = namedtuple('T', "foo bar")
+    T = namedtuple("T", "foo bar")
+
     def default(o):
         if isinstance(o, T):
             return dict(o._asdict())
-        raise TypeError('Unsupported type %s' % (type(o),))
+        raise TypeError("Unsupported type %s" % (type(o),))
+
     packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default)
     unpacked = unpackb(packed, raw=False)
-    assert unpacked == {'foo': 1, 'bar': 42}
+    assert unpacked == {"foo": 1, "bar": 42}
 
 
 def test_tuple():
-    t = ('one', 2, b'three', (4, ))
+    t = ("one", 2, b"three", (4,))
 
     def default(o):
         if isinstance(o, tuple):
             return {
-                '__type__': 'tuple',
-                'value': list(o),
-                }
-        raise TypeError('Unsupported type %s' % (type(o),))
+                "__type__": "tuple",
+                "value": list(o),
+            }
+        raise TypeError("Unsupported type %s" % (type(o),))
 
     def convert(o):
-        if o.get('__type__') == 'tuple':
-            return tuple(o['value'])
+        if o.get("__type__") == "tuple":
+            return tuple(o["value"])
         return o
 
     data = packb(t, strict_types=True, use_bin_type=True, default=default)
@@ -38,7 +40,7 @@ def test_tuple():
 
 
 def test_tuple_ext():
-    t = ('one', 2, b'three', (4, ))
+    t = ("one", 2, b"three", (4,))
 
     MSGPACK_EXT_TYPE_TUPLE = 0
 
@@ -46,7 +48,8 @@ def test_tuple_ext():
         if isinstance(o, tuple):
             # Convert to list and pack
             payload = packb(
-                list(o), strict_types=True, use_bin_type=True, default=default)
+                list(o), strict_types=True, use_bin_type=True, default=default
+            )
             return ExtType(MSGPACK_EXT_TYPE_TUPLE, payload)
         raise TypeError(repr(o))
 
@@ -54,7 +57,7 @@ def test_tuple_ext():
         if code == MSGPACK_EXT_TYPE_TUPLE:
             # Unpack and convert to tuple
             return tuple(unpackb(payload, raw=False, ext_hook=convert))
-        raise ValueError('Unknown Ext code {}'.format(code))
+        raise ValueError("Unknown Ext code {}".format(code))
 
     data = packb(t, strict_types=True, use_bin_type=True, default=default)
     expected = unpackb(data, raw=False, ext_hook=convert)
diff --git a/test/test_subtype.py b/test/test_subtype.py
index 6807508..d91d455 100644
--- a/test/test_subtype.py
+++ b/test/test_subtype.py
@@ -4,16 +4,21 @@
 from msgpack import packb, unpackb
 from collections import namedtuple
 
+
 class MyList(list):
     pass
 
+
 class MyDict(dict):
     pass
 
+
 class MyTuple(tuple):
     pass
 
-MyNamedTuple = namedtuple('MyNamedTuple', 'x y')
+
+MyNamedTuple = namedtuple("MyNamedTuple", "x y")
+
 
 def test_types():
     assert packb(MyDict()) == packb(dict())
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index 55c2f6d..1348e69 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -4,34 +4,34 @@ from msgpack import Timestamp
 
 def test_timestamp():
     # timestamp32
-    ts = Timestamp(2**32 - 1)
+    ts = Timestamp(2 ** 32 - 1)
     assert ts.to_bytes() == b"\xff\xff\xff\xff"
     packed = msgpack.packb(ts)
     assert packed == b"\xd6\xff" + ts.to_bytes()
     unpacked = msgpack.unpackb(packed)
     assert ts == unpacked
-    assert ts.seconds == 2**32 - 1 and ts.nanoseconds == 0
+    assert ts.seconds == 2 ** 32 - 1 and ts.nanoseconds == 0
 
     # timestamp64
-    ts = Timestamp(2**34 - 1, 999999999)
+    ts = Timestamp(2 ** 34 - 1, 999999999)
     assert ts.to_bytes() == b"\xee\x6b\x27\xff\xff\xff\xff\xff"
     packed = msgpack.packb(ts)
     assert packed == b"\xd7\xff" + ts.to_bytes()
     unpacked = msgpack.unpackb(packed)
     assert ts == unpacked
-    assert ts.seconds == 2**34 - 1 and ts.nanoseconds == 999999999
+    assert ts.seconds == 2 ** 34 - 1 and ts.nanoseconds == 999999999
 
     # timestamp96
-    ts = Timestamp(2**63 - 1, 999999999)
+    ts = Timestamp(2 ** 63 - 1, 999999999)
     assert ts.to_bytes() == b"\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff"
     packed = msgpack.packb(ts)
     assert packed == b"\xc7\x0c\xff" + ts.to_bytes()
     unpacked = msgpack.unpackb(packed)
     assert ts == unpacked
-    assert ts.seconds == 2**63 - 1 and ts.nanoseconds == 999999999
+    assert ts.seconds == 2 ** 63 - 1 and ts.nanoseconds == 999999999
 
     # negative fractional
-    ts = Timestamp(-2.3)  #s: -3, ns: 700000000
+    ts = Timestamp(-2.3)  # s: -3, ns: 700000000
     assert ts.to_bytes() == b"\x29\xb9\x27\x00\xff\xff\xff\xff\xff\xff\xff\xfd"
     packed = msgpack.packb(ts)
     assert packed == b"\xc7\x0c\xff" + ts.to_bytes()
diff --git a/test/test_unpack.py b/test/test_unpack.py
index 00a1061..bc74c4d 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -5,7 +5,7 @@ from pytest import raises, mark
 
 
 def test_unpack_array_header_from_file():
-    f = BytesIO(packb([1,2,3,4]))
+    f = BytesIO(packb([1, 2, 3, 4]))
     unpacker = Unpacker(f)
     assert unpacker.read_array_header() == 4
     assert unpacker.unpack() == 1
@@ -16,8 +16,10 @@ def test_unpack_array_header_from_file():
         unpacker.unpack()
 
 
-@mark.skipif("not hasattr(sys, 'getrefcount') == True",
-             reason='sys.getrefcount() is needed to pass this test')
+@mark.skipif(
+    "not hasattr(sys, 'getrefcount') == True",
+    reason="sys.getrefcount() is needed to pass this test",
+)
 def test_unpacker_hook_refcnt():
     result = []
 
@@ -43,12 +45,9 @@ def test_unpacker_hook_refcnt():
 
 
 def test_unpacker_ext_hook():
-
     class MyUnpacker(Unpacker):
-
         def __init__(self):
-            super(MyUnpacker, self).__init__(
-                ext_hook=self._hook, raw=False)
+            super(MyUnpacker, self).__init__(ext_hook=self._hook, raw=False)
 
         def _hook(self, code, data):
             if code == 1:
@@ -57,15 +56,15 @@ def test_unpacker_ext_hook():
                 return ExtType(code, data)
 
     unpacker = MyUnpacker()
-    unpacker.feed(packb({'a': 1}))
-    assert unpacker.unpack() == {'a': 1}
-    unpacker.feed(packb({'a': ExtType(1, b'123')}))
-    assert unpacker.unpack() == {'a': 123}
-    unpacker.feed(packb({'a': ExtType(2, b'321')}))
-    assert unpacker.unpack() == {'a': ExtType(2, b'321')}
+    unpacker.feed(packb({"a": 1}))
+    assert unpacker.unpack() == {"a": 1}
+    unpacker.feed(packb({"a": ExtType(1, b"123")}))
+    assert unpacker.unpack() == {"a": 123}
+    unpacker.feed(packb({"a": ExtType(2, b"321")}))
+    assert unpacker.unpack() == {"a": ExtType(2, b"321")}
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     test_unpack_array_header_from_file()
     test_unpacker_hook_refcnt()
     test_unpacker_ext_hook()

From bc8c86203af8d36152c9c72ea22e895db2ed3fe0 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 5 Dec 2019 18:53:49 +0900
Subject: [PATCH 1486/1648] blacken all files.

---
 benchmark/benchmark.py |  20 +++--
 docs/conf.py           | 164 ++++++++++++++++++++---------------------
 setup.py               | 119 ++++++++++++++++--------------
 3 files changed, 159 insertions(+), 144 deletions(-)

diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py
index 80819c6..82d0ddb 100644
--- a/benchmark/benchmark.py
+++ b/benchmark/benchmark.py
@@ -1,6 +1,8 @@
 from msgpack import fallback
+
 try:
     from msgpack import _unpacker, _packer
+
     has_ext = True
 except ImportError:
     has_ext = False
@@ -9,7 +11,7 @@ import timeit
 
 def profile(name, func):
     times = timeit.repeat(func, number=1000, repeat=4)
-    times = ', '.join(["%8f" % t for t in times])
+    times = ", ".join(["%8f" % t for t in times])
     print("%-30s %40s" % (name, times))
 
 
@@ -18,17 +20,19 @@ def simple(name, data):
         packer = _packer.Packer()
         profile("packing %s (ext)" % name, lambda: packer.pack(data))
     packer = fallback.Packer()
-    profile('packing %s (fallback)' % name, lambda: packer.pack(data))
+    profile("packing %s (fallback)" % name, lambda: packer.pack(data))
 
     data = packer.pack(data)
     if has_ext:
-        profile('unpacking %s (ext)' % name, lambda: _unpacker.unpackb(data))
-    profile('unpacking %s (fallback)' % name, lambda: fallback.unpackb(data))
+        profile("unpacking %s (ext)" % name, lambda: _unpacker.unpackb(data))
+    profile("unpacking %s (fallback)" % name, lambda: fallback.unpackb(data))
+
 
 def main():
-    simple("integers", [7]*10000)
-    simple("bytes", [b'x'*n for n in range(100)]*10)
-    simple("lists", [[]]*10000)
-    simple("dicts", [{}]*10000)
+    simple("integers", [7] * 10000)
+    simple("bytes", [b"x" * n for n in range(100)] * 10)
+    simple("lists", [[]] * 10000)
+    simple("dicts", [{}] * 10000)
+
 
 main()
diff --git a/docs/conf.py b/docs/conf.py
index 47d745a..36fa76e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -16,32 +16,32 @@ import sys, os
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration -----------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
+extensions = ["sphinx.ext.autodoc", "sphinx.ext.viewcode"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'msgpack'
-copyright = u'2013, INADA Naoki'
+project = u"msgpack"
+copyright = u"2013, INADA Naoki"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -49,176 +49,170 @@ copyright = u'2013, INADA Naoki'
 #
 # The short X.Y version.
 # The full version, including alpha/beta/rc tags.
-version = release = '0.5'
+version = release = "0.5"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
-#language = None
+# language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 today_fmt = "%Y-%m-%d"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = ["_build"]
 
 # The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 
 # -- Options for HTML output ---------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'sphinxdoc'
+html_theme = "sphinxdoc"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+# html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # " v documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a  tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'msgpackdoc'
+htmlhelp_basename = "msgpackdoc"
 
 
 # -- Options for LaTeX output --------------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'msgpack.tex', u'msgpack Documentation',
-   u'Author', 'manual'),
+    ("index", "msgpack.tex", u"msgpack Documentation", u"Author", "manual"),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output --------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'msgpack', u'msgpack Documentation',
-     [u'Author'], 1)
-]
+man_pages = [("index", "msgpack", u"msgpack Documentation", [u"Author"], 1)]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output ------------------------------------------------
@@ -227,59 +221,65 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'msgpack', u'msgpack Documentation',
-   u'Author', 'msgpack', 'One line description of project.',
-   'Miscellaneous'),
+    (
+        "index",
+        "msgpack",
+        u"msgpack Documentation",
+        u"Author",
+        "msgpack",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 
 # -- Options for Epub output ---------------------------------------------------
 
 # Bibliographic Dublin Core info.
-epub_title = u'msgpack'
-epub_author = u'Author'
-epub_publisher = u'Author'
-epub_copyright = u'2013, Author'
+epub_title = u"msgpack"
+epub_author = u"Author"
+epub_publisher = u"Author"
+epub_copyright = u"2013, Author"
 
 # The language of the text. It defaults to the language option
 # or en if the language is not set.
-#epub_language = ''
+# epub_language = ''
 
 # The scheme of the identifier. Typical schemes are ISBN or URL.
-#epub_scheme = ''
+# epub_scheme = ''
 
 # The unique identifier of the text. This can be a ISBN number
 # or the project homepage.
-#epub_identifier = ''
+# epub_identifier = ''
 
 # A unique identification for the text.
-#epub_uid = ''
+# epub_uid = ''
 
 # A tuple containing the cover image and cover page html template filenames.
-#epub_cover = ()
+# epub_cover = ()
 
 # HTML files that should be inserted before the pages created by sphinx.
 # The format is a list of tuples containing the path and title.
-#epub_pre_files = []
+# epub_pre_files = []
 
 # HTML files shat should be inserted after the pages created by sphinx.
 # The format is a list of tuples containing the path and title.
-#epub_post_files = []
+# epub_post_files = []
 
 # A list of files that should not be packed into the epub file.
-#epub_exclude_files = []
+# epub_exclude_files = []
 
 # The depth of the table of contents in toc.ncx.
-#epub_tocdepth = 3
+# epub_tocdepth = 3
 
 # Allow duplicate toc entries.
-#epub_tocdup = True
+# epub_tocdup = True
diff --git a/setup.py b/setup.py
index 77b81c6..a8c2306 100755
--- a/setup.py
+++ b/setup.py
@@ -17,11 +17,14 @@ PY2 = sys.version_info[0] == 2
 # for building transitional package.
 TRANSITIONAL = False
 
+
 class NoCython(Exception):
     pass
 
+
 try:
     import Cython.Compiler.Main as cython_compiler
+
     have_cython = True
 except ImportError:
     have_cython = False
@@ -31,16 +34,19 @@ def cythonize(src):
     sys.stderr.write("cythonize: %r\n" % (src,))
     cython_compiler.compile([src], cplus=True)
 
+
 def ensure_source(src):
-    pyx = os.path.splitext(src)[0] + '.pyx'
+    pyx = os.path.splitext(src)[0] + ".pyx"
 
     if not os.path.exists(src):
         if not have_cython:
             raise NoCython
         cythonize(pyx)
-    elif (os.path.exists(pyx) and
-          os.stat(src).st_mtime < os.stat(pyx).st_mtime and
-          have_cython):
+    elif (
+        os.path.exists(pyx)
+        and os.stat(src).st_mtime < os.stat(pyx).st_mtime
+        and have_cython
+    ):
         cythonize(pyx)
     return src
 
@@ -63,77 +69,82 @@ class BuildExt(build_ext):
             print(e)
 
 
-exec(open('msgpack/_version.py').read())
+exec(open("msgpack/_version.py").read())
 
-version_str = '.'.join(str(x) for x in version[:3])
-if len(version) > 3 and version[3] != 'final':
+version_str = ".".join(str(x) for x in version[:3])
+if len(version) > 3 and version[3] != "final":
     version_str += version[3]
 
 # Cython is required for sdist
 class Sdist(sdist):
     def __init__(self, *args, **kwargs):
-        cythonize('msgpack/_cmsgpack.pyx')
+        cythonize("msgpack/_cmsgpack.pyx")
         sdist.__init__(self, *args, **kwargs)
 
-libraries = []
-if sys.platform == 'win32':
-    libraries.append('ws2_32')
 
-if sys.byteorder == 'big':
-    macros = [('__BIG_ENDIAN__', '1')]
+libraries = []
+if sys.platform == "win32":
+    libraries.append("ws2_32")
+
+if sys.byteorder == "big":
+    macros = [("__BIG_ENDIAN__", "1")]
 else:
-    macros = [('__LITTLE_ENDIAN__', '1')]
+    macros = [("__LITTLE_ENDIAN__", "1")]
 
 ext_modules = []
 if not PYPY and not PY2:
-    ext_modules.append(Extension('msgpack._cmsgpack',
-                                 sources=['msgpack/_cmsgpack.cpp'],
-                                 libraries=libraries,
-                                 include_dirs=['.'],
-                                 define_macros=macros,
-                                 ))
+    ext_modules.append(
+        Extension(
+            "msgpack._cmsgpack",
+            sources=["msgpack/_cmsgpack.cpp"],
+            libraries=libraries,
+            include_dirs=["."],
+            define_macros=macros,
+        )
+    )
 del libraries, macros
 
 
-desc = 'MessagePack (de)serializer.'
-with io.open('README.rst', encoding='utf-8') as f:
+desc = "MessagePack (de)serializer."
+with io.open("README.rst", encoding="utf-8") as f:
     long_desc = f.read()
 del f
 
-name = 'msgpack'
+name = "msgpack"
 
 if TRANSITIONAL:
-    name = 'msgpack-python'
+    name = "msgpack-python"
     long_desc = "This package is deprecated.  Install msgpack instead."
 
-setup(name=name,
-      author='INADA Naoki',
-      author_email='songofacandy@gmail.com',
-      version=version_str,
-      cmdclass={'build_ext': BuildExt, 'sdist': Sdist},
-      ext_modules=ext_modules,
-      packages=['msgpack'],
-      description=desc,
-      long_description=long_desc,
-      long_description_content_type="text/x-rst",
-      url='https://msgpack.org/',
-      project_urls = {
-          'Documentation': 'https://msgpack-python.readthedocs.io/',
-          'Source': 'https://github.com/msgpack/msgpack-python',
-          'Tracker': 'https://github.com/msgpack/msgpack-python/issues',
-      },
-      license='Apache 2.0',
-      classifiers=[
-          'Programming Language :: Python :: 2',
-          'Programming Language :: Python :: 2.7',
-          'Programming Language :: Python :: 3',
-          'Programming Language :: Python :: 3.5',
-          'Programming Language :: Python :: 3.6',
-          'Programming Language :: Python :: 3.7',
-          'Programming Language :: Python :: 3.8',
-          'Programming Language :: Python :: Implementation :: CPython',
-          'Programming Language :: Python :: Implementation :: PyPy',
-          'Intended Audience :: Developers',
-          'License :: OSI Approved :: Apache Software License',
-      ],
+setup(
+    name=name,
+    author="INADA Naoki",
+    author_email="songofacandy@gmail.com",
+    version=version_str,
+    cmdclass={"build_ext": BuildExt, "sdist": Sdist},
+    ext_modules=ext_modules,
+    packages=["msgpack"],
+    description=desc,
+    long_description=long_desc,
+    long_description_content_type="text/x-rst",
+    url="https://msgpack.org/",
+    project_urls={
+        "Documentation": "https://msgpack-python.readthedocs.io/",
+        "Source": "https://github.com/msgpack/msgpack-python",
+        "Tracker": "https://github.com/msgpack/msgpack-python/issues",
+    },
+    license="Apache 2.0",
+    classifiers=[
+        "Programming Language :: Python :: 2",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: Implementation :: CPython",
+        "Programming Language :: Python :: Implementation :: PyPy",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: Apache Software License",
+    ],
 )

From af4eea430e2f176f17fff5abe781dd83f55d4657 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 5 Dec 2019 18:54:14 +0900
Subject: [PATCH 1487/1648] travis: Add Black

---
 .travis.yml | 9 ++++++++-
 Makefile    | 4 ++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 7b298af..9d3ae74 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,8 +14,15 @@ python:
 
 matrix:
   include:
+    - name: Black
+      language: python
+      python: 3.8
+      install:
+        - pip install black
+      script:
+        - black --check --diff .
+
     - name: 32bit build
-      sudo: required
       language: python
       services:
         - docker
diff --git a/Makefile b/Makefile
index 5828ed4..a1edc88 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,10 @@
 all: cython
 	python setup.py build_ext -i -f
 
+.PHONY: black
+black:
+	black .
+
 .PHONY: cython
 cython:
 	cython --cplus msgpack/_cmsgpack.pyx

From 9ae43709e42092c7f6a4e990d696d9005fa1623d Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 5 Dec 2019 20:20:53 +0900
Subject: [PATCH 1488/1648] Drop old buffer protocol support (#383)

---
 msgpack/_unpacker.pyx | 60 ++++++++++++++++---------------------------
 1 file changed, 22 insertions(+), 38 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 6dedd30..3c9b7b3 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -109,38 +109,26 @@ def default_read_extended_type(typecode, data):
 cdef inline int get_data_from_buffer(object obj,
                                      Py_buffer *view,
                                      char **buf,
-                                     Py_ssize_t *buffer_len,
-                                     int *new_protocol) except 0:
+                                     Py_ssize_t *buffer_len) except 0:
     cdef object contiguous
     cdef Py_buffer tmp
-    if PyObject_CheckBuffer(obj):
-        new_protocol[0] = 1
-        if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1:
-            raise
-        if view.itemsize != 1:
-            PyBuffer_Release(view)
-            raise BufferError("cannot unpack from multi-byte object")
-        if PyBuffer_IsContiguous(view, b'A') == 0:
-            PyBuffer_Release(view)
-            # create a contiguous copy and get buffer
-            contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C')
-            PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE)
-            # view must hold the only reference to contiguous,
-            # so memory is freed when view is released
-            Py_DECREF(contiguous)
-        buffer_len[0] = view.len
-        buf[0] =  view.buf
-        return 1
-    else:
-        new_protocol[0] = 0
-        if PyObject_AsReadBuffer(obj,  buf, buffer_len) == -1:
-            raise BufferError("could not get memoryview")
-        PyErr_WarnEx(RuntimeWarning,
-                     "using old buffer interface to unpack %s; "
-                     "this leads to unpacking errors if slicing is used and "
-                     "will be removed in a future version" % type(obj),
-                     1)
-        return 1
+    if PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) == -1:
+        raise
+    if view.itemsize != 1:
+        PyBuffer_Release(view)
+        raise BufferError("cannot unpack from multi-byte object")
+    if PyBuffer_IsContiguous(view, b'A') == 0:
+        PyBuffer_Release(view)
+        # create a contiguous copy and get buffer
+        contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C')
+        PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE)
+        # view must hold the only reference to contiguous,
+        # so memory is freed when view is released
+        Py_DECREF(contiguous)
+    buffer_len[0] = view.len
+    buf[0] =  view.buf
+    return 1
+
 
 def unpackb(object packed, *, object object_hook=None, object list_hook=None,
             bint use_list=True, bint raw=True, bint strict_map_key=False,
@@ -172,12 +160,11 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None,
     cdef char* buf = NULL
     cdef Py_ssize_t buf_len
     cdef const char* cerr = NULL
-    cdef int new_protocol = 0
 
     if unicode_errors is not None:
         cerr = unicode_errors
 
-    get_data_from_buffer(packed, &view, &buf, &buf_len, &new_protocol)
+    get_data_from_buffer(packed, &view, &buf, &buf_len)
 
     if max_str_len == -1:
         max_str_len = buf_len
@@ -196,8 +183,7 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None,
                  max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
         ret = unpack_construct(&ctx, buf, buf_len, &off)
     finally:
-        if new_protocol:
-            PyBuffer_Release(&view);
+        PyBuffer_Release(&view);
 
     if ret == 1:
         obj = unpack_data(&ctx)
@@ -392,7 +378,6 @@ cdef class Unpacker(object):
     def feed(self, object next_bytes):
         """Append `next_bytes` to internal buffer."""
         cdef Py_buffer pybuff
-        cdef int new_protocol = 0
         cdef char* buf
         cdef Py_ssize_t buf_len
 
@@ -400,12 +385,11 @@ cdef class Unpacker(object):
             raise AssertionError(
                     "unpacker.feed() is not be able to use with `file_like`.")
 
-        get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len, &new_protocol)
+        get_data_from_buffer(next_bytes, &pybuff, &buf, &buf_len)
         try:
             self.append_buffer(buf, buf_len)
         finally:
-            if new_protocol:
-                PyBuffer_Release(&pybuff)
+            PyBuffer_Release(&pybuff)
 
     cdef append_buffer(self, void* _buf, Py_ssize_t _buf_len):
         cdef:

From 9f4b2d53b77c5ccd96e3ceb359747960cbf03bd4 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 5 Dec 2019 20:47:01 +0900
Subject: [PATCH 1489/1648] Remove deprecated submodule unpack (#385)

---
 msgpack/_unpacker.pyx |  8 --------
 msgpack/fallback.py   | 10 ----------
 2 files changed, 18 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 3c9b7b3..3a9d494 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -200,14 +200,6 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None,
     raise ValueError("Unpack failed: error = %d" % (ret,))
 
 
-def unpack(object stream, **kwargs):
-    PyErr_WarnEx(
-        DeprecationWarning,
-        "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", 1)
-    data = stream.read()
-    return unpackb(data, **kwargs)
-
-
 cdef class Unpacker(object):
     """Streaming unpacker.
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 577e571..3faacbf 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -131,16 +131,6 @@ def _get_data_from_buffer(obj):
     return view
 
 
-def unpack(stream, **kwargs):
-    warnings.warn(
-        "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    data = stream.read()
-    return unpackb(data, **kwargs)
-
-
 def unpackb(packed, **kwargs):
     """
     Unpack an object from `packed`.

From de320488ae494b85a03b60dd33f91b650033d775 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 5 Dec 2019 20:47:20 +0900
Subject: [PATCH 1490/1648] fallback: Remove old buffer protocol support (#384)

---
 msgpack/fallback.py | 16 +---------------
 test/test_buffer.py |  8 ++++----
 2 files changed, 5 insertions(+), 19 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 3faacbf..9de3553 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -111,21 +111,7 @@ def _check_type_strict(obj, t, type=type, tuple=tuple):
 
 
 def _get_data_from_buffer(obj):
-    try:
-        view = memoryview(obj)
-    except TypeError:
-        # try to use legacy buffer protocol if 2.7, otherwise re-raise
-        if PY2:
-            view = memoryview(buffer(obj))
-            warnings.warn(
-                "using old buffer interface to unpack %s; "
-                "this leads to unpacking errors if slicing is used and "
-                "will be removed in a future version" % type(obj),
-                RuntimeWarning,
-                stacklevel=3,
-            )
-        else:
-            raise
+    view = memoryview(obj)
     if view.itemsize != 1:
         raise ValueError("cannot unpack from multi-byte object")
     return view
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 64fbdef..da68b27 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -1,17 +1,17 @@
 #!/usr/bin/env python
 # coding: utf-8
 
+import sys
+import pytest
 from msgpack import packb, unpackb
 
 
+@pytest.mark.skipif(sys.version_info[0] == 2, reason="Python 2 is not supported")
 def test_unpack_buffer():
     from array import array
 
     buf = array("b")
-    try:
-        buf.frombytes(packb((b"foo", b"bar")))
-    except AttributeError:  # PY2
-        buf.fromstring(packb((b"foo", b"bar")))
+    buf.frombytes(packb((b"foo", b"bar")))
     obj = unpackb(buf, use_list=1)
     assert [b"foo", b"bar"] == obj
 

From 7e9905bdfaecde83ddb1a4575e734a10b055fde9 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 5 Dec 2019 21:34:10 +0900
Subject: [PATCH 1491/1648] Use new msgpack spec by default. (#386)

---
 README.rst              | 70 ++++++++++++-----------------------------
 msgpack/_packer.pyx     |  6 ++--
 msgpack/_unpacker.pyx   | 16 ++++------
 msgpack/fallback.py     | 20 +++++-------
 test/test_buffer.py     |  4 +--
 test/test_case.py       | 11 +++----
 test/test_format.py     | 10 ++++--
 test/test_memoryview.py | 39 +++++++----------------
 test/test_newspec.py    |  6 ++--
 test/test_read_size.py  | 10 +++---
 test/test_sequnpack.py  |  9 +++---
 11 files changed, 75 insertions(+), 126 deletions(-)

diff --git a/README.rst b/README.rst
index 82b6c02..f9f074f 100644
--- a/README.rst
+++ b/README.rst
@@ -37,36 +37,16 @@ Sadly, this doesn't work for upgrade install.  After `pip install -U msgpack-pyt
 msgpack is removed and `import msgpack` fail.
 
 
-Deprecating encoding option
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Compatibility with old format
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-encoding and unicode_errors options are deprecated.
+You can use ``use_bin_type=False`` option to pack ``bytes``
+object into raw type in old msgpack spec, instead of bin type in new msgpack spec.
 
-In case of packer, use UTF-8 always.  Storing other than UTF-8 is not recommended.
+You can unpack old msgpack formatk using ``raw=True`` option.
+It unpacks str (raw) type in msgpack into Python bytes.
 
-For backward compatibility, you can use ``use_bin_type=False`` and pack ``bytes``
-object into msgpack raw type.
-
-In case of unpacker, there is new ``raw`` option.  It is ``True`` by default
-for backward compatibility, but it is changed to ``False`` in near future.
-You can use ``raw=False`` instead of ``encoding='utf-8'``.
-
-Planned backward incompatible changes
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-When msgpack 1.0, I planning these breaking changes:
-
-* packer and unpacker: Remove ``encoding`` and ``unicode_errors`` option.
-* packer: Change default of ``use_bin_type`` option from False to True.
-* unpacker: Change default of ``raw`` option from True to False.
-* unpacker: Reduce all ``max_xxx_len`` options for typical usage.
-* unpacker: Remove ``write_bytes`` option from all methods.
-
-To avoid these breaking changes breaks your application, please:
-
-* Don't use deprecated options.
-* Pass ``use_bin_type`` and ``raw`` options explicitly.
-* If your application handle large (>1MB) data, specify ``max_xxx_len`` options too.
+See note in below for detail.
 
 
 Install
@@ -76,6 +56,7 @@ Install
 
    $ pip install msgpack
 
+
 Pure Python implementation
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -100,6 +81,13 @@ Without extension, using pure Python implementation on CPython runs slowly.
 How to use
 ----------
 
+.. note::
+
+   In examples below, I use ``raw=False`` and ``use_bin_type=True`` for users
+   using msgpack < 1.0.
+   These options are default from msgpack 1.0 so you can omit them.
+
+
 One-shot pack & unpack
 ^^^^^^^^^^^^^^^^^^^^^^
 
@@ -252,36 +240,18 @@ Notes
 string and binary type
 ^^^^^^^^^^^^^^^^^^^^^^
 
-Early versions of msgpack didn't distinguish string and binary types (like Python 1).
+Early versions of msgpack didn't distinguish string and binary types.
 The type for representing both string and binary types was named **raw**.
 
-For backward compatibility reasons, msgpack-python will still default all
-strings to byte strings, unless you specify the ``use_bin_type=True`` option in
-the packer. If you do so, it will use a non-standard type called **bin** to
-serialize byte arrays, and **raw** becomes to mean **str**. If you want to
-distinguish **bin** and **raw** in the unpacker, specify ``raw=False``.
-
-Note that Python 2 defaults to byte-arrays over Unicode strings:
+You can pack into and unpack from this old spec using ``use_bin_type=False``
+and ``raw=True`` options.
 
 .. code-block:: pycon
 
     >>> import msgpack
-    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
-    ['spam', 'eggs']
-    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
-                        raw=False)
-    ['spam', u'eggs']
-
-This is the same code in Python 3 (same behaviour, but Python 3 has a
-different default):
-
-.. code-block:: pycon
-
-    >>> import msgpack
-    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs']))
+    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True)
     [b'spam', b'eggs']
-    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True),
-                        raw=False)
+    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False)
     [b'spam', 'eggs']
 
 
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index f3bde3f..8cf3c05 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -80,9 +80,7 @@ cdef class Packer(object):
 
     :param bool use_bin_type:
         Use bin type introduced in msgpack spec 2.0 for bytes.
-        It also enables str8 type for unicode.
-        Current default value is false, but it will be changed to true
-        in future version.  You should specify it explicitly.
+        It also enables str8 type for unicode. (default: True)
 
     :param bool strict_types:
         If set to true, types will be checked to be exact. Derived classes
@@ -113,7 +111,7 @@ cdef class Packer(object):
         self.pk.length = 0
 
     def __init__(self, *, default=None, unicode_errors=None,
-                 bint use_single_float=False, bint autoreset=True, bint use_bin_type=False,
+                 bint use_single_float=False, bint autoreset=True, bint use_bin_type=True,
                  bint strict_types=False):
         self.use_float = use_single_float
         self.strict_types = strict_types
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 3a9d494..f10e99d 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -131,7 +131,7 @@ cdef inline int get_data_from_buffer(object obj,
 
 
 def unpackb(object packed, *, object object_hook=None, object list_hook=None,
-            bint use_list=True, bint raw=True, bint strict_map_key=False,
+            bint use_list=True, bint raw=False, bint strict_map_key=False,
             unicode_errors=None,
             object_pairs_hook=None, ext_hook=ExtType,
             Py_ssize_t max_str_len=-1,
@@ -217,12 +217,8 @@ cdef class Unpacker(object):
         Otherwise, unpack to Python tuple. (default: True)
 
     :param bool raw:
-        If true, unpack msgpack raw to Python bytes (default).
-        Otherwise, unpack to Python str (or unicode on Python 2) by decoding
-        with UTF-8 encoding (recommended).
-        Currently, the default is true, but it will be changed to false in
-        near future.  So you must specify it explicitly for keeping backward
-        compatibility.
+        If true, unpack msgpack raw to Python bytes.
+        Otherwise, unpack to Python str by decoding with UTF-8 encoding (default).
 
     :param bool strict_map_key:
         If true, only str or bytes are accepted for map (dict) keys.
@@ -268,13 +264,13 @@ cdef class Unpacker(object):
 
     Example of streaming deserialize from file-like object::
 
-        unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024)
+        unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024)
         for o in unpacker:
             process(o)
 
     Example of streaming deserialize from socket::
 
-        unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024)
+        unpacker = Unpacker(max_buffer_size=10*1024*1024)
         while True:
             buf = sock.recv(1024**2)
             if not buf:
@@ -309,7 +305,7 @@ cdef class Unpacker(object):
         self.buf = NULL
 
     def __init__(self, file_like=None, *, Py_ssize_t read_size=0,
-                 bint use_list=True, bint raw=True, bint strict_map_key=False,
+                 bint use_list=True, bint raw=False, bint strict_map_key=False,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  unicode_errors=None, Py_ssize_t max_buffer_size=0,
                  object ext_hook=ExtType,
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 9de3553..fa2f3a8 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -158,7 +158,7 @@ else:
 class Unpacker(object):
     """Streaming unpacker.
 
-    arguments:
+    Arguments:
 
     :param file_like:
         File-like object having `.read(n)` method.
@@ -172,12 +172,8 @@ class Unpacker(object):
         Otherwise, unpack to Python tuple. (default: True)
 
     :param bool raw:
-        If true, unpack msgpack raw to Python bytes (default).
-        Otherwise, unpack to Python str (or unicode on Python 2) by decoding
-        with UTF-8 encoding (recommended).
-        Currently, the default is true, but it will be changed to false in
-        near future.  So you must specify it explicitly for keeping backward
-        compatibility.
+        If true, unpack msgpack raw to Python bytes.
+        Otherwise, unpack to Python str by decoding with UTF-8 encoding (default).
 
     :param bool strict_map_key:
         If true, only str or bytes are accepted for map (dict) keys.
@@ -226,13 +222,13 @@ class Unpacker(object):
 
     Example of streaming deserialize from file-like object::
 
-        unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024)
+        unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024)
         for o in unpacker:
             process(o)
 
     Example of streaming deserialize from socket::
 
-        unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024)
+        unpacker = Unpacker(max_buffer_size=10*1024*1024)
         while True:
             buf = sock.recv(1024**2)
             if not buf:
@@ -253,7 +249,7 @@ class Unpacker(object):
         file_like=None,
         read_size=0,
         use_list=True,
-        raw=True,
+        raw=False,
         strict_map_key=False,
         object_hook=None,
         object_pairs_hook=None,
@@ -748,7 +744,7 @@ class Packer(object):
 
     :param bool use_bin_type:
         Use bin type introduced in msgpack spec 2.0 for bytes.
-        It also enables str8 type for unicode.
+        It also enables str8 type for unicode. (default: True)
 
     :param bool strict_types:
         If set to true, types will be checked to be exact. Derived classes
@@ -769,7 +765,7 @@ class Packer(object):
         unicode_errors=None,
         use_single_float=False,
         autoreset=True,
-        use_bin_type=False,
+        use_bin_type=True,
         strict_types=False,
     ):
         self._strict_types = strict_types
diff --git a/test/test_buffer.py b/test/test_buffer.py
index da68b27..62507cf 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -17,7 +17,7 @@ def test_unpack_buffer():
 
 
 def test_unpack_bytearray():
-    buf = bytearray(packb(("foo", "bar")))
+    buf = bytearray(packb((b"foo", b"bar")))
     obj = unpackb(buf, use_list=1)
     assert [b"foo", b"bar"] == obj
     expected_type = bytes
@@ -25,7 +25,7 @@ def test_unpack_bytearray():
 
 
 def test_unpack_memoryview():
-    buf = bytearray(packb(("foo", "bar")))
+    buf = bytearray(packb((b"foo", b"bar")))
     view = memoryview(buf)
     obj = unpackb(view, use_list=1)
     assert [b"foo", b"bar"] == obj
diff --git a/test/test_case.py b/test/test_case.py
index 3bc1b26..3e60e59 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -1,13 +1,12 @@
 #!/usr/bin/env python
 # coding: utf-8
-
 from msgpack import packb, unpackb
 
 
-def check(length, obj):
-    v = packb(obj)
+def check(length, obj, use_bin_type=True):
+    v = packb(obj, use_bin_type=use_bin_type)
     assert len(v) == length, "%r length should be %r but get %r" % (obj, length, len(v))
-    assert unpackb(v, use_list=0) == obj
+    assert unpackb(v, use_list=0, raw=not use_bin_type) == obj
 
 
 def test_1():
@@ -56,7 +55,7 @@ def test_9():
 
 
 def check_raw(overhead, num):
-    check(num + overhead, b" " * num)
+    check(num + overhead, b" " * num, use_bin_type=False)
 
 
 def test_fixraw():
@@ -135,4 +134,4 @@ def test_match():
 
 
 def test_unicode():
-    assert unpackb(packb("foobar"), use_list=1) == b"foobar"
+    assert unpackb(packb(u"foobar"), use_list=1) == u"foobar"
diff --git a/test/test_format.py b/test/test_format.py
index c2cdfbd..8c2f03f 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -4,8 +4,8 @@
 from msgpack import unpackb
 
 
-def check(src, should, use_list=0):
-    assert unpackb(src, use_list=use_list) == should
+def check(src, should, use_list=0, raw=True):
+    assert unpackb(src, use_list=use_list, raw=raw) == should
 
 
 def testSimpleValue():
@@ -59,6 +59,12 @@ def testRaw():
         b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
         (b"", b"a", b"ab", b"", b"a", b"ab"),
     )
+    check(
+        b"\x96\xda\x00\x00\xda\x00\x01a\xda\x00\x02ab\xdb\x00\x00"
+        b"\x00\x00\xdb\x00\x00\x00\x01a\xdb\x00\x00\x00\x02ab",
+        ("", "a", "ab", "", "a", "ab"),
+        raw=False,
+    )
 
 
 def testArray():
diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index e1b63b8..86b2c1f 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -1,50 +1,33 @@
 #!/usr/bin/env python
 # coding: utf-8
 
+import pytest
 from array import array
 from msgpack import packb, unpackb
 import sys
 
 
-# For Python < 3:
-#  - array type only supports old buffer interface
-#  - array.frombytes is not available, must use deprecated array.fromstring
-if sys.version_info[0] < 3:
-
-    def make_memoryview(obj):
-        return memoryview(buffer(obj))
-
-    def make_array(f, data):
-        a = array(f)
-        a.fromstring(data)
-        return a
-
-    def get_data(a):
-        return a.tostring()
+pytestmark = pytest.mark.skipif(
+    sys.version_info[0] < 3, reason="Only Python 3 supports buffer protocol"
+)
 
 
-else:
-    make_memoryview = memoryview
-
-    def make_array(f, data):
-        a = array(f)
-        a.frombytes(data)
-        return a
-
-    def get_data(a):
-        return a.tobytes()
+def make_array(f, data):
+    a = array(f)
+    a.frombytes(data)
+    return a
 
 
 def _runtest(format, nbytes, expected_header, expected_prefix, use_bin_type):
     # create a new array
     original_array = array(format)
     original_array.fromlist([255] * (nbytes // original_array.itemsize))
-    original_data = get_data(original_array)
-    view = make_memoryview(original_array)
+    original_data = original_array.tobytes()
+    view = memoryview(original_array)
 
     # pack, unpack, and reconstruct array
     packed = packb(view, use_bin_type=use_bin_type)
-    unpacked = unpackb(packed)
+    unpacked = unpackb(packed, raw=(not use_bin_type))
     reconstructed_array = make_array(format, unpacked)
 
     # check that we got the right amount of data
diff --git a/test/test_newspec.py b/test/test_newspec.py
index f4f2a23..b7da486 100644
--- a/test/test_newspec.py
+++ b/test/test_newspec.py
@@ -10,14 +10,16 @@ def test_str8():
     assert len(b) == len(data) + 2
     assert b[0:2] == header + b"\x20"
     assert b[2:] == data
-    assert unpackb(b) == data
+    assert unpackb(b, raw=True) == data
+    assert unpackb(b, raw=False) == data.decode()
 
     data = b"x" * 255
     b = packb(data.decode(), use_bin_type=True)
     assert len(b) == len(data) + 2
     assert b[0:2] == header + b"\xff"
     assert b[2:] == data
-    assert unpackb(b) == data
+    assert unpackb(b, raw=True) == data
+    assert unpackb(b, raw=False) == data.decode()
 
 
 def test_bin8():
diff --git a/test/test_read_size.py b/test/test_read_size.py
index 8d8df64..33a7e7d 100644
--- a/test/test_read_size.py
+++ b/test/test_read_size.py
@@ -8,9 +8,9 @@ def test_read_array_header():
     unpacker = Unpacker()
     unpacker.feed(packb(["a", "b", "c"]))
     assert unpacker.read_array_header() == 3
-    assert unpacker.unpack() == b"a"
-    assert unpacker.unpack() == b"b"
-    assert unpacker.unpack() == b"c"
+    assert unpacker.unpack() == "a"
+    assert unpacker.unpack() == "b"
+    assert unpacker.unpack() == "c"
     try:
         unpacker.unpack()
         assert 0, "should raise exception"
@@ -22,8 +22,8 @@ def test_read_map_header():
     unpacker = Unpacker()
     unpacker.feed(packb({"a": "A"}))
     assert unpacker.read_map_header() == 1
-    assert unpacker.unpack() == b"a"
-    assert unpacker.unpack() == b"A"
+    assert unpacker.unpack() == "a"
+    assert unpacker.unpack() == "A"
     try:
         unpacker.unpack()
         assert 0, "should raise exception"
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index e576571..9b69479 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 # coding: utf-8
-
 import io
 from msgpack import Unpacker, BufferFull
 from msgpack import pack
@@ -26,7 +25,7 @@ def test_partialdata():
     with raises(StopIteration):
         next(iter(unpacker))
     unpacker.feed(b"o")
-    assert next(iter(unpacker)) == b"hallo"
+    assert next(iter(unpacker)) == "hallo"
 
 
 def test_foobar():
@@ -98,13 +97,13 @@ def test_readbytes():
 def test_issue124():
     unpacker = Unpacker()
     unpacker.feed(b"\xa1?\xa1!")
-    assert tuple(unpacker) == (b"?", b"!")
+    assert tuple(unpacker) == ("?", "!")
     assert tuple(unpacker) == ()
     unpacker.feed(b"\xa1?\xa1")
-    assert tuple(unpacker) == (b"?",)
+    assert tuple(unpacker) == ("?",)
     assert tuple(unpacker) == ()
     unpacker.feed(b"!")
-    assert tuple(unpacker) == (b"!",)
+    assert tuple(unpacker) == ("!",)
     assert tuple(unpacker) == ()
 
 

From 235c6036eabd89d5a28244091ac1e4f091b49679 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 6 Dec 2019 19:28:23 +0900
Subject: [PATCH 1492/1648] travis: Use codecov (#387)

---
 .travis.yml | 52 +++++++++++++++++++++++++++++-----------------------
 1 file changed, 29 insertions(+), 23 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 9d3ae74..378bc80 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,11 +12,20 @@ python:
   - "3.7"
   - "3.8-dev"
 
+
+_pure: &pure
+  install:
+    - pip install -U pip
+    - pip install -U pytest pytest-cov codecov
+    - pip install .
+  script:
+    - pytest --cov=msgpack -v test
+
 matrix:
   include:
     - name: Black
       language: python
-      python: 3.8
+      python: 3.7
       install:
         - pip install black
       script:
@@ -35,38 +44,35 @@ matrix:
         - docker pull $DOCKER_IMAGE
       script:
         - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
-    - name: "pypy2.7"
-      python: "pypy2.7-7.1.1"
-      install:
-        - pip install -e .
-      script:
-        - py.test -v test
-    - name: "pypy3"
-      python: "pypy3.6-7.1.1"
-      install:
-        - pip install -e .
-      script:
-        - pytest -v test
+
     - name: "Python 2 (fallback)"
       python: "2.7"
-      install:
-        - pip install -U pip
-        - pip install -U pytest
-        - pip install .
-      script:
-        - pytest -v test
+      <<: *pure
+
+    - name: "pypy2.7"
+      python: "pypy2.7-7.1.1"
+      <<: *pure
+
+    - name: "pypy3"
+      python: "pypy3.6-7.1.1"
+      <<: *pure
 
 install:
   - pip install -U pip
-  - pip install -U pytest
-  - pip install -r requirements.txt
+  - pip install -U pytest pytest-cov codecov
+  - pip install -r requirements.txt  # Cython
   - make cython
   - pip install -e .
 
 script:
   - python -c 'import sys; print(hex(sys.maxsize))'
   - python -c 'from msgpack import _cmsgpack'
-  - pytest -v test
-  - MSGPACK_PUREPYTHON=x pytest -v test
+  - pytest --cov=msgpack -v test
+  - MSGPACK_PUREPYTHON=x pytest --cov=msgpack -v test
+
+after_success:
+  - if [ -f .coverage ]; then
+      codecov;
+    fi
 
 # vim: sw=2 ts=2

From 7a8ce0f9ca910a851b6835d26b1d6970a188fa4e Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 6 Dec 2019 20:34:18 +0900
Subject: [PATCH 1493/1648] Remove unused import

---
 Makefile            | 2 +-
 msgpack/fallback.py | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index a1edc88..f8971cc 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ all: cython
 
 .PHONY: black
 black:
-	black .
+	black msgpack/ test/
 
 .PHONY: cython
 cython:
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index fa2f3a8..388a5ab 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -2,7 +2,6 @@
 
 import sys
 import struct
-import warnings
 
 
 PY2 = sys.version_info[0] == 2

From f6f6f328eb2d7b7f1272fa7addb31d2ac5bef207 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 6 Dec 2019 21:16:27 +0900
Subject: [PATCH 1494/1648] Fix fallback Unpacker.read() (#388)

Fixes #352.
---
 msgpack/fallback.py    | 4 +++-
 test/test_sequnpack.py | 9 +++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 388a5ab..85a711b 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -357,7 +357,9 @@ class Unpacker(object):
         return self._buffer[self._buff_i :]
 
     def read_bytes(self, n):
-        return self._read(n)
+        ret = self._read(n)
+        self._consume()
+        return ret
 
     def _read(self, n):
         # (int) -> bytearray
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 9b69479..ad29de8 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -93,6 +93,15 @@ def test_readbytes():
     assert unpacker.unpack() == ord(b"a")
     assert unpacker.unpack() == ord(b"r")
 
+    # Issue 352
+    u = Unpacker()
+    u.feed(b"x")
+    assert bytes(u.read_bytes(1)) == b"x"
+    with raises(StopIteration):
+        next(u)
+    u.feed(b"\1")
+    assert next(u) == 1
+
 
 def test_issue124():
     unpacker = Unpacker()

From 5ba496c79a45c6938f3e850718349cfa51cc38ae Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 6 Dec 2019 21:23:54 +0900
Subject: [PATCH 1495/1648] Move Black from Travis to Github Actions (#390)

---
 .github/workflows/black.yaml | 21 +++++++++++++++++++++
 .travis.yml                  |  8 --------
 2 files changed, 21 insertions(+), 8 deletions(-)
 create mode 100644 .github/workflows/black.yaml

diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml
new file mode 100644
index 0000000..cabd0cc
--- /dev/null
+++ b/.github/workflows/black.yaml
@@ -0,0 +1,21 @@
+name: Black
+
+on: ["push", "pull_request"]
+
+jobs:
+  black:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.x'
+          architecture: 'x64'
+
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Black Code Formatter
+        run: |
+          pip install black
+          black --diff --check msgpack/ test/
diff --git a/.travis.yml b/.travis.yml
index 378bc80..39db1d7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,14 +23,6 @@ _pure: &pure
 
 matrix:
   include:
-    - name: Black
-      language: python
-      python: 3.7
-      install:
-        - pip install black
-      script:
-        - black --check --diff .
-
     - name: 32bit build
       language: python
       services:

From 0fc0eb2f16fcc7d0271792f93a90af389f66dafb Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 6 Dec 2019 21:26:28 +0900
Subject: [PATCH 1496/1648] Update README

---
 README.rst | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.rst b/README.rst
index f9f074f..c13267b 100644
--- a/README.rst
+++ b/README.rst
@@ -34,19 +34,19 @@ I upload transitional package (msgpack-python 0.5 which depending on msgpack)
 for smooth transition from msgpack-python to msgpack.
 
 Sadly, this doesn't work for upgrade install.  After `pip install -U msgpack-python`,
-msgpack is removed and `import msgpack` fail.
+msgpack is removed, and `import msgpack` fail.
 
 
-Compatibility with old format
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Compatibility with the old format
+^^^^^^^^^^^^^^^^^^^^^^----^^^^^^^
 
 You can use ``use_bin_type=False`` option to pack ``bytes``
-object into raw type in old msgpack spec, instead of bin type in new msgpack spec.
+object into raw type in the old msgpack spec, instead of bin type in new msgpack spec.
 
-You can unpack old msgpack formatk using ``raw=True`` option.
+You can unpack old msgpack format using ``raw=True`` option.
 It unpacks str (raw) type in msgpack into Python bytes.
 
-See note in below for detail.
+See note below for detail.
 
 
 Install
@@ -67,7 +67,7 @@ But msgpack provides a pure Python implementation (``msgpack.fallback``)
 for PyPy and Python 2.
 
 Since the [pip](https://pip.pypa.io/) uses the pure Python implementation,
-Python 2 support will not be dropped in foreseeable feature.
+Python 2 support will not be dropped in the foreseeable future.
 
 
 Windows

From d8e3cf0563989a660398318a7c788645124e1d8b Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 6 Dec 2019 22:23:15 +0900
Subject: [PATCH 1497/1648] Make strict_map_key default to True (#392)

---
 msgpack/_unpacker.pyx  | 8 +++-----
 msgpack/fallback.py    | 6 ++----
 test/test_case.py      | 2 +-
 test/test_format.py    | 2 +-
 test/test_limits.py    | 4 ++--
 test/test_obj.py       | 9 ++++++---
 test/test_pack.py      | 6 +++---
 test/test_sequnpack.py | 2 +-
 8 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index f10e99d..53ecf86 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -131,7 +131,7 @@ cdef inline int get_data_from_buffer(object obj,
 
 
 def unpackb(object packed, *, object object_hook=None, object list_hook=None,
-            bint use_list=True, bint raw=False, bint strict_map_key=False,
+            bint use_list=True, bint raw=False, bint strict_map_key=True,
             unicode_errors=None,
             object_pairs_hook=None, ext_hook=ExtType,
             Py_ssize_t max_str_len=-1,
@@ -221,9 +221,7 @@ cdef class Unpacker(object):
         Otherwise, unpack to Python str by decoding with UTF-8 encoding (default).
 
     :param bool strict_map_key:
-        If true, only str or bytes are accepted for map (dict) keys.
-        It's False by default for backward-compatibility.
-        But it will be True from msgpack 1.0.
+        If true (default), only str or bytes are accepted for map (dict) keys.
 
     :param callable object_hook:
         When specified, it should be callable.
@@ -305,7 +303,7 @@ cdef class Unpacker(object):
         self.buf = NULL
 
     def __init__(self, file_like=None, *, Py_ssize_t read_size=0,
-                 bint use_list=True, bint raw=False, bint strict_map_key=False,
+                 bint use_list=True, bint raw=False, bint strict_map_key=True,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  unicode_errors=None, Py_ssize_t max_buffer_size=0,
                  object ext_hook=ExtType,
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 85a711b..7df92f5 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -175,9 +175,7 @@ class Unpacker(object):
         Otherwise, unpack to Python str by decoding with UTF-8 encoding (default).
 
     :param bool strict_map_key:
-        If true, only str or bytes are accepted for map (dict) keys.
-        It's False by default for backward-compatibility.
-        But it will be True from msgpack 1.0.
+        If true (default), only str or bytes are accepted for map (dict) keys.
 
     :param callable object_hook:
         When specified, it should be callable.
@@ -249,7 +247,7 @@ class Unpacker(object):
         read_size=0,
         use_list=True,
         raw=False,
-        strict_map_key=False,
+        strict_map_key=True,
         object_hook=None,
         object_pairs_hook=None,
         list_hook=None,
diff --git a/test/test_case.py b/test/test_case.py
index 3e60e59..a0a3c5a 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -92,7 +92,7 @@ def test_array32():
 
 def match(obj, buf):
     assert packb(obj) == buf
-    assert unpackb(buf, use_list=0) == obj
+    assert unpackb(buf, use_list=0, strict_map_key=False) == obj
 
 
 def test_match():
diff --git a/test/test_format.py b/test/test_format.py
index 8c2f03f..d455f7c 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -5,7 +5,7 @@ from msgpack import unpackb
 
 
 def check(src, should, use_list=0, raw=True):
-    assert unpackb(src, use_list=use_list, raw=raw) == should
+    assert unpackb(src, use_list=use_list, raw=raw, strict_map_key=False) == should
 
 
 def testSimpleValue():
diff --git a/test/test_limits.py b/test/test_limits.py
index 6e85030..65e6bcc 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -87,11 +87,11 @@ def test_max_map_len():
     d = {1: 2, 3: 4, 5: 6}
     packed = packb(d)
 
-    unpacker = Unpacker(max_map_len=3)
+    unpacker = Unpacker(max_map_len=3, strict_map_key=False)
     unpacker.feed(packed)
     assert unpacker.unpack() == d
 
-    unpacker = Unpacker(max_map_len=2)
+    unpacker = Unpacker(max_map_len=2, strict_map_key=False)
     with pytest.raises(UnpackValueError):
         unpacker.feed(packed)
         unpacker.unpack()
diff --git a/test/test_obj.py b/test/test_obj.py
index 0b99cea..86c557c 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -33,7 +33,10 @@ def test_decode_pairs_hook():
     packed = packb([3, {1: 2, 3: 4}])
     prod_sum = 1 * 2 + 3 * 4
     unpacked = unpackb(
-        packed, object_pairs_hook=lambda l: sum(k * v for k, v in l), use_list=1
+        packed,
+        object_pairs_hook=lambda l: sum(k * v for k, v in l),
+        use_list=1,
+        strict_map_key=False,
     )
     assert unpacked[1] == prod_sum
 
@@ -70,10 +73,10 @@ def bad_complex_decoder(o):
 def test_an_exception_in_objecthook1():
     with raises(DecodeError):
         packed = packb({1: {"__complex__": True, "real": 1, "imag": 2}})
-        unpackb(packed, object_hook=bad_complex_decoder)
+        unpackb(packed, object_hook=bad_complex_decoder, strict_map_key=False)
 
 
 def test_an_exception_in_objecthook2():
     with raises(DecodeError):
         packed = packb({1: [{"__complex__": True, "real": 1, "imag": 2}]})
-        unpackb(packed, list_hook=bad_complex_decoder, use_list=1)
+        unpackb(packed, list_hook=bad_complex_decoder, use_list=1, strict_map_key=False)
diff --git a/test/test_pack.py b/test/test_pack.py
index de212ef..932f760 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -14,7 +14,7 @@ from msgpack import packb, unpackb, Unpacker, Packer, pack
 
 
 def check(data, use_list=False):
-    re = unpackb(packb(data), use_list=use_list)
+    re = unpackb(packb(data), use_list=use_list, strict_map_key=False)
     assert re == data
 
 
@@ -166,7 +166,7 @@ def testMapSize(sizes=[0, 5, 50, 1000]):
             bio.write(packer.pack(i * 2))  # value
 
     bio.seek(0)
-    unpacker = Unpacker(bio)
+    unpacker = Unpacker(bio, strict_map_key=False)
     for size in sizes:
         assert unpacker.unpack() == dict((i, i * 2) for i in range(size))
 
@@ -186,7 +186,7 @@ def test_pairlist():
     pairlist = [(b"a", 1), (2, b"b"), (b"foo", b"bar")]
     packer = Packer()
     packed = packer.pack_map_pairs(pairlist)
-    unpacked = unpackb(packed, object_pairs_hook=list)
+    unpacked = unpackb(packed, object_pairs_hook=list, strict_map_key=False)
     assert pairlist == unpacked
 
 
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index ad29de8..6293a45 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -132,7 +132,7 @@ def test_unpack_tell():
         pack(m, stream)
         offsets.append(stream.tell())
     stream.seek(0)
-    unpacker = Unpacker(stream)
+    unpacker = Unpacker(stream, strict_map_key=False)
     for m, o in zip(messages, offsets):
         m2 = next(unpacker)
         assert m == m2

From 5399f8180d23c147b1243d7c39aa19f9a8ba840a Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 9 Dec 2019 17:02:35 +0900
Subject: [PATCH 1498/1648] Update README (#393)

---
 README.rst          | 55 +++++++++++++++++++++++++++++++++++----------
 msgpack/fallback.py |  4 ++--
 2 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/README.rst b/README.rst
index c13267b..d01e963 100644
--- a/README.rst
+++ b/README.rst
@@ -38,7 +38,7 @@ msgpack is removed, and `import msgpack` fail.
 
 
 Compatibility with the old format
-^^^^^^^^^^^^^^^^^^^^^^----^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 You can use ``use_bin_type=False`` option to pack ``bytes``
 object into raw type in the old msgpack spec, instead of bin type in new msgpack spec.
@@ -49,6 +49,32 @@ It unpacks str (raw) type in msgpack into Python bytes.
 See note below for detail.
 
 
+Major breaking changes in msgpack 1.0
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* Python 2
+
+  * The extension module does not support Python 2 anymore.
+    The pure Python implementation (``msgpack.fallback``) is used for Python 2.
+
+* Packer
+
+  * ``use_bin_type=True`` by default.  bytes are encoded in bin type in msgpack.
+    **If you are still sing Python 2, you must use unicode for all string types.**
+    You can use ``use_bin_type=False`` to encode into old msgpack format.
+  * ``encoding`` option is removed.  UTF-8 is used always.
+
+* Unpacker
+
+  * ``raw=False`` by default.  It assumes str types are valid UTF-8 string
+    and decode them to Python str (unicode) object.
+  * ``encdoding`` option is rmeoved.  You can use ``raw=True`` to support old format.
+  * Default value of ``max_buffer_size`` is changed from 0 to 100 MiB.
+  * Default value of ``strict_map_key`` is changed to True to avoid hashdos.
+    You need to pass ``strict_map_key=False`` if you have data which contain map keys
+    which type is not bytes or str.
+
+
 Install
 -------
 
@@ -270,27 +296,32 @@ To use the **ext** type, pass ``msgpack.ExtType`` object to packer.
 You can use it with ``default`` and ``ext_hook``. See below.
 
 
-Note about performance
-----------------------
+Security
+^^^^^^^^
 
-GC
-^^
+To unpacking data received from unreliable source, msgpack provides
+two security options.
+
+``max_buffer_size`` (default: 100*1024*1024) limits the internal buffer size.
+It is used to limit the preallocated list size too.
+
+``strict_map_key`` (default: ``True``) limits the type of map keys to bytes and str.
+While msgpack spec doesn't limit the types of the map keys,
+there is a risk of the hashdos.
+If you need to support other types for map keys, use ``strict_map_key=False``.
+
+
+Performance tips
+^^^^^^^^^^^^^^^^
 
 CPython's GC starts when growing allocated object.
 This means unpacking may cause useless GC.
 You can use ``gc.disable()`` when unpacking large message.
 
-use_list option
-^^^^^^^^^^^^^^^
-
 List is the default sequence type of Python.
 But tuple is lighter than list.
 You can use ``use_list=False`` while unpacking when performance is important.
 
-Python's dict can't use list as key and MessagePack allows array for key of mapping.
-``use_list=False`` allows unpacking such message.
-Another way to unpacking such object is using ``object_pairs_hook``.
-
 
 Development
 -----------
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 7df92f5..3704f9d 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -747,7 +747,7 @@ class Packer(object):
 
     :param bool strict_types:
         If set to true, types will be checked to be exact. Derived classes
-        from serializeable types will not be serialized and will be
+        from serializable types will not be serialized and will be
         treated as unsupported type and forwarded to default.
         Additionally tuples will not be serialized as lists.
         This is useful when trying to implement accurate serialization
@@ -1014,7 +1014,7 @@ class Packer(object):
     def reset(self):
         """Reset internal buffer.
 
-        This method is usaful only when autoreset=False.
+        This method is useful only when autoreset=False.
         """
         self._buffer = StringIO()
 

From c356035a576c38db5ca232ede07b291087f1b8b2 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 9 Dec 2019 17:03:12 +0900
Subject: [PATCH 1499/1648] Unpacker: Change max_buffer_size to 100MiB (#391)

---
 msgpack/_unpacker.pyx | 42 ++++++++++++++++++++++--------------------
 msgpack/fallback.py   | 33 ++++++++++++++++++---------------
 2 files changed, 40 insertions(+), 35 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 53ecf86..0ff633b 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -234,27 +234,28 @@ cdef class Unpacker(object):
         (See also simplejson)
 
     :param int max_buffer_size:
-        Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
+        Limits size of data waiting unpacked.  0 means system's INT_MAX.
+        The default value is 100*1024*1024 (100MiB).
         Raises `BufferFull` exception when it is insufficient.
         You should set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
         Deprecated, use *max_buffer_size* instead.
-        Limits max length of str. (default: max_buffer_size or 1024*1024)
+        Limits max length of str. (default: max_buffer_size)
 
     :param int max_bin_len:
         Deprecated, use *max_buffer_size* instead.
-        Limits max length of bin. (default: max_buffer_size or 1024*1024)
+        Limits max length of bin. (default: max_buffer_size)
 
     :param int max_array_len:
-        Limits max length of array. (default: max_buffer_size or 128*1024)
+        Limits max length of array. (default: max_buffer_size)
 
     :param int max_map_len:
-        Limits max length of map. (default: max_buffer_size//2 or 32*1024)
+        Limits max length of map. (default: max_buffer_size//2)
 
     :param int max_ext_len:
         Deprecated, use *max_buffer_size* instead.
-        Limits max size of ext type. (default: max_buffer_size or 1024*1024)
+        Limits max size of ext type. (default: max_buffer_size)
 
     :param str unicode_errors:
         Error handler used for decoding str type.  (default: `'strict'`)
@@ -262,13 +263,13 @@ cdef class Unpacker(object):
 
     Example of streaming deserialize from file-like object::
 
-        unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024)
+        unpacker = Unpacker(file_like)
         for o in unpacker:
             process(o)
 
     Example of streaming deserialize from socket::
 
-        unpacker = Unpacker(max_buffer_size=10*1024*1024)
+        unpacker = Unpacker()
         while True:
             buf = sock.recv(1024**2)
             if not buf:
@@ -305,7 +306,7 @@ cdef class Unpacker(object):
     def __init__(self, file_like=None, *, Py_ssize_t read_size=0,
                  bint use_list=True, bint raw=False, bint strict_map_key=True,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
-                 unicode_errors=None, Py_ssize_t max_buffer_size=0,
+                 unicode_errors=None, Py_ssize_t max_buffer_size=100*1024*1024,
                  object ext_hook=ExtType,
                  Py_ssize_t max_str_len=-1,
                  Py_ssize_t max_bin_len=-1,
@@ -325,23 +326,24 @@ cdef class Unpacker(object):
             if not PyCallable_Check(self.file_like_read):
                 raise TypeError("`file_like.read` must be a callable.")
 
-        if max_str_len == -1:
-            max_str_len = max_buffer_size or 1024*1024
-        if max_bin_len == -1:
-            max_bin_len = max_buffer_size or 1024*1024
-        if max_array_len == -1:
-            max_array_len = max_buffer_size or 128*1024
-        if max_map_len == -1:
-            max_map_len = max_buffer_size//2 or 32*1024
-        if max_ext_len == -1:
-            max_ext_len = max_buffer_size or 1024*1024
-
         if not max_buffer_size:
             max_buffer_size = INT_MAX
+        if max_str_len == -1:
+            max_str_len = max_buffer_size
+        if max_bin_len == -1:
+            max_bin_len = max_buffer_size
+        if max_array_len == -1:
+            max_array_len = max_buffer_size
+        if max_map_len == -1:
+            max_map_len = max_buffer_size//2
+        if max_ext_len == -1:
+            max_ext_len = max_buffer_size
+
         if read_size > max_buffer_size:
             raise ValueError("read_size should be less or equal to max_buffer_size")
         if not read_size:
             read_size = min(max_buffer_size, 1024**2)
+
         self.max_buffer_size = max_buffer_size
         self.read_size = read_size
         self.buf = PyMem_Malloc(read_size)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 3704f9d..f6ba424 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -193,39 +193,40 @@ class Unpacker(object):
         contains invalid UTF-8 string.
 
     :param int max_buffer_size:
-        Limits size of data waiting unpacked.  0 means system's INT_MAX (default).
+        Limits size of data waiting unpacked.  0 means 2**32-1.
+        The default value is 100*1024*1024 (100MiB).
         Raises `BufferFull` exception when it is insufficient.
         You should set this parameter when unpacking data from untrusted source.
 
     :param int max_str_len:
         Deprecated, use *max_buffer_size* instead.
-        Limits max length of str. (default: max_buffer_size or 1024*1024)
+        Limits max length of str. (default: max_buffer_size)
 
     :param int max_bin_len:
         Deprecated, use *max_buffer_size* instead.
-        Limits max length of bin. (default: max_buffer_size or 1024*1024)
+        Limits max length of bin. (default: max_buffer_size)
 
     :param int max_array_len:
         Limits max length of array.
-        (default: max_buffer_size or 128*1024)
+        (default: max_buffer_size)
 
     :param int max_map_len:
         Limits max length of map.
-        (default: max_buffer_size//2 or 32*1024)
+        (default: max_buffer_size//2)
 
     :param int max_ext_len:
         Deprecated, use *max_buffer_size* instead.
-        Limits max size of ext type.  (default: max_buffer_size or 1024*1024)
+        Limits max size of ext type.  (default: max_buffer_size)
 
     Example of streaming deserialize from file-like object::
 
-        unpacker = Unpacker(file_like, max_buffer_size=10*1024*1024)
+        unpacker = Unpacker(file_like)
         for o in unpacker:
             process(o)
 
     Example of streaming deserialize from socket::
 
-        unpacker = Unpacker(max_buffer_size=10*1024*1024)
+        unpacker = Unpacker(max_buffer_size)
         while True:
             buf = sock.recv(1024**2)
             if not buf:
@@ -252,7 +253,7 @@ class Unpacker(object):
         object_pairs_hook=None,
         list_hook=None,
         unicode_errors=None,
-        max_buffer_size=0,
+        max_buffer_size=100 * 1024 * 1024,
         ext_hook=ExtType,
         max_str_len=-1,
         max_bin_len=-1,
@@ -285,18 +286,20 @@ class Unpacker(object):
         # state, which _buf_checkpoint records.
         self._buf_checkpoint = 0
 
+        if not max_buffer_size:
+            max_buffer_size = 2 ** 31 - 1
         if max_str_len == -1:
-            max_str_len = max_buffer_size or 1024 * 1024
+            max_str_len = max_buffer_size
         if max_bin_len == -1:
-            max_bin_len = max_buffer_size or 1024 * 1024
+            max_bin_len = max_buffer_size
         if max_array_len == -1:
-            max_array_len = max_buffer_size or 128 * 1024
+            max_array_len = max_buffer_size
         if max_map_len == -1:
-            max_map_len = max_buffer_size // 2 or 32 * 1024
+            max_map_len = max_buffer_size // 2
         if max_ext_len == -1:
-            max_ext_len = max_buffer_size or 1024 * 1024
+            max_ext_len = max_buffer_size
 
-        self._max_buffer_size = max_buffer_size or 2 ** 31 - 1
+        self._max_buffer_size = max_buffer_size
         if read_size > self._max_buffer_size:
             raise ValueError("read_size must be smaller than max_buffer_size")
         self._read_size = read_size or min(self._max_buffer_size, 16 * 1024)

From d10f12db8f328130a13df759bc9cb3fa064cc8b8 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 9 Dec 2019 18:12:51 +0900
Subject: [PATCH 1500/1648] typo

---
 msgpack/_packer.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 8cf3c05..2a768b0 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -341,7 +341,7 @@ cdef class Packer(object):
     def reset(self):
         """Reset internal buffer.
 
-        This method is usaful only when autoreset=False.
+        This method is useful only when autoreset=False.
         """
         self.pk.length = 0
 

From 5fd611909319d03200774ea3c7a6ae16dbd26c12 Mon Sep 17 00:00:00 2001
From: Marty B 
Date: Mon, 9 Dec 2019 11:29:47 +0100
Subject: [PATCH 1501/1648] Simplify check for bool type (#362)

---
 msgpack/_packer.pyx | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 2a768b0..1426439 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -153,11 +153,10 @@ cdef class Packer(object):
         while True:
             if o is None:
                 ret = msgpack_pack_nil(&self.pk)
-            elif PyBool_Check(o) if strict_types else isinstance(o, bool):
-                if o:
-                    ret = msgpack_pack_true(&self.pk)
-                else:
-                    ret = msgpack_pack_false(&self.pk)
+            elif o is True:
+                ret = msgpack_pack_true(&self.pk)
+            elif o is False:
+                ret = msgpack_pack_false(&self.pk)
             elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o):
                 # PyInt_Check(long) is True for Python 3.
                 # So we should test long before int.

From 2186455d1579affc33253484d9445f7bdf3f7c29 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 11 Dec 2019 23:48:16 +0900
Subject: [PATCH 1502/1648] Support datetime. (#394)

---
 Makefile               |  2 +-
 msgpack/_cmsgpack.pyx  |  7 ++++
 msgpack/_packer.pyx    | 19 +++++++++--
 msgpack/_unpacker.pyx  | 23 ++++++++++----
 msgpack/ext.py         | 27 +++++++++++++++-
 msgpack/fallback.py    | 48 +++++++++++++++++++++++++---
 msgpack/unpack.h       | 72 ++++++++++++++++++++++++++++++++++++++----
 test/test_timestamp.py | 48 ++++++++++++++++++++++++++--
 8 files changed, 222 insertions(+), 24 deletions(-)

diff --git a/Makefile b/Makefile
index f8971cc..e2f25cf 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ all: cython
 
 .PHONY: black
 black:
-	black msgpack/ test/
+	black msgpack/ test/ setup.py
 
 .PHONY: cython
 cython:
diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx
index 8ebdbf5..1faaac3 100644
--- a/msgpack/_cmsgpack.pyx
+++ b/msgpack/_cmsgpack.pyx
@@ -1,4 +1,11 @@
 # coding: utf-8
 #cython: embedsignature=True, c_string_encoding=ascii, language_level=3
+from cpython.datetime cimport import_datetime, datetime_new
+import_datetime()
+
+import datetime
+cdef object utc = datetime.timezone.utc
+cdef object epoch = datetime_new(1970, 1, 1, 0, 0, 0, 0, tz=utc)
+
 include "_packer.pyx"
 include "_unpacker.pyx"
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 1426439..b470646 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -2,6 +2,10 @@
 
 from cpython cimport *
 from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact
+from cpython.datetime cimport (
+    PyDateTime_CheckExact, PyDelta_CheckExact,
+    datetime_tzinfo, timedelta_days, timedelta_seconds, timedelta_microseconds,
+)
 
 cdef ExtType
 cdef Timestamp
@@ -99,8 +103,9 @@ cdef class Packer(object):
     cdef object _berrors
     cdef const char *unicode_errors
     cdef bint strict_types
-    cdef bool use_float
+    cdef bint use_float
     cdef bint autoreset
+    cdef bint datetime
 
     def __cinit__(self):
         cdef int buf_size = 1024*1024
@@ -110,12 +115,13 @@ cdef class Packer(object):
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
-    def __init__(self, *, default=None, unicode_errors=None,
+    def __init__(self, *, default=None,
                  bint use_single_float=False, bint autoreset=True, bint use_bin_type=True,
-                 bint strict_types=False):
+                 bint strict_types=False, bint datetime=False, unicode_errors=None):
         self.use_float = use_single_float
         self.strict_types = strict_types
         self.autoreset = autoreset
+        self.datetime = datetime
         self.pk.use_bin_type = use_bin_type
         if default is not None:
             if not PyCallable_Check(default):
@@ -262,6 +268,13 @@ cdef class Packer(object):
                 if ret == 0:
                     ret = msgpack_pack_raw_body(&self.pk, view.buf, L)
                 PyBuffer_Release(&view);
+            elif self.datetime and PyDateTime_CheckExact(o) and datetime_tzinfo(o) is not None:
+                delta = o - epoch
+                if not PyDelta_CheckExact(delta):
+                    raise ValueError("failed to calculate delta")
+                llval = timedelta_days(delta) * (24*60*60) + timedelta_seconds(delta)
+                ulval = timedelta_microseconds(delta) * 1000
+                ret = msgpack_pack_timestamp(&self.pk, llval, ulval)
             elif not default_used and self._default:
                 o = self._default(o)
                 default_used = 1
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 0ff633b..43c93a2 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -1,7 +1,6 @@
 # coding: utf-8
 
 from cpython cimport *
-
 cdef extern from "Python.h":
     ctypedef struct PyObject
     cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1
@@ -21,6 +20,8 @@ from .exceptions import (
 )
 from .ext import ExtType, Timestamp
 
+cdef object giga = 1_000_000_000
+
 
 cdef extern from "unpack.h":
     ctypedef struct msgpack_user:
@@ -28,10 +29,13 @@ cdef extern from "unpack.h":
         bint raw
         bint has_pairs_hook # call object_hook with k-v pairs
         bint strict_map_key
+        int timestamp
         PyObject* object_hook
         PyObject* list_hook
         PyObject* ext_hook
         PyObject* timestamp_t
+        PyObject *giga;
+        PyObject *utc;
         char *unicode_errors
         Py_ssize_t max_str_len
         Py_ssize_t max_bin_len
@@ -57,7 +61,8 @@ cdef extern from "unpack.h":
 cdef inline init_ctx(unpack_context *ctx,
                      object object_hook, object object_pairs_hook,
                      object list_hook, object ext_hook,
-                     bint use_list, bint raw, bint strict_map_key,
+                     bint use_list, bint raw, int timestamp,
+                     bint strict_map_key,
                      const char* unicode_errors,
                      Py_ssize_t max_str_len, Py_ssize_t max_bin_len,
                      Py_ssize_t max_array_len, Py_ssize_t max_map_len,
@@ -99,8 +104,14 @@ cdef inline init_ctx(unpack_context *ctx,
             raise TypeError("ext_hook must be a callable.")
         ctx.user.ext_hook = ext_hook
 
+    if timestamp < 0 or 3 < timestamp:
+        raise ValueError("timestamp must be 0..3")
+
     # Add Timestamp type to the user object so it may be used in unpack.h
+    ctx.user.timestamp = timestamp
     ctx.user.timestamp_t = Timestamp
+    ctx.user.giga = giga
+    ctx.user.utc = utc
     ctx.user.unicode_errors = unicode_errors
 
 def default_read_extended_type(typecode, data):
@@ -131,7 +142,7 @@ cdef inline int get_data_from_buffer(object obj,
 
 
 def unpackb(object packed, *, object object_hook=None, object list_hook=None,
-            bint use_list=True, bint raw=False, bint strict_map_key=True,
+            bint use_list=True, bint raw=False, int timestamp=0, bint strict_map_key=True,
             unicode_errors=None,
             object_pairs_hook=None, ext_hook=ExtType,
             Py_ssize_t max_str_len=-1,
@@ -179,7 +190,7 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None,
 
     try:
         init_ctx(&ctx, object_hook, object_pairs_hook, list_hook, ext_hook,
-                 use_list, raw, strict_map_key, cerr,
+                 use_list, raw, timestamp, strict_map_key, cerr,
                  max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len)
         ret = unpack_construct(&ctx, buf, buf_len, &off)
     finally:
@@ -304,7 +315,7 @@ cdef class Unpacker(object):
         self.buf = NULL
 
     def __init__(self, file_like=None, *, Py_ssize_t read_size=0,
-                 bint use_list=True, bint raw=False, bint strict_map_key=True,
+                 bint use_list=True, bint raw=False, int timestamp=0, bint strict_map_key=True,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
                  unicode_errors=None, Py_ssize_t max_buffer_size=100*1024*1024,
                  object ext_hook=ExtType,
@@ -359,7 +370,7 @@ cdef class Unpacker(object):
             cerr = unicode_errors
 
         init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook,
-                 ext_hook, use_list, raw, strict_map_key, cerr,
+                 ext_hook, use_list, raw, timestamp, strict_map_key, cerr,
                  max_str_len, max_bin_len, max_array_len,
                  max_map_len, max_ext_len)
 
diff --git a/msgpack/ext.py b/msgpack/ext.py
index c7efff6..09adb34 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -1,12 +1,18 @@
 # coding: utf-8
 from collections import namedtuple
+import datetime
 import sys
 import struct
 
 
 PY2 = sys.version_info[0] == 2
+
 if not PY2:
     long = int
+    try:
+        _utc = datetime.timezone.utc
+    except AttributeError:
+        _utc = datetime.timezone(datetime.timedelta(0))
 
 
 class ExtType(namedtuple("ExtType", "code data")):
@@ -131,7 +137,7 @@ class Timestamp(object):
             data = struct.pack("!Iq", self.nanoseconds, self.seconds)
         return data
 
-    def to_float_s(self):
+    def to_float(self):
         """Get the timestamp as a floating-point value.
 
         :returns: posix timestamp
@@ -139,6 +145,12 @@ class Timestamp(object):
         """
         return self.seconds + self.nanoseconds / 1e9
 
+    @staticmethod
+    def from_float(unix_float):
+        seconds = int(unix_float)
+        nanoseconds = int((unix_float % 1) * 1000000000)
+        return Timestamp(seconds, nanoseconds)
+
     def to_unix_ns(self):
         """Get the timestamp as a unixtime in nanoseconds.
 
@@ -146,3 +158,16 @@ class Timestamp(object):
         :rtype: int
         """
         return int(self.seconds * 1e9 + self.nanoseconds)
+
+    if not PY2:
+
+        def to_datetime(self):
+            """Get the timestamp as a UTC datetime.
+
+            :rtype: datetime.
+            """
+            return datetime.datetime.fromtimestamp(self.to_float(), _utc)
+
+        @staticmethod
+        def from_datetime(dt):
+            return Timestamp.from_float(dt.timestamp())
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index f6ba424..9ba98bf 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -1,5 +1,6 @@
 """Fallback pure Python implementation of msgpack"""
 
+from datetime import datetime as _DateTime
 import sys
 import struct
 
@@ -174,6 +175,14 @@ class Unpacker(object):
         If true, unpack msgpack raw to Python bytes.
         Otherwise, unpack to Python str by decoding with UTF-8 encoding (default).
 
+    :param int timestamp:
+        Control how timestamp type is unpacked:
+
+            0 - Tiemstamp
+            1 - float  (Seconds from the EPOCH)
+            2 - int  (Nanoseconds from the EPOCH)
+            3 - datetime.datetime  (UTC).  Python 2 is not supported.
+
     :param bool strict_map_key:
         If true (default), only str or bytes are accepted for map (dict) keys.
 
@@ -248,6 +257,7 @@ class Unpacker(object):
         read_size=0,
         use_list=True,
         raw=False,
+        timestamp=0,
         strict_map_key=True,
         object_hook=None,
         object_pairs_hook=None,
@@ -307,6 +317,9 @@ class Unpacker(object):
         self._strict_map_key = bool(strict_map_key)
         self._unicode_errors = unicode_errors
         self._use_list = use_list
+        if not (0 <= timestamp <= 3):
+            raise ValueError("timestamp must be 0..3")
+        self._timestamp = timestamp
         self._list_hook = list_hook
         self._object_hook = object_hook
         self._object_pairs_hook = object_pairs_hook
@@ -672,10 +685,21 @@ class Unpacker(object):
             else:
                 obj = obj.decode("utf_8", self._unicode_errors)
             return obj
-        if typ == TYPE_EXT:
-            return self._ext_hook(n, bytes(obj))
         if typ == TYPE_BIN:
             return bytes(obj)
+        if typ == TYPE_EXT:
+            if n == -1:  # timestamp
+                ts = Timestamp.from_bytes(bytes(obj))
+                if self._timestamp == 1:
+                    return ts.to_float()
+                elif self._timestamp == 2:
+                    return ts.to_unix_ns()
+                elif self._timestamp == 3:
+                    return ts.to_datetime()
+                else:
+                    return ts
+            else:
+                return self._ext_hook(n, bytes(obj))
         assert typ == TYPE_IMMEDIATE
         return obj
 
@@ -756,6 +780,12 @@ class Packer(object):
         This is useful when trying to implement accurate serialization
         for python types.
 
+    :param bool datetime:
+        If set to true, datetime with tzinfo is packed into Timestamp type.
+        Note that the tzinfo is stripped in the timestamp.
+        You can get UTC datetime with `timestamp=3` option of the Unapcker.
+        (Python 2 is not supported).
+
     :param str unicode_errors:
         The error handler for encoding unicode. (default: 'strict')
         DO NOT USE THIS!!  This option is kept for very specific usage.
@@ -764,18 +794,22 @@ class Packer(object):
     def __init__(
         self,
         default=None,
-        unicode_errors=None,
         use_single_float=False,
         autoreset=True,
         use_bin_type=True,
         strict_types=False,
+        datetime=False,
+        unicode_errors=None,
     ):
         self._strict_types = strict_types
         self._use_float = use_single_float
         self._autoreset = autoreset
         self._use_bin_type = use_bin_type
-        self._unicode_errors = unicode_errors or "strict"
         self._buffer = StringIO()
+        if PY2 and datetime:
+            raise ValueError("datetime is not supported in Python 2")
+        self._datetime = bool(datetime)
+        self._unicode_errors = unicode_errors or "strict"
         if default is not None:
             if not callable(default):
                 raise TypeError("default must be callable")
@@ -891,6 +925,12 @@ class Packer(object):
                 return self._pack_map_pairs(
                     len(obj), dict_iteritems(obj), nest_limit - 1
                 )
+
+            if self._datetime and check(obj, _DateTime):
+                obj = Timestamp.from_datetime(obj)
+                default_used = 1
+                continue
+
             if not default_used and self._default is not None:
                 obj = self._default(obj)
                 default_used = 1
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 4380ec5..debdf71 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -24,10 +24,13 @@ typedef struct unpack_user {
     bool raw;
     bool has_pairs_hook;
     bool strict_map_key;
+    int timestamp;
     PyObject *object_hook;
     PyObject *list_hook;
     PyObject *ext_hook;
     PyObject *timestamp_t;
+    PyObject *giga;
+    PyObject *utc;
     const char *unicode_errors;
     Py_ssize_t max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len;
 } unpack_user;
@@ -268,7 +271,7 @@ typedef struct msgpack_timestamp {
 /*
  * Unpack ext buffer to a timestamp. Pulled from msgpack-c timestamp.h.
  */
-static inline int unpack_timestamp(const char* buf, unsigned int buflen, msgpack_timestamp* ts) {
+static int unpack_timestamp(const char* buf, unsigned int buflen, msgpack_timestamp* ts) {
     switch (buflen) {
     case 4:
         ts->tv_nsec = 0;
@@ -292,10 +295,11 @@ static inline int unpack_timestamp(const char* buf, unsigned int buflen, msgpack
     }
 }
 
-static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
-                                      unsigned int length, msgpack_unpack_object* o)
+#include "datetime.h"
+
+static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
+                               unsigned int length, msgpack_unpack_object* o)
 {
-    PyObject *py;
     int8_t typecode = (int8_t)*pos++;
     if (!u->ext_hook) {
         PyErr_SetString(PyExc_AssertionError, "u->ext_hook cannot be NULL");
@@ -305,13 +309,67 @@ static inline int unpack_callback_ext(unpack_user* u, const char* base, const ch
         PyErr_Format(PyExc_ValueError, "%u exceeds max_ext_len(%zd)", length, u->max_ext_len);
         return -1;
     }
+
+    PyObject *py = NULL;
     // length also includes the typecode, so the actual data is length-1
     if (typecode == -1) {
         msgpack_timestamp ts;
-        if (unpack_timestamp(pos, length-1, &ts) == 0) {
+        if (unpack_timestamp(pos, length-1, &ts) < 0) {
+            return -1;
+        }
+
+        if (u->timestamp == 2) {  // int
+            PyObject *a = PyLong_FromLongLong(ts.tv_sec);
+            if (a == NULL) return -1;
+
+            PyObject *c = PyNumber_Multiply(a, u->giga);
+            Py_DECREF(a);
+            if (c == NULL) {
+                return -1;
+            }
+
+            PyObject *b = PyLong_FromUnsignedLong(ts.tv_nsec);
+            if (b == NULL) {
+                Py_DECREF(c);
+                return -1;
+            }
+
+            py = PyNumber_Add(c, b);
+            Py_DECREF(c);
+            Py_DECREF(b);
+        }
+        else if (u->timestamp == 0) {  // Timestamp
             py = PyObject_CallFunction(u->timestamp_t, "(Lk)", ts.tv_sec, ts.tv_nsec);
-        } else {
-            py = NULL;
+        }
+        else { // float or datetime
+            PyObject *a = PyFloat_FromDouble((double)ts.tv_nsec);
+            if (a == NULL) return -1;
+
+            PyObject *b = PyNumber_TrueDivide(a, u->giga);
+            Py_DECREF(a);
+            if (b == NULL) return -1;
+
+            PyObject *c = PyLong_FromLongLong(ts.tv_sec);
+            if (c == NULL) {
+                Py_DECREF(b);
+                return -1;
+            }
+
+            a = PyNumber_Add(b, c);
+            Py_DECREF(b);
+            Py_DECREF(c);
+
+            if (u->timestamp == 3) {  // datetime
+                PyObject *t = PyTuple_Pack(2, a, u->utc);
+                Py_DECREF(a);
+                if (t == NULL) {
+                    return -1;
+                }
+                py = PyDateTime_FromTimestamp(t);
+                Py_DECREF(t);
+            } else { // float
+                py = a;
+            }
         }
     } else {
         py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1);
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index 1348e69..822994c 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -1,5 +1,11 @@
+import pytest
+import sys
+import datetime
 import msgpack
-from msgpack import Timestamp
+from msgpack.ext import Timestamp
+
+if sys.version_info[0] > 2:
+    from msgpack.ext import _utc
 
 
 def test_timestamp():
@@ -42,5 +48,43 @@ def test_timestamp():
 
 def test_timestamp_to():
     t = Timestamp(42, 14)
-    assert t.to_float_s() == 42.000000014
+    assert t.to_float() == 42.000000014
     assert t.to_unix_ns() == 42000000014
+
+
+@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
+def test_timestamp_datetime():
+    t = Timestamp(42, 14)
+    assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc)
+
+
+@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
+def test_unpack_datetime():
+    t = Timestamp(42, 14)
+    packed = msgpack.packb(t)
+    unpacked = msgpack.unpackb(packed, timestamp=3)
+    assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc)
+
+
+@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
+def test_pack_datetime():
+    t = Timestamp(42, 14000)
+    dt = t.to_datetime()
+    assert dt == datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc)
+
+    packed = msgpack.packb(dt, datetime=True)
+    packed2 = msgpack.packb(t)
+    assert packed == packed2
+
+    unpacked = msgpack.unpackb(packed)
+    print(packed, unpacked)
+    assert unpacked == t
+
+    unpacked = msgpack.unpackb(packed, timestamp=3)
+    assert unpacked == dt
+
+    x = []
+    packed = msgpack.packb(dt, datetime=False, default=x.append)
+    assert x
+    assert x[0] == dt
+    assert msgpack.unpackb(packed) is None

From c60e6c7a6ff1815083bf6803ec70f3ac34aaf3bb Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 12 Dec 2019 18:09:07 +0900
Subject: [PATCH 1503/1648] Update README

---
 ChangeLog.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index d44b36a..c70b966 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -6,6 +6,14 @@ Release Date: TBD
 * Remove Python 2 support from the ``msgpack/_cmsgpack``.
   ``msgpack/fallback`` still supports Python 2.
 * Remove ``encoding`` option from the Packer and Unpacker.
+* Unpacker: The default value of ``max_buffer_type`` is changed to 100MiB.
+* Unpacker: ``strict_map_key`` is True by default now.
+* Unpacker: String map keys are interned.
+* Drop old buffer protocol support.
+* Support Timestamp type.
+* Support serializing and decerializing ``datetime`` object
+  with tzinfo.
+* Unpacker: ``Fix Unpacker.read_bytes()`` in fallback implementation. (#352)
 
 
 0.6.2

From 3df431cafd82354e61b39afd6094003e9c313c43 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 12 Dec 2019 18:25:38 +0900
Subject: [PATCH 1504/1648] Prepare 1.0rc1

---
 MANIFEST.in             |   2 +-
 README.rst => README.md | 125 ++++++++++++++--------------------------
 msgpack/_version.py     |   2 +-
 setup.py                |   6 +-
 4 files changed, 49 insertions(+), 86 deletions(-)
 rename README.rst => README.md (83%)

diff --git a/MANIFEST.in b/MANIFEST.in
index e1912ca..57d84a4 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,5 @@
 include setup.py
 include COPYING
-include README.rst
+include README.md
 recursive-include msgpack *.h *.c *.pyx *.cpp
 recursive-include test *.py
diff --git a/README.rst b/README.md
similarity index 83%
rename from README.rst
rename to README.md
index d01e963..897a932 100644
--- a/README.rst
+++ b/README.md
@@ -1,18 +1,9 @@
-======================
-MessagePack for Python
-======================
+# MessagePack for Python
 
-.. image:: https://travis-ci.org/msgpack/msgpack-python.svg?branch=master
-   :target: https://travis-ci.org/msgpack/msgpack-python
-   :alt: Build Status
+[![Build Status](https://travis-ci.org/msgpack/msgpack-python.svg?branch=master)](https://travis-ci.org/msgpack/msgpack-python)
+[![Documentation Status](https://readthedocs.org/projects/msgpack-python/badge/?version=latest)](https://msgpack-python.readthedocs.io/en/latest/?badge=latest)
 
-.. image:: https://readthedocs.org/projects/msgpack-python/badge/?version=latest
-   :target: https://msgpack-python.readthedocs.io/en/latest/?badge=latest
-   :alt: Documentation Status
-
-
-What's this
------------
+## What's this
 
 `MessagePack `_ is an efficient binary serialization format.
 It lets you exchange data among multiple languages like JSON.
@@ -20,11 +11,9 @@ But it's faster and smaller.
 This package provides CPython bindings for reading and writing MessagePack data.
 
 
-Very important notes for existing users
----------------------------------------
+## Very important notes for existing users
 
-PyPI package name
-^^^^^^^^^^^^^^^^^
+### PyPI package name
 
 TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`.
 Do `pip uninstall msgpack-python; pip install msgpack` instead.
@@ -37,8 +26,7 @@ Sadly, this doesn't work for upgrade install.  After `pip install -U msgpack-pyt
 msgpack is removed, and `import msgpack` fail.
 
 
-Compatibility with the old format
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+### Compatibility with the old format
 
 You can use ``use_bin_type=False`` option to pack ``bytes``
 object into raw type in the old msgpack spec, instead of bin type in new msgpack spec.
@@ -49,8 +37,7 @@ It unpacks str (raw) type in msgpack into Python bytes.
 See note below for detail.
 
 
-Major breaking changes in msgpack 1.0
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+### Major breaking changes in msgpack 1.0
 
 * Python 2
 
@@ -75,16 +62,13 @@ Major breaking changes in msgpack 1.0
     which type is not bytes or str.
 
 
-Install
--------
+## Install
 
-::
 
    $ pip install msgpack
 
 
-Pure Python implementation
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+### Pure Python implementation
 
 The extension module in msgpack (``msgpack._cmsgpack``) does not support
 Python 2 and PyPy.
@@ -96,26 +80,20 @@ Since the [pip](https://pip.pypa.io/) uses the pure Python implementation,
 Python 2 support will not be dropped in the foreseeable future.
 
 
-Windows
-^^^^^^^
+### Windows
 
 When you can't use a binary distribution, you need to install Visual Studio
 or Windows SDK on Windows.
 Without extension, using pure Python implementation on CPython runs slowly.
 
 
-How to use
-----------
+## How to use
 
-.. note::
-
-   In examples below, I use ``raw=False`` and ``use_bin_type=True`` for users
-   using msgpack < 1.0.
-   These options are default from msgpack 1.0 so you can omit them.
+NOTE: In examples below, I use ``raw=False`` and ``use_bin_type=True`` for users
+using msgpack < 1.0. These options are default from msgpack 1.0 so you can omit them.
 
 
-One-shot pack & unpack
-^^^^^^^^^^^^^^^^^^^^^^
+### One-shot pack & unpack
 
 Use ``packb`` for packing and ``unpackb`` for unpacking.
 msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with
@@ -124,20 +102,20 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with
 ``pack`` and ``dump`` packs to a file-like object.
 ``unpack`` and ``load`` unpacks from a file-like object.
 
-.. code-block:: pycon
-
+```pycon
    >>> import msgpack
    >>> msgpack.packb([1, 2, 3], use_bin_type=True)
    '\x93\x01\x02\x03'
    >>> msgpack.unpackb(_, raw=False)
    [1, 2, 3]
+```
 
 ``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple:
 
-.. code-block:: pycon
-
+```pycon
    >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False)
    (1, 2, 3)
+```
 
 You should always specify the ``use_list`` keyword argument for backward compatibility.
 See performance issues relating to `use_list option`_ below.
@@ -145,14 +123,12 @@ See performance issues relating to `use_list option`_ below.
 Read the docstring for other options.
 
 
-Streaming unpacking
-^^^^^^^^^^^^^^^^^^^
+### Streaming unpacking
 
 ``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one
 stream (or from bytes provided through its ``feed`` method).
 
-.. code-block:: python
-
+```py
    import msgpack
    from io import BytesIO
 
@@ -165,16 +141,15 @@ stream (or from bytes provided through its ``feed`` method).
    unpacker = msgpack.Unpacker(buf, raw=False)
    for unpacked in unpacker:
        print(unpacked)
+```
 
 
-Packing/unpacking of custom data type
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+### Packing/unpacking of custom data type
 
 It is also possible to pack/unpack custom data types. Here is an example for
 ``datetime.datetime``.
 
-.. code-block:: python
-
+```py
     import datetime
     import msgpack
 
@@ -196,19 +171,18 @@ It is also possible to pack/unpack custom data types. Here is an example for
 
     packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True)
     this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False)
+```
 
 ``Unpacker``'s ``object_hook`` callback receives a dict; the
 ``object_pairs_hook`` callback may instead be used to receive a list of
 key-value pairs.
 
 
-Extended types
-^^^^^^^^^^^^^^
+### Extended types
 
 It is also possible to pack/unpack custom data types using the **ext** type.
 
-.. code-block:: pycon
-
+```pycon
     >>> import msgpack
     >>> import array
     >>> def default(obj):
@@ -228,10 +202,10 @@ It is also possible to pack/unpack custom data types using the **ext** type.
     >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False)
     >>> data == unpacked
     True
+```
 
 
-Advanced unpacking control
-^^^^^^^^^^^^^^^^^^^^^^^^^^
+### Advanced unpacking control
 
 As an alternative to iteration, ``Unpacker`` objects provide ``unpack``,
 ``skip``, ``read_array_header`` and ``read_map_header`` methods. The former two
@@ -243,8 +217,7 @@ in a map, can be unpacked or skipped individually.
 Each of these methods may optionally write the packed data it reads to a
 callback function:
 
-.. code-block:: python
-
+```py
     from io import BytesIO
 
     def distribute(unpacker, get_worker):
@@ -258,13 +231,11 @@ callback function:
             bytestream = BytesIO()
             unpacker.skip(bytestream.write)
             worker.send(bytestream.getvalue())
+```
 
+## Notes
 
-Notes
------
-
-string and binary type
-^^^^^^^^^^^^^^^^^^^^^^
+### string and binary type
 
 Early versions of msgpack didn't distinguish string and binary types.
 The type for representing both string and binary types was named **raw**.
@@ -272,32 +243,29 @@ The type for representing both string and binary types was named **raw**.
 You can pack into and unpack from this old spec using ``use_bin_type=False``
 and ``raw=True`` options.
 
-.. code-block:: pycon
-
+```pycon
     >>> import msgpack
     >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True)
     [b'spam', b'eggs']
     >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False)
     [b'spam', 'eggs']
+```
 
-
-ext type
-^^^^^^^^
+### ext type
 
 To use the **ext** type, pass ``msgpack.ExtType`` object to packer.
 
-.. code-block:: pycon
-
+```pycon
     >>> import msgpack
     >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy'))
     >>> msgpack.unpackb(packed)
     ExtType(code=42, data='xyzzy')
+```
 
 You can use it with ``default`` and ``ext_hook``. See below.
 
 
-Security
-^^^^^^^^
+### Security
 
 To unpacking data received from unreliable source, msgpack provides
 two security options.
@@ -311,8 +279,7 @@ there is a risk of the hashdos.
 If you need to support other types for map keys, use ``strict_map_key=False``.
 
 
-Performance tips
-^^^^^^^^^^^^^^^^
+### Performance tips
 
 CPython's GC starts when growing allocated object.
 This means unpacking may cause useless GC.
@@ -323,17 +290,13 @@ But tuple is lighter than list.
 You can use ``use_list=False`` while unpacking when performance is important.
 
 
-Development
------------
+## Development
 
-Test
-^^^^
+### Test
 
 MessagePack uses `pytest` for testing.
 Run test with following command:
 
+```
     $ make test
-
-
-..
-    vim: filetype=rst
+```
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 1e73a00..5762e8c 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (0, 6, 2)
+version = (1, 0, 0, 'rc1')
diff --git a/setup.py b/setup.py
index a8c2306..ac0dc30 100755
--- a/setup.py
+++ b/setup.py
@@ -106,7 +106,7 @@ del libraries, macros
 
 
 desc = "MessagePack (de)serializer."
-with io.open("README.rst", encoding="utf-8") as f:
+with io.open("README.md", encoding="utf-8") as f:
     long_desc = f.read()
 del f
 
@@ -118,7 +118,7 @@ if TRANSITIONAL:
 
 setup(
     name=name,
-    author="INADA Naoki",
+    author="Inada Naoki",
     author_email="songofacandy@gmail.com",
     version=version_str,
     cmdclass={"build_ext": BuildExt, "sdist": Sdist},
@@ -126,7 +126,7 @@ setup(
     packages=["msgpack"],
     description=desc,
     long_description=long_desc,
-    long_description_content_type="text/x-rst",
+    long_description_content_type="text/markdown",
     url="https://msgpack.org/",
     project_urls={
         "Documentation": "https://msgpack-python.readthedocs.io/",

From a05fc5e7c543d4d925802b6bba1a5542ee8ee3c3 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 12 Dec 2019 18:46:55 +0900
Subject: [PATCH 1505/1648] black

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index 5762e8c..56f67e4 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (1, 0, 0, 'rc1')
+version = (1, 0, 0, "rc1")

From aab29ff277cf88ff85e7ea5e603607a24d8c38a4 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 12 Dec 2019 18:48:16 +0900
Subject: [PATCH 1506/1648] Remove TRANSITIONAL package support

---
 setup.py | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/setup.py b/setup.py
index ac0dc30..2ec9ca7 100755
--- a/setup.py
+++ b/setup.py
@@ -14,10 +14,6 @@ PYPY = hasattr(sys, "pypy_version_info")
 PY2 = sys.version_info[0] == 2
 
 
-# for building transitional package.
-TRANSITIONAL = False
-
-
 class NoCython(Exception):
     pass
 
@@ -110,14 +106,8 @@ with io.open("README.md", encoding="utf-8") as f:
     long_desc = f.read()
 del f
 
-name = "msgpack"
-
-if TRANSITIONAL:
-    name = "msgpack-python"
-    long_desc = "This package is deprecated.  Install msgpack instead."
-
 setup(
-    name=name,
+    name="msgpack",
     author="Inada Naoki",
     author_email="songofacandy@gmail.com",
     version=version_str,

From 887d3a7d22865d36d68fdcb5e653ea61d66f0b61 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 12 Dec 2019 19:43:59 +0900
Subject: [PATCH 1507/1648] Refine Timestamp APIs (#395)

---
 msgpack/ext.py         | 99 ++++++++++++++++++++++++------------------
 msgpack/fallback.py    |  4 +-
 test/test_timestamp.py | 16 ++++---
 3 files changed, 69 insertions(+), 50 deletions(-)

diff --git a/msgpack/ext.py b/msgpack/ext.py
index 09adb34..00b759d 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -7,8 +7,11 @@ import struct
 
 PY2 = sys.version_info[0] == 2
 
-if not PY2:
-    long = int
+if PY2:
+    int_types = (int, long)
+    _utc = None
+else:
+    int_types = int
     try:
         _utc = datetime.timezone.utc
     except AttributeError:
@@ -23,8 +26,6 @@ class ExtType(namedtuple("ExtType", "code data")):
             raise TypeError("code must be int")
         if not isinstance(data, bytes):
             raise TypeError("data must be bytes")
-        if code == -1:
-            return Timestamp.from_bytes(data)
         if not 0 <= code <= 127:
             raise ValueError("code must be 0~127")
         return super(ExtType, cls).__new__(cls, code, data)
@@ -42,34 +43,26 @@ class Timestamp(object):
     def __init__(self, seconds, nanoseconds=0):
         """Initialize a Timestamp object.
 
-        :param seconds: Number of seconds since the UNIX epoch (00:00:00 UTC Jan 1 1970, minus leap seconds). May be
-            negative. If :code:`seconds` includes a fractional part, :code:`nanoseconds` must be 0.
-        :type seconds: int or float
+        :param int seconds:
+            Number of seconds since the UNIX epoch (00:00:00 UTC Jan 1 1970, minus leap seconds).
+            May be negative.
 
-        :param nanoseconds: Number of nanoseconds to add to `seconds` to get fractional time. Maximum is 999_999_999.
-            Default is 0.
-        :type nanoseconds: int
+        :param int nanoseconds:
+            Number of nanoseconds to add to `seconds` to get fractional time.
+            Maximum is 999_999_999.  Default is 0.
 
         Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns.
         """
-        if not isinstance(seconds, (int, long, float)):
-            raise TypeError("seconds must be numeric")
-        if not isinstance(nanoseconds, (int, long)):
+        if not isinstance(seconds, int_types):
+            raise TypeError("seconds must be an interger")
+        if not isinstance(nanoseconds, int_types):
             raise TypeError("nanoseconds must be an integer")
-        if nanoseconds:
-            if nanoseconds < 0 or nanoseconds % 1 != 0 or nanoseconds > (1e9 - 1):
-                raise ValueError(
-                    "nanoseconds must be a non-negative integer less than 999999999."
-                )
-            if not isinstance(seconds, (int, long)):
-                raise ValueError(
-                    "seconds must be an integer if also providing nanoseconds."
-                )
-            self.nanoseconds = nanoseconds
-        else:
-            # round helps with floating point issues
-            self.nanoseconds = int(round(seconds % 1 * 1e9, 0))
-        self.seconds = int(seconds // 1)
+        if not (0 <= nanoseconds < 10 ** 9):
+            raise ValueError(
+                "nanoseconds must be a non-negative integer less than 999999999."
+            )
+        self.seconds = seconds
+        self.nanoseconds = nanoseconds
 
     def __repr__(self):
         """String representation of Timestamp."""
@@ -137,7 +130,18 @@ class Timestamp(object):
             data = struct.pack("!Iq", self.nanoseconds, self.seconds)
         return data
 
-    def to_float(self):
+    @staticmethod
+    def from_unix(unix_sec):
+        """Create a Timestamp from posix timestamp in seconds.
+
+        :param unix_float: Posix timestamp in seconds.
+        :type unix_float: int or float.
+        """
+        seconds = int(unix_sec // 1)
+        nanoseconds = int((unix_sec % 1) * 10 ** 9)
+        return Timestamp(seconds, nanoseconds)
+
+    def to_unix(self):
         """Get the timestamp as a floating-point value.
 
         :returns: posix timestamp
@@ -146,28 +150,37 @@ class Timestamp(object):
         return self.seconds + self.nanoseconds / 1e9
 
     @staticmethod
-    def from_float(unix_float):
-        seconds = int(unix_float)
-        nanoseconds = int((unix_float % 1) * 1000000000)
-        return Timestamp(seconds, nanoseconds)
+    def from_unix_nano(unix_ns):
+        """Create a Timestamp from posix timestamp in nanoseconds.
 
-    def to_unix_ns(self):
+        :param int unix_ns: Posix timestamp in nanoseconds.
+        :rtype: Timestamp
+        """
+        return Timestamp(*divmod(unix_ns, 10 ** 9))
+
+    def to_unix_nano(self):
         """Get the timestamp as a unixtime in nanoseconds.
 
         :returns: posix timestamp in nanoseconds
         :rtype: int
         """
-        return int(self.seconds * 1e9 + self.nanoseconds)
+        return self.seconds * 10 ** 9 + self.nanoseconds
 
-    if not PY2:
+    def to_datetime(self):
+        """Get the timestamp as a UTC datetime.
 
-        def to_datetime(self):
-            """Get the timestamp as a UTC datetime.
+        Python 2 is not supported.
 
-            :rtype: datetime.
-            """
-            return datetime.datetime.fromtimestamp(self.to_float(), _utc)
+        :rtype: datetime.
+        """
+        return datetime.datetime.fromtimestamp(self.to_unix(), _utc)
 
-        @staticmethod
-        def from_datetime(dt):
-            return Timestamp.from_float(dt.timestamp())
+    @staticmethod
+    def from_datetime(dt):
+        """Create a Timestamp from datetime with tzinfo.
+
+        Python 2 is not supported.
+
+        :rtype: Timestamp
+        """
+        return Timestamp.from_unix(dt.timestamp())
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 9ba98bf..08e8d46 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -691,9 +691,9 @@ class Unpacker(object):
             if n == -1:  # timestamp
                 ts = Timestamp.from_bytes(bytes(obj))
                 if self._timestamp == 1:
-                    return ts.to_float()
+                    return ts.to_unix()
                 elif self._timestamp == 2:
-                    return ts.to_unix_ns()
+                    return ts.to_unix_nano()
                 elif self._timestamp == 3:
                     return ts.to_datetime()
                 else:
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index 822994c..ba5611c 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -37,19 +37,25 @@ def test_timestamp():
     assert ts.seconds == 2 ** 63 - 1 and ts.nanoseconds == 999999999
 
     # negative fractional
-    ts = Timestamp(-2.3)  # s: -3, ns: 700000000
+    ts = Timestamp.from_unix(-2.3)  # s: -3, ns: 700000000
+    assert ts.seconds == -3 and ts.nanoseconds == 700000000
     assert ts.to_bytes() == b"\x29\xb9\x27\x00\xff\xff\xff\xff\xff\xff\xff\xfd"
     packed = msgpack.packb(ts)
     assert packed == b"\xc7\x0c\xff" + ts.to_bytes()
     unpacked = msgpack.unpackb(packed)
     assert ts == unpacked
-    assert ts.seconds == -3 and ts.nanoseconds == 700000000
+
+
+def test_timestamp_from():
+    t = Timestamp(42, 14000)
+    assert Timestamp.from_unix(42.000014) == t
+    assert Timestamp.from_unix_nano(42000014000) == t
 
 
 def test_timestamp_to():
-    t = Timestamp(42, 14)
-    assert t.to_float() == 42.000000014
-    assert t.to_unix_ns() == 42000000014
+    t = Timestamp(42, 14000)
+    assert t.to_unix() == 42.000014
+    assert t.to_unix_nano() == 42000014000
 
 
 @pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")

From 9e5ec95e0292dce8485575310f6b69a618fdbefe Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 12 Dec 2019 19:59:06 +0900
Subject: [PATCH 1508/1648] Make Timestamp hashable (#396)

When overriding __eq__, __hash__ should be overridden too.
---
 msgpack/ext.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/msgpack/ext.py b/msgpack/ext.py
index 00b759d..cc34fb2 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -36,6 +36,8 @@ class Timestamp(object):
 
     When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python
     msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and unpack `Timestamp`.
+
+    This class is immutable: Do not override seconds and nanoseconds.
     """
 
     __slots__ = ["seconds", "nanoseconds"]
@@ -78,9 +80,8 @@ class Timestamp(object):
             )
         return False
 
-    def __ne__(self, other):
-        """not-equals method (see :func:`__eq__()`)"""
-        return not self.__eq__(other)
+    def __hash__(self):
+        return hash((self.seconds, self.nanoseconds))
 
     @staticmethod
     def from_bytes(b):

From 5e1fe818e3839c85a38419859bcec6d38979c620 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 12 Dec 2019 20:05:25 +0900
Subject: [PATCH 1509/1648] Reintroduce __ne__

---
 msgpack/ext.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/msgpack/ext.py b/msgpack/ext.py
index cc34fb2..8341c68 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -80,6 +80,10 @@ class Timestamp(object):
             )
         return False
 
+    def __ne__(self, other):
+        """not-equals method (see :func:`__eq__()`)"""
+        return not self.__eq__(other)
+
     def __hash__(self):
         return hash((self.seconds, self.nanoseconds))
 

From 42f5ecfd514cc5797385df0c72258a16fe645c72 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 13 Dec 2019 15:10:32 +0900
Subject: [PATCH 1510/1648] Fix some typo

---
 msgpack/fallback.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 08e8d46..263e74e 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -41,7 +41,7 @@ else:
 
 
 if hasattr(sys, "pypy_version_info"):
-    # cStringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
+    # StringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
     # StringBuilder is fastest.
     from __pypy__ import newlist_hint
 
@@ -147,7 +147,7 @@ def unpackb(packed, **kwargs):
 if sys.version_info < (2, 7, 6):
 
     def _unpack_from(f, b, o=0):
-        """Explicit typcast for legacy struct.unpack_from"""
+        """Explicit type cast for legacy struct.unpack_from"""
         return struct.unpack_from(f, bytes(b), o)
 
 
@@ -178,7 +178,7 @@ class Unpacker(object):
     :param int timestamp:
         Control how timestamp type is unpacked:
 
-            0 - Tiemstamp
+            0 - Timestamp
             1 - float  (Seconds from the EPOCH)
             2 - int  (Nanoseconds from the EPOCH)
             3 - datetime.datetime  (UTC).  Python 2 is not supported.
@@ -749,7 +749,7 @@ class Packer(object):
     """
     MessagePack Packer
 
-    usage:
+    Usage:
 
         packer = Packer()
         astream.write(packer.pack(a))
@@ -783,7 +783,7 @@ class Packer(object):
     :param bool datetime:
         If set to true, datetime with tzinfo is packed into Timestamp type.
         Note that the tzinfo is stripped in the timestamp.
-        You can get UTC datetime with `timestamp=3` option of the Unapcker.
+        You can get UTC datetime with `timestamp=3` option of the Unpacker.
         (Python 2 is not supported).
 
     :param str unicode_errors:

From ebfe55e63738fa41e55ac986ccae5e5f9bc3afbd Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 16 Dec 2019 15:14:34 +0900
Subject: [PATCH 1511/1648] travis: Use build config validation.

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 39db1d7..7af7beb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,3 +1,4 @@
+version: ~> 1.0
 dist: xenial
 language: python
 cache: pip

From 030bb2f1f7ebd12faed937d8ef9846601361fbce Mon Sep 17 00:00:00 2001
From: ossdev07 <39188636+ossdev07@users.noreply.github.com>
Date: Tue, 31 Dec 2019 06:42:21 +0530
Subject: [PATCH 1512/1648] travis: Add test for arm64 (#399)

Signed-off-by: ossdev07 
---
 .travis.yml | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 7af7beb..852674d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,7 +2,9 @@ version: ~> 1.0
 dist: xenial
 language: python
 cache: pip
-
+arch:
+   - amd64
+   - arm64
 python:
   # Available Python (PyPy) can be listed by:
   #
@@ -38,6 +40,21 @@ matrix:
       script:
         - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
 
+    - arch: arm64
+      name: arm64 32bit build
+      language: python
+      services:
+        - docker
+      env:
+        - DOCKER_IMAGE=quay.io/pypa/manylinux2014_aarch64
+      install:
+        - pip install -U pip
+        - pip install -r requirements.txt
+        - make cython
+        - docker pull $DOCKER_IMAGE
+      script:
+        - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
+
     - name: "Python 2 (fallback)"
       python: "2.7"
       <<: *pure

From 1bd6fc36d09cea273a5b47d4f54ed5e3b718582c Mon Sep 17 00:00:00 2001
From: Emilio Tagua <53828441+eb-emilio@users.noreply.github.com>
Date: Wed, 5 Feb 2020 09:20:17 -0300
Subject: [PATCH 1513/1648] Update README.md (#402)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 897a932..48401be 100644
--- a/README.md
+++ b/README.md
@@ -55,7 +55,7 @@ See note below for detail.
 
   * ``raw=False`` by default.  It assumes str types are valid UTF-8 string
     and decode them to Python str (unicode) object.
-  * ``encdoding`` option is rmeoved.  You can use ``raw=True`` to support old format.
+  * ``encoding`` option is removed.  You can use ``raw=True`` to support old format.
   * Default value of ``max_buffer_size`` is changed from 0 to 100 MiB.
   * Default value of ``strict_map_key`` is changed to True to avoid hashdos.
     You need to pass ``strict_map_key=False`` if you have data which contain map keys

From 24950990f4ebeffbf98acd188b171cc60a27095e Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 6 Feb 2020 20:29:33 +0900
Subject: [PATCH 1514/1648] Remove broken example

---
 README.md | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/README.md b/README.md
index 48401be..a4d69c1 100644
--- a/README.md
+++ b/README.md
@@ -214,24 +214,6 @@ the result, or ignoring it. The latter two methods return the number of elements
 in the upcoming container, so that each element in an array, or key-value pair
 in a map, can be unpacked or skipped individually.
 
-Each of these methods may optionally write the packed data it reads to a
-callback function:
-
-```py
-    from io import BytesIO
-
-    def distribute(unpacker, get_worker):
-        nelems = unpacker.read_map_header()
-        for i in range(nelems):
-            # Select a worker for the given key
-            key = unpacker.unpack()
-            worker = get_worker(key)
-
-            # Send the value as a packed message to worker
-            bytestream = BytesIO()
-            unpacker.skip(bytestream.write)
-            worker.send(bytestream.getvalue())
-```
 
 ## Notes
 

From 0dad82116912878dfb172de3f5affe128c7475ce Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 6 Feb 2020 20:35:41 +0900
Subject: [PATCH 1515/1648] Fix markdown

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a4d69c1..46af810 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
 
 ## What's this
 
-`MessagePack `_ is an efficient binary serialization format.
+[MessagePack](https://msgpack.org/) is an efficient binary serialization format.
 It lets you exchange data among multiple languages like JSON.
 But it's faster and smaller.
 This package provides CPython bindings for reading and writing MessagePack data.

From ff1f5f89d997b40e60472c2820ea55cab752c779 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 6 Feb 2020 21:06:04 +0900
Subject: [PATCH 1516/1648] README: `` -> `

---
 README.md | 74 +++++++++++++++++++++++++++----------------------------
 1 file changed, 37 insertions(+), 37 deletions(-)

diff --git a/README.md b/README.md
index 46af810..0355286 100644
--- a/README.md
+++ b/README.md
@@ -28,10 +28,10 @@ msgpack is removed, and `import msgpack` fail.
 
 ### Compatibility with the old format
 
-You can use ``use_bin_type=False`` option to pack ``bytes``
+You can use `use_bin_type=False` option to pack `bytes`
 object into raw type in the old msgpack spec, instead of bin type in new msgpack spec.
 
-You can unpack old msgpack format using ``raw=True`` option.
+You can unpack old msgpack format using `raw=True` option.
 It unpacks str (raw) type in msgpack into Python bytes.
 
 See note below for detail.
@@ -42,23 +42,23 @@ See note below for detail.
 * Python 2
 
   * The extension module does not support Python 2 anymore.
-    The pure Python implementation (``msgpack.fallback``) is used for Python 2.
+    The pure Python implementation (`msgpack.fallback`) is used for Python 2.
 
 * Packer
 
-  * ``use_bin_type=True`` by default.  bytes are encoded in bin type in msgpack.
+  * `use_bin_type=True` by default.  bytes are encoded in bin type in msgpack.
     **If you are still sing Python 2, you must use unicode for all string types.**
-    You can use ``use_bin_type=False`` to encode into old msgpack format.
-  * ``encoding`` option is removed.  UTF-8 is used always.
+    You can use `use_bin_type=False` to encode into old msgpack format.
+  * `encoding` option is removed.  UTF-8 is used always.
 
 * Unpacker
 
-  * ``raw=False`` by default.  It assumes str types are valid UTF-8 string
+  * `raw=False` by default.  It assumes str types are valid UTF-8 string
     and decode them to Python str (unicode) object.
-  * ``encoding`` option is removed.  You can use ``raw=True`` to support old format.
-  * Default value of ``max_buffer_size`` is changed from 0 to 100 MiB.
-  * Default value of ``strict_map_key`` is changed to True to avoid hashdos.
-    You need to pass ``strict_map_key=False`` if you have data which contain map keys
+  * `encoding` option is removed.  You can use `raw=True` to support old format.
+  * Default value of `max_buffer_size` is changed from 0 to 100 MiB.
+  * Default value of `strict_map_key` is changed to True to avoid hashdos.
+    You need to pass `strict_map_key=False` if you have data which contain map keys
     which type is not bytes or str.
 
 
@@ -70,10 +70,10 @@ See note below for detail.
 
 ### Pure Python implementation
 
-The extension module in msgpack (``msgpack._cmsgpack``) does not support
+The extension module in msgpack (`msgpack._cmsgpack`) does not support
 Python 2 and PyPy.
 
-But msgpack provides a pure Python implementation (``msgpack.fallback``)
+But msgpack provides a pure Python implementation (`msgpack.fallback`)
 for PyPy and Python 2.
 
 Since the [pip](https://pip.pypa.io/) uses the pure Python implementation,
@@ -89,18 +89,18 @@ Without extension, using pure Python implementation on CPython runs slowly.
 
 ## How to use
 
-NOTE: In examples below, I use ``raw=False`` and ``use_bin_type=True`` for users
+NOTE: In examples below, I use `raw=False` and `use_bin_type=True` for users
 using msgpack < 1.0. These options are default from msgpack 1.0 so you can omit them.
 
 
 ### One-shot pack & unpack
 
-Use ``packb`` for packing and ``unpackb`` for unpacking.
-msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with
-``json`` and ``pickle``.
+Use `packb` for packing and `unpackb` for unpacking.
+msgpack provides `dumps` and `loads` as an alias for compatibility with
+`json` and `pickle`.
 
-``pack`` and ``dump`` packs to a file-like object.
-``unpack`` and ``load`` unpacks from a file-like object.
+`pack` and `dump` packs to a file-like object.
+`unpack` and `load` unpacks from a file-like object.
 
 ```pycon
    >>> import msgpack
@@ -110,14 +110,14 @@ msgpack provides ``dumps`` and ``loads`` as an alias for compatibility with
    [1, 2, 3]
 ```
 
-``unpack`` unpacks msgpack's array to Python's list, but can also unpack to tuple:
+`unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple:
 
 ```pycon
    >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False)
    (1, 2, 3)
 ```
 
-You should always specify the ``use_list`` keyword argument for backward compatibility.
+You should always specify the `use_list` keyword argument for backward compatibility.
 See performance issues relating to `use_list option`_ below.
 
 Read the docstring for other options.
@@ -125,8 +125,8 @@ Read the docstring for other options.
 
 ### Streaming unpacking
 
-``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one
-stream (or from bytes provided through its ``feed`` method).
+`Unpacker` is a "streaming unpacker". It unpacks multiple objects from one
+stream (or from bytes provided through its `feed` method).
 
 ```py
    import msgpack
@@ -147,7 +147,7 @@ stream (or from bytes provided through its ``feed`` method).
 ### Packing/unpacking of custom data type
 
 It is also possible to pack/unpack custom data types. Here is an example for
-``datetime.datetime``.
+`datetime.datetime`.
 
 ```py
     import datetime
@@ -173,8 +173,8 @@ It is also possible to pack/unpack custom data types. Here is an example for
     this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False)
 ```
 
-``Unpacker``'s ``object_hook`` callback receives a dict; the
-``object_pairs_hook`` callback may instead be used to receive a list of
+`Unpacker`'s `object_hook` callback receives a dict; the
+`object_pairs_hook` callback may instead be used to receive a list of
 key-value pairs.
 
 
@@ -207,8 +207,8 @@ It is also possible to pack/unpack custom data types using the **ext** type.
 
 ### Advanced unpacking control
 
-As an alternative to iteration, ``Unpacker`` objects provide ``unpack``,
-``skip``, ``read_array_header`` and ``read_map_header`` methods. The former two
+As an alternative to iteration, `Unpacker` objects provide `unpack`,
+`skip`, `read_array_header` and `read_map_header` methods. The former two
 read an entire message from the stream, respectively de-serialising and returning
 the result, or ignoring it. The latter two methods return the number of elements
 in the upcoming container, so that each element in an array, or key-value pair
@@ -222,8 +222,8 @@ in a map, can be unpacked or skipped individually.
 Early versions of msgpack didn't distinguish string and binary types.
 The type for representing both string and binary types was named **raw**.
 
-You can pack into and unpack from this old spec using ``use_bin_type=False``
-and ``raw=True`` options.
+You can pack into and unpack from this old spec using `use_bin_type=False`
+and `raw=True` options.
 
 ```pycon
     >>> import msgpack
@@ -235,7 +235,7 @@ and ``raw=True`` options.
 
 ### ext type
 
-To use the **ext** type, pass ``msgpack.ExtType`` object to packer.
+To use the **ext** type, pass `msgpack.ExtType` object to packer.
 
 ```pycon
     >>> import msgpack
@@ -244,7 +244,7 @@ To use the **ext** type, pass ``msgpack.ExtType`` object to packer.
     ExtType(code=42, data='xyzzy')
 ```
 
-You can use it with ``default`` and ``ext_hook``. See below.
+You can use it with `default` and `ext_hook`. See below.
 
 
 ### Security
@@ -252,24 +252,24 @@ You can use it with ``default`` and ``ext_hook``. See below.
 To unpacking data received from unreliable source, msgpack provides
 two security options.
 
-``max_buffer_size`` (default: 100*1024*1024) limits the internal buffer size.
+`max_buffer_size` (default: `100*1024*1024`) limits the internal buffer size.
 It is used to limit the preallocated list size too.
 
-``strict_map_key`` (default: ``True``) limits the type of map keys to bytes and str.
+`strict_map_key` (default: `True`) limits the type of map keys to bytes and str.
 While msgpack spec doesn't limit the types of the map keys,
 there is a risk of the hashdos.
-If you need to support other types for map keys, use ``strict_map_key=False``.
+If you need to support other types for map keys, use `strict_map_key=False`.
 
 
 ### Performance tips
 
 CPython's GC starts when growing allocated object.
 This means unpacking may cause useless GC.
-You can use ``gc.disable()`` when unpacking large message.
+You can use `gc.disable()` when unpacking large message.
 
 List is the default sequence type of Python.
 But tuple is lighter than list.
-You can use ``use_list=False`` while unpacking when performance is important.
+You can use `use_list=False` while unpacking when performance is important.
 
 
 ## Development

From 9d79351e99e435b8ca749d57a313441783f67133 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 6 Feb 2020 22:11:04 +0900
Subject: [PATCH 1517/1648] Add some test for timestamp (#403)

---
 .github/workflows/black.yaml |  2 +-
 msgpack/fallback.py          |  8 +-------
 test/test_format.py          | 19 +++++++------------
 test/test_pack.py            | 14 +++-----------
 test/test_stricttype.py      |  5 +----
 test/test_timestamp.py       | 27 +++++++++++++++++++++++++++
 6 files changed, 40 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml
index cabd0cc..eda8d07 100644
--- a/.github/workflows/black.yaml
+++ b/.github/workflows/black.yaml
@@ -18,4 +18,4 @@ jobs:
       - name: Black Code Formatter
         run: |
           pip install black
-          black --diff --check msgpack/ test/
+          black --diff --check msgpack/ test/ setup.py
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 263e74e..9f6665b 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -77,13 +77,7 @@ else:
     newlist_hint = lambda size: []
 
 
-from .exceptions import (
-    BufferFull,
-    OutOfData,
-    ExtraData,
-    FormatError,
-    StackError,
-)
+from .exceptions import BufferFull, OutOfData, ExtraData, FormatError, StackError
 
 from .ext import ExtType, Timestamp
 
diff --git a/test/test_format.py b/test/test_format.py
index d455f7c..fbbc3f9 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -9,29 +9,24 @@ def check(src, should, use_list=0, raw=True):
 
 
 def testSimpleValue():
-    check(b"\x93\xc0\xc2\xc3", (None, False, True,))
+    check(b"\x93\xc0\xc2\xc3", (None, False, True))
 
 
 def testFixnum():
-    check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", ((0, 64, 127,), (-32, -16, -1,),))
+    check(b"\x92\x93\x00\x40\x7f\x93\xe0\xf0\xff", ((0, 64, 127), (-32, -16, -1)))
 
 
 def testFixArray():
-    check(
-        b"\x92\x90\x91\x91\xc0", ((), ((None,),),),
-    )
+    check(b"\x92\x90\x91\x91\xc0", ((), ((None,),)))
 
 
 def testFixRaw():
-    check(
-        b"\x94\xa0\xa1a\xa2bc\xa3def", (b"", b"a", b"bc", b"def",),
-    )
+    check(b"\x94\xa0\xa1a\xa2bc\xa3def", (b"", b"a", b"bc", b"def"))
 
 
 def testFixMap():
     check(
-        b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80",
-        {False: {None: None}, True: {None: {}}},
+        b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}}
     )
 
 
@@ -40,7 +35,7 @@ def testUnsignedInt():
         b"\x99\xcc\x00\xcc\x80\xcc\xff\xcd\x00\x00\xcd\x80\x00"
         b"\xcd\xff\xff\xce\x00\x00\x00\x00\xce\x80\x00\x00\x00"
         b"\xce\xff\xff\xff\xff",
-        (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295,),
+        (0, 128, 255, 0, 32768, 65535, 0, 2147483648, 4294967295),
     )
 
 
@@ -49,7 +44,7 @@ def testSignedInt():
         b"\x99\xd0\x00\xd0\x80\xd0\xff\xd1\x00\x00\xd1\x80\x00"
         b"\xd1\xff\xff\xd2\x00\x00\x00\x00\xd2\x80\x00\x00\x00"
         b"\xd2\xff\xff\xff\xff",
-        (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1,),
+        (0, -128, -1, 0, -32768, -1, 0, -2147483648, -1),
     )
 
 
diff --git a/test/test_pack.py b/test/test_pack.py
index 932f760..a51d84c 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -49,7 +49,7 @@ def testPack():
         False,
         (),
         ((),),
-        ((), None,),
+        ((), None),
         {None: 0},
         (1 << 23),
     ]
@@ -69,21 +69,13 @@ def testPackUnicode():
 
 
 def testPackBytes():
-    test_data = [
-        b"",
-        b"abcd",
-        (b"defgh",),
-    ]
+    test_data = [b"", b"abcd", (b"defgh",)]
     for td in test_data:
         check(td)
 
 
 def testPackByteArrays():
-    test_data = [
-        bytearray(b""),
-        bytearray(b"abcd"),
-        (bytearray(b"defgh"),),
-    ]
+    test_data = [bytearray(b""), bytearray(b"abcd"), (bytearray(b"defgh"),)]
     for td in test_data:
         check(td)
 
diff --git a/test/test_stricttype.py b/test/test_stricttype.py
index 78e1723..fe9ec6c 100644
--- a/test/test_stricttype.py
+++ b/test/test_stricttype.py
@@ -22,10 +22,7 @@ def test_tuple():
 
     def default(o):
         if isinstance(o, tuple):
-            return {
-                "__type__": "tuple",
-                "value": list(o),
-            }
+            return {"__type__": "tuple", "value": list(o)}
         raise TypeError("Unsupported type %s" % (type(o),))
 
     def convert(o):
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index ba5611c..823fe04 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -46,6 +46,33 @@ def test_timestamp():
     assert ts == unpacked
 
 
+def test_unpack_timestamp():
+    # timestamp 32
+    assert msgpack.unpackb(b"\xd6\xff\x00\x00\x00\x00") == Timestamp(0)
+
+    # timestamp 64
+    assert msgpack.unpackb(b"\xd7\xff" + b"\x00" * 8) == Timestamp(0)
+    with pytest.raises(ValueError):
+        msgpack.unpackb(b"\xd7\xff" + b"\xff" * 8)
+
+    # timestamp 96
+    assert msgpack.unpackb(b"\xc7\x0c\xff" + b"\x00" * 12) == Timestamp(0)
+    with pytest.raises(ValueError):
+        msgpack.unpackb(b"\xc7\x0c\xff" + b"\xff" * 12) == Timestamp(0)
+
+    # Undefined
+    with pytest.raises(ValueError):
+        msgpack.unpackb(b"\xd4\xff\x00")  # fixext 1
+    with pytest.raises(ValueError):
+        msgpack.unpackb(b"\xd5\xff\x00\x00")  # fixext 2
+    with pytest.raises(ValueError):
+        msgpack.unpackb(b"\xc7\x00\xff")  # ext8 (len=0)
+    with pytest.raises(ValueError):
+        msgpack.unpackb(b"\xc7\x03\xff\0\0\0")  # ext8 (len=3)
+    with pytest.raises(ValueError):
+        msgpack.unpackb(b"\xc7\x05\xff\0\0\0\0\0")  # ext8 (len=5)
+
+
 def test_timestamp_from():
     t = Timestamp(42, 14000)
     assert Timestamp.from_unix(42.000014) == t

From f0952f1dd657e3f8437907bfe13885e61d5367fe Mon Sep 17 00:00:00 2001
From: Erik Cederstrand 
Date: Fri, 14 Feb 2020 04:31:09 +0100
Subject: [PATCH 1518/1648] travis: Python 3.9 is the new dev version. (#405)

---
 .travis.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 852674d..5132b4c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,7 +13,8 @@ python:
   - "3.5"
   - "3.6"
   - "3.7"
-  - "3.8-dev"
+  - "3.8"
+  - "3.9-dev"
 
 
 _pure: &pure

From cd6561db520de20c681bb50022cec621c0deda82 Mon Sep 17 00:00:00 2001
From: Alex Willmer 
Date: Fri, 14 Feb 2020 06:45:17 +0000
Subject: [PATCH 1519/1648] build: Don't test C extension on CPython 2.7 under
 Tox (#406)

As the Changelog notes, release 1.0 will drop support for the native
extension on CPython 2.x. So there seems little benefit of testing it.
---
 tox.ini | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 4b059ff..607b182 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,10 @@
 [tox]
-envlist = {py27,py35,py36,py37,py38}-{c,pure},{pypy,pypy3}-pure,py27-x86,py34-x86
+envlist =
+    py27-pure,
+    {py35,py36,py37,py38}-{c,pure},
+    {pypy,pypy3}-pure,
+    py27-x86,
+    py34-x86,
 
 [variants:pure]
 setenv=

From fcb19a0e1a86d80c28447b0008504c3f4e2faf59 Mon Sep 17 00:00:00 2001
From: Alex Willmer 
Date: Fri, 14 Feb 2020 06:51:19 +0000
Subject: [PATCH 1520/1648] Clean msgpack/_cmsgpack.cpp and
 msgpack/_cmsgpack.*.so (#407)

---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index e2f25cf..b4749b3 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,8 @@ serve-doc: all
 .PHONY: clean
 clean:
 	rm -rf build
-	rm -f msgpack/_msgpack.cpp
+	rm -f msgpack/_cmsgpack.cpp
+	rm -f msgpack/_cmsgpack.*.so
 	rm -rf msgpack/__pycache__
 	rm -rf test/__pycache__
 

From 64f59884a1a56a882ae888af354de12d5eb052a8 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 17 Feb 2020 16:58:25 +0900
Subject: [PATCH 1521/1648] Add note

---
 DEVELOP.md | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 DEVELOP.md

diff --git a/DEVELOP.md b/DEVELOP.md
new file mode 100644
index 0000000..0e6e0e5
--- /dev/null
+++ b/DEVELOP.md
@@ -0,0 +1,9 @@
+# Developer's note
+
+## Wheels
+
+Wheels for macOS and Linux are built on Travis and AppVeyr, in
+[methane/msgpack-wheels](https://github.com/methane/msgpack-wheels) repository.
+
+Wheels for Windows are built on Github Actions in this repository.
+

From fa7d7447fc2cc7cc1a8b388618549c98d1712b9c Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 17 Feb 2020 17:07:18 +0900
Subject: [PATCH 1522/1648] 1.0.0

---
 ChangeLog.rst       | 2 +-
 msgpack/_version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index c70b966..55cfd5d 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,7 +1,7 @@
 1.0.0
 =====
 
-Release Date: TBD
+Release Date: 2020-02-17
 
 * Remove Python 2 support from the ``msgpack/_cmsgpack``.
   ``msgpack/fallback`` still supports Python 2.
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 56f67e4..9f55cf5 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (1, 0, 0, "rc1")
+version = (1, 0, 0)

From 12506d8d91ce5a7b76c2d8babe7e2d1e2851d3a2 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 17 Feb 2020 17:12:47 +0900
Subject: [PATCH 1523/1648] update README

---
 DEVELOP.md | 16 ++++++++++++++++
 README.md  | 12 ------------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/DEVELOP.md b/DEVELOP.md
index 0e6e0e5..9c823c3 100644
--- a/DEVELOP.md
+++ b/DEVELOP.md
@@ -7,3 +7,19 @@ Wheels for macOS and Linux are built on Travis and AppVeyr, in
 
 Wheels for Windows are built on Github Actions in this repository.
 
+
+### Build
+
+```
+$ make cython
+```
+
+
+### Test
+
+MessagePack uses `pytest` for testing.
+Run test with following command:
+
+```
+$ make test
+```
diff --git a/README.md b/README.md
index 0355286..78bc0cf 100644
--- a/README.md
+++ b/README.md
@@ -270,15 +270,3 @@ You can use `gc.disable()` when unpacking large message.
 List is the default sequence type of Python.
 But tuple is lighter than list.
 You can use `use_list=False` while unpacking when performance is important.
-
-
-## Development
-
-### Test
-
-MessagePack uses `pytest` for testing.
-Run test with following command:
-
-```
-    $ make test
-```

From 2849f5582ab154ade7fcd8c23109b1c7bd8d1530 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 19 Feb 2020 00:53:01 +0900
Subject: [PATCH 1524/1648] Build linux and macOS wheels on GitHub Actions.
 (#409)

---
 .github/workflows/linux.yml | 67 ++++++++++++++++++++++++++++++++
 .github/workflows/mac.yml   | 76 +++++++++++++++++++++++++++++++++++++
 Makefile                    |  8 ++--
 docker/buildwheel.sh        |  8 +++-
 4 files changed, 154 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/linux.yml
 create mode 100644 .github/workflows/mac.yml

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
new file mode 100644
index 0000000..89bdb4e
--- /dev/null
+++ b/.github/workflows/linux.yml
@@ -0,0 +1,67 @@
+name: Build Linux Wheels
+on:
+  push:
+  pull_request:
+  create:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+
+      - name: Cythonize
+        shell: bash
+        run: |
+          pip install -U pip
+          pip -V
+          pip install -r requirements.txt
+          make cython
+          #python setup.py sdist
+
+      - name: Build wheels
+        shell: bash
+        run: |
+          make linux-wheel
+
+      - name: Run test (3.8)
+        run: |
+          pip install pytest
+          pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
+          pytest -v test
+
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+
+      - name: Run test (3.7)
+        run: |
+          pip install pytest
+          pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
+          pytest -v test
+
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.6
+
+      - name: Run test (3.6)
+        run: |
+          pip install pytest
+          pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
+          pytest -v test
+
+
+      - name: Upload Wheels
+        uses: actions/upload-artifact@v1
+        with:
+          name: linux-wheels
+          path: ./dist/wheelhouse/
diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
new file mode 100644
index 0000000..fb2c67f
--- /dev/null
+++ b/.github/workflows/mac.yml
@@ -0,0 +1,76 @@
+name: Build macOS Wheels
+on:
+  push:
+  pull_request:
+  create:
+
+jobs:
+  build:
+    runs-on: macos-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.8"
+
+      - name: Cythonize
+        run: |
+          pip install -U pip
+          pip install -r requirements.txt
+          make cython
+
+      - name: Build wheel
+        run: |
+          pip install setuptools wheel
+          python setup.py bdist_wheel
+
+      - name: Run test
+        run: |
+          pip install pytest
+          pip install -v msgpack --only-binary :all: -f dist/ --no-index
+          pytest -v test
+
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.7"
+
+      - name: Build wheel
+        run: |
+          pip install setuptools wheel
+          python setup.py bdist_wheel
+
+      - name: Run test
+        run: |
+          pip install pytest
+          pip install -v msgpack --only-binary :all: -f dist/ --no-index
+          pytest -v test
+
+
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.6"
+
+      - name: Build wheel
+        run: |
+          pip install setuptools wheel
+          python setup.py bdist_wheel
+
+      - name: Run test
+        run: |
+          pip install pytest
+          pip install -v msgpack --only-binary :all: -f dist/ --no-index
+          pytest -v test
+
+
+      - name: Upload Wheels
+        uses: actions/upload-artifact@v1
+        with:
+          name: macos-wheels
+          path: ./dist/
diff --git a/Makefile b/Makefile
index b4749b3..2a4c0af 100644
--- a/Makefile
+++ b/Makefile
@@ -30,10 +30,10 @@ clean:
 
 .PHONY: update-docker
 update-docker:
-	docker pull quay.io/pypa/manylinux1_i686
-	docker pull quay.io/pypa/manylinux1_x86_64
+	docker pull quay.io/pypa/manylinux2010_i686
+	docker pull quay.io/pypa/manylinux2010_x86_64
 
 .PHONY: linux-wheel
 linux-wheel:
-	docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686   bash docker/buildwheel.sh
-	docker run --rm -ti -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh
+	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_i686   bash docker/buildwheel.sh
+	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash docker/buildwheel.sh
diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh
index c953127..89a2570 100644
--- a/docker/buildwheel.sh
+++ b/docker/buildwheel.sh
@@ -10,5 +10,11 @@ echo "arch=$ARCH"
 for V in "${PYTHON_VERSIONS[@]}"; do
     PYBIN=/opt/python/$V/bin
     rm -rf build/       # Avoid lib build by narrow Python is used by wide python
-    $PYBIN/python setup.py bdist_wheel -p manylinux1_${ARCH}
+    $PYBIN/python setup.py bdist_wheel
+done
+
+cd dist
+for whl in *.whl; do
+    auditwheel repair "$whl"
+    rm "$whl"
 done

From 2bfc2d0566e24594460078680ad3bd0dc71892ad Mon Sep 17 00:00:00 2001
From: Charles-Axel Dein 
Date: Mon, 24 Feb 2020 09:51:56 +0100
Subject: [PATCH 1525/1648] Upgrade msgpack if already installed (#414)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 78bc0cf..921f7f8 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ This package provides CPython bindings for reading and writing MessagePack data.
 ### PyPI package name
 
 TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`.
-Do `pip uninstall msgpack-python; pip install msgpack` instead.
+Do `pip uninstall msgpack-python; pip install -U msgpack` instead.
 
 Package name on PyPI was changed to msgpack from 0.5.
 I upload transitional package (msgpack-python 0.5 which depending on msgpack)

From 692e0ee8ff66686dd423aae69b248b67c3bf9ed4 Mon Sep 17 00:00:00 2001
From: Dan Salmon 
Date: Wed, 18 Mar 2020 00:29:51 +0000
Subject: [PATCH 1526/1648] Fix typo (#416)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 921f7f8..aeeee3a 100644
--- a/README.md
+++ b/README.md
@@ -47,7 +47,7 @@ See note below for detail.
 * Packer
 
   * `use_bin_type=True` by default.  bytes are encoded in bin type in msgpack.
-    **If you are still sing Python 2, you must use unicode for all string types.**
+    **If you are still using Python 2, you must use unicode for all string types.**
     You can use `use_bin_type=False` to encode into old msgpack format.
   * `encoding` option is removed.  UTF-8 is used always.
 

From 4e10222b5116806864a91fd9f79a70869e0a43c1 Mon Sep 17 00:00:00 2001
From: Kevin Tewouda 
Date: Wed, 13 May 2020 06:41:15 +0200
Subject: [PATCH 1527/1648] Fix an example in README.md (#423)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index aeeee3a..ac52d94 100644
--- a/README.md
+++ b/README.md
@@ -159,7 +159,7 @@ It is also possible to pack/unpack custom data types. Here is an example for
     }
 
     def decode_datetime(obj):
-        if b'__datetime__' in obj:
+        if '__datetime__' in obj:
             obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f")
         return obj
 

From b04690012d5d77cfe5074893686c4d55ec780300 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Sun, 24 May 2020 02:15:04 +0900
Subject: [PATCH 1528/1648] Update doc version

Fixes #425
---
 docs/conf.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 36fa76e..6b432be 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -41,7 +41,7 @@ master_doc = "index"
 
 # General information about the project.
 project = u"msgpack"
-copyright = u"2013, INADA Naoki"
+copyright = u"Inada Naoki"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -49,7 +49,7 @@ copyright = u"2013, INADA Naoki"
 #
 # The short X.Y version.
 # The full version, including alpha/beta/rc tags.
-version = release = "0.5"
+version = release = "1.0"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

From c1b1a23f62d5e0ec39a1910d2e9580ce1c13a1cf Mon Sep 17 00:00:00 2001
From: jfolz 
Date: Mon, 8 Jun 2020 05:14:50 +0200
Subject: [PATCH 1529/1648] Fix Unpacker.tell() (#427)

Fixes #426.

Co-authored-by: folz 
---
 msgpack/_unpacker.pyx | 10 ++++++++--
 msgpack/fallback.py   | 15 ++++++++-------
 test/test_unpack.py   | 29 +++++++++++++++++++++++++++++
 3 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 43c93a2..4340e04 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -484,8 +484,10 @@ cdef class Unpacker(object):
         nread = min(self.buf_tail - self.buf_head, nbytes)
         ret = PyBytes_FromStringAndSize(self.buf + self.buf_head, nread)
         self.buf_head += nread
-        if len(ret) < nbytes and self.file_like is not None:
-            ret += self.file_like.read(nbytes - len(ret))
+        if nread < nbytes and self.file_like is not None:
+            ret += self.file_like.read(nbytes - nread)
+            nread = len(ret)
+        self.stream_offset += nread
         return ret
 
     def unpack(self):
@@ -519,6 +521,10 @@ cdef class Unpacker(object):
         return self._unpack(read_map_header)
 
     def tell(self):
+        """Returns the current position of the Unpacker in bytes, i.e., the
+        number of bytes that were read from the input, also the starting
+        position of the next object.
+        """
         return self.stream_offset
 
     def __iter__(self):
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 9f6665b..1e0bbe9 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -365,18 +365,19 @@ class Unpacker(object):
         return self._buffer[self._buff_i :]
 
     def read_bytes(self, n):
-        ret = self._read(n)
+        ret = self._read(n, raise_outofdata=False)
         self._consume()
         return ret
 
-    def _read(self, n):
+    def _read(self, n, raise_outofdata=True):
         # (int) -> bytearray
-        self._reserve(n)
+        self._reserve(n, raise_outofdata=raise_outofdata)
         i = self._buff_i
-        self._buff_i = i + n
-        return self._buffer[i : i + n]
+        ret = self._buffer[i : i + n]
+        self._buff_i = i + len(ret)
+        return ret
 
-    def _reserve(self, n):
+    def _reserve(self, n, raise_outofdata=True):
         remain_bytes = len(self._buffer) - self._buff_i - n
 
         # Fast path: buffer has n bytes already
@@ -404,7 +405,7 @@ class Unpacker(object):
             self._buffer += read_data
             remain_bytes -= len(read_data)
 
-        if len(self._buffer) < n + self._buff_i:
+        if len(self._buffer) < n + self._buff_i and raise_outofdata:
             self._buff_i = 0  # rollback
             raise OutOfData
 
diff --git a/test/test_unpack.py b/test/test_unpack.py
index bc74c4d..057b7bf 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -3,6 +3,11 @@ import sys
 from msgpack import Unpacker, packb, OutOfData, ExtType
 from pytest import raises, mark
 
+try:
+    from itertools import izip as zip
+except ImportError:
+    pass
+
 
 def test_unpack_array_header_from_file():
     f = BytesIO(packb([1, 2, 3, 4]))
@@ -64,7 +69,31 @@ def test_unpacker_ext_hook():
     assert unpacker.unpack() == {"a": ExtType(2, b"321")}
 
 
+def test_unpacker_tell():
+    objects = 1, 2, u"abc", u"def", u"ghi"
+    packed = b"\x01\x02\xa3abc\xa3def\xa3ghi"
+    positions = 1, 2, 6, 10, 14
+    unpacker = Unpacker(BytesIO(packed))
+    for obj, unp, pos in zip(objects, unpacker, positions):
+        assert obj == unp
+        assert pos == unpacker.tell()
+
+
+def test_unpacker_tell_read_bytes():
+    objects = 1, u"abc", u"ghi"
+    packed = b"\x01\x02\xa3abc\xa3def\xa3ghi"
+    raw_data = b"\x02", b"\xa3def", b""
+    lenghts = 1, 4, 999
+    positions = 1, 6, 14
+    unpacker = Unpacker(BytesIO(packed))
+    for obj, unp, pos, n, raw in zip(objects, unpacker, positions, lenghts, raw_data):
+        assert obj == unp
+        assert pos == unpacker.tell()
+        assert unpacker.read_bytes(n) == raw
+
+
 if __name__ == "__main__":
     test_unpack_array_header_from_file()
     test_unpacker_hook_refcnt()
     test_unpacker_ext_hook()
+    test_unpacker_tell()

From 3508ca524ebb83b0117a0cc9a08986d933ddb022 Mon Sep 17 00:00:00 2001
From: Contextualist 
Date: Sun, 21 Jun 2020 22:27:52 -0400
Subject: [PATCH 1530/1648] Fix benchmark extension module import (#428)

---
 benchmark/benchmark.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py
index 82d0ddb..2e778dd 100644
--- a/benchmark/benchmark.py
+++ b/benchmark/benchmark.py
@@ -1,7 +1,7 @@
 from msgpack import fallback
 
 try:
-    from msgpack import _unpacker, _packer
+    from msgpack import _cmsgpack
 
     has_ext = True
 except ImportError:
@@ -17,14 +17,14 @@ def profile(name, func):
 
 def simple(name, data):
     if has_ext:
-        packer = _packer.Packer()
+        packer = _cmsgpack.Packer()
         profile("packing %s (ext)" % name, lambda: packer.pack(data))
     packer = fallback.Packer()
     profile("packing %s (fallback)" % name, lambda: packer.pack(data))
 
     data = packer.pack(data)
     if has_ext:
-        profile("unpacking %s (ext)" % name, lambda: _unpacker.unpackb(data))
+        profile("unpacking %s (ext)" % name, lambda: _cmsgpack.unpackb(data))
     profile("unpacking %s (fallback)" % name, lambda: fallback.unpackb(data))
 
 

From d9ead81021c7b5f034a0475bf9a88e9612cc8e84 Mon Sep 17 00:00:00 2001
From: Markus Gerstel <2102431+Anthchirp@users.noreply.github.com>
Date: Fri, 26 Jun 2020 10:15:46 +0100
Subject: [PATCH 1531/1648] Fix a typo in the changelog (#429)

---
 ChangeLog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 55cfd5d..d922e84 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -6,7 +6,7 @@ Release Date: 2020-02-17
 * Remove Python 2 support from the ``msgpack/_cmsgpack``.
   ``msgpack/fallback`` still supports Python 2.
 * Remove ``encoding`` option from the Packer and Unpacker.
-* Unpacker: The default value of ``max_buffer_type`` is changed to 100MiB.
+* Unpacker: The default value of ``max_buffer_size`` is changed to 100MiB.
 * Unpacker: ``strict_map_key`` is True by default now.
 * Unpacker: String map keys are interned.
 * Drop old buffer protocol support.

From 5614dd5a898772faa2dfe89bd9f6a5c90c5fafe5 Mon Sep 17 00:00:00 2001
From: Tom Pohl 
Date: Thu, 23 Jul 2020 10:53:55 +0200
Subject: [PATCH 1532/1648] Allow for timestamps before UNIX epoch (#433)

---
 msgpack/ext.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/msgpack/ext.py b/msgpack/ext.py
index 8341c68..4eb9dd6 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -178,7 +178,9 @@ class Timestamp(object):
 
         :rtype: datetime.
         """
-        return datetime.datetime.fromtimestamp(self.to_unix(), _utc)
+        return datetime.datetime.fromtimestamp(0, _utc) + datetime.timedelta(
+            seconds=self.to_unix()
+        )
 
     @staticmethod
     def from_datetime(dt):

From 772c830841a276adb392dd449809764b2826b1f8 Mon Sep 17 00:00:00 2001
From: Peter Fischer 
Date: Fri, 24 Jul 2020 09:29:15 +0200
Subject: [PATCH 1533/1648] Synchronize handling of datetime in Packer
 implementations (#434)

The handling of datetime is different in the cython and Python implementations. In contrast to the docs, timezone is not required in the Python implementation.
---
 msgpack/fallback.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 1e0bbe9..9739d53 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -921,7 +921,7 @@ class Packer(object):
                     len(obj), dict_iteritems(obj), nest_limit - 1
                 )
 
-            if self._datetime and check(obj, _DateTime):
+            if self._datetime and check(obj, _DateTime) and obj.tzinfo is not None:
                 obj = Timestamp.from_datetime(obj)
                 default_used = 1
                 continue

From 8fb709f2e0438862020d8810fa70a81fb5dac7d4 Mon Sep 17 00:00:00 2001
From: Peter Fischer 
Date: Thu, 30 Jul 2020 16:48:51 +0200
Subject: [PATCH 1534/1648] Fix datetime before epoch on windows in cython
 implementation (#436)

Cython implementation still used datetime.from_timestamp method, which does not work on windows.
Update the cython implementation to use utc time and delta and add a regression test to highlight the issue.
---
 Makefile               |  1 +
 msgpack/unpack.h       | 34 +++++++++++++++++++++-------------
 test/test_timestamp.py |  8 ++++++++
 3 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/Makefile b/Makefile
index 2a4c0af..0110ddf 100644
--- a/Makefile
+++ b/Makefile
@@ -25,6 +25,7 @@ clean:
 	rm -rf build
 	rm -f msgpack/_cmsgpack.cpp
 	rm -f msgpack/_cmsgpack.*.so
+	rm -f msgpack/_cmsgpack.*.pyd
 	rm -rf msgpack/__pycache__
 	rm -rf test/__pycache__
 
diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index debdf71..868b96e 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -341,7 +341,26 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos
         else if (u->timestamp == 0) {  // Timestamp
             py = PyObject_CallFunction(u->timestamp_t, "(Lk)", ts.tv_sec, ts.tv_nsec);
         }
-        else { // float or datetime
+        else if (u->timestamp == 3) {  // datetime
+            // Calculate datetime using epoch + delta 
+            // due to limitations PyDateTime_FromTimestamp on Windows with negative timestamps
+            PyObject *epoch = PyDateTimeAPI->DateTime_FromDateAndTime(1970, 1, 1, 0, 0, 0, 0, u->utc, PyDateTimeAPI->DateTimeType);
+            if (epoch == NULL) {
+                return -1;
+            }
+
+            PyObject* d = PyDelta_FromDSU(0, ts.tv_sec, ts.tv_nsec / 1000);
+            if (d == NULL) {
+                Py_DECREF(epoch);
+                return -1;
+            }
+
+            py = PyNumber_Add(epoch, d);
+            
+            Py_DECREF(epoch);
+            Py_DECREF(d);
+        }
+        else { // float
             PyObject *a = PyFloat_FromDouble((double)ts.tv_nsec);
             if (a == NULL) return -1;
 
@@ -358,18 +377,7 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos
             a = PyNumber_Add(b, c);
             Py_DECREF(b);
             Py_DECREF(c);
-
-            if (u->timestamp == 3) {  // datetime
-                PyObject *t = PyTuple_Pack(2, a, u->utc);
-                Py_DECREF(a);
-                if (t == NULL) {
-                    return -1;
-                }
-                py = PyDateTime_FromTimestamp(t);
-                Py_DECREF(t);
-            } else { // float
-                py = a;
-            }
+            py = a;
         }
     } else {
         py = PyObject_CallFunction(u->ext_hook, "(iy#)", (int)typecode, pos, (Py_ssize_t)length-1);
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index 823fe04..edc488a 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -99,6 +99,14 @@ def test_unpack_datetime():
     assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc)
 
 
+@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
+def test_pack_unpack_before_epoch():
+    t_in = datetime.datetime(1960, 1, 1, tzinfo=_utc)
+    packed = msgpack.packb(t_in, datetime=True)
+    unpacked = msgpack.unpackb(packed, timestamp=3)
+    assert unpacked == t_in
+
+
 @pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
 def test_pack_datetime():
     t = Timestamp(42, 14000)

From 44bc2bd439808ad7563ef8a558ad6ccfe175a66a Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 4 Dec 2020 17:23:09 +0900
Subject: [PATCH 1535/1648] Update docstring

---
 msgpack/_packer.pyx   |  8 ++++-
 msgpack/_unpacker.pyx | 80 +++++++++++++++++--------------------------
 msgpack/fallback.py   | 25 +++++++++++++-
 3 files changed, 63 insertions(+), 50 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index b470646..e6cd2c7 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -63,7 +63,7 @@ cdef class Packer(object):
     """
     MessagePack Packer
 
-    usage::
+    Usage::
 
         packer = Packer()
         astream.write(packer.pack(a))
@@ -94,6 +94,12 @@ cdef class Packer(object):
         This is useful when trying to implement accurate serialization
         for python types.
 
+    :param bool datetime:
+        If set to true, datetime with tzinfo is packed into Timestamp type.
+        Note that the tzinfo is stripped in the timestamp.
+        You can get UTC datetime with `timestamp=3` option of the Unpacker.
+        (Python 2 is not supported).
+
     :param str unicode_errors:
         The error handler for encoding unicode. (default: 'strict')
         DO NOT USE THIS!!  This option is kept for very specific usage.
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 4340e04..e4f3f1e 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -212,65 +212,49 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None,
 
 
 cdef class Unpacker(object):
-    """Streaming unpacker.
+    """
+    MessagePack Packer
 
-    Arguments:
+    Usage::
 
-    :param file_like:
-        File-like object having `.read(n)` method.
-        If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
+        packer = Packer()
+        astream.write(packer.pack(a))
+        astream.write(packer.pack(b))
 
-    :param int read_size:
-        Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`)
+    Packer's constructor has some keyword arguments:
 
-    :param bool use_list:
-        If true, unpack msgpack array to Python list.
-        Otherwise, unpack to Python tuple. (default: True)
+    :param callable default:
+        Convert user type to builtin type that Packer supports.
+        See also simplejson's document.
 
-    :param bool raw:
-        If true, unpack msgpack raw to Python bytes.
-        Otherwise, unpack to Python str by decoding with UTF-8 encoding (default).
+    :param bool use_single_float:
+        Use single precision float type for float. (default: False)
 
-    :param bool strict_map_key:
-        If true (default), only str or bytes are accepted for map (dict) keys.
+    :param bool autoreset:
+        Reset buffer after each pack and return its content as `bytes`. (default: True).
+        If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
 
-    :param callable object_hook:
-        When specified, it should be callable.
-        Unpacker calls it with a dict argument after unpacking msgpack map.
-        (See also simplejson)
+    :param bool use_bin_type:
+        Use bin type introduced in msgpack spec 2.0 for bytes.
+        It also enables str8 type for unicode. (default: True)
 
-    :param callable object_pairs_hook:
-        When specified, it should be callable.
-        Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
-        (See also simplejson)
+    :param bool strict_types:
+        If set to true, types will be checked to be exact. Derived classes
+        from serializable types will not be serialized and will be
+        treated as unsupported type and forwarded to default.
+        Additionally tuples will not be serialized as lists.
+        This is useful when trying to implement accurate serialization
+        for python types.
 
-    :param int max_buffer_size:
-        Limits size of data waiting unpacked.  0 means system's INT_MAX.
-        The default value is 100*1024*1024 (100MiB).
-        Raises `BufferFull` exception when it is insufficient.
-        You should set this parameter when unpacking data from untrusted source.
-
-    :param int max_str_len:
-        Deprecated, use *max_buffer_size* instead.
-        Limits max length of str. (default: max_buffer_size)
-
-    :param int max_bin_len:
-        Deprecated, use *max_buffer_size* instead.
-        Limits max length of bin. (default: max_buffer_size)
-
-    :param int max_array_len:
-        Limits max length of array. (default: max_buffer_size)
-
-    :param int max_map_len:
-        Limits max length of map. (default: max_buffer_size//2)
-
-    :param int max_ext_len:
-        Deprecated, use *max_buffer_size* instead.
-        Limits max size of ext type. (default: max_buffer_size)
+    :param bool datetime:
+        If set to true, datetime with tzinfo is packed into Timestamp type.
+        Note that the tzinfo is stripped in the timestamp.
+        You can get UTC datetime with `timestamp=3` option of the Unpacker.
+        (Python 2 is not supported).
 
     :param str unicode_errors:
-        Error handler used for decoding str type.  (default: `'strict'`)
-
+        The error handler for encoding unicode. (default: 'strict')
+        DO NOT USE THIS!!  This option is kept for very specific usage.
 
     Example of streaming deserialize from file-like object::
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 9739d53..0bfa94e 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -744,7 +744,7 @@ class Packer(object):
     """
     MessagePack Packer
 
-    Usage:
+    Usage::
 
         packer = Packer()
         astream.write(packer.pack(a))
@@ -784,6 +784,29 @@ class Packer(object):
     :param str unicode_errors:
         The error handler for encoding unicode. (default: 'strict')
         DO NOT USE THIS!!  This option is kept for very specific usage.
+
+    Example of streaming deserialize from file-like object::
+
+        unpacker = Unpacker(file_like)
+        for o in unpacker:
+            process(o)
+
+    Example of streaming deserialize from socket::
+
+        unpacker = Unpacker()
+        while True:
+            buf = sock.recv(1024**2)
+            if not buf:
+                break
+            unpacker.feed(buf)
+            for o in unpacker:
+                process(o)
+
+    Raises ``ExtraData`` when *packed* contains extra bytes.
+    Raises ``OutOfData`` when *packed* is incomplete.
+    Raises ``FormatError`` when *packed* is not valid msgpack.
+    Raises ``StackError`` when *packed* contains too nested.
+    Other exceptions can be raised during unpacking.
     """
 
     def __init__(

From 2df517999b012737b1cf36f12bac13541a8eb6e1 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 11 Dec 2020 13:39:24 +0900
Subject: [PATCH 1536/1648] Travis: Reduce build

Save credits.
---
 .travis.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 5132b4c..5980d63 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,20 +3,17 @@ dist: xenial
 language: python
 cache: pip
 arch:
-   - amd64
    - arm64
+
 python:
   # Available Python (PyPy) can be listed by:
   #
   #   $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/
-  - "3.4"
-  - "3.5"
   - "3.6"
   - "3.7"
   - "3.8"
   - "3.9-dev"
 
-
 _pure: &pure
   install:
     - pip install -U pip

From 7d6b4dfb516451735150d3f796f0b35ed5f004a4 Mon Sep 17 00:00:00 2001
From: Tsahi Zidenberg <65945052+tsahee@users.noreply.github.com>
Date: Fri, 11 Dec 2020 07:30:49 +0200
Subject: [PATCH 1537/1648] Build arm64 wheels (#439)

---
 .github/workflows/linux.yml | 10 ++++++++++
 Makefile                    |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 89bdb4e..50a0abe 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -30,6 +30,16 @@ jobs:
         run: |
           make linux-wheel
 
+      - name: Install qemu-user-static for docker
+        shell: bash
+        run: |
+          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      - name: Build arm64 wheels
+        shell: bash
+        run: |
+          make linux-arm64-wheel
+
       - name: Run test (3.8)
         run: |
           pip install pytest
diff --git a/Makefile b/Makefile
index 0110ddf..05cca55 100644
--- a/Makefile
+++ b/Makefile
@@ -38,3 +38,7 @@ update-docker:
 linux-wheel:
 	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_i686   bash docker/buildwheel.sh
 	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash docker/buildwheel.sh
+
+.PHONY: linux-arm64-wheel
+linux-arm64-wheel:
+	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_aarch64   bash docker/buildwheel.sh

From d893697eab07a2cf2a02f5115d4e7bae99d07e3f Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 11 Dec 2020 19:16:14 +0900
Subject: [PATCH 1538/1648] v1.0.1

---
 ChangeLog.rst       | 8 ++++++++
 msgpack/_version.py | 2 +-
 setup.py            | 1 +
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index d922e84..bb3a633 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,11 @@
+1.0.1
+=====
+
+* Add Python 3.9 and linux/arm64 wheels. (#439)
+* Fixed Unpacker.tell() after read_bytes() (#426)
+* Fixed unpacking datetime before epoch on Windows (#433)
+* Fixed fallback Packer didn't check DateTime.tzinfo (#434)
+
 1.0.0
 =====
 
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 9f55cf5..95e8129 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (1, 0, 0)
+version = (1, 0, 1)
diff --git a/setup.py b/setup.py
index 2ec9ca7..8e88750 100755
--- a/setup.py
+++ b/setup.py
@@ -132,6 +132,7 @@ setup(
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Intended Audience :: Developers",

From edd56036616e5f7211d5cd840a4d00bb43a7f9ee Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 11 Dec 2020 19:31:24 +0900
Subject: [PATCH 1539/1648] Actions: Add Python 3.9

---
 .github/workflows/linux.yml    | 11 +++++++++++
 .github/workflows/mac.yml      | 17 +++++++++++++++++
 .github/workflows/windows.yaml | 14 ++++++++++++++
 3 files changed, 42 insertions(+)

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 50a0abe..f9707b1 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -47,6 +47,17 @@ jobs:
           pytest -v test
 
 
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.9
+
+      - name: Run test (3.9)
+        run: |
+          pip install pytest
+          pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
+          pytest -v test
+
       - name: Set up Python 3.7
         uses: actions/setup-python@v1
         with:
diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
index fb2c67f..78d944c 100644
--- a/.github/workflows/mac.yml
+++ b/.github/workflows/mac.yml
@@ -35,6 +35,23 @@ jobs:
           pytest -v test
 
 
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.9"
+
+      - name: Build wheel
+        run: |
+          pip install setuptools wheel
+          python setup.py bdist_wheel
+
+      - name: Run test
+        run: |
+          pip install pytest
+          pip install -v msgpack --only-binary :all: -f dist/ --no-index
+          pytest -v test
+
+
       - name: Set up Python 3.7
         uses: actions/setup-python@v1
         with:
diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml
index cecb825..139a5a6 100644
--- a/.github/workflows/windows.yaml
+++ b/.github/workflows/windows.yaml
@@ -63,6 +63,20 @@ jobs:
         run: |
           ci/runtests.sh
 
+      - name: Python 3.9 (amd64)
+        env:
+          PYTHON: "py -3.9-64"
+        shell: bash
+        run: |
+          ci/runtests.sh
+
+      - name: Python 3.9 (x86)
+        env:
+          PYTHON: "py -3.9-32"
+        shell: bash
+        run: |
+          ci/runtests.sh
+
       - name: Upload Wheels
         uses: actions/upload-artifact@v1
         with:

From 8029f95516dbfddf2fea61efb06dc08ded84aab7 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 11 Dec 2020 19:33:20 +0900
Subject: [PATCH 1540/1648] Add Python 3.9 wheels

---
 docker/shared.env | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/shared.env b/docker/shared.env
index 17abdd8..3601a07 100644
--- a/docker/shared.env
+++ b/docker/shared.env
@@ -1,4 +1,5 @@
 PYTHON_VERSIONS=(
+    cp39-cp39
     cp38-cp38
     cp37-cp37m
     cp36-cp36m

From 753b3706d80a7bc5a29147730804e867b97eee57 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 18 Dec 2020 14:21:27 +0900
Subject: [PATCH 1541/1648] Fix overflow in unpacking timestamp to datetime
 (#452)

---
 msgpack/unpack.h       |  6 +++---
 test/test_timestamp.py | 11 +++++++++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 868b96e..34212bc 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -342,21 +342,21 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos
             py = PyObject_CallFunction(u->timestamp_t, "(Lk)", ts.tv_sec, ts.tv_nsec);
         }
         else if (u->timestamp == 3) {  // datetime
-            // Calculate datetime using epoch + delta 
+            // Calculate datetime using epoch + delta
             // due to limitations PyDateTime_FromTimestamp on Windows with negative timestamps
             PyObject *epoch = PyDateTimeAPI->DateTime_FromDateAndTime(1970, 1, 1, 0, 0, 0, 0, u->utc, PyDateTimeAPI->DateTimeType);
             if (epoch == NULL) {
                 return -1;
             }
 
-            PyObject* d = PyDelta_FromDSU(0, ts.tv_sec, ts.tv_nsec / 1000);
+            PyObject* d = PyDelta_FromDSU(ts.tv_sec/(24*3600), ts.tv_sec%(24*3600), ts.tv_nsec / 1000);
             if (d == NULL) {
                 Py_DECREF(epoch);
                 return -1;
             }
 
             py = PyNumber_Add(epoch, d);
-            
+
             Py_DECREF(epoch);
             Py_DECREF(d);
         }
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index edc488a..6a29be7 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -129,3 +129,14 @@ def test_pack_datetime():
     assert x
     assert x[0] == dt
     assert msgpack.unpackb(packed) is None
+
+
+@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
+def test_issue451():
+    # https://github.com/msgpack/msgpack-python/issues/451
+    dt = datetime.datetime(2100, 1, 1, 1, 1, tzinfo=_utc)
+    packed = msgpack.packb(dt, datetime=True)
+    assert packed == b"\xd6\xff\xf4\x86eL"
+
+    unpacked = msgpack.unpackb(packed, timestamp=3)
+    assert dt == unpacked

From 94336cf914c202718e37f27d664517c9a8c79d50 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 18 Dec 2020 16:03:05 +0900
Subject: [PATCH 1542/1648] Fix some travis builds. (#453)

---
 .travis.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 5980d63..4974d26 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,6 +25,7 @@ _pure: &pure
 matrix:
   include:
     - name: 32bit build
+      arch: amd64
       language: python
       services:
         - docker
@@ -58,10 +59,12 @@ matrix:
       <<: *pure
 
     - name: "pypy2.7"
+      arch: amd64
       python: "pypy2.7-7.1.1"
       <<: *pure
 
     - name: "pypy3"
+      arch: amd64
       python: "pypy3.6-7.1.1"
       <<: *pure
 

From 051f9ded1fe2343ffdf55e7a5ec2da1dab7638b9 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 18 Dec 2020 16:13:35 +0900
Subject: [PATCH 1543/1648] format markdown

---
 README.md | 126 +++++++++++++++++++++++++++---------------------------
 1 file changed, 63 insertions(+), 63 deletions(-)

diff --git a/README.md b/README.md
index ac52d94..2d5dd88 100644
--- a/README.md
+++ b/README.md
@@ -64,9 +64,9 @@ See note below for detail.
 
 ## Install
 
-
-   $ pip install msgpack
-
+```
+$ pip install msgpack
+```
 
 ### Pure Python implementation
 
@@ -103,18 +103,18 @@ msgpack provides `dumps` and `loads` as an alias for compatibility with
 `unpack` and `load` unpacks from a file-like object.
 
 ```pycon
-   >>> import msgpack
-   >>> msgpack.packb([1, 2, 3], use_bin_type=True)
-   '\x93\x01\x02\x03'
-   >>> msgpack.unpackb(_, raw=False)
-   [1, 2, 3]
+>>> import msgpack
+>>> msgpack.packb([1, 2, 3], use_bin_type=True)
+'\x93\x01\x02\x03'
+>>> msgpack.unpackb(_, raw=False)
+[1, 2, 3]
 ```
 
 `unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple:
 
 ```pycon
-   >>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False)
-   (1, 2, 3)
+>>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False)
+(1, 2, 3)
 ```
 
 You should always specify the `use_list` keyword argument for backward compatibility.
@@ -129,18 +129,18 @@ Read the docstring for other options.
 stream (or from bytes provided through its `feed` method).
 
 ```py
-   import msgpack
-   from io import BytesIO
+import msgpack
+from io import BytesIO
 
-   buf = BytesIO()
-   for i in range(100):
-      buf.write(msgpack.packb(i, use_bin_type=True))
+buf = BytesIO()
+for i in range(100):
+   buf.write(msgpack.packb(i, use_bin_type=True))
 
-   buf.seek(0)
+buf.seek(0)
 
-   unpacker = msgpack.Unpacker(buf, raw=False)
-   for unpacked in unpacker:
-       print(unpacked)
+unpacker = msgpack.Unpacker(buf, raw=False)
+for unpacked in unpacker:
+    print(unpacked)
 ```
 
 
@@ -150,27 +150,27 @@ It is also possible to pack/unpack custom data types. Here is an example for
 `datetime.datetime`.
 
 ```py
-    import datetime
-    import msgpack
+import datetime
+import msgpack
 
-    useful_dict = {
-        "id": 1,
-        "created": datetime.datetime.now(),
-    }
+useful_dict = {
+    "id": 1,
+    "created": datetime.datetime.now(),
+}
 
-    def decode_datetime(obj):
-        if '__datetime__' in obj:
-            obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f")
-        return obj
+def decode_datetime(obj):
+    if '__datetime__' in obj:
+        obj = datetime.datetime.strptime(obj["as_str"], "%Y%m%dT%H:%M:%S.%f")
+    return obj
 
-    def encode_datetime(obj):
-        if isinstance(obj, datetime.datetime):
-            return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")}
-        return obj
+def encode_datetime(obj):
+    if isinstance(obj, datetime.datetime):
+        return {'__datetime__': True, 'as_str': obj.strftime("%Y%m%dT%H:%M:%S.%f")}
+    return obj
 
 
-    packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True)
-    this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False)
+packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True)
+this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False)
 ```
 
 `Unpacker`'s `object_hook` callback receives a dict; the
@@ -183,25 +183,25 @@ key-value pairs.
 It is also possible to pack/unpack custom data types using the **ext** type.
 
 ```pycon
-    >>> import msgpack
-    >>> import array
-    >>> def default(obj):
-    ...     if isinstance(obj, array.array) and obj.typecode == 'd':
-    ...         return msgpack.ExtType(42, obj.tostring())
-    ...     raise TypeError("Unknown type: %r" % (obj,))
-    ...
-    >>> def ext_hook(code, data):
-    ...     if code == 42:
-    ...         a = array.array('d')
-    ...         a.fromstring(data)
-    ...         return a
-    ...     return ExtType(code, data)
-    ...
-    >>> data = array.array('d', [1.2, 3.4])
-    >>> packed = msgpack.packb(data, default=default, use_bin_type=True)
-    >>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False)
-    >>> data == unpacked
-    True
+>>> import msgpack
+>>> import array
+>>> def default(obj):
+...     if isinstance(obj, array.array) and obj.typecode == 'd':
+...         return msgpack.ExtType(42, obj.tostring())
+...     raise TypeError("Unknown type: %r" % (obj,))
+...
+>>> def ext_hook(code, data):
+...     if code == 42:
+...         a = array.array('d')
+...         a.fromstring(data)
+...         return a
+...     return ExtType(code, data)
+...
+>>> data = array.array('d', [1.2, 3.4])
+>>> packed = msgpack.packb(data, default=default, use_bin_type=True)
+>>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False)
+>>> data == unpacked
+True
 ```
 
 
@@ -226,11 +226,11 @@ You can pack into and unpack from this old spec using `use_bin_type=False`
 and `raw=True` options.
 
 ```pycon
-    >>> import msgpack
-    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True)
-    [b'spam', b'eggs']
-    >>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False)
-    [b'spam', 'eggs']
+>>> import msgpack
+>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True)
+[b'spam', b'eggs']
+>>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False)
+[b'spam', 'eggs']
 ```
 
 ### ext type
@@ -238,10 +238,10 @@ and `raw=True` options.
 To use the **ext** type, pass `msgpack.ExtType` object to packer.
 
 ```pycon
-    >>> import msgpack
-    >>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy'))
-    >>> msgpack.unpackb(packed)
-    ExtType(code=42, data='xyzzy')
+>>> import msgpack
+>>> packed = msgpack.packb(msgpack.ExtType(42, b'xyzzy'))
+>>> msgpack.unpackb(packed)
+ExtType(code=42, data='xyzzy')
 ```
 
 You can use it with `default` and `ext_hook`. See below.

From f34fca7fb55e2c8adde39c34ac48542649a24d11 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 18 Dec 2020 16:21:41 +0900
Subject: [PATCH 1544/1648] Update readme

---
 README.md | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 2d5dd88..d8ce9ba 100644
--- a/README.md
+++ b/README.md
@@ -15,15 +15,10 @@ This package provides CPython bindings for reading and writing MessagePack data.
 
 ### PyPI package name
 
-TL;DR: When upgrading from msgpack-0.4 or earlier, don't do `pip install -U msgpack-python`.
-Do `pip uninstall msgpack-python; pip install -U msgpack` instead.
+Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5.
 
-Package name on PyPI was changed to msgpack from 0.5.
-I upload transitional package (msgpack-python 0.5 which depending on msgpack)
-for smooth transition from msgpack-python to msgpack.
-
-Sadly, this doesn't work for upgrade install.  After `pip install -U msgpack-python`,
-msgpack is removed, and `import msgpack` fail.
+When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before
+`pip install -U msgpack`.
 
 
 ### Compatibility with the old format

From c0516c603f0eb6555117e312f5cdfb383853bc8e Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 18 Dec 2020 16:43:04 +0900
Subject: [PATCH 1545/1648] v1.0.2

---
 ChangeLog.rst       | 5 +++++
 msgpack/_version.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index bb3a633..230cc30 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,8 @@
+1.0.2
+=====
+
+* Fix year 2038 problem regression in 1.0.1. (#451)
+
 1.0.1
 =====
 
diff --git a/msgpack/_version.py b/msgpack/_version.py
index 95e8129..1c83c8e 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (1, 0, 1)
+version = (1, 0, 2)

From 431ef45c8ebff54c2b182dda46e7edb78a4e271b Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 18 Dec 2020 17:43:37 +0900
Subject: [PATCH 1546/1648] Use manylinux1 instead of manylinux2010

---
 Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 05cca55..6f29aed 100644
--- a/Makefile
+++ b/Makefile
@@ -31,13 +31,13 @@ clean:
 
 .PHONY: update-docker
 update-docker:
-	docker pull quay.io/pypa/manylinux2010_i686
-	docker pull quay.io/pypa/manylinux2010_x86_64
+	docker pull quay.io/pypa/manylinux1_i686
+	docker pull quay.io/pypa/manylinux1_x86_64
 
 .PHONY: linux-wheel
 linux-wheel:
-	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_i686   bash docker/buildwheel.sh
-	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash docker/buildwheel.sh
+	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686   bash docker/buildwheel.sh
+	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh
 
 .PHONY: linux-arm64-wheel
 linux-arm64-wheel:

From 3b71818bb0e4cbafdb58895fa47704563448b6ac Mon Sep 17 00:00:00 2001
From: Guy Tuval 
Date: Sat, 2 Jan 2021 08:39:37 +0200
Subject: [PATCH 1547/1648] Refactor fallback read header (#441)

---
 msgpack/fallback.py | 247 +++++++++++++++-----------------------------
 1 file changed, 82 insertions(+), 165 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 0bfa94e..7dbc67a 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -1,5 +1,4 @@
 """Fallback pure Python implementation of msgpack"""
-
 from datetime import datetime as _DateTime
 import sys
 import struct
@@ -148,6 +147,38 @@ if sys.version_info < (2, 7, 6):
 else:
     _unpack_from = struct.unpack_from
 
+_NO_FORMAT_USED = ""
+_MSGPACK_HEADERS = {
+    0xC4: (1, _NO_FORMAT_USED, TYPE_BIN),
+    0xC5: (2, ">H", TYPE_BIN),
+    0xC6: (4, ">I", TYPE_BIN),
+    0xC7: (2, "Bb", TYPE_EXT),
+    0xC8: (3, ">Hb", TYPE_EXT),
+    0xC9: (5, ">Ib", TYPE_EXT),
+    0xCA: (4, ">f"),
+    0xCB: (8, ">d"),
+    0xCC: (1, _NO_FORMAT_USED),
+    0xCD: (2, ">H"),
+    0xCE: (4, ">I"),
+    0xCF: (8, ">Q"),
+    0xD0: (1, "b"),
+    0xD1: (2, ">h"),
+    0xD2: (4, ">i"),
+    0xD3: (8, ">q"),
+    0xD4: (1, "b1s", TYPE_EXT),
+    0xD5: (2, "b2s", TYPE_EXT),
+    0xD6: (4, "b4s", TYPE_EXT),
+    0xD7: (8, "b8s", TYPE_EXT),
+    0xD8: (16, "b16s", TYPE_EXT),
+    0xD9: (1, _NO_FORMAT_USED, TYPE_RAW),
+    0xDA: (2, ">H", TYPE_RAW),
+    0xDB: (4, ">I", TYPE_RAW),
+    0xDC: (2, ">H", TYPE_ARRAY),
+    0xDD: (4, ">I", TYPE_ARRAY),
+    0xDE: (2, ">H", TYPE_MAP),
+    0xDF: (4, ">I", TYPE_MAP),
+}
+
 
 class Unpacker(object):
     """Streaming unpacker.
@@ -409,7 +440,7 @@ class Unpacker(object):
             self._buff_i = 0  # rollback
             raise OutOfData
 
-    def _read_header(self, execute=EX_CONSTRUCT):
+    def _read_header(self):
         typ = TYPE_IMMEDIATE
         n = 0
         obj = None
@@ -442,187 +473,73 @@ class Unpacker(object):
             obj = False
         elif b == 0xC3:
             obj = True
-        elif b == 0xC4:
-            typ = TYPE_BIN
-            self._reserve(1)
-            n = self._buffer[self._buff_i]
-            self._buff_i += 1
+        elif 0xC4 <= b <= 0xC6:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            if len(fmt) > 0:
+                n = _unpack_from(fmt, self._buffer, self._buff_i)[0]
+            else:
+                n = self._buffer[self._buff_i]
+            self._buff_i += size
             if n > self._max_bin_len:
                 raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
             obj = self._read(n)
-        elif b == 0xC5:
-            typ = TYPE_BIN
-            self._reserve(2)
-            n = _unpack_from(">H", self._buffer, self._buff_i)[0]
-            self._buff_i += 2
-            if n > self._max_bin_len:
-                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
-            obj = self._read(n)
-        elif b == 0xC6:
-            typ = TYPE_BIN
-            self._reserve(4)
-            n = _unpack_from(">I", self._buffer, self._buff_i)[0]
-            self._buff_i += 4
-            if n > self._max_bin_len:
-                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
-            obj = self._read(n)
-        elif b == 0xC7:  # ext 8
-            typ = TYPE_EXT
-            self._reserve(2)
-            L, n = _unpack_from("Bb", self._buffer, self._buff_i)
-            self._buff_i += 2
+        elif 0xC7 <= b <= 0xC9:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            L, n = _unpack_from(fmt, self._buffer, self._buff_i)
+            self._buff_i += size
             if L > self._max_ext_len:
                 raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
             obj = self._read(L)
-        elif b == 0xC8:  # ext 16
-            typ = TYPE_EXT
-            self._reserve(3)
-            L, n = _unpack_from(">Hb", self._buffer, self._buff_i)
-            self._buff_i += 3
-            if L > self._max_ext_len:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
-            obj = self._read(L)
-        elif b == 0xC9:  # ext 32
-            typ = TYPE_EXT
-            self._reserve(5)
-            L, n = _unpack_from(">Ib", self._buffer, self._buff_i)
-            self._buff_i += 5
-            if L > self._max_ext_len:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
-            obj = self._read(L)
-        elif b == 0xCA:
-            self._reserve(4)
-            obj = _unpack_from(">f", self._buffer, self._buff_i)[0]
-            self._buff_i += 4
-        elif b == 0xCB:
-            self._reserve(8)
-            obj = _unpack_from(">d", self._buffer, self._buff_i)[0]
-            self._buff_i += 8
-        elif b == 0xCC:
-            self._reserve(1)
-            obj = self._buffer[self._buff_i]
-            self._buff_i += 1
-        elif b == 0xCD:
-            self._reserve(2)
-            obj = _unpack_from(">H", self._buffer, self._buff_i)[0]
-            self._buff_i += 2
-        elif b == 0xCE:
-            self._reserve(4)
-            obj = _unpack_from(">I", self._buffer, self._buff_i)[0]
-            self._buff_i += 4
-        elif b == 0xCF:
-            self._reserve(8)
-            obj = _unpack_from(">Q", self._buffer, self._buff_i)[0]
-            self._buff_i += 8
-        elif b == 0xD0:
-            self._reserve(1)
-            obj = _unpack_from("b", self._buffer, self._buff_i)[0]
-            self._buff_i += 1
-        elif b == 0xD1:
-            self._reserve(2)
-            obj = _unpack_from(">h", self._buffer, self._buff_i)[0]
-            self._buff_i += 2
-        elif b == 0xD2:
-            self._reserve(4)
-            obj = _unpack_from(">i", self._buffer, self._buff_i)[0]
-            self._buff_i += 4
-        elif b == 0xD3:
-            self._reserve(8)
-            obj = _unpack_from(">q", self._buffer, self._buff_i)[0]
-            self._buff_i += 8
-        elif b == 0xD4:  # fixext 1
-            typ = TYPE_EXT
-            if self._max_ext_len < 1:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len))
-            self._reserve(2)
-            n, obj = _unpack_from("b1s", self._buffer, self._buff_i)
-            self._buff_i += 2
-        elif b == 0xD5:  # fixext 2
-            typ = TYPE_EXT
-            if self._max_ext_len < 2:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len))
-            self._reserve(3)
-            n, obj = _unpack_from("b2s", self._buffer, self._buff_i)
-            self._buff_i += 3
-        elif b == 0xD6:  # fixext 4
-            typ = TYPE_EXT
-            if self._max_ext_len < 4:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len))
-            self._reserve(5)
-            n, obj = _unpack_from("b4s", self._buffer, self._buff_i)
-            self._buff_i += 5
-        elif b == 0xD7:  # fixext 8
-            typ = TYPE_EXT
-            if self._max_ext_len < 8:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len))
-            self._reserve(9)
-            n, obj = _unpack_from("b8s", self._buffer, self._buff_i)
-            self._buff_i += 9
-        elif b == 0xD8:  # fixext 16
-            typ = TYPE_EXT
-            if self._max_ext_len < 16:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len))
-            self._reserve(17)
-            n, obj = _unpack_from("b16s", self._buffer, self._buff_i)
-            self._buff_i += 17
-        elif b == 0xD9:
-            typ = TYPE_RAW
-            self._reserve(1)
-            n = self._buffer[self._buff_i]
-            self._buff_i += 1
+        elif 0xCA <= b <= 0xD3:
+            size, fmt = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            if len(fmt) > 0:
+                obj = _unpack_from(fmt, self._buffer, self._buff_i)[0]
+            else:
+                obj = self._buffer[self._buff_i]
+            self._buff_i += size
+        elif 0xD4 <= b <= 0xD8:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            if self._max_ext_len < size:
+                raise ValueError(
+                    "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len)
+                )
+            self._reserve(size + 1)
+            n, obj = _unpack_from(fmt, self._buffer, self._buff_i)
+            self._buff_i += size + 1
+        elif 0xD9 <= b <= 0xDB:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            if len(fmt) > 0:
+                (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
+            else:
+                n = self._buffer[self._buff_i]
+            self._buff_i += size
             if n > self._max_str_len:
                 raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
             obj = self._read(n)
-        elif b == 0xDA:
-            typ = TYPE_RAW
-            self._reserve(2)
-            (n,) = _unpack_from(">H", self._buffer, self._buff_i)
-            self._buff_i += 2
-            if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
-            obj = self._read(n)
-        elif b == 0xDB:
-            typ = TYPE_RAW
-            self._reserve(4)
-            (n,) = _unpack_from(">I", self._buffer, self._buff_i)
-            self._buff_i += 4
-            if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
-            obj = self._read(n)
-        elif b == 0xDC:
-            typ = TYPE_ARRAY
-            self._reserve(2)
-            (n,) = _unpack_from(">H", self._buffer, self._buff_i)
-            self._buff_i += 2
+        elif 0xDC <= b <= 0xDD:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
+            self._buff_i += size
             if n > self._max_array_len:
                 raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
-        elif b == 0xDD:
-            typ = TYPE_ARRAY
-            self._reserve(4)
-            (n,) = _unpack_from(">I", self._buffer, self._buff_i)
-            self._buff_i += 4
-            if n > self._max_array_len:
-                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
-        elif b == 0xDE:
-            self._reserve(2)
-            (n,) = _unpack_from(">H", self._buffer, self._buff_i)
-            self._buff_i += 2
+        elif 0xDE <= b <= 0xDF:
+            size, fmt, typ = _MSGPACK_HEADERS[b]
+            self._reserve(size)
+            (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
+            self._buff_i += size
             if n > self._max_map_len:
                 raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
-            typ = TYPE_MAP
-        elif b == 0xDF:
-            self._reserve(4)
-            (n,) = _unpack_from(">I", self._buffer, self._buff_i)
-            self._buff_i += 4
-            if n > self._max_map_len:
-                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
-            typ = TYPE_MAP
         else:
             raise FormatError("Unknown header: 0x%x" % b)
         return typ, n, obj
 
     def _unpack(self, execute=EX_CONSTRUCT):
-        typ, n, obj = self._read_header(execute)
+        typ, n, obj = self._read_header()
 
         if execute == EX_READ_ARRAY_HEADER:
             if typ != TYPE_ARRAY:

From 02e1f7623cd8d0fcd4763d542fc60e2957ee2046 Mon Sep 17 00:00:00 2001
From: Alex Willmer 
Date: Wed, 27 Jan 2021 01:11:32 +0000
Subject: [PATCH 1548/1648] build: Create tox environments using a known Cython
 version (#408)

This change causes Tox to run the project's setup.py in a virtualenv
(default path is .tox/.package). The required version of Cython is
installed, rather than whatever version is installed system wide.
---
 pyproject.toml   | 8 ++++++++
 requirements.txt | 1 +
 tox.ini          | 1 +
 3 files changed, 10 insertions(+)
 create mode 100644 pyproject.toml

diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..a9eb8aa
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,8 @@
+[build-system]
+requires = [
+    # Also declared in requirements.txt, if updating here please also update
+    # there
+    "Cython~=0.29.13",
+    "setuptools >= 35.0.2",
+]
+build-backend = "setuptools.build_meta"
diff --git a/requirements.txt b/requirements.txt
index a2cce25..180fe85 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
+# Also declared in pyproject.toml, if updating here please also update there
 Cython~=0.29.13
diff --git a/tox.ini b/tox.ini
index 607b182..ace1ba9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,6 +5,7 @@ envlist =
     {pypy,pypy3}-pure,
     py27-x86,
     py34-x86,
+isolated_build = true
 
 [variants:pure]
 setenv=

From cfae52437b0d146f74a14d69cb706712c1fa2c95 Mon Sep 17 00:00:00 2001
From: laike9m 
Date: Wed, 27 Jan 2021 15:33:14 -0800
Subject: [PATCH 1549/1648] Updated readme about Python 2 support (#456)

---
 README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/README.md b/README.md
index d8ce9ba..cb81648 100644
--- a/README.md
+++ b/README.md
@@ -71,8 +71,6 @@ Python 2 and PyPy.
 But msgpack provides a pure Python implementation (`msgpack.fallback`)
 for PyPy and Python 2.
 
-Since the [pip](https://pip.pypa.io/) uses the pure Python implementation,
-Python 2 support will not be dropped in the foreseeable future.
 
 
 ### Windows

From 1e728a2e0b7f263a4c77d6cdb6ec7c7e2a91872f Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 12 Feb 2021 16:20:14 +0900
Subject: [PATCH 1550/1648] fix docstring (#459)

---
 msgpack/_unpacker.pyx | 91 ++++++++++++++++++++++++++++---------------
 msgpack/fallback.py   |  2 +-
 2 files changed, 60 insertions(+), 33 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index e4f3f1e..7bfc3af 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -212,49 +212,76 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None,
 
 
 cdef class Unpacker(object):
-    """
-    MessagePack Packer
+    """Streaming unpacker.
 
-    Usage::
+    Arguments:
 
-        packer = Packer()
-        astream.write(packer.pack(a))
-        astream.write(packer.pack(b))
+    :param file_like:
+        File-like object having `.read(n)` method.
+        If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
 
-    Packer's constructor has some keyword arguments:
+    :param int read_size:
+        Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
 
-    :param callable default:
-        Convert user type to builtin type that Packer supports.
-        See also simplejson's document.
+    :param bool use_list:
+        If true, unpack msgpack array to Python list.
+        Otherwise, unpack to Python tuple. (default: True)
 
-    :param bool use_single_float:
-        Use single precision float type for float. (default: False)
+    :param bool raw:
+        If true, unpack msgpack raw to Python bytes.
+        Otherwise, unpack to Python str by decoding with UTF-8 encoding (default).
 
-    :param bool autoreset:
-        Reset buffer after each pack and return its content as `bytes`. (default: True).
-        If set this to false, use `bytes()` to get content and `.reset()` to clear buffer.
+    :param int timestamp:
+        Control how timestamp type is unpacked:
 
-    :param bool use_bin_type:
-        Use bin type introduced in msgpack spec 2.0 for bytes.
-        It also enables str8 type for unicode. (default: True)
+            0 - Timestamp
+            1 - float  (Seconds from the EPOCH)
+            2 - int  (Nanoseconds from the EPOCH)
+            3 - datetime.datetime  (UTC).  Python 2 is not supported.
 
-    :param bool strict_types:
-        If set to true, types will be checked to be exact. Derived classes
-        from serializable types will not be serialized and will be
-        treated as unsupported type and forwarded to default.
-        Additionally tuples will not be serialized as lists.
-        This is useful when trying to implement accurate serialization
-        for python types.
+    :param bool strict_map_key:
+        If true (default), only str or bytes are accepted for map (dict) keys.
 
-    :param bool datetime:
-        If set to true, datetime with tzinfo is packed into Timestamp type.
-        Note that the tzinfo is stripped in the timestamp.
-        You can get UTC datetime with `timestamp=3` option of the Unpacker.
-        (Python 2 is not supported).
+    :param callable object_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a dict argument after unpacking msgpack map.
+        (See also simplejson)
+
+    :param callable object_pairs_hook:
+        When specified, it should be callable.
+        Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
+        (See also simplejson)
 
     :param str unicode_errors:
-        The error handler for encoding unicode. (default: 'strict')
-        DO NOT USE THIS!!  This option is kept for very specific usage.
+        The error handler for decoding unicode. (default: 'strict')
+        This option should be used only when you have msgpack data which
+        contains invalid UTF-8 string.
+
+    :param int max_buffer_size:
+        Limits size of data waiting unpacked.  0 means 2**32-1.
+        The default value is 100*1024*1024 (100MiB).
+        Raises `BufferFull` exception when it is insufficient.
+        You should set this parameter when unpacking data from untrusted source.
+
+    :param int max_str_len:
+        Deprecated, use *max_buffer_size* instead.
+        Limits max length of str. (default: max_buffer_size)
+
+    :param int max_bin_len:
+        Deprecated, use *max_buffer_size* instead.
+        Limits max length of bin. (default: max_buffer_size)
+
+    :param int max_array_len:
+        Limits max length of array.
+        (default: max_buffer_size)
+
+    :param int max_map_len:
+        Limits max length of map.
+        (default: max_buffer_size//2)
+
+    :param int max_ext_len:
+        Deprecated, use *max_buffer_size* instead.
+        Limits max size of ext type.  (default: max_buffer_size)
 
     Example of streaming deserialize from file-like object::
 
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 7dbc67a..4bcc05d 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -260,7 +260,7 @@ class Unpacker(object):
 
     Example of streaming deserialize from socket::
 
-        unpacker = Unpacker(max_buffer_size)
+        unpacker = Unpacker()
         while True:
             buf = sock.recv(1024**2)
             if not buf:

From 4b0819dca941ba3fd27ea127e835698c33705365 Mon Sep 17 00:00:00 2001
From: Andrey Bienkowski 
Date: Tue, 16 Feb 2021 13:38:06 +0000
Subject: [PATCH 1551/1648] Remove redundant code (#460)

---
 test/test_unpack.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/test/test_unpack.py b/test/test_unpack.py
index 057b7bf..aa4c01f 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -90,10 +90,3 @@ def test_unpacker_tell_read_bytes():
         assert obj == unp
         assert pos == unpacker.tell()
         assert unpacker.read_bytes(n) == raw
-
-
-if __name__ == "__main__":
-    test_unpack_array_header_from_file()
-    test_unpacker_hook_refcnt()
-    test_unpacker_ext_hook()
-    test_unpacker_tell()

From 38357b928a2452d0889d80f0a2a721fa66f94c9a Mon Sep 17 00:00:00 2001
From: Andrey Bienkowski 
Date: Fri, 26 Feb 2021 02:39:36 +0000
Subject: [PATCH 1552/1648] Fix error formatting (#463)

---
 msgpack/fallback.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 4bcc05d..898fe14 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -455,18 +455,20 @@ class Unpacker(object):
             n = b & 0b00011111
             typ = TYPE_RAW
             if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len))
             obj = self._read(n)
         elif b & 0b11110000 == 0b10010000:
             n = b & 0b00001111
             typ = TYPE_ARRAY
             if n > self._max_array_len:
-                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+                raise ValueError(
+                    "%s exceeds max_array_len(%s)" % (n, self._max_array_len)
+                )
         elif b & 0b11110000 == 0b10000000:
             n = b & 0b00001111
             typ = TYPE_MAP
             if n > self._max_map_len:
-                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+                raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len))
         elif b == 0xC0:
             obj = None
         elif b == 0xC2:
@@ -518,7 +520,7 @@ class Unpacker(object):
                 n = self._buffer[self._buff_i]
             self._buff_i += size
             if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len)
+                raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len))
             obj = self._read(n)
         elif 0xDC <= b <= 0xDD:
             size, fmt, typ = _MSGPACK_HEADERS[b]
@@ -526,14 +528,16 @@ class Unpacker(object):
             (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
             self._buff_i += size
             if n > self._max_array_len:
-                raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len)
+                raise ValueError(
+                    "%s exceeds max_array_len(%s)" % (n, self._max_array_len)
+                )
         elif 0xDE <= b <= 0xDF:
             size, fmt, typ = _MSGPACK_HEADERS[b]
             self._reserve(size)
             (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
             self._buff_i += size
             if n > self._max_map_len:
-                raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len)
+                raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len))
         else:
             raise FormatError("Unknown header: 0x%x" % b)
         return typ, n, obj

From 4ace82f1087ffa1ca3c44a27c7dd3338739efd0a Mon Sep 17 00:00:00 2001
From: Andrey Bienkowski 
Date: Fri, 26 Feb 2021 12:08:06 +0000
Subject: [PATCH 1553/1648] Fix tox.ini (#465)

There is no such thing as [variants] in the tox syntax. This resulted
in MSGPACK_PUREPYTHON being unset in the "pure" test environments
---
 tox.ini | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index ace1ba9..29c256d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,10 +7,6 @@ envlist =
     py34-x86,
 isolated_build = true
 
-[variants:pure]
-setenv=
-    MSGPACK_PUREPYTHON=x
-
 [testenv]
 deps=
     pytest
@@ -20,6 +16,8 @@ commands=
     c,x86: python -c 'from msgpack import _cmsgpack'
     c,x86: py.test
     pure: py.test
+setenv=
+    pure: MSGPACK_PUREPYTHON=x
 
 [testenv:py27-x86]
 basepython=python2.7-x86

From 44fd5777050c6583791609d3f77e05427bf878a3 Mon Sep 17 00:00:00 2001
From: Alexander Shadchin 
Date: Sat, 27 Feb 2021 03:30:46 +0300
Subject: [PATCH 1554/1648] Remove unused PyObject_AsReadBuffer definition
 (#468)

Also "old" buffer API was removed in Python 3.10
---
 msgpack/_unpacker.pyx | 1 -
 1 file changed, 1 deletion(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 7bfc3af..9ecfdfb 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -3,7 +3,6 @@
 from cpython cimport *
 cdef extern from "Python.h":
     ctypedef struct PyObject
-    cdef int PyObject_AsReadBuffer(object o, const void** buff, Py_ssize_t* buf_len) except -1
     object PyMemoryView_GetContiguous(object obj, int buffertype, char order)
 
 from libc.stdlib cimport *

From 010de11bedde8d61ced5a382b44a8344d571c305 Mon Sep 17 00:00:00 2001
From: Andrey Bienkowski 
Date: Sat, 27 Feb 2021 01:50:24 +0000
Subject: [PATCH 1555/1648] Make pure-python wheels and eggs possible (#467)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 8e88750..751abff 100755
--- a/setup.py
+++ b/setup.py
@@ -88,7 +88,7 @@ else:
     macros = [("__LITTLE_ENDIAN__", "1")]
 
 ext_modules = []
-if not PYPY and not PY2:
+if not PYPY and not PY2 and not os.environ.get("MSGPACK_PUREPYTHON"):
     ext_modules.append(
         Extension(
             "msgpack._cmsgpack",

From 38dba9634e4efa7886a777b9e7c739dc148da457 Mon Sep 17 00:00:00 2001
From: Vladimir Matveev 
Date: Thu, 18 Mar 2021 14:35:54 -0700
Subject: [PATCH 1556/1648] cimport uint64_t instead of using ctypedef (#473)

---
 msgpack/_unpacker.pyx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 9ecfdfb..27facc0 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -8,7 +8,7 @@ cdef extern from "Python.h":
 from libc.stdlib cimport *
 from libc.string cimport *
 from libc.limits cimport *
-ctypedef unsigned long long uint64_t
+from libc.stdint cimport uint64_t
 
 from .exceptions import (
     BufferFull,

From 09187421eb0ac7061de83c9c112d738aa1edd2cf Mon Sep 17 00:00:00 2001
From: Paul Melis 
Date: Tue, 16 Nov 2021 06:47:40 +0100
Subject: [PATCH 1557/1648] Improve exception message relating to
 strict_map_key (#485)

---
 msgpack/unpack.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 34212bc..23aa622 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -193,7 +193,7 @@ static inline int unpack_callback_map(unpack_user* u, unsigned int n, msgpack_un
 static inline int unpack_callback_map_item(unpack_user* u, unsigned int current, msgpack_unpack_object* c, msgpack_unpack_object k, msgpack_unpack_object v)
 {
     if (u->strict_map_key && !PyUnicode_CheckExact(k) && !PyBytes_CheckExact(k)) {
-        PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key", Py_TYPE(k)->tp_name);
+        PyErr_Format(PyExc_ValueError, "%.100s is not allowed for map key when strict_map_key=True", Py_TYPE(k)->tp_name);
         return -1;
     }
     if (PyUnicode_CheckExact(k)) {

From 9b84e490e7c78ac9bbd76dcf9ce71c1d0c978d81 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 16 Nov 2021 14:53:08 +0900
Subject: [PATCH 1558/1648] Fix black formatting

---
 msgpack/fallback.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 898fe14..4540875 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -385,7 +385,7 @@ class Unpacker(object):
         self._buffer.extend(view)
 
     def _consume(self):
-        """ Gets rid of the used parts of the buffer. """
+        """Gets rid of the used parts of the buffer."""
         self._stream_offset += self._buff_i - self._buf_checkpoint
         self._buf_checkpoint = self._buff_i
 

From b3f72541925cda3b018942db45a59936184727e3 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 16 Nov 2021 16:19:47 +0900
Subject: [PATCH 1559/1648] Support Python 3.10 and Drop Python 3.5 (#487)

* linux: Use manylinux2014
* mac: Drop Python 3.6 too
---
 .github/workflows/linux.yml    | 11 +++++++++++
 .github/workflows/mac.yml      | 34 +++++++++++++++++-----------------
 .github/workflows/windows.yaml | 14 ++++++++++++++
 Makefile                       |  4 ++--
 docker/buildwheel.sh           |  4 +++-
 docker/shared.env              |  2 +-
 setup.py                       |  4 +---
 7 files changed, 49 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index f9707b1..811bc13 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -47,6 +47,17 @@ jobs:
           pytest -v test
 
 
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.10"
+
+      - name: Run test (3.10)
+        run: |
+          pip install pytest
+          pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
+          pytest -v test
+
       - name: Set up Python 3.9
         uses: actions/setup-python@v1
         with:
diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
index 78d944c..4efe2ca 100644
--- a/.github/workflows/mac.yml
+++ b/.github/workflows/mac.yml
@@ -35,6 +35,23 @@ jobs:
           pytest -v test
 
 
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.10"
+
+      - name: Build wheel
+        run: |
+          pip install setuptools wheel
+          python setup.py bdist_wheel
+
+      - name: Run test
+        run: |
+          pip install pytest
+          pip install -v msgpack --only-binary :all: -f dist/ --no-index
+          pytest -v test
+
+
       - name: Set up Python 3.9
         uses: actions/setup-python@v1
         with:
@@ -69,23 +86,6 @@ jobs:
           pytest -v test
 
 
-      - name: Set up Python 3.6
-        uses: actions/setup-python@v1
-        with:
-          python-version: "3.6"
-
-      - name: Build wheel
-        run: |
-          pip install setuptools wheel
-          python setup.py bdist_wheel
-
-      - name: Run test
-        run: |
-          pip install pytest
-          pip install -v msgpack --only-binary :all: -f dist/ --no-index
-          pytest -v test
-
-
       - name: Upload Wheels
         uses: actions/upload-artifact@v1
         with:
diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml
index 139a5a6..debe074 100644
--- a/.github/workflows/windows.yaml
+++ b/.github/workflows/windows.yaml
@@ -77,6 +77,20 @@ jobs:
         run: |
           ci/runtests.sh
 
+      - name: Python 3.10 (amd64)
+        env:
+          PYTHON: "py -3.10-64"
+        shell: bash
+        run: |
+          ci/runtests.sh
+
+      - name: Python 3.10 (x86)
+        env:
+          PYTHON: "py -3.10-32"
+        shell: bash
+        run: |
+          ci/runtests.sh
+
       - name: Upload Wheels
         uses: actions/upload-artifact@v1
         with:
diff --git a/Makefile b/Makefile
index 6f29aed..b50fa80 100644
--- a/Makefile
+++ b/Makefile
@@ -36,8 +36,8 @@ update-docker:
 
 .PHONY: linux-wheel
 linux-wheel:
-	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_i686   bash docker/buildwheel.sh
-	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux1_x86_64 bash docker/buildwheel.sh
+	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_i686   bash docker/buildwheel.sh
+	docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_x86_64 bash docker/buildwheel.sh
 
 .PHONY: linux-arm64-wheel
 linux-arm64-wheel:
diff --git a/docker/buildwheel.sh b/docker/buildwheel.sh
index 89a2570..ff34139 100644
--- a/docker/buildwheel.sh
+++ b/docker/buildwheel.sh
@@ -7,10 +7,12 @@ set -e -x
 ARCH=`uname -p`
 echo "arch=$ARCH"
 
+ls /opt/python
+
 for V in "${PYTHON_VERSIONS[@]}"; do
     PYBIN=/opt/python/$V/bin
     rm -rf build/       # Avoid lib build by narrow Python is used by wide python
-    $PYBIN/python setup.py bdist_wheel
+    $PYBIN/python -m build -w
 done
 
 cd dist
diff --git a/docker/shared.env b/docker/shared.env
index 3601a07..80274ac 100644
--- a/docker/shared.env
+++ b/docker/shared.env
@@ -1,7 +1,7 @@
 PYTHON_VERSIONS=(
+    cp310-cp310
     cp39-cp39
     cp38-cp38
     cp37-cp37m
     cp36-cp36m
-    cp35-cp35m
 )
diff --git a/setup.py b/setup.py
index 751abff..01f125f 100755
--- a/setup.py
+++ b/setup.py
@@ -125,14 +125,12 @@ setup(
     },
     license="Apache 2.0",
     classifiers=[
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
         "Intended Audience :: Developers",

From 8e358617e77d63a391f0c0f91b3e552214f2a49a Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 16 Nov 2021 17:42:42 +0900
Subject: [PATCH 1560/1648] mac: Provide Universal2 wheel (#488)

* mac: Use cibuildwheel
* Do not build wheel for PyPy.
---
 .github/workflows/mac.yml | 51 +++++++++++++++------------------------
 1 file changed, 19 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
index 4efe2ca..85844e7 100644
--- a/.github/workflows/mac.yml
+++ b/.github/workflows/mac.yml
@@ -1,7 +1,6 @@
 name: Build macOS Wheels
 on:
   push:
-  pull_request:
   create:
 
 jobs:
@@ -12,10 +11,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v1
 
-      - name: Set up Python 3.8
+      # Python 3.9
+      - name: Set up Python 3.9
         uses: actions/setup-python@v1
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Cythonize
         run: |
@@ -23,66 +23,53 @@ jobs:
           pip install -r requirements.txt
           make cython
 
-      - name: Build wheel
-        run: |
-          pip install setuptools wheel
-          python setup.py bdist_wheel
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.2.2
+        env:
+          CIBW_ARCHS_MACOS: x86_64 universal2
+          CIBW_SKIP: pp*
 
       - name: Run test
         run: |
+          ls wheelhouse/
           pip install pytest
-          pip install -v msgpack --only-binary :all: -f dist/ --no-index
+          pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index
           pytest -v test
 
-
+      # Python 3.10
       - name: Set up Python 3.10
         uses: actions/setup-python@v1
         with:
           python-version: "3.10"
 
-      - name: Build wheel
-        run: |
-          pip install setuptools wheel
-          python setup.py bdist_wheel
-
       - name: Run test
         run: |
           pip install pytest
-          pip install -v msgpack --only-binary :all: -f dist/ --no-index
+          pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index
           pytest -v test
 
-
-      - name: Set up Python 3.9
+      # Python 3.8
+      - name: Set up Python 3.8
         uses: actions/setup-python@v1
         with:
-          python-version: "3.9"
-
-      - name: Build wheel
-        run: |
-          pip install setuptools wheel
-          python setup.py bdist_wheel
+          python-version: "3.8"
 
       - name: Run test
         run: |
           pip install pytest
-          pip install -v msgpack --only-binary :all: -f dist/ --no-index
+          pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index
           pytest -v test
 
-
+      # Python 3.7
       - name: Set up Python 3.7
         uses: actions/setup-python@v1
         with:
           python-version: "3.7"
 
-      - name: Build wheel
-        run: |
-          pip install setuptools wheel
-          python setup.py bdist_wheel
-
       - name: Run test
         run: |
           pip install pytest
-          pip install -v msgpack --only-binary :all: -f dist/ --no-index
+          pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index
           pytest -v test
 
 
@@ -90,4 +77,4 @@ jobs:
         uses: actions/upload-artifact@v1
         with:
           name: macos-wheels
-          path: ./dist/
+          path: ./wheelhouse/

From cfa05d3fdc6290b4847e4781a06ac0668ea9dc18 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 16 Nov 2021 17:47:16 +0900
Subject: [PATCH 1561/1648] Actions: Run CI only for PRs from forks. (#489)

---
 .github/workflows/black.yaml   | 4 ++++
 .github/workflows/linux.yml    | 4 ++++
 .github/workflows/windows.yaml | 7 ++++---
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml
index eda8d07..c545953 100644
--- a/.github/workflows/black.yaml
+++ b/.github/workflows/black.yaml
@@ -4,6 +4,10 @@ on: ["push", "pull_request"]
 
 jobs:
   black:
+    # We want to run on external PRs, but not on our own internal PRs as they'll be run
+    # by the push to the branch.
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+
     runs-on: ubuntu-latest
     steps:
       - name: Setup Python
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 811bc13..95c672b 100644
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -6,6 +6,10 @@ on:
 
 jobs:
   build:
+    # We want to run on external PRs, but not on our own internal PRs as they'll be run
+    # by the push to the branch.
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml
index debe074..0ce50f5 100644
--- a/.github/workflows/windows.yaml
+++ b/.github/workflows/windows.yaml
@@ -1,14 +1,15 @@
 name: Build and test windows wheels
 on:
   push:
-    branches:
-      - master
-      - test
   pull_request:
   create:
 
 jobs:
   build:
+    # We want to run on external PRs, but not on our own internal PRs as they'll be run
+    # by the push to the branch.
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+
     runs-on: windows-latest
     steps:
       - name: Checkout

From e464cb44fa3af5ad3ecd83f9c045b16981d01bb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Benjamin=20Egelund-M=C3=BCller?= 
Date: Tue, 16 Nov 2021 09:49:47 +0100
Subject: [PATCH 1562/1648] Nicer error when packing a datetime without tzinfo
 (#466)

---
 msgpack/_packer.pyx    |  2 ++
 msgpack/fallback.py    |  4 ++++
 test/test_timestamp.py | 16 ++++++++++++++++
 3 files changed, 22 insertions(+)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index e6cd2c7..396da0c 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -285,6 +285,8 @@ cdef class Packer(object):
                 o = self._default(o)
                 default_used = 1
                 continue
+            elif self.datetime and PyDateTime_CheckExact(o):
+                PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name)
             else:
                 PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name)
             return ret
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 4540875..b27acb2 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -874,6 +874,10 @@ class Packer(object):
                 obj = self._default(obj)
                 default_used = 1
                 continue
+
+            if self._datetime and check(obj, _DateTime):
+                raise ValueError("Cannot serialize %r where tzinfo=None" % (obj,))
+
             raise TypeError("Cannot serialize %r" % (obj,))
 
     def pack(self, obj):
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index 6a29be7..4e26489 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -140,3 +140,19 @@ def test_issue451():
 
     unpacked = msgpack.unpackb(packed, timestamp=3)
     assert dt == unpacked
+
+
+@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
+def test_pack_datetime_without_tzinfo():
+    dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14)
+    with pytest.raises(ValueError, match="where tzinfo=None"):
+        packed = msgpack.packb(dt, datetime=True)
+
+    dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14)
+    packed = msgpack.packb(dt, datetime=True, default=lambda x: None)
+    assert packed == msgpack.packb(None)
+
+    dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc)
+    packed = msgpack.packb(dt, datetime=True)
+    unpacked = msgpack.unpackb(packed, timestamp=3)
+    assert unpacked == dt

From 724e6200fd6b28b3562e48c73f7827a9c19dd11f Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 16 Nov 2021 17:52:01 +0900
Subject: [PATCH 1563/1648] 1.0.3rc1

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index 1c83c8e..3b462fa 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (1, 0, 2)
+version = (1, 0, 3, 'rc1')

From e29b423de71cb6da323bce8742b1328603a7f9be Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 17 Nov 2021 11:03:06 +0900
Subject: [PATCH 1564/1648] black

---
 msgpack/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/_version.py b/msgpack/_version.py
index 3b462fa..b716524 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (1, 0, 3, 'rc1')
+version = (1, 0, 3, "rc1")

From 6129789e9f6ebccdb19b23c2cd1dc9551e57fc53 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 24 Nov 2021 16:18:17 +0900
Subject: [PATCH 1565/1648] Release v1.0.3 (#491)

---
 ChangeLog.rst       | 9 +++++++++
 msgpack/_version.py | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 230cc30..fc6df68 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,12 @@
+1.0.3
+=====
+
+Release Date: 2021-11-24 JST
+
+* Fix Docstring (#459)
+* Fix error formatting (#463)
+* Improve error message about strict_map_key (#485)
+
 1.0.2
 =====
 
diff --git a/msgpack/_version.py b/msgpack/_version.py
index b716524..fb878b3 100644
--- a/msgpack/_version.py
+++ b/msgpack/_version.py
@@ -1 +1 @@
-version = (1, 0, 3, "rc1")
+version = (1, 0, 3)

From bdf0511e29b02427437b03e7d8454bd5076e837f Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 25 Nov 2021 14:43:55 +0900
Subject: [PATCH 1566/1648] Refactor CI (#492)

* Use cibuildwheel to build wheels.
* Use matrix
---
 .github/workflows/black.yaml   |   2 +-
 .github/workflows/linux.yml    | 103 ---------------------------------
 .github/workflows/mac.yml      |  80 -------------------------
 .github/workflows/test.yml     |  45 ++++++++++++++
 .github/workflows/wheel.yml    |  51 ++++++++++++++++
 .github/workflows/windows.yaml |  99 -------------------------------
 .travis.yml                    |  89 ----------------------------
 Makefile                       |   5 +-
 appveyor.yml                   |  50 ----------------
 ci/runtests.bat                |   9 ---
 ci/runtests.sh                 |   8 ---
 11 files changed, 100 insertions(+), 441 deletions(-)
 delete mode 100644 .github/workflows/linux.yml
 delete mode 100644 .github/workflows/mac.yml
 create mode 100644 .github/workflows/test.yml
 create mode 100644 .github/workflows/wheel.yml
 delete mode 100644 .github/workflows/windows.yaml
 delete mode 100644 .travis.yml
 delete mode 100644 appveyor.yml
 delete mode 100644 ci/runtests.bat
 delete mode 100644 ci/runtests.sh

diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml
index c545953..2961ed3 100644
--- a/.github/workflows/black.yaml
+++ b/.github/workflows/black.yaml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: '3.x'
           architecture: 'x64'
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
deleted file mode 100644
index 95c672b..0000000
--- a/.github/workflows/linux.yml
+++ /dev/null
@@ -1,103 +0,0 @@
-name: Build Linux Wheels
-on:
-  push:
-  pull_request:
-  create:
-
-jobs:
-  build:
-    # We want to run on external PRs, but not on our own internal PRs as they'll be run
-    # by the push to the branch.
-    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
-
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
-
-      - name: Cythonize
-        shell: bash
-        run: |
-          pip install -U pip
-          pip -V
-          pip install -r requirements.txt
-          make cython
-          #python setup.py sdist
-
-      - name: Build wheels
-        shell: bash
-        run: |
-          make linux-wheel
-
-      - name: Install qemu-user-static for docker
-        shell: bash
-        run: |
-          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
-      - name: Build arm64 wheels
-        shell: bash
-        run: |
-          make linux-arm64-wheel
-
-      - name: Run test (3.8)
-        run: |
-          pip install pytest
-          pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
-          pytest -v test
-
-
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v1
-        with:
-          python-version: "3.10"
-
-      - name: Run test (3.10)
-        run: |
-          pip install pytest
-          pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
-          pytest -v test
-
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.9
-
-      - name: Run test (3.9)
-        run: |
-          pip install pytest
-          pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
-          pytest -v test
-
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-
-      - name: Run test (3.7)
-        run: |
-          pip install pytest
-          pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
-          pytest -v test
-
-      - name: Set up Python 3.6
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.6
-
-      - name: Run test (3.6)
-        run: |
-          pip install pytest
-          pip install -v msgpack --only-binary :all: --no-index -f dist/wheelhouse
-          pytest -v test
-
-
-      - name: Upload Wheels
-        uses: actions/upload-artifact@v1
-        with:
-          name: linux-wheels
-          path: ./dist/wheelhouse/
diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
deleted file mode 100644
index 85844e7..0000000
--- a/.github/workflows/mac.yml
+++ /dev/null
@@ -1,80 +0,0 @@
-name: Build macOS Wheels
-on:
-  push:
-  create:
-
-jobs:
-  build:
-    runs-on: macos-latest
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-
-      # Python 3.9
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v1
-        with:
-          python-version: "3.9"
-
-      - name: Cythonize
-        run: |
-          pip install -U pip
-          pip install -r requirements.txt
-          make cython
-
-      - name: Build wheels
-        uses: pypa/cibuildwheel@v2.2.2
-        env:
-          CIBW_ARCHS_MACOS: x86_64 universal2
-          CIBW_SKIP: pp*
-
-      - name: Run test
-        run: |
-          ls wheelhouse/
-          pip install pytest
-          pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index
-          pytest -v test
-
-      # Python 3.10
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v1
-        with:
-          python-version: "3.10"
-
-      - name: Run test
-        run: |
-          pip install pytest
-          pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index
-          pytest -v test
-
-      # Python 3.8
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v1
-        with:
-          python-version: "3.8"
-
-      - name: Run test
-        run: |
-          pip install pytest
-          pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index
-          pytest -v test
-
-      # Python 3.7
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: "3.7"
-
-      - name: Run test
-        run: |
-          pip install pytest
-          pip install -v msgpack --only-binary :all: -f wheelhouse/ --no-index
-          pytest -v test
-
-
-      - name: Upload Wheels
-        uses: actions/upload-artifact@v1
-        with:
-          name: macos-wheels
-          path: ./wheelhouse/
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 0000000..6e497e0
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,45 @@
+name: Run tests
+on:
+  push:
+    branches: [main]
+  pull_request:
+  create:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, windows-2022, macos-10.15]
+        py: ["3.10", "3.9", "3.8", "3.7", "3.6"]
+
+    runs-on: ${{ matrix.os }}
+    name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.py }}
+          cache: "pip"
+
+      - name: Build
+        shell: bash
+        run: |
+          pip install -U pip
+          pip install -r requirements.txt pytest
+          make cython
+          pip install .
+
+      - name: Test (C extension)
+        shell: bash
+        run: |
+          pytest -v test
+
+      - name: Test (pure Python fallback)
+        shell: bash
+        run: |
+          MSGPACK_PUREPYTHON=1 pytest -v test
+
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
new file mode 100644
index 0000000..5627b5f
--- /dev/null
+++ b/.github/workflows/wheel.yml
@@ -0,0 +1,51 @@
+name: Build Wheels
+on:
+  push:
+    branches: [main]
+  create:
+
+jobs:
+  build_wheels:
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, windows-2022, macos-10.15]
+    runs-on: ${{ matrix.os }}
+    name: Build wheels on ${{ matrix.os }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+          cache: "pip"
+
+      - name: Prepare
+        shell: bash
+        run: |
+          pip install -r requirements.txt
+          make cython
+
+      - name: Build
+        uses: pypa/cibuildwheel@v2.2.2
+        env:
+          CIBW_TEST_REQUIRES: "pytest"
+          CIBW_TEST_COMMAND: "pytest {package}/test"
+          CIBW_ARCHS_LINUX: auto aarch64
+          CIBW_ARCHS_MACOS: x86_64 universal2 arm64
+          CIBW_SKIP: pp*
+
+      - name: Upload Wheels
+        uses: actions/upload-artifact@v1
+        with:
+          name: Wheels
+          path: wheelhouse
+
diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml
deleted file mode 100644
index 0ce50f5..0000000
--- a/.github/workflows/windows.yaml
+++ /dev/null
@@ -1,99 +0,0 @@
-name: Build and test windows wheels
-on:
-  push:
-  pull_request:
-  create:
-
-jobs:
-  build:
-    # We want to run on external PRs, but not on our own internal PRs as they'll be run
-    # by the push to the branch.
-    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
-
-    runs-on: windows-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-
-      - name: Cythonize
-        shell: bash
-        run: |
-          pip install -U Cython
-          make cython
-          #python setup.py sdist
-
-      - name: Python 3.6 (amd64)
-        env:
-          PYTHON: "py -3.6-64"
-        shell: bash
-        run: |
-          ci/runtests.sh
-
-      - name: Python 3.6 (x86)
-        env:
-          PYTHON: "py -3.6-32"
-        shell: bash
-        run: |
-          ci/runtests.sh
-
-      - name: Python 3.7 (amd64)
-        env:
-          PYTHON: "py -3.7-64"
-        shell: bash
-        run: |
-          ci/runtests.sh
-
-      - name: Python 3.7 (x86)
-        env:
-          PYTHON: "py -3.7-32"
-        shell: bash
-        run: |
-          ci/runtests.sh
-
-      - name: Python 3.8 (amd64)
-        env:
-          PYTHON: "py -3.8-64"
-        shell: bash
-        run: |
-          ci/runtests.sh
-
-      - name: Python 3.8 (x86)
-        env:
-          PYTHON: "py -3.8-32"
-        shell: bash
-        run: |
-          ci/runtests.sh
-
-      - name: Python 3.9 (amd64)
-        env:
-          PYTHON: "py -3.9-64"
-        shell: bash
-        run: |
-          ci/runtests.sh
-
-      - name: Python 3.9 (x86)
-        env:
-          PYTHON: "py -3.9-32"
-        shell: bash
-        run: |
-          ci/runtests.sh
-
-      - name: Python 3.10 (amd64)
-        env:
-          PYTHON: "py -3.10-64"
-        shell: bash
-        run: |
-          ci/runtests.sh
-
-      - name: Python 3.10 (x86)
-        env:
-          PYTHON: "py -3.10-32"
-        shell: bash
-        run: |
-          ci/runtests.sh
-
-      - name: Upload Wheels
-        uses: actions/upload-artifact@v1
-        with:
-          name: win-wheels
-          path: ./dist
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 4974d26..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,89 +0,0 @@
-version: ~> 1.0
-dist: xenial
-language: python
-cache: pip
-arch:
-   - arm64
-
-python:
-  # Available Python (PyPy) can be listed by:
-  #
-  #   $ aws s3 ls s3://travis-python-archives/binaries/ubuntu/16.04/x86_64/
-  - "3.6"
-  - "3.7"
-  - "3.8"
-  - "3.9-dev"
-
-_pure: &pure
-  install:
-    - pip install -U pip
-    - pip install -U pytest pytest-cov codecov
-    - pip install .
-  script:
-    - pytest --cov=msgpack -v test
-
-matrix:
-  include:
-    - name: 32bit build
-      arch: amd64
-      language: python
-      services:
-        - docker
-      env:
-        - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686
-      install:
-        - pip install -U pip
-        - pip install -r requirements.txt
-        - make cython
-        - docker pull $DOCKER_IMAGE
-      script:
-        - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
-
-    - arch: arm64
-      name: arm64 32bit build
-      language: python
-      services:
-        - docker
-      env:
-        - DOCKER_IMAGE=quay.io/pypa/manylinux2014_aarch64
-      install:
-        - pip install -U pip
-        - pip install -r requirements.txt
-        - make cython
-        - docker pull $DOCKER_IMAGE
-      script:
-        - docker run --rm -v `pwd`:/io -w /io $DOCKER_IMAGE /io/docker/runtests.sh
-
-    - name: "Python 2 (fallback)"
-      python: "2.7"
-      <<: *pure
-
-    - name: "pypy2.7"
-      arch: amd64
-      python: "pypy2.7-7.1.1"
-      <<: *pure
-
-    - name: "pypy3"
-      arch: amd64
-      python: "pypy3.6-7.1.1"
-      <<: *pure
-
-install:
-  - pip install -U pip
-  - pip install -U pytest pytest-cov codecov
-  - pip install -r requirements.txt  # Cython
-  - make cython
-  - pip install -e .
-
-script:
-  - python -c 'import sys; print(hex(sys.maxsize))'
-  - python -c 'from msgpack import _cmsgpack'
-  - pytest --cov=msgpack -v test
-  - MSGPACK_PUREPYTHON=x pytest --cov=msgpack -v test
-
-after_success:
-  - if [ -f .coverage ]; then
-      codecov;
-    fi
-
-# vim: sw=2 ts=2
diff --git a/Makefile b/Makefile
index b50fa80..02a4a17 100644
--- a/Makefile
+++ b/Makefile
@@ -31,8 +31,9 @@ clean:
 
 .PHONY: update-docker
 update-docker:
-	docker pull quay.io/pypa/manylinux1_i686
-	docker pull quay.io/pypa/manylinux1_x86_64
+	docker pull quay.io/pypa/manylinux2014_i686
+	docker pull quay.io/pypa/manylinux2014_x86_64
+	docker pull quay.io/pypa/manylinux2014_aarch64
 
 .PHONY: linux-wheel
 linux-wheel:
diff --git a/appveyor.yml b/appveyor.yml
deleted file mode 100644
index f338e17..0000000
--- a/appveyor.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-environment:
-  matrix:
-    # For Python versions available on Appveyor, see
-    # http://www.appveyor.com/docs/installed-software#python
-    - PYTHON: "C:\\Python36"
-
-install:
-  # We need wheel installed to build wheels
-  - "%PYTHON%\\python.exe -m pip install -U pip"
-  - "%PYTHON%\\python.exe -m pip install -U cython"
-  - "%PYTHON%\\Scripts\\cython --cplus msgpack/_cmsgpack.pyx"
-
-build: off
-
-test_script:
-  # Put your test command here.
-  # Note that you must use the environment variable %PYTHON% to refer to
-  # the interpreter you're using - Appveyor does not do anything special
-  # to put the Python version you want to use on PATH.
-  - set PYTHON="C:\\Python27"
-  - ci\\runtests.bat
-  - set PYTHON="C:\\Python27-x64"
-  - ci\\runtests.bat
-  - set PYTHON="C:\\Python36"
-  - ci\\runtests.bat
-  - set PYTHON="C:\\Python36-x64"
-  - ci\\runtests.bat
-  - set PYTHON="C:\\Python37"
-  - ci\\runtests.bat
-  - set PYTHON="C:\\Python37-x64"
-  - ci\\runtests.bat
-  - set PYTHON="C:\\Python38"
-  - ci\\runtests.bat
-  - set PYTHON="C:\\Python38-x64"
-  - ci\\runtests.bat
-
-after_test:
-  # This step builds your wheels.
-  # Again, you need to use %PYTHON% to get the correct interpreter
-
-artifacts:
-  # bdist_wheel puts your built wheel in the dist directory
-  - path: dist\*.whl
-
-#on_success:
-#  You can use this step to upload your artifacts to a public website.
-#  See Appveyor's documentation for more details. Or you can simply
-#  access your wheels from the Appveyor "artifacts" tab for your build.
-
-# vim: set shiftwidth=2
diff --git a/ci/runtests.bat b/ci/runtests.bat
deleted file mode 100644
index 4ae2f70..0000000
--- a/ci/runtests.bat
+++ /dev/null
@@ -1,9 +0,0 @@
-%PYTHON%\python.exe -m pip install -U pip wheel pytest
-%PYTHON%\python.exe setup.py build_ext -i
-%PYTHON%\python.exe setup.py install
-%PYTHON%\python.exe -c "import sys; print(hex(sys.maxsize))"
-%PYTHON%\python.exe -c "from msgpack import _cmsgpack"
-%PYTHON%\python.exe setup.py bdist_wheel
-%PYTHON%\python.exe -m pytest -v test
-SET EL=%ERRORLEVEL%
-exit /b %EL%
diff --git a/ci/runtests.sh b/ci/runtests.sh
deleted file mode 100644
index 5d87f69..0000000
--- a/ci/runtests.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-set -ex
-${PYTHON} -VV
-${PYTHON} -m pip install setuptools wheel pytest
-${PYTHON} setup.py build_ext -if
-${PYTHON} -c "from msgpack import _cmsgpack"
-${PYTHON} setup.py bdist_wheel
-${PYTHON} -m pytest -v test

From 89ea57747ebcb0fad004a92ab00ebf13c10b2d51 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 19 Jan 2022 14:42:28 +0900
Subject: [PATCH 1567/1648] Don't define __*_ENDIAN__ macro on Unix. (#495)

---
 msgpack/fallback.py | 4 ----
 setup.py            | 6 ++----
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index b27acb2..9731a22 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -11,7 +11,6 @@ if PY2:
     def dict_iteritems(d):
         return d.iteritems()
 
-
 else:
     int_types = int
     unicode = str
@@ -32,7 +31,6 @@ if sys.version_info < (3, 5):
             and e.args[0].startswith("maximum recursion depth exceeded")
         )
 
-
 else:
 
     def _is_recursionerror(e):
@@ -68,7 +66,6 @@ if hasattr(sys, "pypy_version_info"):
         def getvalue(self):
             return self.builder.build()
 
-
 else:
     USING_STRINGBUILDER = False
     from io import BytesIO as StringIO
@@ -143,7 +140,6 @@ if sys.version_info < (2, 7, 6):
         """Explicit type cast for legacy struct.unpack_from"""
         return struct.unpack_from(f, bytes(b), o)
 
-
 else:
     _unpack_from = struct.unpack_from
 
diff --git a/setup.py b/setup.py
index 01f125f..502ed33 100755
--- a/setup.py
+++ b/setup.py
@@ -79,12 +79,10 @@ class Sdist(sdist):
 
 
 libraries = []
+macros = []
+
 if sys.platform == "win32":
     libraries.append("ws2_32")
-
-if sys.byteorder == "big":
-    macros = [("__BIG_ENDIAN__", "1")]
-else:
     macros = [("__LITTLE_ENDIAN__", "1")]
 
 ext_modules = []

From cb50b2081b21e5cb4a364d292f55092c98aa1a6f Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 3 Mar 2022 12:29:55 +0900
Subject: [PATCH 1568/1648] Update setuptools and black (#498)

* Use setuptools
* Use black==22.1.0
---
 .github/workflows/black.yaml |  6 +++---
 Makefile                     |  2 +-
 msgpack/__init__.py          |  5 ++++-
 msgpack/_version.py          |  1 -
 msgpack/ext.py               |  8 +++----
 msgpack/fallback.py          | 12 +++++------
 requirements.txt             |  3 +++
 setup.cfg                    | 32 +++++++++++++++++++++++++++
 setup.py                     | 42 ++----------------------------------
 test/test_limits.py          | 12 +++++------
 test/test_memoryview.py      | 28 ++++++++++++------------
 test/test_sequnpack.py       |  4 ++--
 test/test_timestamp.py       | 12 +++++------
 13 files changed, 83 insertions(+), 84 deletions(-)
 delete mode 100644 msgpack/_version.py
 create mode 100644 setup.cfg

diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml
index 2961ed3..be137ae 100644
--- a/.github/workflows/black.yaml
+++ b/.github/workflows/black.yaml
@@ -17,9 +17,9 @@ jobs:
           architecture: 'x64'
 
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
 
       - name: Black Code Formatter
         run: |
-          pip install black
-          black --diff --check msgpack/ test/ setup.py
+          pip install black==22.1.0
+          black -S --diff --check msgpack/ test/ setup.py
diff --git a/Makefile b/Makefile
index 02a4a17..415dcfd 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ all: cython
 
 .PHONY: black
 black:
-	black msgpack/ test/ setup.py
+	black -S msgpack/ test/ setup.py
 
 .PHONY: cython
 cython:
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index d6705e2..1929df3 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,5 +1,4 @@
 # coding: utf-8
-from ._version import version
 from .exceptions import *
 from .ext import ExtType, Timestamp
 
@@ -7,6 +6,10 @@ import os
 import sys
 
 
+version = (1, 0, 4, 'dev')
+__version__ = "1.0.4dev"
+
+
 if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2:
     from .fallback import Packer, unpackb, Unpacker
 else:
diff --git a/msgpack/_version.py b/msgpack/_version.py
deleted file mode 100644
index fb878b3..0000000
--- a/msgpack/_version.py
+++ /dev/null
@@ -1 +0,0 @@
-version = (1, 0, 3)
diff --git a/msgpack/ext.py b/msgpack/ext.py
index 4eb9dd6..25544c5 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -59,7 +59,7 @@ class Timestamp(object):
             raise TypeError("seconds must be an interger")
         if not isinstance(nanoseconds, int_types):
             raise TypeError("nanoseconds must be an integer")
-        if not (0 <= nanoseconds < 10 ** 9):
+        if not (0 <= nanoseconds < 10**9):
             raise ValueError(
                 "nanoseconds must be a non-negative integer less than 999999999."
             )
@@ -143,7 +143,7 @@ class Timestamp(object):
         :type unix_float: int or float.
         """
         seconds = int(unix_sec // 1)
-        nanoseconds = int((unix_sec % 1) * 10 ** 9)
+        nanoseconds = int((unix_sec % 1) * 10**9)
         return Timestamp(seconds, nanoseconds)
 
     def to_unix(self):
@@ -161,7 +161,7 @@ class Timestamp(object):
         :param int unix_ns: Posix timestamp in nanoseconds.
         :rtype: Timestamp
         """
-        return Timestamp(*divmod(unix_ns, 10 ** 9))
+        return Timestamp(*divmod(unix_ns, 10**9))
 
     def to_unix_nano(self):
         """Get the timestamp as a unixtime in nanoseconds.
@@ -169,7 +169,7 @@ class Timestamp(object):
         :returns: posix timestamp in nanoseconds
         :rtype: int
         """
-        return self.seconds * 10 ** 9 + self.nanoseconds
+        return self.seconds * 10**9 + self.nanoseconds
 
     def to_datetime(self):
         """Get the timestamp as a UTC datetime.
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 9731a22..5f215e9 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -318,7 +318,7 @@ class Unpacker(object):
         self._buf_checkpoint = 0
 
         if not max_buffer_size:
-            max_buffer_size = 2 ** 31 - 1
+            max_buffer_size = 2**31 - 1
         if max_str_len == -1:
             max_str_len = max_buffer_size
         if max_bin_len == -1:
@@ -800,20 +800,20 @@ class Packer(object):
                 raise OverflowError("Integer value out of range")
             if check(obj, (bytes, bytearray)):
                 n = len(obj)
-                if n >= 2 ** 32:
+                if n >= 2**32:
                     raise ValueError("%s is too large" % type(obj).__name__)
                 self._pack_bin_header(n)
                 return self._buffer.write(obj)
             if check(obj, unicode):
                 obj = obj.encode("utf-8", self._unicode_errors)
                 n = len(obj)
-                if n >= 2 ** 32:
+                if n >= 2**32:
                     raise ValueError("String is too large")
                 self._pack_raw_header(n)
                 return self._buffer.write(obj)
             if check(obj, memoryview):
                 n = len(obj) * obj.itemsize
-                if n >= 2 ** 32:
+                if n >= 2**32:
                     raise ValueError("Memoryview is too large")
                 self._pack_bin_header(n)
                 return self._buffer.write(obj)
@@ -895,7 +895,7 @@ class Packer(object):
             return ret
 
     def pack_array_header(self, n):
-        if n >= 2 ** 32:
+        if n >= 2**32:
             raise ValueError
         self._pack_array_header(n)
         if self._autoreset:
@@ -904,7 +904,7 @@ class Packer(object):
             return ret
 
     def pack_map_header(self, n):
-        if n >= 2 ** 32:
+        if n >= 2**32:
             raise ValueError
         self._pack_map_header(n)
         if self._autoreset:
diff --git a/requirements.txt b/requirements.txt
index 180fe85..9bdb478 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,5 @@
 # Also declared in pyproject.toml, if updating here please also update there
 Cython~=0.29.13
+
+# dev only tools. no need to add pyproject
+black==22.1.0
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..1cb6ce3
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,32 @@
+[metadata]
+name = msgpack
+#version = attr: msgpack.__version__
+version = attr: msgpack.version
+license = Apache 2.0
+author = Inada Naoki
+author_email = songofacandy@gmail.com
+description = MessagePack serializer
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://msgpack.org/
+
+project_urls =
+    Documentation = https://msgpack-python.readthedocs.io/
+    Source = https://github.com/msgpack/msgpack-python
+    Tracker = https://github.com/msgpack/msgpack-python/issues
+
+classifiers =
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: Implementation :: CPython
+    Programming Language :: Python :: Implementation :: PyPy
+    Intended Audience :: Developers
+    License :: OSI Approved :: Apache Software License
+
+[flake8]
+max_line_length = 100
+
diff --git a/setup.py b/setup.py
index 502ed33..9630cda 100755
--- a/setup.py
+++ b/setup.py
@@ -4,10 +4,9 @@ import io
 import os
 import sys
 from glob import glob
-from distutils.command.sdist import sdist
 from setuptools import setup, Extension
-
-from distutils.command.build_ext import build_ext
+from setuptools.command.build_ext import build_ext
+from setuptools.command.sdist import sdist
 
 
 PYPY = hasattr(sys, "pypy_version_info")
@@ -65,12 +64,6 @@ class BuildExt(build_ext):
             print(e)
 
 
-exec(open("msgpack/_version.py").read())
-
-version_str = ".".join(str(x) for x in version[:3])
-if len(version) > 3 and version[3] != "final":
-    version_str += version[3]
-
 # Cython is required for sdist
 class Sdist(sdist):
     def __init__(self, *args, **kwargs):
@@ -99,39 +92,8 @@ if not PYPY and not PY2 and not os.environ.get("MSGPACK_PUREPYTHON"):
 del libraries, macros
 
 
-desc = "MessagePack (de)serializer."
-with io.open("README.md", encoding="utf-8") as f:
-    long_desc = f.read()
-del f
-
 setup(
-    name="msgpack",
-    author="Inada Naoki",
-    author_email="songofacandy@gmail.com",
-    version=version_str,
     cmdclass={"build_ext": BuildExt, "sdist": Sdist},
     ext_modules=ext_modules,
     packages=["msgpack"],
-    description=desc,
-    long_description=long_desc,
-    long_description_content_type="text/markdown",
-    url="https://msgpack.org/",
-    project_urls={
-        "Documentation": "https://msgpack-python.readthedocs.io/",
-        "Source": "https://github.com/msgpack/msgpack-python",
-        "Tracker": "https://github.com/msgpack/msgpack-python/issues",
-    },
-    license="Apache 2.0",
-    classifiers=[
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: Implementation :: CPython",
-        "Programming Language :: Python :: Implementation :: PyPy",
-        "Intended Audience :: Developers",
-        "License :: OSI Approved :: Apache Software License",
-    ],
 )
diff --git a/test/test_limits.py b/test/test_limits.py
index 65e6bcc..4314c2c 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -16,12 +16,12 @@ from msgpack import (
 
 
 def test_integer():
-    x = -(2 ** 63)
+    x = -(2**63)
     assert unpackb(packb(x)) == x
     with pytest.raises(PackOverflowError):
         packb(x - 1)
 
-    x = 2 ** 64 - 1
+    x = 2**64 - 1
     assert unpackb(packb(x)) == x
     with pytest.raises(PackOverflowError):
         packb(x + 1)
@@ -29,16 +29,16 @@ def test_integer():
 
 def test_array_header():
     packer = Packer()
-    packer.pack_array_header(2 ** 32 - 1)
+    packer.pack_array_header(2**32 - 1)
     with pytest.raises(PackValueError):
-        packer.pack_array_header(2 ** 32)
+        packer.pack_array_header(2**32)
 
 
 def test_map_header():
     packer = Packer()
-    packer.pack_map_header(2 ** 32 - 1)
+    packer.pack_map_header(2**32 - 1)
     with pytest.raises(PackValueError):
-        packer.pack_array_header(2 ** 32)
+        packer.pack_array_header(2**32)
 
 
 def test_max_str_len():
diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index 86b2c1f..84941db 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -53,46 +53,46 @@ def test_fixstr_from_float():
 
 
 def test_str16_from_byte():
-    _runtest("B", 2 ** 8, b"\xda", b"\x01\x00", False)
-    _runtest("B", 2 ** 16 - 1, b"\xda", b"\xff\xff", False)
+    _runtest("B", 2**8, b"\xda", b"\x01\x00", False)
+    _runtest("B", 2**16 - 1, b"\xda", b"\xff\xff", False)
 
 
 def test_str16_from_float():
-    _runtest("f", 2 ** 8, b"\xda", b"\x01\x00", False)
-    _runtest("f", 2 ** 16 - 4, b"\xda", b"\xff\xfc", False)
+    _runtest("f", 2**8, b"\xda", b"\x01\x00", False)
+    _runtest("f", 2**16 - 4, b"\xda", b"\xff\xfc", False)
 
 
 def test_str32_from_byte():
-    _runtest("B", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False)
+    _runtest("B", 2**16, b"\xdb", b"\x00\x01\x00\x00", False)
 
 
 def test_str32_from_float():
-    _runtest("f", 2 ** 16, b"\xdb", b"\x00\x01\x00\x00", False)
+    _runtest("f", 2**16, b"\xdb", b"\x00\x01\x00\x00", False)
 
 
 def test_bin8_from_byte():
     _runtest("B", 1, b"\xc4", b"\x01", True)
-    _runtest("B", 2 ** 8 - 1, b"\xc4", b"\xff", True)
+    _runtest("B", 2**8 - 1, b"\xc4", b"\xff", True)
 
 
 def test_bin8_from_float():
     _runtest("f", 4, b"\xc4", b"\x04", True)
-    _runtest("f", 2 ** 8 - 4, b"\xc4", b"\xfc", True)
+    _runtest("f", 2**8 - 4, b"\xc4", b"\xfc", True)
 
 
 def test_bin16_from_byte():
-    _runtest("B", 2 ** 8, b"\xc5", b"\x01\x00", True)
-    _runtest("B", 2 ** 16 - 1, b"\xc5", b"\xff\xff", True)
+    _runtest("B", 2**8, b"\xc5", b"\x01\x00", True)
+    _runtest("B", 2**16 - 1, b"\xc5", b"\xff\xff", True)
 
 
 def test_bin16_from_float():
-    _runtest("f", 2 ** 8, b"\xc5", b"\x01\x00", True)
-    _runtest("f", 2 ** 16 - 4, b"\xc5", b"\xff\xfc", True)
+    _runtest("f", 2**8, b"\xc5", b"\x01\x00", True)
+    _runtest("f", 2**16 - 4, b"\xc5", b"\xff\xfc", True)
 
 
 def test_bin32_from_byte():
-    _runtest("B", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True)
+    _runtest("B", 2**16, b"\xc6", b"\x00\x01\x00\x00", True)
 
 
 def test_bin32_from_float():
-    _runtest("f", 2 ** 16, b"\xc6", b"\x00\x01\x00\x00", True)
+    _runtest("f", 2**16, b"\xc6", b"\x00\x01\x00\x00", True)
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 6293a45..9f20c07 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -118,8 +118,8 @@ def test_issue124():
 
 def test_unpack_tell():
     stream = io.BytesIO()
-    messages = [2 ** i - 1 for i in range(65)]
-    messages += [-(2 ** i) for i in range(1, 64)]
+    messages = [2**i - 1 for i in range(65)]
+    messages += [-(2**i) for i in range(1, 64)]
     messages += [
         b"hello",
         b"hello" * 1000,
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index 4e26489..253228e 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -10,31 +10,31 @@ if sys.version_info[0] > 2:
 
 def test_timestamp():
     # timestamp32
-    ts = Timestamp(2 ** 32 - 1)
+    ts = Timestamp(2**32 - 1)
     assert ts.to_bytes() == b"\xff\xff\xff\xff"
     packed = msgpack.packb(ts)
     assert packed == b"\xd6\xff" + ts.to_bytes()
     unpacked = msgpack.unpackb(packed)
     assert ts == unpacked
-    assert ts.seconds == 2 ** 32 - 1 and ts.nanoseconds == 0
+    assert ts.seconds == 2**32 - 1 and ts.nanoseconds == 0
 
     # timestamp64
-    ts = Timestamp(2 ** 34 - 1, 999999999)
+    ts = Timestamp(2**34 - 1, 999999999)
     assert ts.to_bytes() == b"\xee\x6b\x27\xff\xff\xff\xff\xff"
     packed = msgpack.packb(ts)
     assert packed == b"\xd7\xff" + ts.to_bytes()
     unpacked = msgpack.unpackb(packed)
     assert ts == unpacked
-    assert ts.seconds == 2 ** 34 - 1 and ts.nanoseconds == 999999999
+    assert ts.seconds == 2**34 - 1 and ts.nanoseconds == 999999999
 
     # timestamp96
-    ts = Timestamp(2 ** 63 - 1, 999999999)
+    ts = Timestamp(2**63 - 1, 999999999)
     assert ts.to_bytes() == b"\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff"
     packed = msgpack.packb(ts)
     assert packed == b"\xc7\x0c\xff" + ts.to_bytes()
     unpacked = msgpack.unpackb(packed)
     assert ts == unpacked
-    assert ts.seconds == 2 ** 63 - 1 and ts.nanoseconds == 999999999
+    assert ts.seconds == 2**63 - 1 and ts.nanoseconds == 999999999
 
     # negative fractional
     ts = Timestamp.from_unix(-2.3)  # s: -3, ns: 700000000

From 849c8063817894f7dba166a19fbdbb9ffd8c2b80 Mon Sep 17 00:00:00 2001
From: Victor Stinner 
Date: Mon, 14 Mar 2022 03:23:11 +0100
Subject: [PATCH 1569/1648] Use PyFloat_Pack8() on Python 3.11a7 (#499)

Python 3.11a7 adds public functions:

* PyFloat_Pack4(), PyFloat_Pack8()
* PyFloat_Unpack4(), PyFloat_Unpack8()

https://bugs.python.org/issue46906
---
 msgpack/pack_template.h   |  9 +++++++++
 msgpack/unpack_template.h | 14 ++++++++++++--
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 0e940b8..7d479b6 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -568,7 +568,12 @@ static inline int msgpack_pack_float(msgpack_packer* x, float d)
 {
     unsigned char buf[5];
     buf[0] = 0xca;
+
+#if PY_VERSION_HEX >= 0x030B00A7
+    PyFloat_Pack4(d, (char *)&buf[1], 0);
+#else
     _PyFloat_Pack4(d, &buf[1], 0);
+#endif
     msgpack_pack_append_buffer(x, buf, 5);
 }
 
@@ -576,7 +581,11 @@ static inline int msgpack_pack_double(msgpack_packer* x, double d)
 {
     unsigned char buf[9];
     buf[0] = 0xcb;
+#if PY_VERSION_HEX >= 0x030B00A7
+    PyFloat_Pack8(d, (char *)&buf[1], 0);
+#else
     _PyFloat_Pack8(d, &buf[1], 0);
+#endif
     msgpack_pack_append_buffer(x, buf, 9);
 }
 
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 9924b9c..8b9fcc1 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -243,10 +243,20 @@ static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize
                                           _msgpack_load32(uint32_t,n)+1,
                                           _ext_zero);
             case CS_FLOAT: {
-                    double f = _PyFloat_Unpack4((unsigned char*)n, 0);
+                    double f;
+#if PY_VERSION_HEX >= 0x030B00A7
+                    f = PyFloat_Unpack4((const char*)n, 0);
+#else
+                    f = _PyFloat_Unpack4((unsigned char*)n, 0);
+#endif
                     push_fixed_value(_float, f); }
             case CS_DOUBLE: {
-                    double f = _PyFloat_Unpack8((unsigned char*)n, 0);
+                    double f;
+#if PY_VERSION_HEX >= 0x030B00A7
+                    f = PyFloat_Unpack8((const char*)n, 0);
+#else
+                    f = _PyFloat_Unpack8((unsigned char*)n, 0);
+#endif
                     push_fixed_value(_double, f); }
             case CS_UINT_8:
                 push_fixed_value(_uint8, *(uint8_t*)n);

From 6a721faa778e4db23f76bc8a0f9adf85f59f69c7 Mon Sep 17 00:00:00 2001
From: Shantanu <12621235+hauntsaninja@users.noreply.github.com>
Date: Mon, 2 May 2022 02:26:53 -0600
Subject: [PATCH 1570/1648] Upgrade black (#505)

---
 .github/workflows/black.yaml | 2 +-
 requirements.txt             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml
index be137ae..0a0a737 100644
--- a/.github/workflows/black.yaml
+++ b/.github/workflows/black.yaml
@@ -21,5 +21,5 @@ jobs:
 
       - name: Black Code Formatter
         run: |
-          pip install black==22.1.0
+          pip install black==22.3.0
           black -S --diff --check msgpack/ test/ setup.py
diff --git a/requirements.txt b/requirements.txt
index 9bdb478..f557888 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@
 Cython~=0.29.13
 
 # dev only tools. no need to add pyproject
-black==22.1.0
+black==22.3.0

From b901b179d1976e09eb074803dcbb17d8d150c69f Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 23 May 2022 04:52:09 +0000
Subject: [PATCH 1571/1648] Update Cython to 0.29.30

---
 pyproject.toml   | 2 +-
 requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index a9eb8aa..195795f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -2,7 +2,7 @@
 requires = [
     # Also declared in requirements.txt, if updating here please also update
     # there
-    "Cython~=0.29.13",
+    "Cython~=0.29.30",
     "setuptools >= 35.0.2",
 ]
 build-backend = "setuptools.build_meta"
diff --git a/requirements.txt b/requirements.txt
index f557888..9f3c1a0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 # Also declared in pyproject.toml, if updating here please also update there
-Cython~=0.29.13
+Cython~=0.29.30
 
 # dev only tools. no need to add pyproject
 black==22.3.0

From b75e3412fb8a2b6d6cd1da1b7063e14f6bfc0337 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 23 May 2022 05:01:08 +0000
Subject: [PATCH 1572/1648] Fix pip upgrade

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6e497e0..2a01926 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,7 +28,7 @@ jobs:
       - name: Build
         shell: bash
         run: |
-          pip install -U pip
+          python -m pip install -U pip
           pip install -r requirements.txt pytest
           make cython
           pip install .

From 500a238028bdebe123b502b07769578b5f0e8a3a Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 24 May 2022 19:46:51 +0900
Subject: [PATCH 1573/1648] Fix Unpacker max_buffer_length handling (#506)

---
 msgpack/_unpacker.pyx  | 28 ++++++++++++----------------
 msgpack/fallback.py    |  2 ++
 test/test_sequnpack.py | 11 ++++++++++-
 3 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 27facc0..8b06661 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -440,34 +440,30 @@ cdef class Unpacker(object):
         self.buf_size = buf_size
         self.buf_tail = tail + _buf_len
 
-    cdef read_from_file(self):
-        next_bytes = self.file_like_read(
-                min(self.read_size,
-                    self.max_buffer_size - (self.buf_tail - self.buf_head)
-                    ))
+    cdef int read_from_file(self) except -1:
+        cdef Py_ssize_t remains = self.max_buffer_size - (self.buf_tail - self.buf_head)
+        if remains <= 0:
+            raise BufferFull
+
+        next_bytes = self.file_like_read(min(self.read_size, remains))
         if next_bytes:
             self.append_buffer(PyBytes_AsString(next_bytes), PyBytes_Size(next_bytes))
         else:
             self.file_like = None
+        return 0
 
     cdef object _unpack(self, execute_fn execute, bint iter=0):
         cdef int ret
         cdef object obj
         cdef Py_ssize_t prev_head
 
-        if self.buf_head >= self.buf_tail and self.file_like is not None:
-            self.read_from_file()
-
         while 1:
             prev_head = self.buf_head
-            if prev_head >= self.buf_tail:
-                if iter:
-                    raise StopIteration("No more data to unpack.")
-                else:
-                    raise OutOfData("No more data to unpack.")
-
-            ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
-            self.stream_offset += self.buf_head - prev_head
+            if prev_head < self.buf_tail:
+                ret = execute(&self.ctx, self.buf, self.buf_tail, &self.buf_head)
+                self.stream_offset += self.buf_head - prev_head
+            else:
+                ret = 0
 
             if ret == 1:
                 obj = unpack_data(&self.ctx)
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 5f215e9..f560c7b 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -423,6 +423,8 @@ class Unpacker(object):
 
         # Read from file
         remain_bytes = -remain_bytes
+        if remain_bytes + len(self._buffer) > self._max_buffer_size:
+            raise BufferFull
         while remain_bytes > 0:
             to_read_bytes = max(self._read_size, remain_bytes)
             read_data = self.file_like.read(to_read_bytes)
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 9f20c07..c091076 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -2,7 +2,7 @@
 # coding: utf-8
 import io
 from msgpack import Unpacker, BufferFull
-from msgpack import pack
+from msgpack import pack, packb
 from msgpack.exceptions import OutOfData
 from pytest import raises
 
@@ -78,6 +78,15 @@ def test_maxbuffersize():
     assert ord("b") == next(unpacker)
 
 
+def test_maxbuffersize_file():
+    buff = io.BytesIO(packb(b"a" * 10) + packb([b"a" * 20] * 2))
+    unpacker = Unpacker(buff, read_size=1, max_buffer_size=19, max_bin_len=20)
+    assert unpacker.unpack() == b"a" * 10
+    # assert unpacker.unpack() == [b"a" * 20]*2
+    with raises(BufferFull):
+        print(unpacker.unpack())
+
+
 def test_readbytes():
     unpacker = Unpacker(read_size=3)
     unpacker.feed(b"foobar")

From 63837a44d855017f1b2f667afa5ac684fd65591d Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 24 May 2022 20:13:07 +0900
Subject: [PATCH 1574/1648] ci: Update action versions. (#507)

---
 .github/workflows/test.yml  | 7 +++----
 .github/workflows/wheel.yml | 6 +++---
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2a01926..2a796aa 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,17 +10,17 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04, windows-2022, macos-10.15]
-        py: ["3.10", "3.9", "3.8", "3.7", "3.6"]
+        py: ["3.11-dev", "3.10", "3.9", "3.8", "3.7", "3.6"]
 
     runs-on: ${{ matrix.os }}
     name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.py }}
           cache: "pip"
@@ -28,7 +28,6 @@ jobs:
       - name: Build
         shell: bash
         run: |
-          python -m pip install -U pip
           pip install -r requirements.txt pytest
           make cython
           pip install .
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 5627b5f..541654d 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up QEMU
         if: runner.os == 'Linux'
@@ -23,7 +23,7 @@ jobs:
           platforms: arm64
 
       - name: Set up Python 3.9
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: 3.9
           cache: "pip"
@@ -35,7 +35,7 @@ jobs:
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v2.2.2
+        uses: pypa/cibuildwheel@v2.5.0
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"

From a34dc945bfe39c1f2f91af2403d906069613ea41 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 25 May 2022 10:00:57 +0900
Subject: [PATCH 1575/1648] 1.0.4rc1

---
 msgpack/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 1929df3..81b2e67 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -6,8 +6,8 @@ import os
 import sys
 
 
-version = (1, 0, 4, 'dev')
-__version__ = "1.0.4dev"
+version = (1, 0, 4, 'rc1')
+__version__ = "1.0.4rc1"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2:

From caadbf2df5a87039a52a5dcf4fc3f151bba70eed Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 25 May 2022 12:10:47 +0900
Subject: [PATCH 1576/1648] Use Actions to publish to PyPI

---
 .github/workflows/test.yml  | 5 +++++
 .github/workflows/wheel.yml | 8 +++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2a796aa..d01d74c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -42,3 +42,8 @@ jobs:
         run: |
           MSGPACK_PUREPYTHON=1 pytest -v test
 
+      - name: Publish Wheels to TestPyPI
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 541654d..9e8dce8 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -43,9 +43,15 @@ jobs:
           CIBW_ARCHS_MACOS: x86_64 universal2 arm64
           CIBW_SKIP: pp*
 
-      - name: Upload Wheels
+      - name: Upload Wheels to artifact
         uses: actions/upload-artifact@v1
         with:
           name: Wheels
           path: wheelhouse
 
+      - name: Publish Wheels to TestPyPI
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages_dir: wheelhouse
+          password: ${{ secrets.PYPI_API_TOKEN }}

From b5acfd53833c3dbd379e539cc6e540cec83d0a99 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 3 Jun 2022 13:46:51 +0900
Subject: [PATCH 1577/1648] Release v1.0.4 (#509)

---
 .github/workflows/wheel.yml |  2 +-
 ChangeLog.rst               | 10 ++++++++++
 msgpack/__init__.py         |  4 ++--
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 9e8dce8..d73898c 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -35,7 +35,7 @@ jobs:
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v2.5.0
+        uses: pypa/cibuildwheel@v2.6.0
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
diff --git a/ChangeLog.rst b/ChangeLog.rst
index fc6df68..a11c814 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,13 @@
+1.0.4
+=====
+
+Release Date: 2022-06-03
+
+* Support Python 3.11 (beta).
+* Don't define `__*_ENDIAN__` macro on Unix. by @methane in https://github.com/msgpack/msgpack-python/pull/495
+* Use PyFloat_Pack8() on Python 3.11a7 by @vstinner in https://github.com/msgpack/msgpack-python/pull/499
+* Fix Unpacker max_buffer_length handling by @methane in https://github.com/msgpack/msgpack-python/pull/506
+
 1.0.3
 =====
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 81b2e67..5071021 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -6,8 +6,8 @@ import os
 import sys
 
 
-version = (1, 0, 4, 'rc1')
-__version__ = "1.0.4rc1"
+version = (1, 0, 4)
+__version__ = "1.0.4"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2:

From 9d45926a596028e39ec59dd909a56eb5e9e8fee7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= 
Date: Tue, 2 Aug 2022 06:19:56 +0200
Subject: [PATCH 1578/1648] Usef `__BYTE_ORDER__` instead of `__BYTE_ORDER`
 (#513)

__BYTE_ORDER__ is common predefined macro available on at least gcc and clang.
__BYTE_ORDER is macro defined in platform specific headers.
---
 msgpack/sysdep.h | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h
index ed9c1bc..ae28f0c 100644
--- a/msgpack/sysdep.h
+++ b/msgpack/sysdep.h
@@ -61,14 +61,12 @@ typedef unsigned int _msgpack_atomic_counter_t;
 #endif
 #endif
 
-#else
-#include   /* __BYTE_ORDER */
 #endif
 
 #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
-#if __BYTE_ORDER == __LITTLE_ENDIAN
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 #define __LITTLE_ENDIAN__
-#elif __BYTE_ORDER == __BIG_ENDIAN
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 #define __BIG_ENDIAN__
 #elif _WIN32
 #define __LITTLE_ENDIAN__

From edca770071fc702e0b4c33f87fb0fa3682b486b4 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 8 Aug 2022 15:08:40 +0900
Subject: [PATCH 1579/1648] Fix build error caused by ntohs, ntohl (#514)

---
 msgpack/sysdep.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/msgpack/sysdep.h b/msgpack/sysdep.h
index ae28f0c..7067300 100644
--- a/msgpack/sysdep.h
+++ b/msgpack/sysdep.h
@@ -61,6 +61,8 @@ typedef unsigned int _msgpack_atomic_counter_t;
 #endif
 #endif
 
+#else /* _WIN32 */
+#include  /* ntohs, ntohl */
 #endif
 
 #if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
@@ -93,7 +95,7 @@ typedef unsigned int _msgpack_atomic_counter_t;
 #ifdef _WIN32
 #  if defined(ntohl)
 #    define _msgpack_be32(x) ntohl(x)
-#  elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400)
+#  elif defined(_byteswap_ulong) || defined(_MSC_VER)
 #    define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x))
 #  else
 #    define _msgpack_be32(x) \
@@ -106,7 +108,7 @@ typedef unsigned int _msgpack_atomic_counter_t;
 #  define _msgpack_be32(x) ntohl(x)
 #endif
 
-#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400)
+#if defined(_byteswap_uint64) || defined(_MSC_VER)
 #  define _msgpack_be64(x) (_byteswap_uint64(x))
 #elif defined(bswap_64)
 #  define _msgpack_be64(x) bswap_64(x)

From 44a80603838ea480e66c9235036ff742f4013200 Mon Sep 17 00:00:00 2001
From: Matthieu Darbois 
Date: Fri, 9 Sep 2022 09:16:12 +0200
Subject: [PATCH 1580/1648] Add python 3.11 wheels (#517)

---
 .github/workflows/wheel.yml | 4 ++--
 setup.cfg                   | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index d73898c..770b565 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -8,7 +8,7 @@ jobs:
   build_wheels:
     strategy:
       matrix:
-        os: [ubuntu-20.04, windows-2022, macos-10.15]
+        os: [ubuntu-22.04, windows-2022, macos-10.15]
     runs-on: ${{ matrix.os }}
     name: Build wheels on ${{ matrix.os }}
 
@@ -35,7 +35,7 @@ jobs:
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v2.6.0
+        uses: pypa/cibuildwheel@v2.9.0
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
diff --git a/setup.cfg b/setup.cfg
index 1cb6ce3..e378284 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -22,6 +22,7 @@ classifiers =
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
     Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: Implementation :: PyPy
     Intended Audience :: Developers

From c3995669f1f821596714240c2cd07943810f8658 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 18 Jan 2023 08:08:58 +0000
Subject: [PATCH 1581/1648] Remove unused code

---
 msgpack/pack.h | 26 --------------------------
 1 file changed, 26 deletions(-)

diff --git a/msgpack/pack.h b/msgpack/pack.h
index 4f3ce1d..1e849ac 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -71,7 +71,6 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_
 static inline int
 msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit)
 {
-#if PY_MAJOR_VERSION >= 3
     assert(PyUnicode_Check(o));
 
     Py_ssize_t len;
@@ -87,31 +86,6 @@ msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit)
     if (ret) return ret;
 
     return msgpack_pack_raw_body(pk, buf, len);
-#else
-    PyObject *bytes;
-    Py_ssize_t len;
-    int ret;
-
-    // py2
-    bytes = PyUnicode_AsUTF8String(o);
-    if (bytes == NULL)
-        return -1;
-
-    len = PyString_GET_SIZE(bytes);
-    if (len > limit) {
-        Py_DECREF(bytes);
-        return -2;
-    }
-
-    ret = msgpack_pack_raw(pk, len);
-    if (ret) {
-        Py_DECREF(bytes);
-        return -1;
-    }
-    ret = msgpack_pack_raw_body(pk, PyString_AS_STRING(bytes), len);
-    Py_DECREF(bytes);
-    return ret;
-#endif
 }
 
 #ifdef __cplusplus

From b82d0b62f187552b8108602d7b0451ac362a29cc Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 18 Jan 2023 19:13:44 +0900
Subject: [PATCH 1582/1648] fallback: Fix packing multidim memoryview (#527)

Fix #526
---
 msgpack/fallback.py     | 2 +-
 test/test_memoryview.py | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index f560c7b..e8cebc1 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -814,7 +814,7 @@ class Packer(object):
                 self._pack_raw_header(n)
                 return self._buffer.write(obj)
             if check(obj, memoryview):
-                n = len(obj) * obj.itemsize
+                n = obj.nbytes
                 if n >= 2**32:
                     raise ValueError("Memoryview is too large")
                 self._pack_bin_header(n)
diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index 84941db..a0939a6 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -96,3 +96,11 @@ def test_bin32_from_byte():
 
 def test_bin32_from_float():
     _runtest("f", 2**16, b"\xc6", b"\x00\x01\x00\x00", True)
+
+
+def test_multidim_memoryview():
+    # See https://github.com/msgpack/msgpack-python/issues/526
+    view = memoryview(b"\00" * 6)
+    data = view.cast(view.format, (3, 2))
+    packed = packb(data)
+    assert packed == b'\xc4\x06\x00\x00\x00\x00\x00\x00'

From 10082295536098d90681da5d7199ca384e8b8ff8 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 18 Jan 2023 19:47:15 +0900
Subject: [PATCH 1583/1648] Release v1.0.5rc1 (#528)

---
 .github/workflows/black.yaml | 4 ++--
 .github/workflows/test.yml   | 6 +++---
 .github/workflows/wheel.yml  | 2 +-
 ChangeLog.rst                | 9 +++++++++
 msgpack/__init__.py          | 4 ++--
 5 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml
index 0a0a737..1e28b7b 100644
--- a/.github/workflows/black.yaml
+++ b/.github/workflows/black.yaml
@@ -11,13 +11,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.x'
           architecture: 'x64'
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Black Code Formatter
         run: |
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d01d74c..5e41167 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -9,8 +9,8 @@ jobs:
   test:
     strategy:
       matrix:
-        os: [ubuntu-20.04, windows-2022, macos-10.15]
-        py: ["3.11-dev", "3.10", "3.9", "3.8", "3.7", "3.6"]
+        os: [ubuntu-22.04, windows-2022, macos-10.15]
+        py: ["3.11", "3.10", "3.9", "3.8", "3.7"]
 
     runs-on: ${{ matrix.os }}
     name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}
@@ -20,7 +20,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.py }}
           cache: "pip"
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 770b565..b2879fe 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -35,7 +35,7 @@ jobs:
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v2.9.0
+        uses: pypa/cibuildwheel@v2.12.0
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
diff --git a/ChangeLog.rst b/ChangeLog.rst
index a11c814..1b7ae2c 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,12 @@
+1.0.5rc1
+========
+
+Release Date: 2023-01-18
+
+* Use ``__BYTE_ORDER__`` instead of ``__BYTE_ORDER`` for portability. (#513, #514)
+* Add Python 3.11 wheels (#517)
+* fallback: Fix packing multidimensional memoryview (#527)
+
 1.0.4
 =====
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 5071021..501b291 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -6,8 +6,8 @@ import os
 import sys
 
 
-version = (1, 0, 4)
-__version__ = "1.0.4"
+version = (1, 0, 5, 'rc', 1)
+__version__ = "1.0.5rc1"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2:

From e3ef909c47e5a245fc9d711e9d974a5f9df99303 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 18 Jan 2023 13:07:24 +0000
Subject: [PATCH 1584/1648] Action: Use setup-python@v4

---
 .github/workflows/wheel.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index b2879fe..2305008 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -22,10 +22,10 @@ jobs:
         with:
           platforms: arm64
 
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v3
+      - name: Set up Python 3.x
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: "3.x"
           cache: "pip"
 
       - name: Prepare

From dcb775031c0b1d575b90e822e81e845ebfda4a2e Mon Sep 17 00:00:00 2001
From: Anthon van der Neut 
Date: Sun, 5 Mar 2023 15:45:38 +0100
Subject: [PATCH 1585/1648] minor type in exception message (#533)

interger -> integer
---
 msgpack/ext.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/msgpack/ext.py b/msgpack/ext.py
index 25544c5..23e0d6b 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -56,7 +56,7 @@ class Timestamp(object):
         Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns.
         """
         if not isinstance(seconds, int_types):
-            raise TypeError("seconds must be an interger")
+            raise TypeError("seconds must be an integer")
         if not isinstance(nanoseconds, int_types):
             raise TypeError("nanoseconds must be an integer")
         if not (0 <= nanoseconds < 10**9):

From aa9ce3e2bbc1d3d0476396892c46e704292455ab Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 3 Jun 2022 14:37:21 +0900
Subject: [PATCH 1586/1648] Action: Run publish on tag creation.

---
 .github/workflows/wheel.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 2305008..5f103a3 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -49,8 +49,8 @@ jobs:
           name: Wheels
           path: wheelhouse
 
-      - name: Publish Wheels to TestPyPI
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+      - name: Publish Wheels to PyPI
+        if: github.event_name == 'create' && github.event.ref_type == 'tag'
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages_dir: wheelhouse

From 4c55f809fe2231130cf99b20538b26b92f1bea31 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 9 Mar 2023 00:43:28 +0900
Subject: [PATCH 1587/1648] Release v1.0.5 (#534)

---
 ChangeLog.rst       | 6 +++---
 msgpack/__init__.py | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 1b7ae2c..5354799 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,7 +1,7 @@
-1.0.5rc1
-========
+1.0.5
+=====
 
-Release Date: 2023-01-18
+Release Date: 2023-03-08
 
 * Use ``__BYTE_ORDER__`` instead of ``__BYTE_ORDER`` for portability. (#513, #514)
 * Add Python 3.11 wheels (#517)
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 501b291..1300b86 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -6,8 +6,8 @@ import os
 import sys
 
 
-version = (1, 0, 5, 'rc', 1)
-__version__ = "1.0.5rc1"
+version = (1, 0, 5)
+__version__ = "1.0.5"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2:

From 35b2d246cfdb19484caa5789512bf71ee378caec Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 9 Mar 2023 00:47:52 +0900
Subject: [PATCH 1588/1648] Action: Update wheel workflow

---
 .github/workflows/wheel.yml | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 5f103a3..6cf2fe9 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -8,7 +8,7 @@ jobs:
   build_wheels:
     strategy:
       matrix:
-        os: [ubuntu-22.04, windows-2022, macos-10.15]
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
     runs-on: ${{ matrix.os }}
     name: Build wheels on ${{ matrix.os }}
 
@@ -48,10 +48,3 @@ jobs:
         with:
           name: Wheels
           path: wheelhouse
-
-      - name: Publish Wheels to PyPI
-        if: github.event_name == 'create' && github.event.ref_type == 'tag'
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          packages_dir: wheelhouse
-          password: ${{ secrets.PYPI_API_TOKEN }}

From 0516c2c2a97ef48a2becf30bc8b2365ca16199f1 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 9 Mar 2023 01:22:38 +0900
Subject: [PATCH 1589/1648] Action: Update test workflow

---
 .github/workflows/test.yml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5e41167..8878102 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -9,7 +9,7 @@ jobs:
   test:
     strategy:
       matrix:
-        os: [ubuntu-22.04, windows-2022, macos-10.15]
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
         py: ["3.11", "3.10", "3.9", "3.8", "3.7"]
 
     runs-on: ${{ matrix.os }}
@@ -41,9 +41,3 @@ jobs:
         shell: bash
         run: |
           MSGPACK_PUREPYTHON=1 pytest -v test
-
-      - name: Publish Wheels to TestPyPI
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          password: ${{ secrets.PYPI_API_TOKEN }}

From 802cbc9495ed059b62b46f057cf3d71f756e2480 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Sat, 1 Apr 2023 00:02:25 +0900
Subject: [PATCH 1590/1648] Add security policy

---
 SECURITY.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 SECURITY.md

diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 0000000..75f0c54
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,5 @@
+## Security contact information
+
+To report a security vulnerability, please use the
+[Tidelift security contact](https://tidelift.com/security).
+Tidelift will coordinate the fix and disclosure.
\ No newline at end of file

From 45f848695c855966d8a46656d8de1a2734d934ae Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Sat, 8 Apr 2023 02:18:25 -0300
Subject: [PATCH 1591/1648] fix: build status badge (#538)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cb81648..7f7c423 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # MessagePack for Python
 
-[![Build Status](https://travis-ci.org/msgpack/msgpack-python.svg?branch=master)](https://travis-ci.org/msgpack/msgpack-python)
+[![Build Status](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml/badge.svg)](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml)
 [![Documentation Status](https://readthedocs.org/projects/msgpack-python/badge/?version=latest)](https://msgpack-python.readthedocs.io/en/latest/?badge=latest)
 
 ## What's this

From feec06206c7f8dd8efeac6177badbf5b256b36e4 Mon Sep 17 00:00:00 2001
From: sblondon 
Date: Sun, 21 May 2023 09:26:39 +0200
Subject: [PATCH 1592/1648] Drop python2 support (#519)

The PR removes python2 references and cases.

Close #518

Co-authored-by: Inada Naoki 
---
 README.md               |  4 +--
 docs/conf.py            | 20 ++++++------
 msgpack/__init__.py     |  2 +-
 msgpack/_packer.pyx     |  1 -
 msgpack/_unpacker.pyx   |  2 +-
 msgpack/ext.py          | 24 +++-----------
 msgpack/fallback.py     | 70 ++++++++++++-----------------------------
 setup.py                |  3 +-
 test/test_buffer.py     |  1 -
 test/test_case.py       |  2 +-
 test/test_except.py     |  2 +-
 test/test_extension.py  | 15 ++-------
 test/test_memoryview.py |  5 ---
 test/test_pack.py       |  6 ----
 test/test_timestamp.py  | 27 +++++++---------
 test/test_unpack.py     |  4 +--
 tox.ini                 | 13 --------
 17 files changed, 58 insertions(+), 143 deletions(-)

diff --git a/README.md b/README.md
index 7f7c423..61f99e1 100644
--- a/README.md
+++ b/README.md
@@ -220,9 +220,9 @@ and `raw=True` options.
 
 ```pycon
 >>> import msgpack
->>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=False), raw=True)
+>>> msgpack.unpackb(msgpack.packb([b'spam', 'eggs'], use_bin_type=False), raw=True)
 [b'spam', b'eggs']
->>> msgpack.unpackb(msgpack.packb([b'spam', u'eggs'], use_bin_type=True), raw=False)
+>>> msgpack.unpackb(msgpack.packb([b'spam', 'eggs'], use_bin_type=True), raw=False)
 [b'spam', 'eggs']
 ```
 
diff --git a/docs/conf.py b/docs/conf.py
index 6b432be..91ce77f 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -40,8 +40,8 @@ source_suffix = ".rst"
 master_doc = "index"
 
 # General information about the project.
-project = u"msgpack"
-copyright = u"Inada Naoki"
+project = "msgpack"
+copyright = "Inada Naoki"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -181,7 +181,7 @@ latex_elements = {
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-    ("index", "msgpack.tex", u"msgpack Documentation", u"Author", "manual"),
+    ("index", "msgpack.tex", "msgpack Documentation", "Author", "manual"),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -209,7 +209,7 @@ latex_documents = [
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [("index", "msgpack", u"msgpack Documentation", [u"Author"], 1)]
+man_pages = [("index", "msgpack", "msgpack Documentation", ["Author"], 1)]
 
 # If true, show URL addresses after external links.
 # man_show_urls = False
@@ -224,8 +224,8 @@ texinfo_documents = [
     (
         "index",
         "msgpack",
-        u"msgpack Documentation",
-        u"Author",
+        "msgpack Documentation",
+        "Author",
         "msgpack",
         "One line description of project.",
         "Miscellaneous",
@@ -245,10 +245,10 @@ texinfo_documents = [
 # -- Options for Epub output ---------------------------------------------------
 
 # Bibliographic Dublin Core info.
-epub_title = u"msgpack"
-epub_author = u"Author"
-epub_publisher = u"Author"
-epub_copyright = u"2013, Author"
+epub_title = "msgpack"
+epub_author = "Author"
+epub_publisher = "Author"
+epub_copyright = "2013, Author"
 
 # The language of the text. It defaults to the language option
 # or en if the language is not set.
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 1300b86..6c10dc2 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -10,7 +10,7 @@ version = (1, 0, 5)
 __version__ = "1.0.5"
 
 
-if os.environ.get("MSGPACK_PUREPYTHON") or sys.version_info[0] == 2:
+if os.environ.get("MSGPACK_PUREPYTHON"):
     from .fallback import Packer, unpackb, Unpacker
 else:
     try:
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 396da0c..074b39f 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -98,7 +98,6 @@ cdef class Packer(object):
         If set to true, datetime with tzinfo is packed into Timestamp type.
         Note that the tzinfo is stripped in the timestamp.
         You can get UTC datetime with `timestamp=3` option of the Unpacker.
-        (Python 2 is not supported).
 
     :param str unicode_errors:
         The error handler for encoding unicode. (default: 'strict')
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 8b06661..d5dc5ea 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -236,7 +236,7 @@ cdef class Unpacker(object):
             0 - Timestamp
             1 - float  (Seconds from the EPOCH)
             2 - int  (Nanoseconds from the EPOCH)
-            3 - datetime.datetime  (UTC).  Python 2 is not supported.
+            3 - datetime.datetime  (UTC).
 
     :param bool strict_map_key:
         If true (default), only str or bytes are accepted for map (dict) keys.
diff --git a/msgpack/ext.py b/msgpack/ext.py
index 23e0d6b..07f96a5 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -5,19 +5,6 @@ import sys
 import struct
 
 
-PY2 = sys.version_info[0] == 2
-
-if PY2:
-    int_types = (int, long)
-    _utc = None
-else:
-    int_types = int
-    try:
-        _utc = datetime.timezone.utc
-    except AttributeError:
-        _utc = datetime.timezone(datetime.timedelta(0))
-
-
 class ExtType(namedtuple("ExtType", "code data")):
     """ExtType represents ext type in msgpack."""
 
@@ -55,9 +42,9 @@ class Timestamp(object):
 
         Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns.
         """
-        if not isinstance(seconds, int_types):
+        if not isinstance(seconds, int):
             raise TypeError("seconds must be an integer")
-        if not isinstance(nanoseconds, int_types):
+        if not isinstance(nanoseconds, int):
             raise TypeError("nanoseconds must be an integer")
         if not (0 <= nanoseconds < 10**9):
             raise ValueError(
@@ -174,11 +161,10 @@ class Timestamp(object):
     def to_datetime(self):
         """Get the timestamp as a UTC datetime.
 
-        Python 2 is not supported.
-
         :rtype: datetime.
         """
-        return datetime.datetime.fromtimestamp(0, _utc) + datetime.timedelta(
+        utc = datetime.timezone.utc
+        return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(
             seconds=self.to_unix()
         )
 
@@ -186,8 +172,6 @@ class Timestamp(object):
     def from_datetime(dt):
         """Create a Timestamp from datetime with tzinfo.
 
-        Python 2 is not supported.
-
         :rtype: Timestamp
         """
         return Timestamp.from_unix(dt.timestamp())
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index e8cebc1..618c362 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -4,22 +4,6 @@ import sys
 import struct
 
 
-PY2 = sys.version_info[0] == 2
-if PY2:
-    int_types = (int, long)
-
-    def dict_iteritems(d):
-        return d.iteritems()
-
-else:
-    int_types = int
-    unicode = str
-    xrange = range
-
-    def dict_iteritems(d):
-        return d.items()
-
-
 if sys.version_info < (3, 5):
     # Ugly hack...
     RecursionError = RuntimeError
@@ -134,15 +118,6 @@ def unpackb(packed, **kwargs):
     return ret
 
 
-if sys.version_info < (2, 7, 6):
-
-    def _unpack_from(f, b, o=0):
-        """Explicit type cast for legacy struct.unpack_from"""
-        return struct.unpack_from(f, bytes(b), o)
-
-else:
-    _unpack_from = struct.unpack_from
-
 _NO_FORMAT_USED = ""
 _MSGPACK_HEADERS = {
     0xC4: (1, _NO_FORMAT_USED, TYPE_BIN),
@@ -202,7 +177,7 @@ class Unpacker(object):
             0 - Timestamp
             1 - float  (Seconds from the EPOCH)
             2 - int  (Nanoseconds from the EPOCH)
-            3 - datetime.datetime  (UTC).  Python 2 is not supported.
+            3 - datetime.datetime  (UTC).
 
     :param bool strict_map_key:
         If true (default), only str or bytes are accepted for map (dict) keys.
@@ -477,7 +452,7 @@ class Unpacker(object):
             size, fmt, typ = _MSGPACK_HEADERS[b]
             self._reserve(size)
             if len(fmt) > 0:
-                n = _unpack_from(fmt, self._buffer, self._buff_i)[0]
+                n = struct.unpack_from(fmt, self._buffer, self._buff_i)[0]
             else:
                 n = self._buffer[self._buff_i]
             self._buff_i += size
@@ -487,7 +462,7 @@ class Unpacker(object):
         elif 0xC7 <= b <= 0xC9:
             size, fmt, typ = _MSGPACK_HEADERS[b]
             self._reserve(size)
-            L, n = _unpack_from(fmt, self._buffer, self._buff_i)
+            L, n = struct.unpack_from(fmt, self._buffer, self._buff_i)
             self._buff_i += size
             if L > self._max_ext_len:
                 raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
@@ -496,7 +471,7 @@ class Unpacker(object):
             size, fmt = _MSGPACK_HEADERS[b]
             self._reserve(size)
             if len(fmt) > 0:
-                obj = _unpack_from(fmt, self._buffer, self._buff_i)[0]
+                obj = struct.unpack_from(fmt, self._buffer, self._buff_i)[0]
             else:
                 obj = self._buffer[self._buff_i]
             self._buff_i += size
@@ -507,13 +482,13 @@ class Unpacker(object):
                     "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len)
                 )
             self._reserve(size + 1)
-            n, obj = _unpack_from(fmt, self._buffer, self._buff_i)
+            n, obj = struct.unpack_from(fmt, self._buffer, self._buff_i)
             self._buff_i += size + 1
         elif 0xD9 <= b <= 0xDB:
             size, fmt, typ = _MSGPACK_HEADERS[b]
             self._reserve(size)
             if len(fmt) > 0:
-                (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
+                (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i)
             else:
                 n = self._buffer[self._buff_i]
             self._buff_i += size
@@ -523,7 +498,7 @@ class Unpacker(object):
         elif 0xDC <= b <= 0xDD:
             size, fmt, typ = _MSGPACK_HEADERS[b]
             self._reserve(size)
-            (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
+            (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i)
             self._buff_i += size
             if n > self._max_array_len:
                 raise ValueError(
@@ -532,7 +507,7 @@ class Unpacker(object):
         elif 0xDE <= b <= 0xDF:
             size, fmt, typ = _MSGPACK_HEADERS[b]
             self._reserve(size)
-            (n,) = _unpack_from(fmt, self._buffer, self._buff_i)
+            (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i)
             self._buff_i += size
             if n > self._max_map_len:
                 raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len))
@@ -554,12 +529,12 @@ class Unpacker(object):
         # TODO should we eliminate the recursion?
         if typ == TYPE_ARRAY:
             if execute == EX_SKIP:
-                for i in xrange(n):
+                for i in range(n):
                     # TODO check whether we need to call `list_hook`
                     self._unpack(EX_SKIP)
                 return
             ret = newlist_hint(n)
-            for i in xrange(n):
+            for i in range(n):
                 ret.append(self._unpack(EX_CONSTRUCT))
             if self._list_hook is not None:
                 ret = self._list_hook(ret)
@@ -567,7 +542,7 @@ class Unpacker(object):
             return ret if self._use_list else tuple(ret)
         if typ == TYPE_MAP:
             if execute == EX_SKIP:
-                for i in xrange(n):
+                for i in range(n):
                     # TODO check whether we need to call hooks
                     self._unpack(EX_SKIP)
                     self._unpack(EX_SKIP)
@@ -575,17 +550,17 @@ class Unpacker(object):
             if self._object_pairs_hook is not None:
                 ret = self._object_pairs_hook(
                     (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT))
-                    for _ in xrange(n)
+                    for _ in range(n)
                 )
             else:
                 ret = {}
-                for _ in xrange(n):
+                for _ in range(n):
                     key = self._unpack(EX_CONSTRUCT)
-                    if self._strict_map_key and type(key) not in (unicode, bytes):
+                    if self._strict_map_key and type(key) not in (str, bytes):
                         raise ValueError(
                             "%s is not allowed for map key" % str(type(key))
                         )
-                    if not PY2 and type(key) is str:
+                    if type(key) is str:
                         key = sys.intern(key)
                     ret[key] = self._unpack(EX_CONSTRUCT)
                 if self._object_hook is not None:
@@ -698,7 +673,6 @@ class Packer(object):
         If set to true, datetime with tzinfo is packed into Timestamp type.
         Note that the tzinfo is stripped in the timestamp.
         You can get UTC datetime with `timestamp=3` option of the Unpacker.
-        (Python 2 is not supported).
 
     :param str unicode_errors:
         The error handler for encoding unicode. (default: 'strict')
@@ -743,8 +717,6 @@ class Packer(object):
         self._autoreset = autoreset
         self._use_bin_type = use_bin_type
         self._buffer = StringIO()
-        if PY2 and datetime:
-            raise ValueError("datetime is not supported in Python 2")
         self._datetime = bool(datetime)
         self._unicode_errors = unicode_errors or "strict"
         if default is not None:
@@ -774,7 +746,7 @@ class Packer(object):
                 if obj:
                     return self._buffer.write(b"\xc3")
                 return self._buffer.write(b"\xc2")
-            if check(obj, int_types):
+            if check(obj, int):
                 if 0 <= obj < 0x80:
                     return self._buffer.write(struct.pack("B", obj))
                 if -0x20 <= obj < 0:
@@ -806,7 +778,7 @@ class Packer(object):
                     raise ValueError("%s is too large" % type(obj).__name__)
                 self._pack_bin_header(n)
                 return self._buffer.write(obj)
-            if check(obj, unicode):
+            if check(obj, str):
                 obj = obj.encode("utf-8", self._unicode_errors)
                 n = len(obj)
                 if n >= 2**32:
@@ -855,13 +827,11 @@ class Packer(object):
             if check(obj, list_types):
                 n = len(obj)
                 self._pack_array_header(n)
-                for i in xrange(n):
+                for i in range(n):
                     self._pack(obj[i], nest_limit - 1)
                 return
             if check(obj, dict):
-                return self._pack_map_pairs(
-                    len(obj), dict_iteritems(obj), nest_limit - 1
-                )
+                return self._pack_map_pairs(len(obj), obj.items(), nest_limit - 1)
 
             if self._datetime and check(obj, _DateTime) and obj.tzinfo is not None:
                 obj = Timestamp.from_datetime(obj)
@@ -1004,7 +974,7 @@ class Packer(object):
 
     def getbuffer(self):
         """Return view of internal buffer."""
-        if USING_STRINGBUILDER or PY2:
+        if USING_STRINGBUILDER:
             return memoryview(self.bytes())
         else:
             return self._buffer.getbuffer()
diff --git a/setup.py b/setup.py
index 9630cda..15ba774 100755
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,6 @@ from setuptools.command.sdist import sdist
 
 
 PYPY = hasattr(sys, "pypy_version_info")
-PY2 = sys.version_info[0] == 2
 
 
 class NoCython(Exception):
@@ -79,7 +78,7 @@ if sys.platform == "win32":
     macros = [("__LITTLE_ENDIAN__", "1")]
 
 ext_modules = []
-if not PYPY and not PY2 and not os.environ.get("MSGPACK_PUREPYTHON"):
+if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"):
     ext_modules.append(
         Extension(
             "msgpack._cmsgpack",
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 62507cf..0447058 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -6,7 +6,6 @@ import pytest
 from msgpack import packb, unpackb
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2, reason="Python 2 is not supported")
 def test_unpack_buffer():
     from array import array
 
diff --git a/test/test_case.py b/test/test_case.py
index a0a3c5a..1c4e322 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -134,4 +134,4 @@ def test_match():
 
 
 def test_unicode():
-    assert unpackb(packb(u"foobar"), use_list=1) == u"foobar"
+    assert unpackb(packb("foobar"), use_list=1) == "foobar"
diff --git a/test/test_except.py b/test/test_except.py
index 5544f2b..745ebec 100644
--- a/test/test_except.py
+++ b/test/test_except.py
@@ -53,7 +53,7 @@ def test_invalidvalue():
 
 
 def test_strict_map_key():
-    valid = {u"unicode": 1, b"bytes": 2}
+    valid = {"unicode": 1, b"bytes": 2}
     packed = packb(valid, use_bin_type=True)
     assert valid == unpackb(packed, raw=False, strict_map_key=True)
 
diff --git a/test/test_extension.py b/test/test_extension.py
index 6b36575..dfbe435 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -55,10 +55,7 @@ def test_extension_type():
         print("ext_hook called", code, data)
         assert code == 123
         obj = array.array("d")
-        try:
-            obj.frombytes(data)
-        except AttributeError:  # PY2
-            obj.fromstring(data)
+        obj.frombytes(data)
         return obj
 
     obj = [42, b"hello", array.array("d", [1.1, 2.2, 3.3])]
@@ -67,20 +64,14 @@ def test_extension_type():
     assert obj == obj2
 
 
-import sys
-
-if sys.version > "3":
-    long = int
-
-
 def test_overriding_hooks():
     def default(obj):
-        if isinstance(obj, long):
+        if isinstance(obj, int):
             return {"__type__": "long", "__data__": str(obj)}
         else:
             return obj
 
-    obj = {"testval": long(1823746192837461928374619)}
+    obj = {"testval": 1823746192837461928374619}
     refobj = {"testval": default(obj["testval"])}
     refout = msgpack.packb(refobj)
     assert isinstance(refout, (str, bytes))
diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index a0939a6..63beab1 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -7,11 +7,6 @@ from msgpack import packb, unpackb
 import sys
 
 
-pytestmark = pytest.mark.skipif(
-    sys.version_info[0] < 3, reason="Only Python 3 supports buffer protocol"
-)
-
-
 def make_array(f, data):
     a = array(f)
     a.frombytes(data)
diff --git a/test/test_pack.py b/test/test_pack.py
index a51d84c..65c9cb1 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -80,9 +80,6 @@ def testPackByteArrays():
         check(td)
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates"
-)
 def testIgnoreUnicodeErrors():
     re = unpackb(
         packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore"
@@ -96,9 +93,6 @@ def testStrictUnicodeUnpack():
         unpackb(packed, raw=False, use_list=1)
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 0), reason="Python 2 passes invalid surrogates"
-)
 def testIgnoreErrorsPack():
     re = unpackb(
         packb("abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors="ignore"),
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index 253228e..af84a2f 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -4,9 +4,6 @@ import datetime
 import msgpack
 from msgpack.ext import Timestamp
 
-if sys.version_info[0] > 2:
-    from msgpack.ext import _utc
-
 
 def test_timestamp():
     # timestamp32
@@ -85,33 +82,33 @@ def test_timestamp_to():
     assert t.to_unix_nano() == 42000014000
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
 def test_timestamp_datetime():
     t = Timestamp(42, 14)
-    assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc)
+    utc = datetime.timezone.utc
+    assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc)
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
 def test_unpack_datetime():
     t = Timestamp(42, 14)
+    utc = datetime.timezone.utc
     packed = msgpack.packb(t)
     unpacked = msgpack.unpackb(packed, timestamp=3)
-    assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=_utc)
+    assert unpacked == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc)
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
 def test_pack_unpack_before_epoch():
-    t_in = datetime.datetime(1960, 1, 1, tzinfo=_utc)
+    utc = datetime.timezone.utc
+    t_in = datetime.datetime(1960, 1, 1, tzinfo=utc)
     packed = msgpack.packb(t_in, datetime=True)
     unpacked = msgpack.unpackb(packed, timestamp=3)
     assert unpacked == t_in
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
 def test_pack_datetime():
     t = Timestamp(42, 14000)
     dt = t.to_datetime()
-    assert dt == datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc)
+    utc = datetime.timezone.utc
+    assert dt == datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=utc)
 
     packed = msgpack.packb(dt, datetime=True)
     packed2 = msgpack.packb(t)
@@ -131,10 +128,10 @@ def test_pack_datetime():
     assert msgpack.unpackb(packed) is None
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
 def test_issue451():
     # https://github.com/msgpack/msgpack-python/issues/451
-    dt = datetime.datetime(2100, 1, 1, 1, 1, tzinfo=_utc)
+    utc = datetime.timezone.utc
+    dt = datetime.datetime(2100, 1, 1, 1, 1, tzinfo=utc)
     packed = msgpack.packb(dt, datetime=True)
     assert packed == b"\xd6\xff\xf4\x86eL"
 
@@ -142,7 +139,6 @@ def test_issue451():
     assert dt == unpacked
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2, reason="datetime support is PY3+ only")
 def test_pack_datetime_without_tzinfo():
     dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14)
     with pytest.raises(ValueError, match="where tzinfo=None"):
@@ -152,7 +148,8 @@ def test_pack_datetime_without_tzinfo():
     packed = msgpack.packb(dt, datetime=True, default=lambda x: None)
     assert packed == msgpack.packb(None)
 
-    dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=_utc)
+    utc = datetime.timezone.utc
+    dt = datetime.datetime(1970, 1, 1, 0, 0, 42, 14, tzinfo=utc)
     packed = msgpack.packb(dt, datetime=True)
     unpacked = msgpack.unpackb(packed, timestamp=3)
     assert unpacked == dt
diff --git a/test/test_unpack.py b/test/test_unpack.py
index aa4c01f..c714102 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -70,7 +70,7 @@ def test_unpacker_ext_hook():
 
 
 def test_unpacker_tell():
-    objects = 1, 2, u"abc", u"def", u"ghi"
+    objects = 1, 2, "abc", "def", "ghi"
     packed = b"\x01\x02\xa3abc\xa3def\xa3ghi"
     positions = 1, 2, 6, 10, 14
     unpacker = Unpacker(BytesIO(packed))
@@ -80,7 +80,7 @@ def test_unpacker_tell():
 
 
 def test_unpacker_tell_read_bytes():
-    objects = 1, u"abc", u"ghi"
+    objects = 1, "abc", "ghi"
     packed = b"\x01\x02\xa3abc\xa3def\xa3ghi"
     raw_data = b"\x02", b"\xa3def", b""
     lenghts = 1, 4, 999
diff --git a/tox.ini b/tox.ini
index 29c256d..1ef2d18 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,9 +1,7 @@
 [tox]
 envlist =
-    py27-pure,
     {py35,py36,py37,py38}-{c,pure},
     {pypy,pypy3}-pure,
-    py27-x86,
     py34-x86,
 isolated_build = true
 
@@ -19,17 +17,6 @@ commands=
 setenv=
     pure: MSGPACK_PUREPYTHON=x
 
-[testenv:py27-x86]
-basepython=python2.7-x86
-deps=
-    pytest
-
-changedir=test
-commands=
-    python -c 'import sys; print(hex(sys.maxsize))'
-    python -c 'from msgpack import _cmsgpack'
-    py.test
-
 [testenv:py34-x86]
 basepython=python3.4-x86
 deps=

From c8d0751fe3375a5e8005b5edf955cd5904aaec2f Mon Sep 17 00:00:00 2001
From: Evgeny Markov 
Date: Tue, 23 May 2023 18:41:08 +0200
Subject: [PATCH 1593/1648] Drop Python 3.6 support (#543)

The following steps have been taken:

1. Black was updated to latest version. The code has been formatted with
the new version.
2. The pyupgrade utility is installed. This helped to remove all the
code that was needed to support Python < 3.7.

Fix #541.

Co-authored-by: Inada Naoki 
---
 Makefile                |  8 ++++-
 msgpack/__init__.py     |  1 -
 msgpack/ext.py          | 21 ++++---------
 msgpack/fallback.py     | 70 ++++++++++++-----------------------------
 pyproject.toml          |  5 +++
 requirements.txt        |  8 +++--
 setup.cfg               |  1 -
 setup.py                |  9 ++----
 test/test_buffer.py     |  1 -
 test/test_case.py       |  7 ++---
 test/test_except.py     |  1 -
 test/test_extension.py  |  7 ++---
 test/test_format.py     |  5 +--
 test/test_limits.py     |  2 --
 test/test_memoryview.py |  1 -
 test/test_newspec.py    |  2 --
 test/test_obj.py        |  1 -
 test/test_pack.py       | 12 +++----
 test/test_seq.py        |  1 -
 test/test_sequnpack.py  |  1 -
 test/test_stricttype.py | 12 +++----
 test/test_subtype.py    |  1 -
 test/test_unpack.py     |  2 +-
 23 files changed, 60 insertions(+), 119 deletions(-)

diff --git a/Makefile b/Makefile
index 415dcfd..e4f22da 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,16 @@
+PYTHON_SOURCES = msgpack test setup.py
+
 .PHONY: all
 all: cython
 	python setup.py build_ext -i -f
 
 .PHONY: black
 black:
-	black -S msgpack/ test/ setup.py
+	black $(PYTHON_SOURCES)
+
+.PHONY: pyupgrade
+pyupgrade:
+	@find $(PYTHON_SOURCES) -name '*.py' -type f -exec pyupgrade --py37-plus '{}' \;
 
 .PHONY: cython
 cython:
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 6c10dc2..638236f 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,4 +1,3 @@
-# coding: utf-8
 from .exceptions import *
 from .ext import ExtType, Timestamp
 
diff --git a/msgpack/ext.py b/msgpack/ext.py
index 07f96a5..9794294 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -1,4 +1,3 @@
-# coding: utf-8
 from collections import namedtuple
 import datetime
 import sys
@@ -15,10 +14,10 @@ class ExtType(namedtuple("ExtType", "code data")):
             raise TypeError("data must be bytes")
         if not 0 <= code <= 127:
             raise ValueError("code must be 0~127")
-        return super(ExtType, cls).__new__(cls, code, data)
+        return super().__new__(cls, code, data)
 
 
-class Timestamp(object):
+class Timestamp:
     """Timestamp represents the Timestamp extension type in msgpack.
 
     When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python
@@ -47,24 +46,18 @@ class Timestamp(object):
         if not isinstance(nanoseconds, int):
             raise TypeError("nanoseconds must be an integer")
         if not (0 <= nanoseconds < 10**9):
-            raise ValueError(
-                "nanoseconds must be a non-negative integer less than 999999999."
-            )
+            raise ValueError("nanoseconds must be a non-negative integer less than 999999999.")
         self.seconds = seconds
         self.nanoseconds = nanoseconds
 
     def __repr__(self):
         """String representation of Timestamp."""
-        return "Timestamp(seconds={0}, nanoseconds={1})".format(
-            self.seconds, self.nanoseconds
-        )
+        return f"Timestamp(seconds={self.seconds}, nanoseconds={self.nanoseconds})"
 
     def __eq__(self, other):
         """Check for equality with another Timestamp object"""
         if type(other) is self.__class__:
-            return (
-                self.seconds == other.seconds and self.nanoseconds == other.nanoseconds
-            )
+            return self.seconds == other.seconds and self.nanoseconds == other.nanoseconds
         return False
 
     def __ne__(self, other):
@@ -164,9 +157,7 @@ class Timestamp(object):
         :rtype: datetime.
         """
         utc = datetime.timezone.utc
-        return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(
-            seconds=self.to_unix()
-        )
+        return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix())
 
     @staticmethod
     def from_datetime(dt):
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 618c362..ac1eaf4 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -4,23 +4,6 @@ import sys
 import struct
 
 
-if sys.version_info < (3, 5):
-    # Ugly hack...
-    RecursionError = RuntimeError
-
-    def _is_recursionerror(e):
-        return (
-            len(e.args) == 1
-            and isinstance(e.args[0], str)
-            and e.args[0].startswith("maximum recursion depth exceeded")
-        )
-
-else:
-
-    def _is_recursionerror(e):
-        return True
-
-
 if hasattr(sys, "pypy_version_info"):
     # StringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
     # StringBuilder is fastest.
@@ -32,7 +15,7 @@ if hasattr(sys, "pypy_version_info"):
         from __pypy__.builders import StringBuilder
     USING_STRINGBUILDER = True
 
-    class StringIO(object):
+    class StringIO:
         def __init__(self, s=b""):
             if s:
                 self.builder = StringBuilder(len(s))
@@ -109,10 +92,8 @@ def unpackb(packed, **kwargs):
         ret = unpacker._unpack()
     except OutOfData:
         raise ValueError("Unpack failed: incomplete input")
-    except RecursionError as e:
-        if _is_recursionerror(e):
-            raise StackError
-        raise
+    except RecursionError:
+        raise StackError
     if unpacker._got_extradata():
         raise ExtraData(ret, unpacker._get_extradata())
     return ret
@@ -151,7 +132,7 @@ _MSGPACK_HEADERS = {
 }
 
 
-class Unpacker(object):
+class Unpacker:
     """Streaming unpacker.
 
     Arguments:
@@ -334,9 +315,7 @@ class Unpacker(object):
         if object_pairs_hook is not None and not callable(object_pairs_hook):
             raise TypeError("`object_pairs_hook` is not callable")
         if object_hook is not None and object_pairs_hook is not None:
-            raise TypeError(
-                "object_pairs_hook and object_hook are mutually " "exclusive"
-            )
+            raise TypeError("object_pairs_hook and object_hook are mutually exclusive")
         if not callable(ext_hook):
             raise TypeError("`ext_hook` is not callable")
 
@@ -428,20 +407,18 @@ class Unpacker(object):
             n = b & 0b00011111
             typ = TYPE_RAW
             if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len))
+                raise ValueError(f"{n} exceeds max_str_len({self._max_str_len})")
             obj = self._read(n)
         elif b & 0b11110000 == 0b10010000:
             n = b & 0b00001111
             typ = TYPE_ARRAY
             if n > self._max_array_len:
-                raise ValueError(
-                    "%s exceeds max_array_len(%s)" % (n, self._max_array_len)
-                )
+                raise ValueError(f"{n} exceeds max_array_len({self._max_array_len})")
         elif b & 0b11110000 == 0b10000000:
             n = b & 0b00001111
             typ = TYPE_MAP
             if n > self._max_map_len:
-                raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len))
+                raise ValueError(f"{n} exceeds max_map_len({self._max_map_len})")
         elif b == 0xC0:
             obj = None
         elif b == 0xC2:
@@ -457,7 +434,7 @@ class Unpacker(object):
                 n = self._buffer[self._buff_i]
             self._buff_i += size
             if n > self._max_bin_len:
-                raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len))
+                raise ValueError(f"{n} exceeds max_bin_len({self._max_bin_len})")
             obj = self._read(n)
         elif 0xC7 <= b <= 0xC9:
             size, fmt, typ = _MSGPACK_HEADERS[b]
@@ -465,7 +442,7 @@ class Unpacker(object):
             L, n = struct.unpack_from(fmt, self._buffer, self._buff_i)
             self._buff_i += size
             if L > self._max_ext_len:
-                raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len))
+                raise ValueError(f"{L} exceeds max_ext_len({self._max_ext_len})")
             obj = self._read(L)
         elif 0xCA <= b <= 0xD3:
             size, fmt = _MSGPACK_HEADERS[b]
@@ -478,9 +455,7 @@ class Unpacker(object):
         elif 0xD4 <= b <= 0xD8:
             size, fmt, typ = _MSGPACK_HEADERS[b]
             if self._max_ext_len < size:
-                raise ValueError(
-                    "%s exceeds max_ext_len(%s)" % (size, self._max_ext_len)
-                )
+                raise ValueError(f"{size} exceeds max_ext_len({self._max_ext_len})")
             self._reserve(size + 1)
             n, obj = struct.unpack_from(fmt, self._buffer, self._buff_i)
             self._buff_i += size + 1
@@ -493,7 +468,7 @@ class Unpacker(object):
                 n = self._buffer[self._buff_i]
             self._buff_i += size
             if n > self._max_str_len:
-                raise ValueError("%s exceeds max_str_len(%s)" % (n, self._max_str_len))
+                raise ValueError(f"{n} exceeds max_str_len({self._max_str_len})")
             obj = self._read(n)
         elif 0xDC <= b <= 0xDD:
             size, fmt, typ = _MSGPACK_HEADERS[b]
@@ -501,16 +476,14 @@ class Unpacker(object):
             (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i)
             self._buff_i += size
             if n > self._max_array_len:
-                raise ValueError(
-                    "%s exceeds max_array_len(%s)" % (n, self._max_array_len)
-                )
+                raise ValueError(f"{n} exceeds max_array_len({self._max_array_len})")
         elif 0xDE <= b <= 0xDF:
             size, fmt, typ = _MSGPACK_HEADERS[b]
             self._reserve(size)
             (n,) = struct.unpack_from(fmt, self._buffer, self._buff_i)
             self._buff_i += size
             if n > self._max_map_len:
-                raise ValueError("%s exceeds max_map_len(%s)" % (n, self._max_map_len))
+                raise ValueError(f"{n} exceeds max_map_len({self._max_map_len})")
         else:
             raise FormatError("Unknown header: 0x%x" % b)
         return typ, n, obj
@@ -549,17 +522,14 @@ class Unpacker(object):
                 return
             if self._object_pairs_hook is not None:
                 ret = self._object_pairs_hook(
-                    (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT))
-                    for _ in range(n)
+                    (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) for _ in range(n)
                 )
             else:
                 ret = {}
                 for _ in range(n):
                     key = self._unpack(EX_CONSTRUCT)
                     if self._strict_map_key and type(key) not in (str, bytes):
-                        raise ValueError(
-                            "%s is not allowed for map key" % str(type(key))
-                        )
+                        raise ValueError("%s is not allowed for map key" % str(type(key)))
                     if type(key) is str:
                         key = sys.intern(key)
                     ret[key] = self._unpack(EX_CONSTRUCT)
@@ -634,7 +604,7 @@ class Unpacker(object):
         return self._stream_offset
 
 
-class Packer(object):
+class Packer:
     """
     MessagePack Packer
 
@@ -844,9 +814,9 @@ class Packer(object):
                 continue
 
             if self._datetime and check(obj, _DateTime):
-                raise ValueError("Cannot serialize %r where tzinfo=None" % (obj,))
+                raise ValueError(f"Cannot serialize {obj!r} where tzinfo=None")
 
-            raise TypeError("Cannot serialize %r" % (obj,))
+            raise TypeError(f"Cannot serialize {obj!r}")
 
     def pack(self, obj):
         try:
@@ -933,7 +903,7 @@ class Packer(object):
 
     def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT):
         self._pack_map_header(n)
-        for (k, v) in pairs:
+        for k, v in pairs:
             self._pack(k, nest_limit - 1)
             self._pack(v, nest_limit - 1)
 
diff --git a/pyproject.toml b/pyproject.toml
index 195795f..86fae1c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,3 +6,8 @@ requires = [
     "setuptools >= 35.0.2",
 ]
 build-backend = "setuptools.build_meta"
+
+[tool.black]
+line-length = 100
+target-version = ["py37"]
+skip_string_normalization = true
diff --git a/requirements.txt b/requirements.txt
index 9f3c1a0..88b5eb9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,7 @@
-# Also declared in pyproject.toml, if updating here please also update there
+# Also declared in pyproject.toml, if updating here please also update there.
 Cython~=0.29.30
 
-# dev only tools. no need to add pyproject
-black==22.3.0
+# Tools required only for development. No need to add it to pyproject.toml file.
+black==23.3.0
+pytest==7.3.1
+pyupgrade==3.3.2
diff --git a/setup.cfg b/setup.cfg
index e378284..d6888fc 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -17,7 +17,6 @@ project_urls =
 
 classifiers =
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
diff --git a/setup.py b/setup.py
index 15ba774..1cd1e8e 100755
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding: utf-8
 import io
 import os
 import sys
@@ -25,7 +24,7 @@ except ImportError:
 
 
 def cythonize(src):
-    sys.stderr.write("cythonize: %r\n" % (src,))
+    sys.stderr.write(f"cythonize: {src!r}\n")
     cython_compiler.compile([src], cplus=True)
 
 
@@ -36,11 +35,7 @@ def ensure_source(src):
         if not have_cython:
             raise NoCython
         cythonize(pyx)
-    elif (
-        os.path.exists(pyx)
-        and os.stat(src).st_mtime < os.stat(pyx).st_mtime
-        and have_cython
-    ):
+    elif os.path.exists(pyx) and os.stat(src).st_mtime < os.stat(pyx).st_mtime and have_cython:
         cythonize(pyx)
     return src
 
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 0447058..7ee674a 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding: utf-8
 
 import sys
 import pytest
diff --git a/test/test_case.py b/test/test_case.py
index 1c4e322..c4c615e 100644
--- a/test/test_case.py
+++ b/test/test_case.py
@@ -1,11 +1,10 @@
 #!/usr/bin/env python
-# coding: utf-8
 from msgpack import packb, unpackb
 
 
 def check(length, obj, use_bin_type=True):
     v = packb(obj, use_bin_type=use_bin_type)
-    assert len(v) == length, "%r length should be %r but get %r" % (obj, length, len(v))
+    assert len(v) == length, f"{obj!r} length should be {length!r} but get {len(v)!r}"
     assert unpackb(v, use_list=0, raw=not use_bin_type) == obj
 
 
@@ -120,11 +119,11 @@ def test_match():
         ),
         ({}, b"\x80"),
         (
-            dict([(x, x) for x in range(15)]),
+            {x: x for x in range(15)},
             b"\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e",
         ),
         (
-            dict([(x, x) for x in range(16)]),
+            {x: x for x in range(16)},
             b"\xde\x00\x10\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07\x07\x08\x08\t\t\n\n\x0b\x0b\x0c\x0c\r\r\x0e\x0e\x0f\x0f",
         ),
     ]
diff --git a/test/test_except.py b/test/test_except.py
index 745ebec..8c0a976 100644
--- a/test/test_except.py
+++ b/test/test_except.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding: utf-8
 
 from pytest import raises
 from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData
diff --git a/test/test_extension.py b/test/test_extension.py
index dfbe435..9e5e6aa 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import array
 import msgpack
 from msgpack import ExtType
@@ -17,9 +16,7 @@ def test_pack_ext_type():
     assert p(b"A" * 16) == b"\xd8\x42" + b"A" * 16  # fixext 16
     assert p(b"ABC") == b"\xc7\x03\x42ABC"  # ext 8
     assert p(b"A" * 0x0123) == b"\xc8\x01\x23\x42" + b"A" * 0x0123  # ext 16
-    assert (
-        p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345
-    )  # ext 32
+    assert p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345  # ext 32
 
 
 def test_unpack_ext_type():
@@ -49,7 +46,7 @@ def test_extension_type():
             except AttributeError:
                 data = obj.tostring()
             return ExtType(typecode, data)
-        raise TypeError("Unknown type object %r" % (obj,))
+        raise TypeError(f"Unknown type object {obj!r}")
 
     def ext_hook(code, data):
         print("ext_hook called", code, data)
diff --git a/test/test_format.py b/test/test_format.py
index fbbc3f9..c06c87d 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding: utf-8
 
 from msgpack import unpackb
 
@@ -25,9 +24,7 @@ def testFixRaw():
 
 
 def testFixMap():
-    check(
-        b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}}
-    )
+    check(b"\x82\xc2\x81\xc0\xc0\xc3\x81\xc0\x80", {False: {None: None}, True: {None: {}}})
 
 
 def testUnsignedInt():
diff --git a/test/test_limits.py b/test/test_limits.py
index 4314c2c..533bc11 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -1,6 +1,4 @@
 #!/usr/bin/env python
-# coding: utf-8
-from __future__ import absolute_import, division, print_function, unicode_literals
 import pytest
 
 from msgpack import (
diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index 63beab1..eaadef7 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding: utf-8
 
 import pytest
 from array import array
diff --git a/test/test_newspec.py b/test/test_newspec.py
index b7da486..a6f4251 100644
--- a/test/test_newspec.py
+++ b/test/test_newspec.py
@@ -1,5 +1,3 @@
-# coding: utf-8
-
 from msgpack import packb, unpackb, ExtType
 
 
diff --git a/test/test_obj.py b/test/test_obj.py
index 86c557c..d3f870d 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding: utf-8
 
 from pytest import raises
 from msgpack import packb, unpackb
diff --git a/test/test_pack.py b/test/test_pack.py
index 65c9cb1..2753e46 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -1,6 +1,4 @@
 #!/usr/bin/env python
-# coding: utf-8
-from __future__ import absolute_import, division, print_function, unicode_literals
 
 from collections import OrderedDict
 from io import BytesIO
@@ -81,9 +79,7 @@ def testPackByteArrays():
 
 
 def testIgnoreUnicodeErrors():
-    re = unpackb(
-        packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore"
-    )
+    re = unpackb(packb(b"abc\xeddef", use_bin_type=False), raw=False, unicode_errors="ignore")
     assert re == "abcdef"
 
 
@@ -108,8 +104,8 @@ def testDecodeBinary():
 
 
 def testPackFloat():
-    assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(str(">f"), 1.0)
-    assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(str(">d"), 1.0)
+    assert packb(1.0, use_single_float=True) == b"\xca" + struct.pack(">f", 1.0)
+    assert packb(1.0, use_single_float=False) == b"\xcb" + struct.pack(">d", 1.0)
 
 
 def testArraySize(sizes=[0, 5, 50, 1000]):
@@ -154,7 +150,7 @@ def testMapSize(sizes=[0, 5, 50, 1000]):
     bio.seek(0)
     unpacker = Unpacker(bio, strict_map_key=False)
     for size in sizes:
-        assert unpacker.unpack() == dict((i, i * 2) for i in range(size))
+        assert unpacker.unpack() == {i: i * 2 for i in range(size)}
 
 
 def test_odict():
diff --git a/test/test_seq.py b/test/test_seq.py
index 0d5d806..def6630 100644
--- a/test/test_seq.py
+++ b/test/test_seq.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding: utf-8
 
 import io
 import msgpack
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index c091076..6b138aa 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding: utf-8
 import io
 from msgpack import Unpacker, BufferFull
 from msgpack import pack, packb
diff --git a/test/test_stricttype.py b/test/test_stricttype.py
index fe9ec6c..9ffaff2 100644
--- a/test/test_stricttype.py
+++ b/test/test_stricttype.py
@@ -1,5 +1,3 @@
-# coding: utf-8
-
 from collections import namedtuple
 from msgpack import packb, unpackb, ExtType
 
@@ -10,7 +8,7 @@ def test_namedtuple():
     def default(o):
         if isinstance(o, T):
             return dict(o._asdict())
-        raise TypeError("Unsupported type %s" % (type(o),))
+        raise TypeError(f"Unsupported type {type(o)}")
 
     packed = packb(T(1, 42), strict_types=True, use_bin_type=True, default=default)
     unpacked = unpackb(packed, raw=False)
@@ -23,7 +21,7 @@ def test_tuple():
     def default(o):
         if isinstance(o, tuple):
             return {"__type__": "tuple", "value": list(o)}
-        raise TypeError("Unsupported type %s" % (type(o),))
+        raise TypeError(f"Unsupported type {type(o)}")
 
     def convert(o):
         if o.get("__type__") == "tuple":
@@ -44,9 +42,7 @@ def test_tuple_ext():
     def default(o):
         if isinstance(o, tuple):
             # Convert to list and pack
-            payload = packb(
-                list(o), strict_types=True, use_bin_type=True, default=default
-            )
+            payload = packb(list(o), strict_types=True, use_bin_type=True, default=default)
             return ExtType(MSGPACK_EXT_TYPE_TUPLE, payload)
         raise TypeError(repr(o))
 
@@ -54,7 +50,7 @@ def test_tuple_ext():
         if code == MSGPACK_EXT_TYPE_TUPLE:
             # Unpack and convert to tuple
             return tuple(unpackb(payload, raw=False, ext_hook=convert))
-        raise ValueError("Unknown Ext code {}".format(code))
+        raise ValueError(f"Unknown Ext code {code}")
 
     data = packb(t, strict_types=True, use_bin_type=True, default=default)
     expected = unpackb(data, raw=False, ext_hook=convert)
diff --git a/test/test_subtype.py b/test/test_subtype.py
index d91d455..d5a9adb 100644
--- a/test/test_subtype.py
+++ b/test/test_subtype.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# coding: utf-8
 
 from msgpack import packb, unpackb
 from collections import namedtuple
diff --git a/test/test_unpack.py b/test/test_unpack.py
index c714102..bf3f960 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -52,7 +52,7 @@ def test_unpacker_hook_refcnt():
 def test_unpacker_ext_hook():
     class MyUnpacker(Unpacker):
         def __init__(self):
-            super(MyUnpacker, self).__init__(ext_hook=self._hook, raw=False)
+            super().__init__(ext_hook=self._hook, raw=False)
 
         def _hook(self, code, data):
             if code == 1:

From e5249f877c18c88a0009f21097d7e48819579e60 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 21 Jul 2023 02:53:58 +0900
Subject: [PATCH 1594/1648] ci: add Python 3.12 and drop 3.7

---
 .github/workflows/test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8878102..76fcf27 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "windows-latest", "macos-latest"]
-        py: ["3.11", "3.10", "3.9", "3.8", "3.7"]
+        py: ["3.12", "3.11", "3.10", "3.9", "3.8"]
 
     runs-on: ${{ matrix.os }}
     name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}
@@ -23,6 +23,7 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.py }}
+          allow-prereleases: true
           cache: "pip"
 
       - name: Build

From 427736bbcc5553b0754616b58154ed26733103b6 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 21 Jul 2023 11:11:04 +0900
Subject: [PATCH 1595/1648] try Cython 3.0 (#548)

---
 pyproject.toml   | 2 +-
 requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 86fae1c..dc8bbee 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -2,7 +2,7 @@
 requires = [
     # Also declared in requirements.txt, if updating here please also update
     # there
-    "Cython~=0.29.30",
+    "Cython~=3.0.0",
     "setuptools >= 35.0.2",
 ]
 build-backend = "setuptools.build_meta"
diff --git a/requirements.txt b/requirements.txt
index 88b5eb9..e27df0f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 # Also declared in pyproject.toml, if updating here please also update there.
-Cython~=0.29.30
+Cython~=3.0.0
 
 # Tools required only for development. No need to add it to pyproject.toml file.
 black==23.3.0

From 7cfced51501b8e0786da5d6b499331ef2f492b29 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 9 Aug 2023 18:09:42 +0900
Subject: [PATCH 1596/1648] start v1.0.6 development

---
 msgpack/__init__.py | 2 +-
 setup.cfg           | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 638236f..2540120 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -6,7 +6,7 @@ import sys
 
 
 version = (1, 0, 5)
-__version__ = "1.0.5"
+__version__ = "1.0.6dev1"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):
diff --git a/setup.cfg b/setup.cfg
index d6888fc..c2e1672 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,7 +1,7 @@
 [metadata]
 name = msgpack
-#version = attr: msgpack.__version__
-version = attr: msgpack.version
+version = attr: msgpack.__version__
+#version = attr: msgpack.version
 license = Apache 2.0
 author = Inada Naoki
 author_email = songofacandy@gmail.com
@@ -17,11 +17,11 @@ project_urls =
 
 classifiers =
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
     Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: Implementation :: PyPy
     Intended Audience :: Developers

From 715126c67b2339381f5ad02f45d8fe367400c749 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 9 Aug 2023 18:20:05 +0900
Subject: [PATCH 1597/1648] CI: update cibuildwheel to v2.15.0 (#551)

---
 .github/workflows/wheel.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 6cf2fe9..c1eb2ed 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -35,7 +35,7 @@ jobs:
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v2.12.0
+        uses: pypa/cibuildwheel@v2.15.0
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"

From 7b75b4f36819c77a12518929fecc09d94e82f5bd Mon Sep 17 00:00:00 2001
From: TW 
Date: Thu, 31 Aug 2023 05:56:24 +0200
Subject: [PATCH 1598/1648] sphinx-related work (#554)

fixes #510
---
 .github/workflows/docs.yaml | 32 ++++++++++++++++++++++++++++++++
 docs/_static/README.txt     |  1 +
 docs/api.rst                |  8 ++++----
 docs/conf.py                |  2 +-
 msgpack/_packer.pyx         |  3 ++-
 msgpack/_unpacker.pyx       |  6 +++---
 msgpack/ext.py              |  4 ++--
 msgpack/fallback.py         |  9 +++++----
 tox.ini                     |  9 +++++++++
 9 files changed, 59 insertions(+), 15 deletions(-)
 create mode 100644 .github/workflows/docs.yaml
 create mode 100644 docs/_static/README.txt

diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
new file mode 100644
index 0000000..a393c6b
--- /dev/null
+++ b/.github/workflows/docs.yaml
@@ -0,0 +1,32 @@
+name: docs
+
+on: ["push", "pull_request"]
+
+jobs:
+  docs:
+    # We want to run on external PRs, but not on our own internal PRs as they'll be run
+    # by the push to the branch.
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+          architecture: 'x64'
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Build
+        shell: bash
+        run: |
+          pip install -r requirements.txt
+          make cython
+          pip install .
+
+      - name: Sphinx Documentation Generator
+        run: |
+          pip install tox
+          tox -e sphinx
diff --git a/docs/_static/README.txt b/docs/_static/README.txt
new file mode 100644
index 0000000..1c70594
--- /dev/null
+++ b/docs/_static/README.txt
@@ -0,0 +1 @@
+Sphinx will copy the contents of docs/_static/ directory to the build location.
diff --git a/docs/api.rst b/docs/api.rst
index 93827e1..f5dfbbd 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -5,19 +5,19 @@ API reference
 
 .. autofunction:: pack
 
-:func:`dump` is alias for :func:`pack`
+``dump()`` is an alias for :func:`pack`
 
 .. autofunction:: packb
 
-:func:`dumps` is alias for :func:`packb`
+``dumps()`` is an alias for :func:`packb`
 
 .. autofunction:: unpack
 
-:func:`load` is alias for :func:`unpack`
+``load()`` is an alias for :func:`unpack`
 
 .. autofunction:: unpackb
 
-:func:`loads` is alias for :func:`unpackb`
+``loads()`` is an alias for :func:`unpackb`
 
 .. autoclass:: Packer
     :members:
diff --git a/docs/conf.py b/docs/conf.py
index 91ce77f..1c1895c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -16,7 +16,7 @@ import sys, os
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-# sys.path.insert(0, os.path.abspath('.'))
+#sys.path.insert(0, os.path.abspath('..'))
 
 # -- General configuration -----------------------------------------------------
 
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 074b39f..3c39867 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -71,7 +71,8 @@ cdef class Packer(object):
 
     Packer's constructor has some keyword arguments:
 
-    :param callable default:
+    :param default:
+        When specified, it should be callable.
         Convert user type to builtin type that Packer supports.
         See also simplejson's document.
 
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index d5dc5ea..56126f4 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -217,7 +217,7 @@ cdef class Unpacker(object):
 
     :param file_like:
         File-like object having `.read(n)` method.
-        If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
+        If specified, unpacker reads serialized data from it and `.feed()` is not usable.
 
     :param int read_size:
         Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
@@ -241,12 +241,12 @@ cdef class Unpacker(object):
     :param bool strict_map_key:
         If true (default), only str or bytes are accepted for map (dict) keys.
 
-    :param callable object_hook:
+    :param object_hook:
         When specified, it should be callable.
         Unpacker calls it with a dict argument after unpacking msgpack map.
         (See also simplejson)
 
-    :param callable object_pairs_hook:
+    :param object_pairs_hook:
         When specified, it should be callable.
         Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
         (See also simplejson)
diff --git a/msgpack/ext.py b/msgpack/ext.py
index 9794294..f7f2d77 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -120,7 +120,7 @@ class Timestamp:
         """Create a Timestamp from posix timestamp in seconds.
 
         :param unix_float: Posix timestamp in seconds.
-        :type unix_float: int or float.
+        :type unix_float: int or float
         """
         seconds = int(unix_sec // 1)
         nanoseconds = int((unix_sec % 1) * 10**9)
@@ -154,7 +154,7 @@ class Timestamp:
     def to_datetime(self):
         """Get the timestamp as a UTC datetime.
 
-        :rtype: datetime.
+        :rtype: `datetime.datetime`
         """
         utc = datetime.timezone.utc
         return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix())
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index ac1eaf4..84b2617 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -139,7 +139,7 @@ class Unpacker:
 
     :param file_like:
         File-like object having `.read(n)` method.
-        If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable.
+        If specified, unpacker reads serialized data from it and `.feed()` is not usable.
 
     :param int read_size:
         Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
@@ -163,12 +163,12 @@ class Unpacker:
     :param bool strict_map_key:
         If true (default), only str or bytes are accepted for map (dict) keys.
 
-    :param callable object_hook:
+    :param object_hook:
         When specified, it should be callable.
         Unpacker calls it with a dict argument after unpacking msgpack map.
         (See also simplejson)
 
-    :param callable object_pairs_hook:
+    :param object_pairs_hook:
         When specified, it should be callable.
         Unpacker calls it with a list of key-value pairs after unpacking msgpack map.
         (See also simplejson)
@@ -616,7 +616,8 @@ class Packer:
 
     Packer's constructor has some keyword arguments:
 
-    :param callable default:
+    :param default:
+        When specified, it should be callable.
         Convert user type to builtin type that Packer supports.
         See also simplejson's document.
 
diff --git a/tox.ini b/tox.ini
index 1ef2d18..369eddc 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,6 +3,7 @@ envlist =
     {py35,py36,py37,py38}-{c,pure},
     {pypy,pypy3}-pure,
     py34-x86,
+    sphinx,
 isolated_build = true
 
 [testenv]
@@ -27,3 +28,11 @@ commands=
     python -c 'import sys; print(hex(sys.maxsize))'
     python -c 'from msgpack import _cmsgpack'
     py.test
+
+
+[testenv:sphinx]
+changedir = docs
+deps =
+    sphinx
+commands =
+    sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html

From 423c6df265d0f964733b31a7e835fe91e4b8ea89 Mon Sep 17 00:00:00 2001
From: TW 
Date: Tue, 5 Sep 2023 03:51:04 +0200
Subject: [PATCH 1599/1648] move project metadata to pyproject.toml (#555)

also: replace flake8 by ruff.
---
 docs/conf.py            |  4 ++--
 msgpack/__init__.py     |  1 -
 msgpack/ext.py          |  8 ++++----
 msgpack/fallback.py     |  2 +-
 pyproject.toml          | 45 +++++++++++++++++++++++++++++++++++++++++
 setup.cfg               | 32 -----------------------------
 setup.py                |  2 --
 test/test_buffer.py     |  2 --
 test/test_memoryview.py |  2 --
 test/test_obj.py        |  4 ++--
 test/test_pack.py       |  4 +---
 test/test_seq.py        |  2 +-
 test/test_subtype.py    |  2 +-
 test/test_timestamp.py  |  1 -
 14 files changed, 57 insertions(+), 54 deletions(-)
 delete mode 100644 setup.cfg

diff --git a/docs/conf.py b/docs/conf.py
index 1c1895c..6eb472a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -11,11 +11,11 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys, os
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
+#import os
+#import sys
 #sys.path.insert(0, os.path.abspath('..'))
 
 # -- General configuration -----------------------------------------------------
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 2540120..9a96c98 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -2,7 +2,6 @@ from .exceptions import *
 from .ext import ExtType, Timestamp
 
 import os
-import sys
 
 
 version = (1, 0, 5)
diff --git a/msgpack/ext.py b/msgpack/ext.py
index f7f2d77..02c2c43 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -1,6 +1,5 @@
 from collections import namedtuple
 import datetime
-import sys
 import struct
 
 
@@ -20,8 +19,9 @@ class ExtType(namedtuple("ExtType", "code data")):
 class Timestamp:
     """Timestamp represents the Timestamp extension type in msgpack.
 
-    When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`. When using pure-Python
-    msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and unpack `Timestamp`.
+    When built with Cython, msgpack uses C methods to pack and unpack `Timestamp`.
+    When using pure-Python msgpack, :func:`to_bytes` and :func:`from_bytes` are used to pack and
+    unpack `Timestamp`.
 
     This class is immutable: Do not override seconds and nanoseconds.
     """
@@ -39,7 +39,7 @@ class Timestamp:
             Number of nanoseconds to add to `seconds` to get fractional time.
             Maximum is 999_999_999.  Default is 0.
 
-        Note: Negative times (before the UNIX epoch) are represented as negative seconds + positive ns.
+        Note: Negative times (before the UNIX epoch) are represented as neg. seconds + pos. ns.
         """
         if not isinstance(seconds, int):
             raise TypeError("seconds must be an integer")
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index 84b2617..a174162 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -530,7 +530,7 @@ class Unpacker:
                     key = self._unpack(EX_CONSTRUCT)
                     if self._strict_map_key and type(key) not in (str, bytes):
                         raise ValueError("%s is not allowed for map key" % str(type(key)))
-                    if type(key) is str:
+                    if isinstance(key, str):
                         key = sys.intern(key)
                     ret[key] = self._unpack(EX_CONSTRUCT)
                 if self._object_hook is not None:
diff --git a/pyproject.toml b/pyproject.toml
index dc8bbee..f37d213 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,7 +7,52 @@ requires = [
 ]
 build-backend = "setuptools.build_meta"
 
+[project]
+name = "msgpack"
+dynamic = ["version"]
+license = {text="Apache 2.0"}
+authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}]
+description = "MessagePack serializer"
+readme = "README.md"
+#keywords = ["python", "msgpack", "messagepack", "serializer", "serialization", "binary"]
+#requires-python = ">=3.8"
+classifiers = [
+#    "Development Status :: 5 - Production/Stable",
+#    "Operating System :: OS Independent",
+#    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+]
+
+[project.urls]
+Homepage = "https://msgpack.org/"
+Documentation = "https://msgpack-python.readthedocs.io/"
+Repository = "https://github.com/msgpack/msgpack-python/"
+Tracker = "https://github.com/msgpack/msgpack-python/issues"
+#Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst"
+
+[tool.setuptools.dynamic]
+version = {attr = "msgpack.__version__"}
+
 [tool.black]
 line-length = 100
 target-version = ["py37"]
 skip_string_normalization = true
+
+[tool.ruff]
+line-length = 100
+target-version = "py38"
+ignore = []
+
+[tool.ruff.per-file-ignores]
+"msgpack/__init__.py" = ["F401", "F403"]
+"msgpack/fallback.py" = ["E731"]
+"test/test_seq.py" = ["E501"]
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index c2e1672..0000000
--- a/setup.cfg
+++ /dev/null
@@ -1,32 +0,0 @@
-[metadata]
-name = msgpack
-version = attr: msgpack.__version__
-#version = attr: msgpack.version
-license = Apache 2.0
-author = Inada Naoki
-author_email = songofacandy@gmail.com
-description = MessagePack serializer
-long_description = file: README.md
-long_description_content_type = text/markdown
-url = https://msgpack.org/
-
-project_urls =
-    Documentation = https://msgpack-python.readthedocs.io/
-    Source = https://github.com/msgpack/msgpack-python
-    Tracker = https://github.com/msgpack/msgpack-python/issues
-
-classifiers =
-    Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.8
-    Programming Language :: Python :: 3.9
-    Programming Language :: Python :: 3.10
-    Programming Language :: Python :: 3.11
-    Programming Language :: Python :: 3.12
-    Programming Language :: Python :: Implementation :: CPython
-    Programming Language :: Python :: Implementation :: PyPy
-    Intended Audience :: Developers
-    License :: OSI Approved :: Apache Software License
-
-[flake8]
-max_line_length = 100
-
diff --git a/setup.py b/setup.py
index 1cd1e8e..7a34c8c 100755
--- a/setup.py
+++ b/setup.py
@@ -1,8 +1,6 @@
 #!/usr/bin/env python
-import io
 import os
 import sys
-from glob import glob
 from setuptools import setup, Extension
 from setuptools.command.build_ext import build_ext
 from setuptools.command.sdist import sdist
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 7ee674a..a3db339 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -1,7 +1,5 @@
 #!/usr/bin/env python
 
-import sys
-import pytest
 from msgpack import packb, unpackb
 
 
diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index eaadef7..dc319a6 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 
-import pytest
 from array import array
 from msgpack import packb, unpackb
-import sys
 
 
 def make_array(f, data):
diff --git a/test/test_obj.py b/test/test_obj.py
index d3f870d..f78bf42 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -33,7 +33,7 @@ def test_decode_pairs_hook():
     prod_sum = 1 * 2 + 3 * 4
     unpacked = unpackb(
         packed,
-        object_pairs_hook=lambda l: sum(k * v for k, v in l),
+        object_pairs_hook=lambda lst: sum(k * v for k, v in lst),
         use_list=1,
         strict_map_key=False,
     )
@@ -48,7 +48,7 @@ def test_only_one_obj_hook():
 def test_bad_hook():
     with raises(TypeError):
         packed = packb([3, 1 + 2j], default=lambda o: o)
-        unpacked = unpackb(packed, use_list=1)
+        unpackb(packed, use_list=1)
 
 
 def _arr_to_str(arr):
diff --git a/test/test_pack.py b/test/test_pack.py
index 2753e46..4232537 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -3,12 +3,10 @@
 from collections import OrderedDict
 from io import BytesIO
 import struct
-import sys
 
 import pytest
-from pytest import raises, xfail
 
-from msgpack import packb, unpackb, Unpacker, Packer, pack
+from msgpack import packb, unpackb, Unpacker, Packer
 
 
 def check(data, use_list=False):
diff --git a/test/test_seq.py b/test/test_seq.py
index def6630..16d9dde 100644
--- a/test/test_seq.py
+++ b/test/test_seq.py
@@ -34,7 +34,7 @@ def test_exceeding_unpacker_read_size():
 
     read_count = 0
     for idx, o in enumerate(unpacker):
-        assert type(o) == bytes
+        assert isinstance(o, bytes)
         assert o == gen_binary_data(idx)
         read_count += 1
 
diff --git a/test/test_subtype.py b/test/test_subtype.py
index d5a9adb..0d1c41a 100644
--- a/test/test_subtype.py
+++ b/test/test_subtype.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-from msgpack import packb, unpackb
+from msgpack import packb
 from collections import namedtuple
 
 
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index af84a2f..db5cc57 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -1,5 +1,4 @@
 import pytest
-import sys
 import datetime
 import msgpack
 from msgpack.ext import Timestamp

From ef15f4a62c25114bec1db91aa4006ae2d3a9fb53 Mon Sep 17 00:00:00 2001
From: TW 
Date: Thu, 7 Sep 2023 14:25:07 +0200
Subject: [PATCH 1600/1648] add a basic .readthedocs.yaml file (#558)

---
 .readthedocs.yaml | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 .readthedocs.yaml

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 0000000..8b5aaf1
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,12 @@
+# Read the Docs configuration file for Sphinx projects.
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details.
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+sphinx:
+  configuration: docs/conf.py

From 41d6239c0a3bfb1daabe1a45ffdbecf4e9aa5469 Mon Sep 17 00:00:00 2001
From: TW 
Date: Tue, 12 Sep 2023 19:51:12 +0200
Subject: [PATCH 1601/1648] fix .readthedocs.yaml, fixes #559 (#560)

---
 .readthedocs.yaml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
index 8b5aaf1..7447895 100644
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -7,6 +7,17 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.11"
+  apt_packages:
+    - build-essential
+  jobs:
+    pre_install:
+      - pip install -r requirements.txt
+      - make cython
+
+python:
+  install:
+    - method: pip
+      path: .
 
 sphinx:
   configuration: docs/conf.py

From 4e10c10aaa8350f23e4b85d27ff131f7b4fd13e2 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 13 Sep 2023 18:40:04 +0900
Subject: [PATCH 1602/1648] prepare for 1.0.6rc1 (#557)

---
 ChangeLog.rst       |  9 +++++++++
 msgpack/__init__.py |  4 ++--
 pyproject.toml      | 12 ++++++------
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 5354799..bf345dd 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,12 @@
+1.0.6rc1
+========
+
+Release Date: 2023-09-13
+
+* Add Python 3.12 wheels (#517)
+* Remove Python 2.7, 3.6, and 3.7 support
+
+
 1.0.5
 =====
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 9a96c98..2e20133 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -4,8 +4,8 @@ from .ext import ExtType, Timestamp
 import os
 
 
-version = (1, 0, 5)
-__version__ = "1.0.6dev1"
+version = (1, 0, 6, "rc", 1)
+__version__ = "1.0.6rc1"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):
diff --git a/pyproject.toml b/pyproject.toml
index f37d213..a63009a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,12 +14,12 @@ license = {text="Apache 2.0"}
 authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}]
 description = "MessagePack serializer"
 readme = "README.md"
-#keywords = ["python", "msgpack", "messagepack", "serializer", "serialization", "binary"]
-#requires-python = ">=3.8"
+keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"]
+requires-python = ">=3.8"
 classifiers = [
-#    "Development Status :: 5 - Production/Stable",
-#    "Operating System :: OS Independent",
-#    "Programming Language :: Python",
+    "Development Status :: 5 - Production/Stable",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
@@ -37,7 +37,7 @@ Homepage = "https://msgpack.org/"
 Documentation = "https://msgpack-python.readthedocs.io/"
 Repository = "https://github.com/msgpack/msgpack-python/"
 Tracker = "https://github.com/msgpack/msgpack-python/issues"
-#Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst"
+Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst"
 
 [tool.setuptools.dynamic]
 version = {attr = "msgpack.__version__"}

From e1d3d5d5c386b8b2fa99c812b4648f6532cab032 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 15 Sep 2023 12:02:06 +0900
Subject: [PATCH 1603/1648] update actions (#563)

---
 .github/workflows/black.yaml | 2 +-
 .github/workflows/docs.yaml  | 2 +-
 .github/workflows/test.yml   | 2 +-
 .github/workflows/wheel.yml  | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml
index 1e28b7b..e091792 100644
--- a/.github/workflows/black.yaml
+++ b/.github/workflows/black.yaml
@@ -17,7 +17,7 @@ jobs:
           architecture: 'x64'
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Black Code Formatter
         run: |
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index a393c6b..80bbba7 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -17,7 +17,7 @@ jobs:
           architecture: 'x64'
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Build
         shell: bash
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 76fcf27..4eb8849 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v4
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index c1eb2ed..0412a38 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -14,11 +14,11 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         if: runner.os == 'Linux'
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: arm64
 

From b1b0edaeedd073f21023ec01a60bfa9da077ad2b Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 21 Sep 2023 14:58:37 +0900
Subject: [PATCH 1604/1648] release v1.0.6 (#564)

---
 ChangeLog.rst       | 6 +++---
 msgpack/__init__.py | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index bf345dd..bad51aa 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,7 +1,7 @@
-1.0.6rc1
-========
+1.0.6
+=====
 
-Release Date: 2023-09-13
+Release Date: 2023-09-21
 
 * Add Python 3.12 wheels (#517)
 * Remove Python 2.7, 3.6, and 3.7 support
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 2e20133..781bcdf 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -4,8 +4,8 @@ from .ext import ExtType, Timestamp
 import os
 
 
-version = (1, 0, 6, "rc", 1)
-__version__ = "1.0.6rc1"
+version = (1, 0, 6)
+__version__ = "1.0.6"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):

From ecf03748c7241a0fb6bef733c7e5d2d68179b670 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 28 Sep 2023 15:03:16 +0900
Subject: [PATCH 1605/1648] remove inline macro for msvc (#567)

---
 msgpack/pack.h | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/msgpack/pack.h b/msgpack/pack.h
index 1e849ac..2453428 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -26,10 +26,6 @@
 extern "C" {
 #endif
 
-#ifdef _MSC_VER
-#define inline __inline
-#endif
-
 typedef struct msgpack_packer {
     char *buf;
     size_t length;

From acd068439233b8f04543c4ee81c18c8dbb681aba Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 28 Sep 2023 15:25:10 +0900
Subject: [PATCH 1606/1648] do not fallback on build error (#568)

---
 setup.py | 26 ++++++--------------------
 1 file changed, 6 insertions(+), 20 deletions(-)
 mode change 100755 => 100644 setup.py

diff --git a/setup.py b/setup.py
old mode 100755
new mode 100644
index 7a34c8c..a13bd81
--- a/setup.py
+++ b/setup.py
@@ -22,6 +22,8 @@ except ImportError:
 
 
 def cythonize(src):
+    if not have_cython:
+        raise Exception("Cython is required for building from checkout")
     sys.stderr.write(f"cythonize: {src!r}\n")
     cython_compiler.compile([src], cplus=True)
 
@@ -29,31 +31,15 @@ def cythonize(src):
 def ensure_source(src):
     pyx = os.path.splitext(src)[0] + ".pyx"
 
-    if not os.path.exists(src):
-        if not have_cython:
-            raise NoCython
+    if not os.path.exists(src) or have_cython and os.stat(src).st_mtime < os.stat(pyx).st_mtime:
         cythonize(pyx)
-    elif os.path.exists(pyx) and os.stat(src).st_mtime < os.stat(pyx).st_mtime and have_cython:
-        cythonize(pyx)
-    return src
 
 
 class BuildExt(build_ext):
     def build_extension(self, ext):
-        try:
-            ext.sources = list(map(ensure_source, ext.sources))
-        except NoCython:
-            print("WARNING")
-            print("Cython is required for building extension from checkout.")
-            print("Install Cython >= 0.16 or install msgpack from PyPI.")
-            print("Falling back to pure Python implementation.")
-            return
-        try:
-            return build_ext.build_extension(self, ext)
-        except Exception as e:
-            print("WARNING: Failed to compile extension modules.")
-            print("msgpack uses fallback pure python implementation.")
-            print(e)
+        for src in ext.sources:
+            ensure_source(src)
+        return build_ext.build_extension(self, ext)
 
 
 # Cython is required for sdist

From 2982e9ff729eae150d67ee608fdf1d01d93d8e3f Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 28 Sep 2023 17:31:52 +0900
Subject: [PATCH 1607/1648] release v1.0.7 (#569)

---
 .gitignore          |  1 +
 ChangeLog.rst       | 13 +++++++++++++
 msgpack/__init__.py |  4 ++--
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 800f1c2..8a06e26 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@ MANIFEST
 build/*
 dist/*
 .tox
+.python-version
 *.pyc
 *.pyo
 *.so
diff --git a/ChangeLog.rst b/ChangeLog.rst
index bad51aa..ca74ebe 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,8 +1,21 @@
+1.0.7
+=====
+
+Release Date: 2023-09-28
+
+* Fix build error of extension module on Windows. (#567)
+* ``setup.py`` doesn't skip build error of extension module. (#568)
+
+
 1.0.6
 =====
 
 Release Date: 2023-09-21
 
+.. note::
+   v1.0.6 Wheels for Windows don't contain extension module.
+   Please upgrade to v1.0.7 or newer.
+
 * Add Python 3.12 wheels (#517)
 * Remove Python 2.7, 3.6, and 3.7 support
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 781bcdf..60a088c 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -4,8 +4,8 @@ from .ext import ExtType, Timestamp
 import os
 
 
-version = (1, 0, 6)
-__version__ = "1.0.6"
+version = (1, 0, 7)
+__version__ = "1.0.7"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):

From c78026102c981d734c6d8fec6b2790ee31212f27 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 15 Nov 2023 23:34:32 +0900
Subject: [PATCH 1608/1648] doc: use sphinx-rtd-theme (#575)

---
 .github/workflows/docs.yaml | 1 -
 .readthedocs.yaml           | 1 +
 docs/conf.py                | 4 +---
 docs/requirements.txt       | 2 ++
 tox.ini                     | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)
 create mode 100644 docs/requirements.txt

diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index 80bbba7..8c8298a 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -24,7 +24,6 @@ jobs:
         run: |
           pip install -r requirements.txt
           make cython
-          pip install .
 
       - name: Sphinx Documentation Generator
         run: |
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
index 7447895..88d8718 100644
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -18,6 +18,7 @@ python:
   install:
     - method: pip
       path: .
+    - requirements: docs/requirements.txt
 
 sphinx:
   configuration: docs/conf.py
diff --git a/docs/conf.py b/docs/conf.py
index 6eb472a..ab0ad3c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-#
 # msgpack documentation build configuration file, created by
 # sphinx-quickstart on Sun Feb 24 14:20:50 2013.
 #
@@ -91,7 +89,7 @@ pygments_style = "sphinx"
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = "sphinxdoc"
+html_theme = "sphinx_rtd_theme"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 0000000..8d45d0b
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,2 @@
+sphinx~=7.2
+sphinx-rtd-theme~=1.3.0
diff --git a/tox.ini b/tox.ini
index 369eddc..49364be 100644
--- a/tox.ini
+++ b/tox.ini
@@ -33,6 +33,6 @@ commands=
 [testenv:sphinx]
 changedir = docs
 deps =
-    sphinx
+    -r docs/requirements.txt
 commands =
     sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html

From 140864249fd0f67dffaeceeb168ffe9cdf6f1964 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 20 Dec 2023 20:46:04 +0900
Subject: [PATCH 1609/1648] exclude C/Cython files from wheel (#577)

---
 pyproject.toml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index a63009a..121b1fd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,6 +39,10 @@ Repository = "https://github.com/msgpack/msgpack-python/"
 Tracker = "https://github.com/msgpack/msgpack-python/issues"
 Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst"
 
+[tool.setuptools]
+# Do not install C/C++/Cython source files
+include-package-data = false
+
 [tool.setuptools.dynamic]
 version = {attr = "msgpack.__version__"}
 

From 039022cecb04b62a29afb8260b81f57a937aaaaa Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 1 Mar 2024 19:24:06 +0900
Subject: [PATCH 1610/1648] update Cython (#581)

---
 .github/workflows/wheel.yml | 10 +++++++++-
 pyproject.toml              |  2 +-
 requirements.txt            |  2 +-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 0412a38..4a8847d 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -3,6 +3,7 @@ on:
   push:
     branches: [main]
   create:
+  workflow_dispatch:
 
 jobs:
   build_wheels:
@@ -23,7 +24,7 @@ jobs:
           platforms: arm64
 
       - name: Set up Python 3.x
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
           cache: "pip"
@@ -43,6 +44,13 @@ jobs:
           CIBW_ARCHS_MACOS: x86_64 universal2 arm64
           CIBW_SKIP: pp*
 
+      - name: Build pure Python wheel
+        env:
+          MSGPACK_PUREPYTHON: "1"
+        run: |
+          pip install build
+          python -m build -w -o wheelhouse
+
       - name: Upload Wheels to artifact
         uses: actions/upload-artifact@v1
         with:
diff --git a/pyproject.toml b/pyproject.toml
index 121b1fd..f9af967 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -2,7 +2,7 @@
 requires = [
     # Also declared in requirements.txt, if updating here please also update
     # there
-    "Cython~=3.0.0",
+    "Cython~=3.0.8",
     "setuptools >= 35.0.2",
 ]
 build-backend = "setuptools.build_meta"
diff --git a/requirements.txt b/requirements.txt
index e27df0f..839dc5f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 # Also declared in pyproject.toml, if updating here please also update there.
-Cython~=3.0.0
+Cython~=3.0.8
 
 # Tools required only for development. No need to add it to pyproject.toml file.
 black==23.3.0

From bf7bf88ad0b3cd7a8cd74e8251521fde743e9af9 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 1 Mar 2024 20:09:55 +0900
Subject: [PATCH 1611/1648] ci: update workflows (#582)

---
 .github/workflows/test.yml  | 14 +++++++++++++-
 .github/workflows/wheel.yml |  6 +++---
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4eb8849..1faeb0c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -20,7 +20,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.py }}
           allow-prereleases: true
@@ -42,3 +42,15 @@ jobs:
         shell: bash
         run: |
           MSGPACK_PUREPYTHON=1 pytest -v test
+
+      - name: build packages
+        shell: bash
+        run: |
+          pip install build
+          python -m build
+
+      - name: upload packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist-${{ matrix.os }}-${{ matrix.py }}
+          path: dist
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 4a8847d..e91325b 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -36,7 +36,7 @@ jobs:
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v2.15.0
+        uses: pypa/cibuildwheel@v2.16.5
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
@@ -52,7 +52,7 @@ jobs:
           python -m build -w -o wheelhouse
 
       - name: Upload Wheels to artifact
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
-          name: Wheels
+          name: wheels-${{ matrix.os }}
           path: wheelhouse

From 9aedf8ed7f632044d42984e9710fefbd97023f71 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 1 Mar 2024 20:35:28 +0900
Subject: [PATCH 1612/1648] Release v1.0.8 (#583)

---
 ChangeLog.rst       | 10 ++++++++++
 msgpack/__init__.py |  4 ++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index ca74ebe..2408bc9 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,13 @@
+1.0.8
+=====
+
+Release Date: 2024-03-01
+
+* Update Cython to 3.0.8. This fixes memory leak when iterating
+  ``Unpacker`` object on Python 3.12.
+* Do not include C/Cython files in binary wheels.
+
+
 1.0.7
 =====
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 60a088c..919b86f 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -4,8 +4,8 @@ from .ext import ExtType, Timestamp
 import os
 
 
-version = (1, 0, 7)
-__version__ = "1.0.7"
+version = (1, 0, 8)
+__version__ = "1.0.8"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):

From e77672200bf89f2093b96a4abd0e4eeb253975b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?hakan=20aky=C3=BCrek?= 
Date: Sat, 20 Apr 2024 00:46:30 +0200
Subject: [PATCH 1613/1648] Avoid using floating points during
 timestamp-datetime conversions (#591)

---
 msgpack/ext.py         |  6 ++++--
 test/test_timestamp.py | 15 +++++++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/msgpack/ext.py b/msgpack/ext.py
index 02c2c43..3940fe0 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -157,7 +157,9 @@ class Timestamp:
         :rtype: `datetime.datetime`
         """
         utc = datetime.timezone.utc
-        return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(seconds=self.to_unix())
+        return datetime.datetime.fromtimestamp(0, utc) + datetime.timedelta(
+            seconds=self.seconds, microseconds=self.nanoseconds // 1000
+        )
 
     @staticmethod
     def from_datetime(dt):
@@ -165,4 +167,4 @@ class Timestamp:
 
         :rtype: Timestamp
         """
-        return Timestamp.from_unix(dt.timestamp())
+        return Timestamp(seconds=int(dt.timestamp()), nanoseconds=dt.microsecond * 1000)
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index db5cc57..f9bc835 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -86,6 +86,21 @@ def test_timestamp_datetime():
     utc = datetime.timezone.utc
     assert t.to_datetime() == datetime.datetime(1970, 1, 1, 0, 0, 42, 0, tzinfo=utc)
 
+    ts = datetime.datetime(2024, 4, 16, 8, 43, 9, 420317, tzinfo=utc)
+    ts2 = datetime.datetime(2024, 4, 16, 8, 43, 9, 420318, tzinfo=utc)
+
+    assert (
+        Timestamp.from_datetime(ts2).nanoseconds - Timestamp.from_datetime(ts).nanoseconds == 1000
+    )
+
+    ts3 = datetime.datetime(2024, 4, 16, 8, 43, 9, 4256)
+    ts4 = datetime.datetime(2024, 4, 16, 8, 43, 9, 4257)
+    assert (
+        Timestamp.from_datetime(ts4).nanoseconds - Timestamp.from_datetime(ts3).nanoseconds == 1000
+    )
+
+    assert Timestamp.from_datetime(ts).to_datetime() == ts
+
 
 def test_unpack_datetime():
     t = Timestamp(42, 14)

From 2eca765533106cc4bbb44ecc565b1034ac50f91b Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 3 May 2024 15:17:54 +0900
Subject: [PATCH 1614/1648] use ruff instead of black (#598)

---
 .github/workflows/docs.yaml                 | 21 +++++++-----
 .github/workflows/{black.yaml => lint.yaml} | 19 +++++------
 Makefile                                    | 14 ++++++--
 docs/requirements.txt                       |  4 +--
 msgpack/fallback.py                         |  1 +
 pyproject.toml                              |  4 +--
 requirements.txt                            |  7 +---
 test/test_memoryview.py                     |  2 +-
 test/test_pack.py                           |  2 +-
 test/test_read_size.py                      |  1 +
 tox.ini                                     | 38 ---------------------
 11 files changed, 40 insertions(+), 73 deletions(-)
 rename .github/workflows/{black.yaml => lint.yaml} (56%)
 delete mode 100644 tox.ini

diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index 8c8298a..08fc2f4 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -10,22 +10,25 @@ jobs:
 
     runs-on: ubuntu-latest
     steps:
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
-          architecture: 'x64'
-
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+          architecture: 'x64'
+          cache: "pip"
+          cache-dependency-path: |
+            requirements.txt
+            docs/requirements.txt
+
       - name: Build
-        shell: bash
         run: |
           pip install -r requirements.txt
           make cython
 
       - name: Sphinx Documentation Generator
         run: |
-          pip install tox
-          tox -e sphinx
+          pip install -r docs/requirements.txt
+          make docs
diff --git a/.github/workflows/black.yaml b/.github/workflows/lint.yaml
similarity index 56%
rename from .github/workflows/black.yaml
rename to .github/workflows/lint.yaml
index e091792..198cf7b 100644
--- a/.github/workflows/black.yaml
+++ b/.github/workflows/lint.yaml
@@ -1,25 +1,22 @@
-name: Black
+name: lint
 
 on: ["push", "pull_request"]
 
 jobs:
-  black:
+  lint:
     # We want to run on external PRs, but not on our own internal PRs as they'll be run
     # by the push to the branch.
     if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
 
     runs-on: ubuntu-latest
     steps:
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
-          architecture: 'x64'
-
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Black Code Formatter
+      - name: ruff check
         run: |
-          pip install black==22.3.0
-          black -S --diff --check msgpack/ test/ setup.py
+          pipx run ruff check --diff msgpack/ test/ setup.py
+
+      - name: ruff format
+        run: |
+          pipx run ruff format --diff msgpack/ test/ setup.py
diff --git a/Makefile b/Makefile
index e4f22da..5c1863c 100644
--- a/Makefile
+++ b/Makefile
@@ -4,9 +4,17 @@ PYTHON_SOURCES = msgpack test setup.py
 all: cython
 	python setup.py build_ext -i -f
 
-.PHONY: black
-black:
-	black $(PYTHON_SOURCES)
+.PHONY: format
+format:
+	pipx run ruff format $(PYTHON_SOURCES)
+
+.PHONY: lint
+lint:
+	pipx run ruff check $(PYTHON_SOURCES)
+
+.PHONY: doc
+doc:
+	cd docs && sphinx-build -n -v -W --keep-going -b html -d doctrees . html
 
 .PHONY: pyupgrade
 pyupgrade:
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 8d45d0b..26002de 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,2 +1,2 @@
-sphinx~=7.2
-sphinx-rtd-theme~=1.3.0
+sphinx~=7.3.7
+sphinx-rtd-theme~=2.0.0
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index a174162..ea4c4ce 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -1,4 +1,5 @@
 """Fallback pure Python implementation of msgpack"""
+
 from datetime import datetime as _DateTime
 import sys
 import struct
diff --git a/pyproject.toml b/pyproject.toml
index f9af967..d99375a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -54,9 +54,9 @@ skip_string_normalization = true
 [tool.ruff]
 line-length = 100
 target-version = "py38"
-ignore = []
+lint.ignore = []
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "msgpack/__init__.py" = ["F401", "F403"]
 "msgpack/fallback.py" = ["E731"]
 "test/test_seq.py" = ["E501"]
diff --git a/requirements.txt b/requirements.txt
index 839dc5f..1164a94 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,2 @@
 # Also declared in pyproject.toml, if updating here please also update there.
-Cython~=3.0.8
-
-# Tools required only for development. No need to add it to pyproject.toml file.
-black==23.3.0
-pytest==7.3.1
-pyupgrade==3.3.2
+Cython~=3.0.10
diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index dc319a6..eff4bca 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -95,4 +95,4 @@ def test_multidim_memoryview():
     view = memoryview(b"\00" * 6)
     data = view.cast(view.format, (3, 2))
     packed = packb(data)
-    assert packed == b'\xc4\x06\x00\x00\x00\x00\x00\x00'
+    assert packed == b"\xc4\x06\x00\x00\x00\x00\x00\x00"
diff --git a/test/test_pack.py b/test/test_pack.py
index 4232537..4a0ef40 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -89,7 +89,7 @@ def testStrictUnicodeUnpack():
 
 def testIgnoreErrorsPack():
     re = unpackb(
-        packb("abc\uDC80\uDCFFdef", use_bin_type=True, unicode_errors="ignore"),
+        packb("abc\udc80\udcffdef", use_bin_type=True, unicode_errors="ignore"),
         raw=False,
         use_list=1,
     )
diff --git a/test/test_read_size.py b/test/test_read_size.py
index 33a7e7d..a7d61fd 100644
--- a/test/test_read_size.py
+++ b/test/test_read_size.py
@@ -1,4 +1,5 @@
 """Test Unpacker's read_array_header and read_map_header methods"""
+
 from msgpack import packb, Unpacker, OutOfData
 
 UnexpectedTypeException = ValueError
diff --git a/tox.ini b/tox.ini
deleted file mode 100644
index 49364be..0000000
--- a/tox.ini
+++ /dev/null
@@ -1,38 +0,0 @@
-[tox]
-envlist =
-    {py35,py36,py37,py38}-{c,pure},
-    {pypy,pypy3}-pure,
-    py34-x86,
-    sphinx,
-isolated_build = true
-
-[testenv]
-deps=
-    pytest
-
-changedir=test
-commands=
-    c,x86: python -c 'from msgpack import _cmsgpack'
-    c,x86: py.test
-    pure: py.test
-setenv=
-    pure: MSGPACK_PUREPYTHON=x
-
-[testenv:py34-x86]
-basepython=python3.4-x86
-deps=
-    pytest
-
-changedir=test
-commands=
-    python -c 'import sys; print(hex(sys.maxsize))'
-    python -c 'from msgpack import _cmsgpack'
-    py.test
-
-
-[testenv:sphinx]
-changedir = docs
-deps =
-    -r docs/requirements.txt
-commands =
-    sphinx-build -n -v -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html

From 0602baf3ea7fb597d1c78a90980071e03a536836 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 3 May 2024 18:20:09 +0900
Subject: [PATCH 1615/1648] update Cython and setuptools (#599)

---
 DEVELOP.md     | 8 --------
 pyproject.toml | 7 +++----
 2 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/DEVELOP.md b/DEVELOP.md
index 9c823c3..27adf8c 100644
--- a/DEVELOP.md
+++ b/DEVELOP.md
@@ -1,13 +1,5 @@
 # Developer's note
 
-## Wheels
-
-Wheels for macOS and Linux are built on Travis and AppVeyr, in
-[methane/msgpack-wheels](https://github.com/methane/msgpack-wheels) repository.
-
-Wheels for Windows are built on Github Actions in this repository.
-
-
 ### Build
 
 ```
diff --git a/pyproject.toml b/pyproject.toml
index d99375a..6254f06 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,9 +1,8 @@
 [build-system]
 requires = [
-    # Also declared in requirements.txt, if updating here please also update
-    # there
-    "Cython~=3.0.8",
-    "setuptools >= 35.0.2",
+    # Also declared in requirements.txt, if updating here please also update there
+    "Cython~=3.0.10",
+    "setuptools >= 69.5.1",
 ]
 build-backend = "setuptools.build_meta"
 

From 3e9a2a7419714c294be0590aab24f2dc040581f5 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Sat, 4 May 2024 16:01:48 +0900
Subject: [PATCH 1616/1648] Stop using c++ (#600)

Python 3.13a6+ & C++ & Cython cause compile error on some compilers.
---
 .github/workflows/test.yml        |  2 +-
 Makefile                          |  2 +-
 msgpack/_unpacker.pyx             |  2 +-
 msgpack/pack.h                    |  2 +
 msgpack/unpack_container_header.h | 51 +++++++++++++++++++
 msgpack/unpack_template.h         | 83 ++++++++-----------------------
 setup.py                          |  6 +--
 7 files changed, 80 insertions(+), 68 deletions(-)
 create mode 100644 msgpack/unpack_container_header.h

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 1faeb0c..530238c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "windows-latest", "macos-latest"]
-        py: ["3.12", "3.11", "3.10", "3.9", "3.8"]
+        py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9", "3.8"]
 
     runs-on: ${{ matrix.os }}
     name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}
diff --git a/Makefile b/Makefile
index 5c1863c..3ce178f 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ pyupgrade:
 
 .PHONY: cython
 cython:
-	cython --cplus msgpack/_cmsgpack.pyx
+	cython msgpack/_cmsgpack.pyx
 
 .PHONY: test
 test: cython
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 56126f4..2771e7b 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -35,7 +35,7 @@ cdef extern from "unpack.h":
         PyObject* timestamp_t
         PyObject *giga;
         PyObject *utc;
-        char *unicode_errors
+        const char *unicode_errors
         Py_ssize_t max_str_len
         Py_ssize_t max_bin_len
         Py_ssize_t max_array_len
diff --git a/msgpack/pack.h b/msgpack/pack.h
index 2453428..901fad7 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -24,6 +24,8 @@
 
 #ifdef __cplusplus
 extern "C" {
+#else
+#define bool char
 #endif
 
 typedef struct msgpack_packer {
diff --git a/msgpack/unpack_container_header.h b/msgpack/unpack_container_header.h
new file mode 100644
index 0000000..c14a3c2
--- /dev/null
+++ b/msgpack/unpack_container_header.h
@@ -0,0 +1,51 @@
+static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off)
+{
+    assert(len >= *off);
+    uint32_t size;
+    const unsigned char *const p = (unsigned char*)data + *off;
+
+#define inc_offset(inc) \
+    if (len - *off < inc) \
+        return 0; \
+    *off += inc;
+
+    switch (*p) {
+    case var_offset:
+        inc_offset(3);
+        size = _msgpack_load16(uint16_t, p + 1);
+        break;
+    case var_offset + 1:
+        inc_offset(5);
+        size = _msgpack_load32(uint32_t, p + 1);
+        break;
+#ifdef USE_CASE_RANGE
+    case fixed_offset + 0x0 ... fixed_offset + 0xf:
+#else
+    case fixed_offset + 0x0:
+    case fixed_offset + 0x1:
+    case fixed_offset + 0x2:
+    case fixed_offset + 0x3:
+    case fixed_offset + 0x4:
+    case fixed_offset + 0x5:
+    case fixed_offset + 0x6:
+    case fixed_offset + 0x7:
+    case fixed_offset + 0x8:
+    case fixed_offset + 0x9:
+    case fixed_offset + 0xa:
+    case fixed_offset + 0xb:
+    case fixed_offset + 0xc:
+    case fixed_offset + 0xd:
+    case fixed_offset + 0xe:
+    case fixed_offset + 0xf:
+#endif
+        ++*off;
+        size = ((unsigned int)*p) & 0x0f;
+        break;
+    default:
+        PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
+        return -1;
+    }
+    unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj);
+    return 1;
+}
+
diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h
index 8b9fcc1..cce29e7 100644
--- a/msgpack/unpack_template.h
+++ b/msgpack/unpack_template.h
@@ -75,8 +75,7 @@ static inline void unpack_clear(unpack_context *ctx)
     Py_CLEAR(ctx->stack[0].obj);
 }
 
-template 
-static inline int unpack_execute(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off)
+static inline int unpack_execute(bool construct, unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off)
 {
     assert(len >= *off);
 
@@ -386,6 +385,7 @@ _end:
 #undef construct_cb
 }
 
+#undef NEXT_CS
 #undef SWITCH_RANGE_BEGIN
 #undef SWITCH_RANGE
 #undef SWITCH_RANGE_DEFAULT
@@ -397,68 +397,27 @@ _end:
 #undef again_fixed_trail_if_zero
 #undef start_container
 
-template 
-static inline int unpack_container_header(unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off)
-{
-    assert(len >= *off);
-    uint32_t size;
-    const unsigned char *const p = (unsigned char*)data + *off;
-
-#define inc_offset(inc) \
-    if (len - *off < inc) \
-        return 0; \
-    *off += inc;
-
-    switch (*p) {
-    case var_offset:
-        inc_offset(3);
-        size = _msgpack_load16(uint16_t, p + 1);
-        break;
-    case var_offset + 1:
-        inc_offset(5);
-        size = _msgpack_load32(uint32_t, p + 1);
-        break;
-#ifdef USE_CASE_RANGE
-    case fixed_offset + 0x0 ... fixed_offset + 0xf:
-#else
-    case fixed_offset + 0x0:
-    case fixed_offset + 0x1:
-    case fixed_offset + 0x2:
-    case fixed_offset + 0x3:
-    case fixed_offset + 0x4:
-    case fixed_offset + 0x5:
-    case fixed_offset + 0x6:
-    case fixed_offset + 0x7:
-    case fixed_offset + 0x8:
-    case fixed_offset + 0x9:
-    case fixed_offset + 0xa:
-    case fixed_offset + 0xb:
-    case fixed_offset + 0xc:
-    case fixed_offset + 0xd:
-    case fixed_offset + 0xe:
-    case fixed_offset + 0xf:
-#endif
-        ++*off;
-        size = ((unsigned int)*p) & 0x0f;
-        break;
-    default:
-        PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream");
-        return -1;
-    }
-    unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj);
-    return 1;
+static int unpack_construct(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) {
+    return unpack_execute(1, ctx, data, len, off);
+}
+static int unpack_skip(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) {
+    return unpack_execute(0, ctx, data, len, off);
 }
 
-#undef SWITCH_RANGE_BEGIN
-#undef SWITCH_RANGE
-#undef SWITCH_RANGE_DEFAULT
-#undef SWITCH_RANGE_END
+#define unpack_container_header read_array_header
+#define fixed_offset 0x90
+#define var_offset 0xdc
+#include "unpack_container_header.h"
+#undef unpack_container_header
+#undef fixed_offset
+#undef var_offset
 
-static const execute_fn unpack_construct = &unpack_execute;
-static const execute_fn unpack_skip = &unpack_execute;
-static const execute_fn read_array_header = &unpack_container_header<0x90, 0xdc>;
-static const execute_fn read_map_header = &unpack_container_header<0x80, 0xde>;
-
-#undef NEXT_CS
+#define unpack_container_header read_map_header
+#define fixed_offset 0x80
+#define var_offset 0xde
+#include "unpack_container_header.h"
+#undef unpack_container_header
+#undef fixed_offset
+#undef var_offset
 
 /* vim: set ts=4 sw=4 sts=4 expandtab  */
diff --git a/setup.py b/setup.py
index a13bd81..dc14a26 100644
--- a/setup.py
+++ b/setup.py
@@ -25,7 +25,7 @@ def cythonize(src):
     if not have_cython:
         raise Exception("Cython is required for building from checkout")
     sys.stderr.write(f"cythonize: {src!r}\n")
-    cython_compiler.compile([src], cplus=True)
+    cython_compiler.compile([src])
 
 
 def ensure_source(src):
@@ -51,17 +51,17 @@ class Sdist(sdist):
 
 libraries = []
 macros = []
+ext_modules = []
 
 if sys.platform == "win32":
     libraries.append("ws2_32")
     macros = [("__LITTLE_ENDIAN__", "1")]
 
-ext_modules = []
 if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"):
     ext_modules.append(
         Extension(
             "msgpack._cmsgpack",
-            sources=["msgpack/_cmsgpack.cpp"],
+            sources=["msgpack/_cmsgpack.c"],
             libraries=libraries,
             include_dirs=["."],
             define_macros=macros,

From b389ccf2f72355e23836ced193b555401508ef81 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Sat, 4 May 2024 16:10:37 +0900
Subject: [PATCH 1617/1648] update README (#561)

---
 README.md | 116 +++++++++++++++++++++++-------------------------------
 1 file changed, 50 insertions(+), 66 deletions(-)

diff --git a/README.md b/README.md
index 61f99e1..d3247bd 100644
--- a/README.md
+++ b/README.md
@@ -10,53 +10,6 @@ It lets you exchange data among multiple languages like JSON.
 But it's faster and smaller.
 This package provides CPython bindings for reading and writing MessagePack data.
 
-
-## Very important notes for existing users
-
-### PyPI package name
-
-Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5.
-
-When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before
-`pip install -U msgpack`.
-
-
-### Compatibility with the old format
-
-You can use `use_bin_type=False` option to pack `bytes`
-object into raw type in the old msgpack spec, instead of bin type in new msgpack spec.
-
-You can unpack old msgpack format using `raw=True` option.
-It unpacks str (raw) type in msgpack into Python bytes.
-
-See note below for detail.
-
-
-### Major breaking changes in msgpack 1.0
-
-* Python 2
-
-  * The extension module does not support Python 2 anymore.
-    The pure Python implementation (`msgpack.fallback`) is used for Python 2.
-
-* Packer
-
-  * `use_bin_type=True` by default.  bytes are encoded in bin type in msgpack.
-    **If you are still using Python 2, you must use unicode for all string types.**
-    You can use `use_bin_type=False` to encode into old msgpack format.
-  * `encoding` option is removed.  UTF-8 is used always.
-
-* Unpacker
-
-  * `raw=False` by default.  It assumes str types are valid UTF-8 string
-    and decode them to Python str (unicode) object.
-  * `encoding` option is removed.  You can use `raw=True` to support old format.
-  * Default value of `max_buffer_size` is changed from 0 to 100 MiB.
-  * Default value of `strict_map_key` is changed to True to avoid hashdos.
-    You need to pass `strict_map_key=False` if you have data which contain map keys
-    which type is not bytes or str.
-
-
 ## Install
 
 ```
@@ -65,12 +18,9 @@ $ pip install msgpack
 
 ### Pure Python implementation
 
-The extension module in msgpack (`msgpack._cmsgpack`) does not support
-Python 2 and PyPy.
-
-But msgpack provides a pure Python implementation (`msgpack.fallback`)
-for PyPy and Python 2.
+The extension module in msgpack (`msgpack._cmsgpack`) does not support PyPy.
 
+But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy.
 
 
 ### Windows
@@ -82,10 +32,6 @@ Without extension, using pure Python implementation on CPython runs slowly.
 
 ## How to use
 
-NOTE: In examples below, I use `raw=False` and `use_bin_type=True` for users
-using msgpack < 1.0. These options are default from msgpack 1.0 so you can omit them.
-
-
 ### One-shot pack & unpack
 
 Use `packb` for packing and `unpackb` for unpacking.
@@ -97,16 +43,16 @@ msgpack provides `dumps` and `loads` as an alias for compatibility with
 
 ```pycon
 >>> import msgpack
->>> msgpack.packb([1, 2, 3], use_bin_type=True)
+>>> msgpack.packb([1, 2, 3])
 '\x93\x01\x02\x03'
->>> msgpack.unpackb(_, raw=False)
+>>> msgpack.unpackb(_)
 [1, 2, 3]
 ```
 
 `unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple:
 
 ```pycon
->>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False, raw=False)
+>>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False)
 (1, 2, 3)
 ```
 
@@ -127,11 +73,11 @@ from io import BytesIO
 
 buf = BytesIO()
 for i in range(100):
-   buf.write(msgpack.packb(i, use_bin_type=True))
+   buf.write(msgpack.packb(i))
 
 buf.seek(0)
 
-unpacker = msgpack.Unpacker(buf, raw=False)
+unpacker = msgpack.Unpacker(buf)
 for unpacked in unpacker:
     print(unpacked)
 ```
@@ -162,8 +108,8 @@ def encode_datetime(obj):
     return obj
 
 
-packed_dict = msgpack.packb(useful_dict, default=encode_datetime, use_bin_type=True)
-this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime, raw=False)
+packed_dict = msgpack.packb(useful_dict, default=encode_datetime)
+this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime)
 ```
 
 `Unpacker`'s `object_hook` callback receives a dict; the
@@ -191,8 +137,8 @@ It is also possible to pack/unpack custom data types using the **ext** type.
 ...     return ExtType(code, data)
 ...
 >>> data = array.array('d', [1.2, 3.4])
->>> packed = msgpack.packb(data, default=default, use_bin_type=True)
->>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook, raw=False)
+>>> packed = msgpack.packb(data, default=default)
+>>> unpacked = msgpack.unpackb(packed, ext_hook=ext_hook)
 >>> data == unpacked
 True
 ```
@@ -210,7 +156,7 @@ in a map, can be unpacked or skipped individually.
 
 ## Notes
 
-### string and binary type
+### string and binary type in old msgpack spec
 
 Early versions of msgpack didn't distinguish string and binary types.
 The type for representing both string and binary types was named **raw**.
@@ -263,3 +209,41 @@ You can use `gc.disable()` when unpacking large message.
 List is the default sequence type of Python.
 But tuple is lighter than list.
 You can use `use_list=False` while unpacking when performance is important.
+
+
+## Major breaking changes in the history
+
+### msgpack 0.5
+
+Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5.
+
+When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before
+`pip install -U msgpack`.
+
+
+### msgpack 1.0
+
+* Python 2 support
+
+  * The extension module does not support Python 2 anymore.
+    The pure Python implementation (`msgpack.fallback`) is used for Python 2.
+  
+  * msgpack 1.0.6 drops official support of Python 2.7, as pip and
+    GitHub Action (setup-python) no longer support Python 2.7.
+
+* Packer
+
+  * Packer uses `use_bin_type=True` by default.
+    Bytes are encoded in bin type in msgpack.
+  * The `encoding` option is removed.  UTF-8 is used always.
+
+* Unpacker
+
+  * Unpacker uses `raw=False` by default.  It assumes str types are valid UTF-8 string
+    and decode them to Python str (unicode) object.
+  * `encoding` option is removed.  You can use `raw=True` to support old format (e.g. unpack into bytes, not str).
+  * Default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attack.
+    You need to pass `max_buffer_size=0` if you have large but safe data.
+  * Default value of `strict_map_key` is changed to True to avoid hashdos.
+    You need to pass `strict_map_key=False` if you have data which contain map keys
+    which type is not bytes or str.

From 526ec9c923c4867c96537c1f09783fe59432f737 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Sat, 4 May 2024 16:49:22 +0900
Subject: [PATCH 1618/1648] update cibuildwheel to 2.17 (#601)

---
 .github/workflows/docs.yaml |  1 -
 .github/workflows/wheel.yml | 15 ++++++---------
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index 08fc2f4..b696b92 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -17,7 +17,6 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.x'
-          architecture: 'x64'
           cache: "pip"
           cache-dependency-path: |
             requirements.txt
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index e91325b..d57e058 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -14,29 +14,25 @@ jobs:
     name: Build wheels on ${{ matrix.os }}
 
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
       - name: Set up QEMU
         if: runner.os == 'Linux'
         uses: docker/setup-qemu-action@v3
         with:
-          platforms: arm64
+          platforms: all
 
-      - name: Set up Python 3.x
-        uses: actions/setup-python@v5
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
           cache: "pip"
-
-      - name: Prepare
+      - name: Cythonize
         shell: bash
         run: |
           pip install -r requirements.txt
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v2.16.5
+        uses: pypa/cibuildwheel@v2.17.0
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
@@ -45,6 +41,7 @@ jobs:
           CIBW_SKIP: pp*
 
       - name: Build pure Python wheel
+        if: runner.os == 'Linux'
         env:
           MSGPACK_PUREPYTHON: "1"
         run: |

From 52f8bc2e557ca6684f2f73d129da8317a88bc431 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Sun, 5 May 2024 23:14:27 +0900
Subject: [PATCH 1619/1648] implement buffer protocol (#602)

Fix #479
---
 msgpack/_packer.pyx      | 10 +++++++---
 msgpack/buff_converter.h |  8 --------
 2 files changed, 7 insertions(+), 11 deletions(-)
 delete mode 100644 msgpack/buff_converter.h

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 3c39867..c201880 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -44,8 +44,6 @@ cdef extern from "pack.h":
     int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds);
     int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit)
 
-cdef extern from "buff_converter.h":
-    object buff_to_buff(char *, Py_ssize_t)
 
 cdef int DEFAULT_RECURSE_LIMIT=511
 cdef long long ITEM_LIMIT = (2**32)-1
@@ -371,4 +369,10 @@ cdef class Packer(object):
 
     def getbuffer(self):
         """Return view of internal buffer."""
-        return buff_to_buff(self.pk.buf, self.pk.length)
+        return memoryview(self)
+
+    def __getbuffer__(self, Py_buffer *buffer, int flags):
+        PyBuffer_FillInfo(buffer, self, self.pk.buf, self.pk.length, 1, flags)
+
+    def __releasebuffer__(self, Py_buffer *buffer):
+        pass
diff --git a/msgpack/buff_converter.h b/msgpack/buff_converter.h
deleted file mode 100644
index 86b4196..0000000
--- a/msgpack/buff_converter.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#include "Python.h"
-
-/* cython does not support this preprocessor check => write it in raw C */
-static PyObject *
-buff_to_buff(char *buff, Py_ssize_t size)
-{
-    return PyMemoryView_FromMemory(buff, size, PyBUF_READ);
-}

From a97b31437d3301fa4c29b7813dabee0e690756e8 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 6 May 2024 00:13:59 +0900
Subject: [PATCH 1620/1648] Remove unused code (#603)

---
 msgpack/_packer.pyx     |   7 --
 msgpack/pack.h          |   3 +-
 msgpack/pack_template.h | 238 ++--------------------------------------
 3 files changed, 8 insertions(+), 240 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index c201880..c7eeda1 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -16,8 +16,6 @@ from .ext import ExtType, Timestamp
 cdef extern from "Python.h":
 
     int PyMemoryView_Check(object obj)
-    char* PyUnicode_AsUTF8AndSize(object obj, Py_ssize_t *l) except NULL
-
 
 cdef extern from "pack.h":
     struct msgpack_packer:
@@ -26,11 +24,9 @@ cdef extern from "pack.h":
         size_t buf_size
         bint use_bin_type
 
-    int msgpack_pack_int(msgpack_packer* pk, int d)
     int msgpack_pack_nil(msgpack_packer* pk)
     int msgpack_pack_true(msgpack_packer* pk)
     int msgpack_pack_false(msgpack_packer* pk)
-    int msgpack_pack_long(msgpack_packer* pk, long d)
     int msgpack_pack_long_long(msgpack_packer* pk, long long d)
     int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d)
     int msgpack_pack_float(msgpack_packer* pk, float d)
@@ -184,9 +180,6 @@ cdef class Packer(object):
                         continue
                     else:
                         raise OverflowError("Integer value out of range")
-            elif PyInt_CheckExact(o) if strict_types else PyInt_Check(o):
-                longval = o
-                ret = msgpack_pack_long(&self.pk, longval)
             elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o):
                 if self.use_float:
                    fval = o
diff --git a/msgpack/pack.h b/msgpack/pack.h
index 901fad7..688eab8 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -21,11 +21,10 @@
 #include "sysdep.h"
 #include 
 #include 
+#include 
 
 #ifdef __cplusplus
 extern "C" {
-#else
-#define bool char
 #endif
 
 typedef struct msgpack_packer {
diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h
index 7d479b6..b8959f0 100644
--- a/msgpack/pack_template.h
+++ b/msgpack/pack_template.h
@@ -37,18 +37,6 @@
  * Integer
  */
 
-#define msgpack_pack_real_uint8(x, d) \
-do { \
-    if(d < (1<<7)) { \
-        /* fixnum */ \
-        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
-    } else { \
-        /* unsigned 8 */ \
-        unsigned char buf[2] = {0xcc, TAKE8_8(d)}; \
-        msgpack_pack_append_buffer(x, buf, 2); \
-    } \
-} while(0)
-
 #define msgpack_pack_real_uint16(x, d) \
 do { \
     if(d < (1<<7)) { \
@@ -123,18 +111,6 @@ do { \
     } \
 } while(0)
 
-#define msgpack_pack_real_int8(x, d) \
-do { \
-    if(d < -(1<<5)) { \
-        /* signed 8 */ \
-        unsigned char buf[2] = {0xd0, TAKE8_8(d)}; \
-        msgpack_pack_append_buffer(x, buf, 2); \
-    } else { \
-        /* fixnum */ \
-        msgpack_pack_append_buffer(x, &TAKE8_8(d), 1); \
-    } \
-} while(0)
-
 #define msgpack_pack_real_int16(x, d) \
 do { \
     if(d < -(1<<5)) { \
@@ -264,49 +240,6 @@ do { \
 } while(0)
 
 
-static inline int msgpack_pack_uint8(msgpack_packer* x, uint8_t d)
-{
-    msgpack_pack_real_uint8(x, d);
-}
-
-static inline int msgpack_pack_uint16(msgpack_packer* x, uint16_t d)
-{
-    msgpack_pack_real_uint16(x, d);
-}
-
-static inline int msgpack_pack_uint32(msgpack_packer* x, uint32_t d)
-{
-    msgpack_pack_real_uint32(x, d);
-}
-
-static inline int msgpack_pack_uint64(msgpack_packer* x, uint64_t d)
-{
-    msgpack_pack_real_uint64(x, d);
-}
-
-static inline int msgpack_pack_int8(msgpack_packer* x, int8_t d)
-{
-    msgpack_pack_real_int8(x, d);
-}
-
-static inline int msgpack_pack_int16(msgpack_packer* x, int16_t d)
-{
-    msgpack_pack_real_int16(x, d);
-}
-
-static inline int msgpack_pack_int32(msgpack_packer* x, int32_t d)
-{
-    msgpack_pack_real_int32(x, d);
-}
-
-static inline int msgpack_pack_int64(msgpack_packer* x, int64_t d)
-{
-    msgpack_pack_real_int64(x, d);
-}
-
-
-//#ifdef msgpack_pack_inline_func_cint
-
 static inline int msgpack_pack_short(msgpack_packer* x, short d)
 {
 #if defined(SIZEOF_SHORT)
@@ -372,192 +305,37 @@ if(sizeof(int) == 2) {
 static inline int msgpack_pack_long(msgpack_packer* x, long d)
 {
 #if defined(SIZEOF_LONG)
-#if SIZEOF_LONG == 2
-    msgpack_pack_real_int16(x, d);
-#elif SIZEOF_LONG == 4
+#if SIZEOF_LONG == 4
     msgpack_pack_real_int32(x, d);
 #else
     msgpack_pack_real_int64(x, d);
 #endif
 
 #elif defined(LONG_MAX)
-#if LONG_MAX == 0x7fffL
-    msgpack_pack_real_int16(x, d);
-#elif LONG_MAX == 0x7fffffffL
+#if LONG_MAX == 0x7fffffffL
     msgpack_pack_real_int32(x, d);
 #else
     msgpack_pack_real_int64(x, d);
 #endif
 
 #else
-if(sizeof(long) == 2) {
-    msgpack_pack_real_int16(x, d);
-} else if(sizeof(long) == 4) {
-    msgpack_pack_real_int32(x, d);
-} else {
-    msgpack_pack_real_int64(x, d);
-}
+    if (sizeof(long) == 4) {
+        msgpack_pack_real_int32(x, d);
+    } else {
+        msgpack_pack_real_int64(x, d);
+    }
 #endif
 }
 
 static inline int msgpack_pack_long_long(msgpack_packer* x, long long d)
 {
-#if defined(SIZEOF_LONG_LONG)
-#if SIZEOF_LONG_LONG == 2
-    msgpack_pack_real_int16(x, d);
-#elif SIZEOF_LONG_LONG == 4
-    msgpack_pack_real_int32(x, d);
-#else
     msgpack_pack_real_int64(x, d);
-#endif
-
-#elif defined(LLONG_MAX)
-#if LLONG_MAX == 0x7fffL
-    msgpack_pack_real_int16(x, d);
-#elif LLONG_MAX == 0x7fffffffL
-    msgpack_pack_real_int32(x, d);
-#else
-    msgpack_pack_real_int64(x, d);
-#endif
-
-#else
-if(sizeof(long long) == 2) {
-    msgpack_pack_real_int16(x, d);
-} else if(sizeof(long long) == 4) {
-    msgpack_pack_real_int32(x, d);
-} else {
-    msgpack_pack_real_int64(x, d);
-}
-#endif
-}
-
-static inline int msgpack_pack_unsigned_short(msgpack_packer* x, unsigned short d)
-{
-#if defined(SIZEOF_SHORT)
-#if SIZEOF_SHORT == 2
-    msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_SHORT == 4
-    msgpack_pack_real_uint32(x, d);
-#else
-    msgpack_pack_real_uint64(x, d);
-#endif
-
-#elif defined(USHRT_MAX)
-#if USHRT_MAX == 0xffffU
-    msgpack_pack_real_uint16(x, d);
-#elif USHRT_MAX == 0xffffffffU
-    msgpack_pack_real_uint32(x, d);
-#else
-    msgpack_pack_real_uint64(x, d);
-#endif
-
-#else
-if(sizeof(unsigned short) == 2) {
-    msgpack_pack_real_uint16(x, d);
-} else if(sizeof(unsigned short) == 4) {
-    msgpack_pack_real_uint32(x, d);
-} else {
-    msgpack_pack_real_uint64(x, d);
-}
-#endif
-}
-
-static inline int msgpack_pack_unsigned_int(msgpack_packer* x, unsigned int d)
-{
-#if defined(SIZEOF_INT)
-#if SIZEOF_INT == 2
-    msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_INT == 4
-    msgpack_pack_real_uint32(x, d);
-#else
-    msgpack_pack_real_uint64(x, d);
-#endif
-
-#elif defined(UINT_MAX)
-#if UINT_MAX == 0xffffU
-    msgpack_pack_real_uint16(x, d);
-#elif UINT_MAX == 0xffffffffU
-    msgpack_pack_real_uint32(x, d);
-#else
-    msgpack_pack_real_uint64(x, d);
-#endif
-
-#else
-if(sizeof(unsigned int) == 2) {
-    msgpack_pack_real_uint16(x, d);
-} else if(sizeof(unsigned int) == 4) {
-    msgpack_pack_real_uint32(x, d);
-} else {
-    msgpack_pack_real_uint64(x, d);
-}
-#endif
-}
-
-static inline int msgpack_pack_unsigned_long(msgpack_packer* x, unsigned long d)
-{
-#if defined(SIZEOF_LONG)
-#if SIZEOF_LONG == 2
-    msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_LONG == 4
-    msgpack_pack_real_uint32(x, d);
-#else
-    msgpack_pack_real_uint64(x, d);
-#endif
-
-#elif defined(ULONG_MAX)
-#if ULONG_MAX == 0xffffUL
-    msgpack_pack_real_uint16(x, d);
-#elif ULONG_MAX == 0xffffffffUL
-    msgpack_pack_real_uint32(x, d);
-#else
-    msgpack_pack_real_uint64(x, d);
-#endif
-
-#else
-if(sizeof(unsigned long) == 2) {
-    msgpack_pack_real_uint16(x, d);
-} else if(sizeof(unsigned long) == 4) {
-    msgpack_pack_real_uint32(x, d);
-} else {
-    msgpack_pack_real_uint64(x, d);
-}
-#endif
 }
 
 static inline int msgpack_pack_unsigned_long_long(msgpack_packer* x, unsigned long long d)
 {
-#if defined(SIZEOF_LONG_LONG)
-#if SIZEOF_LONG_LONG == 2
-    msgpack_pack_real_uint16(x, d);
-#elif SIZEOF_LONG_LONG == 4
-    msgpack_pack_real_uint32(x, d);
-#else
-    msgpack_pack_real_uint64(x, d);
-#endif
-
-#elif defined(ULLONG_MAX)
-#if ULLONG_MAX == 0xffffUL
-    msgpack_pack_real_uint16(x, d);
-#elif ULLONG_MAX == 0xffffffffUL
-    msgpack_pack_real_uint32(x, d);
-#else
-    msgpack_pack_real_uint64(x, d);
-#endif
-
-#else
-if(sizeof(unsigned long long) == 2) {
-    msgpack_pack_real_uint16(x, d);
-} else if(sizeof(unsigned long long) == 4) {
-    msgpack_pack_real_uint32(x, d);
-} else {
     msgpack_pack_real_uint64(x, d);
 }
-#endif
-}
-
-//#undef msgpack_pack_inline_func_cint
-//#endif
-
 
 
 /*
@@ -810,11 +588,9 @@ static inline int msgpack_pack_timestamp(msgpack_packer* x, int64_t seconds, uin
 #undef TAKE8_32
 #undef TAKE8_64
 
-#undef msgpack_pack_real_uint8
 #undef msgpack_pack_real_uint16
 #undef msgpack_pack_real_uint32
 #undef msgpack_pack_real_uint64
-#undef msgpack_pack_real_int8
 #undef msgpack_pack_real_int16
 #undef msgpack_pack_real_int32
 #undef msgpack_pack_real_int64

From bf2413f915474841aafee2c98321dc465e5f0a3e Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 6 May 2024 00:30:07 +0900
Subject: [PATCH 1621/1648] ignore msgpack/*.c

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 8a06e26..341be63 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@ dist/*
 *.so
 *~
 msgpack/__version__.py
+msgpack/*.c
 msgpack/*.cpp
 *.egg-info
 /venv

From 72e65feb0e02449fa191346f26b54d3842ab7e69 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 6 May 2024 00:49:12 +0900
Subject: [PATCH 1622/1648] packer: add `buf_size` option (#604)

And change the default buffer size to 256KiB.

Signed-off-by: Rodrigo Tobar 
Co-authored-by: Rodrigo Tobar 
---
 msgpack/_packer.pyx   | 13 +++++++++----
 msgpack/_unpacker.pyx |  2 +-
 msgpack/fallback.py   | 27 +++++----------------------
 3 files changed, 15 insertions(+), 27 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index c7eeda1..99557d3 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -53,7 +53,7 @@ cdef inline int PyBytesLike_CheckExact(object o):
     return PyBytes_CheckExact(o) or PyByteArray_CheckExact(o)
 
 
-cdef class Packer(object):
+cdef class Packer:
     """
     MessagePack Packer
 
@@ -97,6 +97,11 @@ cdef class Packer(object):
     :param str unicode_errors:
         The error handler for encoding unicode. (default: 'strict')
         DO NOT USE THIS!!  This option is kept for very specific usage.
+
+    :param int buf_size:
+        The size of the internal buffer. (default: 256*1024)
+        Useful if serialisation size can be correctly estimated,
+        avoid unnecessary reallocations.
     """
     cdef msgpack_packer pk
     cdef object _default
@@ -107,8 +112,7 @@ cdef class Packer(object):
     cdef bint autoreset
     cdef bint datetime
 
-    def __cinit__(self):
-        cdef int buf_size = 1024*1024
+    def __cinit__(self, buf_size=256*1024, **_kwargs):
         self.pk.buf =  PyMem_Malloc(buf_size)
         if self.pk.buf == NULL:
             raise MemoryError("Unable to allocate internal buffer.")
@@ -117,7 +121,8 @@ cdef class Packer(object):
 
     def __init__(self, *, default=None,
                  bint use_single_float=False, bint autoreset=True, bint use_bin_type=True,
-                 bint strict_types=False, bint datetime=False, unicode_errors=None):
+                 bint strict_types=False, bint datetime=False, unicode_errors=None,
+                 buf_size=256*1024):
         self.use_float = use_single_float
         self.strict_types = strict_types
         self.autoreset = autoreset
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 2771e7b..34ff330 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -210,7 +210,7 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None,
     raise ValueError("Unpack failed: error = %d" % (ret,))
 
 
-cdef class Unpacker(object):
+cdef class Unpacker:
     """Streaming unpacker.
 
     Arguments:
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index ea4c4ce..cbf0d30 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -232,6 +232,7 @@ class Unpacker:
     def __init__(
         self,
         file_like=None,
+        *,
         read_size=0,
         use_list=True,
         raw=False,
@@ -650,32 +651,13 @@ class Packer:
         The error handler for encoding unicode. (default: 'strict')
         DO NOT USE THIS!!  This option is kept for very specific usage.
 
-    Example of streaming deserialize from file-like object::
-
-        unpacker = Unpacker(file_like)
-        for o in unpacker:
-            process(o)
-
-    Example of streaming deserialize from socket::
-
-        unpacker = Unpacker()
-        while True:
-            buf = sock.recv(1024**2)
-            if not buf:
-                break
-            unpacker.feed(buf)
-            for o in unpacker:
-                process(o)
-
-    Raises ``ExtraData`` when *packed* contains extra bytes.
-    Raises ``OutOfData`` when *packed* is incomplete.
-    Raises ``FormatError`` when *packed* is not valid msgpack.
-    Raises ``StackError`` when *packed* contains too nested.
-    Other exceptions can be raised during unpacking.
+    :param int buf_size:
+        Internal buffer size. This option is used only for C implementation.
     """
 
     def __init__(
         self,
+        *,
         default=None,
         use_single_float=False,
         autoreset=True,
@@ -683,6 +665,7 @@ class Packer:
         strict_types=False,
         datetime=False,
         unicode_errors=None,
+        buf_size=None,
     ):
         self._strict_types = strict_types
         self._use_float = use_single_float

From 3da5818a3a96a4bd39f25a483ee3c22ecb626e9d Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 6 May 2024 02:12:46 +0900
Subject: [PATCH 1623/1648] update readme (#605)

---
 README.md | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index d3247bd..61a03c6 100644
--- a/README.md
+++ b/README.md
@@ -49,17 +49,7 @@ msgpack provides `dumps` and `loads` as an alias for compatibility with
 [1, 2, 3]
 ```
 
-`unpack` unpacks msgpack's array to Python's list, but can also unpack to tuple:
-
-```pycon
->>> msgpack.unpackb(b'\x93\x01\x02\x03', use_list=False)
-(1, 2, 3)
-```
-
-You should always specify the `use_list` keyword argument for backward compatibility.
-See performance issues relating to `use_list option`_ below.
-
-Read the docstring for other options.
+Read the docstring for options.
 
 
 ### Streaming unpacking
@@ -116,6 +106,9 @@ this_dict_again = msgpack.unpackb(packed_dict, object_hook=decode_datetime)
 `object_pairs_hook` callback may instead be used to receive a list of
 key-value pairs.
 
+NOTE: msgpack can encode datetime with tzinfo into standard ext type for now.
+See `datetime` option in `Packer` docstring.
+
 
 ### Extended types
 

From e1068087e0bb3aebd77c4f2407bf9c429f631787 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 6 May 2024 02:13:12 +0900
Subject: [PATCH 1624/1648] cython: better exception handling (#606)

- use `except -1` instead of manual error handling
- use `PyUnicode_AsUTF8AndSize()`
- use `_pack()` and `_pack_inner()` instead of `while True:`
---
 msgpack/_packer.pyx | 308 +++++++++++++++++++-------------------------
 msgpack/pack.h      |  21 ---
 2 files changed, 134 insertions(+), 195 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 99557d3..ad53221 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -24,21 +24,20 @@ cdef extern from "pack.h":
         size_t buf_size
         bint use_bin_type
 
-    int msgpack_pack_nil(msgpack_packer* pk)
-    int msgpack_pack_true(msgpack_packer* pk)
-    int msgpack_pack_false(msgpack_packer* pk)
-    int msgpack_pack_long_long(msgpack_packer* pk, long long d)
-    int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d)
-    int msgpack_pack_float(msgpack_packer* pk, float d)
-    int msgpack_pack_double(msgpack_packer* pk, double d)
-    int msgpack_pack_array(msgpack_packer* pk, size_t l)
-    int msgpack_pack_map(msgpack_packer* pk, size_t l)
-    int msgpack_pack_raw(msgpack_packer* pk, size_t l)
-    int msgpack_pack_bin(msgpack_packer* pk, size_t l)
-    int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
-    int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l)
-    int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds);
-    int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit)
+    int msgpack_pack_nil(msgpack_packer* pk) except -1
+    int msgpack_pack_true(msgpack_packer* pk) except -1
+    int msgpack_pack_false(msgpack_packer* pk) except -1
+    int msgpack_pack_long_long(msgpack_packer* pk, long long d) except -1
+    int msgpack_pack_unsigned_long_long(msgpack_packer* pk, unsigned long long d) except -1
+    int msgpack_pack_float(msgpack_packer* pk, float d) except -1
+    int msgpack_pack_double(msgpack_packer* pk, double d) except -1
+    int msgpack_pack_array(msgpack_packer* pk, size_t l) except -1
+    int msgpack_pack_map(msgpack_packer* pk, size_t l) except -1
+    int msgpack_pack_raw(msgpack_packer* pk, size_t l) except -1
+    int msgpack_pack_bin(msgpack_packer* pk, size_t l) except -1
+    int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l) except -1
+    int msgpack_pack_ext(msgpack_packer* pk, char typecode, size_t l) except -1
+    int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds) except -1
 
 
 cdef int DEFAULT_RECURSE_LIMIT=511
@@ -119,6 +118,10 @@ cdef class Packer:
         self.pk.buf_size = buf_size
         self.pk.length = 0
 
+    def __dealloc__(self):
+        PyMem_Free(self.pk.buf)
+        self.pk.buf = NULL
+
     def __init__(self, *, default=None,
                  bint use_single_float=False, bint autoreset=True, bint use_bin_type=True,
                  bint strict_types=False, bint datetime=False, unicode_errors=None,
@@ -139,155 +142,127 @@ cdef class Packer:
         else:
             self.unicode_errors = self._berrors
 
-    def __dealloc__(self):
-        PyMem_Free(self.pk.buf)
-        self.pk.buf = NULL
-
-    cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1:
+    # returns -2 when default should(o) be called
+    cdef int _pack_inner(self, object o, bint will_default, int nest_limit) except -1:
         cdef long long llval
         cdef unsigned long long ullval
         cdef unsigned long ulval
-        cdef long longval
-        cdef float fval
-        cdef double dval
-        cdef char* rawval
-        cdef int ret
-        cdef dict d
+        cdef const char* rawval
         cdef Py_ssize_t L
-        cdef int default_used = 0
-        cdef bint strict_types = self.strict_types
+        cdef bool strict_types = self.strict_types
         cdef Py_buffer view
 
+        if o is None:
+            msgpack_pack_nil(&self.pk)
+        elif o is True:
+            msgpack_pack_true(&self.pk)
+        elif o is False:
+            msgpack_pack_false(&self.pk)
+        elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o):
+            try:
+                if o > 0:
+                    ullval = o
+                    msgpack_pack_unsigned_long_long(&self.pk, ullval)
+                else:
+                    llval = o
+                    msgpack_pack_long_long(&self.pk, llval)
+            except OverflowError as oe:
+                if will_default:
+                    return -2
+                else:
+                    raise OverflowError("Integer value out of range")
+        elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o):
+            if self.use_float:
+                msgpack_pack_float(&self.pk, o)
+            else:
+                msgpack_pack_double(&self.pk, o)
+        elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o):
+            L = Py_SIZE(o)
+            if L > ITEM_LIMIT:
+                PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name)
+            rawval = o
+            msgpack_pack_bin(&self.pk, L)
+            msgpack_pack_raw_body(&self.pk, rawval, L)
+        elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o):
+            if self.unicode_errors == NULL:
+                rawval = PyUnicode_AsUTF8AndSize(o, &L)
+                if L >ITEM_LIMIT:
+                    raise ValueError("unicode string is too large")
+            else:
+                o = PyUnicode_AsEncodedString(o, NULL, self.unicode_errors)
+                L = Py_SIZE(o)
+                if L > ITEM_LIMIT:
+                    raise ValueError("unicode string is too large")
+                rawval = o
+            msgpack_pack_raw(&self.pk, L)
+            msgpack_pack_raw_body(&self.pk, rawval, L)
+        elif PyDict_CheckExact(o) if strict_types else PyDict_Check(o):
+            L = len(o)
+            if L > ITEM_LIMIT:
+                raise ValueError("dict is too large")
+            msgpack_pack_map(&self.pk, L)
+            for k, v in o.items():
+                self._pack(k, nest_limit)
+                self._pack(v, nest_limit)
+        elif type(o) is ExtType if strict_types else isinstance(o, ExtType):
+            # This should be before Tuple because ExtType is namedtuple.
+            rawval = o.data
+            L = len(o.data)
+            if L > ITEM_LIMIT:
+                raise ValueError("EXT data is too large")
+            msgpack_pack_ext(&self.pk, o.code, L)
+            msgpack_pack_raw_body(&self.pk, rawval, L)
+        elif type(o) is Timestamp:
+            llval = o.seconds
+            ulval = o.nanoseconds
+            msgpack_pack_timestamp(&self.pk, llval, ulval)
+        elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
+            L = Py_SIZE(o)
+            if L > ITEM_LIMIT:
+                raise ValueError("list is too large")
+            msgpack_pack_array(&self.pk, L)
+            for v in o:
+                self._pack(v, nest_limit)
+        elif PyMemoryView_Check(o):
+            PyObject_GetBuffer(o, &view, PyBUF_SIMPLE)
+            L = view.len
+            if L > ITEM_LIMIT:
+                PyBuffer_Release(&view);
+                raise ValueError("memoryview is too large")
+            try:
+                msgpack_pack_bin(&self.pk, L)
+                msgpack_pack_raw_body(&self.pk, view.buf, L)
+            finally:
+                PyBuffer_Release(&view);
+        elif self.datetime and PyDateTime_CheckExact(o) and datetime_tzinfo(o) is not None:
+            delta = o - epoch
+            if not PyDelta_CheckExact(delta):
+                raise ValueError("failed to calculate delta")
+            llval = timedelta_days(delta) * (24*60*60) + timedelta_seconds(delta)
+            ulval = timedelta_microseconds(delta) * 1000
+            msgpack_pack_timestamp(&self.pk, llval, ulval)
+        elif will_default:
+            return -2
+        elif self.datetime and PyDateTime_CheckExact(o):
+            # this should be later than will_default
+            PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name)
+        else:
+            PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name)
+
+    cdef int _pack(self, object o, int nest_limit=DEFAULT_RECURSE_LIMIT) except -1:
+        cdef int ret
         if nest_limit < 0:
             raise ValueError("recursion limit exceeded.")
-
-        while True:
-            if o is None:
-                ret = msgpack_pack_nil(&self.pk)
-            elif o is True:
-                ret = msgpack_pack_true(&self.pk)
-            elif o is False:
-                ret = msgpack_pack_false(&self.pk)
-            elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o):
-                # PyInt_Check(long) is True for Python 3.
-                # So we should test long before int.
-                try:
-                    if o > 0:
-                        ullval = o
-                        ret = msgpack_pack_unsigned_long_long(&self.pk, ullval)
-                    else:
-                        llval = o
-                        ret = msgpack_pack_long_long(&self.pk, llval)
-                except OverflowError as oe:
-                    if not default_used and self._default is not None:
-                        o = self._default(o)
-                        default_used = True
-                        continue
-                    else:
-                        raise OverflowError("Integer value out of range")
-            elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o):
-                if self.use_float:
-                   fval = o
-                   ret = msgpack_pack_float(&self.pk, fval)
-                else:
-                   dval = o
-                   ret = msgpack_pack_double(&self.pk, dval)
-            elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o):
-                L = Py_SIZE(o)
-                if L > ITEM_LIMIT:
-                    PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name)
-                rawval = o
-                ret = msgpack_pack_bin(&self.pk, L)
-                if ret == 0:
-                    ret = msgpack_pack_raw_body(&self.pk, rawval, L)
-            elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o):
-                if self.unicode_errors == NULL:
-                    ret = msgpack_pack_unicode(&self.pk, o, ITEM_LIMIT);
-                    if ret == -2:
-                        raise ValueError("unicode string is too large")
-                else:
-                    o = PyUnicode_AsEncodedString(o, NULL, self.unicode_errors)
-                    L = Py_SIZE(o)
-                    if L > ITEM_LIMIT:
-                        raise ValueError("unicode string is too large")
-                    ret = msgpack_pack_raw(&self.pk, L)
-                    if ret == 0:
-                        rawval = o
-                        ret = msgpack_pack_raw_body(&self.pk, rawval, L)
-            elif PyDict_CheckExact(o):
-                d = o
-                L = len(d)
-                if L > ITEM_LIMIT:
-                    raise ValueError("dict is too large")
-                ret = msgpack_pack_map(&self.pk, L)
-                if ret == 0:
-                    for k, v in d.items():
-                        ret = self._pack(k, nest_limit-1)
-                        if ret != 0: break
-                        ret = self._pack(v, nest_limit-1)
-                        if ret != 0: break
-            elif not strict_types and PyDict_Check(o):
-                L = len(o)
-                if L > ITEM_LIMIT:
-                    raise ValueError("dict is too large")
-                ret = msgpack_pack_map(&self.pk, L)
-                if ret == 0:
-                    for k, v in o.items():
-                        ret = self._pack(k, nest_limit-1)
-                        if ret != 0: break
-                        ret = self._pack(v, nest_limit-1)
-                        if ret != 0: break
-            elif type(o) is ExtType if strict_types else isinstance(o, ExtType):
-                # This should be before Tuple because ExtType is namedtuple.
-                longval = o.code
-                rawval = o.data
-                L = len(o.data)
-                if L > ITEM_LIMIT:
-                    raise ValueError("EXT data is too large")
-                ret = msgpack_pack_ext(&self.pk, longval, L)
-                ret = msgpack_pack_raw_body(&self.pk, rawval, L)
-            elif type(o) is Timestamp:
-                llval = o.seconds
-                ulval = o.nanoseconds
-                ret = msgpack_pack_timestamp(&self.pk, llval, ulval)
-            elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
-                L = Py_SIZE(o)
-                if L > ITEM_LIMIT:
-                    raise ValueError("list is too large")
-                ret = msgpack_pack_array(&self.pk, L)
-                if ret == 0:
-                    for v in o:
-                        ret = self._pack(v, nest_limit-1)
-                        if ret != 0: break
-            elif PyMemoryView_Check(o):
-                if PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) != 0:
-                    raise ValueError("could not get buffer for memoryview")
-                L = view.len
-                if L > ITEM_LIMIT:
-                    PyBuffer_Release(&view);
-                    raise ValueError("memoryview is too large")
-                ret = msgpack_pack_bin(&self.pk, L)
-                if ret == 0:
-                    ret = msgpack_pack_raw_body(&self.pk, view.buf, L)
-                PyBuffer_Release(&view);
-            elif self.datetime and PyDateTime_CheckExact(o) and datetime_tzinfo(o) is not None:
-                delta = o - epoch
-                if not PyDelta_CheckExact(delta):
-                    raise ValueError("failed to calculate delta")
-                llval = timedelta_days(delta) * (24*60*60) + timedelta_seconds(delta)
-                ulval = timedelta_microseconds(delta) * 1000
-                ret = msgpack_pack_timestamp(&self.pk, llval, ulval)
-            elif not default_used and self._default:
+        nest_limit -= 1
+        if self._default is not None:
+            ret = self._pack_inner(o, 1, nest_limit)
+            if ret == -2:
                 o = self._default(o)
-                default_used = 1
-                continue
-            elif self.datetime and PyDateTime_CheckExact(o):
-                PyErr_Format(ValueError, b"can not serialize '%.200s' object where tzinfo=None", Py_TYPE(o).tp_name)
             else:
-                PyErr_Format(TypeError, b"can not serialize '%.200s' object", Py_TYPE(o).tp_name)
-            return ret
+                return ret
+        return self._pack_inner(o, 0, nest_limit)
 
-    cpdef pack(self, object obj):
+    def pack(self, object obj):
         cdef int ret
         try:
             ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
@@ -308,11 +283,7 @@ cdef class Packer:
     def pack_array_header(self, long long size):
         if size > ITEM_LIMIT:
             raise ValueError
-        cdef int ret = msgpack_pack_array(&self.pk, size)
-        if ret == -1:
-            raise MemoryError
-        elif ret:  # should not happen
-            raise TypeError
+        msgpack_pack_array(&self.pk, size)
         if self.autoreset:
             buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
             self.pk.length = 0
@@ -321,11 +292,7 @@ cdef class Packer:
     def pack_map_header(self, long long size):
         if size > ITEM_LIMIT:
             raise ValueError
-        cdef int ret = msgpack_pack_map(&self.pk, size)
-        if ret == -1:
-            raise MemoryError
-        elif ret:  # should not happen
-            raise TypeError
+        msgpack_pack_map(&self.pk, size)
         if self.autoreset:
             buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
             self.pk.length = 0
@@ -338,17 +305,10 @@ cdef class Packer:
         *pairs* should be a sequence of pairs.
         (`len(pairs)` and `for k, v in pairs:` should be supported.)
         """
-        cdef int ret = msgpack_pack_map(&self.pk, len(pairs))
-        if ret == 0:
-            for k, v in pairs:
-                ret = self._pack(k)
-                if ret != 0: break
-                ret = self._pack(v)
-                if ret != 0: break
-        if ret == -1:
-            raise MemoryError
-        elif ret:  # should not happen
-            raise TypeError
+        msgpack_pack_map(&self.pk, len(pairs))
+        for k, v in pairs:
+            self._pack(k)
+            self._pack(v)
         if self.autoreset:
             buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
             self.pk.length = 0
diff --git a/msgpack/pack.h b/msgpack/pack.h
index 688eab8..edf3a3f 100644
--- a/msgpack/pack.h
+++ b/msgpack/pack.h
@@ -64,27 +64,6 @@ static inline int msgpack_pack_write(msgpack_packer* pk, const char *data, size_
 
 #include "pack_template.h"
 
-// return -2 when o is too long
-static inline int
-msgpack_pack_unicode(msgpack_packer *pk, PyObject *o, long long limit)
-{
-    assert(PyUnicode_Check(o));
-
-    Py_ssize_t len;
-    const char* buf = PyUnicode_AsUTF8AndSize(o, &len);
-    if (buf == NULL)
-        return -1;
-
-    if (len > limit) {
-        return -2;
-    }
-
-    int ret = msgpack_pack_raw(pk, len);
-    if (ret) return ret;
-
-    return msgpack_pack_raw_body(pk, buf, len);
-}
-
 #ifdef __cplusplus
 }
 #endif

From e0f0e145f15364819bac80bd0808834a9df0065e Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 6 May 2024 03:33:48 +0900
Subject: [PATCH 1625/1648] better error checks (#607)

* check buffer exports
* add error messages
---
 msgpack/_packer.pyx | 48 +++++++++++++++++++++++++++++++++------------
 test/test_buffer.py | 24 +++++++++++++++++++++--
 2 files changed, 57 insertions(+), 15 deletions(-)

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index ad53221..402b694 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -106,6 +106,7 @@ cdef class Packer:
     cdef object _default
     cdef object _berrors
     cdef const char *unicode_errors
+    cdef size_t exports  # number of exported buffers
     cdef bint strict_types
     cdef bint use_float
     cdef bint autoreset
@@ -117,10 +118,16 @@ cdef class Packer:
             raise MemoryError("Unable to allocate internal buffer.")
         self.pk.buf_size = buf_size
         self.pk.length = 0
+        self.exports = 0
 
     def __dealloc__(self):
         PyMem_Free(self.pk.buf)
         self.pk.buf = NULL
+        assert self.exports == 0
+
+    cdef _check_exports(self):
+        if self.exports > 0:
+            raise BufferError("Existing exports of data: Packer cannot be changed")
 
     def __init__(self, *, default=None,
                  bint use_single_float=False, bint autoreset=True, bint use_bin_type=True,
@@ -149,8 +156,8 @@ cdef class Packer:
         cdef unsigned long ulval
         cdef const char* rawval
         cdef Py_ssize_t L
-        cdef bool strict_types = self.strict_types
         cdef Py_buffer view
+        cdef bint strict = self.strict_types
 
         if o is None:
             msgpack_pack_nil(&self.pk)
@@ -158,7 +165,7 @@ cdef class Packer:
             msgpack_pack_true(&self.pk)
         elif o is False:
             msgpack_pack_false(&self.pk)
-        elif PyLong_CheckExact(o) if strict_types else PyLong_Check(o):
+        elif PyLong_CheckExact(o) if strict else PyLong_Check(o):
             try:
                 if o > 0:
                     ullval = o
@@ -171,19 +178,19 @@ cdef class Packer:
                     return -2
                 else:
                     raise OverflowError("Integer value out of range")
-        elif PyFloat_CheckExact(o) if strict_types else PyFloat_Check(o):
+        elif PyFloat_CheckExact(o) if strict else PyFloat_Check(o):
             if self.use_float:
                 msgpack_pack_float(&self.pk, o)
             else:
                 msgpack_pack_double(&self.pk, o)
-        elif PyBytesLike_CheckExact(o) if strict_types else PyBytesLike_Check(o):
+        elif PyBytesLike_CheckExact(o) if strict else PyBytesLike_Check(o):
             L = Py_SIZE(o)
             if L > ITEM_LIMIT:
                 PyErr_Format(ValueError, b"%.200s object is too large", Py_TYPE(o).tp_name)
             rawval = o
             msgpack_pack_bin(&self.pk, L)
             msgpack_pack_raw_body(&self.pk, rawval, L)
-        elif PyUnicode_CheckExact(o) if strict_types else PyUnicode_Check(o):
+        elif PyUnicode_CheckExact(o) if strict else PyUnicode_Check(o):
             if self.unicode_errors == NULL:
                 rawval = PyUnicode_AsUTF8AndSize(o, &L)
                 if L >ITEM_LIMIT:
@@ -196,7 +203,7 @@ cdef class Packer:
                 rawval = o
             msgpack_pack_raw(&self.pk, L)
             msgpack_pack_raw_body(&self.pk, rawval, L)
-        elif PyDict_CheckExact(o) if strict_types else PyDict_Check(o):
+        elif PyDict_CheckExact(o) if strict else PyDict_Check(o):
             L = len(o)
             if L > ITEM_LIMIT:
                 raise ValueError("dict is too large")
@@ -204,7 +211,7 @@ cdef class Packer:
             for k, v in o.items():
                 self._pack(k, nest_limit)
                 self._pack(v, nest_limit)
-        elif type(o) is ExtType if strict_types else isinstance(o, ExtType):
+        elif type(o) is ExtType if strict else isinstance(o, ExtType):
             # This should be before Tuple because ExtType is namedtuple.
             rawval = o.data
             L = len(o.data)
@@ -216,7 +223,7 @@ cdef class Packer:
             llval = o.seconds
             ulval = o.nanoseconds
             msgpack_pack_timestamp(&self.pk, llval, ulval)
-        elif PyList_CheckExact(o) if strict_types else (PyTuple_Check(o) or PyList_Check(o)):
+        elif PyList_CheckExact(o) if strict else (PyTuple_Check(o) or PyList_Check(o)):
             L = Py_SIZE(o)
             if L > ITEM_LIMIT:
                 raise ValueError("list is too large")
@@ -264,6 +271,7 @@ cdef class Packer:
 
     def pack(self, object obj):
         cdef int ret
+        self._check_exports()
         try:
             ret = self._pack(obj, DEFAULT_RECURSE_LIMIT)
         except:
@@ -277,12 +285,16 @@ cdef class Packer:
             return buf
 
     def pack_ext_type(self, typecode, data):
+        self._check_exports()
+        if len(data) > ITEM_LIMIT:
+            raise ValueError("ext data too large")
         msgpack_pack_ext(&self.pk, typecode, len(data))
         msgpack_pack_raw_body(&self.pk, data, len(data))
 
     def pack_array_header(self, long long size):
+        self._check_exports()
         if size > ITEM_LIMIT:
-            raise ValueError
+            raise ValueError("array too large")
         msgpack_pack_array(&self.pk, size)
         if self.autoreset:
             buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
@@ -290,8 +302,9 @@ cdef class Packer:
             return buf
 
     def pack_map_header(self, long long size):
+        self._check_exports()
         if size > ITEM_LIMIT:
-            raise ValueError
+            raise ValueError("map too learge")
         msgpack_pack_map(&self.pk, size)
         if self.autoreset:
             buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
@@ -305,7 +318,11 @@ cdef class Packer:
         *pairs* should be a sequence of pairs.
         (`len(pairs)` and `for k, v in pairs:` should be supported.)
         """
-        msgpack_pack_map(&self.pk, len(pairs))
+        self._check_exports()
+        size = len(pairs)
+        if size > ITEM_LIMIT:
+            raise ValueError("map too large")
+        msgpack_pack_map(&self.pk, size)
         for k, v in pairs:
             self._pack(k)
             self._pack(v)
@@ -319,6 +336,7 @@ cdef class Packer:
 
         This method is useful only when autoreset=False.
         """
+        self._check_exports()
         self.pk.length = 0
 
     def bytes(self):
@@ -326,11 +344,15 @@ cdef class Packer:
         return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
 
     def getbuffer(self):
-        """Return view of internal buffer."""
+        """Return memoryview of internal buffer.
+
+        Note: Packer now supports buffer protocol. You can use memoryview(packer).
+        """
         return memoryview(self)
 
     def __getbuffer__(self, Py_buffer *buffer, int flags):
         PyBuffer_FillInfo(buffer, self, self.pk.buf, self.pk.length, 1, flags)
+        self.exports += 1
 
     def __releasebuffer__(self, Py_buffer *buffer):
-        pass
+        self.exports -= 1
diff --git a/test/test_buffer.py b/test/test_buffer.py
index a3db339..2165eb5 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -1,6 +1,6 @@
-#!/usr/bin/env python
+from pytest import raises
 
-from msgpack import packb, unpackb
+from msgpack import packb, unpackb, Packer
 
 
 def test_unpack_buffer():
@@ -27,3 +27,23 @@ def test_unpack_memoryview():
     assert [b"foo", b"bar"] == obj
     expected_type = bytes
     assert all(type(s) == expected_type for s in obj)
+
+
+def test_packer_getbuffer():
+    packer = Packer(autoreset=False)
+    packer.pack_array_header(2)
+    packer.pack(42)
+    packer.pack("hello")
+    buffer = packer.getbuffer()
+    assert isinstance(buffer, memoryview)
+    assert bytes(buffer) == b"\x92*\xa5hello"
+
+    if Packer.__module__ == "msgpack._cmsgpack":  # only for Cython
+        # cython Packer supports buffer protocol directly
+        assert bytes(packer) == b"\x92*\xa5hello"
+
+        with raises(BufferError):
+            packer.pack(42)
+        buffer.release()
+        packer.pack(42)
+        assert bytes(packer) == b"\x92*\xa5hello*"

From 33e0e86f4e66e6bb029f118e8f5e166b55e828f7 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 6 May 2024 11:46:31 +0900
Subject: [PATCH 1626/1648] Cleanup code and pyproject (#608)

* use isort
* fallback: use BytesIO instead of StringIO. We had dropped Python 2
already.
---
 Makefile                |  4 +--
 msgpack/__init__.py     | 12 ++++-----
 msgpack/ext.py          |  2 +-
 msgpack/fallback.py     | 54 ++++++++++++++++++-----------------------
 pyproject.toml          | 17 +++++--------
 setup.py                |  4 +--
 test/test_buffer.py     |  2 +-
 test/test_except.py     |  7 +++---
 test/test_extension.py  |  1 +
 test/test_limits.py     |  8 +++---
 test/test_memoryview.py |  1 +
 test/test_newspec.py    |  2 +-
 test/test_obj.py        |  1 +
 test/test_pack.py       |  4 +--
 test/test_read_size.py  |  2 +-
 test/test_seq.py        |  6 ++---
 test/test_sequnpack.py  |  7 +++---
 test/test_stricttype.py |  3 ++-
 test/test_subtype.py    |  3 ++-
 test/test_timestamp.py  |  4 ++-
 test/test_unpack.py     | 11 +++------
 21 files changed, 75 insertions(+), 80 deletions(-)

diff --git a/Makefile b/Makefile
index 3ce178f..51f3e0e 100644
--- a/Makefile
+++ b/Makefile
@@ -6,11 +6,11 @@ all: cython
 
 .PHONY: format
 format:
-	pipx run ruff format $(PYTHON_SOURCES)
+	ruff format $(PYTHON_SOURCES)
 
 .PHONY: lint
 lint:
-	pipx run ruff check $(PYTHON_SOURCES)
+	ruff check $(PYTHON_SOURCES)
 
 .PHONY: doc
 doc:
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index 919b86f..e796efb 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -1,20 +1,20 @@
-from .exceptions import *
-from .ext import ExtType, Timestamp
-
+# ruff: noqa: F401
 import os
 
+from .exceptions import *  # noqa: F403
+from .ext import ExtType, Timestamp
 
 version = (1, 0, 8)
 __version__ = "1.0.8"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):
-    from .fallback import Packer, unpackb, Unpacker
+    from .fallback import Packer, Unpacker, unpackb
 else:
     try:
-        from ._cmsgpack import Packer, unpackb, Unpacker
+        from ._cmsgpack import Packer, Unpacker, unpackb
     except ImportError:
-        from .fallback import Packer, unpackb, Unpacker
+        from .fallback import Packer, Unpacker, unpackb
 
 
 def pack(o, stream, **kwargs):
diff --git a/msgpack/ext.py b/msgpack/ext.py
index 3940fe0..9694819 100644
--- a/msgpack/ext.py
+++ b/msgpack/ext.py
@@ -1,6 +1,6 @@
-from collections import namedtuple
 import datetime
 import struct
+from collections import namedtuple
 
 
 class ExtType(namedtuple("ExtType", "code data")):
diff --git a/msgpack/fallback.py b/msgpack/fallback.py
index cbf0d30..b02e47c 100644
--- a/msgpack/fallback.py
+++ b/msgpack/fallback.py
@@ -1,28 +1,22 @@
 """Fallback pure Python implementation of msgpack"""
 
-from datetime import datetime as _DateTime
-import sys
 import struct
-
+import sys
+from datetime import datetime as _DateTime
 
 if hasattr(sys, "pypy_version_info"):
-    # StringIO is slow on PyPy, StringIO is faster.  However: PyPy's own
-    # StringBuilder is fastest.
     from __pypy__ import newlist_hint
+    from __pypy__.builders import BytesBuilder
 
-    try:
-        from __pypy__.builders import BytesBuilder as StringBuilder
-    except ImportError:
-        from __pypy__.builders import StringBuilder
-    USING_STRINGBUILDER = True
+    _USING_STRINGBUILDER = True
 
-    class StringIO:
+    class BytesIO:
         def __init__(self, s=b""):
             if s:
-                self.builder = StringBuilder(len(s))
+                self.builder = BytesBuilder(len(s))
                 self.builder.append(s)
             else:
-                self.builder = StringBuilder()
+                self.builder = BytesBuilder()
 
         def write(self, s):
             if isinstance(s, memoryview):
@@ -35,17 +29,17 @@ if hasattr(sys, "pypy_version_info"):
             return self.builder.build()
 
 else:
-    USING_STRINGBUILDER = False
-    from io import BytesIO as StringIO
+    from io import BytesIO
 
-    newlist_hint = lambda size: []
+    _USING_STRINGBUILDER = False
+
+    def newlist_hint(size):
+        return []
 
 
-from .exceptions import BufferFull, OutOfData, ExtraData, FormatError, StackError
-
+from .exceptions import BufferFull, ExtraData, FormatError, OutOfData, StackError
 from .ext import ExtType, Timestamp
 
-
 EX_SKIP = 0
 EX_CONSTRUCT = 1
 EX_READ_ARRAY_HEADER = 2
@@ -335,6 +329,7 @@ class Unpacker:
 
         # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython
         self._buffer.extend(view)
+        view.release()
 
     def _consume(self):
         """Gets rid of the used parts of the buffer."""
@@ -671,12 +666,11 @@ class Packer:
         self._use_float = use_single_float
         self._autoreset = autoreset
         self._use_bin_type = use_bin_type
-        self._buffer = StringIO()
+        self._buffer = BytesIO()
         self._datetime = bool(datetime)
         self._unicode_errors = unicode_errors or "strict"
-        if default is not None:
-            if not callable(default):
-                raise TypeError("default must be callable")
+        if default is not None and not callable(default):
+            raise TypeError("default must be callable")
         self._default = default
 
     def _pack(
@@ -807,18 +801,18 @@ class Packer:
         try:
             self._pack(obj)
         except:
-            self._buffer = StringIO()  # force reset
+            self._buffer = BytesIO()  # force reset
             raise
         if self._autoreset:
             ret = self._buffer.getvalue()
-            self._buffer = StringIO()
+            self._buffer = BytesIO()
             return ret
 
     def pack_map_pairs(self, pairs):
         self._pack_map_pairs(len(pairs), pairs)
         if self._autoreset:
             ret = self._buffer.getvalue()
-            self._buffer = StringIO()
+            self._buffer = BytesIO()
             return ret
 
     def pack_array_header(self, n):
@@ -827,7 +821,7 @@ class Packer:
         self._pack_array_header(n)
         if self._autoreset:
             ret = self._buffer.getvalue()
-            self._buffer = StringIO()
+            self._buffer = BytesIO()
             return ret
 
     def pack_map_header(self, n):
@@ -836,7 +830,7 @@ class Packer:
         self._pack_map_header(n)
         if self._autoreset:
             ret = self._buffer.getvalue()
-            self._buffer = StringIO()
+            self._buffer = BytesIO()
             return ret
 
     def pack_ext_type(self, typecode, data):
@@ -925,11 +919,11 @@ class Packer:
 
         This method is useful only when autoreset=False.
         """
-        self._buffer = StringIO()
+        self._buffer = BytesIO()
 
     def getbuffer(self):
         """Return view of internal buffer."""
-        if USING_STRINGBUILDER:
+        if _USING_STRINGBUILDER:
             return memoryview(self.bytes())
         else:
             return self._buffer.getbuffer()
diff --git a/pyproject.toml b/pyproject.toml
index 6254f06..f36c7f4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,17 +45,12 @@ include-package-data = false
 [tool.setuptools.dynamic]
 version = {attr = "msgpack.__version__"}
 
-[tool.black]
-line-length = 100
-target-version = ["py37"]
-skip_string_normalization = true
-
 [tool.ruff]
 line-length = 100
 target-version = "py38"
-lint.ignore = []
-
-[tool.ruff.lint.per-file-ignores]
-"msgpack/__init__.py" = ["F401", "F403"]
-"msgpack/fallback.py" = ["E731"]
-"test/test_seq.py" = ["E501"]
+lint.select = [
+    "E", # pycodestyle
+    "F", # Pyflakes
+    "I", # isort
+    #"UP", pyupgrade
+]
diff --git a/setup.py b/setup.py
index dc14a26..eaca746 100644
--- a/setup.py
+++ b/setup.py
@@ -1,11 +1,11 @@
 #!/usr/bin/env python
 import os
 import sys
-from setuptools import setup, Extension
+
+from setuptools import Extension, setup
 from setuptools.command.build_ext import build_ext
 from setuptools.command.sdist import sdist
 
-
 PYPY = hasattr(sys, "pypy_version_info")
 
 
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 2165eb5..2c5a14c 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -1,6 +1,6 @@
 from pytest import raises
 
-from msgpack import packb, unpackb, Packer
+from msgpack import Packer, packb, unpackb
 
 
 def test_unpack_buffer():
diff --git a/test/test_except.py b/test/test_except.py
index 8c0a976..b77ac80 100644
--- a/test/test_except.py
+++ b/test/test_except.py
@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 
-from pytest import raises
-from msgpack import packb, unpackb, Unpacker, FormatError, StackError, OutOfData
-
 import datetime
 
+from pytest import raises
+
+from msgpack import FormatError, OutOfData, StackError, Unpacker, packb, unpackb
+
 
 class DummyException(Exception):
     pass
diff --git a/test/test_extension.py b/test/test_extension.py
index 9e5e6aa..aaf0fd9 100644
--- a/test/test_extension.py
+++ b/test/test_extension.py
@@ -1,4 +1,5 @@
 import array
+
 import msgpack
 from msgpack import ExtType
 
diff --git a/test/test_limits.py b/test/test_limits.py
index 533bc11..9b92b4d 100644
--- a/test/test_limits.py
+++ b/test/test_limits.py
@@ -2,14 +2,14 @@
 import pytest
 
 from msgpack import (
-    packb,
-    unpackb,
-    Packer,
-    Unpacker,
     ExtType,
+    Packer,
     PackOverflowError,
     PackValueError,
+    Unpacker,
     UnpackValueError,
+    packb,
+    unpackb,
 )
 
 
diff --git a/test/test_memoryview.py b/test/test_memoryview.py
index eff4bca..0a2a6f5 100644
--- a/test/test_memoryview.py
+++ b/test/test_memoryview.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 from array import array
+
 from msgpack import packb, unpackb
 
 
diff --git a/test/test_newspec.py b/test/test_newspec.py
index a6f4251..9e2f9be 100644
--- a/test/test_newspec.py
+++ b/test/test_newspec.py
@@ -1,4 +1,4 @@
-from msgpack import packb, unpackb, ExtType
+from msgpack import ExtType, packb, unpackb
 
 
 def test_str8():
diff --git a/test/test_obj.py b/test/test_obj.py
index f78bf42..23be06d 100644
--- a/test/test_obj.py
+++ b/test/test_obj.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 from pytest import raises
+
 from msgpack import packb, unpackb
 
 
diff --git a/test/test_pack.py b/test/test_pack.py
index 4a0ef40..374d154 100644
--- a/test/test_pack.py
+++ b/test/test_pack.py
@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 
+import struct
 from collections import OrderedDict
 from io import BytesIO
-import struct
 
 import pytest
 
-from msgpack import packb, unpackb, Unpacker, Packer
+from msgpack import Packer, Unpacker, packb, unpackb
 
 
 def check(data, use_list=False):
diff --git a/test/test_read_size.py b/test/test_read_size.py
index a7d61fd..0f6c1b5 100644
--- a/test/test_read_size.py
+++ b/test/test_read_size.py
@@ -1,6 +1,6 @@
 """Test Unpacker's read_array_header and read_map_header methods"""
 
-from msgpack import packb, Unpacker, OutOfData
+from msgpack import OutOfData, Unpacker, packb
 
 UnexpectedTypeException = ValueError
 
diff --git a/test/test_seq.py b/test/test_seq.py
index 16d9dde..8dee462 100644
--- a/test/test_seq.py
+++ b/test/test_seq.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python
-
+# ruff: noqa: E501
+# ignore line length limit for long comments
 import io
-import msgpack
 
+import msgpack
 
 binarydata = bytes(bytearray(range(256)))
 
diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py
index 6b138aa..0f895d7 100644
--- a/test/test_sequnpack.py
+++ b/test/test_sequnpack.py
@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 import io
-from msgpack import Unpacker, BufferFull
-from msgpack import pack, packb
-from msgpack.exceptions import OutOfData
+
 from pytest import raises
 
+from msgpack import BufferFull, Unpacker, pack, packb
+from msgpack.exceptions import OutOfData
+
 
 def test_partialdata():
     unpacker = Unpacker()
diff --git a/test/test_stricttype.py b/test/test_stricttype.py
index 9ffaff2..72776a2 100644
--- a/test/test_stricttype.py
+++ b/test/test_stricttype.py
@@ -1,5 +1,6 @@
 from collections import namedtuple
-from msgpack import packb, unpackb, ExtType
+
+from msgpack import ExtType, packb, unpackb
 
 
 def test_namedtuple():
diff --git a/test/test_subtype.py b/test/test_subtype.py
index 0d1c41a..a911578 100644
--- a/test/test_subtype.py
+++ b/test/test_subtype.py
@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 
-from msgpack import packb
 from collections import namedtuple
 
+from msgpack import packb
+
 
 class MyList(list):
     pass
diff --git a/test/test_timestamp.py b/test/test_timestamp.py
index f9bc835..831141a 100644
--- a/test/test_timestamp.py
+++ b/test/test_timestamp.py
@@ -1,5 +1,7 @@
-import pytest
 import datetime
+
+import pytest
+
 import msgpack
 from msgpack.ext import Timestamp
 
diff --git a/test/test_unpack.py b/test/test_unpack.py
index bf3f960..b17c3c5 100644
--- a/test/test_unpack.py
+++ b/test/test_unpack.py
@@ -1,12 +1,9 @@
-from io import BytesIO
 import sys
-from msgpack import Unpacker, packb, OutOfData, ExtType
-from pytest import raises, mark
+from io import BytesIO
 
-try:
-    from itertools import izip as zip
-except ImportError:
-    pass
+from pytest import mark, raises
+
+from msgpack import ExtType, OutOfData, Unpacker, packb
 
 
 def test_unpack_array_header_from_file():

From 9cea8b6da23ce66f0e78f017c96adcc447deb09a Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 7 May 2024 20:49:23 +0900
Subject: [PATCH 1627/1648] Release v1.1.0rc1 (#609)

---
 ChangeLog.rst       | 20 +++++++++++++++++++-
 msgpack/__init__.py |  4 ++--
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 2408bc9..9b16e41 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,21 @@
+1.1.0rc1
+========
+
+Release Date: 2024-05-07
+
+* Update Cython to 3.0.10 to reduce C warnings and future support for Python 3.13.
+* Stop using C++ mode in Cython to reduce compile error on some compilers.
+* ``Packer()`` has ``buf_size`` option to specify initial size of
+  internal buffer to reduce reallocation.
+* The default internal buffer size of ``Packer()`` is reduced from
+  1MiB to 256KiB to optimize for common use cases. Use ``buf_size``
+  if you are packing large data.
+* ``Timestamp.to_datetime()`` and ``Timestamp.from_datetime()`` become
+  more accurate by avoiding floating point calculations. (#591)
+* The Cython code for ``Unpacker`` has been slightly rewritten for maintainability.
+* The fallback implementation of ``Packer()`` and ``Unpacker()`` now uses keyword-only
+  arguments to improve compatibility with the Cython implementation.
+
 1.0.8
 =====
 
@@ -130,7 +148,7 @@ Important changes
 * unpacker: Default value of input limits are smaller than before to avoid DoS attack.
   If you need to handle large data, you need to specify limits manually. (#319)
 
-* Unpacker doesn't wrap underlaying ``ValueError`` (including ``UnicodeError``) into
+* Unpacker doesn't wrap underlying ``ValueError`` (including ``UnicodeError``) into
   ``UnpackValueError``.  If you want to catch all exception during unpack, you need
   to use ``try ... except Exception`` with minimum try code block. (#323, #233)
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index e796efb..c252544 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -4,8 +4,8 @@ import os
 from .exceptions import *  # noqa: F403
 from .ext import ExtType, Timestamp
 
-version = (1, 0, 8)
-__version__ = "1.0.8"
+version = (1, 1, 0, "rc1")
+__version__ = "1.1.0rc1"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):

From 0b1c47b06b55d91c00c9f7153c4a9440ea878886 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 7 May 2024 22:01:54 +0900
Subject: [PATCH 1628/1648] do not install cython as build dependency (#610)

User can not cythonize during `pip install msgpack`.
So remove cython from build dependency.

If user need to use another Cython, user should download sdist, unzip,
manually cythonize, and `pip install .`.
---
 pyproject.toml |  6 +-----
 setup.py       | 45 ---------------------------------------------
 2 files changed, 1 insertion(+), 50 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index f36c7f4..d041d4c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,9 +1,5 @@
 [build-system]
-requires = [
-    # Also declared in requirements.txt, if updating here please also update there
-    "Cython~=3.0.10",
-    "setuptools >= 69.5.1",
-]
+requires = ["setuptools >= 69.5.1"]
 build-backend = "setuptools.build_meta"
 
 [project]
diff --git a/setup.py b/setup.py
index eaca746..4029e9e 100644
--- a/setup.py
+++ b/setup.py
@@ -3,52 +3,9 @@ import os
 import sys
 
 from setuptools import Extension, setup
-from setuptools.command.build_ext import build_ext
-from setuptools.command.sdist import sdist
 
 PYPY = hasattr(sys, "pypy_version_info")
 
-
-class NoCython(Exception):
-    pass
-
-
-try:
-    import Cython.Compiler.Main as cython_compiler
-
-    have_cython = True
-except ImportError:
-    have_cython = False
-
-
-def cythonize(src):
-    if not have_cython:
-        raise Exception("Cython is required for building from checkout")
-    sys.stderr.write(f"cythonize: {src!r}\n")
-    cython_compiler.compile([src])
-
-
-def ensure_source(src):
-    pyx = os.path.splitext(src)[0] + ".pyx"
-
-    if not os.path.exists(src) or have_cython and os.stat(src).st_mtime < os.stat(pyx).st_mtime:
-        cythonize(pyx)
-
-
-class BuildExt(build_ext):
-    def build_extension(self, ext):
-        for src in ext.sources:
-            ensure_source(src)
-        return build_ext.build_extension(self, ext)
-
-
-# Cython is required for sdist
-class Sdist(sdist):
-    def __init__(self, *args, **kwargs):
-        cythonize("msgpack/_cmsgpack.pyx")
-        sdist.__init__(self, *args, **kwargs)
-
-
 libraries = []
 macros = []
 ext_modules = []
@@ -69,9 +26,7 @@ if not PYPY and not os.environ.get("MSGPACK_PUREPYTHON"):
     )
 del libraries, macros
 
-
 setup(
-    cmdclass={"build_ext": BuildExt, "sdist": Sdist},
     ext_modules=ext_modules,
     packages=["msgpack"],
 )

From 6e11368f5d54f7d4878dc209717495c37be03c68 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 19 Aug 2024 17:35:16 +0900
Subject: [PATCH 1629/1648] update Cython to 3.0.11 (#617)

---
 requirements.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 1164a94..b677f06 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1 @@
-# Also declared in pyproject.toml, if updating here please also update there.
-Cython~=3.0.10
+Cython~=3.0.11

From 9e26d80ab2a02221d3ca36cc0b5ca2268f391204 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 19 Aug 2024 17:56:01 +0900
Subject: [PATCH 1630/1648] update cibuildwheel to 2.20.0 (#618)

---
 .github/workflows/wheel.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index d57e058..01d0bbd 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -32,7 +32,7 @@ jobs:
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v2.17.0
+        uses: pypa/cibuildwheel@v2.20.0
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
@@ -40,13 +40,11 @@ jobs:
           CIBW_ARCHS_MACOS: x86_64 universal2 arm64
           CIBW_SKIP: pp*
 
-      - name: Build pure Python wheel
+      - name: Build sdist
         if: runner.os == 'Linux'
-        env:
-          MSGPACK_PUREPYTHON: "1"
         run: |
           pip install build
-          python -m build -w -o wheelhouse
+          python -m build -s -o wheelhouse
 
       - name: Upload Wheels to artifact
         uses: actions/upload-artifact@v4

From 9d0c7f2f9cab21c3e39d23001cd9d2034824ee61 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 19 Aug 2024 20:36:26 +0900
Subject: [PATCH 1631/1648] Release v1.1.0rc2 (#619)

---
 .github/workflows/wheel.yml | 2 +-
 ChangeLog.rst               | 8 ++++++++
 msgpack/__init__.py         | 4 ++--
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 01d0bbd..5015722 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -38,7 +38,7 @@ jobs:
           CIBW_TEST_COMMAND: "pytest {package}/test"
           CIBW_ARCHS_LINUX: auto aarch64
           CIBW_ARCHS_MACOS: x86_64 universal2 arm64
-          CIBW_SKIP: pp*
+          CIBW_SKIP: "pp* cp38-macosx_*"
 
       - name: Build sdist
         if: runner.os == 'Linux'
diff --git a/ChangeLog.rst b/ChangeLog.rst
index 9b16e41..47328ff 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,11 @@
+1.1.0rc2
+========
+
+Release Date: 2024-08-19
+
+* Update Cython to 3.0.11 for better Python 3.13 support.
+* Update cibuildwheel to 2.20.0 to build Python 3.13 wheels.
+
 1.1.0rc1
 ========
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index c252544..a72e974 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -4,8 +4,8 @@ import os
 from .exceptions import *  # noqa: F403
 from .ext import ExtType, Timestamp
 
-version = (1, 1, 0, "rc1")
-__version__ = "1.1.0rc1"
+version = (1, 1, 0, "rc2")
+__version__ = "1.1.0rc2"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):

From 20a2b8eaa26d76169049ce150e10d1d2aa37d3ab Mon Sep 17 00:00:00 2001
From: Thomas A Caswell 
Date: Wed, 21 Aug 2024 01:56:00 -0400
Subject: [PATCH 1632/1648] use `PyLong_*` instead of `PyInt_*` (#620)

9af421163cb8081414be347038dee7a82b29e8dd in Cython removed back-compatibility `#define`.
---
 msgpack/unpack.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/msgpack/unpack.h b/msgpack/unpack.h
index 23aa622..58a2f4f 100644
--- a/msgpack/unpack.h
+++ b/msgpack/unpack.h
@@ -47,7 +47,7 @@ static inline msgpack_unpack_object unpack_callback_root(unpack_user* u)
 
 static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o)
 {
-    PyObject *p = PyInt_FromLong((long)d);
+    PyObject *p = PyLong_FromLong((long)d);
     if (!p)
         return -1;
     *o = p;
@@ -61,7 +61,7 @@ static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpac
 
 static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o)
 {
-    PyObject *p = PyInt_FromSize_t((size_t)d);
+    PyObject *p = PyLong_FromSize_t((size_t)d);
     if (!p)
         return -1;
     *o = p;
@@ -74,7 +74,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp
     if (d > LONG_MAX) {
         p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d);
     } else {
-        p = PyInt_FromLong((long)d);
+        p = PyLong_FromLong((long)d);
     }
     if (!p)
         return -1;
@@ -84,7 +84,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp
 
 static inline int unpack_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o)
 {
-    PyObject *p = PyInt_FromLong(d);
+    PyObject *p = PyLong_FromLong(d);
     if (!p)
         return -1;
     *o = p;
@@ -107,7 +107,7 @@ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpac
     if (d > LONG_MAX || d < LONG_MIN) {
         p = PyLong_FromLongLong((PY_LONG_LONG)d);
     } else {
-        p = PyInt_FromLong((long)d);
+        p = PyLong_FromLong((long)d);
     }
     *o = p;
     return 0;

From 4587393b1ae2c9ebbd5bc93005b8aea2c8050b27 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Tue, 10 Sep 2024 01:58:00 +0900
Subject: [PATCH 1633/1648] release v1.1.0 (#622)

---
 ChangeLog.rst       | 8 ++++++++
 msgpack/__init__.py | 4 ++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 47328ff..863c6b2 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,11 @@
+1.1.0
+=====
+
+Release Date: 2024-09-10
+
+* use ``PyLong_*`` instead of ``PyInt_*`` for compatibility with
+  future Cython. (#620)
+
 1.1.0rc2
 ========
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index a72e974..b615105 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -4,8 +4,8 @@ import os
 from .exceptions import *  # noqa: F403
 from .ext import ExtType, Timestamp
 
-version = (1, 1, 0, "rc2")
-__version__ = "1.1.0rc2"
+version = (1, 1, 0)
+__version__ = "1.1.0"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):

From 0eeabfb453844b441a4a77097b3d5aa0cb6645b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?=
 <16805946+edgarrmondragon@users.noreply.github.com>
Date: Tue, 8 Oct 2024 03:04:56 -0600
Subject: [PATCH 1634/1648] Add Python 3.13 trove classifier (#626)

---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index d041d4c..e24f2b8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -21,6 +21,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
     "Intended Audience :: Developers",

From 868aa2cd83f39237deb957c68ce7232422a5950b Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Sat, 31 May 2025 12:45:06 +0900
Subject: [PATCH 1635/1648] update Cython to 3.1.1 (#637)

---
 .github/workflows/test.yml |  9 +++++++--
 pyproject.toml             | 12 +++++++-----
 requirements.txt           |  2 +-
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 530238c..23d221c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "windows-latest", "macos-latest"]
-        py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9", "3.8"]
+        py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"]
 
     runs-on: ${{ matrix.os }}
     name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}
@@ -26,10 +26,15 @@ jobs:
           allow-prereleases: true
           cache: "pip"
 
+      - name: Prepare
+        shell: bash
+        run: |
+          pip install -U pip
+          pip install -r requirements.txt pytest
+
       - name: Build
         shell: bash
         run: |
-          pip install -r requirements.txt pytest
           make cython
           pip install .
 
diff --git a/pyproject.toml b/pyproject.toml
index e24f2b8..b162832 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,10 +1,13 @@
 [build-system]
-requires = ["setuptools >= 69.5.1"]
+# 75.3.0 is the latest version supporting Python 3.8
+requires = ["setuptools >= 75.3.0"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "msgpack"
 dynamic = ["version"]
+# `license = "Apache-2.0"` is preferred. But keep old syntax for Python 3.8 compatibility.
+# https://github.com/msgpack/msgpack-python/pull/637
 license = {text="Apache 2.0"}
 authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}]
 description = "MessagePack serializer"
@@ -14,18 +17,17 @@ requires-python = ">=3.8"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Operating System :: OS Independent",
-    "Programming Language :: Python",
-    "Programming Language :: Python :: 3",
+    "Topic :: File Formats",
+    "Intended Audience :: Developers",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
-    "Intended Audience :: Developers",
-    "License :: OSI Approved :: Apache Software License",
 ]
 
 [project.urls]
diff --git a/requirements.txt b/requirements.txt
index b677f06..78a2f38 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1 @@
-Cython~=3.0.11
+Cython~=3.1.1

From cdc764450370ff80e7c83edbe8d015f08f6fb9b3 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Sun, 1 Jun 2025 16:56:44 +0900
Subject: [PATCH 1636/1648] update cibuildwheel to v2.23.3 (#638)

---
 .github/workflows/wheel.yml | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 5015722..d97de1d 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -9,17 +9,12 @@ jobs:
   build_wheels:
     strategy:
       matrix:
-        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+        # macos-13 is for intel
+        os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "macos-13", "macos-latest"]
     runs-on: ${{ matrix.os }}
     name: Build wheels on ${{ matrix.os }}
 
     steps:
-      - name: Set up QEMU
-        if: runner.os == 'Linux'
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: all
-
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
@@ -32,12 +27,10 @@ jobs:
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v2.20.0
+        uses: pypa/cibuildwheel@v2.23.3
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
-          CIBW_ARCHS_LINUX: auto aarch64
-          CIBW_ARCHS_MACOS: x86_64 universal2 arm64
           CIBW_SKIP: "pp* cp38-macosx_*"
 
       - name: Build sdist

From fe9e620a607702b31476f092ad01a387cff4cfbd Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 2 Jun 2025 14:46:53 +0900
Subject: [PATCH 1637/1648] upload to PyPI on create a release (#639)

---
 .github/workflows/wheel.yml | 48 ++++++++++++++++++++++++++++++++++---
 1 file changed, 45 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index d97de1d..686d7dd 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -1,8 +1,10 @@
-name: Build Wheels
+name: Build sdist and Wheels
 on:
   push:
     branches: [main]
-  create:
+  release:
+    types:
+      - published
   workflow_dispatch:
 
 jobs:
@@ -34,7 +36,7 @@ jobs:
           CIBW_SKIP: "pp* cp38-macosx_*"
 
       - name: Build sdist
-        if: runner.os == 'Linux'
+        if: runner.os == 'Linux' && runner.arch == 'X64'
         run: |
           pip install build
           python -m build -s -o wheelhouse
@@ -44,3 +46,43 @@ jobs:
         with:
           name: wheels-${{ matrix.os }}
           path: wheelhouse
+
+  # combine all wheels into one artifact
+  combine_wheels:
+    needs: [build_wheels]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          # unpacks all CIBW artifacts into dist/
+          pattern: wheels-*
+          path: dist
+          merge-multiple: true
+
+      - name: Upload Wheels to artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-all
+          path: dist
+
+  # https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml
+  upload_pypi:
+    needs: [build_wheels]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    if: github.event_name == 'release' && github.event.action == 'published'
+    # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this)
+    # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          # unpacks all CIBW artifacts into dist/
+          pattern: wheels-*
+          path: dist
+          merge-multiple: true
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        #with:
+        # To test: repository-url: https://test.pypi.org/legacy/

From e6445d3b922ca0b9bc82695dd9d1c1529763095a Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 6 Jun 2025 09:38:38 +0900
Subject: [PATCH 1638/1648] v1.1.1rc1

---
 ChangeLog.rst       | 7 +++++++
 docs/conf.py        | 6 +++---
 msgpack/__init__.py | 4 ++--
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 863c6b2..0f67502 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,10 @@
+1.1.1rc1
+========
+
+Release Date: 2025-06-06
+
+* Update Cython to 3.1.1 and cibuildwheel to 2.23.3.
+
 1.1.0
 =====
 
diff --git a/docs/conf.py b/docs/conf.py
index ab0ad3c..28116cd 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -12,9 +12,9 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#import os
-#import sys
-#sys.path.insert(0, os.path.abspath('..'))
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('..'))
 
 # -- General configuration -----------------------------------------------------
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index b615105..b8d83da 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -4,8 +4,8 @@ import os
 from .exceptions import *  # noqa: F403
 from .ext import ExtType, Timestamp
 
-version = (1, 1, 0)
-__version__ = "1.1.0"
+version = (1, 1, 1, "rc1")
+__version__ = "1.1.1rc1"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):

From 42f056f3cfaf2e3ec220db2f864e7613d433ad48 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Fri, 13 Jun 2025 15:41:08 +0900
Subject: [PATCH 1639/1648] v1.1.1

---
 ChangeLog.rst       | 7 +++++++
 msgpack/__init__.py | 4 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/ChangeLog.rst b/ChangeLog.rst
index 0f67502..418c444 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,10 @@
+1.1.1
+=====
+
+Release Date: 2025-06-13
+
+* No change from 1.1.1rc1.
+
 1.1.1rc1
 ========
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index b8d83da..ad68271 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -4,8 +4,8 @@ import os
 from .exceptions import *  # noqa: F403
 from .ext import ExtType, Timestamp
 
-version = (1, 1, 1, "rc1")
-__version__ = "1.1.1rc1"
+version = (1, 1, 1)
+__version__ = "1.1.1"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):

From d9873dab049c60d5fd553223b9be10f1dcb56929 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Sat, 26 Jul 2025 10:59:05 +0900
Subject: [PATCH 1640/1648] ci: update cibuildwheel and drop Python 3.8 (#642)

---
 .github/workflows/test.yml  | 2 +-
 .github/workflows/wheel.yml | 4 ++--
 requirements.txt            | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 23d221c..6089479 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "windows-latest", "macos-latest"]
-        py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"]
+        py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9"]
 
     runs-on: ${{ matrix.os }}
     name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 686d7dd..1260036 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -29,11 +29,11 @@ jobs:
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v2.23.3
+        uses: pypa/cibuildwheel@v3.1.1
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
-          CIBW_SKIP: "pp* cp38-macosx_*"
+          CIBW_SKIP: "pp* cp38-*"
 
       - name: Build sdist
         if: runner.os == 'Linux' && runner.arch == 'X64'
diff --git a/requirements.txt b/requirements.txt
index 78a2f38..5d2e20b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1 @@
-Cython~=3.1.1
+Cython~=3.1.2

From c2a9f1fda5401de2193c43d7a4f93bfc0b2fc1e6 Mon Sep 17 00:00:00 2001
From: MS-GITS <137760120+Greenie0701@users.noreply.github.com>
Date: Fri, 26 Sep 2025 09:47:17 +0530
Subject: [PATCH 1641/1648] ci: add support for building windows on arm wheels
 (#643)

---
 .github/workflows/test.yml  | 12 ++++++++----
 .github/workflows/wheel.yml |  4 ++--
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6089479..26b4ce6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -9,9 +9,13 @@ jobs:
   test:
     strategy:
       matrix:
-        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+        os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"]
         py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9"]
-
+        exclude:
+          - os: windows-11-arm
+            py: "3.9"
+          - os: windows-11-arm
+            py: "3.10"
     runs-on: ${{ matrix.os }}
     name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}
 
@@ -29,8 +33,8 @@ jobs:
       - name: Prepare
         shell: bash
         run: |
-          pip install -U pip
-          pip install -r requirements.txt pytest
+          python -m pip install -U pip
+          python -m pip install -r requirements.txt pytest
 
       - name: Build
         shell: bash
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index 1260036..fb369a1 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         # macos-13 is for intel
-        os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "macos-13", "macos-latest"]
+        os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "windows-11-arm", "macos-13", "macos-latest"]
     runs-on: ${{ matrix.os }}
     name: Build wheels on ${{ matrix.os }}
 
@@ -33,7 +33,7 @@ jobs:
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
-          CIBW_SKIP: "pp* cp38-*"
+          CIBW_SKIP: "pp* cp38-* cp39-win_arm64 cp310-win_arm64"
 
       - name: Build sdist
         if: runner.os == 'Linux' && runner.arch == 'X64'

From 0f3c4be465b5f39ab2d6903797e57ef8a0fd097e Mon Sep 17 00:00:00 2001
From: TW 
Date: Wed, 8 Oct 2025 09:10:46 +0200
Subject: [PATCH 1642/1648] README: fix typos and grammar (#648)

---
 README.md | 64 +++++++++++++++++++++++++++----------------------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/README.md b/README.md
index 61a03c6..1f06324 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
 [![Build Status](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml/badge.svg)](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml)
 [![Documentation Status](https://readthedocs.org/projects/msgpack-python/badge/?version=latest)](https://msgpack-python.readthedocs.io/en/latest/?badge=latest)
 
-## What's this
+## What is this?
 
 [MessagePack](https://msgpack.org/) is an efficient binary serialization format.
 It lets you exchange data among multiple languages like JSON.
@@ -25,9 +25,9 @@ But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy.
 
 ### Windows
 
-When you can't use a binary distribution, you need to install Visual Studio
-or Windows SDK on Windows.
-Without extension, using pure Python implementation on CPython runs slowly.
+If you can't use a binary distribution, you need to install Visual Studio
+or the Windows SDK on Windows.
+Without the extension, the pure Python implementation on CPython runs slowly.
 
 
 ## How to use
@@ -35,11 +35,11 @@ Without extension, using pure Python implementation on CPython runs slowly.
 ### One-shot pack & unpack
 
 Use `packb` for packing and `unpackb` for unpacking.
-msgpack provides `dumps` and `loads` as an alias for compatibility with
+msgpack provides `dumps` and `loads` as aliases for compatibility with
 `json` and `pickle`.
 
-`pack` and `dump` packs to a file-like object.
-`unpack` and `load` unpacks from a file-like object.
+`pack` and `dump` pack to a file-like object.
+`unpack` and `load` unpack from a file-like object.
 
 ```pycon
 >>> import msgpack
@@ -73,7 +73,7 @@ for unpacked in unpacker:
 ```
 
 
-### Packing/unpacking of custom data type
+### Packing/unpacking of custom data types
 
 It is also possible to pack/unpack custom data types. Here is an example for
 `datetime.datetime`.
@@ -140,8 +140,8 @@ True
 ### Advanced unpacking control
 
 As an alternative to iteration, `Unpacker` objects provide `unpack`,
-`skip`, `read_array_header` and `read_map_header` methods. The former two
-read an entire message from the stream, respectively de-serialising and returning
+`skip`, `read_array_header`, and `read_map_header` methods. The former two
+read an entire message from the stream, respectively deserializing and returning
 the result, or ignoring it. The latter two methods return the number of elements
 in the upcoming container, so that each element in an array, or key-value pair
 in a map, can be unpacked or skipped individually.
@@ -149,7 +149,7 @@ in a map, can be unpacked or skipped individually.
 
 ## Notes
 
-### string and binary type in old msgpack spec
+### String and binary types in the old MessagePack spec
 
 Early versions of msgpack didn't distinguish string and binary types.
 The type for representing both string and binary types was named **raw**.
@@ -167,7 +167,7 @@ and `raw=True` options.
 
 ### ext type
 
-To use the **ext** type, pass `msgpack.ExtType` object to packer.
+To use the **ext** type, pass a `msgpack.ExtType` object to the packer.
 
 ```pycon
 >>> import msgpack
@@ -181,26 +181,26 @@ You can use it with `default` and `ext_hook`. See below.
 
 ### Security
 
-To unpacking data received from unreliable source, msgpack provides
+When unpacking data received from an unreliable source, msgpack provides
 two security options.
 
 `max_buffer_size` (default: `100*1024*1024`) limits the internal buffer size.
-It is used to limit the preallocated list size too.
+It is also used to limit preallocated list sizes.
 
 `strict_map_key` (default: `True`) limits the type of map keys to bytes and str.
-While msgpack spec doesn't limit the types of the map keys,
-there is a risk of the hashdos.
+While the MessagePack spec doesn't limit map key types,
+there is a risk of a hash DoS.
 If you need to support other types for map keys, use `strict_map_key=False`.
 
 
 ### Performance tips
 
-CPython's GC starts when growing allocated object.
-This means unpacking may cause useless GC.
-You can use `gc.disable()` when unpacking large message.
+CPython's GC starts when the number of allocated objects grows.
+This means unpacking may trigger unnecessary GC.
+You can use `gc.disable()` when unpacking a large message.
 
-List is the default sequence type of Python.
-But tuple is lighter than list.
+A list is the default sequence type in Python.
+However, a tuple is lighter than a list.
 You can use `use_list=False` while unpacking when performance is important.
 
 
@@ -208,7 +208,7 @@ You can use `use_list=False` while unpacking when performance is important.
 
 ### msgpack 0.5
 
-Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5.
+The package name on PyPI was changed from `msgpack-python` to `msgpack` in 0.5.
 
 When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before
 `pip install -U msgpack`.
@@ -218,25 +218,25 @@ When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` be
 
 * Python 2 support
 
-  * The extension module does not support Python 2 anymore.
+  * The extension module no longer supports Python 2.
     The pure Python implementation (`msgpack.fallback`) is used for Python 2.
   
   * msgpack 1.0.6 drops official support of Python 2.7, as pip and
-    GitHub Action (setup-python) no longer support Python 2.7.
+    GitHub Action "setup-python" no longer supports Python 2.7.
 
 * Packer
 
   * Packer uses `use_bin_type=True` by default.
-    Bytes are encoded in bin type in msgpack.
-  * The `encoding` option is removed.  UTF-8 is used always.
+    Bytes are encoded in the bin type in MessagePack.
+  * The `encoding` option is removed. UTF-8 is always used.
 
 * Unpacker
 
-  * Unpacker uses `raw=False` by default.  It assumes str types are valid UTF-8 string
-    and decode them to Python str (unicode) object.
+  * Unpacker uses `raw=False` by default. It assumes str values are valid UTF-8 strings
+    and decodes them to Python str (Unicode) objects.
   * `encoding` option is removed.  You can use `raw=True` to support old format (e.g. unpack into bytes, not str).
-  * Default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attack.
+  * The default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attacks.
     You need to pass `max_buffer_size=0` if you have large but safe data.
-  * Default value of `strict_map_key` is changed to True to avoid hashdos.
-    You need to pass `strict_map_key=False` if you have data which contain map keys
-    which type is not bytes or str.
+  * The default value of `strict_map_key` is changed to True to avoid hash DoS.
+    You need to pass `strict_map_key=False` if you have data that contain map keys
+    whose type is neither bytes nor str.

From 19b5d33ded09d5a563a677c1a3d75b6246181259 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Wed, 8 Oct 2025 17:56:20 +0900
Subject: [PATCH 1643/1648] release v1.1.2 (#649)

---
 .github/workflows/wheel.yml |  2 +-
 ChangeLog.rst               | 13 +++++++++++++
 msgpack/__init__.py         |  4 ++--
 pyproject.toml              | 18 ++++--------------
 requirements.txt            |  2 +-
 test/test_buffer.py         |  4 ++--
 6 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index fb369a1..fca321a 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -29,7 +29,7 @@ jobs:
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v3.1.1
+        uses: pypa/cibuildwheel@v3.2.0
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
diff --git a/ChangeLog.rst b/ChangeLog.rst
index 418c444..beeab15 100644
--- a/ChangeLog.rst
+++ b/ChangeLog.rst
@@ -1,3 +1,16 @@
+1.1.2
+=====
+
+Release Date: 2025-10-08
+
+This release does not change source code. It updates only building wheels:
+
+* Update Cython to v3.1.4
+* Update cibuildwheel to v3.2.0
+* Drop Python 3.8
+* Add Python 3.14
+* Add windows-arm
+
 1.1.1
 =====
 
diff --git a/msgpack/__init__.py b/msgpack/__init__.py
index ad68271..f3266b7 100644
--- a/msgpack/__init__.py
+++ b/msgpack/__init__.py
@@ -4,8 +4,8 @@ import os
 from .exceptions import *  # noqa: F403
 from .ext import ExtType, Timestamp
 
-version = (1, 1, 1)
-__version__ = "1.1.1"
+version = (1, 1, 2)
+__version__ = "1.1.2"
 
 
 if os.environ.get("MSGPACK_PUREPYTHON"):
diff --git a/pyproject.toml b/pyproject.toml
index b162832..3976adc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,31 +1,21 @@
 [build-system]
-# 75.3.0 is the latest version supporting Python 3.8
-requires = ["setuptools >= 75.3.0"]
+requires = ["setuptools >= 80.9.0"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "msgpack"
 dynamic = ["version"]
-# `license = "Apache-2.0"` is preferred. But keep old syntax for Python 3.8 compatibility.
-# https://github.com/msgpack/msgpack-python/pull/637
-license = {text="Apache 2.0"}
+license = "Apache-2.0"
 authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}]
 description = "MessagePack serializer"
 readme = "README.md"
 keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"]
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Operating System :: OS Independent",
     "Topic :: File Formats",
     "Intended Audience :: Developers",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
-    "Programming Language :: Python :: 3.13",
-    "Programming Language :: Python :: 3.14",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
@@ -46,7 +36,7 @@ version = {attr = "msgpack.__version__"}
 
 [tool.ruff]
 line-length = 100
-target-version = "py38"
+target-version = "py39"
 lint.select = [
     "E", # pycodestyle
     "F", # Pyflakes
diff --git a/requirements.txt b/requirements.txt
index 5d2e20b..b544cca 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1 @@
-Cython~=3.1.2
+Cython==3.1.4
diff --git a/test/test_buffer.py b/test/test_buffer.py
index 2c5a14c..ca09722 100644
--- a/test/test_buffer.py
+++ b/test/test_buffer.py
@@ -17,7 +17,7 @@ def test_unpack_bytearray():
     obj = unpackb(buf, use_list=1)
     assert [b"foo", b"bar"] == obj
     expected_type = bytes
-    assert all(type(s) == expected_type for s in obj)
+    assert all(type(s) is expected_type for s in obj)
 
 
 def test_unpack_memoryview():
@@ -26,7 +26,7 @@ def test_unpack_memoryview():
     obj = unpackb(view, use_list=1)
     assert [b"foo", b"bar"] == obj
     expected_type = bytes
-    assert all(type(s) == expected_type for s in obj)
+    assert all(type(s) is expected_type for s in obj)
 
 
 def test_packer_getbuffer():

From ef4f83df16e552e8c8fb12ceac55f5fda9f7f340 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 9 Oct 2025 13:00:46 +0900
Subject: [PATCH 1644/1648] relax setuptools version (#652)

---
 .github/workflows/test.yml | 4 +---
 MANIFEST.in                | 2 +-
 pyproject.toml             | 2 +-
 requirements.txt           | 2 ++
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 26b4ce6..74c311c 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -33,7 +33,6 @@ jobs:
       - name: Prepare
         shell: bash
         run: |
-          python -m pip install -U pip
           python -m pip install -r requirements.txt pytest
 
       - name: Build
@@ -55,8 +54,7 @@ jobs:
       - name: build packages
         shell: bash
         run: |
-          pip install build
-          python -m build
+          python -m build -nv
 
       - name: upload packages
         uses: actions/upload-artifact@v4
diff --git a/MANIFEST.in b/MANIFEST.in
index 57d84a4..6317706 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,5 @@
 include setup.py
 include COPYING
 include README.md
-recursive-include msgpack *.h *.c *.pyx *.cpp
+recursive-include msgpack *.h *.c *.pyx
 recursive-include test *.py
diff --git a/pyproject.toml b/pyproject.toml
index 3976adc..3633d6b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools >= 80.9.0"]
+requires = ["setuptools >= 77.0.3"]
 build-backend = "setuptools.build_meta"
 
 [project]
diff --git a/requirements.txt b/requirements.txt
index b544cca..f8f8370 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,3 @@
 Cython==3.1.4
+setuptools==77.0.3
+build

From c2546eabc414b6f232d41b7d9e51b9ff9fa93296 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 9 Oct 2025 15:28:01 +0900
Subject: [PATCH 1645/1648] update setuptools requirements to >=78.1.1 (#653)

https://github.com/advisories/GHSA-5rjg-fvgr-3xxf
---
 pyproject.toml   | 2 +-
 requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 3633d6b..a4c2172 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools >= 77.0.3"]
+requires = ["setuptools >= 78.1.1"]
 build-backend = "setuptools.build_meta"
 
 [project]
diff --git a/requirements.txt b/requirements.txt
index f8f8370..2f1c55b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
 Cython==3.1.4
-setuptools==77.0.3
+setuptools==78.1.1
 build

From af456409709f58add777090bdbe6a4696879e49d Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 9 Oct 2025 15:53:08 +0900
Subject: [PATCH 1646/1648] cython: freethreading_compatible (#654)

```
$ v3/bin/python -VV
Python 3.14.0 free-threading build (main, Oct  7 2025, 15:35:12) [Clang 20.1.4 ]

$ v3/bin/python -c 'import sys,msgpack; print(sys._is_gil_enabled())'
False
```
---
 .github/workflows/test.yml |  2 +-
 msgpack/_cmsgpack.pyx      |  3 ++-
 msgpack/_packer.pyx        | 10 ++++++++--
 msgpack/_unpacker.pyx      | 11 +++++++++--
 4 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 74c311c..20410fe 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"]
-        py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9"]
+        py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10", "3.9"]
         exclude:
           - os: windows-11-arm
             py: "3.9"
diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx
index 1faaac3..9680b31 100644
--- a/msgpack/_cmsgpack.pyx
+++ b/msgpack/_cmsgpack.pyx
@@ -1,5 +1,6 @@
-# coding: utf-8
 #cython: embedsignature=True, c_string_encoding=ascii, language_level=3
+#cython: freethreading_compatible = True
+import cython
 from cpython.datetime cimport import_datetime, datetime_new
 import_datetime()
 
diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
index 402b694..94d1462 100644
--- a/msgpack/_packer.pyx
+++ b/msgpack/_packer.pyx
@@ -1,5 +1,3 @@
-# coding: utf-8
-
 from cpython cimport *
 from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact
 from cpython.datetime cimport (
@@ -129,6 +127,7 @@ cdef class Packer:
         if self.exports > 0:
             raise BufferError("Existing exports of data: Packer cannot be changed")
 
+    @cython.critical_section
     def __init__(self, *, default=None,
                  bint use_single_float=False, bint autoreset=True, bint use_bin_type=True,
                  bint strict_types=False, bint datetime=False, unicode_errors=None,
@@ -269,6 +268,7 @@ cdef class Packer:
                 return ret
         return self._pack_inner(o, 0, nest_limit)
 
+    @cython.critical_section
     def pack(self, object obj):
         cdef int ret
         self._check_exports()
@@ -284,6 +284,7 @@ cdef class Packer:
             self.pk.length = 0
             return buf
 
+    @cython.critical_section
     def pack_ext_type(self, typecode, data):
         self._check_exports()
         if len(data) > ITEM_LIMIT:
@@ -291,6 +292,7 @@ cdef class Packer:
         msgpack_pack_ext(&self.pk, typecode, len(data))
         msgpack_pack_raw_body(&self.pk, data, len(data))
 
+    @cython.critical_section
     def pack_array_header(self, long long size):
         self._check_exports()
         if size > ITEM_LIMIT:
@@ -301,6 +303,7 @@ cdef class Packer:
             self.pk.length = 0
             return buf
 
+    @cython.critical_section
     def pack_map_header(self, long long size):
         self._check_exports()
         if size > ITEM_LIMIT:
@@ -311,6 +314,7 @@ cdef class Packer:
             self.pk.length = 0
             return buf
 
+    @cython.critical_section
     def pack_map_pairs(self, object pairs):
         """
         Pack *pairs* as msgpack map type.
@@ -331,6 +335,7 @@ cdef class Packer:
             self.pk.length = 0
             return buf
 
+    @cython.critical_section
     def reset(self):
         """Reset internal buffer.
 
@@ -339,6 +344,7 @@ cdef class Packer:
         self._check_exports()
         self.pk.length = 0
 
+    @cython.critical_section
     def bytes(self):
         """Return internal buffer contents as bytes object"""
         return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length)
diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
index 34ff330..f0cf96d 100644
--- a/msgpack/_unpacker.pyx
+++ b/msgpack/_unpacker.pyx
@@ -1,5 +1,3 @@
-# coding: utf-8
-
 from cpython cimport *
 cdef extern from "Python.h":
     ctypedef struct PyObject
@@ -324,6 +322,7 @@ cdef class Unpacker:
         PyMem_Free(self.buf)
         self.buf = NULL
 
+    @cython.critical_section
     def __init__(self, file_like=None, *, Py_ssize_t read_size=0,
                  bint use_list=True, bint raw=False, int timestamp=0, bint strict_map_key=True,
                  object object_hook=None, object object_pairs_hook=None, object list_hook=None,
@@ -384,6 +383,7 @@ cdef class Unpacker:
                  max_str_len, max_bin_len, max_array_len,
                  max_map_len, max_ext_len)
 
+    @cython.critical_section
     def feed(self, object next_bytes):
         """Append `next_bytes` to internal buffer."""
         cdef Py_buffer pybuff
@@ -484,6 +484,7 @@ cdef class Unpacker:
             else:
                 raise ValueError("Unpack failed: error = %d" % (ret,))
 
+    @cython.critical_section
     def read_bytes(self, Py_ssize_t nbytes):
         """Read a specified number of raw bytes from the stream"""
         cdef Py_ssize_t nread
@@ -496,6 +497,7 @@ cdef class Unpacker:
         self.stream_offset += nread
         return ret
 
+    @cython.critical_section
     def unpack(self):
         """Unpack one object
 
@@ -503,6 +505,7 @@ cdef class Unpacker:
         """
         return self._unpack(unpack_construct)
 
+    @cython.critical_section
     def skip(self):
         """Read and ignore one object, returning None
 
@@ -510,6 +513,7 @@ cdef class Unpacker:
         """
         return self._unpack(unpack_skip)
 
+    @cython.critical_section
     def read_array_header(self):
         """assuming the next object is an array, return its size n, such that
         the next n unpack() calls will iterate over its contents.
@@ -518,6 +522,7 @@ cdef class Unpacker:
         """
         return self._unpack(read_array_header)
 
+    @cython.critical_section
     def read_map_header(self):
         """assuming the next object is a map, return its size n, such that the
         next n * 2 unpack() calls will iterate over its key-value pairs.
@@ -526,6 +531,7 @@ cdef class Unpacker:
         """
         return self._unpack(read_map_header)
 
+    @cython.critical_section
     def tell(self):
         """Returns the current position of the Unpacker in bytes, i.e., the
         number of bytes that were read from the input, also the starting
@@ -536,6 +542,7 @@ cdef class Unpacker:
     def __iter__(self):
         return self
 
+    @cython.critical_section
     def __next__(self):
         return self._unpack(unpack_construct, 1)
 

From c1ecd23dbfaa6c6f2a408e7e5c4727c6bdce7608 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Thu, 9 Oct 2025 18:33:23 +0900
Subject: [PATCH 1647/1648] drop Python 3.9 (#656)

---
 .github/workflows/test.yml  | 8 +++-----
 .github/workflows/wheel.yml | 6 +++---
 pyproject.toml              | 4 ++--
 3 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 20410fe..6b1664a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,10 +10,8 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"]
-        py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10", "3.9"]
+        py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10"]
         exclude:
-          - os: windows-11-arm
-            py: "3.9"
           - os: windows-11-arm
             py: "3.10"
     runs-on: ${{ matrix.os }}
@@ -21,10 +19,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.py }}
           allow-prereleases: true
diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index fca321a..be54e5e 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -17,8 +17,8 @@ jobs:
     name: Build wheels on ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.x"
           cache: "pip"
@@ -33,7 +33,7 @@ jobs:
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
-          CIBW_SKIP: "pp* cp38-* cp39-win_arm64 cp310-win_arm64"
+          CIBW_SKIP: "pp* cp38-* cp39-* cp310-win_arm64"
 
       - name: Build sdist
         if: runner.os == 'Linux' && runner.arch == 'X64'
diff --git a/pyproject.toml b/pyproject.toml
index a4c2172..c69d5a7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,7 +10,7 @@ authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}]
 description = "MessagePack serializer"
 readme = "README.md"
 keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Operating System :: OS Independent",
@@ -36,7 +36,7 @@ version = {attr = "msgpack.__version__"}
 
 [tool.ruff]
 line-length = 100
-target-version = "py39"
+target-version = "py310"
 lint.select = [
     "E", # pycodestyle
     "F", # Pyflakes

From f9806368ae302722f095ea884436b20ed1cddf33 Mon Sep 17 00:00:00 2001
From: Inada Naoki 
Date: Mon, 1 Dec 2025 14:16:03 +0900
Subject: [PATCH 1648/1648] update cython and cibuildwheel (#658)

---
 .github/workflows/wheel.yml | 4 ++--
 requirements.txt            | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
index be54e5e..531abbc 100644
--- a/.github/workflows/wheel.yml
+++ b/.github/workflows/wheel.yml
@@ -17,7 +17,7 @@ jobs:
     name: Build wheels on ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
           python-version: "3.x"
@@ -29,7 +29,7 @@ jobs:
           make cython
 
       - name: Build
-        uses: pypa/cibuildwheel@v3.2.0
+        uses: pypa/cibuildwheel@v3.3.0
         env:
           CIBW_TEST_REQUIRES: "pytest"
           CIBW_TEST_COMMAND: "pytest {package}/test"
diff --git a/requirements.txt b/requirements.txt
index 2f1c55b..9e4643b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-Cython==3.1.4
+Cython==3.2.1
 setuptools==78.1.1
 build